refactor: simplify range code, fix bug in regex-test

pomsky-lang · Nov 25, 2024 · 25f0a6a · 25f0a6a
1 parent e5ecc50
commit 25f0a6a
Show file tree

Hide file tree

Showing 17 changed files with 87 additions and 87 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/pomsky-bin/src/test.rs b/pomsky-bin/src/test.rs
@@ -57,7 +57,7 @@ fn check_test_match(regex: &Regex, test_case: TestCaseMatch, errors: &mut Vec<Di
     let result = regex.captures(test_case.literal.content.as_bytes());
     match result {
         Ok(Some(captures)) => {
-            if captures[0].len() != test_case.literal.content.as_bytes().len() {
+            if captures[0].len() != test_case.literal.content.len() {
                 errors.push(Diagnostic::test_failure(
                     test_case.literal.span,
                     DiagnosticCode::TestNoExactMatch,
@@ -177,7 +177,7 @@ fn check_test_reject(regex: &Regex, test_case: TestCaseReject, errors: &mut Vec<
     let result = regex.captures(test_case.literal.content.as_bytes());
     match result {
         Ok(Some(captures)) => {
-            let is_exact = captures[0].len() == test_case.literal.content.as_bytes().len();
+            let is_exact = captures[0].len() == test_case.literal.content.len();
             if test_case.as_substring || is_exact {
                 let actual_value;
                 let (code, actual_value) = if is_exact || !test_case.as_substring {

diff --git a/pomsky-lib/afl-fuzz/Cargo.lock b/pomsky-lib/afl-fuzz/Cargo.lock
diff --git a/pomsky-lib/afl-fuzz/README.md b/pomsky-lib/afl-fuzz/README.md
@@ -31,3 +31,21 @@ When you found a crash, you might find it in `errors.txt`. If it's not in `error
 ## Report the bug
 
 Please report the bug [here](https://github.com/pomsky-lang/pomsky/issues). If you think it could be a security vulnerability, please disclose it directly per email: [email protected].
+
+## Latest findings
+
+### PCRE
+
+- Lookbehind cannot contain include unbounded repetitions.
+  - Bounded repetitions need an upper bound of _at most_ 255. I.e. `(?<=a{4,255})` is ok.
+  - Nested repetitions reach the limit quicker (TBD)
+- Lookbehind cannot contain `\X`
+
+### Ruby
+
+- Lookaround cannot contain capturing groups
+
+### Python
+
+- Lookbehind requires fixed-width pattern
+- Cannot refer to open capturing group
diff --git a/pomsky-lib/src/exprs/range.rs b/pomsky-lib/src/exprs/range.rs
@@ -469,34 +469,14 @@ impl Class {
         let (a, b) = (self.start, self.end);
         let mut set = UnicodeSet::new();
 
-        match (a, b, a == b) {
-            (0..=9, _, true) => return Regex::Char((a + b'0') as char),
-            (0..=9, 0..=9, _) => {
+        match (a, b) {
+            (0..=9, 0..=9) => {
                 set.add_range_unchecked((a + b'0') as char..=(b + b'0') as char);
             }
-            (10.., _, true) => {
-                set.add_char_unchecked((a + b'a' - 10) as char);
-                set.add_char_unchecked((a + b'A' - 10) as char);
-            }
-            (10.., 10.., _) => {
+            (10.., 10..) => {
                 set.add_range_unchecked((a + b'a' - 10) as char..=(b + b'a' - 10) as char);
                 set.add_range_unchecked((a + b'A' - 10) as char..=(b + b'A' - 10) as char);
             }
-            (9, 10, _) => {
-                set.add_char_unchecked('9');
-                set.add_char_unchecked('a');
-                set.add_char_unchecked('A');
-            }
-            (_, 10, _) => {
-                set.add_range_unchecked((a + b'0') as char..='9');
-                set.add_char_unchecked('a');
-                set.add_char_unchecked('A');
-            }
-            (9, _, _) => {
-                set.add_char_unchecked('9');
-                set.add_range_unchecked('a'..=(b + b'a' - 10) as char);
-                set.add_range_unchecked('A'..=(b + b'A' - 10) as char);
-            }
             _ => {
                 set.add_range_unchecked((a + b'0') as char..='9');
                 set.add_range_unchecked('a'..=(b + b'a' - 10) as char);

diff --git a/pomsky-lib/src/regex/mod.rs b/pomsky-lib/src/regex/mod.rs
@@ -34,8 +34,6 @@ pub(crate) enum Regex {
     Literal(String),
     /// A regex string that is inserted verbatim into the output
     Unescaped(String),
-    /// A literal char
-    Char(char),
     /// A character class, delimited with square brackets
     CharSet(RegexCharSet),
     /// A Unicode grapheme
@@ -66,7 +64,6 @@ impl Regex {
         match self {
             Regex::Literal(str) => Ok(Some(str.chars().count() as u32)),
             Regex::Unescaped(_) => Ok(None),
-            Regex::Char(_) => Ok(Some(1)),
             Regex::CharSet(_) => Ok(Some(1)),
             Regex::Grapheme => Err(CompileErrorKind::UnsupportedInLookbehind {
                 flavor: RegexFlavor::Python,
@@ -115,7 +112,6 @@ impl Regex {
         match self {
             Regex::Literal(_) => Ok(()),
             Regex::Unescaped(_) => Ok(()),
-            Regex::Char(_) => Ok(()),
             Regex::CharSet(_) => Ok(()),
             Regex::Grapheme => Err(CompileErrorKind::UnsupportedInLookbehind {
                 flavor: RegexFlavor::Pcre,
@@ -241,7 +237,6 @@ impl Regex {
                 }
                 Ok(Regex::CharSet(RegexCharSet::new(c.into()).negate()))
             }
-            Regex::Char(c) => Ok(Regex::CharSet(RegexCharSet::new(c.into()).negate())),
             Regex::CharSet(s) => Ok(Regex::CharSet(s.negate())),
             Regex::Boundary(b) => match b {
                 BoundaryKind::Word => Ok(Regex::Boundary(BoundaryKind::NotWord)),
@@ -309,9 +304,6 @@ impl Regex {
             Regex::Unescaped(u) => {
                 buf.push_str(u);
             }
-            &Regex::Char(c) => {
-                literal::codegen_char_esc(c, buf, flavor);
-            }
             Regex::CharSet(c) => c.codegen(buf, flavor),
             Regex::Grapheme => buf.push_str("\\X"),
             Regex::Dot => buf.push('.'),
@@ -330,7 +322,6 @@ impl Regex {
             Regex::Alternation(_) => true,
             Regex::Literal(_)
             | Regex::Unescaped(_)
-            | Regex::Char(_)
             | Regex::Group(_)
             | Regex::CharSet(_)
             | Regex::Grapheme
@@ -353,7 +344,6 @@ impl Regex {
             | Regex::Unescaped(_) => true,
             Regex::Lookaround(_) => matches!(flavor, RegexFlavor::JavaScript),
             Regex::CharSet(_)
-            | Regex::Char(_)
             | Regex::Grapheme
             | Regex::Reference(_)
             | Regex::Dot

diff --git a/pomsky-lib/src/regex/optimize.rs b/pomsky-lib/src/regex/optimize.rs
@@ -107,8 +107,7 @@ impl Regex {
                 Count::One
             }
             Regex::Unescaped(_) => Count::Many,
-            Regex::Char(_)
-            | Regex::CharSet(_)
+            Regex::CharSet(_)
             | Regex::Grapheme
             | Regex::Dot
             | Regex::Boundary(_)

diff --git a/pomsky-lib/src/unicode_set.rs b/pomsky-lib/src/unicode_set.rs
@@ -114,10 +114,6 @@ impl UnicodeSet {
         self.ranges.insert(SetRange { first: *range.start() as u32, last: *range.end() as u32 });
     }
 
-    pub fn add_char_unchecked(&mut self, char: char) {
-        self.ranges.insert(SetRange::single(char as u32));
-    }
-
     fn add(&mut self, mut range_new: SetRange) {
         let lower = SetRange::single(range_new.first.saturating_sub(1));
         let upper = SetRange::single(range_new.last.saturating_add(1));

diff --git a/pomsky-syntax/src/exprs/reference.rs b/pomsky-syntax/src/exprs/reference.rs
@@ -31,7 +31,9 @@ impl Reference {
             ReferenceTarget::Named(n) => buf.write(n),
             ReferenceTarget::Number(i) => buf.write_fmt(i),
             &ReferenceTarget::Relative(o) => {
-                buf.push(if o < 0 { '-' } else { '+' });
+                if o >= 0 {
+                    buf.push('+');
+                }
                 buf.write_fmt(o);
             }
         }

diff --git a/regex-test/Cargo.toml b/regex-test/Cargo.toml
@@ -7,6 +7,6 @@ publish = false
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-regex = "1.10.2"
-pcre2 = "0.2.5"
+regex = "1.11.1"
+pcre2 = "0.2.9"
 onig = "6.4.0"
diff --git a/regex-test/dotnet/TesterAsync.cs b/regex-test/dotnet/TesterAsync.cs
@@ -7,8 +7,12 @@ public static void Main(string[] args) {
         string testLine;
 
         while ((line = Console.ReadLine()) != null) {
+            if (!line.StartsWith("REGEX:")) {
+                continue;
+            }
+
             try {
-                var r = new Regex(line, RegexOptions.Compiled);
+                var r = new Regex(line.Substring(6), RegexOptions.Compiled);
                 Console.WriteLine("success");
 
                 while ((testLine = Console.ReadLine()) != null && testLine.StartsWith("TEST:")) {

diff --git a/regex-test/java/TesterAsync.java b/regex-test/java/TesterAsync.java
@@ -7,8 +7,12 @@ public static void main(String[] args) {
         try (Scanner input = new Scanner(System.in)) {
             while (input.hasNext()) {
                 String regex = input.nextLine();
+                if (!regex.startsWith("REGEX:")) {
+                    continue;
+                }
+
                 try {
-                    Pattern p = Pattern.compile(regex);
+                    Pattern p = Pattern.compile("(?U)" + regex.substring(6));
                     System.out.printf("success\n");
 
                     while (input.hasNext()) {

diff --git a/regex-test/js/tester-deno-async.js b/regex-test/js/tester-deno-async.js
@@ -5,8 +5,12 @@ let regex
 
 for await (const line of readLines(Deno.stdin)) {
   if (regex === undefined) {
+    if (!line.startsWith('REGEX:')) {
+      continue
+    }
+
     try {
-      regex = new RegExp(line, 'u')
+      regex = new RegExp(line.slice(6), 'u')
       console.log('success')
     } catch (e) {
       console.log(substituteLf(e.message))
@@ -17,9 +21,7 @@ for await (const line of readLines(Deno.stdin)) {
     if (regex.test(test)) {
       console.log('test good')
     } else {
-      console.log(
-        substituteLf(`Regex '${regex.source}' does not match '${test}'`)
-      )
+      console.log(substituteLf(`Regex '${regex.source}' does not match '${test}'`))
       regex = undefined
     }
   } else {

diff --git a/regex-test/js/tester-node-async.js b/regex-test/js/tester-node-async.js
@@ -11,8 +11,12 @@ let regex
 
 rl.on('line', (line) => {
   if (regex === undefined) {
+    if (!line.startsWith('REGEX:')) {
+      return
+    }
+
     try {
-      regex = new RegExp(line, 'u')
+      regex = new RegExp(line.slice(6), 'u')
       console.log('success')
     } catch (e) {
       console.log(substituteLf(e.message))
@@ -23,9 +27,7 @@ rl.on('line', (line) => {
     if (regex.test(test)) {
       console.log('test good')
     } else {
-      console.log(
-        substituteLf(`Regex '${regex.source}' does not match '${test}'`)
-      )
+      console.log(substituteLf(`Regex '${regex.source}' does not match '${test}'`))
       regex = undefined
     }
   } else {