group alternations and escape range endpoints in terminal patterns

lark/load_grammar.py

@@ -3,6 +3,7 @@
 
 import hashlib
 import os.path
+import re
 import sys
 from collections import namedtuple
 from copy import copy, deepcopy
@@ -596,23 +597,37 @@ def _literal_to_pattern(literal):
         assert False, 'Invariant failed: literal.type not in ["STRING", "REGEXP"]'
 
 
+def _char_to_regexp(char: str) -> str:
+    """Escape a single character for use inside a regexp character class.
+
+    Characters in 0x80-0xff are written as ``\\xNN`` escapes: under ``use_bytes`` the
+    pattern is later encoded (utf-8 by the dynamic lexer), which would split a literal
+    character into two bytes. Anything else is left to ``re.escape``, which keeps
+    non-ascii characters literal.
+    """
+    if '\x80' <= char <= '\xff':
+        return f'\\x{ord(char):02x}'
+    return re.escape(char)
+
+
 @inline_args
 class PrepareLiterals(Transformer_InPlace):
     def literal(self, literal):
         return ST('pattern', [_literal_to_pattern(literal)])
 
     def range(self, start, end):
         assert start.type == end.type == 'STRING'
-        start = start.value[1:-1]
-        end = end.value[1:-1]
-        assert len(eval_escaping(start)) == len(eval_escaping(end)) == 1
-        regexp = '[%s-%s]' % (start, end)
+        start = eval_escaping(start.value[1:-1])
+        end = eval_escaping(end.value[1:-1])
+        assert len(start) == len(end) == 1
+        regexp = '[%s-%s]' % (_char_to_regexp(start), _char_to_regexp(end))
         return ST('pattern', [PatternRE(regexp)])
 
 
 def _make_joined_pattern(regexp, flags_set) -> PatternRE:
     return PatternRE(regexp, ())
 
+
 class TerminalTreeToPattern(Transformer_NonRecursive):
     def pattern(self, ps):
         p ,= ps
@@ -625,7 +640,9 @@ def expansion(self, items: List[Pattern]) -> Pattern:
         if len(items) == 1:
             return items[0]
 
-        pattern = ''.join(i.to_regexp() for i in items)
+        # A bare '|' binds looser than concatenation, so an item containing one is grouped
+        # before it's joined, or it swallows its neighbors: /a|b/ "c" must not become 'a|bc'.
+        pattern = ''.join(f'(?:{r})' if '|' in r else r for r in (i.to_regexp() for i in items))
         return _make_joined_pattern(pattern, {i.flags for i in items})
 
     def expansions(self, exps: List[Pattern]) -> Pattern:

tests/test_grammar.py

@@ -299,6 +299,34 @@ def test_large_terminal(self):
         for i in (-1, 1000):
             self.assertRaises(UnexpectedInput, l.parse, str(i))
 
+    def test_concat_regexp_with_alternation(self):
+        # A '|' outside a group binds looser than concatenation, so it used to swallow
+        # whatever was concatenated after it: /admin|user/ "_id" compiled to 'admin|user_id'.
+        l = Lark('start: T\nT: /admin|user/ "_id"', parser='lalr')
+        for s in ('admin_id', 'user_id'):
+            self.assertEqual(l.parse(s), Tree('start', [s]))
+        for s in ('admin', 'user'):
+            self.assertRaises(UnexpectedInput, l.parse, s)
+
+        # A '|' that is escaped, inside a character class, or already grouped keeps its meaning
+        for term, matches in [(r'/a\|b/', 'a|bc'), ('/[a|b]/', 'ac'), ('/(a|b)/', 'ac')]:
+            l = Lark('start: T\nT: %s "c"' % term, parser='lalr')
+            self.assertEqual(l.parse(matches), Tree('start', [matches]))
+
+    def test_literal_range_escapes_endpoints(self):
+        # Endpoints were spliced into the character class as written, so a metacharacter
+        # escaped it: "^".."z" became '[^-z]', a negated class matching almost everything.
+        l = Lark('start: T\nT: "^".."z"', parser='lalr')
+        for s in ('^', 'a', 'z'):
+            self.assertEqual(l.parse(s), Tree('start', [s]))
+        for s in ('!', '0', '\t'):
+            self.assertRaises(UnexpectedInput, l.parse, s)
+
+        # Non-ascii endpoints stay literal, so the pattern is the same string as /[а-я]/
+        # (interegular and anonymous-terminal dedup both compare pattern strings)
+        l = Lark('start: T\nT: "а".."я"', parser='lalr')
+        self.assertEqual(l.get_terminal('T').pattern.to_regexp(), '[а-я]')
+
     def test_list_grammar_imports(self):
             grammar = """
             %import .test_templates_import (start, sep)