diff --git a/lark/load_grammar.py b/lark/load_grammar.py index b98f166c..71162288 100644 --- a/lark/load_grammar.py +++ b/lark/load_grammar.py @@ -3,6 +3,7 @@ import hashlib import os.path +import re import sys from collections import namedtuple from copy import copy, deepcopy @@ -596,6 +597,19 @@ def _literal_to_pattern(literal): assert False, 'Invariant failed: literal.type not in ["STRING", "REGEXP"]' +def _char_to_regexp(char: str) -> str: + """Escape a single character for use inside a regexp character class. + + Characters in 0x80-0xff are written as ``\\xNN`` escapes: under ``use_bytes`` the + pattern is later encoded (utf-8 by the dynamic lexer), which would split a literal + character into two bytes. Anything else is left to ``re.escape``, which keeps + non-ascii characters literal. + """ + if '\x80' <= char <= '\xff': + return f'\\x{ord(char):02x}' + return re.escape(char) + + @inline_args class PrepareLiterals(Transformer_InPlace): def literal(self, literal): @@ -603,16 +617,17 @@ def literal(self, literal): def range(self, start, end): assert start.type == end.type == 'STRING' - start = start.value[1:-1] - end = end.value[1:-1] - assert len(eval_escaping(start)) == len(eval_escaping(end)) == 1 - regexp = '[%s-%s]' % (start, end) + start = eval_escaping(start.value[1:-1]) + end = eval_escaping(end.value[1:-1]) + assert len(start) == len(end) == 1 + regexp = '[%s-%s]' % (_char_to_regexp(start), _char_to_regexp(end)) return ST('pattern', [PatternRE(regexp)]) def _make_joined_pattern(regexp, flags_set) -> PatternRE: return PatternRE(regexp, ()) + class TerminalTreeToPattern(Transformer_NonRecursive): def pattern(self, ps): p ,= ps @@ -625,7 +640,9 @@ def expansion(self, items: List[Pattern]) -> Pattern: if len(items) == 1: return items[0] - pattern = ''.join(i.to_regexp() for i in items) + # A bare '|' binds looser than concatenation, so an item containing one is grouped + # before it's joined, or it swallows its neighbors: /a|b/ "c" must not become 'a|bc'. + pattern = ''.join(f'(?:{r})' if '|' in r else r for r in (i.to_regexp() for i in items)) return _make_joined_pattern(pattern, {i.flags for i in items}) def expansions(self, exps: List[Pattern]) -> Pattern: diff --git a/tests/test_grammar.py b/tests/test_grammar.py index fc1d32d3..6638a15c 100644 --- a/tests/test_grammar.py +++ b/tests/test_grammar.py @@ -299,6 +299,34 @@ def test_large_terminal(self): for i in (-1, 1000): self.assertRaises(UnexpectedInput, l.parse, str(i)) + def test_concat_regexp_with_alternation(self): + # A '|' outside a group binds looser than concatenation, so it used to swallow + # whatever was concatenated after it: /admin|user/ "_id" compiled to 'admin|user_id'. + l = Lark('start: T\nT: /admin|user/ "_id"', parser='lalr') + for s in ('admin_id', 'user_id'): + self.assertEqual(l.parse(s), Tree('start', [s])) + for s in ('admin', 'user'): + self.assertRaises(UnexpectedInput, l.parse, s) + + # A '|' that is escaped, inside a character class, or already grouped keeps its meaning + for term, matches in [(r'/a\|b/', 'a|bc'), ('/[a|b]/', 'ac'), ('/(a|b)/', 'ac')]: + l = Lark('start: T\nT: %s "c"' % term, parser='lalr') + self.assertEqual(l.parse(matches), Tree('start', [matches])) + + def test_literal_range_escapes_endpoints(self): + # Endpoints were spliced into the character class as written, so a metacharacter + # escaped it: "^".."z" became '[^-z]', a negated class matching almost everything. + l = Lark('start: T\nT: "^".."z"', parser='lalr') + for s in ('^', 'a', 'z'): + self.assertEqual(l.parse(s), Tree('start', [s])) + for s in ('!', '0', '\t'): + self.assertRaises(UnexpectedInput, l.parse, s) + + # Non-ascii endpoints stay literal, so the pattern is the same string as /[а-я]/ + # (interegular and anonymous-terminal dedup both compare pattern strings) + l = Lark('start: T\nT: "а".."я"', parser='lalr') + self.assertEqual(l.get_terminal('T').pattern.to_regexp(), '[а-я]') + def test_list_grammar_imports(self): grammar = """ %import .test_templates_import (start, sep)