Rooba
/
lark
mirror of https://github.com/lark-parser/lark.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fixed support for hex encoding (\xAA)

This commit is contained in:
Erez Shinan 2019-02-13 13:35:32 +02:00
parent 70d724732d
commit 6b2df208c2
2 changed files with 9 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
lark/lexer.py
View File

@ -72,7 +72,12 @@ class Token(Str):
__slots__ = ('type', 'pos_in_stream', 'value', 'line', 'column', 'end_line', 'end_column') __slots__ = ('type', 'pos_in_stream', 'value', 'line', 'column', 'end_line', 'end_column')
def __new__(cls, type_, value, pos_in_stream=None, line=None, column=None): def __new__(cls, type_, value, pos_in_stream=None, line=None, column=None):
self = super(Token, cls).__new__(cls, value) try:
self = super(Token, cls).__new__(cls, value)
except UnicodeDecodeError:
value = value.decode('latin1')
self = super(Token, cls).__new__(cls, value)
self.type = type_ self.type = type_
self.pos_in_stream = pos_in_stream self.pos_in_stream = pos_in_stream
self.value = value self.value = value

5
tests/test_parser.py
View File

@ -449,11 +449,12 @@ def _make_parser_test(LEXER, PARSER):
g.parse(u'\xa3\u0101\u00a3\u0203\n') g.parse(u'\xa3\u0101\u00a3\u0203\n')
def test_hex_escape(self): def test_hex_escape(self):
g = _Lark(r"""start: A B g = _Lark(r"""start: A B C
A: "\x01" A: "\x01"
B: /\x02/ B: /\x02/
C: "\xABCD"
""") """)
g.parse('\x01\x02') g.parse('\x01\x02\xABCD')
@unittest.skipIf(PARSER == 'cyk', "Takes forever") @unittest.skipIf(PARSER == 'cyk', "Takes forever")
def test_stack_for_ebnf(self): def test_stack_for_ebnf(self):