Rooba
/
lark
mirror of https://github.com/lark-parser/lark.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Bugfix: Some tokens did not recieve and end_line (Issue #472)

This commit is contained in:
Erez Sh 2019-11-13 10:00:11 +02:00
parent 7c5917ba19
commit e39bfa1b18
1 changed files with 8 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
lark/lexer.py
View File

@ -175,24 +175,24 @@ class _Lex:
value, type_ = res
t = None
if type_ not in ignore_types:
t = Token(type_, value, line_ctr.char_pos, line_ctr.line, line_ctr.column)
line_ctr.feed(value, type_ in newline_types)
t.end_line = line_ctr.line
t.end_column = line_ctr.column
if t.type in lexer.callback:
t = lexer.callback[t.type](t)
if not isinstance(t, Token):
raise ValueError("Callbacks must return a token (returned %r)" % t)
last_token = t
yield t
last_token = t
else:
if type_ in lexer.callback:
t = Token(type_, value, line_ctr.char_pos, line_ctr.line, line_ctr.column)
lexer.callback[type_](t)
t2 = Token(type_, value, line_ctr.char_pos, line_ctr.line, line_ctr.column)
lexer.callback[type_](t2)
line_ctr.feed(value, type_ in newline_types)
line_ctr.feed(value, type_ in newline_types)
if t:
t.end_line = line_ctr.line
t.end_column = line_ctr.column
class UnlessCallback: