Rooba
/
lark
mirror of https://github.com/lark-parser/lark.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fixed issue #425, keeping in mind unicode issue #411

This commit is contained in:
Erez Shinan 2019-08-15 18:06:42 +02:00
parent d5036eefdd
commit 7e8488d1a0
2 changed files with 4 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lark/lexer.py
View File

@ -101,7 +101,7 @@ class Token(Str):
self.type = type_
self.pos_in_stream = pos_in_stream
self.value = Str(value)
self.value = value
self.line = line
self.column = column
self.end_line = end_line

6
lark/load_grammar.py
View File

@ -12,7 +12,7 @@ from .parse_tree_builder import ParseTreeBuilder
from .parser_frontends import LALR_TraditionalLexer
from .common import LexerConf, ParserConf
from .grammar import RuleOptions, Rule, Terminal, NonTerminal, Symbol
from .utils import classify, suppress, dedup_list
from .utils import classify, suppress, dedup_list, Str
from .exceptions import GrammarError, UnexpectedCharacters, UnexpectedToken
from .tree import Tree, SlottedTree as ST
@ -451,9 +451,9 @@ class PrepareSymbols(Transformer_InPlace):
if isinstance(v, Tree):
return v
elif v.type == 'RULE':
return NonTerminal(v.value)
return NonTerminal(Str(v.value))
elif v.type == 'TERMINAL':
return Terminal(v.value, filter_out=v.startswith('_'))
return Terminal(Str(v.value), filter_out=v.startswith('_'))
assert False
def _choice_of_rules(rules):