Rooba
/
lark
mirror of https://github.com/lark-parser/lark.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fixed bug where identical anonymous tokens got separate names

This commit is contained in:
Erez Shinan 2017-02-11 21:15:57 +02:00
parent b75fc45a2a
commit ee798f81f2
2 changed files with 4 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@ -73,7 +73,7 @@ These features are planned to be implemented in the near future:
## Comparison to other parsers ## Comparison to other parsers
This is a feature comparison. For benchmarks vs pyparsing, check out the [JSON tutorial](/docs/json_tutorial.md#conclusion). This is a feature comparison. For benchmarks vs other parsers, check out the [JSON tutorial](/docs/json_tutorial.md#conclusion).
| Library | Algorithm | LOC | Grammar | Builds tree? | Library | Algorithm | LOC | Grammar | Builds tree?
|:--------|:----------|:----|:--------|:------------ |:--------|:----------|:----|:--------|:------------

3
lark/load_grammar.py
View File

@ -261,6 +261,7 @@ class ExtractAnonTokens(InlineTransformer):
elif token.type == 'REGEXP': elif token.type == 'REGEXP':
token_name = 'ANONRE_%d' % self.i token_name = 'ANONRE_%d' % self.i
value = token.value
self.i += 1 self.i += 1
else: else:
assert False, x assert False, x
@ -268,6 +269,8 @@ class ExtractAnonTokens(InlineTransformer):
if token_name not in self.token_set: if token_name not in self.token_set:
self.token_set.add(token_name) self.token_set.add(token_name)
self.tokens.append((token_name, token, [])) self.tokens.append((token_name, token, []))
assert value not in self.token_reverse
self.token_reverse[value] = token_name
return Token('TOKEN', token_name, -1) return Token('TOKEN', token_name, -1)