Rooba
/
lark
mirror of https://github.com/lark-parser/lark.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

A few fixes to escaping

This commit is contained in:
Erez Shinan 2017-11-16 08:54:16 +02:00
parent ec7da3bf31
commit d846627fb7
3 changed files with 8 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
examples/python3.g
View File

@ -165,7 +165,7 @@ string: STRING | LONG_STRING
// Tokens // Tokens
NAME: /[a-zA-Z_]\w*/ NAME: /[a-zA-Z_]\w*/
COMMENT: /\#[^\n]*/ COMMENT: /#[^\n]*/
_NEWLINE: ( /\r?\n[\t ]*/ | COMMENT )+ _NEWLINE: ( /\r?\n[\t ]*/ | COMMENT )+
@ -177,8 +177,8 @@ _NEWLINE: ( /\r?\n[\t ]*/ | COMMENT )+
// STRING : /[ub]?r?("(?!"").*?(?<!\\)(\\\\)*?"|'(?!'').*?(?<!\\)(\\\\)*?')/ // STRING : /[ub]?r?("(?!"").*?(?<!\\)(\\\\)*?"|'(?!'').*?(?<!\\)(\\\\)*?')/
// LONG_STRING: /(?s)[ub]?r?(""".*?(?<!\\)(\\\\)*?"""|'''.*?(?<!\\)(\\\\)*?''')/ // LONG_STRING: /(?s)[ub]?r?(""".*?(?<!\\)(\\\\)*?"""|'''.*?(?<!\\)(\\\\)*?''')/
STRING : /(?i)[ub]?r?("(?!"").*?(?<!\\\\)(\\\\\\\\)*?"|'(?!'').*?(?<!\\\\)(\\\\\\\\)*?')/ STRING : /(?i)[ubf]?r?("(?!"").*?(?<!\\\\)(\\\\\\\\)*?"|'(?!'').*?(?<!\\\\)(\\\\\\\\)*?')/
LONG_STRING: /(?i)(?s)[ub]?r?(""".*?(?<!\\\\)(\\\\\\\\)*?"""|'''.*?(?<!\\\\)(\\\\\\\\)*?''')/ LONG_STRING: /(?i)(?s)[ubf]?r?(""".*?(?<!\\\\)(\\\\\\\\)*?"""|'''.*?(?<!\\\\)(\\\\\\\\)*?''')/
DEC_NUMBER: /(?i)[1-9]\d*l?/ DEC_NUMBER: /(?i)[1-9]\d*l?/
HEX_NUMBER: /(?i)0x[\da-f]*l?/ HEX_NUMBER: /(?i)0x[\da-f]*l?/

6
lark/load_grammar.py
View File

@ -77,7 +77,7 @@ TOKENS = {
'_TO': '->', '_TO': '->',
'_IGNORE': r'%ignore', '_IGNORE': r'%ignore',
'_IMPORT': r'%import', '_IMPORT': r'%import',
'NUMBER': '\d+', 'NUMBER': r'\d+',
} }
RULES = { RULES = {
@ -294,7 +294,9 @@ def _literal_to_pattern(literal):
flags = None flags = None
assert v[0] == v[-1] and v[0] in '"/' assert v[0] == v[-1] and v[0] in '"/'
x = v[1:-1].replace("'", r"\'") x = v[1:-1]
x = re.sub(r'(\\[wd/]|\\\[|\\\])', r'\\\1', x)
x = x.replace("'", r"\'")
s = literal_eval("u'''%s'''" % x) s = literal_eval("u'''%s'''" % x)
return { 'STRING': PatternStr, return { 'STRING': PatternStr,
'REGEXP': PatternRE }[literal.type](s, flags) 'REGEXP': PatternRE }[literal.type](s, flags)

2
lark/tools/nearley.py
View File

@ -30,7 +30,7 @@ nearley_grammar = r"""
js: JS? js: JS?
NAME: /[a-zA-Z_$]\w*/ NAME: /[a-zA-Z_$]\w*/
COMMENT: /\#[^\n]*/ COMMENT: /#[^\n]*/
REGEXP: /\[.*?\]/ REGEXP: /\[.*?\]/
STRING: /".*?"/ STRING: /".*?"/