A few fixes to escaping

This commit is contained in:
Erez Shinan 2017-11-16 08:54:16 +02:00
parent ec7da3bf31
commit d846627fb7
3 changed files with 8 additions and 6 deletions

View File

@ -165,7 +165,7 @@ string: STRING | LONG_STRING
// Tokens
NAME: /[a-zA-Z_]\w*/
COMMENT: /\#[^\n]*/
COMMENT: /#[^\n]*/
_NEWLINE: ( /\r?\n[\t ]*/ | COMMENT )+
@ -177,8 +177,8 @@ _NEWLINE: ( /\r?\n[\t ]*/ | COMMENT )+
// STRING : /[ub]?r?("(?!"").*?(?<!\\)(\\\\)*?"|'(?!'').*?(?<!\\)(\\\\)*?')/
// LONG_STRING: /(?s)[ub]?r?(""".*?(?<!\\)(\\\\)*?"""|'''.*?(?<!\\)(\\\\)*?''')/
STRING : /(?i)[ub]?r?("(?!"").*?(?<!\\\\)(\\\\\\\\)*?"|'(?!'').*?(?<!\\\\)(\\\\\\\\)*?')/
LONG_STRING: /(?i)(?s)[ub]?r?(""".*?(?<!\\\\)(\\\\\\\\)*?"""|'''.*?(?<!\\\\)(\\\\\\\\)*?''')/
STRING : /(?i)[ubf]?r?("(?!"").*?(?<!\\\\)(\\\\\\\\)*?"|'(?!'').*?(?<!\\\\)(\\\\\\\\)*?')/
LONG_STRING: /(?i)(?s)[ubf]?r?(""".*?(?<!\\\\)(\\\\\\\\)*?"""|'''.*?(?<!\\\\)(\\\\\\\\)*?''')/
DEC_NUMBER: /(?i)[1-9]\d*l?/
HEX_NUMBER: /(?i)0x[\da-f]*l?/

View File

@ -77,7 +77,7 @@ TOKENS = {
'_TO': '->',
'_IGNORE': r'%ignore',
'_IMPORT': r'%import',
'NUMBER': '\d+',
'NUMBER': r'\d+',
}
RULES = {
@ -294,7 +294,9 @@ def _literal_to_pattern(literal):
flags = None
assert v[0] == v[-1] and v[0] in '"/'
x = v[1:-1].replace("'", r"\'")
x = v[1:-1]
x = re.sub(r'(\\[wd/]|\\\[|\\\])', r'\\\1', x)
x = x.replace("'", r"\'")
s = literal_eval("u'''%s'''" % x)
return { 'STRING': PatternStr,
'REGEXP': PatternRE }[literal.type](s, flags)

View File

@ -30,7 +30,7 @@ nearley_grammar = r"""
js: JS?
NAME: /[a-zA-Z_$]\w*/
COMMENT: /\#[^\n]*/
COMMENT: /#[^\n]*/
REGEXP: /\[.*?\]/
STRING: /".*?"/