diff --git a/lark/common.py b/lark/common.py index 702c3cc..c8336ec 100644 --- a/lark/common.py +++ b/lark/common.py @@ -37,4 +37,3 @@ class ParserConf: self.rules = rules self.callback = callback self.start = start - diff --git a/lark/lark.py b/lark/lark.py index f2fd601..eb0307b 100644 --- a/lark/lark.py +++ b/lark/lark.py @@ -1,10 +1,12 @@ from __future__ import absolute_import import os +import time +from collections import defaultdict -from .utils import STRING_TYPE, inline_args +from .utils import STRING_TYPE from .load_grammar import load_grammar -from .tree import Tree, Transformer +from .tree import Tree from .common import GrammarError, LexerConf, ParserConf from .lexer import Lexer @@ -44,7 +46,8 @@ class LarkOptions(object): assert self.parser in ENGINE_DICT if self.parser == 'earley' and self.transformer: - raise ValueError('Cannot specify an auto-transformer when using the Earley algorithm. Please use your transformer on the resulting parse tree, or use a different algorithm (i.e. lalr)') + raise ValueError('Cannot specify an auto-transformer when using the Earley algorithm.' + 'Please use your transformer on the resulting parse tree, or use a different algorithm (i.e. lalr)') if self.keep_all_tokens: raise NotImplementedError("Not implemented yet!") @@ -52,8 +55,6 @@ class LarkOptions(object): raise ValueError("Unknown options: %s" % o.keys()) -import time -from collections import defaultdict class Profiler: def __init__(self): self.total_time = defaultdict(float) @@ -67,7 +68,7 @@ class Profiler: self.cur_section = name def make_wrapper(self, name, f): - def _f(*args, **kwargs): + def wrapper(*args, **kwargs): last_section = self.cur_section self.enter_section(name) try: @@ -75,7 +76,7 @@ class Profiler: finally: self.enter_section(last_section) - return _f + return wrapper class Lark: diff --git a/lark/lexer.py b/lark/lexer.py index 1f8a287..efdbba3 100644 --- a/lark/lexer.py +++ b/lark/lexer.py @@ -13,7 +13,7 @@ class TokenDef(object): self.value = value def __repr__(self): - return ('%s(%r, %r)' % (type(self).__name__, self.name, self.value)) + return '%s(%r, %r)' % (type(self).__name__, self.name, self.value) class TokenDef__Str(TokenDef): def to_regexp(self): @@ -40,16 +40,16 @@ class UnexpectedInput(LexError): self.context = context class Token(Str): - def __new__(cls, type, value, pos_in_stream=None): + def __new__(cls, type_, value, pos_in_stream=None): inst = Str.__new__(cls, value) - inst.type = type + inst.type = type_ inst.pos_in_stream = pos_in_stream inst.value = value return inst @classmethod - def new_borrow_pos(cls, type, value, borrow_t): - inst = cls(type, value, borrow_t.pos_in_stream) + def new_borrow_pos(cls, type_, value, borrow_t): + inst = cls(type_, value, borrow_t.pos_in_stream) inst.line = borrow_t.line inst.column = borrow_t.column return inst @@ -66,11 +66,11 @@ def _regexp_has_newline(r): return '\n' in r or '\\n' in r or ('(?s)' in r and '.' in r) def _create_unless_callback(strs): - def f(t): + def unless_callback(t): if t in strs: t.type = strs[t] return t - return f + return unless_callback def _create_unless(tokens): tokens_by_type = classify(tokens, type) @@ -169,4 +169,3 @@ class Lexer(object): if lex_pos < len(stream): raise UnexpectedInput(stream, lex_pos, line, lex_pos - col_start_pos) break - diff --git a/lark/load_grammar.py b/lark/load_grammar.py index 78990e0..9a0bb00 100644 --- a/lark/load_grammar.py +++ b/lark/load_grammar.py @@ -53,14 +53,14 @@ _TOKEN_NAMES = { # Grammar Parser TOKENS = { - '_LPAR': '\(', - '_RPAR': '\)', - '_LBRA': '\[', - '_RBRA': '\]', + '_LPAR': r'\(', + '_RPAR': r'\)', + '_LBRA': r'\[', + '_RBRA': r'\]', 'OP': '[+*?](?![a-z])', '_COLON': ':', - '_OR': '\|', - '_DOT': '\.', + '_OR': r'\|', + '_DOT': r'\.', 'RULE': '[_?*]?[a-z][_a-z0-9]*', 'TOKEN': '_?[A-Z][_A-Z0-9]*', 'STRING': r'".*?[^\\]"', @@ -99,7 +99,7 @@ RULES = { 'maybe': ['_LBRA expansions _RBRA'], - 'token': ['TOKEN _COLON tokenvalue _NL', + 'token': ['TOKEN _COLON tokenvalue _NL', 'TOKEN tokenmods _COLON tokenvalue _NL'], '?tokenvalue': ['REGEXP', 'STRING'], @@ -264,7 +264,7 @@ class ExtractAnonTokens(InlineTransformer): value = token.value self.i += 1 else: - assert False, x + assert False, token if token_name not in self.token_set: self.token_set.add(token_name) @@ -373,41 +373,3 @@ class GrammarLoader: return tokendefs, rules load_grammar = GrammarLoader().load_grammar - - - -def test(): - g = """ - start: add - - // Rules - add: mul - | add _add_sym mul - - mul: [mul _add_mul] _atom - - _atom: "-" _atom -> neg - | NUMBER - | "(" add ")" - - // Tokens - NUMBER: /[\d.]+/ - _add_sym: "+" | "-" - _add_mul: "*" | "/" - - WS.ignore.newline: /\s+/ - """ - - g2 = """ - start: a - a: "a" (b*|(c d)+) "b"? - b: "b" - c: "c" - d: "+" | "-" - """ - # print load_grammar(g) - print(GrammarLoader().load_grammar2(g)) - - -if __name__ == '__main__': - test() diff --git a/lark/parse_tree_builder.py b/lark/parse_tree_builder.py index c1302a3..060fc76 100644 --- a/lark/parse_tree_builder.py +++ b/lark/parse_tree_builder.py @@ -5,12 +5,12 @@ class Callback(object): def create_expand1_tree_builder_function(tree_builder): - def f(children): + def expand1(children): if len(children) == 1: return children[0] else: return tree_builder(children) - return f + return expand1 def create_rule_handler(expansion, usermethod): to_include = [(i, sym.startswith('_')) for i, sym in enumerate(expansion) @@ -38,9 +38,10 @@ class ParseTreeBuilder: def _create_tree_builder_function(self, name): tree_class = self.tree_class - def f(children): + def tree_builder_f(children): return tree_class(name, children) - return f + return tree_builder_f + def create_tree_builder(self, rules, transformer): @@ -77,5 +78,3 @@ class ParseTreeBuilder: new_rules.append(( _origin, expansion, _alias )) return new_rules, callback - - diff --git a/lark/parser_frontends.py b/lark/parser_frontends.py index fc016a0..5a71e6d 100644 --- a/lark/parser_frontends.py +++ b/lark/parser_frontends.py @@ -52,4 +52,3 @@ class Earley(WithLexer): ENGINE_DICT = { 'lalr': LALR, 'earley': Earley } - diff --git a/lark/parsers/earley.py b/lark/parsers/earley.py index dc0cf97..a9dbdfc 100644 --- a/lark/parsers/earley.py +++ b/lark/parsers/earley.py @@ -89,7 +89,6 @@ class State(object): new_copy = self.consume_nonterminal(r.name) if r.postprocess: new_copy.data[-1] = r.postprocess([]) - # new_copy.data[-1] = r.postprocess([], self.reference) else: new_copy.data[-1] = [] @@ -154,4 +153,3 @@ class Parser(object): and t.reference == 0 and t.data is not MatchFailed): yield t.data - diff --git a/lark/parsers/lalr_analysis.py b/lark/parsers/lalr_analysis.py index 6ed7b4c..e50de18 100644 --- a/lark/parsers/lalr_analysis.py +++ b/lark/parsers/lalr_analysis.py @@ -192,7 +192,8 @@ class GrammarAnalyzer(object): self.states[state] = {k:v[0] for k, v in lookahead.items()} - x = list(bfs([self.init_state], step)) + for _ in bfs([self.init_state], step): + pass # -- self.enum = list(self.states) @@ -207,4 +208,3 @@ class GrammarAnalyzer(object): self.init_state_idx = self.enum_rev[self.init_state] - diff --git a/lark/tree.py b/lark/tree.py index 34f3c76..58dbf9e 100644 --- a/lark/tree.py +++ b/lark/tree.py @@ -40,7 +40,7 @@ class Tree(object): if pred(self): yield self else: - for i, c in enumerate(self.children): + for c in self.children: if isinstance(c, Tree): for t in c.find_pred(pred): yield t @@ -48,9 +48,6 @@ class Tree(object): def find_data(self, data): return self.find_pred(lambda t: t.data == data) - # def clone(self): - # return Tree(self.data, [c.clone() if isinstance(c, Tree) else c for c in self.children]) - def __deepcopy__(self, memo): return type(self)(self.data, deepcopy(self.children, memo)) @@ -69,7 +66,6 @@ class Transformer(object): else: return f(items) - def __default__(self, data, children): return Tree(data, children) @@ -79,7 +75,6 @@ class InlineTransformer(Transformer): return inline_args(getattr(self, name)).__get__(self) - class Visitor(object): def visit(self, tree): for child in tree.children: @@ -92,5 +87,3 @@ class Visitor(object): def __default__(self, tree): pass - - diff --git a/lark/utils.py b/lark/utils.py index 4f51a66..719204d 100644 --- a/lark/utils.py +++ b/lark/utils.py @@ -1,3 +1,5 @@ +import functools +import types from collections import deque class fzset(frozenset): @@ -49,8 +51,6 @@ except NameError: # Python 3 Str = type(u'') -import functools -import types def inline_args(f): # print '@@', f.__name__, type(f), isinstance(f, types.FunctionType), isinstance(f, types.TypeType), isinstance(f, types.BuiltinFunctionType) if isinstance(f, types.FunctionType):