From 9daacb9082dea08fee0f81eb94f353fd5af559e2 Mon Sep 17 00:00:00 2001 From: Erez Shinan Date: Sun, 13 May 2018 00:42:50 +0300 Subject: [PATCH] Refactored transformers, better code --- lark/__init__.py | 1 - lark/load_grammar.py | 31 ++++++------ lark/transformers.py | 113 ++++++++++++++++++++++++++++--------------- lark/tree.py | 2 - lark/utils.py | 20 ++------ tests/test_parser.py | 4 +- 6 files changed, 98 insertions(+), 73 deletions(-) diff --git a/lark/__init__.py b/lark/__init__.py index a2a67b9..5613855 100644 --- a/lark/__init__.py +++ b/lark/__init__.py @@ -3,6 +3,5 @@ from .transformers import Transformer from .common import ParseError, GrammarError, UnexpectedToken from .lexer import UnexpectedInput, LexError from .lark import Lark -from .utils import inline_args __version__ = "0.5.6" diff --git a/lark/load_grammar.py b/lark/load_grammar.py index c710bf8..49b4b8f 100644 --- a/lark/load_grammar.py +++ b/lark/load_grammar.py @@ -16,7 +16,7 @@ from .grammar import RuleOptions, Rule, Terminal, NonTerminal, Symbol from .utils import classify, suppress from .tree import Tree, SlottedTree as ST -from .transformers import Transformer, ChildrenTransformer, inline_args, Visitor +from .transformers import Transformer, Visitor, children_args, children_args_inline __path__ = os.path.dirname(__file__) IMPORT_PATHS = [os.path.join(__path__, 'grammars')] @@ -138,8 +138,8 @@ RULES = { } -@inline_args -class EBNF_to_BNF(ChildrenTransformer): +@children_args_inline +class EBNF_to_BNF(Transformer): def __init__(self): self.new_rules = [] self.rules_by_expr = {} @@ -232,7 +232,8 @@ class SimplifyRule_Visitor(Visitor): tree.children = list(set(tree.children)) -class RuleTreeToText(ChildrenTransformer): +@children_args +class RuleTreeToText(Transformer): def expansions(self, x): return x def expansion(self, symbols): @@ -243,8 +244,8 @@ class RuleTreeToText(ChildrenTransformer): return expansion, alias.value -@inline_args -class CanonizeTree(ChildrenTransformer): +@children_args_inline +class CanonizeTree(Transformer): def maybe(self, expr): return ST('expr', [expr, Token('OP', '?', -1)]) @@ -254,8 +255,8 @@ class CanonizeTree(ChildrenTransformer): tokenmods, value = args return tokenmods + [value] -@inline_args -class PrepareAnonTerminals(ChildrenTransformer): +@children_args_inline +class PrepareAnonTerminals(Transformer): "Create a unique list of anonymous tokens. Attempt to give meaningful names to them when we add them" def __init__(self, tokens): @@ -354,8 +355,8 @@ def _literal_to_pattern(literal): 'REGEXP': PatternRE }[literal.type](s, flags) -@inline_args -class PrepareLiterals(ChildrenTransformer): +@children_args_inline +class PrepareLiterals(Transformer): def literal(self, literal): return ST('pattern', [_literal_to_pattern(literal)]) @@ -368,7 +369,8 @@ class PrepareLiterals(ChildrenTransformer): return ST('pattern', [PatternRE(regexp)]) -class TokenTreeToPattern(ChildrenTransformer): +@children_args +class TokenTreeToPattern(Transformer): def pattern(self, ps): p ,= ps return p @@ -407,7 +409,8 @@ class TokenTreeToPattern(ChildrenTransformer): def value(self, v): return v[0] -class PrepareSymbols(ChildrenTransformer): +@children_args +class PrepareSymbols(Transformer): def value(self, v): v ,= v if isinstance(v, Tree): @@ -532,8 +535,8 @@ def options_from_rule(name, *x): def symbols_from_strcase(expansion): return [Terminal(x, filter_out=x.startswith('_')) if is_terminal(x) else NonTerminal(x) for x in expansion] -@inline_args -class PrepareGrammar(ChildrenTransformer): +@children_args_inline +class PrepareGrammar(Transformer): def terminal(self, name): return name def nonterminal(self, name): diff --git a/lark/transformers.py b/lark/transformers.py index d5f13a1..41a3980 100644 --- a/lark/transformers.py +++ b/lark/transformers.py @@ -1,7 +1,7 @@ import inspect from functools import wraps -from . import utils +from .utils import smart_decorator from .tree import Tree class Discard(Exception): @@ -13,46 +13,27 @@ class Base: return getattr(self, tree.data, self.__default__)(tree) def __default__(self, tree): + "Default operation on tree (for override)" return tree class Transformer(Base): def _transform_children(self, children): for c in children: try: - yield self._transform(c) if isinstance(c, Tree) else c + yield self._transform_tree(c) if isinstance(c, Tree) else c except Discard: pass - def _transform(self, tree): + def _transform_tree(self, tree): tree = Tree(tree.data, list(self._transform_children(tree.children))) return self._call_userfunc(tree) def transform(self, tree): - return self._transform(tree) + return self._transform_tree(tree) def __mul__(self, other): return TransformerChain(self, other) -class ChildrenTransformer(Transformer): - def _call_userfunc(self, tree): - # Assumes tree is already transformed - try: - f = getattr(self, tree.data) - except AttributeError: - return self.__default__(tree) - else: - return f(tree.children) - -class ChildrenInlineTransformer(Transformer): - def _call_userfunc(self, tree): - # Assumes tree is already transformed - try: - f = getattr(self, tree.data) - except AttributeError: - return self.__default__(tree) - else: - return f(*tree.children) - class TransformerChain(object): def __init__(self, *transformers): @@ -68,14 +49,22 @@ class TransformerChain(object): class Transformer_InPlace(Transformer): - def _transform(self, tree): + def _transform_tree(self, tree): # Cancel recursion return self._call_userfunc(tree) def transform(self, tree): for subtree in tree.iter_subtrees(): subtree.children = list(self._transform_children(subtree.children)) - return self._transform(tree) + return self._transform_tree(tree) + + +class Transformer_InPlaceRecursive(Transformer): + def _transform_tree(self, tree): + tree.children = list(self._transform_children(tree.children)) + return self._call_userfunc(tree) + + class Visitor(Base): "Bottom-up visitor" @@ -85,11 +74,6 @@ class Visitor(Base): self._call_userfunc(subtree) return tree -class Transformer_InPlaceRecursive(Transformer): - def _transform(self, tree): - tree.children = list(self._transform_children(tree.children)) - return self._call_userfunc(tree) - class Visitor_Recursive(Base): def visit(self, tree): for child in tree.children: @@ -101,7 +85,6 @@ class Visitor_Recursive(Base): return tree -from functools import wraps def visit_children_decor(func): @wraps(func) def inner(cls, tree): @@ -126,11 +109,63 @@ class Interpreter(object): return self.visit_children(tree) -def inline_args(obj): - if inspect.isclass(obj) and issubclass(obj, ChildrenTransformer): - class _NewTransformer(ChildrenInlineTransformer, obj): - pass - return _NewTransformer - else: - return utils.inline_args(obj) +def _children_args__func(f): + @wraps(f) + def create_decorator(_f, with_self): + if with_self: + def f(self, tree): + return _f(self, tree.children) + else: + def f(args): + return _f(tree.children) + + return smart_decorator(f, create_decorator) + +def _children_args__class(cls): + def _call_userfunc(self, tree): + # Assumes tree is already transformed + try: + f = getattr(self, tree.data) + except AttributeError: + return self.__default__(tree) + else: + return f(tree.children) + cls._call_userfunc = _call_userfunc + return cls + + +def children_args(obj): + decorator = _children_args__class if issubclass(obj, Base) else _children_args__func + return decorator(obj) + + + +def _children_args_inline__func(f): + @wraps(f) + def create_decorator(_f, with_self): + if with_self: + def f(self, tree): + return _f(self, *tree.children) + else: + def f(args): + return _f(*tree.children) + + return smart_decorator(f, create_decorator) + + +def _children_args_inline__class(cls): + def _call_userfunc(self, tree): + # Assumes tree is already transformed + try: + f = getattr(self, tree.data) + except AttributeError: + return self.__default__(tree) + else: + return f(*tree.children) + cls._call_userfunc = _call_userfunc + return cls + +def children_args_inline(obj): + decorator = _children_args_inline__class if issubclass(obj, Base) else _children_args_inline__func + return decorator(obj) diff --git a/lark/tree.py b/lark/tree.py index a2b0bef..e20c18d 100644 --- a/lark/tree.py +++ b/lark/tree.py @@ -5,8 +5,6 @@ except ImportError: from copy import deepcopy -from .utils import inline_args - class Meta: pass diff --git a/lark/utils.py b/lark/utils.py index 5e09e7d..a4a63dd 100644 --- a/lark/utils.py +++ b/lark/utils.py @@ -50,22 +50,22 @@ except NameError: # Python 3 ###{standalone import types -import functools +from functools import wraps, partial from contextlib import contextmanager Str = type(u'') def smart_decorator(f, create_decorator): if isinstance(f, types.FunctionType): - return functools.wraps(create_decorator(f, True)) + return wraps(create_decorator(f, True)) elif isinstance(f, (type, types.BuiltinFunctionType)): - return functools.wraps(create_decorator(f, False)) + return wraps(create_decorator(f, False)) elif isinstance(f, types.MethodType): - return functools.wraps(create_decorator(f.__func__, True)) + return wraps(create_decorator(f.__func__, True)) - elif isinstance(f, functools.partial): + elif isinstance(f, partial): # wraps does not work for partials in 2.7: https://bugs.python.org/issue3445 return create_decorator(f.__func__, True) @@ -73,16 +73,6 @@ def smart_decorator(f, create_decorator): return create_decorator(f.__func__.__call__, True) -def inline_args(f): - def create_decorator(_f, with_self): - if with_self: - def f(self, args): - return _f(self, *args) - else: - def f(args): - return _f(*args) - - return smart_decorator(f, create_decorator) try: diff --git a/tests/test_parser.py b/tests/test_parser.py index e147595..6531cb6 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -21,8 +21,7 @@ from lark.lark import Lark from lark.common import GrammarError, ParseError, UnexpectedToken from lark.lexer import LexError, UnexpectedInput from lark.tree import Tree -from lark.transformers import ChildrenTransformer as Transformer -# from lark.tree import Transformer +from lark.transformers import Transformer, children_args __path__ = os.path.dirname(__file__) def _read(n, *args): @@ -94,6 +93,7 @@ class TestParsers(unittest.TestCase): self.assertEqual( r.children[0].data, "c" ) def test_embedded_transformer(self): + @children_args class T(Transformer): def a(self, children): return ""