diff --git a/lark-stubs/__init__.pyi b/lark-stubs/__init__.pyi index bb2d018..7545f2d 100644 --- a/lark-stubs/__init__.pyi +++ b/lark-stubs/__init__.pyi @@ -6,4 +6,4 @@ from .exceptions import * from .lexer import * from .lark import * -__version__: str +__version__: str = ... diff --git a/lark-stubs/exceptions.pyi b/lark-stubs/exceptions.pyi index 6c3c4ab..9bdb85d 100644 --- a/lark-stubs/exceptions.pyi +++ b/lark-stubs/exceptions.pyi @@ -23,7 +23,7 @@ class LexError(LarkError): class UnexpectedInput(LarkError): pos_in_stream: int - def get_context(self, text: str, span: int = 40): + def get_context(self, text: str, span: int = ...): ... def match_examples( @@ -39,4 +39,9 @@ class UnexpectedToken(ParseError, UnexpectedInput): class UnexpectedCharacters(LexError, UnexpectedInput): + line: int + column: int + + +class VisitError(LarkError): pass diff --git a/lark-stubs/indenter.pyi b/lark-stubs/indenter.pyi new file mode 100644 index 0000000..11a9058 --- /dev/null +++ b/lark-stubs/indenter.pyi @@ -0,0 +1,53 @@ +# -*- coding: utf-8 -*- + +from typing import Tuple, List, Iterator, Optional +from abc import ABC, abstractmethod +from .lexer import Token + + +class Indenter(ABC): + paren_level: Optional[int] + indent_level: Optional[List[int]] + + def __init__(self): + ... + + def handle_NL(self, token: Token) -> Iterator[Token]: + ... + + def process(self, stream: Iterator[Token]) -> Iterator[Token]: + ... + + @property + def always_accept(self) -> Tuple[str]: + ... + + @property + @abstractmethod + def NL_type(self) -> str: + ... + + @property + @abstractmethod + def OPEN_PAREN_types(self) -> List[str]: + ... + + @property + @abstractmethod + def CLOSE_PAREN_types(self) -> List[str]: + ... + + @property + @abstractmethod + def INDENT_type(self) -> str: + ... + + @property + @abstractmethod + def DEDENT_type(self) -> str: + ... + + @property + @abstractmethod + def tab_len(self) -> int: + ... diff --git a/lark-stubs/lark.pyi b/lark-stubs/lark.pyi index 420cc59..b3aa36d 100644 --- a/lark-stubs/lark.pyi +++ b/lark-stubs/lark.pyi @@ -1,25 +1,57 @@ # -*- coding: utf-8 -*- -from typing import List, Dict, IO, Callable, Union, Optional, Literal +from typing import ( + TypeVar, Type, List, Dict, IO, Iterator, Callable, Union, Optional, + Literal, Protocol, +) from .visitors import Transformer -from .lexer import Lexer, Token +from .lexer import Token, Lexer, TerminalDef from .tree import Tree +_T = TypeVar('_T') _Start = Union[None, str, List[str]] +_Parser = Literal["earley", "lalr", "cyk"] +_Lexer = Union[Literal["auto", "standard", "contextual", "dynamic", "dynamic_complete"], Lexer] +_Ambiguity = Literal["explicit", "resolve"] + + +class PostLex(Protocol): + + def process(self, stream: Iterator[Token]) -> Iterator[Token]: + ... + + +class LarkOptions: + start: _Start + parser: _Parser + lexer: _Lexer + transformer: Optional[Transformer] + postlex: Optional[PostLex] + ambiguity: _Ambiguity + debug: bool + keep_all_tokens: bool + propagate_positions: bool + maybe_placeholders: bool + lexer_callbacks: Dict[str, Callable[[Token], Token]] + cache_grammar: bool class Lark: + source: str + options: LarkOptions + lexer: Lexer + terminals: List[TerminalDef] def __init__( self, grammar: Union[str, IO[str]], *, start: _Start = ..., - parser: Literal["earley", "lalr", "cyk"] = ..., - lexer: Optional[Lexer] = ..., - transformer: Optional[Transformer] = ..., - postlex: Optional[Literal["standard", "contextual"]] = ..., - ambiguity: Literal["explicit", "resolve"] = ..., + parser: _Parser = ..., + lexer: _Lexer = ..., + transformer: Optional[Transformer] = None, + postlex: Optional[PostLex] = None, + ambiguity: _Ambiguity = ..., debug: bool = False, keep_all_tokens: bool = False, propagate_positions: bool = False, @@ -30,3 +62,13 @@ class Lark: def parse(self, text: str, start: _Start = None) -> Tree: ... + + @classmethod + def open(cls: Type[_T], grammar_filename: str, rel_to: Optional[str] = None, **options) -> _T: + ... + + def lex(self, text: str) -> Iterator[Token]: + ... + + def get_terminal(self, name: str) -> TerminalDef: + ... diff --git a/lark-stubs/lexer.pyi b/lark-stubs/lexer.pyi index abe321b..171f6ac 100644 --- a/lark-stubs/lexer.pyi +++ b/lark-stubs/lexer.pyi @@ -1,64 +1,202 @@ # -*- coding: utf-8 -*- -from typing import Tuple, Iterator, Sized +from typing import ( + TypeVar, Type, Tuple, List, Dict, Iterator, Collection, Callable, Optional, + Pattern as REPattern, +) from abc import abstractmethod, ABC +_T = TypeVar('_T') +_MRes = List[Tuple[REPattern, Dict[int, str]]] + class Pattern(ABC): + value: str + flags: Collection[str] + + def __init__(self, value: str, flags: Collection[str] = ...): + ... + + @property + @abstractmethod + def type(self) -> str: + ... @abstractmethod def to_regexp(self) -> str: ... + @property + @abstractmethod + def min_width(self) -> int: + ... + + @property + @abstractmethod + def max_width(self) -> int: + ... + class PatternStr(Pattern): + type: str = ... def to_regexp(self) -> str: ... + @property + def min_width(self) -> int: + ... + + @property + def max_width(self) -> int: + ... + class PatternRE(Pattern): + type: str = ... def to_regexp(self) -> str: ... + @property + def min_width(self) -> int: + ... + + @property + def max_width(self) -> int: + ... + class TerminalDef: name: str pattern: Pattern priority: int + def __init__(self, name: str, pattern: Pattern, priority: int = ...): + ... + class Token(str): type: str pos_in_stream: int + value: str line: int column: int end_line: int end_column: int end_pos: int + def update(self, type_: Optional[str] = None, value: Optional[str] = None) -> Token: + ... -class Lexer(ABC): - - @abstractmethod - def lex(self, stream: Sized) -> Iterator[Token]: + @classmethod + def new_borrow_pos(cls: Type[_T], type_: str, value: str, borrow_t: Token) -> _T: ... +_Callback = Callable[[Token], Token] + + +def build_mres(terminals: Collection[TerminalDef], match_whole: bool = False) -> _MRes: + ... + + +class UnlessCallback: + mres: _MRes + + def __init__(self, mres: _MRes): + ... + + def __call__(self, t: Token) -> Token: + ... + + +class CallChain: + callback1: _Callback + callback2: _Callback + cond: Callable[[Token], bool] + + def __init__( + self, + callback1: _Callback, + callback2: _Callback, + cond: Callable[[Token], bool] + ): + ... + + +class LineCounter: + newline_char: str + char_pos: int + line: int + column: int + line_start_pos: int + + def __init__(self): + ... + + def feed(self, token: str, test_newline: bool = True): + ... + + +class _Lex: + lexer: TraditionalLexer + + def __init__(self, lexer: TraditionalLexer, state: Optional[str] = None): + ... + + def lex( + self, + stream: str, + newline_types: Collection[str], + ignore_types: Collection[str] + ) -> Iterator[Token]: + ... + + +class Lexer(ABC): + lex: Callable[..., Iterator[Token]] + + class TraditionalLexer(Lexer): + terminals: Collection[TerminalDef] + ignore_types: List[str] + newline_types: List[str] + user_callbacks: Dict[str, _Callback] + callback: Dict[str, _Callback] + mres: _MRes + + def __init__( + self, + terminals: Collection[TerminalDef], + ignore: Collection[str] = ..., + user_callbacks: Dict[str, _Callback] = ... + ): + ... def build(self) -> None: ... - def match(self, stream: str, pos: int) -> Tuple[str, str]: + def match(self, stream: str, pos: int) -> Optional[Tuple[str, str]]: ... - def lex(self, stream: Sized) -> Iterator[Token]: + def lex(self, stream: str) -> Iterator[Token]: ... class ContextualLexer(Lexer): + lexers: Dict[str, TraditionalLexer] + root_lexer: TraditionalLexer - def lex(self, stream: Sized) -> Iterator[Token]: + def __init__( + self, + terminals: Collection[TerminalDef], + states: Dict[str, Collection[str]], + ignore: Collection[str] = ..., + always_accept: Collection[str] = ..., + user_callbacks: Dict[str, _Callback] = ... + ): + ... + + def lex(self, stream: str, get_parser_state: Callable[[], str]) -> Iterator[Token]: ... diff --git a/lark-stubs/reconstruct.pyi b/lark-stubs/reconstruct.pyi index 89e51d4..8ac6c14 100644 --- a/lark-stubs/reconstruct.pyi +++ b/lark-stubs/reconstruct.pyi @@ -1,7 +1,31 @@ # -*- coding: utf-8 -*- +from typing import List, Dict, Union from .lark import Lark from .tree import Tree +from .visitors import Transformer_InPlace +from .lexer import TerminalDef + + +class WriteTokensTransformer(Transformer_InPlace): + + def __init__(self, tokens: Dict[str, TerminalDef], term_subs): + ... + + +class MatchTree(Tree): + pass + + +class MakeMatchTree: + name: str + expansion: List[TerminalDef] + + def __init__(self, name: str, expansion: List[TerminalDef]): + ... + + def __call__(self, args: List[Union[str, Tree]]): + ... class Reconstructor: diff --git a/lark-stubs/tree.pyi b/lark-stubs/tree.pyi index b794f53..a24ab35 100644 --- a/lark-stubs/tree.pyi +++ b/lark-stubs/tree.pyi @@ -1,16 +1,31 @@ # -*- coding: utf-8 -*- -from typing import List, Callable, Iterator, Union, Optional -from .lexer import Token +from typing import List, Callable, Iterator, Union, Optional, Literal +from .lexer import TerminalDef + +class Meta: + empty: bool + line: int + column: int + start_pos: int + end_line: int + end_column: int + end_pos: int + orig_expansion: List[TerminalDef] + match_tree: bool class Tree: - data: str children: List[Union[str, Tree]] - meta: Token + meta: Meta - def __init__(self, data: str, children: List[Tree], meta: Optional[Token] = None): + def __init__( + self, + data: str, + children: List[Union[str, Tree]], + meta: Optional[Meta] = None + ): ... def pretty(self, indent_str: str = ...) -> str: @@ -22,13 +37,22 @@ class Tree: def find_data(self, data: str) -> Iterator[Tree]: ... + def expand_kids_by_index(self, *indices: int) -> None: + ... + + def scan_values(self, pred: Callable[[Union[str, Tree]], bool]): + ... + def iter_subtrees(self) -> Iterator[Tree]: ... def iter_subtrees_topdown(self) -> Iterator[Tree]: ... - def __eq__(self, other: object) -> bool: + def copy(self) -> Tree: + ... + + def set(self, data: str, children: List[Union[str, Tree]]) -> None: ... def __hash__(self) -> int: @@ -37,3 +61,12 @@ class Tree: class SlottedTree(Tree): pass + + +def pydot__tree_to_png( + tree: Tree, + filename: str, + rankdir: Literal["TB", "LR", "BT", "RL"] = ..., + **kwargs +) -> None: + ... diff --git a/lark-stubs/visitors.pyi b/lark-stubs/visitors.pyi index ac1367a..2aafd2e 100644 --- a/lark-stubs/visitors.pyi +++ b/lark-stubs/visitors.pyi @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from typing import TypeVar, List, Callable, Generic, Type +from typing import TypeVar, Tuple, List, Callable, Generic, Type from abc import ABC from .tree import Tree @@ -17,6 +17,22 @@ class Transformer(ABC, Generic[_T]): def transform(self, tree: Tree) -> _T: ... + def __mul__(self, other: Transformer[_T]) -> TransformerChain[_T]: + ... + + +class TransformerChain(Generic[_T]): + transformers: Tuple[Transformer[_T], ...] + + def __init__(self, *transformers: Transformer[_T]): + ... + + def transform(self, tree: Tree) -> _T: + ... + + def __mul__(self, other: Transformer[_T]) -> TransformerChain[_T]: + ... + class Transformer_InPlace(Transformer): pass diff --git a/lark/visitors.py b/lark/visitors.py index 8d88c96..3e3612f 100644 --- a/lark/visitors.py +++ b/lark/visitors.py @@ -48,6 +48,7 @@ class Transformer(_Decoratable): Can be used to implement map or reduce. """ __visit_tokens__ = True # For backwards compatibility + def __init__(self, visit_tokens=True): self.__visit_tokens__ = visit_tokens