from typing import Any, Dict, List, Tuple, Union
from .. import Vocab
from .doc import Doc
from .span import Span
from .token import Token
class Retokenizer:
def __init__(self, doc: Doc) -> None: ...
def merge(self, span: Span, attrs: Dict[Union[str, int], Any] = ...) -> None: ...
def split(
self,
token: Token,
orths: List[str],
heads: List[Union[Token, Tuple[Token, int]]],
attrs: Dict[Union[str, int], List[Any]] = ...,
) -> None: ...
def __enter__(self) -> Retokenizer: ...
def __exit__(self, *args: Any) -> None: ...
def normalize_token_attrs(vocab: Vocab, attrs: Dict): ...
def set_token_attrs(py_token: Token, attrs: Dict): ...
