2020-09-21 08:59:07 +00:00
from typing import Dict , List , Union , Optional , Any , Callable , Type , Tuple
2020-08-05 14:00:59 +00:00
from typing import Iterable , TypeVar , TYPE_CHECKING
2022-01-20 12:18:39 +00:00
from . compat import Literal
2019-12-25 11:39:49 +00:00
from enum import Enum
2020-09-29 09:52:45 +00:00
from pydantic import BaseModel , Field , ValidationError , validator , create_model
2022-06-30 09:01:58 +00:00
from pydantic import StrictStr , StrictInt , StrictFloat , StrictBool , ConstrainedStr
2020-09-29 09:52:45 +00:00
from pydantic . main import ModelMetaclass
2021-06-28 09:29:29 +00:00
from thinc . api import Optimizer , ConfigValidationError , Model
2020-09-27 20:21:31 +00:00
from thinc . config import Promise
2019-12-25 11:39:49 +00:00
from collections import defaultdict
2020-09-29 09:52:45 +00:00
import inspect
2022-06-30 09:01:58 +00:00
import re
2019-12-25 11:39:49 +00:00
from . attrs import NAMES
2020-09-18 13:45:55 +00:00
from . lookups import Lookups
2020-09-29 16:08:02 +00:00
from . util import is_cython_func
2019-12-25 11:39:49 +00:00
2020-08-05 14:00:59 +00:00
if TYPE_CHECKING :
# This lets us add type hints for mypy etc. without causing circular imports
from . language import Language # noqa: F401
2020-09-09 08:31:03 +00:00
from . training import Example # noqa: F401
2021-06-28 09:29:29 +00:00
from . vocab import Vocab # noqa: F401
2020-08-05 14:00:59 +00:00
2020-09-27 20:21:31 +00:00
# fmt: off
2020-08-04 13:09:37 +00:00
ItemT = TypeVar ( " ItemT " )
2020-09-27 20:21:31 +00:00
Batcher = Union [ Callable [ [ Iterable [ ItemT ] ] , Iterable [ List [ ItemT ] ] ] , Promise ]
Reader = Union [ Callable [ [ " Language " , str ] , Iterable [ " Example " ] ] , Promise ]
Logger = Union [ Callable [ [ " Language " ] , Tuple [ Callable [ [ Dict [ str , Any ] ] , None ] , Callable ] ] , Promise ]
# fmt: on
2020-08-04 13:09:37 +00:00
2019-12-25 11:39:49 +00:00
2020-07-25 13:01:15 +00:00
def validate ( schema : Type [ BaseModel ] , obj : Dict [ str , Any ] ) - > List [ str ] :
2019-12-25 11:39:49 +00:00
""" Validate data against a given pydantic schema.
2020-07-25 13:01:15 +00:00
obj ( Dict [ str , Any ] ) : JSON - serializable data to validate .
2019-12-25 11:39:49 +00:00
schema ( pydantic . BaseModel ) : The schema to validate against .
2020-07-25 13:01:15 +00:00
RETURNS ( List [ str ] ) : A list of error messages , if available .
2019-12-25 11:39:49 +00:00
"""
try :
schema ( * * obj )
return [ ]
except ValidationError as e :
errors = e . errors ( )
data = defaultdict ( list )
for error in errors :
err_loc = " -> " . join ( [ str ( p ) for p in error . get ( " loc " , [ ] ) ] )
data [ err_loc ] . append ( error . get ( " msg " ) )
🏷 Add Mypy check to CI and ignore all existing Mypy errors (#9167)
* 🚨 Ignore all existing Mypy errors
* 🏗 Add Mypy check to CI
* Add types-mock and types-requests as dev requirements
* Add additional type ignore directives
* Add types packages to dev-only list in reqs test
* Add types-dataclasses for python 3.6
* Add ignore to pretrain
* 🏷 Improve type annotation on `run_command` helper
The `run_command` helper previously declared that it returned an
`Optional[subprocess.CompletedProcess]`, but it isn't actually possible
for the function to return `None`. These changes modify the type
annotation of the `run_command` helper and remove all now-unnecessary
`# type: ignore` directives.
* 🔧 Allow variable type redefinition in limited contexts
These changes modify how Mypy is configured to allow variables to have
their type automatically redefined under certain conditions. The Mypy
documentation contains the following example:
```python
def process(items: List[str]) -> None:
# 'items' has type List[str]
items = [item.split() for item in items]
# 'items' now has type List[List[str]]
...
```
This configuration change is especially helpful in reducing the number
of `# type: ignore` directives needed to handle the common pattern of:
* Accepting a filepath as a string
* Overwriting the variable using `filepath = ensure_path(filepath)`
These changes enable redefinition and remove all `# type: ignore`
directives rendered redundant by this change.
* 🏷 Add type annotation to converters mapping
* 🚨 Fix Mypy error in convert CLI argument verification
* 🏷 Improve type annotation on `resolve_dot_names` helper
* 🏷 Add type annotations for `Vocab` attributes `strings` and `vectors`
* 🏷 Add type annotations for more `Vocab` attributes
* 🏷 Add loose type annotation for gold data compilation
* 🏷 Improve `_format_labels` type annotation
* 🏷 Fix `get_lang_class` type annotation
* 🏷 Loosen return type of `Language.evaluate`
* 🏷 Don't accept `Scorer` in `handle_scores_per_type`
* 🏷 Add `string_to_list` overloads
* 🏷 Fix non-Optional command-line options
* 🙈 Ignore redefinition of `wandb_logger` in `loggers.py`
* ➕ Install `typing_extensions` in Python 3.8+
The `typing_extensions` package states that it should be used when
"writing code that must be compatible with multiple Python versions".
Since SpaCy needs to support multiple Python versions, it should be used
when newer `typing` module members are required. One example of this is
`Literal`, which is available starting with Python 3.8.
Previously SpaCy tried to import `Literal` from `typing`, falling back
to `typing_extensions` if the import failed. However, Mypy doesn't seem
to be able to understand what `Literal` means when the initial import
means. Therefore, these changes modify how `compat` imports `Literal` by
always importing it from `typing_extensions`.
These changes also modify how `typing_extensions` is installed, so that
it is a requirement for all Python versions, including those greater
than or equal to 3.8.
* 🏷 Improve type annotation for `Language.pipe`
These changes add a missing overload variant to the type signature of
`Language.pipe`. Additionally, the type signature is enhanced to allow
type checkers to differentiate between the two overload variants based
on the `as_tuple` parameter.
Fixes #8772
* ➖ Don't install `typing-extensions` in Python 3.8+
After more detailed analysis of how to implement Python version-specific
type annotations using SpaCy, it has been determined that by branching
on a comparison against `sys.version_info` can be statically analyzed by
Mypy well enough to enable us to conditionally use
`typing_extensions.Literal`. This means that we no longer need to
install `typing_extensions` for Python versions greater than or equal to
3.8! 🎉
These changes revert previous changes installing `typing-extensions`
regardless of Python version and modify how we import the `Literal` type
to ensure that Mypy treats it properly.
* resolve mypy errors for Strict pydantic types
* refactor code to avoid missing return statement
* fix types of convert CLI command
* avoid list-set confustion in debug_data
* fix typo and formatting
* small fixes to avoid type ignores
* fix types in profile CLI command and make it more efficient
* type fixes in projects CLI
* put one ignore back
* type fixes for render
* fix render types - the sequel
* fix BaseDefault in language definitions
* fix type of noun_chunks iterator - yields tuple instead of span
* fix types in language-specific modules
* 🏷 Expand accepted inputs of `get_string_id`
`get_string_id` accepts either a string (in which case it returns its
ID) or an ID (in which case it immediately returns the ID). These
changes extend the type annotation of `get_string_id` to indicate that
it can accept either strings or IDs.
* 🏷 Handle override types in `combine_score_weights`
The `combine_score_weights` function allows users to pass an `overrides`
mapping to override data extracted from the `weights` argument. Since it
allows `Optional` dictionary values, the return value may also include
`Optional` dictionary values.
These changes update the type annotations for `combine_score_weights` to
reflect this fact.
* 🏷 Fix tokenizer serialization method signatures in `DummyTokenizer`
* 🏷 Fix redefinition of `wandb_logger`
These changes fix the redefinition of `wandb_logger` by giving a
separate name to each `WandbLogger` version. For
backwards-compatibility, `spacy.train` still exports `wandb_logger_v3`
as `wandb_logger` for now.
* more fixes for typing in language
* type fixes in model definitions
* 🏷 Annotate `_RandomWords.probs` as `NDArray`
* 🏷 Annotate `tok2vec` layers to help Mypy
* 🐛 Fix `_RandomWords.probs` type annotations for Python 3.6
Also remove an import that I forgot to move to the top of the module 😅
* more fixes for matchers and other pipeline components
* quick fix for entity linker
* fixing types for spancat, textcat, etc
* bugfix for tok2vec
* type annotations for scorer
* add runtime_checkable for Protocol
* type and import fixes in tests
* mypy fixes for training utilities
* few fixes in util
* fix import
* 🐵 Remove unused `# type: ignore` directives
* 🏷 Annotate `Language._components`
* 🏷 Annotate `spacy.pipeline.Pipe`
* add doc as property to span.pyi
* small fixes and cleanup
* explicit type annotations instead of via comment
Co-authored-by: Adriane Boyd <adrianeboyd@gmail.com>
Co-authored-by: svlandeg <sofie.vanlandeghem@gmail.com>
Co-authored-by: svlandeg <svlandeg@github.com>
2021-10-14 13:21:40 +00:00
return [ f " [ { loc } ] { ' , ' . join ( msg ) } " for loc , msg in data . items ( ) ] # type: ignore[arg-type]
2019-12-25 11:39:49 +00:00
2020-09-29 09:52:45 +00:00
# Initialization
class ArgSchemaConfig :
extra = " forbid "
arbitrary_types_allowed = True
class ArgSchemaConfigExtra :
extra = " forbid "
arbitrary_types_allowed = True
def get_arg_model (
func : Callable ,
* ,
exclude : Iterable [ str ] = tuple ( ) ,
name : str = " ArgModel " ,
strict : bool = True ,
) - > ModelMetaclass :
""" Generate a pydantic model for function arguments.
func ( Callable ) : The function to generate the schema for .
exclude ( Iterable [ str ] ) : Parameter names to ignore .
name ( str ) : Name of created model class .
strict ( bool ) : Don ' t allow extra arguments if no variable keyword arguments
are allowed on the function .
RETURNS ( ModelMetaclass ) : A pydantic model .
"""
sig_args = { }
try :
sig = inspect . signature ( func )
except ValueError :
# Typically happens if the method is part of a Cython module without
# binding=True. Here we just use an empty model that allows everything.
🏷 Add Mypy check to CI and ignore all existing Mypy errors (#9167)
* 🚨 Ignore all existing Mypy errors
* 🏗 Add Mypy check to CI
* Add types-mock and types-requests as dev requirements
* Add additional type ignore directives
* Add types packages to dev-only list in reqs test
* Add types-dataclasses for python 3.6
* Add ignore to pretrain
* 🏷 Improve type annotation on `run_command` helper
The `run_command` helper previously declared that it returned an
`Optional[subprocess.CompletedProcess]`, but it isn't actually possible
for the function to return `None`. These changes modify the type
annotation of the `run_command` helper and remove all now-unnecessary
`# type: ignore` directives.
* 🔧 Allow variable type redefinition in limited contexts
These changes modify how Mypy is configured to allow variables to have
their type automatically redefined under certain conditions. The Mypy
documentation contains the following example:
```python
def process(items: List[str]) -> None:
# 'items' has type List[str]
items = [item.split() for item in items]
# 'items' now has type List[List[str]]
...
```
This configuration change is especially helpful in reducing the number
of `# type: ignore` directives needed to handle the common pattern of:
* Accepting a filepath as a string
* Overwriting the variable using `filepath = ensure_path(filepath)`
These changes enable redefinition and remove all `# type: ignore`
directives rendered redundant by this change.
* 🏷 Add type annotation to converters mapping
* 🚨 Fix Mypy error in convert CLI argument verification
* 🏷 Improve type annotation on `resolve_dot_names` helper
* 🏷 Add type annotations for `Vocab` attributes `strings` and `vectors`
* 🏷 Add type annotations for more `Vocab` attributes
* 🏷 Add loose type annotation for gold data compilation
* 🏷 Improve `_format_labels` type annotation
* 🏷 Fix `get_lang_class` type annotation
* 🏷 Loosen return type of `Language.evaluate`
* 🏷 Don't accept `Scorer` in `handle_scores_per_type`
* 🏷 Add `string_to_list` overloads
* 🏷 Fix non-Optional command-line options
* 🙈 Ignore redefinition of `wandb_logger` in `loggers.py`
* ➕ Install `typing_extensions` in Python 3.8+
The `typing_extensions` package states that it should be used when
"writing code that must be compatible with multiple Python versions".
Since SpaCy needs to support multiple Python versions, it should be used
when newer `typing` module members are required. One example of this is
`Literal`, which is available starting with Python 3.8.
Previously SpaCy tried to import `Literal` from `typing`, falling back
to `typing_extensions` if the import failed. However, Mypy doesn't seem
to be able to understand what `Literal` means when the initial import
means. Therefore, these changes modify how `compat` imports `Literal` by
always importing it from `typing_extensions`.
These changes also modify how `typing_extensions` is installed, so that
it is a requirement for all Python versions, including those greater
than or equal to 3.8.
* 🏷 Improve type annotation for `Language.pipe`
These changes add a missing overload variant to the type signature of
`Language.pipe`. Additionally, the type signature is enhanced to allow
type checkers to differentiate between the two overload variants based
on the `as_tuple` parameter.
Fixes #8772
* ➖ Don't install `typing-extensions` in Python 3.8+
After more detailed analysis of how to implement Python version-specific
type annotations using SpaCy, it has been determined that by branching
on a comparison against `sys.version_info` can be statically analyzed by
Mypy well enough to enable us to conditionally use
`typing_extensions.Literal`. This means that we no longer need to
install `typing_extensions` for Python versions greater than or equal to
3.8! 🎉
These changes revert previous changes installing `typing-extensions`
regardless of Python version and modify how we import the `Literal` type
to ensure that Mypy treats it properly.
* resolve mypy errors for Strict pydantic types
* refactor code to avoid missing return statement
* fix types of convert CLI command
* avoid list-set confustion in debug_data
* fix typo and formatting
* small fixes to avoid type ignores
* fix types in profile CLI command and make it more efficient
* type fixes in projects CLI
* put one ignore back
* type fixes for render
* fix render types - the sequel
* fix BaseDefault in language definitions
* fix type of noun_chunks iterator - yields tuple instead of span
* fix types in language-specific modules
* 🏷 Expand accepted inputs of `get_string_id`
`get_string_id` accepts either a string (in which case it returns its
ID) or an ID (in which case it immediately returns the ID). These
changes extend the type annotation of `get_string_id` to indicate that
it can accept either strings or IDs.
* 🏷 Handle override types in `combine_score_weights`
The `combine_score_weights` function allows users to pass an `overrides`
mapping to override data extracted from the `weights` argument. Since it
allows `Optional` dictionary values, the return value may also include
`Optional` dictionary values.
These changes update the type annotations for `combine_score_weights` to
reflect this fact.
* 🏷 Fix tokenizer serialization method signatures in `DummyTokenizer`
* 🏷 Fix redefinition of `wandb_logger`
These changes fix the redefinition of `wandb_logger` by giving a
separate name to each `WandbLogger` version. For
backwards-compatibility, `spacy.train` still exports `wandb_logger_v3`
as `wandb_logger` for now.
* more fixes for typing in language
* type fixes in model definitions
* 🏷 Annotate `_RandomWords.probs` as `NDArray`
* 🏷 Annotate `tok2vec` layers to help Mypy
* 🐛 Fix `_RandomWords.probs` type annotations for Python 3.6
Also remove an import that I forgot to move to the top of the module 😅
* more fixes for matchers and other pipeline components
* quick fix for entity linker
* fixing types for spancat, textcat, etc
* bugfix for tok2vec
* type annotations for scorer
* add runtime_checkable for Protocol
* type and import fixes in tests
* mypy fixes for training utilities
* few fixes in util
* fix import
* 🐵 Remove unused `# type: ignore` directives
* 🏷 Annotate `Language._components`
* 🏷 Annotate `spacy.pipeline.Pipe`
* add doc as property to span.pyi
* small fixes and cleanup
* explicit type annotations instead of via comment
Co-authored-by: Adriane Boyd <adrianeboyd@gmail.com>
Co-authored-by: svlandeg <sofie.vanlandeghem@gmail.com>
Co-authored-by: svlandeg <svlandeg@github.com>
2021-10-14 13:21:40 +00:00
return create_model ( name , __config__ = ArgSchemaConfigExtra ) # type: ignore[arg-type, return-value]
2020-09-29 09:52:45 +00:00
has_variable = False
for param in sig . parameters . values ( ) :
if param . name in exclude :
continue
if param . kind == param . VAR_KEYWORD :
# The function allows variable keyword arguments so we shouldn't
# include **kwargs etc. in the schema and switch to non-strict
# mode and pass through all other values
has_variable = True
continue
# If no annotation is specified assume it's anything
annotation = param . annotation if param . annotation != param . empty else Any
2020-09-29 16:08:02 +00:00
# If no default value is specified assume that it's required. Cython
# functions/methods will have param.empty for default value None so we
# need to treat them differently
default_empty = None if is_cython_func ( func ) else . . .
default = param . default if param . default != param . empty else default_empty
2020-09-29 09:52:45 +00:00
sig_args [ param . name ] = ( annotation , default )
is_strict = strict and not has_variable
🏷 Add Mypy check to CI and ignore all existing Mypy errors (#9167)
* 🚨 Ignore all existing Mypy errors
* 🏗 Add Mypy check to CI
* Add types-mock and types-requests as dev requirements
* Add additional type ignore directives
* Add types packages to dev-only list in reqs test
* Add types-dataclasses for python 3.6
* Add ignore to pretrain
* 🏷 Improve type annotation on `run_command` helper
The `run_command` helper previously declared that it returned an
`Optional[subprocess.CompletedProcess]`, but it isn't actually possible
for the function to return `None`. These changes modify the type
annotation of the `run_command` helper and remove all now-unnecessary
`# type: ignore` directives.
* 🔧 Allow variable type redefinition in limited contexts
These changes modify how Mypy is configured to allow variables to have
their type automatically redefined under certain conditions. The Mypy
documentation contains the following example:
```python
def process(items: List[str]) -> None:
# 'items' has type List[str]
items = [item.split() for item in items]
# 'items' now has type List[List[str]]
...
```
This configuration change is especially helpful in reducing the number
of `# type: ignore` directives needed to handle the common pattern of:
* Accepting a filepath as a string
* Overwriting the variable using `filepath = ensure_path(filepath)`
These changes enable redefinition and remove all `# type: ignore`
directives rendered redundant by this change.
* 🏷 Add type annotation to converters mapping
* 🚨 Fix Mypy error in convert CLI argument verification
* 🏷 Improve type annotation on `resolve_dot_names` helper
* 🏷 Add type annotations for `Vocab` attributes `strings` and `vectors`
* 🏷 Add type annotations for more `Vocab` attributes
* 🏷 Add loose type annotation for gold data compilation
* 🏷 Improve `_format_labels` type annotation
* 🏷 Fix `get_lang_class` type annotation
* 🏷 Loosen return type of `Language.evaluate`
* 🏷 Don't accept `Scorer` in `handle_scores_per_type`
* 🏷 Add `string_to_list` overloads
* 🏷 Fix non-Optional command-line options
* 🙈 Ignore redefinition of `wandb_logger` in `loggers.py`
* ➕ Install `typing_extensions` in Python 3.8+
The `typing_extensions` package states that it should be used when
"writing code that must be compatible with multiple Python versions".
Since SpaCy needs to support multiple Python versions, it should be used
when newer `typing` module members are required. One example of this is
`Literal`, which is available starting with Python 3.8.
Previously SpaCy tried to import `Literal` from `typing`, falling back
to `typing_extensions` if the import failed. However, Mypy doesn't seem
to be able to understand what `Literal` means when the initial import
means. Therefore, these changes modify how `compat` imports `Literal` by
always importing it from `typing_extensions`.
These changes also modify how `typing_extensions` is installed, so that
it is a requirement for all Python versions, including those greater
than or equal to 3.8.
* 🏷 Improve type annotation for `Language.pipe`
These changes add a missing overload variant to the type signature of
`Language.pipe`. Additionally, the type signature is enhanced to allow
type checkers to differentiate between the two overload variants based
on the `as_tuple` parameter.
Fixes #8772
* ➖ Don't install `typing-extensions` in Python 3.8+
After more detailed analysis of how to implement Python version-specific
type annotations using SpaCy, it has been determined that by branching
on a comparison against `sys.version_info` can be statically analyzed by
Mypy well enough to enable us to conditionally use
`typing_extensions.Literal`. This means that we no longer need to
install `typing_extensions` for Python versions greater than or equal to
3.8! 🎉
These changes revert previous changes installing `typing-extensions`
regardless of Python version and modify how we import the `Literal` type
to ensure that Mypy treats it properly.
* resolve mypy errors for Strict pydantic types
* refactor code to avoid missing return statement
* fix types of convert CLI command
* avoid list-set confustion in debug_data
* fix typo and formatting
* small fixes to avoid type ignores
* fix types in profile CLI command and make it more efficient
* type fixes in projects CLI
* put one ignore back
* type fixes for render
* fix render types - the sequel
* fix BaseDefault in language definitions
* fix type of noun_chunks iterator - yields tuple instead of span
* fix types in language-specific modules
* 🏷 Expand accepted inputs of `get_string_id`
`get_string_id` accepts either a string (in which case it returns its
ID) or an ID (in which case it immediately returns the ID). These
changes extend the type annotation of `get_string_id` to indicate that
it can accept either strings or IDs.
* 🏷 Handle override types in `combine_score_weights`
The `combine_score_weights` function allows users to pass an `overrides`
mapping to override data extracted from the `weights` argument. Since it
allows `Optional` dictionary values, the return value may also include
`Optional` dictionary values.
These changes update the type annotations for `combine_score_weights` to
reflect this fact.
* 🏷 Fix tokenizer serialization method signatures in `DummyTokenizer`
* 🏷 Fix redefinition of `wandb_logger`
These changes fix the redefinition of `wandb_logger` by giving a
separate name to each `WandbLogger` version. For
backwards-compatibility, `spacy.train` still exports `wandb_logger_v3`
as `wandb_logger` for now.
* more fixes for typing in language
* type fixes in model definitions
* 🏷 Annotate `_RandomWords.probs` as `NDArray`
* 🏷 Annotate `tok2vec` layers to help Mypy
* 🐛 Fix `_RandomWords.probs` type annotations for Python 3.6
Also remove an import that I forgot to move to the top of the module 😅
* more fixes for matchers and other pipeline components
* quick fix for entity linker
* fixing types for spancat, textcat, etc
* bugfix for tok2vec
* type annotations for scorer
* add runtime_checkable for Protocol
* type and import fixes in tests
* mypy fixes for training utilities
* few fixes in util
* fix import
* 🐵 Remove unused `# type: ignore` directives
* 🏷 Annotate `Language._components`
* 🏷 Annotate `spacy.pipeline.Pipe`
* add doc as property to span.pyi
* small fixes and cleanup
* explicit type annotations instead of via comment
Co-authored-by: Adriane Boyd <adrianeboyd@gmail.com>
Co-authored-by: svlandeg <sofie.vanlandeghem@gmail.com>
Co-authored-by: svlandeg <svlandeg@github.com>
2021-10-14 13:21:40 +00:00
sig_args [ " __config__ " ] = ArgSchemaConfig if is_strict else ArgSchemaConfigExtra # type: ignore[assignment]
2022-05-25 07:33:54 +00:00
return create_model ( name , * * sig_args ) # type: ignore[call-overload, arg-type, return-value]
2020-09-29 09:52:45 +00:00
def validate_init_settings (
func : Callable ,
settings : Dict [ str , Any ] ,
* ,
section : Optional [ str ] = None ,
name : str = " " ,
2020-09-29 10:20:26 +00:00
exclude : Iterable [ str ] = ( " get_examples " , " nlp " ) ,
2020-09-29 09:52:45 +00:00
) - > Dict [ str , Any ] :
""" Validate initialization settings against the expected arguments in
the method signature . Will parse values if possible ( e . g . int to string )
and return the updated settings dict . Will raise a ConfigValidationError
if types don ' t match or required values are missing.
func ( Callable ) : The initialize method of a given component etc .
2020-10-05 20:50:14 +00:00
settings ( Dict [ str , Any ] ) : The settings from the respective [ initialize ] block .
2020-09-29 09:52:45 +00:00
section ( str ) : Initialize section , for error message .
name ( str ) : Name of the block in the section .
exclude ( Iterable [ str ] ) : Parameter names to exclude from schema .
RETURNS ( Dict [ str , Any ] ) : The validated settings .
"""
schema = get_arg_model ( func , exclude = exclude , name = " InitArgModel " )
try :
return schema ( * * settings ) . dict ( )
except ValidationError as e :
block = " initialize " if not section else f " initialize. { section } "
title = f " Error validating initialization settings in [ { block } ] "
raise ConfigValidationError (
2020-09-29 19:39:28 +00:00
title = title , errors = e . errors ( ) , config = settings , parent = name
2020-09-29 09:52:45 +00:00
) from None
2019-12-25 11:39:49 +00:00
# Matcher token patterns
2020-07-25 13:01:15 +00:00
def validate_token_pattern ( obj : list ) - > List [ str ] :
2019-12-25 11:39:49 +00:00
# Try to convert non-string keys (e.g. {ORTH: "foo"} -> {"ORTH": "foo"})
get_key = lambda k : NAMES [ k ] if isinstance ( k , int ) and k < len ( NAMES ) else k
if isinstance ( obj , list ) :
converted = [ ]
for pattern in obj :
if isinstance ( pattern , dict ) :
pattern = { get_key ( k ) : v for k , v in pattern . items ( ) }
converted . append ( pattern )
obj = converted
return validate ( TokenPatternSchema , { " pattern " : obj } )
class TokenPatternString ( BaseModel ) :
2020-09-04 12:05:55 +00:00
REGEX : Optional [ StrictStr ] = Field ( None , alias = " regex " )
IN : Optional [ List [ StrictStr ] ] = Field ( None , alias = " in " )
NOT_IN : Optional [ List [ StrictStr ] ] = Field ( None , alias = " not_in " )
2020-09-24 14:55:09 +00:00
IS_SUBSET : Optional [ List [ StrictStr ] ] = Field ( None , alias = " is_subset " )
IS_SUPERSET : Optional [ List [ StrictStr ] ] = Field ( None , alias = " is_superset " )
2021-08-02 17:39:26 +00:00
INTERSECTS : Optional [ List [ StrictStr ] ] = Field ( None , alias = " intersects " )
2019-12-25 11:39:49 +00:00
class Config :
extra = " forbid "
2020-09-04 12:05:55 +00:00
allow_population_by_field_name = True # allow alias and field name
2019-12-25 11:39:49 +00:00
2020-08-25 15:13:11 +00:00
@validator ( " * " , pre = True , each_item = True , allow_reuse = True )
2019-12-25 11:39:49 +00:00
def raise_for_none ( cls , v ) :
if v is None :
raise ValueError ( " None / null is not allowed " )
return v
class TokenPatternNumber ( BaseModel ) :
2020-09-04 12:05:55 +00:00
REGEX : Optional [ StrictStr ] = Field ( None , alias = " regex " )
IN : Optional [ List [ StrictInt ] ] = Field ( None , alias = " in " )
NOT_IN : Optional [ List [ StrictInt ] ] = Field ( None , alias = " not_in " )
2021-08-02 17:39:26 +00:00
IS_SUBSET : Optional [ List [ StrictInt ] ] = Field ( None , alias = " is_subset " )
IS_SUPERSET : Optional [ List [ StrictInt ] ] = Field ( None , alias = " is_superset " )
INTERSECTS : Optional [ List [ StrictInt ] ] = Field ( None , alias = " intersects " )
2022-10-14 06:16:49 +00:00
EQ : Optional [ Union [ StrictInt , StrictFloat ] ] = Field ( None , alias = " == " )
NEQ : Optional [ Union [ StrictInt , StrictFloat ] ] = Field ( None , alias = " != " )
GEQ : Optional [ Union [ StrictInt , StrictFloat ] ] = Field ( None , alias = " >= " )
LEQ : Optional [ Union [ StrictInt , StrictFloat ] ] = Field ( None , alias = " <= " )
GT : Optional [ Union [ StrictInt , StrictFloat ] ] = Field ( None , alias = " > " )
LT : Optional [ Union [ StrictInt , StrictFloat ] ] = Field ( None , alias = " < " )
2019-12-25 11:39:49 +00:00
class Config :
extra = " forbid "
2020-09-04 12:05:55 +00:00
allow_population_by_field_name = True # allow alias and field name
2019-12-25 11:39:49 +00:00
2020-08-25 15:13:11 +00:00
@validator ( " * " , pre = True , each_item = True , allow_reuse = True )
2019-12-25 11:39:49 +00:00
def raise_for_none ( cls , v ) :
if v is None :
raise ValueError ( " None / null is not allowed " )
return v
2022-06-30 09:01:58 +00:00
class TokenPatternOperatorSimple ( str , Enum ) :
🏷 Add Mypy check to CI and ignore all existing Mypy errors (#9167)
* 🚨 Ignore all existing Mypy errors
* 🏗 Add Mypy check to CI
* Add types-mock and types-requests as dev requirements
* Add additional type ignore directives
* Add types packages to dev-only list in reqs test
* Add types-dataclasses for python 3.6
* Add ignore to pretrain
* 🏷 Improve type annotation on `run_command` helper
The `run_command` helper previously declared that it returned an
`Optional[subprocess.CompletedProcess]`, but it isn't actually possible
for the function to return `None`. These changes modify the type
annotation of the `run_command` helper and remove all now-unnecessary
`# type: ignore` directives.
* 🔧 Allow variable type redefinition in limited contexts
These changes modify how Mypy is configured to allow variables to have
their type automatically redefined under certain conditions. The Mypy
documentation contains the following example:
```python
def process(items: List[str]) -> None:
# 'items' has type List[str]
items = [item.split() for item in items]
# 'items' now has type List[List[str]]
...
```
This configuration change is especially helpful in reducing the number
of `# type: ignore` directives needed to handle the common pattern of:
* Accepting a filepath as a string
* Overwriting the variable using `filepath = ensure_path(filepath)`
These changes enable redefinition and remove all `# type: ignore`
directives rendered redundant by this change.
* 🏷 Add type annotation to converters mapping
* 🚨 Fix Mypy error in convert CLI argument verification
* 🏷 Improve type annotation on `resolve_dot_names` helper
* 🏷 Add type annotations for `Vocab` attributes `strings` and `vectors`
* 🏷 Add type annotations for more `Vocab` attributes
* 🏷 Add loose type annotation for gold data compilation
* 🏷 Improve `_format_labels` type annotation
* 🏷 Fix `get_lang_class` type annotation
* 🏷 Loosen return type of `Language.evaluate`
* 🏷 Don't accept `Scorer` in `handle_scores_per_type`
* 🏷 Add `string_to_list` overloads
* 🏷 Fix non-Optional command-line options
* 🙈 Ignore redefinition of `wandb_logger` in `loggers.py`
* ➕ Install `typing_extensions` in Python 3.8+
The `typing_extensions` package states that it should be used when
"writing code that must be compatible with multiple Python versions".
Since SpaCy needs to support multiple Python versions, it should be used
when newer `typing` module members are required. One example of this is
`Literal`, which is available starting with Python 3.8.
Previously SpaCy tried to import `Literal` from `typing`, falling back
to `typing_extensions` if the import failed. However, Mypy doesn't seem
to be able to understand what `Literal` means when the initial import
means. Therefore, these changes modify how `compat` imports `Literal` by
always importing it from `typing_extensions`.
These changes also modify how `typing_extensions` is installed, so that
it is a requirement for all Python versions, including those greater
than or equal to 3.8.
* 🏷 Improve type annotation for `Language.pipe`
These changes add a missing overload variant to the type signature of
`Language.pipe`. Additionally, the type signature is enhanced to allow
type checkers to differentiate between the two overload variants based
on the `as_tuple` parameter.
Fixes #8772
* ➖ Don't install `typing-extensions` in Python 3.8+
After more detailed analysis of how to implement Python version-specific
type annotations using SpaCy, it has been determined that by branching
on a comparison against `sys.version_info` can be statically analyzed by
Mypy well enough to enable us to conditionally use
`typing_extensions.Literal`. This means that we no longer need to
install `typing_extensions` for Python versions greater than or equal to
3.8! 🎉
These changes revert previous changes installing `typing-extensions`
regardless of Python version and modify how we import the `Literal` type
to ensure that Mypy treats it properly.
* resolve mypy errors for Strict pydantic types
* refactor code to avoid missing return statement
* fix types of convert CLI command
* avoid list-set confustion in debug_data
* fix typo and formatting
* small fixes to avoid type ignores
* fix types in profile CLI command and make it more efficient
* type fixes in projects CLI
* put one ignore back
* type fixes for render
* fix render types - the sequel
* fix BaseDefault in language definitions
* fix type of noun_chunks iterator - yields tuple instead of span
* fix types in language-specific modules
* 🏷 Expand accepted inputs of `get_string_id`
`get_string_id` accepts either a string (in which case it returns its
ID) or an ID (in which case it immediately returns the ID). These
changes extend the type annotation of `get_string_id` to indicate that
it can accept either strings or IDs.
* 🏷 Handle override types in `combine_score_weights`
The `combine_score_weights` function allows users to pass an `overrides`
mapping to override data extracted from the `weights` argument. Since it
allows `Optional` dictionary values, the return value may also include
`Optional` dictionary values.
These changes update the type annotations for `combine_score_weights` to
reflect this fact.
* 🏷 Fix tokenizer serialization method signatures in `DummyTokenizer`
* 🏷 Fix redefinition of `wandb_logger`
These changes fix the redefinition of `wandb_logger` by giving a
separate name to each `WandbLogger` version. For
backwards-compatibility, `spacy.train` still exports `wandb_logger_v3`
as `wandb_logger` for now.
* more fixes for typing in language
* type fixes in model definitions
* 🏷 Annotate `_RandomWords.probs` as `NDArray`
* 🏷 Annotate `tok2vec` layers to help Mypy
* 🐛 Fix `_RandomWords.probs` type annotations for Python 3.6
Also remove an import that I forgot to move to the top of the module 😅
* more fixes for matchers and other pipeline components
* quick fix for entity linker
* fixing types for spancat, textcat, etc
* bugfix for tok2vec
* type annotations for scorer
* add runtime_checkable for Protocol
* type and import fixes in tests
* mypy fixes for training utilities
* few fixes in util
* fix import
* 🐵 Remove unused `# type: ignore` directives
* 🏷 Annotate `Language._components`
* 🏷 Annotate `spacy.pipeline.Pipe`
* add doc as property to span.pyi
* small fixes and cleanup
* explicit type annotations instead of via comment
Co-authored-by: Adriane Boyd <adrianeboyd@gmail.com>
Co-authored-by: svlandeg <sofie.vanlandeghem@gmail.com>
Co-authored-by: svlandeg <svlandeg@github.com>
2021-10-14 13:21:40 +00:00
plus : StrictStr = StrictStr ( " + " )
2022-06-30 09:01:58 +00:00
star : StrictStr = StrictStr ( " * " )
🏷 Add Mypy check to CI and ignore all existing Mypy errors (#9167)
* 🚨 Ignore all existing Mypy errors
* 🏗 Add Mypy check to CI
* Add types-mock and types-requests as dev requirements
* Add additional type ignore directives
* Add types packages to dev-only list in reqs test
* Add types-dataclasses for python 3.6
* Add ignore to pretrain
* 🏷 Improve type annotation on `run_command` helper
The `run_command` helper previously declared that it returned an
`Optional[subprocess.CompletedProcess]`, but it isn't actually possible
for the function to return `None`. These changes modify the type
annotation of the `run_command` helper and remove all now-unnecessary
`# type: ignore` directives.
* 🔧 Allow variable type redefinition in limited contexts
These changes modify how Mypy is configured to allow variables to have
their type automatically redefined under certain conditions. The Mypy
documentation contains the following example:
```python
def process(items: List[str]) -> None:
# 'items' has type List[str]
items = [item.split() for item in items]
# 'items' now has type List[List[str]]
...
```
This configuration change is especially helpful in reducing the number
of `# type: ignore` directives needed to handle the common pattern of:
* Accepting a filepath as a string
* Overwriting the variable using `filepath = ensure_path(filepath)`
These changes enable redefinition and remove all `# type: ignore`
directives rendered redundant by this change.
* 🏷 Add type annotation to converters mapping
* 🚨 Fix Mypy error in convert CLI argument verification
* 🏷 Improve type annotation on `resolve_dot_names` helper
* 🏷 Add type annotations for `Vocab` attributes `strings` and `vectors`
* 🏷 Add type annotations for more `Vocab` attributes
* 🏷 Add loose type annotation for gold data compilation
* 🏷 Improve `_format_labels` type annotation
* 🏷 Fix `get_lang_class` type annotation
* 🏷 Loosen return type of `Language.evaluate`
* 🏷 Don't accept `Scorer` in `handle_scores_per_type`
* 🏷 Add `string_to_list` overloads
* 🏷 Fix non-Optional command-line options
* 🙈 Ignore redefinition of `wandb_logger` in `loggers.py`
* ➕ Install `typing_extensions` in Python 3.8+
The `typing_extensions` package states that it should be used when
"writing code that must be compatible with multiple Python versions".
Since SpaCy needs to support multiple Python versions, it should be used
when newer `typing` module members are required. One example of this is
`Literal`, which is available starting with Python 3.8.
Previously SpaCy tried to import `Literal` from `typing`, falling back
to `typing_extensions` if the import failed. However, Mypy doesn't seem
to be able to understand what `Literal` means when the initial import
means. Therefore, these changes modify how `compat` imports `Literal` by
always importing it from `typing_extensions`.
These changes also modify how `typing_extensions` is installed, so that
it is a requirement for all Python versions, including those greater
than or equal to 3.8.
* 🏷 Improve type annotation for `Language.pipe`
These changes add a missing overload variant to the type signature of
`Language.pipe`. Additionally, the type signature is enhanced to allow
type checkers to differentiate between the two overload variants based
on the `as_tuple` parameter.
Fixes #8772
* ➖ Don't install `typing-extensions` in Python 3.8+
After more detailed analysis of how to implement Python version-specific
type annotations using SpaCy, it has been determined that by branching
on a comparison against `sys.version_info` can be statically analyzed by
Mypy well enough to enable us to conditionally use
`typing_extensions.Literal`. This means that we no longer need to
install `typing_extensions` for Python versions greater than or equal to
3.8! 🎉
These changes revert previous changes installing `typing-extensions`
regardless of Python version and modify how we import the `Literal` type
to ensure that Mypy treats it properly.
* resolve mypy errors for Strict pydantic types
* refactor code to avoid missing return statement
* fix types of convert CLI command
* avoid list-set confustion in debug_data
* fix typo and formatting
* small fixes to avoid type ignores
* fix types in profile CLI command and make it more efficient
* type fixes in projects CLI
* put one ignore back
* type fixes for render
* fix render types - the sequel
* fix BaseDefault in language definitions
* fix type of noun_chunks iterator - yields tuple instead of span
* fix types in language-specific modules
* 🏷 Expand accepted inputs of `get_string_id`
`get_string_id` accepts either a string (in which case it returns its
ID) or an ID (in which case it immediately returns the ID). These
changes extend the type annotation of `get_string_id` to indicate that
it can accept either strings or IDs.
* 🏷 Handle override types in `combine_score_weights`
The `combine_score_weights` function allows users to pass an `overrides`
mapping to override data extracted from the `weights` argument. Since it
allows `Optional` dictionary values, the return value may also include
`Optional` dictionary values.
These changes update the type annotations for `combine_score_weights` to
reflect this fact.
* 🏷 Fix tokenizer serialization method signatures in `DummyTokenizer`
* 🏷 Fix redefinition of `wandb_logger`
These changes fix the redefinition of `wandb_logger` by giving a
separate name to each `WandbLogger` version. For
backwards-compatibility, `spacy.train` still exports `wandb_logger_v3`
as `wandb_logger` for now.
* more fixes for typing in language
* type fixes in model definitions
* 🏷 Annotate `_RandomWords.probs` as `NDArray`
* 🏷 Annotate `tok2vec` layers to help Mypy
* 🐛 Fix `_RandomWords.probs` type annotations for Python 3.6
Also remove an import that I forgot to move to the top of the module 😅
* more fixes for matchers and other pipeline components
* quick fix for entity linker
* fixing types for spancat, textcat, etc
* bugfix for tok2vec
* type annotations for scorer
* add runtime_checkable for Protocol
* type and import fixes in tests
* mypy fixes for training utilities
* few fixes in util
* fix import
* 🐵 Remove unused `# type: ignore` directives
* 🏷 Annotate `Language._components`
* 🏷 Annotate `spacy.pipeline.Pipe`
* add doc as property to span.pyi
* small fixes and cleanup
* explicit type annotations instead of via comment
Co-authored-by: Adriane Boyd <adrianeboyd@gmail.com>
Co-authored-by: svlandeg <sofie.vanlandeghem@gmail.com>
Co-authored-by: svlandeg <svlandeg@github.com>
2021-10-14 13:21:40 +00:00
question : StrictStr = StrictStr ( " ? " )
exclamation : StrictStr = StrictStr ( " ! " )
2019-12-25 11:39:49 +00:00
2022-06-30 09:01:58 +00:00
class TokenPatternOperatorMinMax ( ConstrainedStr ) :
2022-08-09 08:59:36 +00:00
regex = re . compile ( r " ^( { \ d+}| { \ d+, \ d*}| { \ d*, \ d+})$ " )
2022-06-30 09:01:58 +00:00
TokenPatternOperator = Union [ TokenPatternOperatorSimple , TokenPatternOperatorMinMax ]
2019-12-25 11:39:49 +00:00
StringValue = Union [ TokenPatternString , StrictStr ]
NumberValue = Union [ TokenPatternNumber , StrictInt , StrictFloat ]
UnderscoreValue = Union [
2020-09-17 09:48:04 +00:00
TokenPatternString , TokenPatternNumber , str , int , float , list , bool
2019-12-25 11:39:49 +00:00
]
2022-01-20 12:18:39 +00:00
IobValue = Literal [ " " , " I " , " O " , " B " , 0 , 1 , 2 , 3 ]
2019-12-25 11:39:49 +00:00
class TokenPattern ( BaseModel ) :
orth : Optional [ StringValue ] = None
text : Optional [ StringValue ] = None
lower : Optional [ StringValue ] = None
pos : Optional [ StringValue ] = None
tag : Optional [ StringValue ] = None
2020-09-24 14:55:09 +00:00
morph : Optional [ StringValue ] = None
2019-12-25 11:39:49 +00:00
dep : Optional [ StringValue ] = None
lemma : Optional [ StringValue ] = None
shape : Optional [ StringValue ] = None
ent_type : Optional [ StringValue ] = None
2022-01-20 12:18:39 +00:00
ent_iob : Optional [ IobValue ] = None
2021-11-24 09:37:10 +00:00
ent_id : Optional [ StringValue ] = None
ent_kb_id : Optional [ StringValue ] = None
2019-12-25 11:39:49 +00:00
norm : Optional [ StringValue ] = None
length : Optional [ NumberValue ] = None
2020-02-18 13:32:53 +00:00
spacy : Optional [ StrictBool ] = None
2019-12-25 11:39:49 +00:00
is_alpha : Optional [ StrictBool ] = None
is_ascii : Optional [ StrictBool ] = None
is_digit : Optional [ StrictBool ] = None
is_lower : Optional [ StrictBool ] = None
is_upper : Optional [ StrictBool ] = None
is_title : Optional [ StrictBool ] = None
is_punct : Optional [ StrictBool ] = None
is_space : Optional [ StrictBool ] = None
is_bracket : Optional [ StrictBool ] = None
is_quote : Optional [ StrictBool ] = None
is_left_punct : Optional [ StrictBool ] = None
is_right_punct : Optional [ StrictBool ] = None
is_currency : Optional [ StrictBool ] = None
is_stop : Optional [ StrictBool ] = None
is_sent_start : Optional [ StrictBool ] = None
2020-03-26 13:05:40 +00:00
sent_start : Optional [ StrictBool ] = None
2019-12-25 11:39:49 +00:00
like_num : Optional [ StrictBool ] = None
like_url : Optional [ StrictBool ] = None
like_email : Optional [ StrictBool ] = None
op : Optional [ TokenPatternOperator ] = None
underscore : Optional [ Dict [ StrictStr , UnderscoreValue ] ] = Field ( None , alias = " _ " )
class Config :
extra = " forbid "
allow_population_by_field_name = True
alias_generator = lambda value : value . upper ( )
2020-08-25 15:13:11 +00:00
@validator ( " * " , pre = True , allow_reuse = True )
2019-12-25 11:39:49 +00:00
def raise_for_none ( cls , v ) :
if v is None :
raise ValueError ( " None / null is not allowed " )
return v
class TokenPatternSchema ( BaseModel ) :
2020-10-15 22:41:19 +00:00
pattern : List [ TokenPattern ] = Field ( . . . , min_items = 1 )
2019-12-25 11:39:49 +00:00
class Config :
extra = " forbid "
# Model meta
class ModelMetaSchema ( BaseModel ) :
# fmt: off
lang : StrictStr = Field ( . . . , title = " Two-letter language code, e.g. ' en ' " )
name : StrictStr = Field ( . . . , title = " Model name " )
version : StrictStr = Field ( . . . , title = " Model version " )
2020-08-18 12:39:40 +00:00
spacy_version : StrictStr = Field ( " " , title = " Compatible spaCy version identifier " )
parent_package : StrictStr = Field ( " spacy " , title = " Name of parent spaCy package, e.g. spacy or spacy-nightly " )
2020-12-10 12:36:46 +00:00
requirements : List [ StrictStr ] = Field ( [ ] , title = " Additional Python package dependencies, used for the Python package setup " )
2020-08-18 12:39:40 +00:00
pipeline : List [ StrictStr ] = Field ( [ ] , title = " Names of pipeline components " )
description : StrictStr = Field ( " " , title = " Model description " )
license : StrictStr = Field ( " " , title = " Model license " )
author : StrictStr = Field ( " " , title = " Model author name " )
email : StrictStr = Field ( " " , title = " Model author email " )
url : StrictStr = Field ( " " , title = " Model author URL " )
sources : Optional [ Union [ List [ StrictStr ] , List [ Dict [ str , str ] ] ] ] = Field ( None , title = " Training data sources " )
vectors : Dict [ str , Any ] = Field ( { } , title = " Included word vectors " )
2020-09-11 09:38:33 +00:00
labels : Dict [ str , List [ str ] ] = Field ( { } , title = " Component labels, keyed by component name " )
2020-10-11 10:30:57 +00:00
performance : Dict [ str , Any ] = Field ( { } , title = " Accuracy and speed numbers " )
2020-08-18 12:39:40 +00:00
spacy_git_version : StrictStr = Field ( " " , title = " Commit of spaCy version used " )
2019-12-25 11:39:49 +00:00
# fmt: on
2020-07-10 11:31:27 +00:00
# Config schema
# We're not setting any defaults here (which is too messy) and are making all
# fields required, so we can raise validation errors for missing values. To
# provide a default, we include a separate .cfg file with all values and
# check that against this schema in the test suite to make sure it's always
# up to date.
class ConfigSchemaTraining ( BaseModel ) :
# fmt: off
2020-09-17 09:38:59 +00:00
dev_corpus : StrictStr = Field ( . . . , title = " Path in the config to the dev data " )
train_corpus : StrictStr = Field ( . . . , title = " Path in the config to the training data " )
2020-08-04 13:09:37 +00:00
batcher : Batcher = Field ( . . . , title = " Batcher for the training data " )
2020-07-10 11:31:27 +00:00
dropout : StrictFloat = Field ( . . . , title = " Dropout rate " )
patience : StrictInt = Field ( . . . , title = " How many steps to continue without improvement in evaluation score " )
max_epochs : StrictInt = Field ( . . . , title = " Maximum number of epochs to train for " )
max_steps : StrictInt = Field ( . . . , title = " Maximum number of update steps to train for " )
eval_frequency : StrictInt = Field ( . . . , title = " How often to evaluate during training (steps) " )
2020-07-10 18:52:00 +00:00
seed : Optional [ StrictInt ] = Field ( . . . , title = " Random seed " )
2020-09-18 23:17:02 +00:00
gpu_allocator : Optional [ StrictStr ] = Field ( . . . , title = " Memory allocator when running on GPU " )
2020-07-10 11:31:27 +00:00
accumulate_gradient : StrictInt = Field ( . . . , title = " Whether to divide the batch up into substeps " )
2020-09-24 08:27:33 +00:00
score_weights : Dict [ StrictStr , Optional [ Union [ StrictFloat , StrictInt ] ] ] = Field ( . . . , title = " Scores to report and their weights for selecting final model " )
2020-07-10 11:31:27 +00:00
optimizer : Optimizer = Field ( . . . , title = " The optimizer to use " )
2020-08-26 13:24:33 +00:00
logger : Logger = Field ( . . . , title = " The logger to track training progress " )
2020-08-04 21:39:19 +00:00
frozen_components : List [ str ] = Field ( . . . , title = " Pipeline components that shouldn ' t be updated during training " )
2021-04-26 14:53:53 +00:00
annotating_components : List [ str ] = Field ( . . . , title = " Pipeline components that should set annotations during training " )
2020-09-24 10:40:25 +00:00
before_to_disk : Optional [ Callable [ [ " Language " ] , " Language " ] ] = Field ( . . . , title = " Optional callback to modify nlp object after training, before it ' s saved to disk " )
2022-11-23 16:54:58 +00:00
before_update : Optional [ Callable [ [ " Language " , Dict [ str , Any ] ] , None ] ] = Field ( . . . , title = " Optional callback that is invoked at the start of each training step " )
2020-07-10 11:31:27 +00:00
# fmt: on
class Config :
extra = " forbid "
2020-06-21 11:44:00 +00:00
arbitrary_types_allowed = True
2020-07-10 11:31:27 +00:00
class ConfigSchemaNlp ( BaseModel ) :
2020-07-22 11:42:59 +00:00
# fmt: off
2020-07-10 11:31:27 +00:00
lang : StrictStr = Field ( . . . , title = " The base language to use " )
2020-07-22 11:42:59 +00:00
pipeline : List [ StrictStr ] = Field ( . . . , title = " The pipeline component names in order " )
2020-08-28 13:20:14 +00:00
disabled : List [ StrictStr ] = Field ( . . . , title = " Pipeline components to disable by default " )
2020-07-22 11:42:59 +00:00
tokenizer : Callable = Field ( . . . , title = " The tokenizer to use " )
2020-08-05 17:47:54 +00:00
before_creation : Optional [ Callable [ [ Type [ " Language " ] ] , Type [ " Language " ] ] ] = Field ( . . . , title = " Optional callback to modify Language class before initialization " )
after_creation : Optional [ Callable [ [ " Language " ] , " Language " ] ] = Field ( . . . , title = " Optional callback to modify nlp object after creation and before the pipeline is constructed " )
after_pipeline_creation : Optional [ Callable [ [ " Language " ] , " Language " ] ] = Field ( . . . , title = " Optional callback to modify nlp object after the pipeline is constructed " )
2020-12-09 08:13:26 +00:00
batch_size : Optional [ int ] = Field ( . . . , title = " Default batch size " )
2020-07-22 11:42:59 +00:00
# fmt: on
2019-12-25 11:39:49 +00:00
class Config :
extra = " forbid "
2020-07-10 11:31:27 +00:00
arbitrary_types_allowed = True
2020-08-24 13:56:03 +00:00
class ConfigSchemaPretrainEmpty ( BaseModel ) :
class Config :
extra = " forbid "
2020-07-11 11:03:53 +00:00
class ConfigSchemaPretrain ( BaseModel ) :
# fmt: off
max_epochs : StrictInt = Field ( . . . , title = " Maximum number of epochs to train for " )
dropout : StrictFloat = Field ( . . . , title = " Dropout rate " )
2021-08-12 09:14:48 +00:00
n_save_every : Optional [ StrictInt ] = Field ( . . . , title = " Saving additional temporary model after n batches within an epoch " )
n_save_epoch : Optional [ StrictInt ] = Field ( . . . , title = " Saving model after every n epoch " )
2020-07-11 11:03:53 +00:00
optimizer : Optimizer = Field ( . . . , title = " The optimizer to use " )
2020-09-17 09:38:59 +00:00
corpus : StrictStr = Field ( . . . , title = " Path in the config to the training data " )
2020-09-13 12:05:05 +00:00
batcher : Batcher = Field ( . . . , title = " Batcher for the training data " )
component : str = Field ( . . . , title = " Component to find the layer to pretrain " )
layer : str = Field ( . . . , title = " Layer to pretrain. Whole model if empty. " )
2021-06-28 09:29:29 +00:00
objective : Callable [ [ " Vocab " , Model ] , Model ] = Field ( . . . , title = " A function that creates the pretraining objective. " )
2020-07-11 11:03:53 +00:00
# fmt: on
class Config :
extra = " forbid "
arbitrary_types_allowed = True
2020-09-29 14:47:55 +00:00
class ConfigSchemaInit ( BaseModel ) :
2020-09-28 09:56:14 +00:00
# fmt: off
2020-09-29 14:47:55 +00:00
vocab_data : Optional [ StrictStr ] = Field ( . . . , title = " Path to JSON-formatted vocabulary file " )
2020-09-28 09:56:14 +00:00
lookups : Optional [ Lookups ] = Field ( . . . , title = " Vocabulary lookups, e.g. lexeme normalization " )
vectors : Optional [ StrictStr ] = Field ( . . . , title = " Path to vectors " )
init_tok2vec : Optional [ StrictStr ] = Field ( . . . , title = " Path to pretrained tok2vec weights " )
2020-09-29 14:47:55 +00:00
tokenizer : Dict [ StrictStr , Any ] = Field ( . . . , help = " Arguments to be passed into Tokenizer.initialize " )
2020-10-08 19:33:49 +00:00
components : Dict [ StrictStr , Dict [ StrictStr , Any ] ] = Field ( . . . , help = " Arguments for TrainablePipe.initialize methods of pipeline components, keyed by component " )
2021-01-12 10:29:31 +00:00
before_init : Optional [ Callable [ [ " Language " ] , " Language " ] ] = Field ( . . . , title = " Optional callback to modify nlp object before initialization " )
after_init : Optional [ Callable [ [ " Language " ] , " Language " ] ] = Field ( . . . , title = " Optional callback to modify nlp object after initialization " )
2020-09-28 09:56:14 +00:00
# fmt: on
class Config :
extra = " forbid "
arbitrary_types_allowed = True
2020-07-10 11:31:27 +00:00
class ConfigSchema ( BaseModel ) :
training : ConfigSchemaTraining
nlp : ConfigSchemaNlp
🏷 Add Mypy check to CI and ignore all existing Mypy errors (#9167)
* 🚨 Ignore all existing Mypy errors
* 🏗 Add Mypy check to CI
* Add types-mock and types-requests as dev requirements
* Add additional type ignore directives
* Add types packages to dev-only list in reqs test
* Add types-dataclasses for python 3.6
* Add ignore to pretrain
* 🏷 Improve type annotation on `run_command` helper
The `run_command` helper previously declared that it returned an
`Optional[subprocess.CompletedProcess]`, but it isn't actually possible
for the function to return `None`. These changes modify the type
annotation of the `run_command` helper and remove all now-unnecessary
`# type: ignore` directives.
* 🔧 Allow variable type redefinition in limited contexts
These changes modify how Mypy is configured to allow variables to have
their type automatically redefined under certain conditions. The Mypy
documentation contains the following example:
```python
def process(items: List[str]) -> None:
# 'items' has type List[str]
items = [item.split() for item in items]
# 'items' now has type List[List[str]]
...
```
This configuration change is especially helpful in reducing the number
of `# type: ignore` directives needed to handle the common pattern of:
* Accepting a filepath as a string
* Overwriting the variable using `filepath = ensure_path(filepath)`
These changes enable redefinition and remove all `# type: ignore`
directives rendered redundant by this change.
* 🏷 Add type annotation to converters mapping
* 🚨 Fix Mypy error in convert CLI argument verification
* 🏷 Improve type annotation on `resolve_dot_names` helper
* 🏷 Add type annotations for `Vocab` attributes `strings` and `vectors`
* 🏷 Add type annotations for more `Vocab` attributes
* 🏷 Add loose type annotation for gold data compilation
* 🏷 Improve `_format_labels` type annotation
* 🏷 Fix `get_lang_class` type annotation
* 🏷 Loosen return type of `Language.evaluate`
* 🏷 Don't accept `Scorer` in `handle_scores_per_type`
* 🏷 Add `string_to_list` overloads
* 🏷 Fix non-Optional command-line options
* 🙈 Ignore redefinition of `wandb_logger` in `loggers.py`
* ➕ Install `typing_extensions` in Python 3.8+
The `typing_extensions` package states that it should be used when
"writing code that must be compatible with multiple Python versions".
Since SpaCy needs to support multiple Python versions, it should be used
when newer `typing` module members are required. One example of this is
`Literal`, which is available starting with Python 3.8.
Previously SpaCy tried to import `Literal` from `typing`, falling back
to `typing_extensions` if the import failed. However, Mypy doesn't seem
to be able to understand what `Literal` means when the initial import
means. Therefore, these changes modify how `compat` imports `Literal` by
always importing it from `typing_extensions`.
These changes also modify how `typing_extensions` is installed, so that
it is a requirement for all Python versions, including those greater
than or equal to 3.8.
* 🏷 Improve type annotation for `Language.pipe`
These changes add a missing overload variant to the type signature of
`Language.pipe`. Additionally, the type signature is enhanced to allow
type checkers to differentiate between the two overload variants based
on the `as_tuple` parameter.
Fixes #8772
* ➖ Don't install `typing-extensions` in Python 3.8+
After more detailed analysis of how to implement Python version-specific
type annotations using SpaCy, it has been determined that by branching
on a comparison against `sys.version_info` can be statically analyzed by
Mypy well enough to enable us to conditionally use
`typing_extensions.Literal`. This means that we no longer need to
install `typing_extensions` for Python versions greater than or equal to
3.8! 🎉
These changes revert previous changes installing `typing-extensions`
regardless of Python version and modify how we import the `Literal` type
to ensure that Mypy treats it properly.
* resolve mypy errors for Strict pydantic types
* refactor code to avoid missing return statement
* fix types of convert CLI command
* avoid list-set confustion in debug_data
* fix typo and formatting
* small fixes to avoid type ignores
* fix types in profile CLI command and make it more efficient
* type fixes in projects CLI
* put one ignore back
* type fixes for render
* fix render types - the sequel
* fix BaseDefault in language definitions
* fix type of noun_chunks iterator - yields tuple instead of span
* fix types in language-specific modules
* 🏷 Expand accepted inputs of `get_string_id`
`get_string_id` accepts either a string (in which case it returns its
ID) or an ID (in which case it immediately returns the ID). These
changes extend the type annotation of `get_string_id` to indicate that
it can accept either strings or IDs.
* 🏷 Handle override types in `combine_score_weights`
The `combine_score_weights` function allows users to pass an `overrides`
mapping to override data extracted from the `weights` argument. Since it
allows `Optional` dictionary values, the return value may also include
`Optional` dictionary values.
These changes update the type annotations for `combine_score_weights` to
reflect this fact.
* 🏷 Fix tokenizer serialization method signatures in `DummyTokenizer`
* 🏷 Fix redefinition of `wandb_logger`
These changes fix the redefinition of `wandb_logger` by giving a
separate name to each `WandbLogger` version. For
backwards-compatibility, `spacy.train` still exports `wandb_logger_v3`
as `wandb_logger` for now.
* more fixes for typing in language
* type fixes in model definitions
* 🏷 Annotate `_RandomWords.probs` as `NDArray`
* 🏷 Annotate `tok2vec` layers to help Mypy
* 🐛 Fix `_RandomWords.probs` type annotations for Python 3.6
Also remove an import that I forgot to move to the top of the module 😅
* more fixes for matchers and other pipeline components
* quick fix for entity linker
* fixing types for spancat, textcat, etc
* bugfix for tok2vec
* type annotations for scorer
* add runtime_checkable for Protocol
* type and import fixes in tests
* mypy fixes for training utilities
* few fixes in util
* fix import
* 🐵 Remove unused `# type: ignore` directives
* 🏷 Annotate `Language._components`
* 🏷 Annotate `spacy.pipeline.Pipe`
* add doc as property to span.pyi
* small fixes and cleanup
* explicit type annotations instead of via comment
Co-authored-by: Adriane Boyd <adrianeboyd@gmail.com>
Co-authored-by: svlandeg <sofie.vanlandeghem@gmail.com>
Co-authored-by: svlandeg <svlandeg@github.com>
2021-10-14 13:21:40 +00:00
pretraining : Union [ ConfigSchemaPretrain , ConfigSchemaPretrainEmpty ] = { } # type: ignore[assignment]
2020-07-22 11:42:59 +00:00
components : Dict [ str , Dict [ str , Any ] ]
2020-09-17 09:38:59 +00:00
corpora : Dict [ str , Reader ]
2020-09-28 09:56:14 +00:00
initialize : ConfigSchemaInit
2020-07-10 11:31:27 +00:00
class Config :
extra = " allow "
arbitrary_types_allowed = True
2020-06-21 11:44:00 +00:00
2020-09-29 18:38:35 +00:00
CONFIG_SCHEMAS = {
" nlp " : ConfigSchemaNlp ,
" training " : ConfigSchemaTraining ,
" pretraining " : ConfigSchemaPretrain ,
" initialize " : ConfigSchemaInit ,
}
2020-09-27 20:21:31 +00:00
2020-06-21 11:44:00 +00:00
# Project config Schema
2020-08-25 09:54:53 +00:00
class ProjectConfigAssetGitItem ( BaseModel ) :
# fmt: off
repo : StrictStr = Field ( . . . , title = " URL of Git repo to download from " )
path : StrictStr = Field ( . . . , title = " File path or sub-directory to download (used for sparse checkout) " )
branch : StrictStr = Field ( " master " , title = " Branch to clone from " )
# fmt: on
class ProjectConfigAssetURL ( BaseModel ) :
2020-06-27 12:15:41 +00:00
# fmt: off
2020-06-21 11:44:00 +00:00
dest : StrictStr = Field ( . . . , title = " Destination of downloaded asset " )
2020-07-07 18:51:50 +00:00
url : Optional [ StrictStr ] = Field ( None , title = " URL of asset " )
2022-10-14 06:16:49 +00:00
checksum : Optional [ str ] = Field ( None , title = " MD5 hash of file " , regex = r " ([a-fA-F \ d] {32} ) " )
2020-08-25 15:14:27 +00:00
description : StrictStr = Field ( " " , title = " Description of asset " )
2020-06-27 12:15:41 +00:00
# fmt: on
2020-06-21 11:44:00 +00:00
2020-08-25 09:54:53 +00:00
class ProjectConfigAssetGit ( BaseModel ) :
# fmt: off
git : ProjectConfigAssetGitItem = Field ( . . . , title = " Git repo information " )
2022-10-14 06:16:49 +00:00
checksum : Optional [ str ] = Field ( None , title = " MD5 hash of file " , regex = r " ([a-fA-F \ d] {32} ) " )
2020-08-25 15:14:27 +00:00
description : Optional [ StrictStr ] = Field ( None , title = " Description of asset " )
2020-08-25 09:54:53 +00:00
# fmt: on
2020-06-21 11:44:00 +00:00
class ProjectConfigCommand ( BaseModel ) :
# fmt: off
name : StrictStr = Field ( . . . , title = " Name of command " )
help : Optional [ StrictStr ] = Field ( None , title = " Command description " )
script : List [ StrictStr ] = Field ( [ ] , title = " List of CLI commands to run, in order " )
2020-07-09 21:51:18 +00:00
deps : List [ StrictStr ] = Field ( [ ] , title = " File dependencies required by this command " )
outputs : List [ StrictStr ] = Field ( [ ] , title = " Outputs produced by this command " )
outputs_no_cache : List [ StrictStr ] = Field ( [ ] , title = " Outputs not tracked by DVC (DVC only) " )
no_skip : bool = Field ( False , title = " Never skip this command, even if nothing changed " )
2020-06-21 11:44:00 +00:00
# fmt: on
2020-06-22 12:53:31 +00:00
class Config :
title = " A single named command specified in a project config "
extra = " forbid "
2020-06-21 11:44:00 +00:00
class ProjectConfigSchema ( BaseModel ) :
# fmt: off
2020-08-23 16:32:09 +00:00
vars : Dict [ StrictStr , Any ] = Field ( { } , title = " Optional variables to substitute in commands " )
2021-02-10 02:45:27 +00:00
env : Dict [ StrictStr , Any ] = Field ( { } , title = " Optional variable names to substitute in commands, mapped to environment variable names " )
2020-08-25 09:54:53 +00:00
assets : List [ Union [ ProjectConfigAssetURL , ProjectConfigAssetGit ] ] = Field ( [ ] , title = " Data assets " )
2020-07-08 23:42:51 +00:00
workflows : Dict [ StrictStr , List [ StrictStr ] ] = Field ( { } , title = " Named workflows, mapped to list of project commands to run in order " )
2020-06-21 11:44:00 +00:00
commands : List [ ProjectConfigCommand ] = Field ( [ ] , title = " Project command shortucts " )
2020-08-25 15:14:27 +00:00
title : Optional [ str ] = Field ( None , title = " Project title " )
2020-10-05 18:00:42 +00:00
spacy_version : Optional [ StrictStr ] = Field ( None , title = " spaCy version range that the project is compatible with " )
2020-06-21 11:44:00 +00:00
# fmt: on
class Config :
title = " Schema for project configuration file "
2020-08-19 11:33:15 +00:00
# Recommendations for init config workflows
class RecommendationTrfItem ( BaseModel ) :
name : str
size_factor : int
class RecommendationTrf ( BaseModel ) :
efficiency : RecommendationTrfItem
accuracy : RecommendationTrfItem
class RecommendationSchema ( BaseModel ) :
word_vectors : Optional [ str ] = None
transformer : Optional [ RecommendationTrf ] = None
has_letters : bool = True
2022-06-02 12:03:47 +00:00
class DocJSONSchema ( BaseModel ) :
"""
JSON / dict format for JSON representation of Doc objects .
"""
cats : Optional [ Dict [ StrictStr , StrictFloat ] ] = Field (
None , title = " Categories with corresponding probabilities "
)
ents : Optional [ List [ Dict [ StrictStr , Union [ StrictInt , StrictStr ] ] ] ] = Field (
None , title = " Information on entities "
)
sents : Optional [ List [ Dict [ StrictStr , StrictInt ] ] ] = Field (
None , title = " Indices of sentences ' start and end indices "
)
text : StrictStr = Field ( . . . , title = " Document text " )
2022-10-14 09:04:55 +00:00
spans : Optional [
Dict [ StrictStr , List [ Dict [ StrictStr , Union [ StrictStr , StrictInt ] ] ] ]
] = Field ( None , title = " Span information - end/start indices, label, KB ID " )
2022-06-02 12:03:47 +00:00
tokens : List [ Dict [ StrictStr , Union [ StrictStr , StrictInt ] ] ] = Field (
. . . , title = " Token information - ID, start, annotations "
)
2022-08-23 08:05:02 +00:00
underscore_doc : Optional [ Dict [ StrictStr , Any ] ] = Field (
None ,
title = " Any custom data stored in the document ' s _ attribute " ,
alias = " _ " ,
)
2022-10-19 13:52:47 +00:00
underscore_token : Optional [ Dict [ StrictStr , List [ Dict [ StrictStr , Any ] ] ] ] = Field (
2022-08-23 08:05:02 +00:00
None , title = " Any custom data stored in the token ' s _ attribute "
)
2022-10-19 13:52:47 +00:00
underscore_span : Optional [ Dict [ StrictStr , List [ Dict [ StrictStr , Any ] ] ] ] = Field (
2022-08-23 08:05:02 +00:00
None , title = " Any custom data stored in the span ' s _ attribute "
2022-06-02 12:03:47 +00:00
)