2023-06-14 15:48:41 +00:00
import re
2017-03-21 01:06:29 +00:00
import shutil
2023-06-14 15:48:41 +00:00
import sys
2021-08-17 12:05:13 +00:00
from collections import defaultdict
2023-06-14 15:48:41 +00:00
from pathlib import Path
from typing import Any , Dict , List , Optional , Tuple , Union , cast
💫 Replace ujson, msgpack and dill/pickle/cloudpickle with srsly (#3003)
Remove hacks and wrappers, keep code in sync across our libraries and move spaCy a few steps closer to only depending on packages with binary wheels 🎉
See here: https://github.com/explosion/srsly
Serialization is hard, especially across Python versions and multiple platforms. After dealing with many subtle bugs over the years (encodings, locales, large files) our libraries like spaCy and Prodigy have steadily grown a number of utility functions to wrap the multiple serialization formats we need to support (especially json, msgpack and pickle). These wrapping functions ended up duplicated across our codebases, so we wanted to put them in one place.
At the same time, we noticed that having a lot of small dependencies was making maintainence harder, and making installation slower. To solve this, we've made srsly standalone, by including the component packages directly within it. This way we can provide all the serialization utilities we need in a single binary wheel.
srsly currently includes forks of the following packages:
ujson
msgpack
msgpack-numpy
cloudpickle
* WIP: replace json/ujson with srsly
* Replace ujson in examples
Use regular json instead of srsly to make code easier to read and follow
* Update requirements
* Fix imports
* Fix typos
* Replace msgpack with srsly
* Fix warning
2018-12-03 00:28:22 +00:00
import srsly
2023-06-14 15:48:41 +00:00
from catalogue import RegistryError
from thinc . api import Config
from wasabi import MarkdownRenderer , Printer , get_raw_input
2017-03-20 21:50:13 +00:00
2023-06-14 15:48:41 +00:00
from . . import about , util
from . . schemas import ModelMetaSchema , validate
from . _util import SDIST_SUFFIX , WHEEL_SUFFIX , Arg , Opt , app , string_to_list
2017-03-20 21:50:13 +00:00
2020-06-21 11:44:00 +00:00
@app.command ( " package " )
2020-06-21 19:35:01 +00:00
def package_cli (
2020-01-01 12:15:46 +00:00
# fmt: off
2020-09-03 11:13:03 +00:00
input_dir : Path = Arg ( . . . , help = " Directory with pipeline data " , exists = True , file_okay = False ) ,
2020-06-21 19:35:01 +00:00
output_dir : Path = Arg ( . . . , help = " Output parent directory " , exists = True , file_okay = False ) ,
2021-01-30 02:11:36 +00:00
code_paths : str = Opt ( " " , " --code " , " -c " , help = " Comma-separated paths to Python file with additional code (registered functions) to be included in the package " ) ,
2020-06-27 18:36:08 +00:00
meta_path : Optional [ Path ] = Opt ( None , " --meta-path " , " --meta " , " -m " , help = " Path to meta.json " , exists = True , dir_okay = False ) ,
2021-06-30 09:23:26 +00:00
create_meta : bool = Opt ( False , " --create-meta " , " -C " , help = " Create meta.json, even if one exists " ) ,
2020-09-11 09:38:28 +00:00
name : Optional [ str ] = Opt ( None , " --name " , " -n " , help = " Package name to override meta " ) ,
2020-06-27 18:36:08 +00:00
version : Optional [ str ] = Opt ( None , " --version " , " -v " , help = " Package version to override meta " ) ,
2021-01-30 02:11:36 +00:00
build : str = Opt ( " sdist " , " --build " , " -b " , help = " Comma-separated formats to build: sdist and/or wheel, or none. " ) ,
2020-09-03 11:13:03 +00:00
force : bool = Opt ( False , " --force " , " -f " , " -F " , help = " Force overwriting existing data in output directory " ) ,
2020-01-01 12:15:46 +00:00
# fmt: on
) :
2017-05-27 18:01:46 +00:00
"""
2020-09-03 11:13:03 +00:00
Generate an installable Python package for a pipeline . Includes binary data ,
2020-07-12 11:53:49 +00:00
meta and required installation files . A new directory will be created in the
2020-09-03 11:13:03 +00:00
specified output directory , and the data will be copied over . If
2020-07-12 11:53:49 +00:00
- - create - meta is set and a meta . json already exists in the output directory ,
the existing values will be used as the defaults in the command - line prompt .
After packaging , " python setup.py sdist " is run in the package directory ,
which will create a . tar . gz archive that can be installed via " pip install " .
2020-09-04 10:58:50 +00:00
2020-12-10 12:36:46 +00:00
If additional code files are provided ( e . g . Python files containing custom
registered functions like pipeline components ) , they are copied into the
package and imported in the __init__ . py .
2021-01-30 09:09:38 +00:00
DOCS : https : / / spacy . io / api / cli #package
2017-05-22 10:28:58 +00:00
"""
2021-01-30 02:11:36 +00:00
create_sdist , create_wheel = get_build_formats ( string_to_list ( build ) )
code_paths = [ Path ( p . strip ( ) ) for p in string_to_list ( code_paths ) ]
2020-06-21 19:35:01 +00:00
package (
input_dir ,
output_dir ,
meta_path = meta_path ,
2020-12-10 12:36:46 +00:00
code_paths = code_paths ,
2020-09-11 09:38:28 +00:00
name = name ,
2020-06-27 18:36:08 +00:00
version = version ,
2020-06-21 19:35:01 +00:00
create_meta = create_meta ,
2021-01-30 02:11:36 +00:00
create_sdist = create_sdist ,
create_wheel = create_wheel ,
2020-06-21 19:35:01 +00:00
force = force ,
silent = False ,
)
def package (
input_dir : Path ,
output_dir : Path ,
meta_path : Optional [ Path ] = None ,
2020-12-10 12:36:46 +00:00
code_paths : List [ Path ] = [ ] ,
2020-09-11 09:38:28 +00:00
name : Optional [ str ] = None ,
2020-06-27 18:36:08 +00:00
version : Optional [ str ] = None ,
2020-06-21 19:35:01 +00:00
create_meta : bool = False ,
2020-07-27 14:52:23 +00:00
create_sdist : bool = True ,
2021-01-30 00:54:02 +00:00
create_wheel : bool = False ,
2020-06-21 19:35:01 +00:00
force : bool = False ,
silent : bool = True ,
) - > None :
msg = Printer ( no_print = silent , pretty = not silent )
2017-05-07 21:25:29 +00:00
input_path = util . ensure_path ( input_dir )
output_path = util . ensure_path ( output_dir )
2017-08-12 19:44:15 +00:00
meta_path = util . ensure_path ( meta_path )
2021-01-30 00:54:02 +00:00
if create_wheel and not has_wheel ( ) :
err = " Generating a binary .whl file requires wheel to be installed "
msg . fail ( err , " pip install wheel " , exits = 1 )
2017-05-07 21:25:29 +00:00
if not input_path or not input_path . exists ( ) :
2020-09-03 11:13:03 +00:00
msg . fail ( " Can ' t locate pipeline data " , input_path , exits = 1 )
2017-05-07 21:25:29 +00:00
if not output_path or not output_path . exists ( ) :
2018-12-08 10:49:43 +00:00
msg . fail ( " Output directory not found " , output_path , exits = 1 )
2021-01-30 02:11:36 +00:00
if create_sdist or create_wheel :
opts = [ " sdist " if create_sdist else " " , " wheel " if create_wheel else " " ]
msg . info ( f " Building package artifacts: { ' , ' . join ( opt for opt in opts if opt ) } " )
2020-12-10 12:36:46 +00:00
for code_path in code_paths :
if not code_path . exists ( ) :
msg . fail ( " Can ' t find code file " , code_path , exits = 1 )
# Import the code here so it's available when model is loaded (via
# get_meta helper). Also verifies that everything works
util . import_file ( code_path . stem , code_path )
if code_paths :
msg . good ( f " Including { len ( code_paths ) } Python module(s) with custom code " )
2017-05-07 21:25:29 +00:00
if meta_path and not meta_path . exists ( ) :
2020-09-03 11:13:03 +00:00
msg . fail ( " Can ' t find pipeline meta.json " , meta_path , exits = 1 )
2020-06-21 19:35:01 +00:00
meta_path = meta_path or input_dir / " meta.json "
if not meta_path . exists ( ) or not meta_path . is_file ( ) :
2020-09-03 11:13:03 +00:00
msg . fail ( " Can ' t load pipeline meta.json " , meta_path , exits = 1 )
2020-06-21 19:35:01 +00:00
meta = srsly . read_json ( meta_path )
2020-06-27 18:36:08 +00:00
meta = get_meta ( input_dir , meta )
2021-08-17 12:05:13 +00:00
if meta [ " requirements " ] :
msg . good (
f " Including { len ( meta [ ' requirements ' ] ) } package requirement(s) from "
f " meta and config " ,
" , " . join ( meta [ " requirements " ] ) ,
)
2020-09-11 09:38:28 +00:00
if name is not None :
2022-02-10 07:15:23 +00:00
if not name . isidentifier ( ) :
msg . fail (
f " Model name ( ' { name } ' ) is not a valid module name. "
" This is required so it can be imported as a module. " ,
" We recommend names that use ASCII A-Z, a-z, _ (underscore), "
" and 0-9. "
" For specific details see: https://docs.python.org/3/reference/lexical_analysis.html#identifiers " ,
exits = 1 ,
)
if not _is_permitted_package_name ( name ) :
msg . fail (
f " Model name ( ' { name } ' ) is not a permitted package name. "
" This is required to correctly load the model with spacy.load. " ,
" We recommend names that use ASCII A-Z, a-z, _ (underscore), "
" and 0-9. "
" For specific details see: https://www.python.org/dev/peps/pep-0426/#name " ,
exits = 1 ,
)
2020-09-11 09:38:28 +00:00
meta [ " name " ] = name
2020-06-27 18:36:08 +00:00
if version is not None :
meta [ " version " ] = version
2020-06-21 19:35:01 +00:00
if not create_meta : # only print if user doesn't want to overwrite
msg . good ( " Loaded meta.json from file " , meta_path )
else :
2020-06-27 18:36:08 +00:00
meta = generate_meta ( meta , msg )
2020-06-21 19:35:01 +00:00
errors = validate ( ModelMetaSchema , meta )
if errors :
2020-09-03 11:13:03 +00:00
msg . fail ( " Invalid pipeline meta.json " )
2020-08-25 15:13:33 +00:00
print ( " \n " . join ( errors ) )
sys . exit ( 1 )
2021-04-26 14:53:21 +00:00
model_name = meta [ " name " ]
2021-06-11 08:20:24 +00:00
if not model_name . startswith ( meta [ " lang " ] + " _ " ) :
2021-04-26 14:53:21 +00:00
model_name = f " { meta [ ' lang ' ] } _ { model_name } "
2018-11-30 19:16:14 +00:00
model_name_v = model_name + " - " + meta [ " version " ]
2020-06-21 19:35:01 +00:00
main_path = output_dir / model_name_v
2017-03-20 21:50:13 +00:00
package_path = main_path / model_name
2017-03-21 01:06:53 +00:00
if package_path . exists ( ) :
if force :
2019-12-22 00:53:56 +00:00
shutil . rmtree ( str ( package_path ) )
2017-03-21 01:06:53 +00:00
else :
2018-11-30 19:16:14 +00:00
msg . fail (
2018-12-08 10:49:43 +00:00
" Package directory already exists " ,
" Please delete the directory and try again, or use the "
2019-12-22 00:53:56 +00:00
" `--force` flag to overwrite existing directories. " ,
2018-11-30 19:16:14 +00:00
exits = 1 ,
)
2017-03-21 01:06:53 +00:00
Path . mkdir ( package_path , parents = True )
2020-06-21 19:35:01 +00:00
shutil . copytree ( str ( input_dir ) , str ( package_path / model_name_v ) )
2021-06-11 08:20:24 +00:00
for file_name in FILENAMES_DOCS :
file_path = package_path / model_name_v / file_name
if file_path . exists ( ) :
2021-06-18 13:48:53 +00:00
shutil . copy ( str ( file_path ) , str ( main_path ) )
2021-06-22 02:06:25 +00:00
readme_path = main_path / " README.md "
if not readme_path . exists ( ) :
readme = generate_readme ( meta )
create_file ( readme_path , readme )
create_file ( package_path / model_name_v / " README.md " , readme )
2021-07-12 09:18:52 +00:00
msg . good ( " Generated README.md from meta.json " )
else :
msg . info ( " Using existing README.md from pipeline directory " )
2020-12-10 12:36:46 +00:00
imports = [ ]
for code_path in code_paths :
imports . append ( code_path . stem )
shutil . copy ( str ( code_path ) , str ( package_path ) )
2018-12-19 13:36:08 +00:00
create_file ( main_path / " meta.json " , srsly . json_dumps ( meta , indent = 2 ) )
2018-11-30 19:16:14 +00:00
create_file ( main_path / " setup.py " , TEMPLATE_SETUP )
create_file ( main_path / " MANIFEST.in " , TEMPLATE_MANIFEST )
2020-12-10 12:36:46 +00:00
init_py = TEMPLATE_INIT . format (
imports = " \n " . join ( f " from . import { m } " for m in imports )
)
create_file ( package_path / " __init__.py " , init_py )
2022-02-10 07:15:23 +00:00
msg . good ( f " Successfully created package directory ' { model_name_v } ' " , main_path )
2020-07-27 14:52:23 +00:00
if create_sdist :
with util . working_dir ( main_path ) :
2020-09-20 14:21:43 +00:00
util . run_command ( [ sys . executable , " setup.py " , " sdist " ] , capture = False )
2021-01-30 10:03:25 +00:00
zip_file = main_path / " dist " / f " { model_name_v } { SDIST_SUFFIX } "
2020-07-27 14:52:23 +00:00
msg . good ( f " Successfully created zipped Python package " , zip_file )
2021-01-30 00:54:02 +00:00
if create_wheel :
with util . working_dir ( main_path ) :
util . run_command ( [ sys . executable , " setup.py " , " bdist_wheel " ] , capture = False )
2022-02-10 07:15:23 +00:00
wheel_name_squashed = re . sub ( " _+ " , " _ " , model_name_v )
wheel = main_path / " dist " / f " { wheel_name_squashed } { WHEEL_SUFFIX } "
2021-01-30 00:54:02 +00:00
msg . good ( f " Successfully created binary wheel " , wheel )
2022-02-10 07:15:23 +00:00
if " __ " in model_name :
msg . warn (
f " Model name ( ' { model_name } ' ) contains a run of underscores. "
" Runs of underscores are not significant in installed package names. " ,
)
2021-01-30 00:54:02 +00:00
def has_wheel ( ) - > bool :
try :
import wheel # noqa: F401
return True
except ImportError :
return False
2017-03-21 01:06:53 +00:00
2021-08-17 12:05:13 +00:00
def get_third_party_dependencies (
config : Config , exclude : List [ str ] = util . SimpleFrozenList ( )
) - > List [ str ] :
""" If the config includes references to registered functions that are
provided by third - party packages ( spacy - transformers , other libraries ) , we
want to include them in meta [ " requirements " ] so that the package specifies
them as dependencies and the user won ' t have to do it manually.
We do this by :
- traversing the config to check for registered function ( @ keys )
- looking up the functions and getting their module
- looking up the module version and generating an appropriate version range
config ( Config ) : The pipeline config .
exclude ( list ) : List of packages to exclude ( e . g . that already exist in meta ) .
RETURNS ( list ) : The versioned requirements .
"""
2021-09-08 09:46:40 +00:00
own_packages = ( " spacy " , " spacy-legacy " , " spacy-nightly " , " thinc " , " srsly " )
2021-08-17 12:05:13 +00:00
distributions = util . packages_distributions ( )
funcs = defaultdict ( set )
2021-09-15 21:16:01 +00:00
# We only want to look at runtime-relevant sections, not [training] or [initialize]
for section in ( " nlp " , " components " ) :
for path , value in util . walk_dict ( config [ section ] ) :
if path [ - 1 ] . startswith ( " @ " ) : # collect all function references by registry
funcs [ path [ - 1 ] [ 1 : ] ] . add ( value )
2021-08-25 12:58:01 +00:00
for component in config . get ( " components " , { } ) . values ( ) :
if " factory " in component :
funcs [ " factories " ] . add ( component [ " factory " ] )
2021-08-17 12:05:13 +00:00
modules = set ( )
2021-11-29 07:31:02 +00:00
lang = config [ " nlp " ] [ " lang " ]
2021-08-17 12:05:13 +00:00
for reg_name , func_names in funcs . items ( ) :
for func_name in func_names :
2021-11-29 07:31:02 +00:00
# Try the lang-specific version and fall back
try :
func_info = util . registry . find ( reg_name , lang + " . " + func_name )
except RegistryError :
try :
func_info = util . registry . find ( reg_name , func_name )
except RegistryError as regerr :
# lang-specific version being absent is not actually an issue
raise regerr from None
🏷 Add Mypy check to CI and ignore all existing Mypy errors (#9167)
* 🚨 Ignore all existing Mypy errors
* 🏗 Add Mypy check to CI
* Add types-mock and types-requests as dev requirements
* Add additional type ignore directives
* Add types packages to dev-only list in reqs test
* Add types-dataclasses for python 3.6
* Add ignore to pretrain
* 🏷 Improve type annotation on `run_command` helper
The `run_command` helper previously declared that it returned an
`Optional[subprocess.CompletedProcess]`, but it isn't actually possible
for the function to return `None`. These changes modify the type
annotation of the `run_command` helper and remove all now-unnecessary
`# type: ignore` directives.
* 🔧 Allow variable type redefinition in limited contexts
These changes modify how Mypy is configured to allow variables to have
their type automatically redefined under certain conditions. The Mypy
documentation contains the following example:
```python
def process(items: List[str]) -> None:
# 'items' has type List[str]
items = [item.split() for item in items]
# 'items' now has type List[List[str]]
...
```
This configuration change is especially helpful in reducing the number
of `# type: ignore` directives needed to handle the common pattern of:
* Accepting a filepath as a string
* Overwriting the variable using `filepath = ensure_path(filepath)`
These changes enable redefinition and remove all `# type: ignore`
directives rendered redundant by this change.
* 🏷 Add type annotation to converters mapping
* 🚨 Fix Mypy error in convert CLI argument verification
* 🏷 Improve type annotation on `resolve_dot_names` helper
* 🏷 Add type annotations for `Vocab` attributes `strings` and `vectors`
* 🏷 Add type annotations for more `Vocab` attributes
* 🏷 Add loose type annotation for gold data compilation
* 🏷 Improve `_format_labels` type annotation
* 🏷 Fix `get_lang_class` type annotation
* 🏷 Loosen return type of `Language.evaluate`
* 🏷 Don't accept `Scorer` in `handle_scores_per_type`
* 🏷 Add `string_to_list` overloads
* 🏷 Fix non-Optional command-line options
* 🙈 Ignore redefinition of `wandb_logger` in `loggers.py`
* ➕ Install `typing_extensions` in Python 3.8+
The `typing_extensions` package states that it should be used when
"writing code that must be compatible with multiple Python versions".
Since SpaCy needs to support multiple Python versions, it should be used
when newer `typing` module members are required. One example of this is
`Literal`, which is available starting with Python 3.8.
Previously SpaCy tried to import `Literal` from `typing`, falling back
to `typing_extensions` if the import failed. However, Mypy doesn't seem
to be able to understand what `Literal` means when the initial import
means. Therefore, these changes modify how `compat` imports `Literal` by
always importing it from `typing_extensions`.
These changes also modify how `typing_extensions` is installed, so that
it is a requirement for all Python versions, including those greater
than or equal to 3.8.
* 🏷 Improve type annotation for `Language.pipe`
These changes add a missing overload variant to the type signature of
`Language.pipe`. Additionally, the type signature is enhanced to allow
type checkers to differentiate between the two overload variants based
on the `as_tuple` parameter.
Fixes #8772
* ➖ Don't install `typing-extensions` in Python 3.8+
After more detailed analysis of how to implement Python version-specific
type annotations using SpaCy, it has been determined that by branching
on a comparison against `sys.version_info` can be statically analyzed by
Mypy well enough to enable us to conditionally use
`typing_extensions.Literal`. This means that we no longer need to
install `typing_extensions` for Python versions greater than or equal to
3.8! 🎉
These changes revert previous changes installing `typing-extensions`
regardless of Python version and modify how we import the `Literal` type
to ensure that Mypy treats it properly.
* resolve mypy errors for Strict pydantic types
* refactor code to avoid missing return statement
* fix types of convert CLI command
* avoid list-set confustion in debug_data
* fix typo and formatting
* small fixes to avoid type ignores
* fix types in profile CLI command and make it more efficient
* type fixes in projects CLI
* put one ignore back
* type fixes for render
* fix render types - the sequel
* fix BaseDefault in language definitions
* fix type of noun_chunks iterator - yields tuple instead of span
* fix types in language-specific modules
* 🏷 Expand accepted inputs of `get_string_id`
`get_string_id` accepts either a string (in which case it returns its
ID) or an ID (in which case it immediately returns the ID). These
changes extend the type annotation of `get_string_id` to indicate that
it can accept either strings or IDs.
* 🏷 Handle override types in `combine_score_weights`
The `combine_score_weights` function allows users to pass an `overrides`
mapping to override data extracted from the `weights` argument. Since it
allows `Optional` dictionary values, the return value may also include
`Optional` dictionary values.
These changes update the type annotations for `combine_score_weights` to
reflect this fact.
* 🏷 Fix tokenizer serialization method signatures in `DummyTokenizer`
* 🏷 Fix redefinition of `wandb_logger`
These changes fix the redefinition of `wandb_logger` by giving a
separate name to each `WandbLogger` version. For
backwards-compatibility, `spacy.train` still exports `wandb_logger_v3`
as `wandb_logger` for now.
* more fixes for typing in language
* type fixes in model definitions
* 🏷 Annotate `_RandomWords.probs` as `NDArray`
* 🏷 Annotate `tok2vec` layers to help Mypy
* 🐛 Fix `_RandomWords.probs` type annotations for Python 3.6
Also remove an import that I forgot to move to the top of the module 😅
* more fixes for matchers and other pipeline components
* quick fix for entity linker
* fixing types for spancat, textcat, etc
* bugfix for tok2vec
* type annotations for scorer
* add runtime_checkable for Protocol
* type and import fixes in tests
* mypy fixes for training utilities
* few fixes in util
* fix import
* 🐵 Remove unused `# type: ignore` directives
* 🏷 Annotate `Language._components`
* 🏷 Annotate `spacy.pipeline.Pipe`
* add doc as property to span.pyi
* small fixes and cleanup
* explicit type annotations instead of via comment
Co-authored-by: Adriane Boyd <adrianeboyd@gmail.com>
Co-authored-by: svlandeg <sofie.vanlandeghem@gmail.com>
Co-authored-by: svlandeg <svlandeg@github.com>
2021-10-14 13:21:40 +00:00
module_name = func_info . get ( " module " ) # type: ignore[attr-defined]
2021-08-17 12:05:13 +00:00
if module_name : # the code is part of a module, not a --code file
2023-02-15 11:32:53 +00:00
modules . add ( func_info [ " module " ] . split ( " . " ) [ 0 ] ) # type: ignore[union-attr]
2021-08-17 12:05:13 +00:00
dependencies = [ ]
for module_name in modules :
if module_name in distributions :
dist = distributions . get ( module_name )
if dist :
pkg = dist [ 0 ]
if pkg in own_packages or pkg in exclude :
continue
version = util . get_package_version ( pkg )
🏷 Add Mypy check to CI and ignore all existing Mypy errors (#9167)
* 🚨 Ignore all existing Mypy errors
* 🏗 Add Mypy check to CI
* Add types-mock and types-requests as dev requirements
* Add additional type ignore directives
* Add types packages to dev-only list in reqs test
* Add types-dataclasses for python 3.6
* Add ignore to pretrain
* 🏷 Improve type annotation on `run_command` helper
The `run_command` helper previously declared that it returned an
`Optional[subprocess.CompletedProcess]`, but it isn't actually possible
for the function to return `None`. These changes modify the type
annotation of the `run_command` helper and remove all now-unnecessary
`# type: ignore` directives.
* 🔧 Allow variable type redefinition in limited contexts
These changes modify how Mypy is configured to allow variables to have
their type automatically redefined under certain conditions. The Mypy
documentation contains the following example:
```python
def process(items: List[str]) -> None:
# 'items' has type List[str]
items = [item.split() for item in items]
# 'items' now has type List[List[str]]
...
```
This configuration change is especially helpful in reducing the number
of `# type: ignore` directives needed to handle the common pattern of:
* Accepting a filepath as a string
* Overwriting the variable using `filepath = ensure_path(filepath)`
These changes enable redefinition and remove all `# type: ignore`
directives rendered redundant by this change.
* 🏷 Add type annotation to converters mapping
* 🚨 Fix Mypy error in convert CLI argument verification
* 🏷 Improve type annotation on `resolve_dot_names` helper
* 🏷 Add type annotations for `Vocab` attributes `strings` and `vectors`
* 🏷 Add type annotations for more `Vocab` attributes
* 🏷 Add loose type annotation for gold data compilation
* 🏷 Improve `_format_labels` type annotation
* 🏷 Fix `get_lang_class` type annotation
* 🏷 Loosen return type of `Language.evaluate`
* 🏷 Don't accept `Scorer` in `handle_scores_per_type`
* 🏷 Add `string_to_list` overloads
* 🏷 Fix non-Optional command-line options
* 🙈 Ignore redefinition of `wandb_logger` in `loggers.py`
* ➕ Install `typing_extensions` in Python 3.8+
The `typing_extensions` package states that it should be used when
"writing code that must be compatible with multiple Python versions".
Since SpaCy needs to support multiple Python versions, it should be used
when newer `typing` module members are required. One example of this is
`Literal`, which is available starting with Python 3.8.
Previously SpaCy tried to import `Literal` from `typing`, falling back
to `typing_extensions` if the import failed. However, Mypy doesn't seem
to be able to understand what `Literal` means when the initial import
means. Therefore, these changes modify how `compat` imports `Literal` by
always importing it from `typing_extensions`.
These changes also modify how `typing_extensions` is installed, so that
it is a requirement for all Python versions, including those greater
than or equal to 3.8.
* 🏷 Improve type annotation for `Language.pipe`
These changes add a missing overload variant to the type signature of
`Language.pipe`. Additionally, the type signature is enhanced to allow
type checkers to differentiate between the two overload variants based
on the `as_tuple` parameter.
Fixes #8772
* ➖ Don't install `typing-extensions` in Python 3.8+
After more detailed analysis of how to implement Python version-specific
type annotations using SpaCy, it has been determined that by branching
on a comparison against `sys.version_info` can be statically analyzed by
Mypy well enough to enable us to conditionally use
`typing_extensions.Literal`. This means that we no longer need to
install `typing_extensions` for Python versions greater than or equal to
3.8! 🎉
These changes revert previous changes installing `typing-extensions`
regardless of Python version and modify how we import the `Literal` type
to ensure that Mypy treats it properly.
* resolve mypy errors for Strict pydantic types
* refactor code to avoid missing return statement
* fix types of convert CLI command
* avoid list-set confustion in debug_data
* fix typo and formatting
* small fixes to avoid type ignores
* fix types in profile CLI command and make it more efficient
* type fixes in projects CLI
* put one ignore back
* type fixes for render
* fix render types - the sequel
* fix BaseDefault in language definitions
* fix type of noun_chunks iterator - yields tuple instead of span
* fix types in language-specific modules
* 🏷 Expand accepted inputs of `get_string_id`
`get_string_id` accepts either a string (in which case it returns its
ID) or an ID (in which case it immediately returns the ID). These
changes extend the type annotation of `get_string_id` to indicate that
it can accept either strings or IDs.
* 🏷 Handle override types in `combine_score_weights`
The `combine_score_weights` function allows users to pass an `overrides`
mapping to override data extracted from the `weights` argument. Since it
allows `Optional` dictionary values, the return value may also include
`Optional` dictionary values.
These changes update the type annotations for `combine_score_weights` to
reflect this fact.
* 🏷 Fix tokenizer serialization method signatures in `DummyTokenizer`
* 🏷 Fix redefinition of `wandb_logger`
These changes fix the redefinition of `wandb_logger` by giving a
separate name to each `WandbLogger` version. For
backwards-compatibility, `spacy.train` still exports `wandb_logger_v3`
as `wandb_logger` for now.
* more fixes for typing in language
* type fixes in model definitions
* 🏷 Annotate `_RandomWords.probs` as `NDArray`
* 🏷 Annotate `tok2vec` layers to help Mypy
* 🐛 Fix `_RandomWords.probs` type annotations for Python 3.6
Also remove an import that I forgot to move to the top of the module 😅
* more fixes for matchers and other pipeline components
* quick fix for entity linker
* fixing types for spancat, textcat, etc
* bugfix for tok2vec
* type annotations for scorer
* add runtime_checkable for Protocol
* type and import fixes in tests
* mypy fixes for training utilities
* few fixes in util
* fix import
* 🐵 Remove unused `# type: ignore` directives
* 🏷 Annotate `Language._components`
* 🏷 Annotate `spacy.pipeline.Pipe`
* add doc as property to span.pyi
* small fixes and cleanup
* explicit type annotations instead of via comment
Co-authored-by: Adriane Boyd <adrianeboyd@gmail.com>
Co-authored-by: svlandeg <sofie.vanlandeghem@gmail.com>
Co-authored-by: svlandeg <svlandeg@github.com>
2021-10-14 13:21:40 +00:00
version_range = util . get_minor_version_range ( version ) # type: ignore[arg-type]
2021-08-17 12:05:13 +00:00
dependencies . append ( f " { pkg } { version_range } " )
return dependencies
2021-01-30 02:11:36 +00:00
def get_build_formats ( formats : List [ str ] ) - > Tuple [ bool , bool ] :
supported = [ " sdist " , " wheel " , " none " ]
for form in formats :
if form not in supported :
msg = Printer ( )
err = f " Unknown build format: { form } . Supported: { ' , ' . join ( supported ) } "
msg . fail ( err , exits = 1 )
if not formats or " none " in formats :
return ( False , False )
return ( " sdist " in formats , " wheel " in formats )
2020-06-21 19:35:01 +00:00
def create_file ( file_path : Path , contents : str ) - > None :
2017-03-20 21:50:13 +00:00
file_path . touch ( )
2018-11-30 19:16:14 +00:00
file_path . open ( " w " , encoding = " utf-8 " ) . write ( contents )
2017-03-20 21:50:13 +00:00
2020-06-27 18:36:08 +00:00
def get_meta (
model_path : Union [ str , Path ] , existing_meta : Dict [ str , Any ]
2020-06-21 19:35:01 +00:00
) - > Dict [ str , Any ] :
🏷 Add Mypy check to CI and ignore all existing Mypy errors (#9167)
* 🚨 Ignore all existing Mypy errors
* 🏗 Add Mypy check to CI
* Add types-mock and types-requests as dev requirements
* Add additional type ignore directives
* Add types packages to dev-only list in reqs test
* Add types-dataclasses for python 3.6
* Add ignore to pretrain
* 🏷 Improve type annotation on `run_command` helper
The `run_command` helper previously declared that it returned an
`Optional[subprocess.CompletedProcess]`, but it isn't actually possible
for the function to return `None`. These changes modify the type
annotation of the `run_command` helper and remove all now-unnecessary
`# type: ignore` directives.
* 🔧 Allow variable type redefinition in limited contexts
These changes modify how Mypy is configured to allow variables to have
their type automatically redefined under certain conditions. The Mypy
documentation contains the following example:
```python
def process(items: List[str]) -> None:
# 'items' has type List[str]
items = [item.split() for item in items]
# 'items' now has type List[List[str]]
...
```
This configuration change is especially helpful in reducing the number
of `# type: ignore` directives needed to handle the common pattern of:
* Accepting a filepath as a string
* Overwriting the variable using `filepath = ensure_path(filepath)`
These changes enable redefinition and remove all `# type: ignore`
directives rendered redundant by this change.
* 🏷 Add type annotation to converters mapping
* 🚨 Fix Mypy error in convert CLI argument verification
* 🏷 Improve type annotation on `resolve_dot_names` helper
* 🏷 Add type annotations for `Vocab` attributes `strings` and `vectors`
* 🏷 Add type annotations for more `Vocab` attributes
* 🏷 Add loose type annotation for gold data compilation
* 🏷 Improve `_format_labels` type annotation
* 🏷 Fix `get_lang_class` type annotation
* 🏷 Loosen return type of `Language.evaluate`
* 🏷 Don't accept `Scorer` in `handle_scores_per_type`
* 🏷 Add `string_to_list` overloads
* 🏷 Fix non-Optional command-line options
* 🙈 Ignore redefinition of `wandb_logger` in `loggers.py`
* ➕ Install `typing_extensions` in Python 3.8+
The `typing_extensions` package states that it should be used when
"writing code that must be compatible with multiple Python versions".
Since SpaCy needs to support multiple Python versions, it should be used
when newer `typing` module members are required. One example of this is
`Literal`, which is available starting with Python 3.8.
Previously SpaCy tried to import `Literal` from `typing`, falling back
to `typing_extensions` if the import failed. However, Mypy doesn't seem
to be able to understand what `Literal` means when the initial import
means. Therefore, these changes modify how `compat` imports `Literal` by
always importing it from `typing_extensions`.
These changes also modify how `typing_extensions` is installed, so that
it is a requirement for all Python versions, including those greater
than or equal to 3.8.
* 🏷 Improve type annotation for `Language.pipe`
These changes add a missing overload variant to the type signature of
`Language.pipe`. Additionally, the type signature is enhanced to allow
type checkers to differentiate between the two overload variants based
on the `as_tuple` parameter.
Fixes #8772
* ➖ Don't install `typing-extensions` in Python 3.8+
After more detailed analysis of how to implement Python version-specific
type annotations using SpaCy, it has been determined that by branching
on a comparison against `sys.version_info` can be statically analyzed by
Mypy well enough to enable us to conditionally use
`typing_extensions.Literal`. This means that we no longer need to
install `typing_extensions` for Python versions greater than or equal to
3.8! 🎉
These changes revert previous changes installing `typing-extensions`
regardless of Python version and modify how we import the `Literal` type
to ensure that Mypy treats it properly.
* resolve mypy errors for Strict pydantic types
* refactor code to avoid missing return statement
* fix types of convert CLI command
* avoid list-set confustion in debug_data
* fix typo and formatting
* small fixes to avoid type ignores
* fix types in profile CLI command and make it more efficient
* type fixes in projects CLI
* put one ignore back
* type fixes for render
* fix render types - the sequel
* fix BaseDefault in language definitions
* fix type of noun_chunks iterator - yields tuple instead of span
* fix types in language-specific modules
* 🏷 Expand accepted inputs of `get_string_id`
`get_string_id` accepts either a string (in which case it returns its
ID) or an ID (in which case it immediately returns the ID). These
changes extend the type annotation of `get_string_id` to indicate that
it can accept either strings or IDs.
* 🏷 Handle override types in `combine_score_weights`
The `combine_score_weights` function allows users to pass an `overrides`
mapping to override data extracted from the `weights` argument. Since it
allows `Optional` dictionary values, the return value may also include
`Optional` dictionary values.
These changes update the type annotations for `combine_score_weights` to
reflect this fact.
* 🏷 Fix tokenizer serialization method signatures in `DummyTokenizer`
* 🏷 Fix redefinition of `wandb_logger`
These changes fix the redefinition of `wandb_logger` by giving a
separate name to each `WandbLogger` version. For
backwards-compatibility, `spacy.train` still exports `wandb_logger_v3`
as `wandb_logger` for now.
* more fixes for typing in language
* type fixes in model definitions
* 🏷 Annotate `_RandomWords.probs` as `NDArray`
* 🏷 Annotate `tok2vec` layers to help Mypy
* 🐛 Fix `_RandomWords.probs` type annotations for Python 3.6
Also remove an import that I forgot to move to the top of the module 😅
* more fixes for matchers and other pipeline components
* quick fix for entity linker
* fixing types for spancat, textcat, etc
* bugfix for tok2vec
* type annotations for scorer
* add runtime_checkable for Protocol
* type and import fixes in tests
* mypy fixes for training utilities
* few fixes in util
* fix import
* 🐵 Remove unused `# type: ignore` directives
* 🏷 Annotate `Language._components`
* 🏷 Annotate `spacy.pipeline.Pipe`
* add doc as property to span.pyi
* small fixes and cleanup
* explicit type annotations instead of via comment
Co-authored-by: Adriane Boyd <adrianeboyd@gmail.com>
Co-authored-by: svlandeg <sofie.vanlandeghem@gmail.com>
Co-authored-by: svlandeg <svlandeg@github.com>
2021-10-14 13:21:40 +00:00
meta : Dict [ str , Any ] = {
2020-06-27 18:36:08 +00:00
" lang " : " en " ,
2020-09-03 11:13:03 +00:00
" name " : " pipeline " ,
2020-06-27 18:36:08 +00:00
" version " : " 0.0.0 " ,
2020-08-25 15:13:33 +00:00
" description " : " " ,
" author " : " " ,
" email " : " " ,
" url " : " " ,
2020-06-27 18:36:08 +00:00
" license " : " MIT " ,
}
2017-10-25 14:03:26 +00:00
nlp = util . load_model_from_path ( Path ( model_path ) )
2021-07-12 09:18:52 +00:00
meta . update ( nlp . meta )
2021-08-17 12:05:13 +00:00
meta [ " spacy_version " ] = util . get_minor_version_range ( about . __version__ )
2022-09-29 08:44:06 +00:00
meta . update ( existing_meta )
2018-11-30 19:16:14 +00:00
meta [ " vectors " ] = {
" width " : nlp . vocab . vectors_length ,
" vectors " : len ( nlp . vocab . vectors ) ,
" keys " : nlp . vocab . vectors . n_keys ,
2018-12-27 18:55:40 +00:00
" name " : nlp . vocab . vectors . name ,
2018-11-30 19:16:14 +00:00
}
2020-06-27 18:36:08 +00:00
if about . __title__ != " spacy " :
meta [ " parent_package " ] = about . __title__
2021-08-17 12:05:13 +00:00
meta . setdefault ( " requirements " , [ ] )
# Update the requirements with all third-party packages in the config
existing_reqs = [ util . split_requirement ( req ) [ 0 ] for req in meta [ " requirements " ] ]
reqs = get_third_party_dependencies ( nlp . config , exclude = existing_reqs )
meta [ " requirements " ] . extend ( reqs )
2020-06-27 18:36:08 +00:00
return meta
def generate_meta ( existing_meta : Dict [ str , Any ] , msg : Printer ) - > Dict [ str , Any ] :
meta = existing_meta or { }
settings = [
2020-09-03 11:13:03 +00:00
( " lang " , " Pipeline language " , meta . get ( " lang " , " en " ) ) ,
( " name " , " Pipeline name " , meta . get ( " name " , " pipeline " ) ) ,
( " version " , " Package version " , meta . get ( " version " , " 0.0.0 " ) ) ,
( " description " , " Package description " , meta . get ( " description " , None ) ) ,
2020-06-27 18:36:08 +00:00
( " author " , " Author " , meta . get ( " author " , None ) ) ,
( " email " , " Author email " , meta . get ( " email " , None ) ) ,
( " url " , " Author website " , meta . get ( " url " , None ) ) ,
( " license " , " License " , meta . get ( " license " , " MIT " ) ) ,
]
2018-12-08 10:49:43 +00:00
msg . divider ( " Generating meta.json " )
msg . text (
2020-09-03 11:13:03 +00:00
" Enter the package settings for your pipeline. The following information "
" will be read from your pipeline data: pipeline, vectors. "
2018-12-08 10:49:43 +00:00
)
2017-03-20 21:50:13 +00:00
for setting , desc , default in settings :
2018-11-30 19:16:14 +00:00
response = get_raw_input ( desc , default )
meta [ setting ] = default if response == " " and default else response
2017-05-27 18:02:01 +00:00
return meta
2017-04-16 11:13:17 +00:00
2021-06-22 02:06:25 +00:00
def generate_readme ( meta : Dict [ str , Any ] ) - > str :
"""
Generate a Markdown - formatted README text from a model meta . json . Used
within the GitHub release notes and as content for README . md file added
to model packages .
"""
md = MarkdownRenderer ( )
lang = meta [ " lang " ]
name = f " { lang } _ { meta [ ' name ' ] } "
version = meta [ " version " ]
pipeline = " , " . join ( [ md . code ( p ) for p in meta . get ( " pipeline " , [ ] ) ] )
components = " , " . join ( [ md . code ( p ) for p in meta . get ( " components " , [ ] ) ] )
vecs = meta . get ( " vectors " , { } )
vectors = f " { vecs . get ( ' keys ' , 0 ) } keys, { vecs . get ( ' vectors ' , 0 ) } unique vectors ( { vecs . get ( ' width ' , 0 ) } dimensions) "
2021-06-24 01:55:50 +00:00
author = meta . get ( " author " ) or " n/a "
2021-06-22 02:06:25 +00:00
notes = meta . get ( " notes " , " " )
2021-06-24 01:55:50 +00:00
license_name = meta . get ( " license " )
sources = _format_sources ( meta . get ( " sources " ) )
description = meta . get ( " description " )
🏷 Add Mypy check to CI and ignore all existing Mypy errors (#9167)
* 🚨 Ignore all existing Mypy errors
* 🏗 Add Mypy check to CI
* Add types-mock and types-requests as dev requirements
* Add additional type ignore directives
* Add types packages to dev-only list in reqs test
* Add types-dataclasses for python 3.6
* Add ignore to pretrain
* 🏷 Improve type annotation on `run_command` helper
The `run_command` helper previously declared that it returned an
`Optional[subprocess.CompletedProcess]`, but it isn't actually possible
for the function to return `None`. These changes modify the type
annotation of the `run_command` helper and remove all now-unnecessary
`# type: ignore` directives.
* 🔧 Allow variable type redefinition in limited contexts
These changes modify how Mypy is configured to allow variables to have
their type automatically redefined under certain conditions. The Mypy
documentation contains the following example:
```python
def process(items: List[str]) -> None:
# 'items' has type List[str]
items = [item.split() for item in items]
# 'items' now has type List[List[str]]
...
```
This configuration change is especially helpful in reducing the number
of `# type: ignore` directives needed to handle the common pattern of:
* Accepting a filepath as a string
* Overwriting the variable using `filepath = ensure_path(filepath)`
These changes enable redefinition and remove all `# type: ignore`
directives rendered redundant by this change.
* 🏷 Add type annotation to converters mapping
* 🚨 Fix Mypy error in convert CLI argument verification
* 🏷 Improve type annotation on `resolve_dot_names` helper
* 🏷 Add type annotations for `Vocab` attributes `strings` and `vectors`
* 🏷 Add type annotations for more `Vocab` attributes
* 🏷 Add loose type annotation for gold data compilation
* 🏷 Improve `_format_labels` type annotation
* 🏷 Fix `get_lang_class` type annotation
* 🏷 Loosen return type of `Language.evaluate`
* 🏷 Don't accept `Scorer` in `handle_scores_per_type`
* 🏷 Add `string_to_list` overloads
* 🏷 Fix non-Optional command-line options
* 🙈 Ignore redefinition of `wandb_logger` in `loggers.py`
* ➕ Install `typing_extensions` in Python 3.8+
The `typing_extensions` package states that it should be used when
"writing code that must be compatible with multiple Python versions".
Since SpaCy needs to support multiple Python versions, it should be used
when newer `typing` module members are required. One example of this is
`Literal`, which is available starting with Python 3.8.
Previously SpaCy tried to import `Literal` from `typing`, falling back
to `typing_extensions` if the import failed. However, Mypy doesn't seem
to be able to understand what `Literal` means when the initial import
means. Therefore, these changes modify how `compat` imports `Literal` by
always importing it from `typing_extensions`.
These changes also modify how `typing_extensions` is installed, so that
it is a requirement for all Python versions, including those greater
than or equal to 3.8.
* 🏷 Improve type annotation for `Language.pipe`
These changes add a missing overload variant to the type signature of
`Language.pipe`. Additionally, the type signature is enhanced to allow
type checkers to differentiate between the two overload variants based
on the `as_tuple` parameter.
Fixes #8772
* ➖ Don't install `typing-extensions` in Python 3.8+
After more detailed analysis of how to implement Python version-specific
type annotations using SpaCy, it has been determined that by branching
on a comparison against `sys.version_info` can be statically analyzed by
Mypy well enough to enable us to conditionally use
`typing_extensions.Literal`. This means that we no longer need to
install `typing_extensions` for Python versions greater than or equal to
3.8! 🎉
These changes revert previous changes installing `typing-extensions`
regardless of Python version and modify how we import the `Literal` type
to ensure that Mypy treats it properly.
* resolve mypy errors for Strict pydantic types
* refactor code to avoid missing return statement
* fix types of convert CLI command
* avoid list-set confustion in debug_data
* fix typo and formatting
* small fixes to avoid type ignores
* fix types in profile CLI command and make it more efficient
* type fixes in projects CLI
* put one ignore back
* type fixes for render
* fix render types - the sequel
* fix BaseDefault in language definitions
* fix type of noun_chunks iterator - yields tuple instead of span
* fix types in language-specific modules
* 🏷 Expand accepted inputs of `get_string_id`
`get_string_id` accepts either a string (in which case it returns its
ID) or an ID (in which case it immediately returns the ID). These
changes extend the type annotation of `get_string_id` to indicate that
it can accept either strings or IDs.
* 🏷 Handle override types in `combine_score_weights`
The `combine_score_weights` function allows users to pass an `overrides`
mapping to override data extracted from the `weights` argument. Since it
allows `Optional` dictionary values, the return value may also include
`Optional` dictionary values.
These changes update the type annotations for `combine_score_weights` to
reflect this fact.
* 🏷 Fix tokenizer serialization method signatures in `DummyTokenizer`
* 🏷 Fix redefinition of `wandb_logger`
These changes fix the redefinition of `wandb_logger` by giving a
separate name to each `WandbLogger` version. For
backwards-compatibility, `spacy.train` still exports `wandb_logger_v3`
as `wandb_logger` for now.
* more fixes for typing in language
* type fixes in model definitions
* 🏷 Annotate `_RandomWords.probs` as `NDArray`
* 🏷 Annotate `tok2vec` layers to help Mypy
* 🐛 Fix `_RandomWords.probs` type annotations for Python 3.6
Also remove an import that I forgot to move to the top of the module 😅
* more fixes for matchers and other pipeline components
* quick fix for entity linker
* fixing types for spancat, textcat, etc
* bugfix for tok2vec
* type annotations for scorer
* add runtime_checkable for Protocol
* type and import fixes in tests
* mypy fixes for training utilities
* few fixes in util
* fix import
* 🐵 Remove unused `# type: ignore` directives
* 🏷 Annotate `Language._components`
* 🏷 Annotate `spacy.pipeline.Pipe`
* add doc as property to span.pyi
* small fixes and cleanup
* explicit type annotations instead of via comment
Co-authored-by: Adriane Boyd <adrianeboyd@gmail.com>
Co-authored-by: svlandeg <sofie.vanlandeghem@gmail.com>
Co-authored-by: svlandeg <svlandeg@github.com>
2021-10-14 13:21:40 +00:00
label_scheme = _format_label_scheme ( cast ( Dict [ str , Any ] , meta . get ( " labels " ) ) )
accuracy = _format_accuracy ( cast ( Dict [ str , Any ] , meta . get ( " performance " ) ) )
2021-06-22 02:06:25 +00:00
table_data = [
( md . bold ( " Name " ) , md . code ( name ) ) ,
( md . bold ( " Version " ) , md . code ( version ) ) ,
( md . bold ( " spaCy " ) , md . code ( meta [ " spacy_version " ] ) ) ,
( md . bold ( " Default Pipeline " ) , pipeline ) ,
( md . bold ( " Components " ) , components ) ,
( md . bold ( " Vectors " ) , vectors ) ,
2021-06-24 01:55:50 +00:00
( md . bold ( " Sources " ) , sources or " n/a " ) ,
( md . bold ( " License " ) , md . code ( license_name ) if license_name else " n/a " ) ,
2021-06-22 02:06:25 +00:00
( md . bold ( " Author " ) , md . link ( author , meta [ " url " ] ) if " url " in meta else author ) ,
]
# Put together Markdown body
2021-06-24 01:55:50 +00:00
if description :
md . add ( description )
2021-06-22 02:06:25 +00:00
md . add ( md . table ( table_data , [ " Feature " , " Description " ] ) )
2021-06-24 01:55:50 +00:00
if label_scheme :
md . add ( md . title ( 3 , " Label Scheme " ) )
md . add ( label_scheme )
if accuracy :
md . add ( md . title ( 3 , " Accuracy " ) )
md . add ( accuracy )
2021-06-22 02:06:25 +00:00
if notes :
md . add ( notes )
return md . text
def _format_sources ( data : Any ) - > str :
if not data or not isinstance ( data , list ) :
return " n/a "
sources = [ ]
for source in data :
if not isinstance ( source , dict ) :
source = { " name " : source }
name = source . get ( " name " )
if not name :
continue
url = source . get ( " url " )
author = source . get ( " author " )
result = name if not url else " [ {} ]( {} ) " . format ( name , url )
if author :
result + = " ( {} ) " . format ( author )
sources . append ( result )
return " <br /> " . join ( sources )
def _format_accuracy ( data : Dict [ str , Any ] , exclude : List [ str ] = [ " speed " ] ) - > str :
if not data :
return " "
md = MarkdownRenderer ( )
scalars = [ ( k , v ) for k , v in data . items ( ) if isinstance ( v , ( int , float ) ) ]
scores = [
( md . code ( acc . upper ( ) ) , f " { score * 100 : .2f } " )
for acc , score in scalars
if acc not in exclude
]
md . add ( md . table ( scores , [ " Type " , " Score " ] ) )
return md . text
def _format_label_scheme ( data : Dict [ str , Any ] ) - > str :
if not data :
return " "
md = MarkdownRenderer ( )
n_labels = 0
n_pipes = 0
label_data = [ ]
for pipe , labels in data . items ( ) :
if not labels :
continue
col1 = md . bold ( md . code ( pipe ) )
col2 = " , " . join (
2021-11-19 07:51:46 +00:00
[ md . code ( str ( label ) . replace ( " | " , " \\ | " ) ) for label in labels ]
2021-06-22 02:06:25 +00:00
) # noqa: W605
label_data . append ( ( col1 , col2 ) )
n_labels + = len ( labels )
n_pipes + = 1
if not label_data :
return " "
label_info = f " View label scheme ( { n_labels } labels for { n_pipes } components) "
md . add ( " <details> " )
md . add ( f " <summary> { label_info } </summary> " )
md . add ( md . table ( label_data , [ " Component " , " Labels " ] ) )
md . add ( " </details> " )
return md . text
2022-02-10 07:15:23 +00:00
def _is_permitted_package_name ( package_name : str ) - > bool :
# regex from: https://www.python.org/dev/peps/pep-0426/#name
permitted_match = re . search (
r " ^([A-Z0-9]|[A-Z0-9][A-Z0-9._-]*[A-Z0-9])$ " , package_name , re . IGNORECASE
)
return permitted_match is not None
2017-11-07 11:15:35 +00:00
TEMPLATE_SETUP = """
#!/usr/bin/env python
import io
import json
from os import path , walk
from shutil import copy
from setuptools import setup
def load_meta ( fp ) :
with io . open ( fp , encoding = ' utf8 ' ) as f :
return json . load ( f )
2021-06-18 13:48:53 +00:00
def load_readme ( fp ) :
if path . exists ( fp ) :
with io . open ( fp , encoding = ' utf8 ' ) as f :
return f . read ( )
return " "
2017-11-07 11:15:35 +00:00
def list_files ( data_dir ) :
output = [ ]
for root , _ , filenames in walk ( data_dir ) :
for filename in filenames :
if not filename . startswith ( ' . ' ) :
output . append ( path . join ( root , filename ) )
output = [ path . relpath ( p , path . dirname ( data_dir ) ) for p in output ]
output . append ( ' meta.json ' )
return output
def list_requirements ( meta ) :
parent_package = meta . get ( ' parent_package ' , ' spacy ' )
2020-05-30 13:01:58 +00:00
requirements = [ parent_package + meta [ ' spacy_version ' ] ]
2017-11-07 11:15:35 +00:00
if ' setup_requires ' in meta :
requirements + = meta [ ' setup_requires ' ]
2019-07-27 11:34:57 +00:00
if ' requirements ' in meta :
requirements + = meta [ ' requirements ' ]
2017-11-07 11:15:35 +00:00
return requirements
def setup_package ( ) :
root = path . abspath ( path . dirname ( __file__ ) )
meta_path = path . join ( root , ' meta.json ' )
meta = load_meta ( meta_path )
2021-06-18 13:48:53 +00:00
readme_path = path . join ( root , ' README.md ' )
readme = load_readme ( readme_path )
2017-11-07 11:15:35 +00:00
model_name = str ( meta [ ' lang ' ] + ' _ ' + meta [ ' name ' ] )
model_dir = path . join ( model_name , model_name + ' - ' + meta [ ' version ' ] )
copy ( meta_path , path . join ( model_name ) )
copy ( meta_path , model_dir )
setup (
name = model_name ,
2020-06-27 18:36:08 +00:00
description = meta . get ( ' description ' ) ,
2021-06-18 13:48:53 +00:00
long_description = readme ,
2020-06-27 18:36:08 +00:00
author = meta . get ( ' author ' ) ,
author_email = meta . get ( ' email ' ) ,
url = meta . get ( ' url ' ) ,
2017-11-07 11:15:35 +00:00
version = meta [ ' version ' ] ,
2020-06-27 18:36:08 +00:00
license = meta . get ( ' license ' ) ,
2017-11-07 11:15:35 +00:00
packages = [ model_name ] ,
package_data = { model_name : list_files ( model_dir ) } ,
install_requires = list_requirements ( meta ) ,
zip_safe = False ,
2020-05-22 13:42:46 +00:00
entry_points = { ' spacy_models ' : [ ' {m} = {m} ' . format ( m = model_name ) ] }
2017-11-07 11:15:35 +00:00
)
if __name__ == ' __main__ ' :
setup_package ( )
2021-04-26 14:53:21 +00:00
""" .lstrip()
2017-11-07 11:15:35 +00:00
TEMPLATE_MANIFEST = """
include meta . json
2020-11-30 12:43:58 +00:00
include LICENSE
2021-06-18 13:48:53 +00:00
include LICENSES_SOURCES
include README . md
2017-11-07 11:15:35 +00:00
""" .strip()
TEMPLATE_INIT = """
from pathlib import Path
from spacy . util import load_model_from_init_py , get_model_meta
2020-12-10 12:36:46 +00:00
{ imports }
2017-11-07 11:15:35 +00:00
__version__ = get_model_meta ( Path ( __file__ ) . parent ) [ ' version ' ]
def load ( * * overrides ) :
return load_model_from_init_py ( __file__ , * * overrides )
2021-04-26 14:53:21 +00:00
""" .lstrip()
2021-06-11 08:20:24 +00:00
FILENAMES_DOCS = [ " LICENSE " , " LICENSES_SOURCES " , " README.md " ]