2017-03-12 12:07:28 +00:00
|
|
|
|
# coding: utf8
|
2017-03-15 16:35:57 +00:00
|
|
|
|
from __future__ import unicode_literals, print_function
|
2017-04-15 10:05:47 +00:00
|
|
|
|
|
2017-05-18 09:36:53 +00:00
|
|
|
|
import os
|
2017-09-16 18:27:59 +00:00
|
|
|
|
import pkg_resources
|
2017-05-07 21:24:51 +00:00
|
|
|
|
import importlib
|
2019-02-01 07:05:22 +00:00
|
|
|
|
import re
|
2017-04-15 10:05:47 +00:00
|
|
|
|
from pathlib import Path
|
2017-05-21 14:05:05 +00:00
|
|
|
|
import random
|
2017-05-31 11:42:39 +00:00
|
|
|
|
from collections import OrderedDict
|
2017-09-21 00:16:35 +00:00
|
|
|
|
from thinc.neural._classes.model import Model
|
2018-03-27 17:22:52 +00:00
|
|
|
|
from thinc.neural.ops import NumpyOps
|
2017-10-17 16:20:52 +00:00
|
|
|
|
import functools
|
2017-11-10 18:05:18 +00:00
|
|
|
|
import itertools
|
2018-02-13 11:52:48 +00:00
|
|
|
|
import numpy.random
|
💫 Replace ujson, msgpack and dill/pickle/cloudpickle with srsly (#3003)
Remove hacks and wrappers, keep code in sync across our libraries and move spaCy a few steps closer to only depending on packages with binary wheels 🎉
See here: https://github.com/explosion/srsly
Serialization is hard, especially across Python versions and multiple platforms. After dealing with many subtle bugs over the years (encodings, locales, large files) our libraries like spaCy and Prodigy have steadily grown a number of utility functions to wrap the multiple serialization formats we need to support (especially json, msgpack and pickle). These wrapping functions ended up duplicated across our codebases, so we wanted to put them in one place.
At the same time, we noticed that having a lot of small dependencies was making maintainence harder, and making installation slower. To solve this, we've made srsly standalone, by including the component packages directly within it. This way we can provide all the serialization utilities we need in a single binary wheel.
srsly currently includes forks of the following packages:
ujson
msgpack
msgpack-numpy
cloudpickle
* WIP: replace json/ujson with srsly
* Replace ujson in examples
Use regular json instead of srsly to make code easier to read and follow
* Update requirements
* Fix imports
* Fix typos
* Replace msgpack with srsly
* Fix warning
2018-12-03 00:28:22 +00:00
|
|
|
|
import srsly
|
2019-02-07 08:44:31 +00:00
|
|
|
|
from jsonschema import Draft4Validator
|
|
|
|
|
|
2018-11-30 19:16:14 +00:00
|
|
|
|
|
2018-12-08 11:37:38 +00:00
|
|
|
|
try:
|
|
|
|
|
import cupy.random
|
|
|
|
|
except ImportError:
|
|
|
|
|
cupy = None
|
|
|
|
|
|
2017-10-27 12:39:09 +00:00
|
|
|
|
from .symbols import ORTH
|
2018-12-01 03:55:48 +00:00
|
|
|
|
from .compat import cupy, CudaStream, path2str, basestring_, unicode_
|
💫 Replace ujson, msgpack and dill/pickle/cloudpickle with srsly (#3003)
Remove hacks and wrappers, keep code in sync across our libraries and move spaCy a few steps closer to only depending on packages with binary wheels 🎉
See here: https://github.com/explosion/srsly
Serialization is hard, especially across Python versions and multiple platforms. After dealing with many subtle bugs over the years (encodings, locales, large files) our libraries like spaCy and Prodigy have steadily grown a number of utility functions to wrap the multiple serialization formats we need to support (especially json, msgpack and pickle). These wrapping functions ended up duplicated across our codebases, so we wanted to put them in one place.
At the same time, we noticed that having a lot of small dependencies was making maintainence harder, and making installation slower. To solve this, we've made srsly standalone, by including the component packages directly within it. This way we can provide all the serialization utilities we need in a single binary wheel.
srsly currently includes forks of the following packages:
ujson
msgpack
msgpack-numpy
cloudpickle
* WIP: replace json/ujson with srsly
* Replace ujson in examples
Use regular json instead of srsly to make code easier to read and follow
* Update requirements
* Fix imports
* Fix typos
* Replace msgpack with srsly
* Fix warning
2018-12-03 00:28:22 +00:00
|
|
|
|
from .compat import import_file
|
2019-03-10 18:16:45 +00:00
|
|
|
|
from .errors import Errors, Warnings, deprecation_warning
|
2017-10-27 12:39:09 +00:00
|
|
|
|
|
2017-03-20 21:48:32 +00:00
|
|
|
|
|
2016-03-25 17:54:45 +00:00
|
|
|
|
LANGUAGES = {}
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 16:03:03 +00:00
|
|
|
|
_data_path = Path(__file__).parent / "data"
|
2017-10-27 12:39:09 +00:00
|
|
|
|
_PRINT_ENV = False
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def set_env_log(value):
|
|
|
|
|
global _PRINT_ENV
|
|
|
|
|
_PRINT_ENV = value
|
2016-03-25 17:54:45 +00:00
|
|
|
|
|
|
|
|
|
|
2017-05-13 23:31:10 +00:00
|
|
|
|
def get_lang_class(lang):
|
|
|
|
|
"""Import and load a Language class.
|
2016-03-25 17:54:45 +00:00
|
|
|
|
|
2017-05-13 23:31:10 +00:00
|
|
|
|
lang (unicode): Two-letter language code, e.g. 'en'.
|
|
|
|
|
RETURNS (Language): Language class.
|
|
|
|
|
"""
|
|
|
|
|
global LANGUAGES
|
2018-12-20 22:58:43 +00:00
|
|
|
|
# Check if an entry point is exposed for the language code
|
|
|
|
|
entry_point = get_entry_point("spacy_languages", lang)
|
|
|
|
|
if entry_point is not None:
|
|
|
|
|
LANGUAGES[lang] = entry_point
|
|
|
|
|
return entry_point
|
2017-10-27 12:39:09 +00:00
|
|
|
|
if lang not in LANGUAGES:
|
2017-05-13 23:31:10 +00:00
|
|
|
|
try:
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 16:03:03 +00:00
|
|
|
|
module = importlib.import_module(".lang.%s" % lang, "spacy")
|
2019-02-13 15:52:25 +00:00
|
|
|
|
except ImportError as err:
|
|
|
|
|
raise ImportError(Errors.E048.format(lang=lang, err=err))
|
2017-05-13 23:31:10 +00:00
|
|
|
|
LANGUAGES[lang] = getattr(module, module.__all__[0])
|
2016-03-25 17:54:45 +00:00
|
|
|
|
return LANGUAGES[lang]
|
|
|
|
|
|
|
|
|
|
|
2017-05-13 23:31:10 +00:00
|
|
|
|
def set_lang_class(name, cls):
|
|
|
|
|
"""Set a custom Language class name that can be loaded via get_lang_class.
|
2017-05-13 19:22:49 +00:00
|
|
|
|
|
2017-05-13 23:31:10 +00:00
|
|
|
|
name (unicode): Name of Language class.
|
|
|
|
|
cls (Language): Language class.
|
2017-05-13 19:22:49 +00:00
|
|
|
|
"""
|
2017-05-13 23:31:10 +00:00
|
|
|
|
global LANGUAGES
|
|
|
|
|
LANGUAGES[name] = cls
|
2017-05-08 21:50:45 +00:00
|
|
|
|
|
|
|
|
|
|
2017-01-09 22:40:26 +00:00
|
|
|
|
def get_data_path(require_exists=True):
|
2017-05-13 19:22:49 +00:00
|
|
|
|
"""Get path to spaCy data directory.
|
|
|
|
|
|
2017-05-13 23:30:29 +00:00
|
|
|
|
require_exists (bool): Only return path if it exists, otherwise None.
|
|
|
|
|
RETURNS (Path or None): Data path or None.
|
2017-05-13 19:22:49 +00:00
|
|
|
|
"""
|
2017-01-09 22:40:26 +00:00
|
|
|
|
if not require_exists:
|
|
|
|
|
return _data_path
|
|
|
|
|
else:
|
|
|
|
|
return _data_path if _data_path.exists() else None
|
2016-09-24 18:26:17 +00:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def set_data_path(path):
|
2017-05-13 19:22:49 +00:00
|
|
|
|
"""Set path to spaCy data directory.
|
|
|
|
|
|
2017-05-13 23:30:29 +00:00
|
|
|
|
path (unicode or Path): Path to new data directory.
|
2017-05-13 19:22:49 +00:00
|
|
|
|
"""
|
2016-09-24 18:26:17 +00:00
|
|
|
|
global _data_path
|
2017-04-15 10:11:16 +00:00
|
|
|
|
_data_path = ensure_path(path)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def ensure_path(path):
|
2017-05-13 23:30:29 +00:00
|
|
|
|
"""Ensure string is converted to a Path.
|
|
|
|
|
|
|
|
|
|
path: Anything. If string, it's converted to Path.
|
|
|
|
|
RETURNS: Path or original argument.
|
|
|
|
|
"""
|
2017-04-15 10:11:16 +00:00
|
|
|
|
if isinstance(path, basestring_):
|
|
|
|
|
return Path(path)
|
|
|
|
|
else:
|
|
|
|
|
return path
|
2016-09-24 18:26:17 +00:00
|
|
|
|
|
|
|
|
|
|
2017-05-29 12:10:10 +00:00
|
|
|
|
def load_model(name, **overrides):
|
2017-05-27 22:22:00 +00:00
|
|
|
|
"""Load a model from a shortcut link, package or data path.
|
2017-05-13 19:22:49 +00:00
|
|
|
|
|
2017-05-13 23:30:29 +00:00
|
|
|
|
name (unicode): Package name, shortcut link or model path.
|
2017-05-29 12:10:10 +00:00
|
|
|
|
**overrides: Specific overrides, like pipeline components to disable.
|
2017-05-27 22:22:00 +00:00
|
|
|
|
RETURNS (Language): `Language` class with the loaded model.
|
2017-05-13 19:22:49 +00:00
|
|
|
|
"""
|
2017-05-08 21:51:15 +00:00
|
|
|
|
data_path = get_data_path()
|
|
|
|
|
if not data_path or not data_path.exists():
|
2018-04-03 13:50:31 +00:00
|
|
|
|
raise IOError(Errors.E049.format(path=path2str(data_path)))
|
2017-10-27 12:39:09 +00:00
|
|
|
|
if isinstance(name, basestring_): # in data dir / shortcut
|
|
|
|
|
if name in set([d.name for d in data_path.iterdir()]):
|
2017-06-05 11:02:31 +00:00
|
|
|
|
return load_model_from_link(name, **overrides)
|
2017-10-27 12:39:09 +00:00
|
|
|
|
if is_package(name): # installed as package
|
2017-06-05 11:02:31 +00:00
|
|
|
|
return load_model_from_package(name, **overrides)
|
2017-10-27 12:39:09 +00:00
|
|
|
|
if Path(name).exists(): # path to model data directory
|
2017-06-05 11:02:31 +00:00
|
|
|
|
return load_model_from_path(Path(name), **overrides)
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 16:03:03 +00:00
|
|
|
|
elif hasattr(name, "exists"): # Path or Path-like to model data
|
2017-06-05 11:02:31 +00:00
|
|
|
|
return load_model_from_path(name, **overrides)
|
2018-04-03 13:50:31 +00:00
|
|
|
|
raise IOError(Errors.E050.format(name=name))
|
2017-05-08 21:51:15 +00:00
|
|
|
|
|
|
|
|
|
|
2017-06-05 11:02:31 +00:00
|
|
|
|
def load_model_from_link(name, **overrides):
|
|
|
|
|
"""Load a model from a shortcut link, or directory in spaCy data path."""
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 16:03:03 +00:00
|
|
|
|
path = get_data_path() / name / "__init__.py"
|
2017-06-05 11:02:31 +00:00
|
|
|
|
try:
|
2017-08-18 19:57:06 +00:00
|
|
|
|
cls = import_file(name, path)
|
2017-06-05 11:02:31 +00:00
|
|
|
|
except AttributeError:
|
2018-04-03 13:50:31 +00:00
|
|
|
|
raise IOError(Errors.E051.format(name=name))
|
2017-06-05 11:02:31 +00:00
|
|
|
|
return cls.load(**overrides)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def load_model_from_package(name, **overrides):
|
|
|
|
|
"""Load a model from an installed package."""
|
|
|
|
|
cls = importlib.import_module(name)
|
|
|
|
|
return cls.load(**overrides)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def load_model_from_path(model_path, meta=False, **overrides):
|
|
|
|
|
"""Load a model from a data directory path. Creates Language class with
|
|
|
|
|
pipeline from meta.json and then calls from_disk() with path."""
|
|
|
|
|
if not meta:
|
|
|
|
|
meta = get_model_meta(model_path)
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 16:03:03 +00:00
|
|
|
|
cls = get_lang_class(meta["lang"])
|
2017-10-06 22:25:54 +00:00
|
|
|
|
nlp = cls(meta=meta, **overrides)
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 16:03:03 +00:00
|
|
|
|
pipeline = meta.get("pipeline", [])
|
|
|
|
|
disable = overrides.get("disable", [])
|
2017-10-06 22:29:08 +00:00
|
|
|
|
if pipeline is True:
|
|
|
|
|
pipeline = nlp.Defaults.pipe_names
|
|
|
|
|
elif pipeline in (False, None):
|
|
|
|
|
pipeline = []
|
|
|
|
|
for name in pipeline:
|
|
|
|
|
if name not in disable:
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 16:03:03 +00:00
|
|
|
|
config = meta.get("pipeline_args", {}).get(name, {})
|
2017-10-06 22:29:08 +00:00
|
|
|
|
component = nlp.create_pipe(name, config=config)
|
|
|
|
|
nlp.add_pipe(component, name=name)
|
2017-06-05 11:02:31 +00:00
|
|
|
|
return nlp.from_disk(model_path)
|
|
|
|
|
|
|
|
|
|
|
2017-05-29 12:10:10 +00:00
|
|
|
|
def load_model_from_init_py(init_file, **overrides):
|
2017-05-27 22:22:00 +00:00
|
|
|
|
"""Helper function to use in the `load()` method of a model package's
|
|
|
|
|
__init__.py.
|
|
|
|
|
|
|
|
|
|
init_file (unicode): Path to model's __init__.py, i.e. `__file__`.
|
2017-05-29 12:10:10 +00:00
|
|
|
|
**overrides: Specific overrides, like pipeline components to disable.
|
2017-05-27 22:22:00 +00:00
|
|
|
|
RETURNS (Language): `Language` class with loaded model.
|
|
|
|
|
"""
|
|
|
|
|
model_path = Path(init_file).parent
|
2017-05-29 12:10:10 +00:00
|
|
|
|
meta = get_model_meta(model_path)
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 16:03:03 +00:00
|
|
|
|
data_dir = "%s_%s-%s" % (meta["lang"], meta["name"], meta["version"])
|
2017-05-29 12:10:10 +00:00
|
|
|
|
data_path = model_path / data_dir
|
|
|
|
|
if not model_path.exists():
|
2018-04-03 13:50:31 +00:00
|
|
|
|
raise IOError(Errors.E052.format(path=path2str(data_path)))
|
2017-06-05 11:02:31 +00:00
|
|
|
|
return load_model_from_path(data_path, meta, **overrides)
|
2017-05-27 22:22:00 +00:00
|
|
|
|
|
|
|
|
|
|
2017-05-29 12:10:10 +00:00
|
|
|
|
def get_model_meta(path):
|
|
|
|
|
"""Get model meta.json from a directory path and validate its contents.
|
2017-05-27 22:22:00 +00:00
|
|
|
|
|
2017-05-29 12:10:10 +00:00
|
|
|
|
path (unicode or Path): Path to model directory.
|
|
|
|
|
RETURNS (dict): The model's meta data.
|
2017-05-27 22:22:00 +00:00
|
|
|
|
"""
|
2017-05-29 12:10:10 +00:00
|
|
|
|
model_path = ensure_path(path)
|
|
|
|
|
if not model_path.exists():
|
2018-04-03 13:50:31 +00:00
|
|
|
|
raise IOError(Errors.E052.format(path=path2str(model_path)))
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 16:03:03 +00:00
|
|
|
|
meta_path = model_path / "meta.json"
|
2017-05-27 22:22:00 +00:00
|
|
|
|
if not meta_path.is_file():
|
2018-04-03 13:50:31 +00:00
|
|
|
|
raise IOError(Errors.E053.format(path=meta_path))
|
💫 Replace ujson, msgpack and dill/pickle/cloudpickle with srsly (#3003)
Remove hacks and wrappers, keep code in sync across our libraries and move spaCy a few steps closer to only depending on packages with binary wheels 🎉
See here: https://github.com/explosion/srsly
Serialization is hard, especially across Python versions and multiple platforms. After dealing with many subtle bugs over the years (encodings, locales, large files) our libraries like spaCy and Prodigy have steadily grown a number of utility functions to wrap the multiple serialization formats we need to support (especially json, msgpack and pickle). These wrapping functions ended up duplicated across our codebases, so we wanted to put them in one place.
At the same time, we noticed that having a lot of small dependencies was making maintainence harder, and making installation slower. To solve this, we've made srsly standalone, by including the component packages directly within it. This way we can provide all the serialization utilities we need in a single binary wheel.
srsly currently includes forks of the following packages:
ujson
msgpack
msgpack-numpy
cloudpickle
* WIP: replace json/ujson with srsly
* Replace ujson in examples
Use regular json instead of srsly to make code easier to read and follow
* Update requirements
* Fix imports
* Fix typos
* Replace msgpack with srsly
* Fix warning
2018-12-03 00:28:22 +00:00
|
|
|
|
meta = srsly.read_json(meta_path)
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 16:03:03 +00:00
|
|
|
|
for setting in ["lang", "name", "version"]:
|
2017-08-29 09:21:44 +00:00
|
|
|
|
if setting not in meta or not meta[setting]:
|
2018-04-03 13:50:31 +00:00
|
|
|
|
raise ValueError(Errors.E054.format(setting=setting))
|
2017-05-29 12:10:10 +00:00
|
|
|
|
return meta
|
2017-05-27 22:22:00 +00:00
|
|
|
|
|
|
|
|
|
|
2017-05-13 19:22:49 +00:00
|
|
|
|
def is_package(name):
|
|
|
|
|
"""Check if string maps to a package installed via pip.
|
|
|
|
|
|
2017-05-13 23:30:29 +00:00
|
|
|
|
name (unicode): Name of package.
|
|
|
|
|
RETURNS (bool): True if installed package, False if not.
|
2017-05-08 21:51:15 +00:00
|
|
|
|
"""
|
2017-09-29 18:55:17 +00:00
|
|
|
|
name = name.lower() # compare package name against lowercase name
|
2017-09-16 18:27:59 +00:00
|
|
|
|
packages = pkg_resources.working_set.by_key.keys()
|
2017-05-08 21:51:15 +00:00
|
|
|
|
for package in packages:
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 16:03:03 +00:00
|
|
|
|
if package.lower().replace("-", "_") == name:
|
2017-05-08 21:51:15 +00:00
|
|
|
|
return True
|
|
|
|
|
return False
|
|
|
|
|
|
|
|
|
|
|
2017-05-27 22:22:00 +00:00
|
|
|
|
def get_package_path(name):
|
|
|
|
|
"""Get the path to an installed package.
|
2017-05-13 19:22:49 +00:00
|
|
|
|
|
2017-05-27 22:22:00 +00:00
|
|
|
|
name (unicode): Package name.
|
|
|
|
|
RETURNS (Path): Path to installed package.
|
2017-05-13 19:22:49 +00:00
|
|
|
|
"""
|
2017-09-29 18:55:17 +00:00
|
|
|
|
name = name.lower() # use lowercase version to be safe
|
2017-05-08 21:51:15 +00:00
|
|
|
|
# Here we're importing the module just to find it. This is worryingly
|
|
|
|
|
# indirect, but it's otherwise very difficult to find the package.
|
2017-05-29 08:51:19 +00:00
|
|
|
|
pkg = importlib.import_module(name)
|
2017-05-27 22:22:00 +00:00
|
|
|
|
return Path(pkg.__file__).parent
|
2017-05-08 21:51:15 +00:00
|
|
|
|
|
|
|
|
|
|
2018-05-22 16:29:45 +00:00
|
|
|
|
def get_entry_points(key):
|
|
|
|
|
"""Get registered entry points from other packages for a given key, e.g.
|
|
|
|
|
'spacy_factories' and return them as a dictionary, keyed by name.
|
|
|
|
|
|
|
|
|
|
key (unicode): Entry point name.
|
|
|
|
|
RETURNS (dict): Entry points, keyed by name.
|
|
|
|
|
"""
|
|
|
|
|
result = {}
|
|
|
|
|
for entry_point in pkg_resources.iter_entry_points(key):
|
|
|
|
|
result[entry_point.name] = entry_point.load()
|
|
|
|
|
return result
|
|
|
|
|
|
|
|
|
|
|
2018-12-20 22:58:43 +00:00
|
|
|
|
def get_entry_point(key, value):
|
|
|
|
|
"""Check if registered entry point is available for a given name and
|
|
|
|
|
load it. Otherwise, return None.
|
|
|
|
|
|
|
|
|
|
key (unicode): Entry point name.
|
|
|
|
|
value (unicode): Name of entry point to load.
|
|
|
|
|
RETURNS: The loaded entry point or None.
|
|
|
|
|
"""
|
|
|
|
|
for entry_point in pkg_resources.iter_entry_points(key):
|
|
|
|
|
if entry_point.name == value:
|
|
|
|
|
return entry_point.load()
|
|
|
|
|
|
|
|
|
|
|
2017-05-18 12:13:14 +00:00
|
|
|
|
def is_in_jupyter():
|
2017-05-20 23:12:09 +00:00
|
|
|
|
"""Check if user is running spaCy from a Jupyter notebook by detecting the
|
|
|
|
|
IPython kernel. Mainly used for the displaCy visualizer.
|
2017-05-18 12:13:14 +00:00
|
|
|
|
RETURNS (bool): True if in Jupyter, False if not.
|
|
|
|
|
"""
|
2018-12-20 16:32:04 +00:00
|
|
|
|
# https://stackoverflow.com/a/39662359/6400719
|
2017-05-18 12:13:14 +00:00
|
|
|
|
try:
|
2018-12-20 16:32:04 +00:00
|
|
|
|
shell = get_ipython().__class__.__name__
|
|
|
|
|
if shell == "ZMQInteractiveShell":
|
|
|
|
|
return True # Jupyter notebook or qtconsole
|
2017-05-18 12:13:14 +00:00
|
|
|
|
except NameError:
|
2018-12-20 16:32:04 +00:00
|
|
|
|
return False # Probably standard Python interpreter
|
2017-05-18 12:13:14 +00:00
|
|
|
|
return False
|
|
|
|
|
|
|
|
|
|
|
2017-05-13 22:37:53 +00:00
|
|
|
|
def get_cuda_stream(require=False):
|
2018-03-27 17:22:52 +00:00
|
|
|
|
if CudaStream is None:
|
|
|
|
|
return None
|
|
|
|
|
elif isinstance(Model.ops, NumpyOps):
|
|
|
|
|
return None
|
|
|
|
|
else:
|
|
|
|
|
return CudaStream()
|
2017-05-15 19:46:08 +00:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def get_async(stream, numpy_array):
|
|
|
|
|
if cupy is None:
|
|
|
|
|
return numpy_array
|
|
|
|
|
else:
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 16:03:03 +00:00
|
|
|
|
array = cupy.ndarray(numpy_array.shape, order="C", dtype=numpy_array.dtype)
|
2017-05-18 09:36:53 +00:00
|
|
|
|
array.set(numpy_array, stream=stream)
|
|
|
|
|
return array
|
|
|
|
|
|
2017-05-26 10:37:45 +00:00
|
|
|
|
|
2017-05-18 09:36:53 +00:00
|
|
|
|
def env_opt(name, default=None):
|
2017-05-18 13:32:03 +00:00
|
|
|
|
if type(default) is float:
|
|
|
|
|
type_convert = float
|
2017-05-18 09:36:53 +00:00
|
|
|
|
else:
|
2017-05-18 13:32:03 +00:00
|
|
|
|
type_convert = int
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 16:03:03 +00:00
|
|
|
|
if "SPACY_" + name.upper() in os.environ:
|
|
|
|
|
value = type_convert(os.environ["SPACY_" + name.upper()])
|
2017-05-31 12:14:11 +00:00
|
|
|
|
if _PRINT_ENV:
|
|
|
|
|
print(name, "=", repr(value), "via", "$SPACY_" + name.upper())
|
2017-05-18 13:32:03 +00:00
|
|
|
|
return value
|
|
|
|
|
elif name in os.environ:
|
|
|
|
|
value = type_convert(os.environ[name])
|
2017-05-31 12:14:11 +00:00
|
|
|
|
if _PRINT_ENV:
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 16:03:03 +00:00
|
|
|
|
print(name, "=", repr(value), "via", "$" + name)
|
2017-05-18 13:32:03 +00:00
|
|
|
|
return value
|
|
|
|
|
else:
|
2017-05-31 12:14:11 +00:00
|
|
|
|
if _PRINT_ENV:
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 16:03:03 +00:00
|
|
|
|
print(name, "=", repr(default), "by default")
|
2017-05-18 09:36:53 +00:00
|
|
|
|
return default
|
2017-05-13 22:37:53 +00:00
|
|
|
|
|
|
|
|
|
|
2016-09-24 18:26:17 +00:00
|
|
|
|
def read_regex(path):
|
2017-04-15 10:11:16 +00:00
|
|
|
|
path = ensure_path(path)
|
2016-09-24 18:26:17 +00:00
|
|
|
|
with path.open() as file_:
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 16:03:03 +00:00
|
|
|
|
entries = file_.read().split("\n")
|
|
|
|
|
expression = "|".join(
|
|
|
|
|
["^" + re.escape(piece) for piece in entries if piece.strip()]
|
|
|
|
|
)
|
2016-09-24 18:26:17 +00:00
|
|
|
|
return re.compile(expression)
|
|
|
|
|
|
|
|
|
|
|
2016-09-25 12:49:53 +00:00
|
|
|
|
def compile_prefix_regex(entries):
|
2019-02-24 17:39:59 +00:00
|
|
|
|
"""Compile a sequence of prefix rules into a regex object.
|
2019-02-24 17:34:10 +00:00
|
|
|
|
|
|
|
|
|
entries (tuple): The prefix rules, e.g. spacy.lang.punctuation.TOKENIZER_PREFIXES.
|
|
|
|
|
RETURNS (regex object): The regex object. to be used for Tokenizer.prefix_search.
|
|
|
|
|
"""
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 16:03:03 +00:00
|
|
|
|
if "(" in entries:
|
2017-01-08 19:33:28 +00:00
|
|
|
|
# Handle deprecated data
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 16:03:03 +00:00
|
|
|
|
expression = "|".join(
|
|
|
|
|
["^" + re.escape(piece) for piece in entries if piece.strip()]
|
|
|
|
|
)
|
2017-01-08 19:33:28 +00:00
|
|
|
|
return re.compile(expression)
|
|
|
|
|
else:
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 16:03:03 +00:00
|
|
|
|
expression = "|".join(["^" + piece for piece in entries if piece.strip()])
|
2017-01-08 19:33:28 +00:00
|
|
|
|
return re.compile(expression)
|
2016-09-24 18:26:17 +00:00
|
|
|
|
|
|
|
|
|
|
2016-09-25 12:49:53 +00:00
|
|
|
|
def compile_suffix_regex(entries):
|
2019-02-24 17:39:59 +00:00
|
|
|
|
"""Compile a sequence of suffix rules into a regex object.
|
2019-02-24 17:34:10 +00:00
|
|
|
|
|
|
|
|
|
entries (tuple): The suffix rules, e.g. spacy.lang.punctuation.TOKENIZER_SUFFIXES.
|
|
|
|
|
RETURNS (regex object): The regex object. to be used for Tokenizer.suffix_search.
|
|
|
|
|
"""
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 16:03:03 +00:00
|
|
|
|
expression = "|".join([piece + "$" for piece in entries if piece.strip()])
|
2016-09-24 18:26:17 +00:00
|
|
|
|
return re.compile(expression)
|
|
|
|
|
|
|
|
|
|
|
2016-09-25 12:49:53 +00:00
|
|
|
|
def compile_infix_regex(entries):
|
2019-02-24 17:39:59 +00:00
|
|
|
|
"""Compile a sequence of infix rules into a regex object.
|
2019-02-24 17:34:10 +00:00
|
|
|
|
|
|
|
|
|
entries (tuple): The infix rules, e.g. spacy.lang.punctuation.TOKENIZER_INFIXES.
|
|
|
|
|
RETURNS (regex object): The regex object. to be used for Tokenizer.infix_finditer.
|
|
|
|
|
"""
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 16:03:03 +00:00
|
|
|
|
expression = "|".join([piece for piece in entries if piece.strip()])
|
2016-09-24 18:26:17 +00:00
|
|
|
|
return re.compile(expression)
|
|
|
|
|
|
|
|
|
|
|
2017-06-03 17:44:47 +00:00
|
|
|
|
def add_lookups(default_func, *lookups):
|
|
|
|
|
"""Extend an attribute function with special cases. If a word is in the
|
|
|
|
|
lookups, the value is returned. Otherwise the previous function is used.
|
|
|
|
|
|
|
|
|
|
default_func (callable): The default function to execute.
|
|
|
|
|
*lookups (dict): Lookup dictionary mapping string to attribute value.
|
|
|
|
|
RETURNS (callable): Lexical attribute getter.
|
|
|
|
|
"""
|
2017-10-17 16:20:52 +00:00
|
|
|
|
# This is implemented as functools.partial instead of a closure, to allow
|
|
|
|
|
# pickle to work.
|
|
|
|
|
return functools.partial(_get_attr_unless_lookup, default_func, lookups)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _get_attr_unless_lookup(default_func, lookups, string):
|
|
|
|
|
for lookup in lookups:
|
|
|
|
|
if string in lookup:
|
|
|
|
|
return lookup[string]
|
|
|
|
|
return default_func(string)
|
2017-06-03 17:44:47 +00:00
|
|
|
|
|
|
|
|
|
|
2017-05-08 13:42:12 +00:00
|
|
|
|
def update_exc(base_exceptions, *addition_dicts):
|
2017-05-13 19:22:49 +00:00
|
|
|
|
"""Update and validate tokenizer exceptions. Will overwrite exceptions.
|
|
|
|
|
|
2017-05-13 23:30:29 +00:00
|
|
|
|
base_exceptions (dict): Base exceptions.
|
|
|
|
|
*addition_dicts (dict): Exceptions to add to the base dict, in order.
|
|
|
|
|
RETURNS (dict): Combined tokenizer exceptions.
|
2017-05-13 19:22:49 +00:00
|
|
|
|
"""
|
2017-05-08 13:42:12 +00:00
|
|
|
|
exc = dict(base_exceptions)
|
|
|
|
|
for additions in addition_dicts:
|
|
|
|
|
for orth, token_attrs in additions.items():
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 16:03:03 +00:00
|
|
|
|
if not all(isinstance(attr[ORTH], unicode_) for attr in token_attrs):
|
2018-04-03 13:50:31 +00:00
|
|
|
|
raise ValueError(Errors.E055.format(key=orth, orths=token_attrs))
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 16:03:03 +00:00
|
|
|
|
described_orth = "".join(attr[ORTH] for attr in token_attrs)
|
2017-05-08 13:42:12 +00:00
|
|
|
|
if orth != described_orth:
|
2018-04-03 13:50:31 +00:00
|
|
|
|
raise ValueError(Errors.E056.format(key=orth, orths=described_orth))
|
2017-05-08 13:42:12 +00:00
|
|
|
|
exc.update(additions)
|
2017-05-13 19:22:25 +00:00
|
|
|
|
exc = expand_exc(exc, "'", "’")
|
2017-05-08 13:42:12 +00:00
|
|
|
|
return exc
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def expand_exc(excs, search, replace):
|
2017-05-13 19:22:49 +00:00
|
|
|
|
"""Find string in tokenizer exceptions, duplicate entry and replace string.
|
|
|
|
|
For example, to add additional versions with typographic apostrophes.
|
|
|
|
|
|
2017-05-13 23:30:29 +00:00
|
|
|
|
excs (dict): Tokenizer exceptions.
|
|
|
|
|
search (unicode): String to find and replace.
|
|
|
|
|
replace (unicode): Replacement.
|
|
|
|
|
RETURNS (dict): Combined tokenizer exceptions.
|
2017-05-13 19:22:49 +00:00
|
|
|
|
"""
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 16:03:03 +00:00
|
|
|
|
|
2017-05-08 13:42:12 +00:00
|
|
|
|
def _fix_token(token, search, replace):
|
|
|
|
|
fixed = dict(token)
|
|
|
|
|
fixed[ORTH] = fixed[ORTH].replace(search, replace)
|
|
|
|
|
return fixed
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 16:03:03 +00:00
|
|
|
|
|
2017-05-13 19:22:25 +00:00
|
|
|
|
new_excs = dict(excs)
|
2017-05-08 13:42:12 +00:00
|
|
|
|
for token_string, tokens in excs.items():
|
|
|
|
|
if search in token_string:
|
|
|
|
|
new_key = token_string.replace(search, replace)
|
|
|
|
|
new_value = [_fix_token(t, search, replace) for t in tokens]
|
2017-05-13 19:22:25 +00:00
|
|
|
|
new_excs[new_key] = new_value
|
|
|
|
|
return new_excs
|
2017-05-08 13:42:12 +00:00
|
|
|
|
|
|
|
|
|
|
2015-10-07 08:25:35 +00:00
|
|
|
|
def normalize_slice(length, start, stop, step=None):
|
|
|
|
|
if not (step is None or step == 1):
|
2018-04-03 13:50:31 +00:00
|
|
|
|
raise ValueError(Errors.E057)
|
2015-10-07 08:25:35 +00:00
|
|
|
|
if start is None:
|
2017-10-27 12:39:09 +00:00
|
|
|
|
start = 0
|
2015-10-07 08:25:35 +00:00
|
|
|
|
elif start < 0:
|
2017-10-27 12:39:09 +00:00
|
|
|
|
start += length
|
2015-10-07 08:25:35 +00:00
|
|
|
|
start = min(length, max(0, start))
|
|
|
|
|
if stop is None:
|
2017-10-27 12:39:09 +00:00
|
|
|
|
stop = length
|
2015-10-07 08:25:35 +00:00
|
|
|
|
elif stop < 0:
|
2017-10-27 12:39:09 +00:00
|
|
|
|
stop += length
|
2015-10-07 08:25:35 +00:00
|
|
|
|
stop = min(length, max(start, stop))
|
|
|
|
|
return start, stop
|
|
|
|
|
|
|
|
|
|
|
2017-11-06 22:45:36 +00:00
|
|
|
|
def minibatch(items, size=8):
|
|
|
|
|
"""Iterate over batches of items. `size` may be an iterator,
|
|
|
|
|
so that batch-size can vary on each step.
|
|
|
|
|
"""
|
|
|
|
|
if isinstance(size, int):
|
2017-11-06 23:22:43 +00:00
|
|
|
|
size_ = itertools.repeat(size)
|
2017-11-06 22:45:36 +00:00
|
|
|
|
else:
|
|
|
|
|
size_ = size
|
|
|
|
|
items = iter(items)
|
|
|
|
|
while True:
|
|
|
|
|
batch_size = next(size_)
|
2018-12-03 01:19:12 +00:00
|
|
|
|
batch = list(itertools.islice(items, int(batch_size)))
|
2017-11-06 22:45:36 +00:00
|
|
|
|
if len(batch) == 0:
|
|
|
|
|
break
|
|
|
|
|
yield list(batch)
|
|
|
|
|
|
|
|
|
|
|
2017-05-25 21:16:10 +00:00
|
|
|
|
def compounding(start, stop, compound):
|
2017-05-27 22:04:04 +00:00
|
|
|
|
"""Yield an infinite series of compounding values. Each time the
|
2017-05-25 21:16:10 +00:00
|
|
|
|
generator is called, a value is produced by multiplying the previous
|
|
|
|
|
value by the compound rate.
|
|
|
|
|
|
2017-05-27 22:04:04 +00:00
|
|
|
|
EXAMPLE:
|
2017-05-25 21:16:10 +00:00
|
|
|
|
>>> sizes = compounding(1., 10., 1.5)
|
|
|
|
|
>>> assert next(sizes) == 1.
|
|
|
|
|
>>> assert next(sizes) == 1 * 1.5
|
|
|
|
|
>>> assert next(sizes) == 1.5 * 1.5
|
2017-05-27 22:04:04 +00:00
|
|
|
|
"""
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 16:03:03 +00:00
|
|
|
|
|
2017-05-25 21:16:10 +00:00
|
|
|
|
def clip(value):
|
2017-10-27 12:39:09 +00:00
|
|
|
|
return max(value, stop) if (start > stop) else min(value, stop)
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 16:03:03 +00:00
|
|
|
|
|
2017-05-25 21:16:10 +00:00
|
|
|
|
curr = float(start)
|
|
|
|
|
while True:
|
|
|
|
|
yield clip(curr)
|
|
|
|
|
curr *= compound
|
|
|
|
|
|
|
|
|
|
|
2018-09-14 16:37:16 +00:00
|
|
|
|
def stepping(start, stop, steps):
|
|
|
|
|
"""Yield an infinite series of values that step from a start value to a
|
|
|
|
|
final value over some number of steps. Each step is (stop-start)/steps.
|
|
|
|
|
|
|
|
|
|
After the final value is reached, the generator continues yielding that
|
|
|
|
|
value.
|
|
|
|
|
|
|
|
|
|
EXAMPLE:
|
|
|
|
|
>>> sizes = stepping(1., 200., 100)
|
|
|
|
|
>>> assert next(sizes) == 1.
|
|
|
|
|
>>> assert next(sizes) == 1 * (200.-1.) / 100
|
|
|
|
|
>>> assert next(sizes) == 1 + (200.-1.) / 100 + (200.-1.) / 100
|
|
|
|
|
"""
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 16:03:03 +00:00
|
|
|
|
|
2018-09-14 16:37:16 +00:00
|
|
|
|
def clip(value):
|
|
|
|
|
return max(value, stop) if (start > stop) else min(value, stop)
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 16:03:03 +00:00
|
|
|
|
|
2018-09-14 16:37:16 +00:00
|
|
|
|
curr = float(start)
|
|
|
|
|
while True:
|
|
|
|
|
yield clip(curr)
|
|
|
|
|
curr += (stop - start) / steps
|
|
|
|
|
|
|
|
|
|
|
2017-05-25 21:16:10 +00:00
|
|
|
|
def decaying(start, stop, decay):
|
2017-05-27 22:04:04 +00:00
|
|
|
|
"""Yield an infinite series of linearly decaying values."""
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 16:03:03 +00:00
|
|
|
|
|
2017-05-25 21:16:10 +00:00
|
|
|
|
def clip(value):
|
2017-10-27 12:39:09 +00:00
|
|
|
|
return max(value, stop) if (start > stop) else min(value, stop)
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 16:03:03 +00:00
|
|
|
|
|
|
|
|
|
nr_upd = 1.0
|
2017-05-25 21:16:10 +00:00
|
|
|
|
while True:
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 16:03:03 +00:00
|
|
|
|
yield clip(start * 1.0 / (1.0 + decay * nr_upd))
|
2017-05-25 21:16:10 +00:00
|
|
|
|
nr_upd += 1
|
|
|
|
|
|
|
|
|
|
|
2018-11-15 22:44:07 +00:00
|
|
|
|
def minibatch_by_words(items, size, tuples=True, count_words=len):
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 16:03:03 +00:00
|
|
|
|
"""Create minibatches of a given number of words."""
|
2018-03-27 17:23:02 +00:00
|
|
|
|
if isinstance(size, int):
|
|
|
|
|
size_ = itertools.repeat(size)
|
|
|
|
|
else:
|
|
|
|
|
size_ = size
|
|
|
|
|
items = iter(items)
|
|
|
|
|
while True:
|
|
|
|
|
batch_size = next(size_)
|
|
|
|
|
batch = []
|
|
|
|
|
while batch_size >= 0:
|
|
|
|
|
try:
|
2018-11-15 22:44:07 +00:00
|
|
|
|
if tuples:
|
|
|
|
|
doc, gold = next(items)
|
|
|
|
|
else:
|
|
|
|
|
doc = next(items)
|
2018-03-27 17:23:02 +00:00
|
|
|
|
except StopIteration:
|
|
|
|
|
if batch:
|
|
|
|
|
yield batch
|
|
|
|
|
return
|
|
|
|
|
batch_size -= count_words(doc)
|
2018-11-15 22:44:07 +00:00
|
|
|
|
if tuples:
|
|
|
|
|
batch.append((doc, gold))
|
|
|
|
|
else:
|
|
|
|
|
batch.append(doc)
|
2018-03-27 17:23:02 +00:00
|
|
|
|
if batch:
|
|
|
|
|
yield batch
|
|
|
|
|
|
|
|
|
|
|
2017-11-06 22:45:36 +00:00
|
|
|
|
def itershuffle(iterable, bufsize=1000):
|
|
|
|
|
"""Shuffle an iterator. This works by holding `bufsize` items back
|
|
|
|
|
and yielding them sometime later. Obviously, this is not unbiased –
|
|
|
|
|
but should be good enough for batching. Larger bufsize means less bias.
|
|
|
|
|
From https://gist.github.com/andres-erbsen/1307752
|
|
|
|
|
|
|
|
|
|
iterable (iterable): Iterator to shuffle.
|
|
|
|
|
bufsize (int): Items to hold back.
|
|
|
|
|
YIELDS (iterable): The shuffled iterator.
|
|
|
|
|
"""
|
|
|
|
|
iterable = iter(iterable)
|
|
|
|
|
buf = []
|
|
|
|
|
try:
|
|
|
|
|
while True:
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 16:03:03 +00:00
|
|
|
|
for i in range(random.randint(1, bufsize - len(buf))):
|
2018-03-27 17:23:02 +00:00
|
|
|
|
buf.append(next(iterable))
|
2017-11-06 22:45:36 +00:00
|
|
|
|
random.shuffle(buf)
|
|
|
|
|
for i in range(random.randint(1, bufsize)):
|
|
|
|
|
if buf:
|
|
|
|
|
yield buf.pop()
|
|
|
|
|
else:
|
|
|
|
|
break
|
|
|
|
|
except StopIteration:
|
|
|
|
|
random.shuffle(buf)
|
|
|
|
|
while buf:
|
|
|
|
|
yield buf.pop()
|
|
|
|
|
raise StopIteration
|
|
|
|
|
|
|
|
|
|
|
2017-05-29 08:13:42 +00:00
|
|
|
|
def to_bytes(getters, exclude):
|
2017-05-31 11:42:39 +00:00
|
|
|
|
serialized = OrderedDict()
|
2017-05-29 08:13:42 +00:00
|
|
|
|
for key, getter in getters.items():
|
2019-03-10 18:16:45 +00:00
|
|
|
|
# Split to support file names like meta.json
|
|
|
|
|
if key.split(".")[0] not in exclude:
|
2017-05-29 08:13:42 +00:00
|
|
|
|
serialized[key] = getter()
|
💫 Replace ujson, msgpack and dill/pickle/cloudpickle with srsly (#3003)
Remove hacks and wrappers, keep code in sync across our libraries and move spaCy a few steps closer to only depending on packages with binary wheels 🎉
See here: https://github.com/explosion/srsly
Serialization is hard, especially across Python versions and multiple platforms. After dealing with many subtle bugs over the years (encodings, locales, large files) our libraries like spaCy and Prodigy have steadily grown a number of utility functions to wrap the multiple serialization formats we need to support (especially json, msgpack and pickle). These wrapping functions ended up duplicated across our codebases, so we wanted to put them in one place.
At the same time, we noticed that having a lot of small dependencies was making maintainence harder, and making installation slower. To solve this, we've made srsly standalone, by including the component packages directly within it. This way we can provide all the serialization utilities we need in a single binary wheel.
srsly currently includes forks of the following packages:
ujson
msgpack
msgpack-numpy
cloudpickle
* WIP: replace json/ujson with srsly
* Replace ujson in examples
Use regular json instead of srsly to make code easier to read and follow
* Update requirements
* Fix imports
* Fix typos
* Replace msgpack with srsly
* Fix warning
2018-12-03 00:28:22 +00:00
|
|
|
|
return srsly.msgpack_dumps(serialized)
|
2017-05-29 08:13:42 +00:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def from_bytes(bytes_data, setters, exclude):
|
💫 Replace ujson, msgpack and dill/pickle/cloudpickle with srsly (#3003)
Remove hacks and wrappers, keep code in sync across our libraries and move spaCy a few steps closer to only depending on packages with binary wheels 🎉
See here: https://github.com/explosion/srsly
Serialization is hard, especially across Python versions and multiple platforms. After dealing with many subtle bugs over the years (encodings, locales, large files) our libraries like spaCy and Prodigy have steadily grown a number of utility functions to wrap the multiple serialization formats we need to support (especially json, msgpack and pickle). These wrapping functions ended up duplicated across our codebases, so we wanted to put them in one place.
At the same time, we noticed that having a lot of small dependencies was making maintainence harder, and making installation slower. To solve this, we've made srsly standalone, by including the component packages directly within it. This way we can provide all the serialization utilities we need in a single binary wheel.
srsly currently includes forks of the following packages:
ujson
msgpack
msgpack-numpy
cloudpickle
* WIP: replace json/ujson with srsly
* Replace ujson in examples
Use regular json instead of srsly to make code easier to read and follow
* Update requirements
* Fix imports
* Fix typos
* Replace msgpack with srsly
* Fix warning
2018-12-03 00:28:22 +00:00
|
|
|
|
msg = srsly.msgpack_loads(bytes_data)
|
2017-05-29 08:13:42 +00:00
|
|
|
|
for key, setter in setters.items():
|
2019-03-10 18:16:45 +00:00
|
|
|
|
# Split to support file names like meta.json
|
|
|
|
|
if key.split(".")[0] not in exclude and key in msg:
|
2017-05-29 08:13:42 +00:00
|
|
|
|
setter(msg[key])
|
|
|
|
|
return msg
|
|
|
|
|
|
|
|
|
|
|
2017-05-31 11:42:39 +00:00
|
|
|
|
def to_disk(path, writers, exclude):
|
|
|
|
|
path = ensure_path(path)
|
|
|
|
|
if not path.exists():
|
|
|
|
|
path.mkdir()
|
|
|
|
|
for key, writer in writers.items():
|
2019-03-10 18:16:45 +00:00
|
|
|
|
# Split to support file names like meta.json
|
|
|
|
|
if key.split(".")[0] not in exclude:
|
2017-05-31 11:42:39 +00:00
|
|
|
|
writer(path / key)
|
|
|
|
|
return path
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def from_disk(path, readers, exclude):
|
|
|
|
|
path = ensure_path(path)
|
|
|
|
|
for key, reader in readers.items():
|
2019-03-10 18:16:45 +00:00
|
|
|
|
# Split to support file names like meta.json
|
|
|
|
|
if key.split(".")[0] not in exclude:
|
2017-10-16 18:55:00 +00:00
|
|
|
|
reader(path / key)
|
2017-05-31 11:42:39 +00:00
|
|
|
|
return path
|
|
|
|
|
|
|
|
|
|
|
2017-05-14 15:50:23 +00:00
|
|
|
|
def minify_html(html):
|
|
|
|
|
"""Perform a template-specific, rudimentary HTML minification for displaCy.
|
2017-10-27 12:39:09 +00:00
|
|
|
|
Disclaimer: NOT a general-purpose solution, only removes indentation and
|
|
|
|
|
newlines.
|
2017-05-14 15:50:23 +00:00
|
|
|
|
|
|
|
|
|
html (unicode): Markup to minify.
|
|
|
|
|
RETURNS (unicode): "Minified" HTML.
|
|
|
|
|
"""
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 16:03:03 +00:00
|
|
|
|
return html.strip().replace(" ", "").replace("\n", "")
|
2017-09-21 00:16:35 +00:00
|
|
|
|
|
|
|
|
|
|
escape html in displacy.render (#2378) (closes #2361)
## Description
Fix for issue #2361 :
replace &, <, >, " with &amp; , &lt; , &gt; , &quot; in before rendering svg
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [ ] I ran the tests, and all new and existing tests passed.
(As discussed in the comments to #2361)
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-05-28 16:36:41 +00:00
|
|
|
|
def escape_html(text):
|
|
|
|
|
"""Replace <, >, &, " with their HTML encoded representation. Intended to
|
|
|
|
|
prevent HTML errors in rendered displaCy markup.
|
|
|
|
|
|
|
|
|
|
text (unicode): The original text.
|
|
|
|
|
RETURNS (unicode): Equivalent text to be safely used within HTML.
|
|
|
|
|
"""
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 16:03:03 +00:00
|
|
|
|
text = text.replace("&", "&")
|
|
|
|
|
text = text.replace("<", "<")
|
|
|
|
|
text = text.replace(">", ">")
|
|
|
|
|
text = text.replace('"', """)
|
escape html in displacy.render (#2378) (closes #2361)
## Description
Fix for issue #2361 :
replace &, <, >, " with &amp; , &lt; , &gt; , &quot; in before rendering svg
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [ ] I ran the tests, and all new and existing tests passed.
(As discussed in the comments to #2361)
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-05-28 16:36:41 +00:00
|
|
|
|
return text
|
|
|
|
|
|
|
|
|
|
|
2017-09-21 00:16:35 +00:00
|
|
|
|
def use_gpu(gpu_id):
|
2017-10-03 20:47:31 +00:00
|
|
|
|
try:
|
|
|
|
|
import cupy.cuda.device
|
|
|
|
|
except ImportError:
|
|
|
|
|
return None
|
2017-09-21 00:16:35 +00:00
|
|
|
|
from thinc.neural.ops import CupyOps
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 16:03:03 +00:00
|
|
|
|
|
2017-09-21 00:16:35 +00:00
|
|
|
|
device = cupy.cuda.device.Device(gpu_id)
|
|
|
|
|
device.use()
|
|
|
|
|
Model.ops = CupyOps()
|
|
|
|
|
Model.Ops = CupyOps
|
|
|
|
|
return device
|
2018-02-13 11:42:23 +00:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def fix_random_seed(seed=0):
|
2018-02-13 11:52:48 +00:00
|
|
|
|
random.seed(seed)
|
|
|
|
|
numpy.random.seed(seed)
|
2018-12-08 11:37:38 +00:00
|
|
|
|
if cupy is not None:
|
|
|
|
|
cupy.random.seed(seed)
|
2018-05-20 13:13:37 +00:00
|
|
|
|
|
|
|
|
|
|
2019-02-12 14:47:26 +00:00
|
|
|
|
def get_json_validator(schema):
|
|
|
|
|
# We're using a helper function here to make it easier to change the
|
|
|
|
|
# validator that's used (e.g. different draft implementation), without
|
|
|
|
|
# having to change it all across the codebase.
|
2019-02-07 08:44:31 +00:00
|
|
|
|
# TODO: replace with (stable) Draft6Validator, if available
|
2019-02-12 14:47:26 +00:00
|
|
|
|
return Draft4Validator(schema)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def validate_schema(schema):
|
|
|
|
|
"""Validate a given schema. This just checks if the schema itself is valid."""
|
|
|
|
|
validator = get_json_validator(schema)
|
2019-02-07 08:44:31 +00:00
|
|
|
|
validator.check_schema(schema)
|
|
|
|
|
|
|
|
|
|
|
2019-02-12 14:47:26 +00:00
|
|
|
|
def validate_json(data, validator):
|
2019-02-07 08:44:31 +00:00
|
|
|
|
"""Validate data against a given JSON schema (see https://json-schema.org).
|
|
|
|
|
|
|
|
|
|
data: JSON-serializable data to validate.
|
2019-02-12 14:47:26 +00:00
|
|
|
|
validator (jsonschema.DraftXValidator): The validator.
|
2019-02-07 08:44:31 +00:00
|
|
|
|
RETURNS (list): A list of error messages, if available.
|
|
|
|
|
"""
|
|
|
|
|
errors = []
|
|
|
|
|
for err in sorted(validator.iter_errors(data), key=lambda e: e.path):
|
|
|
|
|
if err.path:
|
|
|
|
|
err_path = "[{}]".format(" -> ".join([str(p) for p in err.path]))
|
|
|
|
|
else:
|
|
|
|
|
err_path = ""
|
2019-02-12 14:47:26 +00:00
|
|
|
|
msg = err.message + " " + err_path
|
|
|
|
|
if err.context: # Error has suberrors, e.g. if schema uses anyOf
|
|
|
|
|
suberrs = [" - {}".format(suberr.message) for suberr in err.context]
|
|
|
|
|
msg += ":\n{}".format("".join(suberrs))
|
|
|
|
|
errors.append(msg)
|
2019-02-07 08:44:31 +00:00
|
|
|
|
return errors
|
|
|
|
|
|
|
|
|
|
|
2019-03-10 18:16:45 +00:00
|
|
|
|
def get_serialization_exclude(serializers, exclude, kwargs):
|
|
|
|
|
"""Helper function to validate serialization args and manage transition from
|
|
|
|
|
keyword arguments (pre v2.1) to exclude argument.
|
|
|
|
|
"""
|
|
|
|
|
exclude = list(exclude)
|
|
|
|
|
# Split to support file names like meta.json
|
|
|
|
|
options = [name.split(".")[0] for name in serializers]
|
|
|
|
|
for key, value in kwargs.items():
|
|
|
|
|
if key in ("vocab",) and value is False:
|
|
|
|
|
deprecation_warning(Warnings.W015.format(arg=key))
|
|
|
|
|
exclude.append(key)
|
|
|
|
|
elif key.split(".")[0] in options:
|
|
|
|
|
raise ValueError(Errors.E128.format(arg=key))
|
|
|
|
|
# TODO: user warning?
|
|
|
|
|
return exclude
|
|
|
|
|
|
|
|
|
|
|
2018-05-20 13:13:37 +00:00
|
|
|
|
class SimpleFrozenDict(dict):
|
|
|
|
|
"""Simplified implementation of a frozen dict, mainly used as default
|
|
|
|
|
function or method argument (for arguments that should default to empty
|
|
|
|
|
dictionary). Will raise an error if user or spaCy attempts to add to dict.
|
|
|
|
|
"""
|
💫 Tidy up and auto-format .py files (#2983)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Use [`black`](https://github.com/ambv/black) to auto-format all `.py` files.
- [x] Update flake8 config to exclude very large files (lemmatization tables etc.)
- [x] Update code to be compatible with flake8 rules
- [x] Fix various small bugs, inconsistencies and messy stuff in the language data
- [x] Update docs to explain new code style (`black`, `flake8`, when to use `# fmt: off` and `# fmt: on` and what `# noqa` means)
Once #2932 is merged, which auto-formats and tidies up the CLI, we'll be able to run `flake8 spacy` actually get meaningful results.
At the moment, the code style and linting isn't applied automatically, but I'm hoping that the new [GitHub Actions](https://github.com/features/actions) will let us auto-format pull requests and post comments with relevant linting information.
### Types of change
enhancement, code style
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-11-30 16:03:03 +00:00
|
|
|
|
|
2018-05-20 13:13:37 +00:00
|
|
|
|
def __setitem__(self, key, value):
|
|
|
|
|
raise NotImplementedError(Errors.E095)
|
|
|
|
|
|
|
|
|
|
def pop(self, key, default=None):
|
|
|
|
|
raise NotImplementedError(Errors.E095)
|
|
|
|
|
|
|
|
|
|
def update(self, other):
|
|
|
|
|
raise NotImplementedError(Errors.E095)
|
2019-01-10 14:40:37 +00:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class DummyTokenizer(object):
|
|
|
|
|
# add dummy methods for to_bytes, from_bytes, to_disk and from_disk to
|
|
|
|
|
# allow serialization (see #1557)
|
2019-03-10 18:16:45 +00:00
|
|
|
|
def to_bytes(self, **kwargs):
|
2019-02-07 20:00:04 +00:00
|
|
|
|
return b""
|
2019-01-10 14:40:37 +00:00
|
|
|
|
|
2019-03-10 18:16:45 +00:00
|
|
|
|
def from_bytes(self, _bytes_data, **kwargs):
|
2019-01-10 14:40:37 +00:00
|
|
|
|
return self
|
|
|
|
|
|
2019-03-10 18:16:45 +00:00
|
|
|
|
def to_disk(self, _path, **kwargs):
|
2019-01-10 14:40:37 +00:00
|
|
|
|
return None
|
|
|
|
|
|
2019-03-10 18:16:45 +00:00
|
|
|
|
def from_disk(self, _path, **kwargs):
|
2019-01-10 14:40:37 +00:00
|
|
|
|
return self
|