2020-06-21 19:35:01 +00:00
|
|
|
from typing import Optional, Union, Any, Dict
|
2017-03-21 01:06:29 +00:00
|
|
|
import shutil
|
2017-03-20 21:50:13 +00:00
|
|
|
from pathlib import Path
|
2020-06-21 19:35:01 +00:00
|
|
|
from wasabi import Printer, get_raw_input
|
💫 Replace ujson, msgpack and dill/pickle/cloudpickle with srsly (#3003)
Remove hacks and wrappers, keep code in sync across our libraries and move spaCy a few steps closer to only depending on packages with binary wheels 🎉
See here: https://github.com/explosion/srsly
Serialization is hard, especially across Python versions and multiple platforms. After dealing with many subtle bugs over the years (encodings, locales, large files) our libraries like spaCy and Prodigy have steadily grown a number of utility functions to wrap the multiple serialization formats we need to support (especially json, msgpack and pickle). These wrapping functions ended up duplicated across our codebases, so we wanted to put them in one place.
At the same time, we noticed that having a lot of small dependencies was making maintainence harder, and making installation slower. To solve this, we've made srsly standalone, by including the component packages directly within it. This way we can provide all the serialization utilities we need in a single binary wheel.
srsly currently includes forks of the following packages:
ujson
msgpack
msgpack-numpy
cloudpickle
* WIP: replace json/ujson with srsly
* Replace ujson in examples
Use regular json instead of srsly to make code easier to read and follow
* Update requirements
* Fix imports
* Fix typos
* Replace msgpack with srsly
* Fix warning
2018-12-03 00:28:22 +00:00
|
|
|
import srsly
|
2020-06-21 19:35:01 +00:00
|
|
|
import sys
|
2017-03-20 21:50:13 +00:00
|
|
|
|
2020-07-10 15:57:40 +00:00
|
|
|
from ._util import app, Arg, Opt
|
2020-06-21 19:35:01 +00:00
|
|
|
from ..schemas import validate, ModelMetaSchema
|
2017-03-20 21:50:13 +00:00
|
|
|
from .. import util
|
2017-05-07 21:25:29 +00:00
|
|
|
from .. import about
|
2017-03-20 21:50:13 +00:00
|
|
|
|
|
|
|
|
2020-06-21 11:44:00 +00:00
|
|
|
@app.command("package")
|
2020-06-21 19:35:01 +00:00
|
|
|
def package_cli(
|
2020-01-01 12:15:46 +00:00
|
|
|
# fmt: off
|
2020-09-03 11:13:03 +00:00
|
|
|
input_dir: Path = Arg(..., help="Directory with pipeline data", exists=True, file_okay=False),
|
2020-06-21 19:35:01 +00:00
|
|
|
output_dir: Path = Arg(..., help="Output parent directory", exists=True, file_okay=False),
|
2020-06-27 18:36:08 +00:00
|
|
|
meta_path: Optional[Path] = Opt(None, "--meta-path", "--meta", "-m", help="Path to meta.json", exists=True, dir_okay=False),
|
2020-06-21 22:57:28 +00:00
|
|
|
create_meta: bool = Opt(False, "--create-meta", "-c", "-C", help="Create meta.json, even if one exists"),
|
2020-09-11 09:38:28 +00:00
|
|
|
name: Optional[str] = Opt(None, "--name", "-n", help="Package name to override meta"),
|
2020-06-27 18:36:08 +00:00
|
|
|
version: Optional[str] = Opt(None, "--version", "-v", help="Package version to override meta"),
|
2020-07-27 14:52:23 +00:00
|
|
|
no_sdist: bool = Opt(False, "--no-sdist", "-NS", help="Don't build .tar.gz sdist, can be set if you want to run this step manually"),
|
2020-09-03 11:13:03 +00:00
|
|
|
force: bool = Opt(False, "--force", "-f", "-F", help="Force overwriting existing data in output directory"),
|
2020-01-01 12:15:46 +00:00
|
|
|
# fmt: on
|
|
|
|
):
|
2017-05-27 18:01:46 +00:00
|
|
|
"""
|
2020-09-03 11:13:03 +00:00
|
|
|
Generate an installable Python package for a pipeline. Includes binary data,
|
2020-07-12 11:53:49 +00:00
|
|
|
meta and required installation files. A new directory will be created in the
|
2020-09-03 11:13:03 +00:00
|
|
|
specified output directory, and the data will be copied over. If
|
2020-07-12 11:53:49 +00:00
|
|
|
--create-meta is set and a meta.json already exists in the output directory,
|
|
|
|
the existing values will be used as the defaults in the command-line prompt.
|
|
|
|
After packaging, "python setup.py sdist" is run in the package directory,
|
|
|
|
which will create a .tar.gz archive that can be installed via "pip install".
|
2020-09-04 10:58:50 +00:00
|
|
|
|
|
|
|
DOCS: https://nightly.spacy.io/api/cli#package
|
2017-05-22 10:28:58 +00:00
|
|
|
"""
|
2020-06-21 19:35:01 +00:00
|
|
|
package(
|
|
|
|
input_dir,
|
|
|
|
output_dir,
|
|
|
|
meta_path=meta_path,
|
2020-09-11 09:38:28 +00:00
|
|
|
name=name,
|
2020-06-27 18:36:08 +00:00
|
|
|
version=version,
|
2020-06-21 19:35:01 +00:00
|
|
|
create_meta=create_meta,
|
2020-07-27 14:52:23 +00:00
|
|
|
create_sdist=not no_sdist,
|
2020-06-21 19:35:01 +00:00
|
|
|
force=force,
|
|
|
|
silent=False,
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
def package(
|
|
|
|
input_dir: Path,
|
|
|
|
output_dir: Path,
|
|
|
|
meta_path: Optional[Path] = None,
|
2020-09-11 09:38:28 +00:00
|
|
|
name: Optional[str] = None,
|
2020-06-27 18:36:08 +00:00
|
|
|
version: Optional[str] = None,
|
2020-06-21 19:35:01 +00:00
|
|
|
create_meta: bool = False,
|
2020-07-27 14:52:23 +00:00
|
|
|
create_sdist: bool = True,
|
2020-06-21 19:35:01 +00:00
|
|
|
force: bool = False,
|
|
|
|
silent: bool = True,
|
|
|
|
) -> None:
|
|
|
|
msg = Printer(no_print=silent, pretty=not silent)
|
2017-05-07 21:25:29 +00:00
|
|
|
input_path = util.ensure_path(input_dir)
|
|
|
|
output_path = util.ensure_path(output_dir)
|
2017-08-12 19:44:15 +00:00
|
|
|
meta_path = util.ensure_path(meta_path)
|
2017-05-07 21:25:29 +00:00
|
|
|
if not input_path or not input_path.exists():
|
2020-09-03 11:13:03 +00:00
|
|
|
msg.fail("Can't locate pipeline data", input_path, exits=1)
|
2017-05-07 21:25:29 +00:00
|
|
|
if not output_path or not output_path.exists():
|
2018-12-08 10:49:43 +00:00
|
|
|
msg.fail("Output directory not found", output_path, exits=1)
|
2017-05-07 21:25:29 +00:00
|
|
|
if meta_path and not meta_path.exists():
|
2020-09-03 11:13:03 +00:00
|
|
|
msg.fail("Can't find pipeline meta.json", meta_path, exits=1)
|
2020-06-21 19:35:01 +00:00
|
|
|
meta_path = meta_path or input_dir / "meta.json"
|
|
|
|
if not meta_path.exists() or not meta_path.is_file():
|
2020-09-03 11:13:03 +00:00
|
|
|
msg.fail("Can't load pipeline meta.json", meta_path, exits=1)
|
2020-06-21 19:35:01 +00:00
|
|
|
meta = srsly.read_json(meta_path)
|
2020-06-27 18:36:08 +00:00
|
|
|
meta = get_meta(input_dir, meta)
|
2020-09-11 09:38:28 +00:00
|
|
|
if name is not None:
|
|
|
|
meta["name"] = name
|
2020-06-27 18:36:08 +00:00
|
|
|
if version is not None:
|
|
|
|
meta["version"] = version
|
2020-06-21 19:35:01 +00:00
|
|
|
if not create_meta: # only print if user doesn't want to overwrite
|
|
|
|
msg.good("Loaded meta.json from file", meta_path)
|
|
|
|
else:
|
2020-06-27 18:36:08 +00:00
|
|
|
meta = generate_meta(meta, msg)
|
2020-06-21 19:35:01 +00:00
|
|
|
errors = validate(ModelMetaSchema, meta)
|
|
|
|
if errors:
|
2020-09-03 11:13:03 +00:00
|
|
|
msg.fail("Invalid pipeline meta.json")
|
2020-08-25 15:13:33 +00:00
|
|
|
print("\n".join(errors))
|
|
|
|
sys.exit(1)
|
2018-11-30 19:16:14 +00:00
|
|
|
model_name = meta["lang"] + "_" + meta["name"]
|
|
|
|
model_name_v = model_name + "-" + meta["version"]
|
2020-06-21 19:35:01 +00:00
|
|
|
main_path = output_dir / model_name_v
|
2017-03-20 21:50:13 +00:00
|
|
|
package_path = main_path / model_name
|
2017-03-21 01:06:53 +00:00
|
|
|
if package_path.exists():
|
|
|
|
if force:
|
2019-12-22 00:53:56 +00:00
|
|
|
shutil.rmtree(str(package_path))
|
2017-03-21 01:06:53 +00:00
|
|
|
else:
|
2018-11-30 19:16:14 +00:00
|
|
|
msg.fail(
|
2018-12-08 10:49:43 +00:00
|
|
|
"Package directory already exists",
|
|
|
|
"Please delete the directory and try again, or use the "
|
2019-12-22 00:53:56 +00:00
|
|
|
"`--force` flag to overwrite existing directories.",
|
2018-11-30 19:16:14 +00:00
|
|
|
exits=1,
|
|
|
|
)
|
2017-03-21 01:06:53 +00:00
|
|
|
Path.mkdir(package_path, parents=True)
|
2020-06-21 19:35:01 +00:00
|
|
|
shutil.copytree(str(input_dir), str(package_path / model_name_v))
|
2020-11-30 12:43:58 +00:00
|
|
|
license_path = package_path / model_name_v / "LICENSE"
|
|
|
|
if license_path.exists():
|
|
|
|
shutil.move(str(license_path), str(main_path))
|
2018-12-19 13:36:08 +00:00
|
|
|
create_file(main_path / "meta.json", srsly.json_dumps(meta, indent=2))
|
2018-11-30 19:16:14 +00:00
|
|
|
create_file(main_path / "setup.py", TEMPLATE_SETUP)
|
|
|
|
create_file(main_path / "MANIFEST.in", TEMPLATE_MANIFEST)
|
|
|
|
create_file(package_path / "__init__.py", TEMPLATE_INIT)
|
2019-12-22 00:53:56 +00:00
|
|
|
msg.good(f"Successfully created package '{model_name_v}'", main_path)
|
2020-07-27 14:52:23 +00:00
|
|
|
if create_sdist:
|
|
|
|
with util.working_dir(main_path):
|
2020-09-20 14:21:43 +00:00
|
|
|
util.run_command([sys.executable, "setup.py", "sdist"], capture=False)
|
2020-07-27 14:52:23 +00:00
|
|
|
zip_file = main_path / "dist" / f"{model_name_v}.tar.gz"
|
|
|
|
msg.good(f"Successfully created zipped Python package", zip_file)
|
2017-03-21 01:06:53 +00:00
|
|
|
|
|
|
|
|
2020-06-21 19:35:01 +00:00
|
|
|
def create_file(file_path: Path, contents: str) -> None:
|
2017-03-20 21:50:13 +00:00
|
|
|
file_path.touch()
|
2018-11-30 19:16:14 +00:00
|
|
|
file_path.open("w", encoding="utf-8").write(contents)
|
2017-03-20 21:50:13 +00:00
|
|
|
|
|
|
|
|
2020-06-27 18:36:08 +00:00
|
|
|
def get_meta(
|
|
|
|
model_path: Union[str, Path], existing_meta: Dict[str, Any]
|
2020-06-21 19:35:01 +00:00
|
|
|
) -> Dict[str, Any]:
|
2020-06-27 18:36:08 +00:00
|
|
|
meta = {
|
|
|
|
"lang": "en",
|
2020-09-03 11:13:03 +00:00
|
|
|
"name": "pipeline",
|
2020-06-27 18:36:08 +00:00
|
|
|
"version": "0.0.0",
|
2020-08-25 15:13:33 +00:00
|
|
|
"description": "",
|
|
|
|
"author": "",
|
|
|
|
"email": "",
|
|
|
|
"url": "",
|
2020-06-27 18:36:08 +00:00
|
|
|
"license": "MIT",
|
|
|
|
}
|
|
|
|
meta.update(existing_meta)
|
2017-10-25 14:03:26 +00:00
|
|
|
nlp = util.load_model_from_path(Path(model_path))
|
2020-05-30 13:01:58 +00:00
|
|
|
meta["spacy_version"] = util.get_model_version_range(about.__version__)
|
2018-11-30 19:16:14 +00:00
|
|
|
meta["vectors"] = {
|
|
|
|
"width": nlp.vocab.vectors_length,
|
|
|
|
"vectors": len(nlp.vocab.vectors),
|
|
|
|
"keys": nlp.vocab.vectors.n_keys,
|
2018-12-27 18:55:40 +00:00
|
|
|
"name": nlp.vocab.vectors.name,
|
2018-11-30 19:16:14 +00:00
|
|
|
}
|
2020-06-27 18:36:08 +00:00
|
|
|
if about.__title__ != "spacy":
|
|
|
|
meta["parent_package"] = about.__title__
|
|
|
|
return meta
|
|
|
|
|
|
|
|
|
|
|
|
def generate_meta(existing_meta: Dict[str, Any], msg: Printer) -> Dict[str, Any]:
|
|
|
|
meta = existing_meta or {}
|
|
|
|
settings = [
|
2020-09-03 11:13:03 +00:00
|
|
|
("lang", "Pipeline language", meta.get("lang", "en")),
|
|
|
|
("name", "Pipeline name", meta.get("name", "pipeline")),
|
|
|
|
("version", "Package version", meta.get("version", "0.0.0")),
|
|
|
|
("description", "Package description", meta.get("description", None)),
|
2020-06-27 18:36:08 +00:00
|
|
|
("author", "Author", meta.get("author", None)),
|
|
|
|
("email", "Author email", meta.get("email", None)),
|
|
|
|
("url", "Author website", meta.get("url", None)),
|
|
|
|
("license", "License", meta.get("license", "MIT")),
|
|
|
|
]
|
2018-12-08 10:49:43 +00:00
|
|
|
msg.divider("Generating meta.json")
|
|
|
|
msg.text(
|
2020-09-03 11:13:03 +00:00
|
|
|
"Enter the package settings for your pipeline. The following information "
|
|
|
|
"will be read from your pipeline data: pipeline, vectors."
|
2018-12-08 10:49:43 +00:00
|
|
|
)
|
2017-03-20 21:50:13 +00:00
|
|
|
for setting, desc, default in settings:
|
2018-11-30 19:16:14 +00:00
|
|
|
response = get_raw_input(desc, default)
|
|
|
|
meta[setting] = default if response == "" and default else response
|
2017-05-27 18:02:01 +00:00
|
|
|
return meta
|
2017-04-16 11:13:17 +00:00
|
|
|
|
|
|
|
|
2017-11-07 11:15:35 +00:00
|
|
|
TEMPLATE_SETUP = """
|
|
|
|
#!/usr/bin/env python
|
|
|
|
import io
|
|
|
|
import json
|
|
|
|
from os import path, walk
|
|
|
|
from shutil import copy
|
|
|
|
from setuptools import setup
|
|
|
|
|
|
|
|
|
|
|
|
def load_meta(fp):
|
|
|
|
with io.open(fp, encoding='utf8') as f:
|
|
|
|
return json.load(f)
|
|
|
|
|
|
|
|
|
|
|
|
def list_files(data_dir):
|
|
|
|
output = []
|
|
|
|
for root, _, filenames in walk(data_dir):
|
|
|
|
for filename in filenames:
|
|
|
|
if not filename.startswith('.'):
|
|
|
|
output.append(path.join(root, filename))
|
|
|
|
output = [path.relpath(p, path.dirname(data_dir)) for p in output]
|
|
|
|
output.append('meta.json')
|
|
|
|
return output
|
|
|
|
|
|
|
|
|
|
|
|
def list_requirements(meta):
|
|
|
|
parent_package = meta.get('parent_package', 'spacy')
|
2020-05-30 13:01:58 +00:00
|
|
|
requirements = [parent_package + meta['spacy_version']]
|
2017-11-07 11:15:35 +00:00
|
|
|
if 'setup_requires' in meta:
|
|
|
|
requirements += meta['setup_requires']
|
2019-07-27 11:34:57 +00:00
|
|
|
if 'requirements' in meta:
|
|
|
|
requirements += meta['requirements']
|
2017-11-07 11:15:35 +00:00
|
|
|
return requirements
|
|
|
|
|
|
|
|
|
|
|
|
def setup_package():
|
|
|
|
root = path.abspath(path.dirname(__file__))
|
|
|
|
meta_path = path.join(root, 'meta.json')
|
|
|
|
meta = load_meta(meta_path)
|
|
|
|
model_name = str(meta['lang'] + '_' + meta['name'])
|
|
|
|
model_dir = path.join(model_name, model_name + '-' + meta['version'])
|
|
|
|
|
|
|
|
copy(meta_path, path.join(model_name))
|
|
|
|
copy(meta_path, model_dir)
|
|
|
|
|
|
|
|
setup(
|
|
|
|
name=model_name,
|
2020-06-27 18:36:08 +00:00
|
|
|
description=meta.get('description'),
|
|
|
|
author=meta.get('author'),
|
|
|
|
author_email=meta.get('email'),
|
|
|
|
url=meta.get('url'),
|
2017-11-07 11:15:35 +00:00
|
|
|
version=meta['version'],
|
2020-06-27 18:36:08 +00:00
|
|
|
license=meta.get('license'),
|
2017-11-07 11:15:35 +00:00
|
|
|
packages=[model_name],
|
|
|
|
package_data={model_name: list_files(model_dir)},
|
|
|
|
install_requires=list_requirements(meta),
|
|
|
|
zip_safe=False,
|
2020-05-22 13:42:46 +00:00
|
|
|
entry_points={'spacy_models': ['{m} = {m}'.format(m=model_name)]}
|
2017-11-07 11:15:35 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
if __name__ == '__main__':
|
|
|
|
setup_package()
|
|
|
|
""".strip()
|
|
|
|
|
|
|
|
|
|
|
|
TEMPLATE_MANIFEST = """
|
|
|
|
include meta.json
|
2020-11-30 12:43:58 +00:00
|
|
|
include LICENSE
|
2017-11-07 11:15:35 +00:00
|
|
|
""".strip()
|
|
|
|
|
|
|
|
|
|
|
|
TEMPLATE_INIT = """
|
|
|
|
from pathlib import Path
|
|
|
|
from spacy.util import load_model_from_init_py, get_model_meta
|
|
|
|
|
|
|
|
|
|
|
|
__version__ = get_model_meta(Path(__file__).parent)['version']
|
|
|
|
|
|
|
|
|
|
|
|
def load(**overrides):
|
|
|
|
return load_model_from_init_py(__file__, **overrides)
|
|
|
|
""".strip()
|