mirror of https://github.com/explosion/spaCy.git
243 lines
8.1 KiB
Python
243 lines
8.1 KiB
Python
from typing import Optional, Union, Any, Dict
|
|
import shutil
|
|
from pathlib import Path
|
|
from wasabi import Printer, get_raw_input
|
|
import srsly
|
|
import sys
|
|
|
|
from ._util import app, Arg, Opt
|
|
from ..schemas import validate, ModelMetaSchema
|
|
from .. import util
|
|
from .. import about
|
|
|
|
|
|
@app.command("package")
|
|
def package_cli(
|
|
# fmt: off
|
|
input_dir: Path = Arg(..., help="Directory with model data", exists=True, file_okay=False),
|
|
output_dir: Path = Arg(..., help="Output parent directory", exists=True, file_okay=False),
|
|
meta_path: Optional[Path] = Opt(None, "--meta-path", "--meta", "-m", help="Path to meta.json", exists=True, dir_okay=False),
|
|
create_meta: bool = Opt(False, "--create-meta", "-c", "-C", help="Create meta.json, even if one exists"),
|
|
version: Optional[str] = Opt(None, "--version", "-v", help="Package version to override meta"),
|
|
force: bool = Opt(False, "--force", "-f", "-F", help="Force overwriting existing model in output directory"),
|
|
# fmt: on
|
|
):
|
|
"""
|
|
Generate Python package for model data, including meta and required
|
|
installation files. A new directory will be created in the specified
|
|
output directory, and model data will be copied over. If --create-meta is
|
|
set and a meta.json already exists in the output directory, the existing
|
|
values will be used as the defaults in the command-line prompt.
|
|
"""
|
|
package(
|
|
input_dir,
|
|
output_dir,
|
|
meta_path=meta_path,
|
|
version=version,
|
|
create_meta=create_meta,
|
|
force=force,
|
|
silent=False,
|
|
)
|
|
|
|
|
|
def package(
|
|
input_dir: Path,
|
|
output_dir: Path,
|
|
meta_path: Optional[Path] = None,
|
|
version: Optional[str] = None,
|
|
create_meta: bool = False,
|
|
force: bool = False,
|
|
silent: bool = True,
|
|
) -> None:
|
|
msg = Printer(no_print=silent, pretty=not silent)
|
|
input_path = util.ensure_path(input_dir)
|
|
output_path = util.ensure_path(output_dir)
|
|
meta_path = util.ensure_path(meta_path)
|
|
if not input_path or not input_path.exists():
|
|
msg.fail("Can't locate model data", input_path, exits=1)
|
|
if not output_path or not output_path.exists():
|
|
msg.fail("Output directory not found", output_path, exits=1)
|
|
if meta_path and not meta_path.exists():
|
|
msg.fail("Can't find model meta.json", meta_path, exits=1)
|
|
|
|
meta_path = meta_path or input_dir / "meta.json"
|
|
if not meta_path.exists() or not meta_path.is_file():
|
|
msg.fail("Can't load model meta.json", meta_path, exits=1)
|
|
meta = srsly.read_json(meta_path)
|
|
meta = get_meta(input_dir, meta)
|
|
if version is not None:
|
|
meta["version"] = version
|
|
if not create_meta: # only print if user doesn't want to overwrite
|
|
msg.good("Loaded meta.json from file", meta_path)
|
|
else:
|
|
meta = generate_meta(meta, msg)
|
|
errors = validate(ModelMetaSchema, meta)
|
|
if errors:
|
|
msg.fail("Invalid model meta.json", "\n".join(errors), exits=1)
|
|
model_name = meta["lang"] + "_" + meta["name"]
|
|
model_name_v = model_name + "-" + meta["version"]
|
|
main_path = output_dir / model_name_v
|
|
package_path = main_path / model_name
|
|
|
|
if package_path.exists():
|
|
if force:
|
|
shutil.rmtree(str(package_path))
|
|
else:
|
|
msg.fail(
|
|
"Package directory already exists",
|
|
"Please delete the directory and try again, or use the "
|
|
"`--force` flag to overwrite existing directories.",
|
|
exits=1,
|
|
)
|
|
Path.mkdir(package_path, parents=True)
|
|
shutil.copytree(str(input_dir), str(package_path / model_name_v))
|
|
create_file(main_path / "meta.json", srsly.json_dumps(meta, indent=2))
|
|
create_file(main_path / "setup.py", TEMPLATE_SETUP)
|
|
create_file(main_path / "MANIFEST.in", TEMPLATE_MANIFEST)
|
|
create_file(package_path / "__init__.py", TEMPLATE_INIT)
|
|
msg.good(f"Successfully created package '{model_name_v}'", main_path)
|
|
with util.working_dir(main_path):
|
|
util.run_command([sys.executable, "setup.py", "sdist"])
|
|
zip_file = main_path / "dist" / f"{model_name_v}.tar.gz"
|
|
msg.good(f"Successfully created zipped Python package", zip_file)
|
|
|
|
|
|
def create_file(file_path: Path, contents: str) -> None:
|
|
file_path.touch()
|
|
file_path.open("w", encoding="utf-8").write(contents)
|
|
|
|
|
|
def get_meta(
|
|
model_path: Union[str, Path], existing_meta: Dict[str, Any]
|
|
) -> Dict[str, Any]:
|
|
meta = {
|
|
"lang": "en",
|
|
"name": "model",
|
|
"version": "0.0.0",
|
|
"description": None,
|
|
"author": None,
|
|
"email": None,
|
|
"url": None,
|
|
"license": "MIT",
|
|
}
|
|
meta.update(existing_meta)
|
|
nlp = util.load_model_from_path(Path(model_path))
|
|
meta["spacy_version"] = util.get_model_version_range(about.__version__)
|
|
meta["pipeline"] = nlp.pipe_names
|
|
meta["vectors"] = {
|
|
"width": nlp.vocab.vectors_length,
|
|
"vectors": len(nlp.vocab.vectors),
|
|
"keys": nlp.vocab.vectors.n_keys,
|
|
"name": nlp.vocab.vectors.name,
|
|
}
|
|
if about.__title__ != "spacy":
|
|
meta["parent_package"] = about.__title__
|
|
return meta
|
|
|
|
|
|
def generate_meta(existing_meta: Dict[str, Any], msg: Printer) -> Dict[str, Any]:
|
|
meta = existing_meta or {}
|
|
settings = [
|
|
("lang", "Model language", meta.get("lang", "en")),
|
|
("name", "Model name", meta.get("name", "model")),
|
|
("version", "Model version", meta.get("version", "0.0.0")),
|
|
("description", "Model description", meta.get("description", None)),
|
|
("author", "Author", meta.get("author", None)),
|
|
("email", "Author email", meta.get("email", None)),
|
|
("url", "Author website", meta.get("url", None)),
|
|
("license", "License", meta.get("license", "MIT")),
|
|
]
|
|
msg.divider("Generating meta.json")
|
|
msg.text(
|
|
"Enter the package settings for your model. The following information "
|
|
"will be read from your model data: pipeline, vectors."
|
|
)
|
|
for setting, desc, default in settings:
|
|
response = get_raw_input(desc, default)
|
|
meta[setting] = default if response == "" and default else response
|
|
return meta
|
|
|
|
|
|
TEMPLATE_SETUP = """
|
|
#!/usr/bin/env python
|
|
import io
|
|
import json
|
|
from os import path, walk
|
|
from shutil import copy
|
|
from setuptools import setup
|
|
|
|
|
|
def load_meta(fp):
|
|
with io.open(fp, encoding='utf8') as f:
|
|
return json.load(f)
|
|
|
|
|
|
def list_files(data_dir):
|
|
output = []
|
|
for root, _, filenames in walk(data_dir):
|
|
for filename in filenames:
|
|
if not filename.startswith('.'):
|
|
output.append(path.join(root, filename))
|
|
output = [path.relpath(p, path.dirname(data_dir)) for p in output]
|
|
output.append('meta.json')
|
|
return output
|
|
|
|
|
|
def list_requirements(meta):
|
|
parent_package = meta.get('parent_package', 'spacy')
|
|
requirements = [parent_package + meta['spacy_version']]
|
|
if 'setup_requires' in meta:
|
|
requirements += meta['setup_requires']
|
|
if 'requirements' in meta:
|
|
requirements += meta['requirements']
|
|
return requirements
|
|
|
|
|
|
def setup_package():
|
|
root = path.abspath(path.dirname(__file__))
|
|
meta_path = path.join(root, 'meta.json')
|
|
meta = load_meta(meta_path)
|
|
model_name = str(meta['lang'] + '_' + meta['name'])
|
|
model_dir = path.join(model_name, model_name + '-' + meta['version'])
|
|
|
|
copy(meta_path, path.join(model_name))
|
|
copy(meta_path, model_dir)
|
|
|
|
setup(
|
|
name=model_name,
|
|
description=meta.get('description'),
|
|
author=meta.get('author'),
|
|
author_email=meta.get('email'),
|
|
url=meta.get('url'),
|
|
version=meta['version'],
|
|
license=meta.get('license'),
|
|
packages=[model_name],
|
|
package_data={model_name: list_files(model_dir)},
|
|
install_requires=list_requirements(meta),
|
|
zip_safe=False,
|
|
entry_points={'spacy_models': ['{m} = {m}'.format(m=model_name)]}
|
|
)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
setup_package()
|
|
""".strip()
|
|
|
|
|
|
TEMPLATE_MANIFEST = """
|
|
include meta.json
|
|
""".strip()
|
|
|
|
|
|
TEMPLATE_INIT = """
|
|
from pathlib import Path
|
|
from spacy.util import load_model_from_init_py, get_model_meta
|
|
|
|
|
|
__version__ = get_model_meta(Path(__file__).parent)['version']
|
|
|
|
|
|
def load(**overrides):
|
|
return load_model_from_init_py(__file__, **overrides)
|
|
""".strip()
|