diff --git a/spacy/about.py b/spacy/about.py index 4e03256c9..0c9ac70ef 100644 --- a/spacy/about.py +++ b/spacy/about.py @@ -15,4 +15,3 @@ __docs_models__ = 'https://spacy.io/usage/models' __download_url__ = 'https://github.com/explosion/spacy-models/releases/download' __compatibility__ = 'https://raw.githubusercontent.com/explosion/spacy-models/master/compatibility.json' __shortcuts__ = 'https://raw.githubusercontent.com/explosion/spacy-models/master/shortcuts-v2.json' -__model_files__ = 'https://raw.githubusercontent.com/explosion/spacy-dev-resources/develop/templates/model/' diff --git a/spacy/cli/package.py b/spacy/cli/package.py index 5ee8a2b1e..8c590b49b 100644 --- a/spacy/cli/package.py +++ b/spacy/cli/package.py @@ -3,7 +3,6 @@ from __future__ import unicode_literals import plac import shutil -import requests from pathlib import Path from ..compat import path2str, json_dumps @@ -38,9 +37,6 @@ def package(cmd, input_dir, output_dir, meta_path=None, create_meta=False, if meta_path and not meta_path.exists(): prints(meta_path, title="meta.json not found", exits=1) - template_setup = get_template('setup.py') - template_manifest = get_template('MANIFEST.in') - template_init = get_template('xx_model_name/__init__.py') meta_path = meta_path or input_path / 'meta.json' if meta_path.is_file(): meta = util.read_json(meta_path) @@ -58,9 +54,9 @@ def package(cmd, input_dir, output_dir, meta_path=None, create_meta=False, shutil.copytree(path2str(input_path), path2str(package_path / model_name_v)) create_file(main_path / 'meta.json', json_dumps(meta)) - create_file(main_path / 'setup.py', template_setup) - create_file(main_path / 'MANIFEST.in', template_manifest) - create_file(package_path / '__init__.py', template_init) + create_file(main_path / 'setup.py', TEMPLATE_SETUP) + create_file(main_path / 'MANIFEST.in', TEMPLATE_MANIFEST) + create_file(package_path / '__init__.py', TEMPLATE_INIT) prints(main_path, "To build the package, run `python setup.py sdist` in " "this directory.", title="Successfully created package '%s'" % model_name_v) @@ -120,9 +116,88 @@ def validate_meta(meta, keys): return meta -def get_template(filepath): - r = requests.get(about.__model_files__ + filepath) - if r.status_code != 200: - prints("Couldn't fetch template files from GitHub.", - title="Server error (%d)" % r.status_code, exits=1) - return r.text +TEMPLATE_SETUP = """ +#!/usr/bin/env python +# coding: utf8 +from __future__ import unicode_literals + +import io +import json +from os import path, walk +from shutil import copy +from setuptools import setup + + +def load_meta(fp): + with io.open(fp, encoding='utf8') as f: + return json.load(f) + + +def list_files(data_dir): + output = [] + for root, _, filenames in walk(data_dir): + for filename in filenames: + if not filename.startswith('.'): + output.append(path.join(root, filename)) + output = [path.relpath(p, path.dirname(data_dir)) for p in output] + output.append('meta.json') + return output + + +def list_requirements(meta): + parent_package = meta.get('parent_package', 'spacy') + requirements = [parent_package + meta['spacy_version']] + if 'setup_requires' in meta: + requirements += meta['setup_requires'] + return requirements + + +def setup_package(): + root = path.abspath(path.dirname(__file__)) + meta_path = path.join(root, 'meta.json') + meta = load_meta(meta_path) + model_name = str(meta['lang'] + '_' + meta['name']) + model_dir = path.join(model_name, model_name + '-' + meta['version']) + + copy(meta_path, path.join(model_name)) + copy(meta_path, model_dir) + + setup( + name=model_name, + description=meta['description'], + author=meta['author'], + author_email=meta['email'], + url=meta['url'], + version=meta['version'], + license=meta['license'], + packages=[model_name], + package_data={model_name: list_files(model_dir)}, + install_requires=list_requirements(meta), + zip_safe=False, + ) + + +if __name__ == '__main__': + setup_package() +""".strip() + + +TEMPLATE_MANIFEST = """ +include meta.json +""".strip() + + +TEMPLATE_INIT = """ +# coding: utf8 +from __future__ import unicode_literals + +from pathlib import Path +from spacy.util import load_model_from_init_py, get_model_meta + + +__version__ = get_model_meta(Path(__file__).parent)['version'] + + +def load(**overrides): + return load_model_from_init_py(__file__, **overrides) +""".strip() diff --git a/website/api/_top-level/_util.jade b/website/api/_top-level/_util.jade index a08cf1276..ef12846ad 100644 --- a/website/api/_top-level/_util.jade +++ b/website/api/_top-level/_util.jade @@ -148,7 +148,7 @@ p p | A helper function to use in the #[code load()] method of a model package's - | #[+src(gh("spacy-dev-resources", "templates/model/en_model_name/__init__.py")) #[code __init__.py]]. + | #[+src(gh("spacy-models", "template/model/xx_model_name/__init__.py")) #[code __init__.py]]. +aside-code("Example"). from spacy.util import load_model_from_init_py diff --git a/website/api/cli.jade b/website/api/cli.jade index cd1cb22fb..12f5e8713 100644 --- a/website/api/cli.jade +++ b/website/api/cli.jade @@ -541,13 +541,9 @@ p | from an existing model data directory. All data files are copied over. | If the path to a #[code meta.json] is supplied, or a #[code meta.json] is | found in the input directory, this file is used. Otherwise, the data can - | be entered directly from the command line. The required file templates - | are downloaded from - | #[+src(gh("spacy-dev-resources", "templates/model")) GitHub] to make - | sure you're always using the latest versions. This means you need to be - | connected to the internet to use this command. After packaging, you - | can run #[code python setup.py sdist] from the newly created directory - | to turn your model into an installable archive file. + | be entered directly from the command line. After packaging, you can run + | #[code python setup.py sdist] from the newly created directory to turn + | your model into an installable archive file. +code(false, "bash", "$", false, false, true). spacy package [input_dir] [output_dir] [--meta-path] [--create-meta] [--force] diff --git a/website/usage/_training/_saving-loading.jade b/website/usage/_training/_saving-loading.jade index 885b3ad24..2a6f9380b 100644 --- a/website/usage/_training/_saving-loading.jade +++ b/website/usage/_training/_saving-loading.jade @@ -59,8 +59,8 @@ p This command will create a model package directory that should look like this: └── en_example_model-1.0.0 # model data p - | You can also find templates for all files in our - | #[+src(gh("spacy-dev-resources", "templates/model")) spaCy dev resources]. + | You can also find templates for all files on + | #[+src(gh("spacy-models", "template")) GitHub]. | If you're creating the package manually, keep in mind that the directories | need to be named according to the naming conventions of | #[code lang_name] and #[code lang_name-version].