2017-03-15 16:37:18 +00:00
|
|
|
# coding: utf8
|
|
|
|
from __future__ import unicode_literals
|
2016-03-24 10:19:43 +00:00
|
|
|
|
2017-05-22 10:28:58 +00:00
|
|
|
import plac
|
2018-05-20 18:26:56 +00:00
|
|
|
import requests
|
2017-03-18 11:59:41 +00:00
|
|
|
import os
|
2017-03-18 18:29:36 +00:00
|
|
|
import subprocess
|
|
|
|
import sys
|
2018-11-30 19:16:14 +00:00
|
|
|
from wasabi import Printer
|
2017-03-18 18:29:36 +00:00
|
|
|
|
2018-05-20 18:26:56 +00:00
|
|
|
from .link import link
|
2018-11-30 19:16:14 +00:00
|
|
|
from ..util import get_package_path
|
2017-03-18 14:14:48 +00:00
|
|
|
from .. import about
|
2016-03-24 10:19:43 +00:00
|
|
|
|
|
|
|
|
2018-11-30 19:16:14 +00:00
|
|
|
msg = Printer()
|
|
|
|
|
|
|
|
|
2017-05-22 10:28:58 +00:00
|
|
|
@plac.annotations(
|
2018-11-30 19:16:14 +00:00
|
|
|
model=("Model to download (shortcut or name)", "positional", None, str),
|
|
|
|
direct=("Force direct download of name + version", "flag", "d", bool),
|
2019-07-27 14:35:42 +00:00
|
|
|
pip_args=("Additional arguments to be passed to `pip install` on model install"),
|
2018-11-30 19:16:14 +00:00
|
|
|
)
|
2018-05-20 18:26:56 +00:00
|
|
|
def download(model, direct=False, *pip_args):
|
2017-05-27 18:01:46 +00:00
|
|
|
"""
|
|
|
|
Download compatible model from default download path using pip. Model
|
2017-05-22 10:28:58 +00:00
|
|
|
can be shortcut, model name or, if --direct flag is set, full model name
|
2018-11-30 19:16:14 +00:00
|
|
|
with version. For direct downloads, the compatibility check will be skipped.
|
2017-05-22 10:28:58 +00:00
|
|
|
"""
|
2019-03-07 21:07:31 +00:00
|
|
|
dl_tpl = "{m}-{v}/{m}-{v}.tar.gz#egg={m}=={v}"
|
2017-03-15 16:37:18 +00:00
|
|
|
if direct:
|
2019-03-07 20:07:19 +00:00
|
|
|
components = model.split("-")
|
|
|
|
model_name = "".join(components[:-1])
|
|
|
|
version = components[-1]
|
2019-03-07 21:07:31 +00:00
|
|
|
dl = download_model(dl_tpl.format(m=model_name, v=version), pip_args)
|
2017-03-15 16:37:18 +00:00
|
|
|
else:
|
2018-05-20 18:26:56 +00:00
|
|
|
shortcuts = get_json(about.__shortcuts__, "available shortcuts")
|
2017-05-07 21:25:29 +00:00
|
|
|
model_name = shortcuts.get(model, model)
|
2018-05-20 18:26:56 +00:00
|
|
|
compatibility = get_compatibility()
|
2017-03-16 18:54:51 +00:00
|
|
|
version = get_version(model_name, compatibility)
|
2018-11-30 19:16:14 +00:00
|
|
|
dl = download_model(dl_tpl.format(m=model_name, v=version), pip_args)
|
2018-04-03 13:50:31 +00:00
|
|
|
if dl != 0: # if download subprocess doesn't return 0, exit
|
2018-01-03 20:03:36 +00:00
|
|
|
sys.exit(dl)
|
2019-03-10 12:02:24 +00:00
|
|
|
msg.good(
|
|
|
|
"Download and installation successful",
|
|
|
|
"You can now load the model via spacy.load('{}')".format(model_name),
|
|
|
|
)
|
|
|
|
# Only create symlink if the model is installed via a shortcut like 'en'.
|
|
|
|
# There's no real advantage over an additional symlink for en_core_web_sm
|
|
|
|
# and if anything, it's more error prone and causes more confusion.
|
|
|
|
if model in shortcuts:
|
|
|
|
try:
|
|
|
|
# Get package path here because link uses
|
|
|
|
# pip.get_installed_distributions() to check if model is a
|
|
|
|
# package, which fails if model was just installed via
|
|
|
|
# subprocess
|
|
|
|
package_path = get_package_path(model_name)
|
|
|
|
link(model_name, model, force=True, model_path=package_path)
|
|
|
|
except: # noqa: E722
|
|
|
|
# Dirty, but since spacy.download and the auto-linking is
|
|
|
|
# mostly a convenience wrapper, it's best to show a success
|
|
|
|
# message and loading instructions, even if linking fails.
|
|
|
|
msg.warn(
|
|
|
|
"Download successful but linking failed",
|
|
|
|
"Creating a shortcut link for '{}' didn't work (maybe you "
|
|
|
|
"don't have admin permissions?), but you can still load "
|
|
|
|
"the model via its full package name: "
|
|
|
|
"nlp = spacy.load('{}')".format(model, model_name),
|
|
|
|
)
|
2017-03-16 22:23:26 +00:00
|
|
|
|
2018-01-03 19:14:50 +00:00
|
|
|
|
2018-05-20 18:26:56 +00:00
|
|
|
def get_json(url, desc):
|
|
|
|
r = requests.get(url)
|
|
|
|
if r.status_code != 200:
|
2018-11-30 19:16:14 +00:00
|
|
|
msg.fail(
|
2018-12-08 10:49:43 +00:00
|
|
|
"Server error ({})".format(r.status_code),
|
|
|
|
"Couldn't fetch {}. Please find a model for your spaCy "
|
|
|
|
"installation (v{}), and download it manually. For more "
|
|
|
|
"details, see the documentation: "
|
|
|
|
"https://spacy.io/usage/models".format(desc, about.__version__),
|
2018-11-30 19:16:14 +00:00
|
|
|
exits=1,
|
|
|
|
)
|
2018-05-20 18:26:56 +00:00
|
|
|
return r.json()
|
2017-04-26 16:00:28 +00:00
|
|
|
|
|
|
|
|
2018-05-20 18:26:56 +00:00
|
|
|
def get_compatibility():
|
2017-04-26 16:00:28 +00:00
|
|
|
version = about.__version__
|
2018-11-30 19:16:14 +00:00
|
|
|
version = version.rsplit(".dev", 1)[0]
|
2018-05-20 18:26:56 +00:00
|
|
|
comp_table = get_json(about.__compatibility__, "compatibility table")
|
2018-11-30 19:16:14 +00:00
|
|
|
comp = comp_table["spacy"]
|
2017-03-15 16:37:18 +00:00
|
|
|
if version not in comp:
|
2018-12-08 10:49:43 +00:00
|
|
|
msg.fail("No compatible models found for v{} of spaCy".format(version), exits=1)
|
2017-04-26 16:00:28 +00:00
|
|
|
return comp[version]
|
2017-03-15 16:37:18 +00:00
|
|
|
|
|
|
|
|
|
|
|
def get_version(model, comp):
|
2018-11-30 19:16:14 +00:00
|
|
|
model = model.rsplit(".dev", 1)[0]
|
2017-03-15 16:37:18 +00:00
|
|
|
if model not in comp:
|
2018-11-30 19:16:14 +00:00
|
|
|
msg.fail(
|
2018-12-08 10:49:43 +00:00
|
|
|
"No compatible model found for '{}' "
|
|
|
|
"(spaCy v{}).".format(model, about.__version__),
|
2018-11-30 19:16:14 +00:00
|
|
|
exits=1,
|
|
|
|
)
|
2017-03-15 16:37:18 +00:00
|
|
|
return comp[model][0]
|
|
|
|
|
|
|
|
|
2018-05-20 18:26:56 +00:00
|
|
|
def download_model(filename, user_pip_args=None):
|
2018-11-30 19:16:14 +00:00
|
|
|
download_url = about.__download_url__ + "/" + filename
|
|
|
|
pip_args = ["--no-cache-dir", "--no-deps"]
|
2018-05-20 18:26:56 +00:00
|
|
|
if user_pip_args:
|
|
|
|
pip_args.extend(user_pip_args)
|
2018-11-30 19:16:14 +00:00
|
|
|
cmd = [sys.executable, "-m", "pip", "install"] + pip_args + [download_url]
|
2018-05-20 18:26:56 +00:00
|
|
|
return subprocess.call(cmd, env=os.environ.copy())
|