2017-03-15 16:37:18 +00:00
|
|
|
# coding: utf8
|
|
|
|
from __future__ import unicode_literals
|
2016-03-24 10:19:43 +00:00
|
|
|
|
2017-05-22 10:28:58 +00:00
|
|
|
import plac
|
2017-03-18 11:59:41 +00:00
|
|
|
import os
|
2017-03-18 18:29:36 +00:00
|
|
|
import subprocess
|
|
|
|
import sys
|
2018-03-28 10:46:07 +00:00
|
|
|
import ujson
|
2017-03-18 18:29:36 +00:00
|
|
|
|
2017-05-07 21:25:29 +00:00
|
|
|
from .link import link
|
2018-04-03 13:50:31 +00:00
|
|
|
from ._messages import Messages
|
2017-08-09 09:52:38 +00:00
|
|
|
from ..util import prints, get_package_path
|
2018-03-28 22:08:16 +00:00
|
|
|
from ..compat import url_read, HTTPError
|
2017-03-18 14:14:48 +00:00
|
|
|
from .. import about
|
2016-03-24 10:19:43 +00:00
|
|
|
|
|
|
|
|
2017-05-22 10:28:58 +00:00
|
|
|
@plac.annotations(
|
2017-10-27 12:38:39 +00:00
|
|
|
model=("model to download, shortcut or name)", "positional", None, str),
|
2017-05-22 10:28:58 +00:00
|
|
|
direct=("force direct download. Needs model name with version and won't "
|
2018-05-03 16:37:02 +00:00
|
|
|
"perform compatibility check", "flag", "d", bool),
|
2018-05-07 21:02:11 +00:00
|
|
|
insecure=("insecure mode - disables the verification of certificates",
|
|
|
|
"flag", "i", bool),
|
|
|
|
ca_file=("specify a certificate authority file to use for certificates "
|
|
|
|
"validation. Ignored if --insecure is used", "option", "c"))
|
|
|
|
def download(model, direct=False, insecure=False, ca_file=None):
|
2017-05-27 18:01:46 +00:00
|
|
|
"""
|
|
|
|
Download compatible model from default download path using pip. Model
|
2017-05-22 10:28:58 +00:00
|
|
|
can be shortcut, model name or, if --direct flag is set, full model name
|
|
|
|
with version.
|
2018-05-07 21:02:11 +00:00
|
|
|
The --insecure optional flag can be used to disable ssl verification
|
|
|
|
The --ca-file option can be used to provide a local CA file
|
2018-05-03 16:37:02 +00:00
|
|
|
used for certificate verification.
|
2017-05-22 10:28:58 +00:00
|
|
|
"""
|
2018-05-03 16:37:02 +00:00
|
|
|
|
2018-05-07 21:02:11 +00:00
|
|
|
# ssl_verify is the argument handled to the 'verify' parameter
|
|
|
|
# of requests package. It must be either None, a boolean,
|
|
|
|
# or a string containing the path to CA file
|
|
|
|
ssl_verify = None
|
|
|
|
if insecure:
|
|
|
|
ca_file = None
|
|
|
|
ssl_verify = False
|
2018-05-03 16:37:02 +00:00
|
|
|
else:
|
2018-05-07 21:02:11 +00:00
|
|
|
if ca_file is not None:
|
|
|
|
ssl_verify = ca_file
|
2018-05-03 16:37:02 +00:00
|
|
|
|
|
|
|
# Download the model
|
2017-03-15 16:37:18 +00:00
|
|
|
if direct:
|
2017-08-23 10:36:31 +00:00
|
|
|
dl = download_model('{m}/{m}.tar.gz'.format(m=model))
|
2017-03-15 16:37:18 +00:00
|
|
|
else:
|
2018-05-07 21:02:11 +00:00
|
|
|
shortcuts = get_json(about.__shortcuts__, "available shortcuts", ssl_verify)
|
2017-05-07 21:25:29 +00:00
|
|
|
model_name = shortcuts.get(model, model)
|
2018-05-07 21:02:11 +00:00
|
|
|
compatibility = get_compatibility(ssl_verify)
|
2017-03-16 18:54:51 +00:00
|
|
|
version = get_version(model_name, compatibility)
|
2017-10-27 12:38:39 +00:00
|
|
|
dl = download_model('{m}-{v}/{m}-{v}.tar.gz'.format(m=model_name,
|
|
|
|
v=version))
|
2018-04-03 13:50:31 +00:00
|
|
|
if dl != 0: # if download subprocess doesn't return 0, exit
|
2018-01-03 20:03:36 +00:00
|
|
|
sys.exit(dl)
|
|
|
|
try:
|
|
|
|
# Get package path here because link uses
|
|
|
|
# pip.get_installed_distributions() to check if model is a
|
|
|
|
# package, which fails if model was just installed via
|
|
|
|
# subprocess
|
|
|
|
package_path = get_package_path(model_name)
|
2018-05-07 21:02:11 +00:00
|
|
|
link(model_name, model, force=True, model_path=package_path)
|
2018-01-03 20:03:36 +00:00
|
|
|
except:
|
|
|
|
# Dirty, but since spacy.download and the auto-linking is
|
|
|
|
# mostly a convenience wrapper, it's best to show a success
|
|
|
|
# message and loading instructions, even if linking fails.
|
2018-04-03 13:50:31 +00:00
|
|
|
prints(Messages.M001.format(name=model_name), title=Messages.M002)
|
2017-03-16 22:23:26 +00:00
|
|
|
|
2018-01-03 19:14:50 +00:00
|
|
|
|
2018-05-07 21:02:11 +00:00
|
|
|
def get_json(url, desc, ssl_verify):
|
2018-03-28 10:46:07 +00:00
|
|
|
try:
|
2018-05-07 21:02:11 +00:00
|
|
|
data = url_read(url, verify=ssl_verify)
|
2018-03-28 22:08:16 +00:00
|
|
|
except HTTPError as e:
|
2018-04-03 13:50:31 +00:00
|
|
|
prints(Messages.M004.format(desc, about.__version__),
|
|
|
|
title=Messages.M003.format(e.code, e.reason), exits=1)
|
2018-03-28 22:08:16 +00:00
|
|
|
return ujson.loads(data)
|
2017-04-26 16:00:28 +00:00
|
|
|
|
|
|
|
|
2018-05-07 21:02:11 +00:00
|
|
|
def get_compatibility(ssl_verify):
|
2017-04-26 16:00:28 +00:00
|
|
|
version = about.__version__
|
2017-11-08 17:40:21 +00:00
|
|
|
version = version.rsplit('.dev', 1)[0]
|
2018-05-07 21:02:11 +00:00
|
|
|
comp_table = get_json(about.__compatibility__, "compatibility table", ssl_verify)
|
2017-04-26 16:00:28 +00:00
|
|
|
comp = comp_table['spacy']
|
2017-03-15 16:37:18 +00:00
|
|
|
if version not in comp:
|
2018-04-03 13:50:31 +00:00
|
|
|
prints(Messages.M006.format(version=version), title=Messages.M005,
|
|
|
|
exits=1)
|
2017-04-26 16:00:28 +00:00
|
|
|
return comp[version]
|
2017-03-15 16:37:18 +00:00
|
|
|
|
|
|
|
|
|
|
|
def get_version(model, comp):
|
2017-11-08 17:03:52 +00:00
|
|
|
model = model.rsplit('.dev', 1)[0]
|
2017-03-15 16:37:18 +00:00
|
|
|
if model not in comp:
|
2018-04-03 13:50:31 +00:00
|
|
|
prints(Messages.M007.format(name=model, version=about.__version__),
|
|
|
|
title=Messages.M005, exits=1)
|
2017-03-15 16:37:18 +00:00
|
|
|
return comp[model][0]
|
|
|
|
|
|
|
|
|
|
|
|
def download_model(filename):
|
2017-03-18 18:29:36 +00:00
|
|
|
download_url = about.__download_url__ + '/' + filename
|
2017-10-27 12:38:39 +00:00
|
|
|
return subprocess.call(
|
2018-01-03 16:40:37 +00:00
|
|
|
[sys.executable, '-m', 'pip', 'install', '--no-cache-dir', '--no-deps',
|
2017-10-27 12:38:39 +00:00
|
|
|
download_url], env=os.environ.copy())
|