spaCy/spacy/deprecated.py

161 lines
5.6 KiB
Python
Raw Normal View History

2017-04-15 11:05:15 +00:00
# coding: utf8
from __future__ import unicode_literals
from pathlib import Path
from . import about
from . import util
2017-05-07 21:25:29 +00:00
from .util import prints
from .compat import path2str
from .cli import download
from .cli import link
def read_lang_data(package):
tokenization = package.load_json(('tokenizer', 'specials.json'))
with package.open(('tokenizer', 'prefix.txt'), default=None) as file_:
prefix = read_prefix(file_) if file_ is not None else None
with package.open(('tokenizer', 'suffix.txt'), default=None) as file_:
suffix = read_suffix(file_) if file_ is not None else None
with package.open(('tokenizer', 'infix.txt'), default=None) as file_:
infix = read_infix(file_) if file_ is not None else None
return tokenization, prefix, suffix, infix
def align_tokens(ref, indices): # Deprecated, surely?
start = 0
queue = list(indices)
for token in ref:
end = start + len(token)
emit = []
while queue and queue[0][1] <= end:
emit.append(queue.pop(0))
yield token, emit
start = end
assert not queue
def detokenize(token_rules, words): # Deprecated?
"""
To align with treebanks, return a list of "chunks", where a chunk is a
sequence of tokens that are separated by whitespace in actual strings. Each
chunk should be a tuple of token indices, e.g.
>>> detokenize(["ca<SEP>n't", '<SEP>!'], ["I", "ca", "n't", "!"])
[(0,), (1, 2, 3)]
"""
string = ' '.join(words)
for subtoks in token_rules:
# Algorithmically this is dumb, but writing a little list-based match
# machine? Ain't nobody got time for that.
string = string.replace(subtoks.replace('<SEP>', ' '), subtoks)
positions = []
i = 0
for chunk in string.split():
subtoks = chunk.split('<SEP>')
positions.append(tuple(range(i, i+len(subtoks))))
i += len(subtoks)
return positions
2017-04-15 10:12:31 +00:00
def match_best_version(target_name, target_version, path):
path = util.ensure_path(path)
if path is None or not path.exists():
return None
matches = []
for data_name in path.iterdir():
name, version = split_data_name(data_name.parts[-1])
if name == target_name:
matches.append((tuple(float(v) for v in version.split('.')), data_name))
if matches:
return Path(max(matches)[1])
else:
return None
def split_data_name(name):
return name.split('-', 1) if '-' in name else (name, '')
def fix_glove_vectors_loading(overrides):
"""
Special-case hack for loading the GloVe vectors, to support deprecated
<1.0 stuff. Phase this out once the data is fixed.
"""
if 'data_dir' in overrides and 'path' not in overrides:
raise ValueError("The argument 'data_dir' has been renamed to 'path'")
if overrides.get('path') is False:
return overrides
if overrides.get('path') in (None, True):
data_path = util.get_data_path()
else:
path = util.ensure_path(overrides['path'])
data_path = path.parent
vec_path = None
if 'add_vectors' not in overrides:
if 'vectors' in overrides:
2017-04-15 10:12:31 +00:00
vec_path = match_best_version(overrides['vectors'], None, data_path)
if vec_path is None:
return overrides
else:
2017-04-15 10:12:31 +00:00
vec_path = match_best_version('en_glove_cc_300_1m_vectors', None, data_path)
if vec_path is not None:
vec_path = vec_path / 'vocab' / 'vec.bin'
if vec_path is not None:
overrides['add_vectors'] = lambda vocab: vocab.load_vectors_from_bin_loc(vec_path)
return overrides
def resolve_model_name(name):
"""
If spaCy is loaded with 'de', check if symlink already exists. If
2017-04-16 18:34:37 +00:00
not, user may have upgraded from older version and have old models installed.
Check if old model directory exists and if so, return that instead and create
2017-03-17 18:26:28 +00:00
shortcut link. If English model is found and no shortcut exists, raise error
and tell user to install new model.
"""
if name == 'en' or name == 'de':
versions = ['1.0.0', '1.1.0']
2017-05-07 21:25:29 +00:00
data_path = util.get_data_path()
model_path = data_path / name
2017-05-07 21:25:29 +00:00
v_model_paths = [data_path / '%s-%s' % (name, v) for v in versions]
2017-03-17 18:26:28 +00:00
if not model_path.exists(): # no shortcut found
for v_path in v_model_paths:
2017-03-17 18:26:28 +00:00
if v_path.exists(): # versioned model directory found
if name == 'de':
link(v_path, name)
return name
else:
raise ValueError(
2017-05-07 21:25:29 +00:00
"Found English model at %s. This model is not "
2017-03-17 18:26:28 +00:00
"compatible with the current version. See "
"https://spacy.io/docs/usage/models to download the "
2017-05-07 21:25:29 +00:00
"new model." % path2str(v_path))
return name
class ModelDownload():
"""
Replace download modules within en and de with deprecation warning and
download default language model (using shortcut). Use classmethods to allow
importing ModelDownload as download and calling download.en() etc.
"""
@classmethod
def load(self, lang):
2017-05-07 21:25:29 +00:00
prints("The spacy.%s.download command is now deprecated. Please use "
"python -m spacy download [model name or shortcut] instead. For "
"more info, see the docs: %s." % (lang, about.__docs__),
"Downloading default '%s' model now..." % lang,
title="Warning: deprecated command")
download(lang)
@classmethod
def en(cls, *args, **kwargs):
cls.load('en')
@classmethod
def de(cls, *args, **kwargs):
cls.load('de')