diff --git a/requirements.txt b/requirements.txt index add083a05..a104b68ba 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,8 +13,11 @@ numpy>=1.15.0 requests>=2.13.0,<3.0.0 plac>=0.9.6,<1.2.0 tqdm>=4.38.0,<5.0.0 -importlib_metadata>=0.20; python_version < "3.8" pydantic>=1.3.0,<2.0.0 +# Official Python utilities +setuptools +packaging +importlib_metadata>=0.20; python_version < "3.8" # Development dependencies cython>=0.25 pytest>=4.6.5 diff --git a/setup.cfg b/setup.cfg index c5c39b447..ae09d071c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -50,11 +50,13 @@ install_requires = ml_datasets>=0.1.1 # Third-party dependencies tqdm>=4.38.0,<5.0.0 - setuptools numpy>=1.15.0 plac>=0.9.6,<1.2.0 requests>=2.13.0,<3.0.0 pydantic>=1.3.0,<2.0.0 + # Official Python utilities + setuptools + packaging importlib_metadata>=0.20; python_version < "3.8" [options.extras_require] diff --git a/spacy/cli/download.py b/spacy/cli/download.py index af132bbbe..3d56822a5 100644 --- a/spacy/cli/download.py +++ b/spacy/cli/download.py @@ -5,7 +5,7 @@ import sys from wasabi import msg from .. import about -from ..util import is_package +from ..util import is_package, get_base_version def download( @@ -63,8 +63,7 @@ def get_json(url, desc): def get_compatibility(): - version = about.__version__ - version = version.rsplit(".dev", 1)[0] + version = get_base_version(about.__version__) comp_table = get_json(about.__compatibility__, "compatibility table") comp = comp_table["spacy"] if version not in comp: @@ -73,7 +72,7 @@ def get_compatibility(): def get_version(model, comp): - model = model.rsplit(".dev", 1)[0] + model = get_base_version(model) if model not in comp: msg.fail( f"No compatible model found for '{model}' (spaCy v{about.__version__})", diff --git a/spacy/cli/package.py b/spacy/cli/package.py index cf93c872f..153e61ba3 100644 --- a/spacy/cli/package.py +++ b/spacy/cli/package.py @@ -90,7 +90,7 @@ def generate_meta(model_path, existing_meta, msg): ("license", "License", meta.get("license", "MIT")), ] nlp = util.load_model_from_path(Path(model_path)) - meta["spacy_version"] = about.__version__ + meta["spacy_version"] = util.get_model_version_range(about.__version__) meta["pipeline"] = nlp.pipe_names meta["vectors"] = { "width": nlp.vocab.vectors_length, diff --git a/spacy/cli/train.py b/spacy/cli/train.py index c205fa5b2..590ce4f13 100644 --- a/spacy/cli/train.py +++ b/spacy/cli/train.py @@ -467,7 +467,6 @@ def train( # Update model meta.json meta["lang"] = nlp.lang meta["pipeline"] = nlp.pipe_names - meta["spacy_version"] = about.__version__ if beam_width == 1: meta["speed"] = { "nwords": nwords, diff --git a/spacy/cli/validate.py b/spacy/cli/validate.py index c39cadc7b..080cd77e2 100644 --- a/spacy/cli/validate.py +++ b/spacy/cli/validate.py @@ -4,8 +4,8 @@ import requests from wasabi import msg from .. import about -from ..util import get_package_version, get_installed_models, split_version -from ..util import get_package_path, get_model_meta, is_compatible_model +from ..util import get_package_version, get_installed_models, get_base_version +from ..util import get_package_path, get_model_meta, is_compatible_version def validate(): @@ -14,7 +14,7 @@ def validate(): with the installed models. Should be run after `pip install -U spacy`. """ model_pkgs, compat = get_model_pkgs() - spacy_version = about.__version__.rsplit(".dev", 1)[0] + spacy_version = get_base_version(about.__version__) current_compat = compat.get(spacy_version, {}) if not current_compat: msg.warn(f"No compatible models found for v{spacy_version} of spaCy") @@ -78,13 +78,12 @@ def get_model_pkgs(): version = get_package_version(pkg_name) if package in compat: is_compat = version in compat[package] - v_maj, v_min = split_version(about.__version__) - spacy_version = f"{v_maj}.{v_min}" + spacy_version = about.__version__ else: model_path = get_package_path(package) model_meta = get_model_meta(model_path) - is_compat = is_compatible_model(model_meta) spacy_version = model_meta.get("spacy_version", "n/a") + is_compat = is_compatible_version(about.__version__, spacy_version) pkgs[pkg_name] = { "name": package, "version": version, diff --git a/spacy/errors.py b/spacy/errors.py index 932bb1eff..da2cfdf04 100644 --- a/spacy/errors.py +++ b/spacy/errors.py @@ -104,6 +104,12 @@ class Warnings(object): "string \"Field1=Value1,Value2|Field2=Value3\".") # TODO: fix numbering after merging develop into master + W095 = ("Model '{model}' ({model_version}) requires spaCy {version} and is " + "incompatible with the current version ({current}). This may lead " + "to unexpected results or runtime errors. To resolve this, " + "download a newer compatible model or retrain your custom model " + "with the current spaCy version. For more details and available " + "updates, run: python -m spacy validate") W096 = ("The method 'disable_pipes' has become deprecated - use 'select_pipes' " "instead.") W097 = ("No Model config was provided to create the '{name}' component, " diff --git a/spacy/language.py b/spacy/language.py index 551b8c9af..61d69b63e 100644 --- a/spacy/language.py +++ b/spacy/language.py @@ -191,13 +191,14 @@ class Language(object): @property def meta(self): + spacy_version = util.get_model_version_range(about.__version__) if self.vocab.lang: self._meta.setdefault("lang", self.vocab.lang) else: self._meta.setdefault("lang", self.lang) self._meta.setdefault("name", "model") self._meta.setdefault("version", "0.0.0") - self._meta.setdefault("spacy_version", about.__version__) + self._meta.setdefault("spacy_version", spacy_version) self._meta.setdefault("description", "") self._meta.setdefault("author", "") self._meta.setdefault("email", "") diff --git a/spacy/tests/test_misc.py b/spacy/tests/test_misc.py index 9e67ae83b..e4b4e570c 100644 --- a/spacy/tests/test_misc.py +++ b/spacy/tests/test_misc.py @@ -94,8 +94,18 @@ def test_ascii_filenames(): @pytest.mark.parametrize( - "version,compatible", - [(spacy_version, True), ("2.0.0", False), (">=1.2.3,<4.5.6", False)], + "version,constraint,compatible", + [ + (spacy_version, spacy_version, True), + (spacy_version, f">={spacy_version}", True), + ("3.0.0", "2.0.0", False), + ("3.2.1", ">=2.0.0", True), + ("2.2.10a1", ">=1.0.0,<2.1.1", False), + ("3.0.0.dev3", ">=1.2.3,<4.5.6", True), + ("n/a", ">=1.2.3,<4.5.6", None), + ("1.2.3", "n/a", None), + ("n/a", "n/a", None), + ], ) -def test_is_compatible_model(version, compatible): - assert util.is_compatible_model({"spacy_version": version}) is compatible +def test_is_compatible_version(version, constraint, compatible): + assert util.is_compatible_version(version, constraint) is compatible diff --git a/spacy/util.py b/spacy/util.py index b614c29c7..79134400c 100644 --- a/spacy/util.py +++ b/spacy/util.py @@ -14,6 +14,8 @@ import srsly import catalogue import sys import warnings +from packaging.specifiers import SpecifierSet, InvalidSpecifier +from packaging.version import Version, InvalidVersion try: @@ -236,33 +238,46 @@ def get_package_version(name): return None -def split_version(version): - """RETURNS (tuple): Two integers, the major and minor spaCy version.""" - pieces = version.split(".", 3) - return int(pieces[0]), int(pieces[1]) +def is_compatible_version(version, constraint, prereleases=True): + """Check if a version (e.g. "2.0.0") is compatible given a version + constraint (e.g. ">=1.9.0,<2.2.1"). If the constraint is a specific version, + it's interpreted as =={version}. - -def is_compatible_model(meta): - """Check if a model is compatible with the current version of spaCy, based - on its meta.json. We compare the version of spaCy the model was created with - with the current version. If the minor version is different, it's considered - incompatible. - - meta (dict): The model's meta. - RETURNS (bool / None): Whether the model is compatible with the current - spaCy or None if we don't have enough info. + version (str): The version to check. + constraint (str): The constraint string. + prereleases (bool): Whether to allow prereleases. If set to False, + prerelease versions will be considered incompatible. + RETURNS (bool / None): Whether the version is compatible, or None if the + version or constraint are invalid. """ - cur_v = about.__version__ - pkg_v = meta.get("spacy_version") - if not pkg_v or not isinstance(pkg_v, str): + # Handle cases where exact version is provided as constraint + if constraint[0].isdigit(): + constraint = f"=={constraint}" + try: + spec = SpecifierSet(constraint) + version = Version(version) + except (InvalidSpecifier, InvalidVersion): return None - # Handle spacy_version values like >=x,=1.2.3,<1.3.0 based on a given spaCy + version. Models are always compatible across patch versions but not + across minor or major versions. + """ + release = Version(spacy_version).release + return f">={spacy_version},<{release[0]}.{release[1] + 1}.0" + + +def get_base_version(version): + """Generate the base version without any prerelease identifiers. + + version (str): The version, e.g. "3.0.0.dev1". + RETURNS (str): The base version, e.g. "3.0.0". + """ + return Version(version).base_version def load_config(path, create_objects=False): @@ -315,6 +330,16 @@ def get_model_meta(path): for setting in ["lang", "name", "version"]: if setting not in meta or not meta[setting]: raise ValueError(Errors.E054.format(setting=setting)) + if "spacy_version" in meta: + if not is_compatible_version(about.__version__, meta["spacy_version"]): + warnings.warn( + Warnings.W095.format( + model=f"{meta['lang']}_{meta['name']}", + model_version=meta["version"], + version=meta["spacy_version"], + current=about.__version__, + ) + ) return meta