Tidy up setup and update requirements tests

.gitignore

@@ -5,6 +5,11 @@ corpora/
 keys/
 *.json.gz
 
+# Tests
+spacy/tests/package/setup.cfg
+spacy/tests/package/pyproject.toml
+spacy/tests/package/requirements.txt
+
 # Website
 website/.cache/
 website/public/

pyproject.toml

@@ -6,6 +6,6 @@ requires = [
     "cymem>=2.0.2,<2.1.0",
     "preshed>=3.0.2,<3.1.0",
     "murmurhash>=0.28.0,<1.1.0",
-    "thinc==7.4.0.dev0",
+    "thinc==8.0.0a0",
 ]
 build-backend = "setuptools.build_meta"

requirements.txt

@@ -15,7 +15,7 @@ plac>=0.9.6,<1.2.0
 tqdm>=4.38.0,<5.0.0
 # Optional dependencies
 jsonschema>=2.6.0,<3.1.0
-pydantic>=1.0.0,<2.0.0
+pydantic>=1.3.0,<2.0.0
 # Development dependencies
 cython>=0.25
 pytest>=4.6.5

setup.py

@@ -7,15 +7,19 @@ from distutils import ccompiler, msvccompiler
 from setuptools import Extension, setup, find_packages
 import numpy
 from pathlib import Path
+import shutil
 from Cython.Build import cythonize
 from Cython.Compiler import Options
 
 
+ROOT = Path(__file__).parent
+PACKAGE_ROOT = ROOT / "spacy"
+
+
 # Preserve `__doc__` on functions and classes
 # http://docs.cython.org/en/latest/src/userguide/source_files_and_compilation.html#compiler-options
 Options.docstrings = True
 
-
 PACKAGES = find_packages()
 MOD_NAMES = [
     "spacy.parts_of_speech",
@@ -60,6 +64,12 @@ COMPILER_DIRECTIVES = {
     "embedsignature": True,
     "annotation_typing": False,
 }
+# Files to copy into the package that are otherwise not included
+COPY_FILES = {
+    ROOT / "setup.cfg": PACKAGE_ROOT / "tests" / "package",
+    ROOT / "pyproject.toml": PACKAGE_ROOT / "tests" / "package",
+    ROOT / "requirements.txt": PACKAGE_ROOT / "tests" / "package",
+}
 
 
 def is_new_osx():
@@ -115,25 +125,27 @@ def clean(path):
 
 
 def setup_package():
-    root = Path(__file__).parent
-
     if len(sys.argv) > 1 and sys.argv[1] == "clean":
-        return clean(root / "spacy")
+        return clean(PACKAGE_ROOT)
 
-    with (root / "spacy" / "about.py").open("r") as f:
+    with (PACKAGE_ROOT / "about.py").open("r") as f:
         about = {}
         exec(f.read(), about)
 
+    for copy_file, target_dir in COPY_FILES.items():
+        shutil.copy(str(copy_file), str(target_dir))
+        print(f"Copied {copy_file} -> {target_dir}")
+
     include_dirs = [
         get_python_inc(plat_specific=True),
         numpy.get_include(),
-        str(root / "include"),
+        str(ROOT / "include"),
     ]
     if (
         ccompiler.new_compiler().compiler_type == "msvc"
         and msvccompiler.get_build_version() == 9
     ):
-        include_dirs.append(str(root / "include" / "msvc9"))
+        include_dirs.append(str(ROOT / "include" / "msvc9"))
     ext_modules = []
     for name in MOD_NAMES:
         mod_path = name.replace(".", "/") + ".pyx"

spacy/tests/package/test_requirements.py

@@ -0,0 +1,76 @@
+import re
+from pathlib import Path
+
+
+def test_build_dependencies():
+    # Check that library requirements are pinned exactly the same across different setup files.
+    libs_ignore_requirements = [
+        "pytest",
+        "pytest-timeout",
+        "mock",
+        "flake8",
+        "jsonschema",
+    ]
+    libs_ignore_setup = ["fugashi", "natto-py", "pythainlp"]
+
+    # check requirements.txt
+    req_dict = {}
+
+    root_dir = Path(__file__).parent
+    req_file = root_dir / "requirements.txt"
+    with req_file.open() as f:
+        lines = f.readlines()
+        for line in lines:
+            line = line.strip()
+            if not line.startswith("#"):
+                lib, v = _parse_req(line)
+                if lib and lib not in libs_ignore_requirements:
+                    req_dict[lib] = v
+    # check setup.cfg and compare to requirements.txt
+    # also fails when there are missing or additional libs
+    setup_file = root_dir / "setup.cfg"
+    with setup_file.open() as f:
+        lines = f.readlines()
+
+    setup_keys = set()
+    for line in lines:
+        line = line.strip()
+        if not line.startswith("#"):
+            lib, v = _parse_req(line)
+            if lib and not lib.startswith("cupy") and lib not in libs_ignore_setup:
+                req_v = req_dict.get(lib, None)
+                assert (
+                    req_v is not None
+                ), "{} in setup.cfg but not in requirements.txt".format(lib)
+                assert (lib + v) == (lib + req_v), (
+                    "{} has different version in setup.cfg and in requirements.txt: "
+                    "{} and {} respectively".format(lib, v, req_v)
+                )
+                setup_keys.add(lib)
+    assert sorted(setup_keys) == sorted(
+        req_dict.keys()
+    )  # if fail: requirements.txt contains a lib not in setup.cfg
+
+    # check pyproject.toml and compare the versions of the libs to requirements.txt
+    # does not fail when there are missing or additional libs
+    toml_file = root_dir / "pyproject.toml"
+    with toml_file.open() as f:
+        lines = f.readlines()
+    for line in lines:
+        line = line.strip().strip(",").strip('"')
+        if not line.startswith("#"):
+            lib, v = _parse_req(line)
+            if lib:
+                req_v = req_dict.get(lib, None)
+                assert (lib + v) == (lib + req_v), (
+                    "{} has different version in pyproject.toml and in requirements.txt: "
+                    "{} and {} respectively".format(lib, v, req_v)
+                )
+
+
+def _parse_req(line):
+    lib = re.match(r"^[a-z0-9\-]*", line).group(0)
+    v = line.replace(lib, "").strip()
+    if not re.match(r"^[<>=][<>=].*", v):
+        return None, None
+    return lib, v

spacy/tests/test_requirements.py

@@ -1,83 +0,0 @@
-import re
-from pathlib import Path
-
-
-def test_build_dependencies(en_vocab):
-    # Check that library requirements are pinned exactly the same across different setup files.
-    libs_ignore_requirements = ["pytest", "pytest-timeout", "mock", "flake8", "jsonschema"]
-    libs_ignore_setup = ["fugashi", "natto-py", "pythainlp"]
-
-    # check requirements.txt
-    req_dict = {}
-
-    root_dir = None
-    # when running tests locally, the file is 3 levels up. On the CI, it's 2 levels up.
-    roots = [Path(__file__).parent.parent, Path(__file__).parent.parent.parent]  # or whatever
-    print()
-    for r in roots:
-        print("inspecting dir", r, "-->", [f.name for f in r.glob(pattern="*.*")])
-        req_file = r / "requirements.txt"
-        if req_file.exists():
-            root_dir = r
-            with req_file.open() as f:
-                lines = f.readlines()
-                for line in lines:
-                    line = line.strip()
-                    if not line.startswith("#"):
-                        lib, v = _parse_req(line)
-                        if lib and lib not in libs_ignore_requirements:
-                            req_dict[lib] = v
-
-    assert root_dir is not None, "Could not find the root directory of requirements.txt"
-
-    # check setup.cfg and compare to requirements.txt
-    # also fails when there are missing or additional libs
-    setup_file = root_dir / "setup.cfg"
-    with setup_file.open() as f:
-        lines = f.readlines()
-
-    # import configparser
-    # config = configparser.ConfigParser()
-    # config.read(setup_file)
-    # print("SECTIONS", config.sections())
-    # print("options", config['options'])
-    # for key in config['options']:
-    #     print("key", key)
-    # print("setup_requires *", config['options']['setup_requires'], "*")
-    # lines = config['options']['setup_requires']
-    # lines += config['options']['install_requires']
-
-    setup_keys = set()
-    for line in lines:
-        line = line.strip()
-        if not line.startswith("#"):
-            lib, v = _parse_req(line)
-            if lib and not lib.startswith("cupy") and lib not in libs_ignore_setup:
-                req_v = req_dict.get(lib, None)
-                assert req_v is not None, "{} in setup.cfg but not in requirements.txt".format(lib)
-                assert (lib+v) == (lib+req_v), "{} has different version in setup.cfg and in requirements.txt: " \
-                                               "{} and {} respectively".format(lib, v, req_v)
-                setup_keys.add(lib)
-    assert sorted(setup_keys) == sorted(req_dict.keys())  # if fail: requirements.txt contains a lib not in setup.cfg
-
-    # check pyproject.toml and compare the versions of the libs to requirements.txt
-    # does not fail when there are missing or additional libs
-    toml_file = root_dir / "pyproject.toml"
-    with toml_file.open() as f:
-        lines = f.readlines()
-    for line in lines:
-        line = line.strip().strip(",").strip("\"")
-        if not line.startswith("#"):
-            lib, v = _parse_req(line)
-            if lib:
-                req_v = req_dict.get(lib, None)
-                assert (lib+v) == (lib+req_v), "{} has different version in pyproject.toml and in requirements.txt: " \
-                                               "{} and {} respectively".format(lib, v, req_v)
-
-
-def _parse_req(line):
-    lib = re.match(r"^[a-z0-9\-]*", line).group(0)
-    v = line.replace(lib, "").strip()
-    if not re.match(r"^[<>=][<>=].*", v):
-        return None, None
-    return lib, v