From c1a5ece65f18b4955c8e7e72bdf815c78290d6f4 Mon Sep 17 00:00:00 2001 From: Ines Montani Date: Tue, 25 Feb 2020 15:46:39 +0100 Subject: [PATCH] Tidy up setup and update requirements tests --- .gitignore | 5 ++ pyproject.toml | 2 +- requirements.txt | 2 +- setup.py | 26 ++++++-- spacy/tests/package/test_requirements.py | 76 ++++++++++++++++++++++ spacy/tests/test_requirements.py | 83 ------------------------ 6 files changed, 102 insertions(+), 92 deletions(-) create mode 100644 spacy/tests/package/test_requirements.py delete mode 100644 spacy/tests/test_requirements.py diff --git a/.gitignore b/.gitignore index a0af6d4d2..f39607b76 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,11 @@ corpora/ keys/ *.json.gz +# Tests +spacy/tests/package/setup.cfg +spacy/tests/package/pyproject.toml +spacy/tests/package/requirements.txt + # Website website/.cache/ website/public/ diff --git a/pyproject.toml b/pyproject.toml index 8a6ababf3..8d3652a2f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,6 +6,6 @@ requires = [ "cymem>=2.0.2,<2.1.0", "preshed>=3.0.2,<3.1.0", "murmurhash>=0.28.0,<1.1.0", - "thinc==7.4.0.dev0", + "thinc==8.0.0a0", ] build-backend = "setuptools.build_meta" diff --git a/requirements.txt b/requirements.txt index bb6bf9804..f3a7cc162 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,7 +15,7 @@ plac>=0.9.6,<1.2.0 tqdm>=4.38.0,<5.0.0 # Optional dependencies jsonschema>=2.6.0,<3.1.0 -pydantic>=1.0.0,<2.0.0 +pydantic>=1.3.0,<2.0.0 # Development dependencies cython>=0.25 pytest>=4.6.5 diff --git a/setup.py b/setup.py index 31f22ba3f..d850a74ac 100755 --- a/setup.py +++ b/setup.py @@ -7,15 +7,19 @@ from distutils import ccompiler, msvccompiler from setuptools import Extension, setup, find_packages import numpy from pathlib import Path +import shutil from Cython.Build import cythonize from Cython.Compiler import Options +ROOT = Path(__file__).parent +PACKAGE_ROOT = ROOT / "spacy" + + # Preserve `__doc__` on functions and classes # http://docs.cython.org/en/latest/src/userguide/source_files_and_compilation.html#compiler-options Options.docstrings = True - PACKAGES = find_packages() MOD_NAMES = [ "spacy.parts_of_speech", @@ -60,6 +64,12 @@ COMPILER_DIRECTIVES = { "embedsignature": True, "annotation_typing": False, } +# Files to copy into the package that are otherwise not included +COPY_FILES = { + ROOT / "setup.cfg": PACKAGE_ROOT / "tests" / "package", + ROOT / "pyproject.toml": PACKAGE_ROOT / "tests" / "package", + ROOT / "requirements.txt": PACKAGE_ROOT / "tests" / "package", +} def is_new_osx(): @@ -115,25 +125,27 @@ def clean(path): def setup_package(): - root = Path(__file__).parent - if len(sys.argv) > 1 and sys.argv[1] == "clean": - return clean(root / "spacy") + return clean(PACKAGE_ROOT) - with (root / "spacy" / "about.py").open("r") as f: + with (PACKAGE_ROOT / "about.py").open("r") as f: about = {} exec(f.read(), about) + for copy_file, target_dir in COPY_FILES.items(): + shutil.copy(str(copy_file), str(target_dir)) + print(f"Copied {copy_file} -> {target_dir}") + include_dirs = [ get_python_inc(plat_specific=True), numpy.get_include(), - str(root / "include"), + str(ROOT / "include"), ] if ( ccompiler.new_compiler().compiler_type == "msvc" and msvccompiler.get_build_version() == 9 ): - include_dirs.append(str(root / "include" / "msvc9")) + include_dirs.append(str(ROOT / "include" / "msvc9")) ext_modules = [] for name in MOD_NAMES: mod_path = name.replace(".", "/") + ".pyx" diff --git a/spacy/tests/package/test_requirements.py b/spacy/tests/package/test_requirements.py new file mode 100644 index 000000000..59a8569ee --- /dev/null +++ b/spacy/tests/package/test_requirements.py @@ -0,0 +1,76 @@ +import re +from pathlib import Path + + +def test_build_dependencies(): + # Check that library requirements are pinned exactly the same across different setup files. + libs_ignore_requirements = [ + "pytest", + "pytest-timeout", + "mock", + "flake8", + "jsonschema", + ] + libs_ignore_setup = ["fugashi", "natto-py", "pythainlp"] + + # check requirements.txt + req_dict = {} + + root_dir = Path(__file__).parent + req_file = root_dir / "requirements.txt" + with req_file.open() as f: + lines = f.readlines() + for line in lines: + line = line.strip() + if not line.startswith("#"): + lib, v = _parse_req(line) + if lib and lib not in libs_ignore_requirements: + req_dict[lib] = v + # check setup.cfg and compare to requirements.txt + # also fails when there are missing or additional libs + setup_file = root_dir / "setup.cfg" + with setup_file.open() as f: + lines = f.readlines() + + setup_keys = set() + for line in lines: + line = line.strip() + if not line.startswith("#"): + lib, v = _parse_req(line) + if lib and not lib.startswith("cupy") and lib not in libs_ignore_setup: + req_v = req_dict.get(lib, None) + assert ( + req_v is not None + ), "{} in setup.cfg but not in requirements.txt".format(lib) + assert (lib + v) == (lib + req_v), ( + "{} has different version in setup.cfg and in requirements.txt: " + "{} and {} respectively".format(lib, v, req_v) + ) + setup_keys.add(lib) + assert sorted(setup_keys) == sorted( + req_dict.keys() + ) # if fail: requirements.txt contains a lib not in setup.cfg + + # check pyproject.toml and compare the versions of the libs to requirements.txt + # does not fail when there are missing or additional libs + toml_file = root_dir / "pyproject.toml" + with toml_file.open() as f: + lines = f.readlines() + for line in lines: + line = line.strip().strip(",").strip('"') + if not line.startswith("#"): + lib, v = _parse_req(line) + if lib: + req_v = req_dict.get(lib, None) + assert (lib + v) == (lib + req_v), ( + "{} has different version in pyproject.toml and in requirements.txt: " + "{} and {} respectively".format(lib, v, req_v) + ) + + +def _parse_req(line): + lib = re.match(r"^[a-z0-9\-]*", line).group(0) + v = line.replace(lib, "").strip() + if not re.match(r"^[<>=][<>=].*", v): + return None, None + return lib, v diff --git a/spacy/tests/test_requirements.py b/spacy/tests/test_requirements.py deleted file mode 100644 index fc5aeeddd..000000000 --- a/spacy/tests/test_requirements.py +++ /dev/null @@ -1,83 +0,0 @@ -import re -from pathlib import Path - - -def test_build_dependencies(en_vocab): - # Check that library requirements are pinned exactly the same across different setup files. - libs_ignore_requirements = ["pytest", "pytest-timeout", "mock", "flake8", "jsonschema"] - libs_ignore_setup = ["fugashi", "natto-py", "pythainlp"] - - # check requirements.txt - req_dict = {} - - root_dir = None - # when running tests locally, the file is 3 levels up. On the CI, it's 2 levels up. - roots = [Path(__file__).parent.parent, Path(__file__).parent.parent.parent] # or whatever - print() - for r in roots: - print("inspecting dir", r, "-->", [f.name for f in r.glob(pattern="*.*")]) - req_file = r / "requirements.txt" - if req_file.exists(): - root_dir = r - with req_file.open() as f: - lines = f.readlines() - for line in lines: - line = line.strip() - if not line.startswith("#"): - lib, v = _parse_req(line) - if lib and lib not in libs_ignore_requirements: - req_dict[lib] = v - - assert root_dir is not None, "Could not find the root directory of requirements.txt" - - # check setup.cfg and compare to requirements.txt - # also fails when there are missing or additional libs - setup_file = root_dir / "setup.cfg" - with setup_file.open() as f: - lines = f.readlines() - - # import configparser - # config = configparser.ConfigParser() - # config.read(setup_file) - # print("SECTIONS", config.sections()) - # print("options", config['options']) - # for key in config['options']: - # print("key", key) - # print("setup_requires *", config['options']['setup_requires'], "*") - # lines = config['options']['setup_requires'] - # lines += config['options']['install_requires'] - - setup_keys = set() - for line in lines: - line = line.strip() - if not line.startswith("#"): - lib, v = _parse_req(line) - if lib and not lib.startswith("cupy") and lib not in libs_ignore_setup: - req_v = req_dict.get(lib, None) - assert req_v is not None, "{} in setup.cfg but not in requirements.txt".format(lib) - assert (lib+v) == (lib+req_v), "{} has different version in setup.cfg and in requirements.txt: " \ - "{} and {} respectively".format(lib, v, req_v) - setup_keys.add(lib) - assert sorted(setup_keys) == sorted(req_dict.keys()) # if fail: requirements.txt contains a lib not in setup.cfg - - # check pyproject.toml and compare the versions of the libs to requirements.txt - # does not fail when there are missing or additional libs - toml_file = root_dir / "pyproject.toml" - with toml_file.open() as f: - lines = f.readlines() - for line in lines: - line = line.strip().strip(",").strip("\"") - if not line.startswith("#"): - lib, v = _parse_req(line) - if lib: - req_v = req_dict.get(lib, None) - assert (lib+v) == (lib+req_v), "{} has different version in pyproject.toml and in requirements.txt: " \ - "{} and {} respectively".format(lib, v, req_v) - - -def _parse_req(line): - lib = re.match(r"^[a-z0-9\-]*", line).group(0) - v = line.replace(lib, "").strip() - if not re.match(r"^[<>=][<>=].*", v): - return None, None - return lib, v