From 207994871106bde872bd76224fbb4cf195f01e66 Mon Sep 17 00:00:00 2001 From: svlandeg Date: Wed, 12 Feb 2020 22:49:21 +0100 Subject: [PATCH 01/20] add build dependencies back to pyproject.toml --- pyproject.toml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index fed528d4a..8a6ababf3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,11 @@ [build-system] -requires = ["setuptools"] +requires = [ + "setuptools", + "wheel", + "cython>=0.25", + "cymem>=2.0.2,<2.1.0", + "preshed>=3.0.2,<3.1.0", + "murmurhash>=0.28.0,<1.1.0", + "thinc==7.4.0.dev0", +] build-backend = "setuptools.build_meta" From 34986c7bfd1d4634861a5c4b54cf90ef18090ff4 Mon Sep 17 00:00:00 2001 From: svlandeg Date: Wed, 12 Feb 2020 22:49:50 +0100 Subject: [PATCH 02/20] test versions of required libs across different places --- spacy/tests/test_requirements.py | 61 ++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 spacy/tests/test_requirements.py diff --git a/spacy/tests/test_requirements.py b/spacy/tests/test_requirements.py new file mode 100644 index 000000000..8c2b49b85 --- /dev/null +++ b/spacy/tests/test_requirements.py @@ -0,0 +1,61 @@ +import re +from pathlib import Path + + +def test_build_dependencies(en_vocab): + libs_ignore_requirements = ["pytest", "pytest-timeout", "mock", "flake8", "jsonschema"] + libs_ignore_setup = ["fugashi", "natto-py", "pythainlp"] + + # check requirements.txt + root_dir = Path(__file__).parent.parent.parent + req_file = root_dir / "requirements.txt" + req_dict = {} + with req_file.open() as f: + lines = f.readlines() + for line in lines: + line = line.strip() + if not line.startswith("#"): + lib, v = _parse_req(line) + if lib and lib not in libs_ignore_requirements: + req_dict[lib] = v + + # check setup.cfg and compare to requirements.txt + # also fails when there are missing or additional libs + setup_file = root_dir / "setup.cfg" + with setup_file.open() as f: + lines = f.readlines() + setup_keys = set() + for line in lines: + line = line.strip() + if not line.startswith("#"): + lib, v = _parse_req(line) + if lib and not lib.startswith("cupy") and lib not in libs_ignore_setup: + req_v = req_dict.get(lib, None) + assert req_v is not None # if fail: setup.cfg contains a lib not in requirements.txt + assert (lib+v) == (lib+req_v) # if fail: setup.cfg & requirements.txt have conflicting versions + setup_keys.add(lib) + assert sorted(setup_keys) == sorted(req_dict.keys()) # if fail: requirements.txt contains a lib not in setup.cfg + + # check pyproject.toml and compare the versions of the libs to requirements.txt + # does not fail when there are missing or additional libs + toml_file = root_dir / "pyproject.toml" + with toml_file.open() as f: + lines = f.readlines() + toml_keys = set() + for line in lines: + line = line.strip() + line = line.strip(",") + line = line.strip("\"") + if not line.startswith("#"): + lib, v = _parse_req(line) + if lib: + req_v = req_dict.get(lib, None) + assert (lib+v) == (lib+req_v) # if fail: pyproject.toml & requirements.txt have conflicting versions + toml_keys.add(lib) + +def _parse_req(line): + lib = re.match(r"^[a-z0-9\-]*", line).group(0) + v = line.replace(lib, "").strip() + if not re.match(r"^[<>=][<>=].*", v): + return None, None + return lib, v \ No newline at end of file From 6bbd81656967fd93dfeb9af40c9194536b31a135 Mon Sep 17 00:00:00 2001 From: svlandeg Date: Wed, 12 Feb 2020 22:50:27 +0100 Subject: [PATCH 03/20] formatting --- spacy/tests/test_requirements.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/spacy/tests/test_requirements.py b/spacy/tests/test_requirements.py index 8c2b49b85..cb2f51725 100644 --- a/spacy/tests/test_requirements.py +++ b/spacy/tests/test_requirements.py @@ -34,7 +34,7 @@ def test_build_dependencies(en_vocab): assert req_v is not None # if fail: setup.cfg contains a lib not in requirements.txt assert (lib+v) == (lib+req_v) # if fail: setup.cfg & requirements.txt have conflicting versions setup_keys.add(lib) - assert sorted(setup_keys) == sorted(req_dict.keys()) # if fail: requirements.txt contains a lib not in setup.cfg + assert sorted(setup_keys) == sorted(req_dict.keys()) # if fail: requirements.txt contains a lib not in setup.cfg # check pyproject.toml and compare the versions of the libs to requirements.txt # does not fail when there are missing or additional libs @@ -53,9 +53,10 @@ def test_build_dependencies(en_vocab): assert (lib+v) == (lib+req_v) # if fail: pyproject.toml & requirements.txt have conflicting versions toml_keys.add(lib) + def _parse_req(line): lib = re.match(r"^[a-z0-9\-]*", line).group(0) v = line.replace(lib, "").strip() if not re.match(r"^[<>=][<>=].*", v): return None, None - return lib, v \ No newline at end of file + return lib, v From 2729d9164d02a6795ccf93f0b9414856644e6dbc Mon Sep 17 00:00:00 2001 From: svlandeg Date: Wed, 12 Feb 2020 22:59:37 +0100 Subject: [PATCH 04/20] cleanup --- spacy/tests/test_requirements.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/spacy/tests/test_requirements.py b/spacy/tests/test_requirements.py index cb2f51725..320fc5763 100644 --- a/spacy/tests/test_requirements.py +++ b/spacy/tests/test_requirements.py @@ -3,6 +3,7 @@ from pathlib import Path def test_build_dependencies(en_vocab): + # Check that library requirements are pinned exactly the same across different setup files. libs_ignore_requirements = ["pytest", "pytest-timeout", "mock", "flake8", "jsonschema"] libs_ignore_setup = ["fugashi", "natto-py", "pythainlp"] @@ -41,17 +42,13 @@ def test_build_dependencies(en_vocab): toml_file = root_dir / "pyproject.toml" with toml_file.open() as f: lines = f.readlines() - toml_keys = set() for line in lines: - line = line.strip() - line = line.strip(",") - line = line.strip("\"") + line = line.strip().strip(",").strip("\"") if not line.startswith("#"): lib, v = _parse_req(line) if lib: req_v = req_dict.get(lib, None) assert (lib+v) == (lib+req_v) # if fail: pyproject.toml & requirements.txt have conflicting versions - toml_keys.add(lib) def _parse_req(line): From b20351792acba1bcd28998bed80171f5b6caa59f Mon Sep 17 00:00:00 2001 From: svlandeg Date: Wed, 19 Feb 2020 15:51:53 +0200 Subject: [PATCH 05/20] assert prints for more clarity --- spacy/tests/test_requirements.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/spacy/tests/test_requirements.py b/spacy/tests/test_requirements.py index 320fc5763..aaa562722 100644 --- a/spacy/tests/test_requirements.py +++ b/spacy/tests/test_requirements.py @@ -32,8 +32,9 @@ def test_build_dependencies(en_vocab): lib, v = _parse_req(line) if lib and not lib.startswith("cupy") and lib not in libs_ignore_setup: req_v = req_dict.get(lib, None) - assert req_v is not None # if fail: setup.cfg contains a lib not in requirements.txt - assert (lib+v) == (lib+req_v) # if fail: setup.cfg & requirements.txt have conflicting versions + assert req_v is not None, "{} in setup.cfg but not in requirements.txt".format(lib) + assert (lib+v) == (lib+req_v), "{} has different version in setup.cfg and in requirements.txt: " \ + "{} and {} respectively".format(lib, v, req_v) setup_keys.add(lib) assert sorted(setup_keys) == sorted(req_dict.keys()) # if fail: requirements.txt contains a lib not in setup.cfg @@ -48,7 +49,8 @@ def test_build_dependencies(en_vocab): lib, v = _parse_req(line) if lib: req_v = req_dict.get(lib, None) - assert (lib+v) == (lib+req_v) # if fail: pyproject.toml & requirements.txt have conflicting versions + assert (lib+v) == (lib+req_v), "{} has different version in pyproject.toml and in requirements.txt: " \ + "{} and {} respectively".format(lib, v, req_v) def _parse_req(line): From 303c4bcd4ca50569f7987c980ff2e4eb7e9c8a63 Mon Sep 17 00:00:00 2001 From: svlandeg Date: Wed, 19 Feb 2020 15:52:55 +0200 Subject: [PATCH 06/20] include requirements in manifest --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) diff --git a/MANIFEST.in b/MANIFEST.in index 1947b9140..64886cd19 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -4,5 +4,6 @@ include LICENSE include README.md include bin/spacy include pyproject.toml +include requirements.txt recursive-exclude spacy/lang *.json recursive-include spacy/lang *.json.gz From 5c2f6454706b4522cc58efc8ffca132caeba27f9 Mon Sep 17 00:00:00 2001 From: svlandeg Date: Wed, 19 Feb 2020 16:15:56 +0200 Subject: [PATCH 07/20] root dir one level up --- spacy/tests/test_requirements.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spacy/tests/test_requirements.py b/spacy/tests/test_requirements.py index aaa562722..5bbccf362 100644 --- a/spacy/tests/test_requirements.py +++ b/spacy/tests/test_requirements.py @@ -8,7 +8,7 @@ def test_build_dependencies(en_vocab): libs_ignore_setup = ["fugashi", "natto-py", "pythainlp"] # check requirements.txt - root_dir = Path(__file__).parent.parent.parent + root_dir = Path(__file__).parent.parent req_file = root_dir / "requirements.txt" req_dict = {} with req_file.open() as f: From 9834527f2c373708252c37998b7573291fc9da63 Mon Sep 17 00:00:00 2001 From: svlandeg Date: Wed, 19 Feb 2020 16:22:48 +0200 Subject: [PATCH 08/20] hack to switch between CLI folder setup and local setup --- spacy/tests/test_requirements.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/spacy/tests/test_requirements.py b/spacy/tests/test_requirements.py index 5bbccf362..a6fa20d6b 100644 --- a/spacy/tests/test_requirements.py +++ b/spacy/tests/test_requirements.py @@ -8,11 +8,21 @@ def test_build_dependencies(en_vocab): libs_ignore_setup = ["fugashi", "natto-py", "pythainlp"] # check requirements.txt - root_dir = Path(__file__).parent.parent - req_file = root_dir / "requirements.txt" - req_dict = {} - with req_file.open() as f: - lines = f.readlines() + try: + # for CLI usage + root_dir = Path(__file__).parent.parent + req_file = root_dir / "requirements.txt" + req_dict = {} + with req_file.open() as f: + lines = f.readlines() + except FileNotFoundError as e: + # for local usage + root_dir = Path(__file__).parent.parent.parent + req_file = root_dir / "requirements.txt" + req_dict = {} + with req_file.open() as f: + lines = f.readlines() + for line in lines: line = line.strip() if not line.startswith("#"): From 9f1447bf7160dfdc354d8eb386ee169a330dbbca Mon Sep 17 00:00:00 2001 From: svlandeg Date: Wed, 19 Feb 2020 17:09:29 +0200 Subject: [PATCH 09/20] where areth thou, file ? --- spacy/tests/test_requirements.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/spacy/tests/test_requirements.py b/spacy/tests/test_requirements.py index a6fa20d6b..23ba792df 100644 --- a/spacy/tests/test_requirements.py +++ b/spacy/tests/test_requirements.py @@ -8,20 +8,26 @@ def test_build_dependencies(en_vocab): libs_ignore_setup = ["fugashi", "natto-py", "pythainlp"] # check requirements.txt + req_dict = {} try: # for CLI usage root_dir = Path(__file__).parent.parent req_file = root_dir / "requirements.txt" - req_dict = {} with req_file.open() as f: lines = f.readlines() except FileNotFoundError as e: - # for local usage - root_dir = Path(__file__).parent.parent.parent - req_file = root_dir / "requirements.txt" - req_dict = {} - with req_file.open() as f: - lines = f.readlines() + try: + # for local usage + root_dir = Path(__file__).parent.parent.parent + req_file = root_dir / "requirements.txt" + with req_file.open() as f: + lines = f.readlines() + except FileNotFoundError as e: + # where areth thou ? + root_dir = Path(__file__).parent.parent.parent.parent + req_file = root_dir / "requirements.txt" + with req_file.open() as f: + lines = f.readlines() for line in lines: line = line.strip() From 783da088eac9429852b48af38e32c4e219a95d57 Mon Sep 17 00:00:00 2001 From: svlandeg Date: Sun, 23 Feb 2020 16:21:21 +0100 Subject: [PATCH 10/20] avoid try except --- spacy/tests/test_requirements.py | 40 ++++++++++++-------------------- 1 file changed, 15 insertions(+), 25 deletions(-) diff --git a/spacy/tests/test_requirements.py b/spacy/tests/test_requirements.py index 23ba792df..644e6f8f9 100644 --- a/spacy/tests/test_requirements.py +++ b/spacy/tests/test_requirements.py @@ -9,32 +9,22 @@ def test_build_dependencies(en_vocab): # check requirements.txt req_dict = {} - try: - # for CLI usage - root_dir = Path(__file__).parent.parent - req_file = root_dir / "requirements.txt" - with req_file.open() as f: - lines = f.readlines() - except FileNotFoundError as e: - try: - # for local usage - root_dir = Path(__file__).parent.parent.parent - req_file = root_dir / "requirements.txt" - with req_file.open() as f: - lines = f.readlines() - except FileNotFoundError as e: - # where areth thou ? - root_dir = Path(__file__).parent.parent.parent.parent - req_file = root_dir / "requirements.txt" - with req_file.open() as f: - lines = f.readlines() - for line in lines: - line = line.strip() - if not line.startswith("#"): - lib, v = _parse_req(line) - if lib and lib not in libs_ignore_requirements: - req_dict[lib] = v + root_dir = None + # when running tests locally, the file is 3 levels up. On the CI, it's 2 levels up. + roots = [Path(__file__).parent.parent, Path(__file__).parent.parent.parent] # or whatever + for r in roots: + req_file = root_dir / "requirements.txt" + if req_file.exists(): + root_dir = r + with req_file.open() as f: + lines = f.readlines() + for line in lines: + line = line.strip() + if not line.startswith("#"): + lib, v = _parse_req(line) + if lib and lib not in libs_ignore_requirements: + req_dict[lib] = v # check setup.cfg and compare to requirements.txt # also fails when there are missing or additional libs From 0f55e5170414d90b048609eb44fbf8f27d085074 Mon Sep 17 00:00:00 2001 From: svlandeg Date: Sun, 23 Feb 2020 16:33:58 +0100 Subject: [PATCH 11/20] assert we found the root_dir --- spacy/tests/test_requirements.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/spacy/tests/test_requirements.py b/spacy/tests/test_requirements.py index 644e6f8f9..21636766d 100644 --- a/spacy/tests/test_requirements.py +++ b/spacy/tests/test_requirements.py @@ -26,6 +26,8 @@ def test_build_dependencies(en_vocab): if lib and lib not in libs_ignore_requirements: req_dict[lib] = v + assert root_dir is not None, "Could not find the root directory of requirements.txt" + # check setup.cfg and compare to requirements.txt # also fails when there are missing or additional libs setup_file = root_dir / "setup.cfg" From 58568bd0cd96b2f72f8b4ea81cfcc269ad93d1f5 Mon Sep 17 00:00:00 2001 From: svlandeg Date: Sun, 23 Feb 2020 16:45:37 +0100 Subject: [PATCH 12/20] fix --- spacy/tests/test_requirements.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spacy/tests/test_requirements.py b/spacy/tests/test_requirements.py index 21636766d..7922f1f18 100644 --- a/spacy/tests/test_requirements.py +++ b/spacy/tests/test_requirements.py @@ -14,7 +14,7 @@ def test_build_dependencies(en_vocab): # when running tests locally, the file is 3 levels up. On the CI, it's 2 levels up. roots = [Path(__file__).parent.parent, Path(__file__).parent.parent.parent] # or whatever for r in roots: - req_file = root_dir / "requirements.txt" + req_file = r / "requirements.txt" if req_file.exists(): root_dir = r with req_file.open() as f: From d821c95eb05f3ad0b82601487093559d1d686a2c Mon Sep 17 00:00:00 2001 From: svlandeg Date: Sun, 23 Feb 2020 17:38:33 +0100 Subject: [PATCH 13/20] debugging prints --- spacy/tests/test_requirements.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/spacy/tests/test_requirements.py b/spacy/tests/test_requirements.py index 7922f1f18..fc5aeeddd 100644 --- a/spacy/tests/test_requirements.py +++ b/spacy/tests/test_requirements.py @@ -13,7 +13,9 @@ def test_build_dependencies(en_vocab): root_dir = None # when running tests locally, the file is 3 levels up. On the CI, it's 2 levels up. roots = [Path(__file__).parent.parent, Path(__file__).parent.parent.parent] # or whatever + print() for r in roots: + print("inspecting dir", r, "-->", [f.name for f in r.glob(pattern="*.*")]) req_file = r / "requirements.txt" if req_file.exists(): root_dir = r @@ -33,6 +35,18 @@ def test_build_dependencies(en_vocab): setup_file = root_dir / "setup.cfg" with setup_file.open() as f: lines = f.readlines() + + # import configparser + # config = configparser.ConfigParser() + # config.read(setup_file) + # print("SECTIONS", config.sections()) + # print("options", config['options']) + # for key in config['options']: + # print("key", key) + # print("setup_requires *", config['options']['setup_requires'], "*") + # lines = config['options']['setup_requires'] + # lines += config['options']['install_requires'] + setup_keys = set() for line in lines: line = line.strip() From 6f846c2cbf1a0a2b4ceaffb83dd8e3d43a22e03e Mon Sep 17 00:00:00 2001 From: svlandeg Date: Mon, 24 Feb 2020 09:19:08 +0100 Subject: [PATCH 14/20] removing --pyargs for testing purposes --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 054365336..316ac0c68 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -99,5 +99,5 @@ jobs: pip install dist/$SDIST displayName: 'Install from sdist' - - script: python -m pytest --pyargs spacy + - script: python -m pytest spacy displayName: 'Run tests' From 217c16c7a9f6c08c078d56fb34bc6497e8c38131 Mon Sep 17 00:00:00 2001 From: svlandeg Date: Mon, 24 Feb 2020 09:38:43 +0100 Subject: [PATCH 15/20] running tests BEFORE deleting them ? --- azure-pipelines.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 316ac0c68..2ebc381cd 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -84,6 +84,9 @@ jobs: pip install -r requirements.txt displayName: 'Install dependencies' + - script: python -m pytest spacy + displayName: 'Run tests' + - script: | python setup.py build_ext --inplace python setup.py sdist --formats=gztar @@ -99,5 +102,3 @@ jobs: pip install dist/$SDIST displayName: 'Install from sdist' - - script: python -m pytest spacy - displayName: 'Run tests' From d5bfebe1c5d4772004965a450b57a3ca3119bcd2 Mon Sep 17 00:00:00 2001 From: svlandeg Date: Mon, 24 Feb 2020 10:04:24 +0100 Subject: [PATCH 16/20] it's moving day --- azure-pipelines.yml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 2ebc381cd..779037c96 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -84,21 +84,20 @@ jobs: pip install -r requirements.txt displayName: 'Install dependencies' - - script: python -m pytest spacy - displayName: 'Run tests' - - script: | python setup.py build_ext --inplace python setup.py sdist --formats=gztar displayName: 'Compile and build sdist' - - task: DeleteFiles@1 - inputs: - contents: 'spacy' - displayName: 'Delete source directory' - - bash: | SDIST=$(python -c "import os;print(os.listdir('./dist')[-1])" 2>&1) pip install dist/$SDIST displayName: 'Install from sdist' + - script: python -m pytest spacy + displayName: 'Run tests' + + - task: DeleteFiles@1 + inputs: + contents: 'spacy' + displayName: 'Delete source directory' From c1a5ece65f18b4955c8e7e72bdf815c78290d6f4 Mon Sep 17 00:00:00 2001 From: Ines Montani Date: Tue, 25 Feb 2020 15:46:39 +0100 Subject: [PATCH 17/20] Tidy up setup and update requirements tests --- .gitignore | 5 ++ pyproject.toml | 2 +- requirements.txt | 2 +- setup.py | 26 ++++++-- spacy/tests/package/test_requirements.py | 76 ++++++++++++++++++++++ spacy/tests/test_requirements.py | 83 ------------------------ 6 files changed, 102 insertions(+), 92 deletions(-) create mode 100644 spacy/tests/package/test_requirements.py delete mode 100644 spacy/tests/test_requirements.py diff --git a/.gitignore b/.gitignore index a0af6d4d2..f39607b76 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,11 @@ corpora/ keys/ *.json.gz +# Tests +spacy/tests/package/setup.cfg +spacy/tests/package/pyproject.toml +spacy/tests/package/requirements.txt + # Website website/.cache/ website/public/ diff --git a/pyproject.toml b/pyproject.toml index 8a6ababf3..8d3652a2f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,6 +6,6 @@ requires = [ "cymem>=2.0.2,<2.1.0", "preshed>=3.0.2,<3.1.0", "murmurhash>=0.28.0,<1.1.0", - "thinc==7.4.0.dev0", + "thinc==8.0.0a0", ] build-backend = "setuptools.build_meta" diff --git a/requirements.txt b/requirements.txt index bb6bf9804..f3a7cc162 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,7 +15,7 @@ plac>=0.9.6,<1.2.0 tqdm>=4.38.0,<5.0.0 # Optional dependencies jsonschema>=2.6.0,<3.1.0 -pydantic>=1.0.0,<2.0.0 +pydantic>=1.3.0,<2.0.0 # Development dependencies cython>=0.25 pytest>=4.6.5 diff --git a/setup.py b/setup.py index 31f22ba3f..d850a74ac 100755 --- a/setup.py +++ b/setup.py @@ -7,15 +7,19 @@ from distutils import ccompiler, msvccompiler from setuptools import Extension, setup, find_packages import numpy from pathlib import Path +import shutil from Cython.Build import cythonize from Cython.Compiler import Options +ROOT = Path(__file__).parent +PACKAGE_ROOT = ROOT / "spacy" + + # Preserve `__doc__` on functions and classes # http://docs.cython.org/en/latest/src/userguide/source_files_and_compilation.html#compiler-options Options.docstrings = True - PACKAGES = find_packages() MOD_NAMES = [ "spacy.parts_of_speech", @@ -60,6 +64,12 @@ COMPILER_DIRECTIVES = { "embedsignature": True, "annotation_typing": False, } +# Files to copy into the package that are otherwise not included +COPY_FILES = { + ROOT / "setup.cfg": PACKAGE_ROOT / "tests" / "package", + ROOT / "pyproject.toml": PACKAGE_ROOT / "tests" / "package", + ROOT / "requirements.txt": PACKAGE_ROOT / "tests" / "package", +} def is_new_osx(): @@ -115,25 +125,27 @@ def clean(path): def setup_package(): - root = Path(__file__).parent - if len(sys.argv) > 1 and sys.argv[1] == "clean": - return clean(root / "spacy") + return clean(PACKAGE_ROOT) - with (root / "spacy" / "about.py").open("r") as f: + with (PACKAGE_ROOT / "about.py").open("r") as f: about = {} exec(f.read(), about) + for copy_file, target_dir in COPY_FILES.items(): + shutil.copy(str(copy_file), str(target_dir)) + print(f"Copied {copy_file} -> {target_dir}") + include_dirs = [ get_python_inc(plat_specific=True), numpy.get_include(), - str(root / "include"), + str(ROOT / "include"), ] if ( ccompiler.new_compiler().compiler_type == "msvc" and msvccompiler.get_build_version() == 9 ): - include_dirs.append(str(root / "include" / "msvc9")) + include_dirs.append(str(ROOT / "include" / "msvc9")) ext_modules = [] for name in MOD_NAMES: mod_path = name.replace(".", "/") + ".pyx" diff --git a/spacy/tests/package/test_requirements.py b/spacy/tests/package/test_requirements.py new file mode 100644 index 000000000..59a8569ee --- /dev/null +++ b/spacy/tests/package/test_requirements.py @@ -0,0 +1,76 @@ +import re +from pathlib import Path + + +def test_build_dependencies(): + # Check that library requirements are pinned exactly the same across different setup files. + libs_ignore_requirements = [ + "pytest", + "pytest-timeout", + "mock", + "flake8", + "jsonschema", + ] + libs_ignore_setup = ["fugashi", "natto-py", "pythainlp"] + + # check requirements.txt + req_dict = {} + + root_dir = Path(__file__).parent + req_file = root_dir / "requirements.txt" + with req_file.open() as f: + lines = f.readlines() + for line in lines: + line = line.strip() + if not line.startswith("#"): + lib, v = _parse_req(line) + if lib and lib not in libs_ignore_requirements: + req_dict[lib] = v + # check setup.cfg and compare to requirements.txt + # also fails when there are missing or additional libs + setup_file = root_dir / "setup.cfg" + with setup_file.open() as f: + lines = f.readlines() + + setup_keys = set() + for line in lines: + line = line.strip() + if not line.startswith("#"): + lib, v = _parse_req(line) + if lib and not lib.startswith("cupy") and lib not in libs_ignore_setup: + req_v = req_dict.get(lib, None) + assert ( + req_v is not None + ), "{} in setup.cfg but not in requirements.txt".format(lib) + assert (lib + v) == (lib + req_v), ( + "{} has different version in setup.cfg and in requirements.txt: " + "{} and {} respectively".format(lib, v, req_v) + ) + setup_keys.add(lib) + assert sorted(setup_keys) == sorted( + req_dict.keys() + ) # if fail: requirements.txt contains a lib not in setup.cfg + + # check pyproject.toml and compare the versions of the libs to requirements.txt + # does not fail when there are missing or additional libs + toml_file = root_dir / "pyproject.toml" + with toml_file.open() as f: + lines = f.readlines() + for line in lines: + line = line.strip().strip(",").strip('"') + if not line.startswith("#"): + lib, v = _parse_req(line) + if lib: + req_v = req_dict.get(lib, None) + assert (lib + v) == (lib + req_v), ( + "{} has different version in pyproject.toml and in requirements.txt: " + "{} and {} respectively".format(lib, v, req_v) + ) + + +def _parse_req(line): + lib = re.match(r"^[a-z0-9\-]*", line).group(0) + v = line.replace(lib, "").strip() + if not re.match(r"^[<>=][<>=].*", v): + return None, None + return lib, v diff --git a/spacy/tests/test_requirements.py b/spacy/tests/test_requirements.py deleted file mode 100644 index fc5aeeddd..000000000 --- a/spacy/tests/test_requirements.py +++ /dev/null @@ -1,83 +0,0 @@ -import re -from pathlib import Path - - -def test_build_dependencies(en_vocab): - # Check that library requirements are pinned exactly the same across different setup files. - libs_ignore_requirements = ["pytest", "pytest-timeout", "mock", "flake8", "jsonschema"] - libs_ignore_setup = ["fugashi", "natto-py", "pythainlp"] - - # check requirements.txt - req_dict = {} - - root_dir = None - # when running tests locally, the file is 3 levels up. On the CI, it's 2 levels up. - roots = [Path(__file__).parent.parent, Path(__file__).parent.parent.parent] # or whatever - print() - for r in roots: - print("inspecting dir", r, "-->", [f.name for f in r.glob(pattern="*.*")]) - req_file = r / "requirements.txt" - if req_file.exists(): - root_dir = r - with req_file.open() as f: - lines = f.readlines() - for line in lines: - line = line.strip() - if not line.startswith("#"): - lib, v = _parse_req(line) - if lib and lib not in libs_ignore_requirements: - req_dict[lib] = v - - assert root_dir is not None, "Could not find the root directory of requirements.txt" - - # check setup.cfg and compare to requirements.txt - # also fails when there are missing or additional libs - setup_file = root_dir / "setup.cfg" - with setup_file.open() as f: - lines = f.readlines() - - # import configparser - # config = configparser.ConfigParser() - # config.read(setup_file) - # print("SECTIONS", config.sections()) - # print("options", config['options']) - # for key in config['options']: - # print("key", key) - # print("setup_requires *", config['options']['setup_requires'], "*") - # lines = config['options']['setup_requires'] - # lines += config['options']['install_requires'] - - setup_keys = set() - for line in lines: - line = line.strip() - if not line.startswith("#"): - lib, v = _parse_req(line) - if lib and not lib.startswith("cupy") and lib not in libs_ignore_setup: - req_v = req_dict.get(lib, None) - assert req_v is not None, "{} in setup.cfg but not in requirements.txt".format(lib) - assert (lib+v) == (lib+req_v), "{} has different version in setup.cfg and in requirements.txt: " \ - "{} and {} respectively".format(lib, v, req_v) - setup_keys.add(lib) - assert sorted(setup_keys) == sorted(req_dict.keys()) # if fail: requirements.txt contains a lib not in setup.cfg - - # check pyproject.toml and compare the versions of the libs to requirements.txt - # does not fail when there are missing or additional libs - toml_file = root_dir / "pyproject.toml" - with toml_file.open() as f: - lines = f.readlines() - for line in lines: - line = line.strip().strip(",").strip("\"") - if not line.startswith("#"): - lib, v = _parse_req(line) - if lib: - req_v = req_dict.get(lib, None) - assert (lib+v) == (lib+req_v), "{} has different version in pyproject.toml and in requirements.txt: " \ - "{} and {} respectively".format(lib, v, req_v) - - -def _parse_req(line): - lib = re.match(r"^[a-z0-9\-]*", line).group(0) - v = line.replace(lib, "").strip() - if not re.match(r"^[<>=][<>=].*", v): - return None, None - return lib, v From 436b26fe0fb28118d41351aa4f94bb4bb9932cd0 Mon Sep 17 00:00:00 2001 From: Ines Montani Date: Tue, 25 Feb 2020 15:48:29 +0100 Subject: [PATCH 18/20] Revert other changes --- MANIFEST.in | 1 - azure-pipelines.yml | 12 ++++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 64886cd19..1947b9140 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -4,6 +4,5 @@ include LICENSE include README.md include bin/spacy include pyproject.toml -include requirements.txt recursive-exclude spacy/lang *.json recursive-include spacy/lang *.json.gz diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 5a5e8f03a..d34da39f7 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -83,15 +83,15 @@ jobs: python setup.py sdist --formats=gztar displayName: 'Compile and build sdist' + - task: DeleteFiles@1 + inputs: + contents: 'spacy' + displayName: 'Delete source directory' + - bash: | SDIST=$(python -c "import os;print(os.listdir('./dist')[-1])" 2>&1) pip install dist/$SDIST displayName: 'Install from sdist' - - script: python -m pytest spacy + - script: python -m pytest --pyargs spacy displayName: 'Run tests' - - - task: DeleteFiles@1 - inputs: - contents: 'spacy' - displayName: 'Delete source directory' From 912572e04a6fe15c515c005847f054c989d5e6f1 Mon Sep 17 00:00:00 2001 From: Ines Montani Date: Tue, 25 Feb 2020 16:01:58 +0100 Subject: [PATCH 19/20] Only copy if file exists (not if installed from sdist etc.) --- setup.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index d850a74ac..d9021836f 100755 --- a/setup.py +++ b/setup.py @@ -133,8 +133,9 @@ def setup_package(): exec(f.read(), about) for copy_file, target_dir in COPY_FILES.items(): - shutil.copy(str(copy_file), str(target_dir)) - print(f"Copied {copy_file} -> {target_dir}") + if copy_file.exists(): + shutil.copy(str(copy_file), str(target_dir)) + print(f"Copied {copy_file} -> {target_dir}") include_dirs = [ get_python_inc(plat_specific=True), From b6a6cff70857b9edd0a1d2fa6a2fe62deb7a4290 Mon Sep 17 00:00:00 2001 From: Ines Montani Date: Tue, 25 Feb 2020 16:17:23 +0100 Subject: [PATCH 20/20] Add blis to pyproject.toml --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 8d3652a2f..71e523c7c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,5 +7,6 @@ requires = [ "preshed>=3.0.2,<3.1.0", "murmurhash>=0.28.0,<1.1.0", "thinc==8.0.0a0", + "blis>=0.4.0,<0.5.0" ] build-backend = "setuptools.build_meta"