Rooba
/
spaCy
mirror of https://github.com/explosion/spaCy.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Tidy up setup and update requirements tests

This commit is contained in:
Ines Montani 2020-02-25 15:46:39 +01:00
parent 5d21d3e8b9
commit c1a5ece65f
6 changed files with 102 additions and 92 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
.gitignore vendored
View File

@ -5,6 +5,11 @@ corpora/
keys/
*.json.gz
# Tests
spacy/tests/package/setup.cfg
spacy/tests/package/pyproject.toml
spacy/tests/package/requirements.txt
# Website
website/.cache/
website/public/

2
pyproject.toml
View File

@ -6,6 +6,6 @@ requires = [
"cymem>=2.0.2,<2.1.0",
"preshed>=3.0.2,<3.1.0",
"murmurhash>=0.28.0,<1.1.0",
"thinc==7.4.0.dev0",
"thinc==8.0.0a0",
]
build-backend = "setuptools.build_meta"

2
requirements.txt
View File

@ -15,7 +15,7 @@ plac>=0.9.6,<1.2.0
tqdm>=4.38.0,<5.0.0
# Optional dependencies
jsonschema>=2.6.0,<3.1.0
pydantic>=1.0.0,<2.0.0
pydantic>=1.3.0,<2.0.0
# Development dependencies
cython>=0.25
pytest>=4.6.5

26
setup.py
View File

@ -7,15 +7,19 @@ from distutils import ccompiler, msvccompiler
from setuptools import Extension, setup, find_packages
import numpy
from pathlib import Path
import shutil
from Cython.Build import cythonize
from Cython.Compiler import Options
ROOT = Path(__file__).parent
PACKAGE_ROOT = ROOT / "spacy"
# Preserve `__doc__` on functions and classes
# http://docs.cython.org/en/latest/src/userguide/source_files_and_compilation.html#compiler-options
Options.docstrings = True
PACKAGES = find_packages()
MOD_NAMES = [
"spacy.parts_of_speech",
@ -60,6 +64,12 @@ COMPILER_DIRECTIVES = {
"embedsignature": True,
"annotation_typing": False,
}
# Files to copy into the package that are otherwise not included
COPY_FILES = {
ROOT / "setup.cfg": PACKAGE_ROOT / "tests" / "package",
ROOT / "pyproject.toml": PACKAGE_ROOT / "tests" / "package",
ROOT / "requirements.txt": PACKAGE_ROOT / "tests" / "package",
}
def is_new_osx():
@ -115,25 +125,27 @@ def clean(path):
def setup_package():
root = Path(__file__).parent
if len(sys.argv) > 1 and sys.argv[1] == "clean":
return clean(root / "spacy")
return clean(PACKAGE_ROOT)
with (root / "spacy" / "about.py").open("r") as f:
with (PACKAGE_ROOT / "about.py").open("r") as f:
about = {}
exec(f.read(), about)
for copy_file, target_dir in COPY_FILES.items():
shutil.copy(str(copy_file), str(target_dir))
print(f"Copied {copy_file} -> {target_dir}")
include_dirs = [
get_python_inc(plat_specific=True),
numpy.get_include(),
str(root / "include"),
str(ROOT / "include"),
]
if (
ccompiler.new_compiler().compiler_type == "msvc"
and msvccompiler.get_build_version() == 9
):
include_dirs.append(str(root / "include" / "msvc9"))
include_dirs.append(str(ROOT / "include" / "msvc9"))
ext_modules = []
for name in MOD_NAMES:
mod_path = name.replace(".", "/") + ".pyx"

76
spacy/tests/package/test_requirements.py Normal file
View File

@ -0,0 +1,76 @@
import re
from pathlib import Path
def test_build_dependencies():
# Check that library requirements are pinned exactly the same across different setup files.
libs_ignore_requirements = [
"pytest",
"pytest-timeout",
"mock",
"flake8",
"jsonschema",
]
libs_ignore_setup = ["fugashi", "natto-py", "pythainlp"]
# check requirements.txt
req_dict = {}
root_dir = Path(__file__).parent
req_file = root_dir / "requirements.txt"
with req_file.open() as f:
lines = f.readlines()
for line in lines:
line = line.strip()
if not line.startswith("#"):
lib, v = _parse_req(line)
if lib and lib not in libs_ignore_requirements:
req_dict[lib] = v
# check setup.cfg and compare to requirements.txt
# also fails when there are missing or additional libs
setup_file = root_dir / "setup.cfg"
with setup_file.open() as f:
lines = f.readlines()
setup_keys = set()
for line in lines:
line = line.strip()
if not line.startswith("#"):
lib, v = _parse_req(line)
if lib and not lib.startswith("cupy") and lib not in libs_ignore_setup:
req_v = req_dict.get(lib, None)
assert (
req_v is not None
), "{} in setup.cfg but not in requirements.txt".format(lib)
assert (lib + v) == (lib + req_v), (
"{} has different version in setup.cfg and in requirements.txt: "
"{} and {} respectively".format(lib, v, req_v)
)
setup_keys.add(lib)
assert sorted(setup_keys) == sorted(
req_dict.keys()
) # if fail: requirements.txt contains a lib not in setup.cfg
# check pyproject.toml and compare the versions of the libs to requirements.txt
# does not fail when there are missing or additional libs
toml_file = root_dir / "pyproject.toml"
with toml_file.open() as f:
lines = f.readlines()
for line in lines:
line = line.strip().strip(",").strip('"')
if not line.startswith("#"):
lib, v = _parse_req(line)
if lib:
req_v = req_dict.get(lib, None)
assert (lib + v) == (lib + req_v), (
"{} has different version in pyproject.toml and in requirements.txt: "
"{} and {} respectively".format(lib, v, req_v)
)
def _parse_req(line):
lib = re.match(r"^[a-z0-9\-]*", line).group(0)
v = line.replace(lib, "").strip()
if not re.match(r"^[<>=][<>=].*", v):
return None, None
return lib, v

83
spacy/tests/test_requirements.py
View File

@ -1,83 +0,0 @@
import re
from pathlib import Path
def test_build_dependencies(en_vocab):
# Check that library requirements are pinned exactly the same across different setup files.
libs_ignore_requirements = ["pytest", "pytest-timeout", "mock", "flake8", "jsonschema"]
libs_ignore_setup = ["fugashi", "natto-py", "pythainlp"]
# check requirements.txt
req_dict = {}
root_dir = None
# when running tests locally, the file is 3 levels up. On the CI, it's 2 levels up.
roots = [Path(__file__).parent.parent, Path(__file__).parent.parent.parent] # or whatever
print()
for r in roots:
print("inspecting dir", r, "-->", [f.name for f in r.glob(pattern="*.*")])
req_file = r / "requirements.txt"
if req_file.exists():
root_dir = r
with req_file.open() as f:
lines = f.readlines()
for line in lines:
line = line.strip()
if not line.startswith("#"):
lib, v = _parse_req(line)
if lib and lib not in libs_ignore_requirements:
req_dict[lib] = v
assert root_dir is not None, "Could not find the root directory of requirements.txt"
# check setup.cfg and compare to requirements.txt
# also fails when there are missing or additional libs
setup_file = root_dir / "setup.cfg"
with setup_file.open() as f:
lines = f.readlines()
# import configparser
# config = configparser.ConfigParser()
# config.read(setup_file)
# print("SECTIONS", config.sections())
# print("options", config['options'])
# for key in config['options']:
# print("key", key)
# print("setup_requires *", config['options']['setup_requires'], "*")
# lines = config['options']['setup_requires']
# lines += config['options']['install_requires']
setup_keys = set()
for line in lines:
line = line.strip()
if not line.startswith("#"):
lib, v = _parse_req(line)
if lib and not lib.startswith("cupy") and lib not in libs_ignore_setup:
req_v = req_dict.get(lib, None)
assert req_v is not None, "{} in setup.cfg but not in requirements.txt".format(lib)
assert (lib+v) == (lib+req_v), "{} has different version in setup.cfg and in requirements.txt: " \
"{} and {} respectively".format(lib, v, req_v)
setup_keys.add(lib)
assert sorted(setup_keys) == sorted(req_dict.keys()) # if fail: requirements.txt contains a lib not in setup.cfg
# check pyproject.toml and compare the versions of the libs to requirements.txt
# does not fail when there are missing or additional libs
toml_file = root_dir / "pyproject.toml"
with toml_file.open() as f:
lines = f.readlines()
for line in lines:
line = line.strip().strip(",").strip("\"")
if not line.startswith("#"):
lib, v = _parse_req(line)
if lib:
req_v = req_dict.get(lib, None)
assert (lib+v) == (lib+req_v), "{} has different version in pyproject.toml and in requirements.txt: " \
"{} and {} respectively".format(lib, v, req_v)
def _parse_req(line):
lib = re.match(r"^[a-z0-9\-]*", line).group(0)
v = line.replace(lib, "").strip()
if not re.match(r"^[<>=][<>=].*", v):
return None, None
return lib, v