refactor setup.py

setup.py

@@ -4,6 +4,7 @@ import os
 import shutil
 import subprocess
 import sys
+import contextlib
 from distutils.command.build_ext import build_ext
 from distutils.sysconfig import get_python_inc
 
@@ -133,7 +134,7 @@ def get_version_info():
     FULLVERSION = VERSION
     if os.path.exists('.git'):
         GIT_REVISION = git_version()
-    elif os.path.exists('spacy/about.py'):
+    elif os.path.exists(os.path.join('spacy', 'about.py')):
         # must be a source distribution, use existing version file
         try:
             from spacy.about import git_revision as GIT_REVISION
@@ -150,7 +151,7 @@ def get_version_info():
     return FULLVERSION, GIT_REVISION
 
 
-def write_version_py(filename='spacy/about.py'):
+def write_version(path):
     cnt = """# THIS FILE IS GENERATED FROM SPACY SETUP.PY
 short_version = '%(version)s'
 version = '%(version)s'
@@ -162,100 +163,120 @@ if not release:
 """
     FULLVERSION, GIT_REVISION = get_version_info()
 
-    with open(filename, 'w') as f:
+    with open(path, 'w') as f:
         f.write(cnt % {'version': VERSION,
                        'full_version' : FULLVERSION,
                        'git_revision' : GIT_REVISION,
                        'isrelease': str(ISRELEASED)})
 
 
-def generate_cython():
+def generate_cython(root, source):
-    cwd = os.path.abspath(os.path.dirname(__file__))
     print('Cythonizing sources')
     p = subprocess.call([sys.executable,
-                         os.path.join(cwd, 'bin', 'cythonize.py'),
+                         os.path.join(root, 'bin', 'cythonize.py'),
-                         'spacy'],
+                         source])
-                         cwd=cwd)
     if p != 0:
         raise RuntimeError('Running cythonize failed')
 
 
-def clean():
+def import_include(module_name):
+    try:
+        return __import__(module_name, globals(), locals(), [], -1)
+    except ImportError:
+        raise ImportError('Unable to import %s. Create a virtual environment '
+                          'and install all dependencies from requirements.txt, '
+                          'e.g., run "pip install -r requirements.txt".' % module_name)
+
+
+def copy_include(src, dst, path):
+    assert os.path.isdir(src)
+    assert os.path.isdir(dst)
+    shutil.copytree(
+        os.path.join(src, path),
+        os.path.join(dst, path))
+
+
+def prepare_includes(path):
+    include_dir = os.path.join(path, 'include')
+    if os.path.exists(include_dir):
+        shutil.rmtree(include_dir)
+    os.mkdir(include_dir)
+
+    numpy = import_include('numpy')
+    copy_include(numpy.get_include(), include_dir, 'numpy')
+
+    murmurhash = import_include('murmurhash')
+    copy_include(
+        os.path.join(os.path.dirname(murmurhash.__file__), 'headers'),
+        include_dir, 'murmurhash')
+
+
+def is_source_release(path):
+    return os.path.exists(os.path.join(path, 'PKG-INFO'))
+
+
+def clean(path):
     for name in MOD_NAMES:
         name = name.replace('.', '/')
         for ext in ['.so', '.html', '.cpp', '.c']:
-            if os.path.exists(name + ext):
+            file_path = os.path.join(path, name + ext)
-                os.unlink(name + ext)
+            if os.path.exists(file_path):
+                os.unlink(file_path)
+
+
+@contextlib.contextmanager
+def chdir(new_dir):
+    old_dir = os.getcwd()
+    try:
+        os.chdir(new_dir)
+        sys.path.insert(0, new_dir)
+        yield
+    finally:
+        del sys.path[0]
+        os.chdir(old_dir)
 
 
 def setup_package():
-    src_path = os.path.dirname(os.path.abspath(sys.argv[0]))
+    root = os.path.abspath(os.path.dirname(__file__))
-    old_path = os.getcwd()
-    os.chdir(src_path)
-    sys.path.insert(0, src_path)
 
-    # Rewrite the version file everytime
+    if len(sys.argv) > 1 and sys.argv[1] == 'clean':
-    write_version_py()
+        return clean(root)
 
-    include_dirs = [
+    with chdir(root):
-        get_python_inc(plat_specific=True),
+        write_version(os.path.join(root, 'spacy', 'about.py'))
-        os.path.join(src_path, 'include')]
 
-    ext_modules = []
+        include_dirs = [
-    for mod_name in MOD_NAMES:
+            get_python_inc(plat_specific=True),
-        mod_path = mod_name.replace('.', '/') + '.cpp'
+            os.path.join(root, 'include')]
-        ext_modules.append(
-            Extension(mod_name, [mod_path],
-                language='c++', include_dirs=include_dirs))
 
-    metadata = dict(
+        ext_modules = []
-        name='spacy',
+        for mod_name in MOD_NAMES:
-        packages=PACKAGES,
+            mod_path = mod_name.replace('.', '/') + '.cpp'
-        description='Industrial-strength NLP',
+            ext_modules.append(
-        author='Matthew Honnibal',
+                Extension(mod_name, [mod_path],
-        author_email='matt@spacy.io',
+                    language='c++', include_dirs=include_dirs))
-        version=VERSION,
-        url='https://spacy.io',
-        license='MIT',
-        ext_modules=ext_modules,
-        install_requires=['numpy', 'murmurhash == 0.24', 'cymem == 1.30', 'preshed == 0.44',
-                          'thinc == 4.0.0', 'text_unidecode', 'plac', 'six',
-                          'ujson', 'cloudpickle', 'sputnik == 0.6.2'],
-        cmdclass = {
-            'build_ext': build_ext_subclass},
-    )
 
-    # Run build
+        if not is_source_release(root):
-    cwd = os.path.abspath(os.path.dirname(__file__))
+            generate_cython(root, 'spacy')
-    if not os.path.exists(os.path.join(cwd, 'PKG-INFO')):
+            prepare_includes(root)
-        # Generate Cython sources, unless building from source release
-        generate_cython()
 
-        # sync include dirs from native dependencies
+        setup(
-        include_dir = os.path.join(src_path, 'include')
+            name='spacy',
-        if os.path.exists(include_dir):
+            packages=PACKAGES,
-            shutil.rmtree(include_dir)
+            description='Industrial-strength NLP',
-        os.mkdir(include_dir)
+            author='Matthew Honnibal',
-
+            author_email='matt@spacy.io',
-        import numpy
+            version=VERSION,
-        shutil.copytree(
+            url='https://spacy.io',
-            os.path.join(numpy.get_include(), 'numpy'),
+            license='MIT',
-            os.path.join(include_dir, 'numpy'))
+            ext_modules=ext_modules,
-
+            install_requires=['numpy', 'murmurhash == 0.24', 'cymem == 1.30', 'preshed == 0.44',
-        import murmurhash
+                              'thinc == 4.0.0', 'text_unidecode', 'plac', 'six',
-        shutil.copytree(
+                              'ujson', 'cloudpickle', 'sputnik == 0.6.2'],
-            os.path.join(os.path.dirname(murmurhash.__file__), 'headers', 'murmurhash'),
+            cmdclass = {
-            os.path.join(include_dir, 'murmurhash'))
+                'build_ext': build_ext_subclass},
-
+        )
-    try:
-        setup(**metadata)
-    finally:
-        del sys.path[0]
-        os.chdir(old_path)
 
 
 if __name__ == '__main__':
-    if sys.argv[1] == 'clean':
+    setup_package()
-        clean()
-    else:
-        setup_package()

tox.ini

@@ -10,4 +10,4 @@ deps =
     pytest
 commands =
     python -m spacy.en.download
-    python -m pytest {toxinidir}/spacy/ --models --vectors --slow
+    python -m pytest {toxinidir}/spacy/ -x --models --vectors --slow