2014-07-05 18:49:34 +00:00
|
|
|
#!/usr/bin/env python
|
2016-01-15 17:57:01 +00:00
|
|
|
from __future__ import print_function
|
2016-10-18 22:27:57 +00:00
|
|
|
import io
|
2015-12-13 10:49:17 +00:00
|
|
|
import os
|
|
|
|
import subprocess
|
2014-07-05 18:49:34 +00:00
|
|
|
import sys
|
2015-12-13 22:32:23 +00:00
|
|
|
import contextlib
|
2015-10-12 22:31:59 +00:00
|
|
|
from distutils.command.build_ext import build_ext
|
2015-12-13 10:49:17 +00:00
|
|
|
from distutils.sysconfig import get_python_inc
|
2016-04-28 20:10:43 +00:00
|
|
|
from distutils import ccompiler, msvccompiler
|
2017-05-03 18:10:59 +00:00
|
|
|
from setuptools import Extension, setup, find_packages
|
2015-12-13 10:49:17 +00:00
|
|
|
|
|
|
|
|
2016-11-06 10:58:26 +00:00
|
|
|
PACKAGE_DATA = {'': ['*.pyx', '*.pxd', '*.txt', '*.tokens']}
|
|
|
|
|
|
|
|
|
2017-05-03 18:10:59 +00:00
|
|
|
PACKAGES = find_packages()
|
2015-12-13 10:49:17 +00:00
|
|
|
|
|
|
|
|
|
|
|
MOD_NAMES = [
|
2018-03-27 17:23:02 +00:00
|
|
|
'spacy._align',
|
2015-12-13 10:49:17 +00:00
|
|
|
'spacy.parts_of_speech',
|
|
|
|
'spacy.strings',
|
|
|
|
'spacy.lexeme',
|
|
|
|
'spacy.vocab',
|
|
|
|
'spacy.attrs',
|
|
|
|
'spacy.morphology',
|
2016-10-15 23:47:12 +00:00
|
|
|
'spacy.pipeline',
|
2015-12-13 10:49:17 +00:00
|
|
|
'spacy.syntax.stateclass',
|
2016-02-01 02:00:53 +00:00
|
|
|
'spacy.syntax._state',
|
2015-12-13 10:49:17 +00:00
|
|
|
'spacy.tokenizer',
|
2017-05-13 23:20:28 +00:00
|
|
|
'spacy.syntax.nn_parser',
|
2018-05-07 13:54:27 +00:00
|
|
|
'spacy.syntax._parser_model',
|
2018-05-07 22:19:52 +00:00
|
|
|
'spacy.syntax._beam_utils',
|
2016-03-01 09:09:08 +00:00
|
|
|
'spacy.syntax.nonproj',
|
2015-12-13 10:49:17 +00:00
|
|
|
'spacy.syntax.transition_system',
|
|
|
|
'spacy.syntax.arc_eager',
|
|
|
|
'spacy.gold',
|
|
|
|
'spacy.tokens.doc',
|
|
|
|
'spacy.tokens.span',
|
|
|
|
'spacy.tokens.token',
|
Add doc.retokenize() context manager (#2172)
This patch takes a step towards #1487 by introducing the
doc.retokenize() context manager, to handle merging spans, and soon
splitting tokens.
The idea is to do merging and splitting like this:
with doc.retokenize() as retokenizer:
for start, end, label in matches:
retokenizer.merge(doc[start : end], attrs={'ent_type': label})
The retokenizer accumulates the merge requests, and applies them
together at the end of the block. This will allow retokenization to be
more efficient, and much less error prone.
A retokenizer.split() function will then be added, to handle splitting a
single token into multiple tokens. These methods take `Span` and `Token`
objects; if the user wants to go directly from offsets, they can append
to the .merges and .splits lists on the retokenizer.
The doc.merge() method's behaviour remains unchanged, so this patch
should be 100% backwards incompatible (modulo bugs). Internally,
doc.merge() fixes up the arguments (to handle the various deprecated styles),
opens the retokenizer, and makes the single merge.
We can later start making deprecation warnings on direct calls to doc.merge(),
to migrate people to use of the retokenize context manager.
2018-04-03 12:10:35 +00:00
|
|
|
'spacy.tokens._retokenize',
|
2015-12-13 10:49:17 +00:00
|
|
|
'spacy.matcher',
|
|
|
|
'spacy.syntax.ner',
|
2016-03-10 12:01:34 +00:00
|
|
|
'spacy.symbols',
|
2017-06-05 10:32:22 +00:00
|
|
|
'spacy.vectors',
|
2017-06-05 10:45:29 +00:00
|
|
|
]
|
2015-12-13 10:49:17 +00:00
|
|
|
|
|
|
|
|
2016-11-06 10:58:26 +00:00
|
|
|
COMPILE_OPTIONS = {
|
2016-04-12 08:12:57 +00:00
|
|
|
'msvc': ['/Ox', '/EHsc'],
|
2018-01-14 13:34:36 +00:00
|
|
|
'mingw32' : ['-O2', '-Wno-strict-prototypes', '-Wno-unused-function'],
|
|
|
|
'other' : ['-O2', '-Wno-strict-prototypes', '-Wno-unused-function']
|
2016-02-05 13:43:52 +00:00
|
|
|
}
|
2015-10-12 22:31:59 +00:00
|
|
|
|
2016-02-05 13:43:52 +00:00
|
|
|
|
2016-11-06 10:58:26 +00:00
|
|
|
LINK_OPTIONS = {
|
2016-02-05 13:43:52 +00:00
|
|
|
'msvc' : [],
|
|
|
|
'mingw32': [],
|
|
|
|
'other' : []
|
|
|
|
}
|
|
|
|
|
2016-12-20 10:05:06 +00:00
|
|
|
|
2016-11-06 10:58:26 +00:00
|
|
|
# I don't understand this very well yet. See Issue #267
|
|
|
|
# Fingers crossed!
|
2017-10-27 10:16:58 +00:00
|
|
|
USE_OPENMP_DEFAULT = '0' if sys.platform != 'darwin' else None
|
2016-11-06 10:58:26 +00:00
|
|
|
if os.environ.get('USE_OPENMP', USE_OPENMP_DEFAULT) == '1':
|
|
|
|
if sys.platform == 'darwin':
|
|
|
|
COMPILE_OPTIONS['other'].append('-fopenmp')
|
|
|
|
LINK_OPTIONS['other'].append('-fopenmp')
|
|
|
|
PACKAGE_DATA['spacy.platform.darwin.lib'] = ['*.dylib']
|
|
|
|
PACKAGES.append('spacy.platform.darwin.lib')
|
|
|
|
|
|
|
|
elif sys.platform == 'win32':
|
|
|
|
COMPILE_OPTIONS['msvc'].append('/openmp')
|
|
|
|
|
|
|
|
else:
|
|
|
|
COMPILE_OPTIONS['other'].append('-fopenmp')
|
|
|
|
LINK_OPTIONS['other'].append('-fopenmp')
|
2015-01-06 01:34:55 +00:00
|
|
|
|
2017-05-07 16:36:35 +00:00
|
|
|
|
2016-04-19 17:50:42 +00:00
|
|
|
# By subclassing build_extensions we have the actual compiler that will be used which is really known only after finalize_options
|
|
|
|
# http://stackoverflow.com/questions/724664/python-distutils-how-to-get-a-compiler-that-is-going-to-be-used
|
2015-12-13 10:49:17 +00:00
|
|
|
class build_ext_options:
|
|
|
|
def build_options(self):
|
|
|
|
for e in self.extensions:
|
2016-11-06 10:58:26 +00:00
|
|
|
e.extra_compile_args += COMPILE_OPTIONS.get(
|
|
|
|
self.compiler.compiler_type, COMPILE_OPTIONS['other'])
|
2015-12-13 10:49:17 +00:00
|
|
|
for e in self.extensions:
|
2016-11-06 10:58:26 +00:00
|
|
|
e.extra_link_args += LINK_OPTIONS.get(
|
|
|
|
self.compiler.compiler_type, LINK_OPTIONS['other'])
|
2015-01-17 05:19:54 +00:00
|
|
|
|
|
|
|
|
2015-12-13 10:49:17 +00:00
|
|
|
class build_ext_subclass(build_ext, build_ext_options):
|
|
|
|
def build_extensions(self):
|
|
|
|
build_ext_options.build_options(self)
|
|
|
|
build_ext.build_extensions(self)
|
2015-01-17 05:19:54 +00:00
|
|
|
|
|
|
|
|
2015-12-13 22:32:23 +00:00
|
|
|
def generate_cython(root, source):
|
2015-12-13 10:49:17 +00:00
|
|
|
print('Cythonizing sources')
|
2015-12-13 11:51:23 +00:00
|
|
|
p = subprocess.call([sys.executable,
|
2015-12-13 22:32:23 +00:00
|
|
|
os.path.join(root, 'bin', 'cythonize.py'),
|
2017-02-16 18:01:25 +00:00
|
|
|
source], env=os.environ)
|
2015-12-13 10:49:17 +00:00
|
|
|
if p != 0:
|
|
|
|
raise RuntimeError('Running cythonize failed')
|
|
|
|
|
|
|
|
|
2015-12-13 22:32:23 +00:00
|
|
|
def is_source_release(path):
|
|
|
|
return os.path.exists(os.path.join(path, 'PKG-INFO'))
|
|
|
|
|
|
|
|
|
|
|
|
def clean(path):
|
2015-12-13 10:49:17 +00:00
|
|
|
for name in MOD_NAMES:
|
|
|
|
name = name.replace('.', '/')
|
|
|
|
for ext in ['.so', '.html', '.cpp', '.c']:
|
2015-12-13 22:32:23 +00:00
|
|
|
file_path = os.path.join(path, name + ext)
|
|
|
|
if os.path.exists(file_path):
|
|
|
|
os.unlink(file_path)
|
2015-01-25 03:49:10 +00:00
|
|
|
|
|
|
|
|
2015-12-13 22:32:23 +00:00
|
|
|
@contextlib.contextmanager
|
|
|
|
def chdir(new_dir):
|
|
|
|
old_dir = os.getcwd()
|
2015-12-13 10:49:17 +00:00
|
|
|
try:
|
2015-12-13 22:32:23 +00:00
|
|
|
os.chdir(new_dir)
|
|
|
|
sys.path.insert(0, new_dir)
|
|
|
|
yield
|
2015-12-13 10:49:17 +00:00
|
|
|
finally:
|
|
|
|
del sys.path[0]
|
2015-12-13 22:32:23 +00:00
|
|
|
os.chdir(old_dir)
|
|
|
|
|
|
|
|
|
|
|
|
def setup_package():
|
|
|
|
root = os.path.abspath(os.path.dirname(__file__))
|
|
|
|
|
|
|
|
if len(sys.argv) > 1 and sys.argv[1] == 'clean':
|
|
|
|
return clean(root)
|
|
|
|
|
|
|
|
with chdir(root):
|
2016-10-18 22:05:53 +00:00
|
|
|
with io.open(os.path.join(root, 'spacy', 'about.py'), encoding='utf8') as f:
|
2016-03-13 17:12:32 +00:00
|
|
|
about = {}
|
2016-01-15 17:57:01 +00:00
|
|
|
exec(f.read(), about)
|
2015-12-13 22:32:23 +00:00
|
|
|
|
2016-10-18 22:05:53 +00:00
|
|
|
with io.open(os.path.join(root, 'README.rst'), encoding='utf8') as f:
|
2016-03-13 17:12:32 +00:00
|
|
|
readme = f.read()
|
|
|
|
|
2015-12-13 22:32:23 +00:00
|
|
|
include_dirs = [
|
|
|
|
get_python_inc(plat_specific=True),
|
|
|
|
os.path.join(root, 'include')]
|
|
|
|
|
2016-04-28 20:10:43 +00:00
|
|
|
if (ccompiler.new_compiler().compiler_type == 'msvc'
|
2016-11-06 10:58:26 +00:00
|
|
|
and msvccompiler.get_build_version() == 9):
|
2016-04-28 20:10:43 +00:00
|
|
|
include_dirs.append(os.path.join(root, 'include', 'msvc9'))
|
|
|
|
|
2015-12-13 22:32:23 +00:00
|
|
|
ext_modules = []
|
|
|
|
for mod_name in MOD_NAMES:
|
|
|
|
mod_path = mod_name.replace('.', '/') + '.cpp'
|
2016-11-06 10:58:26 +00:00
|
|
|
extra_link_args = []
|
|
|
|
# ???
|
|
|
|
# Imported from patch from @mikepb
|
|
|
|
# See Issue #267. Running blind here...
|
|
|
|
if sys.platform == 'darwin':
|
|
|
|
dylib_path = ['..' for _ in range(mod_name.count('.'))]
|
|
|
|
dylib_path = '/'.join(dylib_path)
|
|
|
|
dylib_path = '@loader_path/%s/spacy/platform/darwin/lib' % dylib_path
|
|
|
|
extra_link_args.append('-Wl,-rpath,%s' % dylib_path)
|
2015-12-13 22:32:23 +00:00
|
|
|
ext_modules.append(
|
|
|
|
Extension(mod_name, [mod_path],
|
2016-11-06 10:58:26 +00:00
|
|
|
language='c++', include_dirs=include_dirs,
|
|
|
|
extra_link_args=extra_link_args))
|
2015-12-13 22:32:23 +00:00
|
|
|
|
|
|
|
if not is_source_release(root):
|
|
|
|
generate_cython(root, 'spacy')
|
|
|
|
|
|
|
|
setup(
|
2016-02-15 00:33:39 +00:00
|
|
|
name=about['__title__'],
|
2016-01-15 17:01:02 +00:00
|
|
|
zip_safe=False,
|
2015-12-13 22:32:23 +00:00
|
|
|
packages=PACKAGES,
|
2016-11-06 10:58:26 +00:00
|
|
|
package_data=PACKAGE_DATA,
|
2016-01-15 17:57:01 +00:00
|
|
|
description=about['__summary__'],
|
2016-03-13 17:12:32 +00:00
|
|
|
long_description=readme,
|
2016-01-15 17:57:01 +00:00
|
|
|
author=about['__author__'],
|
|
|
|
author_email=about['__email__'],
|
|
|
|
version=about['__version__'],
|
|
|
|
url=about['__uri__'],
|
|
|
|
license=about['__license__'],
|
2015-12-13 22:32:23 +00:00
|
|
|
ext_modules=ext_modules,
|
2017-08-09 10:17:30 +00:00
|
|
|
scripts=['bin/spacy'],
|
2016-03-13 17:12:32 +00:00
|
|
|
install_requires=[
|
2016-04-19 17:50:42 +00:00
|
|
|
'numpy>=1.7',
|
2017-06-01 07:56:12 +00:00
|
|
|
'murmurhash>=0.28,<0.29',
|
2016-04-19 17:50:42 +00:00
|
|
|
'cymem>=1.30,<1.32',
|
2017-03-11 18:33:56 +00:00
|
|
|
'preshed>=1.0.0,<2.0.0',
|
2018-05-20 17:00:34 +00:00
|
|
|
'thinc>=6.11.1,<6.12.0',
|
2017-03-18 14:14:02 +00:00
|
|
|
'plac<1.0.0,>=0.9.6',
|
2016-10-13 12:19:57 +00:00
|
|
|
'pathlib',
|
2017-03-08 13:11:54 +00:00
|
|
|
'ujson>=1.35',
|
2018-05-16 21:19:01 +00:00
|
|
|
'regex==2017.4.5',
|
2018-04-29 13:47:54 +00:00
|
|
|
'dill>=0.2,<0.3'],
|
2018-03-27 17:23:02 +00:00
|
|
|
setup_requires=['wheel'],
|
2018-05-19 16:12:23 +00:00
|
|
|
extras_require={
|
|
|
|
'cuda': ['cupy>=4.0'],
|
|
|
|
'cuda80': ['cupy-cuda80>=4.0'],
|
|
|
|
'cuda90': ['cupy-cuda90>=4.0'],
|
|
|
|
'cuda91': ['cupy-cuda91>=4.0'],
|
|
|
|
},
|
2016-03-12 12:47:10 +00:00
|
|
|
classifiers=[
|
|
|
|
'Development Status :: 5 - Production/Stable',
|
|
|
|
'Environment :: Console',
|
|
|
|
'Intended Audience :: Developers',
|
|
|
|
'Intended Audience :: Science/Research',
|
|
|
|
'License :: OSI Approved :: MIT License',
|
|
|
|
'Operating System :: POSIX :: Linux',
|
|
|
|
'Operating System :: MacOS :: MacOS X',
|
|
|
|
'Operating System :: Microsoft :: Windows',
|
2016-03-14 00:46:33 +00:00
|
|
|
'Programming Language :: Cython',
|
2017-11-26 17:46:24 +00:00
|
|
|
'Programming Language :: Python :: 2',
|
2016-03-12 12:47:10 +00:00
|
|
|
'Programming Language :: Python :: 2.7',
|
2017-11-26 17:46:24 +00:00
|
|
|
'Programming Language :: Python :: 3',
|
2016-03-12 12:47:10 +00:00
|
|
|
'Programming Language :: Python :: 3.4',
|
|
|
|
'Programming Language :: Python :: 3.5',
|
2017-03-19 00:40:24 +00:00
|
|
|
'Programming Language :: Python :: 3.6',
|
2016-03-12 12:47:10 +00:00
|
|
|
'Topic :: Scientific/Engineering'],
|
2015-12-13 22:32:23 +00:00
|
|
|
cmdclass = {
|
|
|
|
'build_ext': build_ext_subclass},
|
|
|
|
)
|
2015-01-04 18:30:56 +00:00
|
|
|
|
|
|
|
|
|
|
|
if __name__ == '__main__':
|
2015-12-13 22:32:23 +00:00
|
|
|
setup_package()
|