2017-01-11 12:56:32 +00:00
|
|
|
|
# coding: utf-8
|
|
|
|
|
from __future__ import unicode_literals
|
2015-06-07 15:53:14 +00:00
|
|
|
|
|
2017-01-13 01:23:50 +00:00
|
|
|
|
from io import StringIO, BytesIO
|
2017-01-12 22:38:47 +00:00
|
|
|
|
from pathlib import Path
|
2017-01-11 12:56:32 +00:00
|
|
|
|
import pytest
|
|
|
|
|
|
2017-05-29 20:14:31 +00:00
|
|
|
|
from .util import load_test_model
|
|
|
|
|
from ..tokens import Doc
|
|
|
|
|
from ..strings import StringStore
|
|
|
|
|
from .. import util
|
|
|
|
|
|
2017-01-11 12:56:32 +00:00
|
|
|
|
|
2017-10-14 11:11:18 +00:00
|
|
|
|
# These languages are used for generic tokenizer tests – only add a language
|
|
|
|
|
# here if it's using spaCy's tokenizer (not a different library)
|
|
|
|
|
# TODO: re-implement generic tokenizer tests
|
2017-10-31 14:50:13 +00:00
|
|
|
|
_languages = ['bn', 'da', 'de', 'en', 'es', 'fi', 'fr', 'ga', 'he', 'hu', 'id',
|
2017-12-01 14:04:32 +00:00
|
|
|
|
'it', 'nb', 'nl', 'pl', 'pt', 'ru', 'sv', 'tr', 'xx']
|
2017-09-04 13:19:18 +00:00
|
|
|
|
_models = {'en': ['en_core_web_sm'],
|
2017-05-29 20:14:31 +00:00
|
|
|
|
'de': ['de_core_news_md'],
|
2017-11-01 22:02:00 +00:00
|
|
|
|
'fr': ['fr_core_news_sm'],
|
2018-03-28 14:02:59 +00:00
|
|
|
|
'xx': ['xx_ent_web_md'],
|
|
|
|
|
'en_core_web_md': ['en_core_web_md'],
|
|
|
|
|
'es_core_news_md': ['es_core_news_md']}
|
2017-05-29 20:14:31 +00:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# only used for tests that require loading the models
|
|
|
|
|
# in all other cases, use specific instances
|
|
|
|
|
|
2017-06-05 00:09:27 +00:00
|
|
|
|
@pytest.fixture(params=_models['en'])
|
2017-05-29 20:14:31 +00:00
|
|
|
|
def EN(request):
|
|
|
|
|
return load_test_model(request.param)
|
|
|
|
|
|
|
|
|
|
|
2017-06-05 00:09:27 +00:00
|
|
|
|
@pytest.fixture(params=_models['de'])
|
2017-05-29 20:14:31 +00:00
|
|
|
|
def DE(request):
|
|
|
|
|
return load_test_model(request.param)
|
|
|
|
|
|
|
|
|
|
|
2017-06-05 00:09:27 +00:00
|
|
|
|
@pytest.fixture(params=_models['fr'])
|
2017-05-29 20:14:31 +00:00
|
|
|
|
def FR(request):
|
|
|
|
|
return load_test_model(request.param)
|
2017-01-11 12:56:32 +00:00
|
|
|
|
|
|
|
|
|
|
2017-11-21 19:23:59 +00:00
|
|
|
|
@pytest.fixture()
|
|
|
|
|
def RU(request):
|
2017-11-26 17:54:48 +00:00
|
|
|
|
pymorphy = pytest.importorskip('pymorphy2')
|
2017-11-30 09:29:38 +00:00
|
|
|
|
return util.get_lang_class('ru')()
|
2017-11-21 19:23:59 +00:00
|
|
|
|
|
|
|
|
|
|
2017-06-05 00:26:13 +00:00
|
|
|
|
#@pytest.fixture(params=_languages)
|
|
|
|
|
#def tokenizer(request):
|
|
|
|
|
#lang = util.get_lang_class(request.param)
|
|
|
|
|
#return lang.Defaults.create_tokenizer()
|
|
|
|
|
|
2017-10-14 11:11:18 +00:00
|
|
|
|
|
2017-06-05 00:26:13 +00:00
|
|
|
|
@pytest.fixture
|
|
|
|
|
def tokenizer():
|
|
|
|
|
return util.get_lang_class('xx').Defaults.create_tokenizer()
|
2017-01-11 12:56:32 +00:00
|
|
|
|
|
|
|
|
|
|
2017-06-05 00:09:27 +00:00
|
|
|
|
@pytest.fixture
|
2017-01-11 12:56:32 +00:00
|
|
|
|
def en_tokenizer():
|
2017-05-13 23:31:10 +00:00
|
|
|
|
return util.get_lang_class('en').Defaults.create_tokenizer()
|
2015-06-07 15:53:14 +00:00
|
|
|
|
|
2016-09-26 09:57:54 +00:00
|
|
|
|
|
2017-06-05 00:09:27 +00:00
|
|
|
|
@pytest.fixture
|
2017-01-11 12:56:32 +00:00
|
|
|
|
def en_vocab():
|
2017-05-13 23:31:10 +00:00
|
|
|
|
return util.get_lang_class('en').Defaults.create_vocab()
|
2017-01-11 12:56:32 +00:00
|
|
|
|
|
|
|
|
|
|
2017-06-05 00:09:27 +00:00
|
|
|
|
@pytest.fixture
|
2017-10-06 22:38:51 +00:00
|
|
|
|
def en_parser(en_vocab):
|
|
|
|
|
nlp = util.get_lang_class('en')(en_vocab)
|
|
|
|
|
return nlp.create_pipe('parser')
|
2017-05-08 22:02:21 +00:00
|
|
|
|
|
2017-01-11 20:29:59 +00:00
|
|
|
|
|
2017-06-05 00:09:27 +00:00
|
|
|
|
@pytest.fixture
|
2017-04-06 16:48:45 +00:00
|
|
|
|
def es_tokenizer():
|
2017-05-13 23:31:10 +00:00
|
|
|
|
return util.get_lang_class('es').Defaults.create_tokenizer()
|
2017-04-06 16:48:45 +00:00
|
|
|
|
|
2017-01-11 20:29:59 +00:00
|
|
|
|
|
2017-06-05 00:09:27 +00:00
|
|
|
|
@pytest.fixture
|
2017-01-11 12:56:32 +00:00
|
|
|
|
def de_tokenizer():
|
2017-05-13 23:31:10 +00:00
|
|
|
|
return util.get_lang_class('de').Defaults.create_tokenizer()
|
2017-01-11 12:56:32 +00:00
|
|
|
|
|
|
|
|
|
|
2017-06-05 00:09:27 +00:00
|
|
|
|
@pytest.fixture
|
2017-01-24 09:55:02 +00:00
|
|
|
|
def fr_tokenizer():
|
2017-05-13 23:31:10 +00:00
|
|
|
|
return util.get_lang_class('fr').Defaults.create_tokenizer()
|
2017-01-24 09:55:02 +00:00
|
|
|
|
|
|
|
|
|
|
2017-06-05 00:09:27 +00:00
|
|
|
|
@pytest.fixture
|
2017-01-11 12:56:32 +00:00
|
|
|
|
def hu_tokenizer():
|
2017-05-13 23:31:10 +00:00
|
|
|
|
return util.get_lang_class('hu').Defaults.create_tokenizer()
|
2017-01-11 12:56:32 +00:00
|
|
|
|
|
2017-01-12 15:49:19 +00:00
|
|
|
|
|
2017-06-05 00:09:27 +00:00
|
|
|
|
@pytest.fixture
|
2017-02-04 11:47:29 +00:00
|
|
|
|
def fi_tokenizer():
|
2017-05-13 23:31:10 +00:00
|
|
|
|
return util.get_lang_class('fi').Defaults.create_tokenizer()
|
2017-02-04 11:47:29 +00:00
|
|
|
|
|
|
|
|
|
|
2017-08-20 05:17:14 +00:00
|
|
|
|
@pytest.fixture
|
|
|
|
|
def id_tokenizer():
|
|
|
|
|
return util.get_lang_class('id').Defaults.create_tokenizer()
|
2017-02-04 11:47:29 +00:00
|
|
|
|
|
2017-10-14 11:11:18 +00:00
|
|
|
|
|
2017-06-05 00:09:27 +00:00
|
|
|
|
@pytest.fixture
|
2017-02-04 14:21:34 +00:00
|
|
|
|
def sv_tokenizer():
|
2017-05-13 23:31:10 +00:00
|
|
|
|
return util.get_lang_class('sv').Defaults.create_tokenizer()
|
2017-02-04 14:21:34 +00:00
|
|
|
|
|
|
|
|
|
|
2017-06-05 00:09:27 +00:00
|
|
|
|
@pytest.fixture
|
2017-03-05 01:11:26 +00:00
|
|
|
|
def bn_tokenizer():
|
2017-05-13 23:31:10 +00:00
|
|
|
|
return util.get_lang_class('bn').Defaults.create_tokenizer()
|
2017-03-05 01:11:26 +00:00
|
|
|
|
|
2017-10-31 14:50:13 +00:00
|
|
|
|
|
2017-09-11 09:31:41 +00:00
|
|
|
|
@pytest.fixture
|
|
|
|
|
def ga_tokenizer():
|
|
|
|
|
return util.get_lang_class('ga').Defaults.create_tokenizer()
|
|
|
|
|
|
2017-04-06 16:48:45 +00:00
|
|
|
|
|
2017-06-05 00:09:27 +00:00
|
|
|
|
@pytest.fixture
|
2017-03-24 15:27:44 +00:00
|
|
|
|
def he_tokenizer():
|
2017-05-13 23:31:10 +00:00
|
|
|
|
return util.get_lang_class('he').Defaults.create_tokenizer()
|
2017-03-24 15:27:44 +00:00
|
|
|
|
|
2017-10-14 11:11:18 +00:00
|
|
|
|
|
2017-06-05 00:09:27 +00:00
|
|
|
|
@pytest.fixture
|
2017-04-26 21:21:41 +00:00
|
|
|
|
def nb_tokenizer():
|
2017-05-13 23:31:10 +00:00
|
|
|
|
return util.get_lang_class('nb').Defaults.create_tokenizer()
|
2017-05-08 22:02:21 +00:00
|
|
|
|
|
2017-07-03 13:43:06 +00:00
|
|
|
|
@pytest.fixture
|
|
|
|
|
def da_tokenizer():
|
|
|
|
|
return util.get_lang_class('da').Defaults.create_tokenizer()
|
2017-03-24 15:27:44 +00:00
|
|
|
|
|
2017-10-14 11:11:39 +00:00
|
|
|
|
@pytest.fixture
|
|
|
|
|
def ja_tokenizer():
|
2018-05-03 16:38:26 +00:00
|
|
|
|
janome = pytest.importorskip("MeCab")
|
2017-10-14 11:11:39 +00:00
|
|
|
|
return util.get_lang_class('ja').Defaults.create_tokenizer()
|
|
|
|
|
|
2017-09-26 14:36:27 +00:00
|
|
|
|
@pytest.fixture
|
|
|
|
|
def th_tokenizer():
|
2017-09-26 16:54:15 +00:00
|
|
|
|
pythainlp = pytest.importorskip("pythainlp")
|
2017-09-26 14:36:27 +00:00
|
|
|
|
return util.get_lang_class('th').Defaults.create_tokenizer()
|
|
|
|
|
|
2017-12-01 14:04:32 +00:00
|
|
|
|
@pytest.fixture
|
|
|
|
|
def tr_tokenizer():
|
|
|
|
|
return util.get_lang_class('tr').Defaults.create_tokenizer()
|
|
|
|
|
|
2017-03-24 15:27:44 +00:00
|
|
|
|
|
2017-11-21 19:23:59 +00:00
|
|
|
|
@pytest.fixture
|
|
|
|
|
def ru_tokenizer():
|
2017-11-26 18:04:44 +00:00
|
|
|
|
pymorphy = pytest.importorskip('pymorphy2')
|
2017-11-21 19:23:59 +00:00
|
|
|
|
return util.get_lang_class('ru').Defaults.create_tokenizer()
|
|
|
|
|
|
|
|
|
|
|
2017-01-12 14:05:40 +00:00
|
|
|
|
@pytest.fixture
|
|
|
|
|
def stringstore():
|
|
|
|
|
return StringStore()
|
2017-01-11 12:56:32 +00:00
|
|
|
|
|
2017-01-12 15:49:19 +00:00
|
|
|
|
|
2017-06-05 00:09:27 +00:00
|
|
|
|
@pytest.fixture
|
2017-01-12 20:56:32 +00:00
|
|
|
|
def en_entityrecognizer():
|
2017-05-13 23:31:10 +00:00
|
|
|
|
return util.get_lang_class('en').Defaults.create_entity()
|
2017-01-12 20:56:32 +00:00
|
|
|
|
|
|
|
|
|
|
2017-01-11 12:56:32 +00:00
|
|
|
|
@pytest.fixture
|
|
|
|
|
def text_file():
|
|
|
|
|
return StringIO()
|
|
|
|
|
|
2017-10-14 11:11:18 +00:00
|
|
|
|
|
2017-01-13 01:23:50 +00:00
|
|
|
|
@pytest.fixture
|
|
|
|
|
def text_file_b():
|
|
|
|
|
return BytesIO()
|
|
|
|
|
|
2017-01-11 12:56:32 +00:00
|
|
|
|
|
2015-07-22 23:19:03 +00:00
|
|
|
|
def pytest_addoption(parser):
|
|
|
|
|
parser.addoption("--models", action="store_true",
|
|
|
|
|
help="include tests that require full models")
|
|
|
|
|
parser.addoption("--vectors", action="store_true",
|
|
|
|
|
help="include word vectors tests")
|
|
|
|
|
parser.addoption("--slow", action="store_true",
|
|
|
|
|
help="include slow tests")
|
|
|
|
|
|
2017-05-29 20:10:16 +00:00
|
|
|
|
for lang in _languages + ['all']:
|
|
|
|
|
parser.addoption("--%s" % lang, action="store_true", help="Use %s models" % lang)
|
2018-03-28 14:02:59 +00:00
|
|
|
|
for model in _models:
|
|
|
|
|
if model not in _languages:
|
|
|
|
|
parser.addoption("--%s" % model, action="store_true", help="Use %s model" % model)
|
2017-05-29 20:10:16 +00:00
|
|
|
|
|
2015-07-22 23:19:03 +00:00
|
|
|
|
|
|
|
|
|
def pytest_runtest_setup(item):
|
|
|
|
|
for opt in ['models', 'vectors', 'slow']:
|
|
|
|
|
if opt in item.keywords and not item.config.getoption("--%s" % opt):
|
|
|
|
|
pytest.skip("need --%s option to run" % opt)
|
2017-05-29 20:10:16 +00:00
|
|
|
|
|
|
|
|
|
# Check if test is marked with models and has arguments set, i.e. specific
|
|
|
|
|
# language. If so, skip test if flag not set.
|
|
|
|
|
if item.get_marker('models'):
|
|
|
|
|
for arg in item.get_marker('models').args:
|
|
|
|
|
if not item.config.getoption("--%s" % arg) and not item.config.getoption("--all"):
|
2017-06-05 00:09:27 +00:00
|
|
|
|
pytest.skip("need --%s or --all option to run" % arg)
|