2018-07-24 21:38:44 +00:00
|
|
|
import pytest
|
2020-07-22 11:42:59 +00:00
|
|
|
from spacy import registry
|
2018-11-27 00:09:36 +00:00
|
|
|
from spacy.pipeline import Tagger, DependencyParser, EntityRecognizer
|
2020-06-01 21:38:48 +00:00
|
|
|
from spacy.pipeline import TextCategorizer, SentenceRecognizer
|
2020-07-22 11:42:59 +00:00
|
|
|
from spacy.pipeline.dep_parser import DEFAULT_PARSER_MODEL
|
|
|
|
from spacy.pipeline.tagger import DEFAULT_TAGGER_MODEL
|
|
|
|
from spacy.pipeline.textcat import DEFAULT_TEXTCAT_MODEL
|
|
|
|
from spacy.pipeline.senter import DEFAULT_SENTER_MODEL
|
2020-08-28 13:20:14 +00:00
|
|
|
from spacy.lang.en import English
|
|
|
|
import spacy
|
2018-07-24 21:38:44 +00:00
|
|
|
|
|
|
|
from ..util import make_tempdir
|
|
|
|
|
|
|
|
|
|
|
|
test_parsers = [DependencyParser, EntityRecognizer]
|
|
|
|
|
|
|
|
|
|
|
|
@pytest.fixture
|
|
|
|
def parser(en_vocab):
|
2020-06-20 12:15:04 +00:00
|
|
|
config = {
|
|
|
|
"learn_tokens": False,
|
|
|
|
"min_action_freq": 30,
|
2020-07-22 11:42:59 +00:00
|
|
|
"update_with_oracle_cut_size": 100,
|
2020-06-20 12:15:04 +00:00
|
|
|
}
|
2020-07-25 13:01:15 +00:00
|
|
|
cfg = {"model": DEFAULT_PARSER_MODEL}
|
|
|
|
model = registry.make_from_config(cfg, validate=True)["model"]
|
2020-07-22 11:42:59 +00:00
|
|
|
parser = DependencyParser(en_vocab, model, **config)
|
2018-11-27 00:09:36 +00:00
|
|
|
parser.add_label("nsubj")
|
2018-07-24 21:38:44 +00:00
|
|
|
return parser
|
|
|
|
|
|
|
|
|
|
|
|
@pytest.fixture
|
|
|
|
def blank_parser(en_vocab):
|
2020-07-22 11:42:59 +00:00
|
|
|
config = {
|
|
|
|
"learn_tokens": False,
|
|
|
|
"min_action_freq": 30,
|
|
|
|
"update_with_oracle_cut_size": 100,
|
|
|
|
}
|
2020-07-25 13:01:15 +00:00
|
|
|
cfg = {"model": DEFAULT_PARSER_MODEL}
|
|
|
|
model = registry.make_from_config(cfg, validate=True)["model"]
|
2020-07-22 11:42:59 +00:00
|
|
|
parser = DependencyParser(en_vocab, model, **config)
|
2018-07-24 21:38:44 +00:00
|
|
|
return parser
|
|
|
|
|
|
|
|
|
|
|
|
@pytest.fixture
|
|
|
|
def taggers(en_vocab):
|
2020-07-25 13:01:15 +00:00
|
|
|
cfg = {"model": DEFAULT_TAGGER_MODEL}
|
|
|
|
model = registry.make_from_config(cfg, validate=True)["model"]
|
2020-08-07 13:27:13 +00:00
|
|
|
tagger1 = Tagger(en_vocab, model)
|
|
|
|
tagger2 = Tagger(en_vocab, model)
|
2020-02-27 17:42:27 +00:00
|
|
|
return tagger1, tagger2
|
2018-07-24 21:38:44 +00:00
|
|
|
|
|
|
|
|
2018-11-27 00:09:36 +00:00
|
|
|
@pytest.mark.parametrize("Parser", test_parsers)
|
2018-07-24 21:38:44 +00:00
|
|
|
def test_serialize_parser_roundtrip_bytes(en_vocab, Parser):
|
2020-07-22 11:42:59 +00:00
|
|
|
config = {
|
|
|
|
"learn_tokens": False,
|
|
|
|
"min_action_freq": 0,
|
|
|
|
"update_with_oracle_cut_size": 100,
|
|
|
|
}
|
2020-07-25 13:01:15 +00:00
|
|
|
cfg = {"model": DEFAULT_PARSER_MODEL}
|
|
|
|
model = registry.make_from_config(cfg, validate=True)["model"]
|
2020-07-22 11:42:59 +00:00
|
|
|
parser = Parser(en_vocab, model, **config)
|
|
|
|
new_parser = Parser(en_vocab, model, **config)
|
2019-09-10 17:45:16 +00:00
|
|
|
new_parser = new_parser.from_bytes(parser.to_bytes(exclude=["vocab"]))
|
2020-02-27 17:42:27 +00:00
|
|
|
bytes_2 = new_parser.to_bytes(exclude=["vocab"])
|
|
|
|
bytes_3 = parser.to_bytes(exclude=["vocab"])
|
|
|
|
assert len(bytes_2) == len(bytes_3)
|
|
|
|
assert bytes_2 == bytes_3
|
2018-07-24 21:38:44 +00:00
|
|
|
|
|
|
|
|
2018-11-27 00:09:36 +00:00
|
|
|
@pytest.mark.parametrize("Parser", test_parsers)
|
2018-07-24 21:38:44 +00:00
|
|
|
def test_serialize_parser_roundtrip_disk(en_vocab, Parser):
|
2020-07-22 11:42:59 +00:00
|
|
|
config = {
|
|
|
|
"learn_tokens": False,
|
|
|
|
"min_action_freq": 0,
|
|
|
|
"update_with_oracle_cut_size": 100,
|
|
|
|
}
|
2020-07-25 13:01:15 +00:00
|
|
|
cfg = {"model": DEFAULT_PARSER_MODEL}
|
|
|
|
model = registry.make_from_config(cfg, validate=True)["model"]
|
2020-07-22 11:42:59 +00:00
|
|
|
parser = Parser(en_vocab, model, **config)
|
2018-07-24 21:38:44 +00:00
|
|
|
with make_tempdir() as d:
|
2018-11-27 00:09:36 +00:00
|
|
|
file_path = d / "parser"
|
2018-07-24 21:38:44 +00:00
|
|
|
parser.to_disk(file_path)
|
2020-07-22 11:42:59 +00:00
|
|
|
parser_d = Parser(en_vocab, model, **config)
|
2018-07-24 21:38:44 +00:00
|
|
|
parser_d = parser_d.from_disk(file_path)
|
2019-09-10 17:45:16 +00:00
|
|
|
parser_bytes = parser.to_bytes(exclude=["model", "vocab"])
|
|
|
|
parser_d_bytes = parser_d.to_bytes(exclude=["model", "vocab"])
|
2020-02-27 17:42:27 +00:00
|
|
|
assert len(parser_bytes) == len(parser_d_bytes)
|
2019-03-10 18:16:45 +00:00
|
|
|
assert parser_bytes == parser_d_bytes
|
2018-07-24 21:38:44 +00:00
|
|
|
|
|
|
|
|
|
|
|
def test_to_from_bytes(parser, blank_parser):
|
|
|
|
assert parser.model is not True
|
2020-02-27 17:42:27 +00:00
|
|
|
assert blank_parser.model is not True
|
2018-07-24 21:38:44 +00:00
|
|
|
assert blank_parser.moves.n_moves != parser.moves.n_moves
|
2019-09-10 17:45:16 +00:00
|
|
|
bytes_data = parser.to_bytes(exclude=["vocab"])
|
2020-02-27 17:42:27 +00:00
|
|
|
# the blank parser needs to be resized before we can call from_bytes
|
2020-05-18 20:23:33 +00:00
|
|
|
blank_parser.model.attrs["resize_output"](blank_parser.model, parser.moves.n_moves)
|
2018-07-24 21:38:44 +00:00
|
|
|
blank_parser.from_bytes(bytes_data)
|
|
|
|
assert blank_parser.model is not True
|
|
|
|
assert blank_parser.moves.n_moves == parser.moves.n_moves
|
|
|
|
|
|
|
|
|
2018-11-27 00:09:36 +00:00
|
|
|
@pytest.mark.skip(
|
|
|
|
reason="This seems to be a dict ordering bug somewhere. Only failing on some platforms."
|
|
|
|
)
|
2018-07-24 21:38:44 +00:00
|
|
|
def test_serialize_tagger_roundtrip_bytes(en_vocab, taggers):
|
2018-11-30 16:43:08 +00:00
|
|
|
tagger1 = taggers[0]
|
2018-07-24 21:38:44 +00:00
|
|
|
tagger1_b = tagger1.to_bytes()
|
|
|
|
tagger1 = tagger1.from_bytes(tagger1_b)
|
|
|
|
assert tagger1.to_bytes() == tagger1_b
|
2020-07-25 13:01:15 +00:00
|
|
|
cfg = {"model": DEFAULT_TAGGER_MODEL}
|
|
|
|
model = registry.make_from_config(cfg, validate=True)["model"]
|
2020-07-22 11:42:59 +00:00
|
|
|
new_tagger1 = Tagger(en_vocab, model).from_bytes(tagger1_b)
|
2020-02-27 17:42:27 +00:00
|
|
|
new_tagger1_b = new_tagger1.to_bytes()
|
|
|
|
assert len(new_tagger1_b) == len(tagger1_b)
|
|
|
|
assert new_tagger1_b == tagger1_b
|
2018-07-24 21:38:44 +00:00
|
|
|
|
|
|
|
|
|
|
|
def test_serialize_tagger_roundtrip_disk(en_vocab, taggers):
|
|
|
|
tagger1, tagger2 = taggers
|
|
|
|
with make_tempdir() as d:
|
2018-11-27 00:09:36 +00:00
|
|
|
file_path1 = d / "tagger1"
|
|
|
|
file_path2 = d / "tagger2"
|
2018-07-24 21:38:44 +00:00
|
|
|
tagger1.to_disk(file_path1)
|
|
|
|
tagger2.to_disk(file_path2)
|
2020-07-25 13:01:15 +00:00
|
|
|
cfg = {"model": DEFAULT_TAGGER_MODEL}
|
|
|
|
model = registry.make_from_config(cfg, validate=True)["model"]
|
2020-08-07 13:27:13 +00:00
|
|
|
tagger1_d = Tagger(en_vocab, model).from_disk(file_path1)
|
|
|
|
tagger2_d = Tagger(en_vocab, model).from_disk(file_path2)
|
2018-07-24 21:38:44 +00:00
|
|
|
assert tagger1_d.to_bytes() == tagger2_d.to_bytes()
|
|
|
|
|
|
|
|
|
|
|
|
def test_serialize_textcat_empty(en_vocab):
|
|
|
|
# See issue #1105
|
2020-07-25 13:01:15 +00:00
|
|
|
cfg = {"model": DEFAULT_TEXTCAT_MODEL}
|
|
|
|
model = registry.make_from_config(cfg, validate=True)["model"]
|
2020-09-13 12:15:52 +00:00
|
|
|
textcat = TextCategorizer(en_vocab, model, labels=["ENTITY", "ACTION", "MODIFIER"], threshold=0.5)
|
2019-09-10 17:45:16 +00:00
|
|
|
textcat.to_bytes(exclude=["vocab"])
|
2019-03-10 18:16:45 +00:00
|
|
|
|
|
|
|
|
|
|
|
@pytest.mark.parametrize("Parser", test_parsers)
|
|
|
|
def test_serialize_pipe_exclude(en_vocab, Parser):
|
2020-07-25 13:01:15 +00:00
|
|
|
cfg = {"model": DEFAULT_PARSER_MODEL}
|
|
|
|
model = registry.make_from_config(cfg, validate=True)["model"]
|
2020-07-22 11:42:59 +00:00
|
|
|
config = {
|
|
|
|
"learn_tokens": False,
|
|
|
|
"min_action_freq": 0,
|
|
|
|
"update_with_oracle_cut_size": 100,
|
|
|
|
}
|
2020-07-25 13:01:15 +00:00
|
|
|
|
2019-03-10 18:16:45 +00:00
|
|
|
def get_new_parser():
|
2020-07-22 11:42:59 +00:00
|
|
|
new_parser = Parser(en_vocab, model, **config)
|
2019-03-10 18:16:45 +00:00
|
|
|
return new_parser
|
|
|
|
|
2020-07-22 11:42:59 +00:00
|
|
|
parser = Parser(en_vocab, model, **config)
|
2019-03-10 18:16:45 +00:00
|
|
|
parser.cfg["foo"] = "bar"
|
2019-09-10 17:45:16 +00:00
|
|
|
new_parser = get_new_parser().from_bytes(parser.to_bytes(exclude=["vocab"]))
|
2019-03-10 18:16:45 +00:00
|
|
|
assert "foo" in new_parser.cfg
|
2019-09-10 17:45:16 +00:00
|
|
|
new_parser = get_new_parser().from_bytes(
|
|
|
|
parser.to_bytes(exclude=["vocab"]), exclude=["cfg"]
|
|
|
|
)
|
2019-03-10 18:16:45 +00:00
|
|
|
assert "foo" not in new_parser.cfg
|
2019-09-10 17:45:16 +00:00
|
|
|
new_parser = get_new_parser().from_bytes(
|
|
|
|
parser.to_bytes(exclude=["cfg"]), exclude=["vocab"]
|
|
|
|
)
|
2019-03-10 18:16:45 +00:00
|
|
|
assert "foo" not in new_parser.cfg
|
2019-11-28 10:10:07 +00:00
|
|
|
|
|
|
|
|
|
|
|
def test_serialize_sentencerecognizer(en_vocab):
|
2020-07-25 13:01:15 +00:00
|
|
|
cfg = {"model": DEFAULT_SENTER_MODEL}
|
|
|
|
model = registry.make_from_config(cfg, validate=True)["model"]
|
2020-07-22 11:42:59 +00:00
|
|
|
sr = SentenceRecognizer(en_vocab, model)
|
2019-11-28 10:10:07 +00:00
|
|
|
sr_b = sr.to_bytes()
|
2020-07-22 11:42:59 +00:00
|
|
|
sr_d = SentenceRecognizer(en_vocab, model).from_bytes(sr_b)
|
2019-11-28 10:10:07 +00:00
|
|
|
assert sr.to_bytes() == sr_d.to_bytes()
|
2020-08-28 13:20:14 +00:00
|
|
|
|
|
|
|
|
|
|
|
def test_serialize_pipeline_disable_enable():
|
|
|
|
nlp = English()
|
|
|
|
nlp.add_pipe("ner")
|
|
|
|
nlp.add_pipe("tagger")
|
|
|
|
nlp.disable_pipe("tagger")
|
|
|
|
assert nlp.config["nlp"]["disabled"] == ["tagger"]
|
|
|
|
config = nlp.config.copy()
|
|
|
|
nlp2 = English.from_config(config)
|
|
|
|
assert nlp2.pipe_names == ["ner"]
|
2020-08-28 19:04:02 +00:00
|
|
|
assert nlp2.component_names == ["ner", "tagger"]
|
2020-08-29 10:08:33 +00:00
|
|
|
assert nlp2.disabled == ["tagger"]
|
2020-08-28 13:20:14 +00:00
|
|
|
assert nlp2.config["nlp"]["disabled"] == ["tagger"]
|
|
|
|
with make_tempdir() as d:
|
|
|
|
nlp2.to_disk(d)
|
|
|
|
nlp3 = spacy.load(d)
|
|
|
|
assert nlp3.pipe_names == ["ner"]
|
2020-08-28 19:04:02 +00:00
|
|
|
assert nlp3.component_names == ["ner", "tagger"]
|
2020-08-28 13:20:14 +00:00
|
|
|
with make_tempdir() as d:
|
|
|
|
nlp3.to_disk(d)
|
|
|
|
nlp4 = spacy.load(d, disable=["ner"])
|
|
|
|
assert nlp4.pipe_names == []
|
2020-08-28 19:04:02 +00:00
|
|
|
assert nlp4.component_names == ["ner", "tagger"]
|
2020-08-29 10:08:33 +00:00
|
|
|
assert nlp4.disabled == ["ner", "tagger"]
|
2020-08-28 13:20:14 +00:00
|
|
|
with make_tempdir() as d:
|
|
|
|
nlp.to_disk(d)
|
|
|
|
nlp5 = spacy.load(d, exclude=["tagger"])
|
|
|
|
assert nlp5.pipe_names == ["ner"]
|
2020-08-28 19:04:02 +00:00
|
|
|
assert nlp5.component_names == ["ner"]
|
2020-08-29 10:08:33 +00:00
|
|
|
assert nlp5.disabled == []
|