2017-10-09 01:42:35 +00:00
|
|
|
import pytest
|
2020-02-18 14:38:18 +00:00
|
|
|
from thinc.api import Adam, NumpyOps
|
2018-07-24 21:38:44 +00:00
|
|
|
from spacy.attrs import NORM
|
|
|
|
from spacy.gold import GoldParse
|
|
|
|
from spacy.vocab import Vocab
|
2020-02-27 17:42:27 +00:00
|
|
|
|
2020-05-19 14:20:03 +00:00
|
|
|
from spacy.pipeline.defaults import default_parser, default_ner
|
2018-07-24 21:38:44 +00:00
|
|
|
from spacy.tokens import Doc
|
2019-03-23 11:35:29 +00:00
|
|
|
from spacy.pipeline import DependencyParser, EntityRecognizer
|
|
|
|
from spacy.util import fix_random_seed
|
2017-10-09 01:42:35 +00:00
|
|
|
|
|
|
|
|
|
|
|
@pytest.fixture
|
|
|
|
def vocab():
|
|
|
|
return Vocab(lex_attr_getters={NORM: lambda s: s})
|
|
|
|
|
|
|
|
|
|
|
|
@pytest.fixture
|
|
|
|
def parser(vocab):
|
2020-02-27 17:42:27 +00:00
|
|
|
parser = DependencyParser(vocab, default_parser())
|
2019-03-23 11:35:29 +00:00
|
|
|
return parser
|
|
|
|
|
|
|
|
|
|
|
|
def test_init_parser(parser):
|
|
|
|
pass
|
|
|
|
|
|
|
|
|
|
|
|
def _train_parser(parser):
|
|
|
|
fix_random_seed(1)
|
2018-11-27 00:09:36 +00:00
|
|
|
parser.add_label("left")
|
2017-10-09 01:42:35 +00:00
|
|
|
parser.begin_training([], **parser.cfg)
|
2020-05-15 11:25:00 +00:00
|
|
|
sgd = Adam(0.001)
|
2017-10-09 01:42:35 +00:00
|
|
|
|
2019-03-23 12:46:25 +00:00
|
|
|
for i in range(5):
|
2017-10-09 01:42:35 +00:00
|
|
|
losses = {}
|
2019-03-23 11:35:29 +00:00
|
|
|
doc = Doc(parser.vocab, words=["a", "b", "c", "d"])
|
2018-11-27 00:09:36 +00:00
|
|
|
gold = GoldParse(doc, heads=[1, 1, 3, 3], deps=["left", "ROOT", "left", "ROOT"])
|
2019-11-11 16:35:27 +00:00
|
|
|
parser.update((doc, gold), sgd=sgd, losses=losses)
|
2017-10-09 01:42:35 +00:00
|
|
|
return parser
|
|
|
|
|
2018-07-24 21:38:44 +00:00
|
|
|
|
2017-10-09 01:42:35 +00:00
|
|
|
def test_add_label(parser):
|
2019-03-23 11:35:29 +00:00
|
|
|
parser = _train_parser(parser)
|
2018-11-27 00:09:36 +00:00
|
|
|
parser.add_label("right")
|
2020-05-15 11:25:00 +00:00
|
|
|
sgd = Adam(0.001)
|
2020-01-29 16:06:46 +00:00
|
|
|
for i in range(100):
|
2017-10-09 01:42:35 +00:00
|
|
|
losses = {}
|
2018-11-27 00:09:36 +00:00
|
|
|
doc = Doc(parser.vocab, words=["a", "b", "c", "d"])
|
|
|
|
gold = GoldParse(
|
|
|
|
doc, heads=[1, 1, 3, 3], deps=["right", "ROOT", "left", "ROOT"]
|
|
|
|
)
|
2019-11-11 16:35:27 +00:00
|
|
|
parser.update((doc, gold), sgd=sgd, losses=losses)
|
2018-11-27 00:09:36 +00:00
|
|
|
doc = Doc(parser.vocab, words=["a", "b", "c", "d"])
|
2017-10-09 01:42:35 +00:00
|
|
|
doc = parser(doc)
|
2018-11-27 00:09:36 +00:00
|
|
|
assert doc[0].dep_ == "right"
|
|
|
|
assert doc[2].dep_ == "left"
|
2019-03-23 11:35:29 +00:00
|
|
|
|
|
|
|
|
|
|
|
def test_add_label_deserializes_correctly():
|
2020-02-27 17:42:27 +00:00
|
|
|
ner1 = EntityRecognizer(Vocab(), default_ner())
|
2019-03-23 11:35:29 +00:00
|
|
|
ner1.add_label("C")
|
|
|
|
ner1.add_label("B")
|
|
|
|
ner1.add_label("A")
|
|
|
|
ner1.begin_training([])
|
2020-02-27 17:42:27 +00:00
|
|
|
ner2 = EntityRecognizer(Vocab(), default_ner())
|
|
|
|
|
|
|
|
# the second model needs to be resized before we can call from_bytes
|
2020-05-18 20:23:33 +00:00
|
|
|
ner2.model.attrs["resize_output"](ner2.model, ner1.moves.n_moves)
|
2020-02-27 17:42:27 +00:00
|
|
|
ner2.from_bytes(ner1.to_bytes())
|
2019-03-23 11:35:29 +00:00
|
|
|
assert ner1.moves.n_moves == ner2.moves.n_moves
|
|
|
|
for i in range(ner1.moves.n_moves):
|
|
|
|
assert ner1.moves.get_class_name(i) == ner2.moves.get_class_name(i)
|
2019-09-11 16:29:35 +00:00
|
|
|
|
|
|
|
|
|
|
|
@pytest.mark.parametrize(
|
2020-02-28 10:57:41 +00:00
|
|
|
"pipe_cls,n_moves,model",
|
|
|
|
[(DependencyParser, 5, default_parser()), (EntityRecognizer, 4, default_ner())],
|
2019-09-11 16:29:35 +00:00
|
|
|
)
|
2020-02-27 17:42:27 +00:00
|
|
|
def test_add_label_get_label(pipe_cls, n_moves, model):
|
2019-09-11 16:29:35 +00:00
|
|
|
"""Test that added labels are returned correctly. This test was added to
|
|
|
|
test for a bug in DependencyParser.labels that'd cause it to fail when
|
|
|
|
splitting the move names.
|
|
|
|
"""
|
|
|
|
labels = ["A", "B", "C"]
|
2020-02-27 17:42:27 +00:00
|
|
|
pipe = pipe_cls(Vocab(), model)
|
2019-09-11 16:29:35 +00:00
|
|
|
for label in labels:
|
|
|
|
pipe.add_label(label)
|
|
|
|
assert len(pipe.move_names) == len(labels) * n_moves
|
|
|
|
pipe_labels = sorted(list(pipe.labels))
|
|
|
|
assert pipe_labels == labels
|