2017-10-08 22:02:23 +00:00
|
|
|
import pytest
|
2020-02-18 14:38:18 +00:00
|
|
|
from thinc.api import Adam
|
2018-07-24 21:38:44 +00:00
|
|
|
from spacy.attrs import NORM
|
|
|
|
from spacy.gold import GoldParse
|
|
|
|
from spacy.vocab import Vocab
|
2020-02-27 17:42:27 +00:00
|
|
|
|
2020-05-19 14:20:03 +00:00
|
|
|
from spacy.pipeline.defaults import default_parser
|
2018-07-24 21:38:44 +00:00
|
|
|
from spacy.tokens import Doc
|
|
|
|
from spacy.pipeline import DependencyParser
|
2017-10-08 22:02:23 +00:00
|
|
|
|
|
|
|
|
|
|
|
@pytest.fixture
|
|
|
|
def vocab():
|
|
|
|
return Vocab(lex_attr_getters={NORM: lambda s: s})
|
|
|
|
|
2018-07-24 21:38:44 +00:00
|
|
|
|
2017-10-08 22:02:23 +00:00
|
|
|
@pytest.fixture
|
|
|
|
def parser(vocab):
|
2020-06-12 00:02:07 +00:00
|
|
|
config = {"learn_tokens": False, "min_action_freq": 30, "beam_width": 1, "beam_update_prob": 1.0}
|
|
|
|
parser = DependencyParser(vocab, default_parser(), **config)
|
2018-11-27 00:09:36 +00:00
|
|
|
parser.cfg["token_vector_width"] = 4
|
|
|
|
parser.cfg["hidden_width"] = 32
|
|
|
|
# parser.add_label('right')
|
|
|
|
parser.add_label("left")
|
2017-10-08 22:02:23 +00:00
|
|
|
parser.begin_training([], **parser.cfg)
|
2020-01-29 16:06:46 +00:00
|
|
|
sgd = Adam(0.001)
|
2017-10-08 22:02:23 +00:00
|
|
|
|
|
|
|
for i in range(10):
|
|
|
|
losses = {}
|
2018-11-27 00:09:36 +00:00
|
|
|
doc = Doc(vocab, words=["a", "b", "c", "d"])
|
|
|
|
gold = GoldParse(doc, heads=[1, 1, 3, 3], deps=["left", "ROOT", "left", "ROOT"])
|
2019-11-11 16:35:27 +00:00
|
|
|
parser.update((doc, gold), sgd=sgd, losses=losses)
|
2017-10-08 22:02:23 +00:00
|
|
|
return parser
|
|
|
|
|
2018-07-24 21:38:44 +00:00
|
|
|
|
2017-10-08 22:02:23 +00:00
|
|
|
def test_no_sentences(parser):
|
2018-11-27 00:09:36 +00:00
|
|
|
doc = Doc(parser.vocab, words=["a", "b", "c", "d"])
|
2017-10-08 22:02:23 +00:00
|
|
|
doc = parser(doc)
|
2017-10-12 19:18:22 +00:00
|
|
|
assert len(list(doc.sents)) >= 1
|
2017-10-08 22:02:23 +00:00
|
|
|
|
|
|
|
|
|
|
|
def test_sents_1(parser):
|
2018-11-27 00:09:36 +00:00
|
|
|
doc = Doc(parser.vocab, words=["a", "b", "c", "d"])
|
2017-10-08 22:02:23 +00:00
|
|
|
doc[2].sent_start = True
|
|
|
|
doc = parser(doc)
|
2017-10-08 22:29:37 +00:00
|
|
|
assert len(list(doc.sents)) >= 2
|
2018-11-27 00:09:36 +00:00
|
|
|
doc = Doc(parser.vocab, words=["a", "b", "c", "d"])
|
2017-10-08 22:02:23 +00:00
|
|
|
doc[1].sent_start = False
|
|
|
|
doc[2].sent_start = True
|
|
|
|
doc[3].sent_start = False
|
|
|
|
doc = parser(doc)
|
|
|
|
assert len(list(doc.sents)) == 2
|
|
|
|
|
|
|
|
|
|
|
|
def test_sents_1_2(parser):
|
2018-11-27 00:09:36 +00:00
|
|
|
doc = Doc(parser.vocab, words=["a", "b", "c", "d"])
|
2017-10-08 22:02:23 +00:00
|
|
|
doc[1].sent_start = True
|
|
|
|
doc[2].sent_start = True
|
|
|
|
doc = parser(doc)
|
2017-11-03 13:36:08 +00:00
|
|
|
assert len(list(doc.sents)) >= 3
|
2017-10-08 22:02:23 +00:00
|
|
|
|
|
|
|
|
|
|
|
def test_sents_1_3(parser):
|
2018-11-27 00:09:36 +00:00
|
|
|
doc = Doc(parser.vocab, words=["a", "b", "c", "d"])
|
2017-10-08 22:02:23 +00:00
|
|
|
doc[1].sent_start = True
|
|
|
|
doc[3].sent_start = True
|
|
|
|
doc = parser(doc)
|
2017-10-11 06:38:34 +00:00
|
|
|
assert len(list(doc.sents)) >= 3
|
2018-11-27 00:09:36 +00:00
|
|
|
doc = Doc(parser.vocab, words=["a", "b", "c", "d"])
|
2017-10-08 22:02:23 +00:00
|
|
|
doc[1].sent_start = True
|
|
|
|
doc[2].sent_start = False
|
|
|
|
doc[3].sent_start = True
|
|
|
|
doc = parser(doc)
|
|
|
|
assert len(list(doc.sents)) == 3
|