spaCy/spacy/tests/pipeline/test_morphologizer.py

64 lines
1.8 KiB
Python
Raw Normal View History

import pytest
from spacy import util
from spacy.lang.en import English
from spacy.language import Language
from spacy.tests.util import make_tempdir
def test_label_types():
nlp = Language()
nlp.add_pipe(nlp.create_pipe("morphologizer"))
nlp.get_pipe("morphologizer").add_label("Feat=A")
with pytest.raises(ValueError):
nlp.get_pipe("morphologizer").add_label(9)
TRAIN_DATA = [
2020-06-20 12:15:04 +00:00
(
"I like green eggs",
{
"morphs": ["Feat=N", "Feat=V", "Feat=J", "Feat=N"],
"pos": ["NOUN", "VERB", "ADJ", "NOUN"],
},
),
(
"Eat blue ham",
{"morphs": ["Feat=V", "Feat=J", "Feat=N"], "pos": ["VERB", "ADJ", "NOUN"]},
),
]
def test_overfitting_IO():
# Simple test to try and quickly overfit the morphologizer - ensuring the ML models work correctly
nlp = English()
morphologizer = nlp.create_pipe("morphologizer")
for inst in TRAIN_DATA:
for morph, pos in zip(inst[1]["morphs"], inst[1]["pos"]):
morphologizer.add_label(morph + "|POS=" + pos)
nlp.add_pipe(morphologizer)
optimizer = nlp.begin_training()
for i in range(50):
losses = {}
nlp.update(TRAIN_DATA, sgd=optimizer, losses=losses)
assert losses["morphologizer"] < 0.00001
# test the trained model
test_text = "I like blue eggs"
doc = nlp(test_text)
2020-06-20 12:15:04 +00:00
gold_morphs = [
"Feat=N|POS=NOUN",
"Feat=V|POS=VERB",
"Feat=J|POS=ADJ",
"Feat=N|POS=NOUN",
]
assert gold_morphs == [t.morph_ for t in doc]
# Also test the results are still the same after IO
with make_tempdir() as tmp_dir:
nlp.to_disk(tmp_dir)
nlp2 = util.load_model_from_path(tmp_dir)
doc2 = nlp2(test_text)
assert gold_morphs == [t.morph_ for t in doc2]