2018-07-24 21:38:44 +00:00
|
|
|
import random
|
2021-12-04 19:34:48 +00:00
|
|
|
|
2018-07-24 21:38:44 +00:00
|
|
|
import numpy.random
|
2021-12-04 19:34:48 +00:00
|
|
|
import pytest
|
2021-04-22 12:58:29 +00:00
|
|
|
from numpy.testing import assert_almost_equal
|
2021-12-04 19:34:48 +00:00
|
|
|
from thinc.api import Config, compounding, fix_random_seed, get_current_ops
|
|
|
|
from wasabi import msg
|
|
|
|
|
|
|
|
import spacy
|
2020-02-27 17:42:27 +00:00
|
|
|
from spacy import util
|
2021-12-04 19:34:48 +00:00
|
|
|
from spacy.cli.evaluate import print_prf_per_type, print_textcats_auc_per_cat
|
2020-02-27 17:42:27 +00:00
|
|
|
from spacy.lang.en import English
|
2018-07-24 21:38:44 +00:00
|
|
|
from spacy.language import Language
|
|
|
|
from spacy.pipeline import TextCategorizer
|
2021-12-04 19:34:48 +00:00
|
|
|
from spacy.pipeline.textcat import single_label_bow_config
|
|
|
|
from spacy.pipeline.textcat import single_label_cnn_config
|
|
|
|
from spacy.pipeline.textcat import single_label_default_config
|
|
|
|
from spacy.pipeline.textcat_multilabel import multi_label_bow_config
|
|
|
|
from spacy.pipeline.textcat_multilabel import multi_label_cnn_config
|
|
|
|
from spacy.pipeline.textcat_multilabel import multi_label_default_config
|
2020-07-22 11:42:59 +00:00
|
|
|
from spacy.pipeline.tok2vec import DEFAULT_TOK2VEC_MODEL
|
2020-09-24 08:31:17 +00:00
|
|
|
from spacy.scorer import Scorer
|
2021-12-04 19:34:48 +00:00
|
|
|
from spacy.tokens import Doc, DocBin
|
2020-09-28 13:09:59 +00:00
|
|
|
from spacy.training import Example
|
2021-12-04 19:34:48 +00:00
|
|
|
from spacy.training.initialize import init_nlp
|
2017-11-06 21:04:29 +00:00
|
|
|
|
2020-03-29 17:40:36 +00:00
|
|
|
from ..util import make_tempdir
|
|
|
|
|
2021-01-06 02:07:14 +00:00
|
|
|
TRAIN_DATA_SINGLE_LABEL = [
|
2020-01-29 16:06:46 +00:00
|
|
|
("I'm so happy.", {"cats": {"POSITIVE": 1.0, "NEGATIVE": 0.0}}),
|
|
|
|
("I'm so angry", {"cats": {"POSITIVE": 0.0, "NEGATIVE": 1.0}}),
|
|
|
|
]
|
|
|
|
|
2021-01-06 02:07:14 +00:00
|
|
|
TRAIN_DATA_MULTI_LABEL = [
|
|
|
|
("I'm angry and confused", {"cats": {"ANGRY": 1.0, "CONFUSED": 1.0, "HAPPY": 0.0}}),
|
|
|
|
("I'm confused but happy", {"cats": {"ANGRY": 0.0, "CONFUSED": 1.0, "HAPPY": 1.0}}),
|
|
|
|
]
|
|
|
|
|
2017-11-07 00:25:54 +00:00
|
|
|
|
2021-01-06 02:07:14 +00:00
|
|
|
def make_get_examples_single_label(nlp):
|
2020-10-03 15:07:38 +00:00
|
|
|
train_examples = []
|
2021-01-06 02:07:14 +00:00
|
|
|
for t in TRAIN_DATA_SINGLE_LABEL:
|
|
|
|
train_examples.append(Example.from_dict(nlp.make_doc(t[0]), t[1]))
|
|
|
|
|
|
|
|
def get_examples():
|
|
|
|
return train_examples
|
|
|
|
|
|
|
|
return get_examples
|
|
|
|
|
|
|
|
|
|
|
|
def make_get_examples_multi_label(nlp):
|
|
|
|
train_examples = []
|
|
|
|
for t in TRAIN_DATA_MULTI_LABEL:
|
2020-10-03 15:07:38 +00:00
|
|
|
train_examples.append(Example.from_dict(nlp.make_doc(t[0]), t[1]))
|
|
|
|
|
|
|
|
def get_examples():
|
|
|
|
return train_examples
|
|
|
|
|
|
|
|
return get_examples
|
|
|
|
|
|
|
|
|
2021-12-04 19:34:48 +00:00
|
|
|
@pytest.mark.issue(3611)
|
|
|
|
def test_issue3611():
|
|
|
|
"""Test whether adding n-grams in the textcat works even when n > token length of some docs"""
|
|
|
|
unique_classes = ["offensive", "inoffensive"]
|
|
|
|
x_train = [
|
|
|
|
"This is an offensive text",
|
|
|
|
"This is the second offensive text",
|
|
|
|
"inoff",
|
|
|
|
]
|
|
|
|
y_train = ["offensive", "offensive", "inoffensive"]
|
|
|
|
nlp = spacy.blank("en")
|
|
|
|
# preparing the data
|
|
|
|
train_data = []
|
|
|
|
for text, train_instance in zip(x_train, y_train):
|
|
|
|
cat_dict = {label: label == train_instance for label in unique_classes}
|
|
|
|
train_data.append(Example.from_dict(nlp.make_doc(text), {"cats": cat_dict}))
|
|
|
|
# add a text categorizer component
|
|
|
|
model = {
|
|
|
|
"@architectures": "spacy.TextCatBOW.v1",
|
|
|
|
"exclusive_classes": True,
|
|
|
|
"ngram_size": 2,
|
|
|
|
"no_output_layer": False,
|
|
|
|
}
|
|
|
|
textcat = nlp.add_pipe("textcat", config={"model": model}, last=True)
|
|
|
|
for label in unique_classes:
|
|
|
|
textcat.add_label(label)
|
|
|
|
# training the network
|
|
|
|
with nlp.select_pipes(enable="textcat"):
|
|
|
|
optimizer = nlp.initialize()
|
|
|
|
for i in range(3):
|
|
|
|
losses = {}
|
|
|
|
batches = util.minibatch(train_data, size=compounding(4.0, 32.0, 1.001))
|
|
|
|
|
|
|
|
for batch in batches:
|
|
|
|
nlp.update(examples=batch, sgd=optimizer, drop=0.1, losses=losses)
|
|
|
|
|
|
|
|
|
|
|
|
@pytest.mark.issue(4030)
|
|
|
|
def test_issue4030():
|
|
|
|
"""Test whether textcat works fine with empty doc"""
|
|
|
|
unique_classes = ["offensive", "inoffensive"]
|
|
|
|
x_train = [
|
|
|
|
"This is an offensive text",
|
|
|
|
"This is the second offensive text",
|
|
|
|
"inoff",
|
|
|
|
]
|
|
|
|
y_train = ["offensive", "offensive", "inoffensive"]
|
|
|
|
nlp = spacy.blank("en")
|
|
|
|
# preparing the data
|
|
|
|
train_data = []
|
|
|
|
for text, train_instance in zip(x_train, y_train):
|
|
|
|
cat_dict = {label: label == train_instance for label in unique_classes}
|
|
|
|
train_data.append(Example.from_dict(nlp.make_doc(text), {"cats": cat_dict}))
|
|
|
|
# add a text categorizer component
|
|
|
|
model = {
|
|
|
|
"@architectures": "spacy.TextCatBOW.v1",
|
|
|
|
"exclusive_classes": True,
|
|
|
|
"ngram_size": 2,
|
|
|
|
"no_output_layer": False,
|
|
|
|
}
|
|
|
|
textcat = nlp.add_pipe("textcat", config={"model": model}, last=True)
|
|
|
|
for label in unique_classes:
|
|
|
|
textcat.add_label(label)
|
|
|
|
# training the network
|
|
|
|
with nlp.select_pipes(enable="textcat"):
|
|
|
|
optimizer = nlp.initialize()
|
|
|
|
for i in range(3):
|
|
|
|
losses = {}
|
|
|
|
batches = util.minibatch(train_data, size=compounding(4.0, 32.0, 1.001))
|
|
|
|
|
|
|
|
for batch in batches:
|
|
|
|
nlp.update(examples=batch, sgd=optimizer, drop=0.1, losses=losses)
|
|
|
|
# processing of an empty doc should result in 0.0 for all categories
|
|
|
|
doc = nlp("")
|
|
|
|
assert doc.cats["offensive"] == 0.0
|
|
|
|
assert doc.cats["inoffensive"] == 0.0
|
|
|
|
|
|
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
|
|
"textcat_config",
|
|
|
|
[
|
|
|
|
single_label_default_config,
|
|
|
|
single_label_bow_config,
|
|
|
|
single_label_cnn_config,
|
|
|
|
multi_label_default_config,
|
|
|
|
multi_label_bow_config,
|
|
|
|
multi_label_cnn_config,
|
|
|
|
],
|
|
|
|
)
|
|
|
|
@pytest.mark.issue(5551)
|
|
|
|
def test_issue5551(textcat_config):
|
|
|
|
"""Test that after fixing the random seed, the results of the pipeline are truly identical"""
|
|
|
|
component = "textcat"
|
|
|
|
|
|
|
|
pipe_cfg = Config().from_str(textcat_config)
|
|
|
|
results = []
|
|
|
|
for i in range(3):
|
|
|
|
fix_random_seed(0)
|
|
|
|
nlp = English()
|
|
|
|
text = "Once hot, form ping-pong-ball-sized balls of the mixture, each weighing roughly 25 g."
|
|
|
|
annots = {"cats": {"Labe1": 1.0, "Label2": 0.0, "Label3": 0.0}}
|
|
|
|
pipe = nlp.add_pipe(component, config=pipe_cfg, last=True)
|
|
|
|
for label in set(annots["cats"]):
|
|
|
|
pipe.add_label(label)
|
|
|
|
# Train
|
|
|
|
nlp.initialize()
|
|
|
|
doc = nlp.make_doc(text)
|
|
|
|
nlp.update([Example.from_dict(doc, annots)])
|
|
|
|
# Store the result of each iteration
|
|
|
|
result = pipe.model.predict([doc])
|
|
|
|
results.append(result[0])
|
|
|
|
# All results should be the same because of the fixed seed
|
|
|
|
assert len(results) == 3
|
|
|
|
ops = get_current_ops()
|
|
|
|
assert_almost_equal(ops.to_numpy(results[0]), ops.to_numpy(results[1]), decimal=5)
|
|
|
|
assert_almost_equal(ops.to_numpy(results[0]), ops.to_numpy(results[2]), decimal=5)
|
|
|
|
|
|
|
|
|
|
|
|
CONFIG_ISSUE_6908 = """
|
|
|
|
[paths]
|
|
|
|
train = "TRAIN_PLACEHOLDER"
|
|
|
|
raw = null
|
|
|
|
init_tok2vec = null
|
|
|
|
vectors = null
|
|
|
|
|
|
|
|
[system]
|
|
|
|
seed = 0
|
|
|
|
gpu_allocator = null
|
|
|
|
|
|
|
|
[nlp]
|
|
|
|
lang = "en"
|
|
|
|
pipeline = ["textcat"]
|
|
|
|
tokenizer = {"@tokenizers":"spacy.Tokenizer.v1"}
|
|
|
|
disabled = []
|
|
|
|
before_creation = null
|
|
|
|
after_creation = null
|
|
|
|
after_pipeline_creation = null
|
|
|
|
batch_size = 1000
|
|
|
|
|
|
|
|
[components]
|
|
|
|
|
|
|
|
[components.textcat]
|
|
|
|
factory = "TEXTCAT_PLACEHOLDER"
|
|
|
|
|
|
|
|
[corpora]
|
|
|
|
|
|
|
|
[corpora.train]
|
|
|
|
@readers = "spacy.Corpus.v1"
|
|
|
|
path = ${paths:train}
|
|
|
|
|
|
|
|
[corpora.dev]
|
|
|
|
@readers = "spacy.Corpus.v1"
|
|
|
|
path = ${paths:train}
|
|
|
|
|
|
|
|
|
|
|
|
[training]
|
|
|
|
train_corpus = "corpora.train"
|
|
|
|
dev_corpus = "corpora.dev"
|
|
|
|
seed = ${system.seed}
|
|
|
|
gpu_allocator = ${system.gpu_allocator}
|
|
|
|
frozen_components = []
|
|
|
|
before_to_disk = null
|
|
|
|
|
|
|
|
[pretraining]
|
|
|
|
|
|
|
|
[initialize]
|
|
|
|
vectors = ${paths.vectors}
|
|
|
|
init_tok2vec = ${paths.init_tok2vec}
|
|
|
|
vocab_data = null
|
|
|
|
lookups = null
|
|
|
|
before_init = null
|
|
|
|
after_init = null
|
|
|
|
|
|
|
|
[initialize.components]
|
|
|
|
|
|
|
|
[initialize.components.textcat]
|
|
|
|
labels = ['label1', 'label2']
|
|
|
|
|
|
|
|
[initialize.tokenizer]
|
|
|
|
"""
|
|
|
|
|
|
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
|
|
"component_name",
|
|
|
|
["textcat", "textcat_multilabel"],
|
|
|
|
)
|
|
|
|
@pytest.mark.issue(6908)
|
|
|
|
def test_issue6908(component_name):
|
|
|
|
"""Test intializing textcat with labels in a list"""
|
|
|
|
|
|
|
|
def create_data(out_file):
|
|
|
|
nlp = spacy.blank("en")
|
|
|
|
doc = nlp.make_doc("Some text")
|
|
|
|
doc.cats = {"label1": 0, "label2": 1}
|
|
|
|
out_data = DocBin(docs=[doc]).to_bytes()
|
|
|
|
with out_file.open("wb") as file_:
|
|
|
|
file_.write(out_data)
|
|
|
|
|
|
|
|
with make_tempdir() as tmp_path:
|
|
|
|
train_path = tmp_path / "train.spacy"
|
|
|
|
create_data(train_path)
|
|
|
|
config_str = CONFIG_ISSUE_6908.replace("TEXTCAT_PLACEHOLDER", component_name)
|
|
|
|
config_str = config_str.replace("TRAIN_PLACEHOLDER", train_path.as_posix())
|
|
|
|
config = util.load_config_from_str(config_str)
|
|
|
|
init_nlp(config)
|
|
|
|
|
|
|
|
|
|
|
|
@pytest.mark.issue(7019)
|
|
|
|
def test_issue7019():
|
|
|
|
scores = {"LABEL_A": 0.39829102, "LABEL_B": 0.938298329382, "LABEL_C": None}
|
|
|
|
print_textcats_auc_per_cat(msg, scores)
|
|
|
|
scores = {
|
|
|
|
"LABEL_A": {"p": 0.3420302, "r": 0.3929020, "f": 0.49823928932},
|
|
|
|
"LABEL_B": {"p": None, "r": None, "f": None},
|
|
|
|
}
|
|
|
|
print_prf_per_type(msg, scores, name="foo", type="bar")
|
|
|
|
|
|
|
|
|
2018-08-15 14:56:55 +00:00
|
|
|
@pytest.mark.skip(reason="Test is flakey when run with others")
|
2017-11-06 21:04:29 +00:00
|
|
|
def test_simple_train():
|
|
|
|
nlp = Language()
|
2020-07-22 11:42:59 +00:00
|
|
|
textcat = nlp.add_pipe("textcat")
|
|
|
|
textcat.add_label("answer")
|
2020-09-28 19:35:09 +00:00
|
|
|
nlp.initialize()
|
2017-11-06 21:04:29 +00:00
|
|
|
for i in range(5):
|
2018-11-27 00:09:36 +00:00
|
|
|
for text, answer in [
|
|
|
|
("aaaa", 1.0),
|
|
|
|
("bbbb", 0),
|
|
|
|
("aa", 1.0),
|
|
|
|
("bbbbbbbbb", 0.0),
|
|
|
|
("aaaaaa", 1),
|
|
|
|
]:
|
2019-11-11 16:35:27 +00:00
|
|
|
nlp.update((text, {"cats": {"answer": answer}}))
|
2018-11-27 00:09:36 +00:00
|
|
|
doc = nlp("aaa")
|
|
|
|
assert "answer" in doc.cats
|
|
|
|
assert doc.cats["answer"] >= 0.5
|
2018-07-24 21:38:44 +00:00
|
|
|
|
|
|
|
|
|
|
|
@pytest.mark.skip(reason="Test is flakey when run with others")
|
|
|
|
def test_textcat_learns_multilabel():
|
|
|
|
random.seed(5)
|
|
|
|
numpy.random.seed(5)
|
|
|
|
docs = []
|
|
|
|
nlp = Language()
|
2018-11-27 00:09:36 +00:00
|
|
|
letters = ["a", "b", "c"]
|
2018-07-24 21:38:44 +00:00
|
|
|
for w1 in letters:
|
|
|
|
for w2 in letters:
|
2018-11-27 00:09:36 +00:00
|
|
|
cats = {letter: float(w2 == letter) for letter in letters}
|
|
|
|
docs.append((Doc(nlp.vocab, words=["d"] * 3 + [w1, w2] + ["d"] * 3), cats))
|
2018-07-24 21:38:44 +00:00
|
|
|
random.shuffle(docs)
|
2020-06-26 17:34:12 +00:00
|
|
|
textcat = TextCategorizer(nlp.vocab, width=8)
|
2018-07-24 21:38:44 +00:00
|
|
|
for letter in letters:
|
2020-06-26 17:34:12 +00:00
|
|
|
textcat.add_label(letter)
|
2020-09-28 19:35:09 +00:00
|
|
|
optimizer = textcat.initialize(lambda: [])
|
2018-07-24 21:38:44 +00:00
|
|
|
for i in range(30):
|
|
|
|
losses = {}
|
2020-06-26 17:34:12 +00:00
|
|
|
examples = [Example.from_dict(doc, {"cats": cats}) for doc, cat in docs]
|
|
|
|
textcat.update(examples, sgd=optimizer, losses=losses)
|
2018-07-24 21:38:44 +00:00
|
|
|
random.shuffle(docs)
|
|
|
|
for w1 in letters:
|
|
|
|
for w2 in letters:
|
2018-11-27 00:09:36 +00:00
|
|
|
doc = Doc(nlp.vocab, words=["d"] * 3 + [w1, w2] + ["d"] * 3)
|
|
|
|
truth = {letter: w2 == letter for letter in letters}
|
2020-06-26 17:34:12 +00:00
|
|
|
textcat(doc)
|
2018-07-24 21:38:44 +00:00
|
|
|
for cat, score in doc.cats.items():
|
|
|
|
if not truth[cat]:
|
|
|
|
assert score < 0.5
|
|
|
|
else:
|
|
|
|
assert score > 0.5
|
2019-11-21 15:24:10 +00:00
|
|
|
|
|
|
|
|
2021-01-06 02:07:14 +00:00
|
|
|
@pytest.mark.parametrize("name", ["textcat", "textcat_multilabel"])
|
|
|
|
def test_label_types(name):
|
2019-11-21 15:24:10 +00:00
|
|
|
nlp = Language()
|
2021-01-06 02:07:14 +00:00
|
|
|
textcat = nlp.add_pipe(name)
|
2020-07-22 11:42:59 +00:00
|
|
|
textcat.add_label("answer")
|
2019-11-21 15:24:10 +00:00
|
|
|
with pytest.raises(ValueError):
|
2020-07-22 11:42:59 +00:00
|
|
|
textcat.add_label(9)
|
2021-07-06 10:35:22 +00:00
|
|
|
# textcat requires at least two labels
|
|
|
|
if name == "textcat":
|
|
|
|
with pytest.raises(ValueError):
|
|
|
|
nlp.initialize()
|
|
|
|
else:
|
|
|
|
nlp.initialize()
|
2020-01-29 16:06:46 +00:00
|
|
|
|
|
|
|
|
2021-01-06 02:07:14 +00:00
|
|
|
@pytest.mark.parametrize("name", ["textcat", "textcat_multilabel"])
|
|
|
|
def test_no_label(name):
|
2020-09-08 20:44:25 +00:00
|
|
|
nlp = Language()
|
2021-01-06 02:07:14 +00:00
|
|
|
nlp.add_pipe(name)
|
2020-09-08 20:44:25 +00:00
|
|
|
with pytest.raises(ValueError):
|
2020-09-28 19:35:09 +00:00
|
|
|
nlp.initialize()
|
2020-09-08 20:44:25 +00:00
|
|
|
|
|
|
|
|
2021-01-06 02:07:14 +00:00
|
|
|
@pytest.mark.parametrize(
|
|
|
|
"name,get_examples",
|
|
|
|
[
|
|
|
|
("textcat", make_get_examples_single_label),
|
|
|
|
("textcat_multilabel", make_get_examples_multi_label),
|
|
|
|
],
|
|
|
|
)
|
|
|
|
def test_implicit_label(name, get_examples):
|
2020-09-08 20:44:25 +00:00
|
|
|
nlp = Language()
|
2021-01-06 02:07:14 +00:00
|
|
|
nlp.add_pipe(name)
|
|
|
|
nlp.initialize(get_examples=get_examples(nlp))
|
2020-09-08 20:44:25 +00:00
|
|
|
|
|
|
|
|
2021-06-28 09:48:00 +00:00
|
|
|
# fmt: off
|
2021-06-16 09:45:00 +00:00
|
|
|
@pytest.mark.parametrize(
|
|
|
|
"name,textcat_config",
|
|
|
|
[
|
|
|
|
# BOW
|
|
|
|
("textcat", {"@architectures": "spacy.TextCatBOW.v1", "exclusive_classes": True, "no_output_layer": False, "ngram_size": 3}),
|
|
|
|
("textcat", {"@architectures": "spacy.TextCatBOW.v1", "exclusive_classes": True, "no_output_layer": True, "ngram_size": 3}),
|
|
|
|
("textcat_multilabel", {"@architectures": "spacy.TextCatBOW.v1", "exclusive_classes": False, "no_output_layer": False, "ngram_size": 3}),
|
|
|
|
("textcat_multilabel", {"@architectures": "spacy.TextCatBOW.v1", "exclusive_classes": False, "no_output_layer": True, "ngram_size": 3}),
|
|
|
|
# ENSEMBLE
|
|
|
|
("textcat", {"@architectures": "spacy.TextCatEnsemble.v2", "tok2vec": DEFAULT_TOK2VEC_MODEL, "linear_model": {"@architectures": "spacy.TextCatBOW.v1", "exclusive_classes": True, "no_output_layer": False, "ngram_size": 3}}),
|
|
|
|
("textcat", {"@architectures": "spacy.TextCatEnsemble.v2", "tok2vec": DEFAULT_TOK2VEC_MODEL, "linear_model": {"@architectures": "spacy.TextCatBOW.v1", "exclusive_classes": True, "no_output_layer": True, "ngram_size": 3}}),
|
|
|
|
("textcat_multilabel", {"@architectures": "spacy.TextCatEnsemble.v2", "tok2vec": DEFAULT_TOK2VEC_MODEL, "linear_model": {"@architectures": "spacy.TextCatBOW.v1", "exclusive_classes": False, "no_output_layer": False, "ngram_size": 3}}),
|
|
|
|
("textcat_multilabel", {"@architectures": "spacy.TextCatEnsemble.v2", "tok2vec": DEFAULT_TOK2VEC_MODEL, "linear_model": {"@architectures": "spacy.TextCatBOW.v1", "exclusive_classes": False, "no_output_layer": True, "ngram_size": 3}}),
|
|
|
|
# CNN
|
|
|
|
("textcat", {"@architectures": "spacy.TextCatCNN.v1", "tok2vec": DEFAULT_TOK2VEC_MODEL, "exclusive_classes": True}),
|
|
|
|
("textcat_multilabel", {"@architectures": "spacy.TextCatCNN.v1", "tok2vec": DEFAULT_TOK2VEC_MODEL, "exclusive_classes": False}),
|
|
|
|
],
|
|
|
|
)
|
2021-06-28 09:48:00 +00:00
|
|
|
# fmt: on
|
2021-06-16 09:45:00 +00:00
|
|
|
def test_no_resize(name, textcat_config):
|
|
|
|
"""The old textcat architectures weren't resizable"""
|
2020-09-08 20:44:25 +00:00
|
|
|
nlp = Language()
|
2021-06-16 09:45:00 +00:00
|
|
|
pipe_config = {"model": textcat_config}
|
|
|
|
textcat = nlp.add_pipe(name, config=pipe_config)
|
2020-09-08 20:44:25 +00:00
|
|
|
textcat.add_label("POSITIVE")
|
|
|
|
textcat.add_label("NEGATIVE")
|
2020-09-28 19:35:09 +00:00
|
|
|
nlp.initialize()
|
2021-06-16 09:45:00 +00:00
|
|
|
assert textcat.model.maybe_get_dim("nO") in [2, None]
|
2020-09-08 20:44:25 +00:00
|
|
|
# this throws an error because the textcat can't be resized after initialization
|
|
|
|
with pytest.raises(ValueError):
|
|
|
|
textcat.add_label("NEUTRAL")
|
|
|
|
|
|
|
|
|
2021-06-28 09:48:00 +00:00
|
|
|
# fmt: off
|
2021-06-16 09:45:00 +00:00
|
|
|
@pytest.mark.parametrize(
|
|
|
|
"name,textcat_config",
|
|
|
|
[
|
|
|
|
# BOW
|
|
|
|
("textcat", {"@architectures": "spacy.TextCatBOW.v2", "exclusive_classes": True, "no_output_layer": False, "ngram_size": 3}),
|
|
|
|
("textcat", {"@architectures": "spacy.TextCatBOW.v2", "exclusive_classes": True, "no_output_layer": True, "ngram_size": 3}),
|
|
|
|
("textcat_multilabel", {"@architectures": "spacy.TextCatBOW.v2", "exclusive_classes": False, "no_output_layer": False, "ngram_size": 3}),
|
|
|
|
("textcat_multilabel", {"@architectures": "spacy.TextCatBOW.v2", "exclusive_classes": False, "no_output_layer": True, "ngram_size": 3}),
|
|
|
|
# CNN
|
|
|
|
("textcat", {"@architectures": "spacy.TextCatCNN.v2", "tok2vec": DEFAULT_TOK2VEC_MODEL, "exclusive_classes": True}),
|
|
|
|
("textcat_multilabel", {"@architectures": "spacy.TextCatCNN.v2", "tok2vec": DEFAULT_TOK2VEC_MODEL, "exclusive_classes": False}),
|
|
|
|
],
|
|
|
|
)
|
2021-06-28 09:48:00 +00:00
|
|
|
# fmt: on
|
2021-06-16 09:45:00 +00:00
|
|
|
def test_resize(name, textcat_config):
|
|
|
|
"""The new textcat architectures are resizable"""
|
|
|
|
nlp = Language()
|
|
|
|
pipe_config = {"model": textcat_config}
|
|
|
|
textcat = nlp.add_pipe(name, config=pipe_config)
|
|
|
|
textcat.add_label("POSITIVE")
|
|
|
|
textcat.add_label("NEGATIVE")
|
|
|
|
assert textcat.model.maybe_get_dim("nO") in [2, None]
|
|
|
|
nlp.initialize()
|
|
|
|
assert textcat.model.maybe_get_dim("nO") in [2, None]
|
|
|
|
textcat.add_label("NEUTRAL")
|
|
|
|
assert textcat.model.maybe_get_dim("nO") in [3, None]
|
|
|
|
|
|
|
|
|
2021-06-28 09:48:00 +00:00
|
|
|
# fmt: off
|
2021-06-16 09:45:00 +00:00
|
|
|
@pytest.mark.parametrize(
|
|
|
|
"name,textcat_config",
|
|
|
|
[
|
|
|
|
# BOW
|
|
|
|
("textcat", {"@architectures": "spacy.TextCatBOW.v2", "exclusive_classes": True, "no_output_layer": False, "ngram_size": 3}),
|
|
|
|
("textcat", {"@architectures": "spacy.TextCatBOW.v2", "exclusive_classes": True, "no_output_layer": True, "ngram_size": 3}),
|
|
|
|
("textcat_multilabel", {"@architectures": "spacy.TextCatBOW.v2", "exclusive_classes": False, "no_output_layer": False, "ngram_size": 3}),
|
|
|
|
("textcat_multilabel", {"@architectures": "spacy.TextCatBOW.v2", "exclusive_classes": False, "no_output_layer": True, "ngram_size": 3}),
|
|
|
|
# CNN
|
|
|
|
("textcat", {"@architectures": "spacy.TextCatCNN.v2", "tok2vec": DEFAULT_TOK2VEC_MODEL, "exclusive_classes": True}),
|
|
|
|
("textcat_multilabel", {"@architectures": "spacy.TextCatCNN.v2", "tok2vec": DEFAULT_TOK2VEC_MODEL, "exclusive_classes": False}),
|
|
|
|
],
|
|
|
|
)
|
2021-06-28 09:48:00 +00:00
|
|
|
# fmt: on
|
2021-06-16 09:45:00 +00:00
|
|
|
def test_resize_same_results(name, textcat_config):
|
|
|
|
# Ensure that the resized textcat classifiers still produce the same results for old labels
|
|
|
|
fix_random_seed(0)
|
|
|
|
nlp = English()
|
|
|
|
pipe_config = {"model": textcat_config}
|
|
|
|
textcat = nlp.add_pipe(name, config=pipe_config)
|
|
|
|
|
|
|
|
train_examples = []
|
|
|
|
for text, annotations in TRAIN_DATA_SINGLE_LABEL:
|
|
|
|
train_examples.append(Example.from_dict(nlp.make_doc(text), annotations))
|
|
|
|
optimizer = nlp.initialize(get_examples=lambda: train_examples)
|
|
|
|
assert textcat.model.maybe_get_dim("nO") in [2, None]
|
|
|
|
|
|
|
|
for i in range(5):
|
|
|
|
losses = {}
|
|
|
|
nlp.update(train_examples, sgd=optimizer, losses=losses)
|
|
|
|
|
|
|
|
# test the trained model before resizing
|
|
|
|
test_text = "I am happy."
|
|
|
|
doc = nlp(test_text)
|
|
|
|
assert len(doc.cats) == 2
|
|
|
|
pos_pred = doc.cats["POSITIVE"]
|
|
|
|
neg_pred = doc.cats["NEGATIVE"]
|
|
|
|
|
|
|
|
# test the trained model again after resizing
|
|
|
|
textcat.add_label("NEUTRAL")
|
|
|
|
doc = nlp(test_text)
|
|
|
|
assert len(doc.cats) == 3
|
|
|
|
assert doc.cats["POSITIVE"] == pos_pred
|
|
|
|
assert doc.cats["NEGATIVE"] == neg_pred
|
|
|
|
assert doc.cats["NEUTRAL"] <= 1
|
|
|
|
|
|
|
|
for i in range(5):
|
|
|
|
losses = {}
|
|
|
|
nlp.update(train_examples, sgd=optimizer, losses=losses)
|
|
|
|
|
|
|
|
# test the trained model again after training further with new label
|
|
|
|
doc = nlp(test_text)
|
|
|
|
assert len(doc.cats) == 3
|
|
|
|
assert doc.cats["POSITIVE"] != pos_pred
|
|
|
|
assert doc.cats["NEGATIVE"] != neg_pred
|
|
|
|
for cat in doc.cats:
|
|
|
|
assert doc.cats[cat] <= 1
|
|
|
|
|
|
|
|
|
2021-01-06 02:07:14 +00:00
|
|
|
def test_error_with_multi_labels():
|
2020-09-08 20:44:25 +00:00
|
|
|
nlp = Language()
|
2021-01-15 00:57:36 +00:00
|
|
|
nlp.add_pipe("textcat")
|
2021-01-06 02:07:14 +00:00
|
|
|
train_examples = []
|
|
|
|
for text, annotations in TRAIN_DATA_MULTI_LABEL:
|
|
|
|
train_examples.append(Example.from_dict(nlp.make_doc(text), annotations))
|
|
|
|
with pytest.raises(ValueError):
|
2021-01-15 00:57:36 +00:00
|
|
|
nlp.initialize(get_examples=lambda: train_examples)
|
2021-01-06 02:07:14 +00:00
|
|
|
|
|
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
|
|
"name,get_examples, train_data",
|
|
|
|
[
|
|
|
|
("textcat", make_get_examples_single_label, TRAIN_DATA_SINGLE_LABEL),
|
|
|
|
("textcat_multilabel", make_get_examples_multi_label, TRAIN_DATA_MULTI_LABEL),
|
|
|
|
],
|
|
|
|
)
|
|
|
|
def test_initialize_examples(name, get_examples, train_data):
|
|
|
|
nlp = Language()
|
|
|
|
textcat = nlp.add_pipe(name)
|
|
|
|
for text, annotations in train_data:
|
2020-09-08 20:44:25 +00:00
|
|
|
for label, value in annotations.get("cats").items():
|
|
|
|
textcat.add_label(label)
|
|
|
|
# you shouldn't really call this more than once, but for testing it should be fine
|
2020-09-28 19:35:09 +00:00
|
|
|
nlp.initialize()
|
2021-01-06 02:07:14 +00:00
|
|
|
nlp.initialize(get_examples=get_examples(nlp))
|
2020-10-08 19:33:49 +00:00
|
|
|
with pytest.raises(TypeError):
|
2020-09-28 19:35:09 +00:00
|
|
|
nlp.initialize(get_examples=lambda: None)
|
2020-10-08 19:33:49 +00:00
|
|
|
with pytest.raises(TypeError):
|
2020-10-03 15:07:38 +00:00
|
|
|
nlp.initialize(get_examples=get_examples())
|
2020-09-08 20:44:25 +00:00
|
|
|
|
|
|
|
|
2020-02-27 17:42:27 +00:00
|
|
|
def test_overfitting_IO():
|
2020-12-08 22:29:15 +00:00
|
|
|
# Simple test to try and quickly overfit the single-label textcat component - ensuring the ML models work correctly
|
2020-04-02 12:46:32 +00:00
|
|
|
fix_random_seed(0)
|
2020-02-27 17:42:27 +00:00
|
|
|
nlp = English()
|
2021-01-06 02:07:14 +00:00
|
|
|
textcat = nlp.add_pipe("textcat")
|
|
|
|
|
2020-07-06 11:02:36 +00:00
|
|
|
train_examples = []
|
2021-01-06 02:07:14 +00:00
|
|
|
for text, annotations in TRAIN_DATA_SINGLE_LABEL:
|
2020-07-06 11:02:36 +00:00
|
|
|
train_examples.append(Example.from_dict(nlp.make_doc(text), annotations))
|
2020-09-28 19:35:09 +00:00
|
|
|
optimizer = nlp.initialize(get_examples=lambda: train_examples)
|
2020-09-08 20:44:25 +00:00
|
|
|
assert textcat.model.get_dim("nO") == 2
|
2020-01-29 16:06:46 +00:00
|
|
|
|
|
|
|
for i in range(50):
|
|
|
|
losses = {}
|
2020-07-06 11:02:36 +00:00
|
|
|
nlp.update(train_examples, sgd=optimizer, losses=losses)
|
2020-02-27 17:42:27 +00:00
|
|
|
assert losses["textcat"] < 0.01
|
2020-01-29 16:06:46 +00:00
|
|
|
|
|
|
|
# test the trained model
|
|
|
|
test_text = "I am happy."
|
|
|
|
doc = nlp(test_text)
|
|
|
|
cats = doc.cats
|
2020-09-02 11:07:41 +00:00
|
|
|
assert cats["POSITIVE"] > 0.9
|
|
|
|
assert cats["POSITIVE"] + cats["NEGATIVE"] == pytest.approx(1.0, 0.001)
|
2020-02-27 17:42:27 +00:00
|
|
|
|
|
|
|
# Also test the results are still the same after IO
|
|
|
|
with make_tempdir() as tmp_dir:
|
|
|
|
nlp.to_disk(tmp_dir)
|
|
|
|
nlp2 = util.load_model_from_path(tmp_dir)
|
|
|
|
doc2 = nlp2(test_text)
|
|
|
|
cats2 = doc2.cats
|
2020-09-02 11:07:41 +00:00
|
|
|
assert cats2["POSITIVE"] > 0.9
|
|
|
|
assert cats2["POSITIVE"] + cats2["NEGATIVE"] == pytest.approx(1.0, 0.001)
|
2020-03-29 17:40:36 +00:00
|
|
|
|
Refactor the Scorer to improve flexibility (#5731)
* Refactor the Scorer to improve flexibility
Refactor the `Scorer` to improve flexibility for arbitrary pipeline
components.
* Individual pipeline components provide their own `evaluate` methods
that score a list of `Example`s and return a dictionary of scores
* `Scorer` is initialized either:
* with a provided pipeline containing components to be scored
* with a default pipeline containing the built-in statistical
components (senter, tagger, morphologizer, parser, ner)
* `Scorer.score` evaluates a list of `Example`s and returns a dictionary
of scores referring to the scores provided by the components in the
pipeline
Significant differences:
* `tags_acc` is renamed to `tag_acc` to be consistent with `token_acc`
and the new `morph_acc`, `pos_acc`, and `lemma_acc`
* Scoring is no longer cumulative: `Scorer.score` scores a list of
examples rather than a single example and does not retain any state
about previously scored examples
* PRF values in the returned scores are no longer multiplied by 100
* Add kwargs to Morphologizer.evaluate
* Create generalized scoring methods in Scorer
* Generalized static scoring methods are added to `Scorer`
* Methods require an attribute (either on Token or Doc) that is
used to key the returned scores
Naming differences:
* `uas`, `las`, and `las_per_type` in the scores dict are renamed to
`dep_uas`, `dep_las`, and `dep_las_per_type`
Scoring differences:
* `Doc.sents` is now scored as spans rather than on sentence-initial
token positions so that `Doc.sents` and `Doc.ents` can be scored with
the same method (this lowers scores since a single incorrect sentence
start results in two incorrect spans)
* Simplify / extend hasattr check for eval method
* Add hasattr check to tokenizer scoring
* Simplify to hasattr check for component scoring
* Reset Example alignment if docs are set
Reset the Example alignment if either doc is set in case the
tokenization has changed.
* Add PRF tokenization scoring for tokens as spans
Add PRF scores for tokens as character spans. The scores are:
* token_acc: # correct tokens / # gold tokens
* token_p/r/f: PRF for (token.idx, token.idx + len(token))
* Add docstring to Scorer.score_tokenization
* Rename component.evaluate() to component.score()
* Update Scorer API docs
* Update scoring for positive_label in textcat
* Fix TextCategorizer.score kwargs
* Update Language.evaluate docs
* Update score names in default config
2020-07-25 10:53:02 +00:00
|
|
|
# Test scoring
|
2020-09-14 15:08:00 +00:00
|
|
|
scores = nlp.evaluate(train_examples)
|
2020-08-06 14:24:13 +00:00
|
|
|
assert scores["cats_micro_f"] == 1.0
|
2021-01-06 02:07:14 +00:00
|
|
|
assert scores["cats_macro_f"] == 1.0
|
|
|
|
assert scores["cats_macro_auc"] == 1.0
|
2020-07-27 09:17:52 +00:00
|
|
|
assert scores["cats_score"] == 1.0
|
|
|
|
assert "cats_score_desc" in scores
|
Refactor the Scorer to improve flexibility (#5731)
* Refactor the Scorer to improve flexibility
Refactor the `Scorer` to improve flexibility for arbitrary pipeline
components.
* Individual pipeline components provide their own `evaluate` methods
that score a list of `Example`s and return a dictionary of scores
* `Scorer` is initialized either:
* with a provided pipeline containing components to be scored
* with a default pipeline containing the built-in statistical
components (senter, tagger, morphologizer, parser, ner)
* `Scorer.score` evaluates a list of `Example`s and returns a dictionary
of scores referring to the scores provided by the components in the
pipeline
Significant differences:
* `tags_acc` is renamed to `tag_acc` to be consistent with `token_acc`
and the new `morph_acc`, `pos_acc`, and `lemma_acc`
* Scoring is no longer cumulative: `Scorer.score` scores a list of
examples rather than a single example and does not retain any state
about previously scored examples
* PRF values in the returned scores are no longer multiplied by 100
* Add kwargs to Morphologizer.evaluate
* Create generalized scoring methods in Scorer
* Generalized static scoring methods are added to `Scorer`
* Methods require an attribute (either on Token or Doc) that is
used to key the returned scores
Naming differences:
* `uas`, `las`, and `las_per_type` in the scores dict are renamed to
`dep_uas`, `dep_las`, and `dep_las_per_type`
Scoring differences:
* `Doc.sents` is now scored as spans rather than on sentence-initial
token positions so that `Doc.sents` and `Doc.ents` can be scored with
the same method (this lowers scores since a single incorrect sentence
start results in two incorrect spans)
* Simplify / extend hasattr check for eval method
* Add hasattr check to tokenizer scoring
* Simplify to hasattr check for component scoring
* Reset Example alignment if docs are set
Reset the Example alignment if either doc is set in case the
tokenization has changed.
* Add PRF tokenization scoring for tokens as spans
Add PRF scores for tokens as character spans. The scores are:
* token_acc: # correct tokens / # gold tokens
* token_p/r/f: PRF for (token.idx, token.idx + len(token))
* Add docstring to Scorer.score_tokenization
* Rename component.evaluate() to component.score()
* Update Scorer API docs
* Update scoring for positive_label in textcat
* Fix TextCategorizer.score kwargs
* Update Language.evaluate docs
* Update score names in default config
2020-07-25 10:53:02 +00:00
|
|
|
|
2020-10-13 19:07:13 +00:00
|
|
|
# Make sure that running pipe twice, or comparing to call, always amounts to the same predictions
|
|
|
|
texts = ["Just a sentence.", "I like green eggs.", "I am happy.", "I eat ham."]
|
2020-12-08 22:29:15 +00:00
|
|
|
batch_cats_1 = [doc.cats for doc in nlp.pipe(texts)]
|
|
|
|
batch_cats_2 = [doc.cats for doc in nlp.pipe(texts)]
|
|
|
|
no_batch_cats = [doc.cats for doc in [nlp(text) for text in texts]]
|
2021-04-22 12:58:29 +00:00
|
|
|
for cats_1, cats_2 in zip(batch_cats_1, batch_cats_2):
|
|
|
|
for cat in cats_1:
|
|
|
|
assert_almost_equal(cats_1[cat], cats_2[cat], decimal=5)
|
|
|
|
for cats_1, cats_2 in zip(batch_cats_1, no_batch_cats):
|
|
|
|
for cat in cats_1:
|
|
|
|
assert_almost_equal(cats_1[cat], cats_2[cat], decimal=5)
|
2020-12-08 22:29:15 +00:00
|
|
|
|
|
|
|
|
2021-01-15 01:51:02 +00:00
|
|
|
def test_overfitting_IO_multi():
|
2021-01-06 02:07:14 +00:00
|
|
|
# Simple test to try and quickly overfit the multi-label textcat component - ensuring the ML models work correctly
|
|
|
|
fix_random_seed(0)
|
|
|
|
nlp = English()
|
|
|
|
textcat = nlp.add_pipe("textcat_multilabel")
|
|
|
|
|
|
|
|
train_examples = []
|
|
|
|
for text, annotations in TRAIN_DATA_MULTI_LABEL:
|
|
|
|
train_examples.append(Example.from_dict(nlp.make_doc(text), annotations))
|
|
|
|
optimizer = nlp.initialize(get_examples=lambda: train_examples)
|
|
|
|
assert textcat.model.get_dim("nO") == 3
|
|
|
|
|
|
|
|
for i in range(100):
|
|
|
|
losses = {}
|
|
|
|
nlp.update(train_examples, sgd=optimizer, losses=losses)
|
|
|
|
assert losses["textcat_multilabel"] < 0.01
|
|
|
|
|
|
|
|
# test the trained model
|
|
|
|
test_text = "I am confused but happy."
|
|
|
|
doc = nlp(test_text)
|
|
|
|
cats = doc.cats
|
|
|
|
assert cats["HAPPY"] > 0.9
|
|
|
|
assert cats["CONFUSED"] > 0.9
|
|
|
|
|
|
|
|
# Also test the results are still the same after IO
|
|
|
|
with make_tempdir() as tmp_dir:
|
|
|
|
nlp.to_disk(tmp_dir)
|
|
|
|
nlp2 = util.load_model_from_path(tmp_dir)
|
|
|
|
doc2 = nlp2(test_text)
|
|
|
|
cats2 = doc2.cats
|
|
|
|
assert cats2["HAPPY"] > 0.9
|
|
|
|
assert cats2["CONFUSED"] > 0.9
|
|
|
|
|
|
|
|
# Test scoring
|
|
|
|
scores = nlp.evaluate(train_examples)
|
|
|
|
assert scores["cats_micro_f"] == 1.0
|
|
|
|
assert scores["cats_macro_f"] == 1.0
|
|
|
|
assert "cats_score_desc" in scores
|
|
|
|
|
|
|
|
# Make sure that running pipe twice, or comparing to call, always amounts to the same predictions
|
|
|
|
texts = ["Just a sentence.", "I like green eggs.", "I am happy.", "I eat ham."]
|
|
|
|
batch_deps_1 = [doc.cats for doc in nlp.pipe(texts)]
|
|
|
|
batch_deps_2 = [doc.cats for doc in nlp.pipe(texts)]
|
|
|
|
no_batch_deps = [doc.cats for doc in [nlp(text) for text in texts]]
|
2021-04-22 12:58:29 +00:00
|
|
|
for cats_1, cats_2 in zip(batch_deps_1, batch_deps_2):
|
|
|
|
for cat in cats_1:
|
|
|
|
assert_almost_equal(cats_1[cat], cats_2[cat], decimal=5)
|
|
|
|
for cats_1, cats_2 in zip(batch_deps_1, no_batch_deps):
|
|
|
|
for cat in cats_1:
|
|
|
|
assert_almost_equal(cats_1[cat], cats_2[cat], decimal=5)
|
2021-01-06 02:07:14 +00:00
|
|
|
|
|
|
|
|
2020-03-29 17:40:36 +00:00
|
|
|
# fmt: off
|
|
|
|
@pytest.mark.parametrize(
|
2021-01-06 02:07:14 +00:00
|
|
|
"name,train_data,textcat_config",
|
2020-03-29 17:40:36 +00:00
|
|
|
[
|
2021-06-16 09:45:00 +00:00
|
|
|
("textcat_multilabel", TRAIN_DATA_MULTI_LABEL, {"@architectures": "spacy.TextCatBOW.v2", "exclusive_classes": False, "ngram_size": 1, "no_output_layer": False}),
|
|
|
|
("textcat", TRAIN_DATA_SINGLE_LABEL, {"@architectures": "spacy.TextCatBOW.v2", "exclusive_classes": True, "ngram_size": 4, "no_output_layer": False}),
|
|
|
|
("textcat_multilabel", TRAIN_DATA_MULTI_LABEL, {"@architectures": "spacy.TextCatBOW.v2", "exclusive_classes": False, "ngram_size": 3, "no_output_layer": True}),
|
|
|
|
("textcat", TRAIN_DATA_SINGLE_LABEL, {"@architectures": "spacy.TextCatBOW.v2", "exclusive_classes": True, "ngram_size": 2, "no_output_layer": True}),
|
|
|
|
("textcat_multilabel", TRAIN_DATA_MULTI_LABEL, {"@architectures": "spacy.TextCatEnsemble.v2", "tok2vec": DEFAULT_TOK2VEC_MODEL, "linear_model": {"@architectures": "spacy.TextCatBOW.v2", "exclusive_classes": False, "ngram_size": 1, "no_output_layer": False}}),
|
|
|
|
("textcat", TRAIN_DATA_SINGLE_LABEL, {"@architectures": "spacy.TextCatEnsemble.v2", "tok2vec": DEFAULT_TOK2VEC_MODEL, "linear_model": {"@architectures": "spacy.TextCatBOW.v2", "exclusive_classes": True, "ngram_size": 5, "no_output_layer": False}}),
|
|
|
|
("textcat", TRAIN_DATA_SINGLE_LABEL, {"@architectures": "spacy.TextCatCNN.v2", "tok2vec": DEFAULT_TOK2VEC_MODEL, "exclusive_classes": True}),
|
|
|
|
("textcat_multilabel", TRAIN_DATA_MULTI_LABEL, {"@architectures": "spacy.TextCatCNN.v2", "tok2vec": DEFAULT_TOK2VEC_MODEL, "exclusive_classes": False}),
|
2020-03-29 17:40:36 +00:00
|
|
|
],
|
|
|
|
)
|
|
|
|
# fmt: on
|
2021-01-06 02:07:14 +00:00
|
|
|
def test_textcat_configs(name, train_data, textcat_config):
|
2020-03-29 17:40:36 +00:00
|
|
|
pipe_config = {"model": textcat_config}
|
|
|
|
nlp = English()
|
2021-01-06 02:07:14 +00:00
|
|
|
textcat = nlp.add_pipe(name, config=pipe_config)
|
2020-07-06 11:02:36 +00:00
|
|
|
train_examples = []
|
2021-01-06 02:07:14 +00:00
|
|
|
for text, annotations in train_data:
|
2020-07-06 11:02:36 +00:00
|
|
|
train_examples.append(Example.from_dict(nlp.make_doc(text), annotations))
|
2020-03-29 17:40:36 +00:00
|
|
|
for label, value in annotations.get("cats").items():
|
|
|
|
textcat.add_label(label)
|
2020-09-28 19:35:09 +00:00
|
|
|
optimizer = nlp.initialize()
|
2020-03-29 17:40:36 +00:00
|
|
|
for i in range(5):
|
|
|
|
losses = {}
|
2020-07-06 11:02:36 +00:00
|
|
|
nlp.update(train_examples, sgd=optimizer, losses=losses)
|
2020-09-14 15:08:00 +00:00
|
|
|
|
|
|
|
|
|
|
|
def test_positive_class():
|
|
|
|
nlp = English()
|
2020-10-03 15:07:38 +00:00
|
|
|
textcat = nlp.add_pipe("textcat")
|
2021-01-06 02:07:14 +00:00
|
|
|
get_examples = make_get_examples_single_label(nlp)
|
2020-10-03 15:07:38 +00:00
|
|
|
textcat.initialize(get_examples, labels=["POS", "NEG"], positive_label="POS")
|
2020-09-14 15:08:00 +00:00
|
|
|
assert textcat.labels == ("POS", "NEG")
|
2021-01-06 02:07:14 +00:00
|
|
|
assert textcat.cfg["positive_label"] == "POS"
|
|
|
|
|
|
|
|
textcat_multilabel = nlp.add_pipe("textcat_multilabel")
|
|
|
|
get_examples = make_get_examples_multi_label(nlp)
|
|
|
|
with pytest.raises(TypeError):
|
2021-01-15 00:57:36 +00:00
|
|
|
textcat_multilabel.initialize(
|
|
|
|
get_examples, labels=["POS", "NEG"], positive_label="POS"
|
|
|
|
)
|
2021-01-06 02:07:14 +00:00
|
|
|
textcat_multilabel.initialize(get_examples, labels=["FICTION", "DRAMA"])
|
|
|
|
assert textcat_multilabel.labels == ("FICTION", "DRAMA")
|
|
|
|
assert "positive_label" not in textcat_multilabel.cfg
|
2020-09-14 15:08:00 +00:00
|
|
|
|
|
|
|
|
|
|
|
def test_positive_class_not_present():
|
|
|
|
nlp = English()
|
2020-10-03 15:07:38 +00:00
|
|
|
textcat = nlp.add_pipe("textcat")
|
2021-01-06 02:07:14 +00:00
|
|
|
get_examples = make_get_examples_single_label(nlp)
|
2020-09-14 15:08:00 +00:00
|
|
|
with pytest.raises(ValueError):
|
2020-10-03 15:07:38 +00:00
|
|
|
textcat.initialize(get_examples, labels=["SOME", "THING"], positive_label="POS")
|
2020-09-14 15:08:00 +00:00
|
|
|
|
|
|
|
|
|
|
|
def test_positive_class_not_binary():
|
|
|
|
nlp = English()
|
2020-10-03 15:07:38 +00:00
|
|
|
textcat = nlp.add_pipe("textcat")
|
2021-01-06 02:07:14 +00:00
|
|
|
get_examples = make_get_examples_multi_label(nlp)
|
2020-09-14 15:08:00 +00:00
|
|
|
with pytest.raises(ValueError):
|
2021-01-15 00:57:36 +00:00
|
|
|
textcat.initialize(
|
|
|
|
get_examples, labels=["SOME", "THING", "POS"], positive_label="POS"
|
|
|
|
)
|
2020-09-24 08:31:17 +00:00
|
|
|
|
2020-09-29 19:39:28 +00:00
|
|
|
|
2020-09-24 08:31:17 +00:00
|
|
|
def test_textcat_evaluation():
|
|
|
|
train_examples = []
|
|
|
|
nlp = English()
|
|
|
|
ref1 = nlp("one")
|
|
|
|
ref1.cats = {"winter": 1.0, "summer": 1.0, "spring": 1.0, "autumn": 1.0}
|
|
|
|
pred1 = nlp("one")
|
|
|
|
pred1.cats = {"winter": 1.0, "summer": 0.0, "spring": 1.0, "autumn": 1.0}
|
|
|
|
train_examples.append(Example(pred1, ref1))
|
|
|
|
|
|
|
|
ref2 = nlp("two")
|
|
|
|
ref2.cats = {"winter": 0.0, "summer": 0.0, "spring": 1.0, "autumn": 1.0}
|
|
|
|
pred2 = nlp("two")
|
|
|
|
pred2.cats = {"winter": 1.0, "summer": 0.0, "spring": 0.0, "autumn": 1.0}
|
|
|
|
train_examples.append(Example(pred2, ref2))
|
|
|
|
|
2020-09-29 19:39:28 +00:00
|
|
|
scores = Scorer().score_cats(
|
|
|
|
train_examples, "cats", labels=["winter", "summer", "spring", "autumn"]
|
|
|
|
)
|
|
|
|
assert scores["cats_f_per_type"]["winter"]["p"] == 1 / 2
|
|
|
|
assert scores["cats_f_per_type"]["winter"]["r"] == 1 / 1
|
2020-09-24 08:31:17 +00:00
|
|
|
assert scores["cats_f_per_type"]["summer"]["p"] == 0
|
2020-09-29 19:39:28 +00:00
|
|
|
assert scores["cats_f_per_type"]["summer"]["r"] == 0 / 1
|
|
|
|
assert scores["cats_f_per_type"]["spring"]["p"] == 1 / 1
|
|
|
|
assert scores["cats_f_per_type"]["spring"]["r"] == 1 / 2
|
|
|
|
assert scores["cats_f_per_type"]["autumn"]["p"] == 2 / 2
|
|
|
|
assert scores["cats_f_per_type"]["autumn"]["r"] == 2 / 2
|
|
|
|
|
|
|
|
assert scores["cats_micro_p"] == 4 / 5
|
|
|
|
assert scores["cats_micro_r"] == 4 / 6
|
2021-03-09 12:04:22 +00:00
|
|
|
|
|
|
|
|
|
|
|
def test_textcat_threshold():
|
|
|
|
# Ensure the scorer can be called with a different threshold
|
|
|
|
nlp = English()
|
|
|
|
nlp.add_pipe("textcat")
|
|
|
|
|
|
|
|
train_examples = []
|
|
|
|
for text, annotations in TRAIN_DATA_SINGLE_LABEL:
|
|
|
|
train_examples.append(Example.from_dict(nlp.make_doc(text), annotations))
|
|
|
|
nlp.initialize(get_examples=lambda: train_examples)
|
|
|
|
|
|
|
|
# score the model (it's not actually trained but that doesn't matter)
|
|
|
|
scores = nlp.evaluate(train_examples)
|
|
|
|
assert 0 <= scores["cats_score"] <= 1
|
|
|
|
|
|
|
|
scores = nlp.evaluate(train_examples, scorer_cfg={"threshold": 1.0})
|
|
|
|
assert scores["cats_f_per_type"]["POSITIVE"]["r"] == 0
|
|
|
|
|
|
|
|
scores = nlp.evaluate(train_examples, scorer_cfg={"threshold": 0})
|
|
|
|
macro_f = scores["cats_score"]
|
|
|
|
assert scores["cats_f_per_type"]["POSITIVE"]["r"] == 1.0
|
|
|
|
|
2021-06-28 09:48:00 +00:00
|
|
|
scores = nlp.evaluate(
|
|
|
|
train_examples, scorer_cfg={"threshold": 0, "positive_label": "POSITIVE"}
|
|
|
|
)
|
2021-03-09 12:04:22 +00:00
|
|
|
pos_f = scores["cats_score"]
|
|
|
|
assert scores["cats_f_per_type"]["POSITIVE"]["r"] == 1.0
|
|
|
|
assert pos_f > macro_f
|
|
|
|
|
|
|
|
|
|
|
|
def test_textcat_multi_threshold():
|
|
|
|
# Ensure the scorer can be called with a different threshold
|
|
|
|
nlp = English()
|
|
|
|
nlp.add_pipe("textcat_multilabel")
|
|
|
|
|
|
|
|
train_examples = []
|
|
|
|
for text, annotations in TRAIN_DATA_SINGLE_LABEL:
|
|
|
|
train_examples.append(Example.from_dict(nlp.make_doc(text), annotations))
|
|
|
|
nlp.initialize(get_examples=lambda: train_examples)
|
|
|
|
|
|
|
|
# score the model (it's not actually trained but that doesn't matter)
|
|
|
|
scores = nlp.evaluate(train_examples)
|
|
|
|
assert 0 <= scores["cats_score"] <= 1
|
|
|
|
|
|
|
|
scores = nlp.evaluate(train_examples, scorer_cfg={"threshold": 1.0})
|
|
|
|
assert scores["cats_f_per_type"]["POSITIVE"]["r"] == 0
|
|
|
|
|
|
|
|
scores = nlp.evaluate(train_examples, scorer_cfg={"threshold": 0})
|
|
|
|
assert scores["cats_f_per_type"]["POSITIVE"]["r"] == 1.0
|