2020-09-08 20:44:25 +00:00
|
|
|
from itertools import islice
|
2020-07-25 13:01:15 +00:00
|
|
|
from typing import Iterable, Tuple, Optional, Dict, List, Callable, Iterator, Any
|
2020-07-22 11:42:59 +00:00
|
|
|
from thinc.api import get_array_module, Model, Optimizer, set_dropout_rate, Config
|
2020-08-09 20:28:29 +00:00
|
|
|
from thinc.types import Floats2d
|
2020-07-22 11:42:59 +00:00
|
|
|
import numpy
|
|
|
|
|
|
|
|
from .pipe import Pipe
|
|
|
|
from ..language import Language
|
2020-09-09 08:31:03 +00:00
|
|
|
from ..training import Example, validate_examples
|
2020-07-22 11:42:59 +00:00
|
|
|
from ..errors import Errors
|
Refactor the Scorer to improve flexibility (#5731)
* Refactor the Scorer to improve flexibility
Refactor the `Scorer` to improve flexibility for arbitrary pipeline
components.
* Individual pipeline components provide their own `evaluate` methods
that score a list of `Example`s and return a dictionary of scores
* `Scorer` is initialized either:
* with a provided pipeline containing components to be scored
* with a default pipeline containing the built-in statistical
components (senter, tagger, morphologizer, parser, ner)
* `Scorer.score` evaluates a list of `Example`s and returns a dictionary
of scores referring to the scores provided by the components in the
pipeline
Significant differences:
* `tags_acc` is renamed to `tag_acc` to be consistent with `token_acc`
and the new `morph_acc`, `pos_acc`, and `lemma_acc`
* Scoring is no longer cumulative: `Scorer.score` scores a list of
examples rather than a single example and does not retain any state
about previously scored examples
* PRF values in the returned scores are no longer multiplied by 100
* Add kwargs to Morphologizer.evaluate
* Create generalized scoring methods in Scorer
* Generalized static scoring methods are added to `Scorer`
* Methods require an attribute (either on Token or Doc) that is
used to key the returned scores
Naming differences:
* `uas`, `las`, and `las_per_type` in the scores dict are renamed to
`dep_uas`, `dep_las`, and `dep_las_per_type`
Scoring differences:
* `Doc.sents` is now scored as spans rather than on sentence-initial
token positions so that `Doc.sents` and `Doc.ents` can be scored with
the same method (this lowers scores since a single incorrect sentence
start results in two incorrect spans)
* Simplify / extend hasattr check for eval method
* Add hasattr check to tokenizer scoring
* Simplify to hasattr check for component scoring
* Reset Example alignment if docs are set
Reset the Example alignment if either doc is set in case the
tokenization has changed.
* Add PRF tokenization scoring for tokens as spans
Add PRF scores for tokens as character spans. The scores are:
* token_acc: # correct tokens / # gold tokens
* token_p/r/f: PRF for (token.idx, token.idx + len(token))
* Add docstring to Scorer.score_tokenization
* Rename component.evaluate() to component.score()
* Update Scorer API docs
* Update scoring for positive_label in textcat
* Fix TextCategorizer.score kwargs
* Update Language.evaluate docs
* Update score names in default config
2020-07-25 10:53:02 +00:00
|
|
|
from ..scorer import Scorer
|
2020-07-22 11:42:59 +00:00
|
|
|
from .. import util
|
|
|
|
from ..tokens import Doc
|
|
|
|
from ..vocab import Vocab
|
|
|
|
|
|
|
|
|
|
|
|
default_model_config = """
|
|
|
|
[model]
|
2020-07-26 13:11:43 +00:00
|
|
|
@architectures = "spacy.TextCatEnsemble.v1"
|
2020-07-22 11:42:59 +00:00
|
|
|
exclusive_classes = false
|
|
|
|
pretrained_vectors = null
|
|
|
|
width = 64
|
|
|
|
conv_depth = 2
|
|
|
|
embed_size = 2000
|
|
|
|
window_size = 1
|
|
|
|
ngram_size = 1
|
|
|
|
dropout = null
|
|
|
|
"""
|
|
|
|
DEFAULT_TEXTCAT_MODEL = Config().from_str(default_model_config)["model"]
|
|
|
|
|
|
|
|
bow_model_config = """
|
|
|
|
[model]
|
|
|
|
@architectures = "spacy.TextCatBOW.v1"
|
|
|
|
exclusive_classes = false
|
2020-08-06 17:30:53 +00:00
|
|
|
ngram_size = 1
|
|
|
|
no_output_layer = false
|
2020-07-22 11:42:59 +00:00
|
|
|
"""
|
|
|
|
|
|
|
|
cnn_model_config = """
|
|
|
|
[model]
|
|
|
|
@architectures = "spacy.TextCatCNN.v1"
|
|
|
|
exclusive_classes = false
|
|
|
|
|
|
|
|
[model.tok2vec]
|
|
|
|
@architectures = "spacy.HashEmbedCNN.v1"
|
|
|
|
pretrained_vectors = null
|
|
|
|
width = 96
|
|
|
|
depth = 4
|
|
|
|
embed_size = 2000
|
|
|
|
window_size = 1
|
|
|
|
maxout_pieces = 3
|
|
|
|
subword_features = true
|
|
|
|
"""
|
|
|
|
|
|
|
|
|
|
|
|
@Language.factory(
|
|
|
|
"textcat",
|
|
|
|
assigns=["doc.cats"],
|
2020-09-14 15:08:00 +00:00
|
|
|
default_config={
|
|
|
|
"labels": [],
|
|
|
|
"threshold": 0.5,
|
|
|
|
"positive_label": None,
|
|
|
|
"model": DEFAULT_TEXTCAT_MODEL,
|
|
|
|
},
|
2020-07-28 11:37:31 +00:00
|
|
|
scores=[
|
|
|
|
"cats_score",
|
|
|
|
"cats_score_desc",
|
|
|
|
"cats_p",
|
|
|
|
"cats_r",
|
|
|
|
"cats_f",
|
|
|
|
"cats_macro_f",
|
|
|
|
"cats_macro_auc",
|
|
|
|
"cats_f_per_type",
|
|
|
|
"cats_macro_auc_per_type",
|
|
|
|
],
|
2020-07-27 10:27:40 +00:00
|
|
|
default_score_weights={"cats_score": 1.0},
|
2020-07-22 11:42:59 +00:00
|
|
|
)
|
|
|
|
def make_textcat(
|
2020-08-09 20:28:29 +00:00
|
|
|
nlp: Language,
|
|
|
|
name: str,
|
|
|
|
model: Model[List[Doc], List[Floats2d]],
|
2020-09-14 15:08:00 +00:00
|
|
|
labels: List[str],
|
2020-09-13 12:15:52 +00:00
|
|
|
threshold: float,
|
2020-09-14 15:08:00 +00:00
|
|
|
positive_label: Optional[str],
|
2020-07-22 11:42:59 +00:00
|
|
|
) -> "TextCategorizer":
|
2020-08-09 13:07:09 +00:00
|
|
|
"""Create a TextCategorizer compoment. The text categorizer predicts categories
|
|
|
|
over a whole document. It can learn one or more labels, and the labels can
|
|
|
|
be mutually exclusive (i.e. one true label per doc) or non-mutually exclusive
|
|
|
|
(i.e. zero or more labels may be true per doc). The multi-label setting is
|
|
|
|
controlled by the model instance that's provided.
|
|
|
|
|
|
|
|
model (Model[List[Doc], List[Floats2d]]): A model instance that predicts
|
|
|
|
scores for each category.
|
|
|
|
labels (list): A list of categories to learn. If empty, the model infers the
|
|
|
|
categories from the data.
|
2020-09-13 12:15:52 +00:00
|
|
|
threshold (float): Cutoff to consider a prediction "positive".
|
2020-09-14 15:08:00 +00:00
|
|
|
positive_label (Optional[str]): The positive label for a binary task with exclusive classes, None otherwise.
|
2020-08-09 13:07:09 +00:00
|
|
|
"""
|
2020-09-14 15:08:00 +00:00
|
|
|
return TextCategorizer(
|
|
|
|
nlp.vocab,
|
|
|
|
model,
|
|
|
|
name,
|
|
|
|
labels=labels,
|
|
|
|
threshold=threshold,
|
|
|
|
positive_label=positive_label,
|
|
|
|
)
|
2020-07-22 11:42:59 +00:00
|
|
|
|
|
|
|
|
|
|
|
class TextCategorizer(Pipe):
|
|
|
|
"""Pipeline component for text classification.
|
|
|
|
|
2020-09-04 10:58:50 +00:00
|
|
|
DOCS: https://nightly.spacy.io/api/textcategorizer
|
2020-07-22 11:42:59 +00:00
|
|
|
"""
|
|
|
|
|
|
|
|
def __init__(
|
|
|
|
self,
|
|
|
|
vocab: Vocab,
|
|
|
|
model: Model,
|
|
|
|
name: str = "textcat",
|
|
|
|
*,
|
2020-09-14 15:08:00 +00:00
|
|
|
labels: List[str],
|
2020-09-13 12:15:52 +00:00
|
|
|
threshold: float,
|
2020-09-14 15:08:00 +00:00
|
|
|
positive_label: Optional[str],
|
2020-07-22 11:42:59 +00:00
|
|
|
) -> None:
|
2020-07-27 16:11:45 +00:00
|
|
|
"""Initialize a text categorizer.
|
|
|
|
|
|
|
|
vocab (Vocab): The shared vocabulary.
|
|
|
|
model (thinc.api.Model): The Thinc Model powering the pipeline component.
|
|
|
|
name (str): The component instance name, used to add entries to the
|
|
|
|
losses during training.
|
2020-09-14 15:08:00 +00:00
|
|
|
labels (List[str]): The labels to use.
|
2020-09-13 12:15:52 +00:00
|
|
|
threshold (float): Cutoff to consider a prediction "positive".
|
2020-09-14 15:08:00 +00:00
|
|
|
positive_label (Optional[str]): The positive label for a binary task with exclusive classes, None otherwise.
|
2020-07-27 16:11:45 +00:00
|
|
|
|
2020-09-04 10:58:50 +00:00
|
|
|
DOCS: https://nightly.spacy.io/api/textcategorizer#init
|
2020-07-27 16:11:45 +00:00
|
|
|
"""
|
2020-07-22 11:42:59 +00:00
|
|
|
self.vocab = vocab
|
|
|
|
self.model = model
|
|
|
|
self.name = name
|
|
|
|
self._rehearsal_model = None
|
2020-09-14 15:08:00 +00:00
|
|
|
cfg = {
|
|
|
|
"labels": labels,
|
|
|
|
"threshold": threshold,
|
|
|
|
"positive_label": positive_label,
|
|
|
|
}
|
2020-07-22 11:42:59 +00:00
|
|
|
self.cfg = dict(cfg)
|
|
|
|
|
|
|
|
@property
|
|
|
|
def labels(self) -> Tuple[str]:
|
2020-07-27 16:11:45 +00:00
|
|
|
"""RETURNS (Tuple[str]): The labels currently added to the component.
|
|
|
|
|
2020-09-04 10:58:50 +00:00
|
|
|
DOCS: https://nightly.spacy.io/api/textcategorizer#labels
|
2020-07-27 16:11:45 +00:00
|
|
|
"""
|
2020-09-14 15:08:00 +00:00
|
|
|
return tuple(self.cfg["labels"])
|
2020-07-22 11:42:59 +00:00
|
|
|
|
|
|
|
@labels.setter
|
2020-09-14 15:08:00 +00:00
|
|
|
def labels(self, value: List[str]) -> None:
|
2020-07-22 11:42:59 +00:00
|
|
|
self.cfg["labels"] = tuple(value)
|
|
|
|
|
2020-07-27 16:11:45 +00:00
|
|
|
def pipe(self, stream: Iterable[Doc], *, batch_size: int = 128) -> Iterator[Doc]:
|
|
|
|
"""Apply the pipe to a stream of documents. This usually happens under
|
|
|
|
the hood when the nlp object is called on a text and all components are
|
|
|
|
applied to the Doc.
|
|
|
|
|
|
|
|
stream (Iterable[Doc]): A stream of documents.
|
|
|
|
batch_size (int): The number of documents to buffer.
|
2020-07-28 11:37:31 +00:00
|
|
|
YIELDS (Doc): Processed documents in order.
|
2020-07-27 16:11:45 +00:00
|
|
|
|
2020-09-04 10:58:50 +00:00
|
|
|
DOCS: https://nightly.spacy.io/api/textcategorizer#pipe
|
2020-07-27 16:11:45 +00:00
|
|
|
"""
|
2020-07-22 11:42:59 +00:00
|
|
|
for docs in util.minibatch(stream, size=batch_size):
|
|
|
|
scores = self.predict(docs)
|
|
|
|
self.set_annotations(docs, scores)
|
|
|
|
yield from docs
|
|
|
|
|
|
|
|
def predict(self, docs: Iterable[Doc]):
|
2020-07-27 16:11:45 +00:00
|
|
|
"""Apply the pipeline's model to a batch of docs, without modifying them.
|
|
|
|
|
|
|
|
docs (Iterable[Doc]): The documents to predict.
|
|
|
|
RETURNS: The models prediction for each document.
|
|
|
|
|
2020-09-04 10:58:50 +00:00
|
|
|
DOCS: https://nightly.spacy.io/api/textcategorizer#predict
|
2020-07-27 16:11:45 +00:00
|
|
|
"""
|
2020-07-22 11:42:59 +00:00
|
|
|
tensors = [doc.tensor for doc in docs]
|
|
|
|
if not any(len(doc) for doc in docs):
|
|
|
|
# Handle cases where there are no tokens in any docs.
|
|
|
|
xp = get_array_module(tensors)
|
|
|
|
scores = xp.zeros((len(docs), len(self.labels)))
|
|
|
|
return scores
|
|
|
|
scores = self.model.predict(docs)
|
|
|
|
scores = self.model.ops.asarray(scores)
|
|
|
|
return scores
|
|
|
|
|
|
|
|
def set_annotations(self, docs: Iterable[Doc], scores) -> None:
|
2020-08-10 11:45:22 +00:00
|
|
|
"""Modify a batch of [`Doc`](/api/doc) objects, using pre-computed scores.
|
2020-07-27 16:11:45 +00:00
|
|
|
|
|
|
|
docs (Iterable[Doc]): The documents to modify.
|
|
|
|
scores: The scores to set, produced by TextCategorizer.predict.
|
|
|
|
|
2020-09-04 10:58:50 +00:00
|
|
|
DOCS: https://nightly.spacy.io/api/textcategorizer#set_annotations
|
2020-07-27 16:11:45 +00:00
|
|
|
"""
|
2020-07-22 11:42:59 +00:00
|
|
|
for i, doc in enumerate(docs):
|
|
|
|
for j, label in enumerate(self.labels):
|
|
|
|
doc.cats[label] = float(scores[i, j])
|
|
|
|
|
|
|
|
def update(
|
|
|
|
self,
|
|
|
|
examples: Iterable[Example],
|
|
|
|
*,
|
|
|
|
drop: float = 0.0,
|
|
|
|
set_annotations: bool = False,
|
|
|
|
sgd: Optional[Optimizer] = None,
|
|
|
|
losses: Optional[Dict[str, float]] = None,
|
|
|
|
) -> Dict[str, float]:
|
2020-07-27 16:11:45 +00:00
|
|
|
"""Learn from a batch of documents and gold-standard information,
|
|
|
|
updating the pipe's model. Delegates to predict and get_loss.
|
|
|
|
|
|
|
|
examples (Iterable[Example]): A batch of Example objects.
|
|
|
|
drop (float): The dropout rate.
|
|
|
|
set_annotations (bool): Whether or not to update the Example objects
|
|
|
|
with the predictions.
|
|
|
|
sgd (thinc.api.Optimizer): The optimizer.
|
|
|
|
losses (Dict[str, float]): Optional record of the loss during training.
|
|
|
|
Updated using the component name as the key.
|
|
|
|
RETURNS (Dict[str, float]): The updated losses dictionary.
|
|
|
|
|
2020-09-04 10:58:50 +00:00
|
|
|
DOCS: https://nightly.spacy.io/api/textcategorizer#update
|
2020-07-27 16:11:45 +00:00
|
|
|
"""
|
2020-07-22 11:42:59 +00:00
|
|
|
if losses is None:
|
|
|
|
losses = {}
|
|
|
|
losses.setdefault(self.name, 0.0)
|
2020-08-11 21:29:31 +00:00
|
|
|
validate_examples(examples, "TextCategorizer.update")
|
|
|
|
if not any(len(eg.predicted) if eg.predicted else 0 for eg in examples):
|
|
|
|
# Handle cases where there are no tokens in any docs.
|
|
|
|
return losses
|
2020-07-22 11:42:59 +00:00
|
|
|
set_dropout_rate(self.model, drop)
|
|
|
|
scores, bp_scores = self.model.begin_update([eg.predicted for eg in examples])
|
|
|
|
loss, d_scores = self.get_loss(examples, scores)
|
|
|
|
bp_scores(d_scores)
|
|
|
|
if sgd is not None:
|
|
|
|
self.model.finish_update(sgd)
|
|
|
|
losses[self.name] += loss
|
|
|
|
if set_annotations:
|
|
|
|
docs = [eg.predicted for eg in examples]
|
|
|
|
self.set_annotations(docs, scores=scores)
|
|
|
|
return losses
|
|
|
|
|
|
|
|
def rehearse(
|
|
|
|
self,
|
|
|
|
examples: Iterable[Example],
|
2020-07-27 16:11:45 +00:00
|
|
|
*,
|
2020-07-22 11:42:59 +00:00
|
|
|
drop: float = 0.0,
|
|
|
|
sgd: Optional[Optimizer] = None,
|
|
|
|
losses: Optional[Dict[str, float]] = None,
|
2020-07-27 16:11:45 +00:00
|
|
|
) -> Dict[str, float]:
|
|
|
|
"""Perform a "rehearsal" update from a batch of data. Rehearsal updates
|
|
|
|
teach the current model to make predictions similar to an initial model,
|
|
|
|
to try to address the "catastrophic forgetting" problem. This feature is
|
|
|
|
experimental.
|
|
|
|
|
|
|
|
examples (Iterable[Example]): A batch of Example objects.
|
|
|
|
drop (float): The dropout rate.
|
|
|
|
sgd (thinc.api.Optimizer): The optimizer.
|
|
|
|
losses (Dict[str, float]): Optional record of the loss during training.
|
|
|
|
Updated using the component name as the key.
|
|
|
|
RETURNS (Dict[str, float]): The updated losses dictionary.
|
|
|
|
|
2020-09-04 10:58:50 +00:00
|
|
|
DOCS: https://nightly.spacy.io/api/textcategorizer#rehearse
|
2020-07-27 16:11:45 +00:00
|
|
|
"""
|
2020-07-30 21:30:54 +00:00
|
|
|
if losses is not None:
|
|
|
|
losses.setdefault(self.name, 0.0)
|
2020-07-22 11:42:59 +00:00
|
|
|
if self._rehearsal_model is None:
|
2020-07-30 21:30:54 +00:00
|
|
|
return losses
|
2020-08-11 21:29:31 +00:00
|
|
|
validate_examples(examples, "TextCategorizer.rehearse")
|
|
|
|
docs = [eg.predicted for eg in examples]
|
2020-07-22 11:42:59 +00:00
|
|
|
if not any(len(doc) for doc in docs):
|
|
|
|
# Handle cases where there are no tokens in any docs.
|
2020-07-30 21:30:54 +00:00
|
|
|
return losses
|
2020-07-22 11:42:59 +00:00
|
|
|
set_dropout_rate(self.model, drop)
|
|
|
|
scores, bp_scores = self.model.begin_update(docs)
|
|
|
|
target = self._rehearsal_model(examples)
|
|
|
|
gradient = scores - target
|
|
|
|
bp_scores(gradient)
|
|
|
|
if sgd is not None:
|
|
|
|
self.model.finish_update(sgd)
|
|
|
|
if losses is not None:
|
|
|
|
losses[self.name] += (gradient ** 2).sum()
|
2020-07-27 16:11:45 +00:00
|
|
|
return losses
|
2020-07-22 11:42:59 +00:00
|
|
|
|
|
|
|
def _examples_to_truth(
|
|
|
|
self, examples: List[Example]
|
|
|
|
) -> Tuple[numpy.ndarray, numpy.ndarray]:
|
|
|
|
truths = numpy.zeros((len(examples), len(self.labels)), dtype="f")
|
|
|
|
not_missing = numpy.ones((len(examples), len(self.labels)), dtype="f")
|
|
|
|
for i, eg in enumerate(examples):
|
|
|
|
for j, label in enumerate(self.labels):
|
|
|
|
if label in eg.reference.cats:
|
|
|
|
truths[i, j] = eg.reference.cats[label]
|
|
|
|
else:
|
|
|
|
not_missing[i, j] = 0.0
|
|
|
|
truths = self.model.ops.asarray(truths)
|
|
|
|
return truths, not_missing
|
|
|
|
|
|
|
|
def get_loss(self, examples: Iterable[Example], scores) -> Tuple[float, float]:
|
2020-07-27 16:11:45 +00:00
|
|
|
"""Find the loss and gradient of loss for the batch of documents and
|
|
|
|
their predicted scores.
|
|
|
|
|
|
|
|
examples (Iterable[Examples]): The batch of examples.
|
|
|
|
scores: Scores representing the model's predictions.
|
|
|
|
RETUTNRS (Tuple[float, float]): The loss and the gradient.
|
|
|
|
|
2020-09-04 10:58:50 +00:00
|
|
|
DOCS: https://nightly.spacy.io/api/textcategorizer#get_loss
|
2020-07-27 16:11:45 +00:00
|
|
|
"""
|
2020-08-11 21:29:31 +00:00
|
|
|
validate_examples(examples, "TextCategorizer.get_loss")
|
2020-07-22 11:42:59 +00:00
|
|
|
truths, not_missing = self._examples_to_truth(examples)
|
|
|
|
not_missing = self.model.ops.asarray(not_missing)
|
|
|
|
d_scores = (scores - truths) / scores.shape[0]
|
|
|
|
d_scores *= not_missing
|
|
|
|
mean_square_error = (d_scores ** 2).sum(axis=1).mean()
|
|
|
|
return float(mean_square_error), d_scores
|
|
|
|
|
|
|
|
def add_label(self, label: str) -> int:
|
2020-07-27 16:11:45 +00:00
|
|
|
"""Add a new label to the pipe.
|
|
|
|
|
|
|
|
label (str): The label to add.
|
2020-07-28 11:37:31 +00:00
|
|
|
RETURNS (int): 0 if label is already present, otherwise 1.
|
2020-07-27 16:11:45 +00:00
|
|
|
|
2020-09-04 10:58:50 +00:00
|
|
|
DOCS: https://nightly.spacy.io/api/textcategorizer#add_label
|
2020-07-27 16:11:45 +00:00
|
|
|
"""
|
2020-07-22 11:42:59 +00:00
|
|
|
if not isinstance(label, str):
|
|
|
|
raise ValueError(Errors.E187)
|
|
|
|
if label in self.labels:
|
|
|
|
return 0
|
2020-09-08 20:44:25 +00:00
|
|
|
self._allow_extra_label()
|
2020-07-22 11:42:59 +00:00
|
|
|
self.labels = tuple(list(self.labels) + [label])
|
|
|
|
return 1
|
|
|
|
|
|
|
|
def begin_training(
|
|
|
|
self,
|
2020-08-11 21:29:31 +00:00
|
|
|
get_examples: Callable[[], Iterable[Example]],
|
2020-07-27 16:11:45 +00:00
|
|
|
*,
|
2020-07-22 11:42:59 +00:00
|
|
|
pipeline: Optional[List[Tuple[str, Callable[[Doc], Doc]]]] = None,
|
|
|
|
sgd: Optional[Optimizer] = None,
|
|
|
|
) -> Optimizer:
|
2020-09-08 20:44:25 +00:00
|
|
|
"""Initialize the pipe for training, using a representative set
|
|
|
|
of data examples.
|
2020-07-27 16:11:45 +00:00
|
|
|
|
2020-09-08 20:44:25 +00:00
|
|
|
get_examples (Callable[[], Iterable[Example]]): Function that
|
|
|
|
returns a representative sample of gold-standard Example objects.
|
2020-07-27 16:11:45 +00:00
|
|
|
pipeline (List[Tuple[str, Callable]]): Optional list of pipeline
|
|
|
|
components that this component is part of. Corresponds to
|
|
|
|
nlp.pipeline.
|
|
|
|
sgd (thinc.api.Optimizer): Optional optimizer. Will be created with
|
|
|
|
create_optimizer if it doesn't exist.
|
|
|
|
RETURNS (thinc.api.Optimizer): The optimizer.
|
|
|
|
|
2020-09-04 10:58:50 +00:00
|
|
|
DOCS: https://nightly.spacy.io/api/textcategorizer#begin_training
|
2020-07-27 16:11:45 +00:00
|
|
|
"""
|
2020-09-08 20:44:25 +00:00
|
|
|
self._ensure_examples(get_examples)
|
2020-08-11 21:29:31 +00:00
|
|
|
subbatch = [] # Select a subbatch of examples to initialize the model
|
2020-09-08 20:44:25 +00:00
|
|
|
for example in islice(get_examples(), 10):
|
2020-08-11 21:29:31 +00:00
|
|
|
if len(subbatch) < 2:
|
|
|
|
subbatch.append(example)
|
|
|
|
for cat in example.y.cats:
|
2020-07-22 11:42:59 +00:00
|
|
|
self.add_label(cat)
|
2020-09-08 20:44:25 +00:00
|
|
|
doc_sample = [eg.reference for eg in subbatch]
|
|
|
|
label_sample, _ = self._examples_to_truth(subbatch)
|
|
|
|
self._require_labels()
|
|
|
|
assert len(doc_sample) > 0, Errors.E923.format(name=self.name)
|
|
|
|
assert len(label_sample) > 0, Errors.E923.format(name=self.name)
|
|
|
|
self.model.initialize(X=doc_sample, Y=label_sample)
|
2020-07-22 11:42:59 +00:00
|
|
|
if sgd is None:
|
|
|
|
sgd = self.create_optimizer()
|
|
|
|
return sgd
|
Refactor the Scorer to improve flexibility (#5731)
* Refactor the Scorer to improve flexibility
Refactor the `Scorer` to improve flexibility for arbitrary pipeline
components.
* Individual pipeline components provide their own `evaluate` methods
that score a list of `Example`s and return a dictionary of scores
* `Scorer` is initialized either:
* with a provided pipeline containing components to be scored
* with a default pipeline containing the built-in statistical
components (senter, tagger, morphologizer, parser, ner)
* `Scorer.score` evaluates a list of `Example`s and returns a dictionary
of scores referring to the scores provided by the components in the
pipeline
Significant differences:
* `tags_acc` is renamed to `tag_acc` to be consistent with `token_acc`
and the new `morph_acc`, `pos_acc`, and `lemma_acc`
* Scoring is no longer cumulative: `Scorer.score` scores a list of
examples rather than a single example and does not retain any state
about previously scored examples
* PRF values in the returned scores are no longer multiplied by 100
* Add kwargs to Morphologizer.evaluate
* Create generalized scoring methods in Scorer
* Generalized static scoring methods are added to `Scorer`
* Methods require an attribute (either on Token or Doc) that is
used to key the returned scores
Naming differences:
* `uas`, `las`, and `las_per_type` in the scores dict are renamed to
`dep_uas`, `dep_las`, and `dep_las_per_type`
Scoring differences:
* `Doc.sents` is now scored as spans rather than on sentence-initial
token positions so that `Doc.sents` and `Doc.ents` can be scored with
the same method (this lowers scores since a single incorrect sentence
start results in two incorrect spans)
* Simplify / extend hasattr check for eval method
* Add hasattr check to tokenizer scoring
* Simplify to hasattr check for component scoring
* Reset Example alignment if docs are set
Reset the Example alignment if either doc is set in case the
tokenization has changed.
* Add PRF tokenization scoring for tokens as spans
Add PRF scores for tokens as character spans. The scores are:
* token_acc: # correct tokens / # gold tokens
* token_p/r/f: PRF for (token.idx, token.idx + len(token))
* Add docstring to Scorer.score_tokenization
* Rename component.evaluate() to component.score()
* Update Scorer API docs
* Update scoring for positive_label in textcat
* Fix TextCategorizer.score kwargs
* Update Language.evaluate docs
* Update score names in default config
2020-07-25 10:53:02 +00:00
|
|
|
|
2020-09-14 15:08:00 +00:00
|
|
|
def score(self, examples: Iterable[Example], **kwargs) -> Dict[str, Any]:
|
2020-07-27 16:11:45 +00:00
|
|
|
"""Score a batch of examples.
|
|
|
|
|
|
|
|
examples (Iterable[Example]): The examples to score.
|
|
|
|
RETURNS (Dict[str, Any]): The scores, produced by Scorer.score_cats.
|
|
|
|
|
2020-09-04 10:58:50 +00:00
|
|
|
DOCS: https://nightly.spacy.io/api/textcategorizer#score
|
2020-07-27 16:11:45 +00:00
|
|
|
"""
|
2020-08-11 21:29:31 +00:00
|
|
|
validate_examples(examples, "TextCategorizer.score")
|
2020-07-25 13:01:15 +00:00
|
|
|
return Scorer.score_cats(
|
|
|
|
examples,
|
|
|
|
"cats",
|
|
|
|
labels=self.labels,
|
Refactor the Scorer to improve flexibility (#5731)
* Refactor the Scorer to improve flexibility
Refactor the `Scorer` to improve flexibility for arbitrary pipeline
components.
* Individual pipeline components provide their own `evaluate` methods
that score a list of `Example`s and return a dictionary of scores
* `Scorer` is initialized either:
* with a provided pipeline containing components to be scored
* with a default pipeline containing the built-in statistical
components (senter, tagger, morphologizer, parser, ner)
* `Scorer.score` evaluates a list of `Example`s and returns a dictionary
of scores referring to the scores provided by the components in the
pipeline
Significant differences:
* `tags_acc` is renamed to `tag_acc` to be consistent with `token_acc`
and the new `morph_acc`, `pos_acc`, and `lemma_acc`
* Scoring is no longer cumulative: `Scorer.score` scores a list of
examples rather than a single example and does not retain any state
about previously scored examples
* PRF values in the returned scores are no longer multiplied by 100
* Add kwargs to Morphologizer.evaluate
* Create generalized scoring methods in Scorer
* Generalized static scoring methods are added to `Scorer`
* Methods require an attribute (either on Token or Doc) that is
used to key the returned scores
Naming differences:
* `uas`, `las`, and `las_per_type` in the scores dict are renamed to
`dep_uas`, `dep_las`, and `dep_las_per_type`
Scoring differences:
* `Doc.sents` is now scored as spans rather than on sentence-initial
token positions so that `Doc.sents` and `Doc.ents` can be scored with
the same method (this lowers scores since a single incorrect sentence
start results in two incorrect spans)
* Simplify / extend hasattr check for eval method
* Add hasattr check to tokenizer scoring
* Simplify to hasattr check for component scoring
* Reset Example alignment if docs are set
Reset the Example alignment if either doc is set in case the
tokenization has changed.
* Add PRF tokenization scoring for tokens as spans
Add PRF scores for tokens as character spans. The scores are:
* token_acc: # correct tokens / # gold tokens
* token_p/r/f: PRF for (token.idx, token.idx + len(token))
* Add docstring to Scorer.score_tokenization
* Rename component.evaluate() to component.score()
* Update Scorer API docs
* Update scoring for positive_label in textcat
* Fix TextCategorizer.score kwargs
* Update Language.evaluate docs
* Update score names in default config
2020-07-25 10:53:02 +00:00
|
|
|
multi_label=self.model.attrs["multi_label"],
|
2020-09-14 15:08:00 +00:00
|
|
|
positive_label=self.cfg["positive_label"],
|
2020-09-13 12:15:52 +00:00
|
|
|
threshold=self.cfg["threshold"],
|
2020-07-25 13:01:15 +00:00
|
|
|
**kwargs,
|
Refactor the Scorer to improve flexibility (#5731)
* Refactor the Scorer to improve flexibility
Refactor the `Scorer` to improve flexibility for arbitrary pipeline
components.
* Individual pipeline components provide their own `evaluate` methods
that score a list of `Example`s and return a dictionary of scores
* `Scorer` is initialized either:
* with a provided pipeline containing components to be scored
* with a default pipeline containing the built-in statistical
components (senter, tagger, morphologizer, parser, ner)
* `Scorer.score` evaluates a list of `Example`s and returns a dictionary
of scores referring to the scores provided by the components in the
pipeline
Significant differences:
* `tags_acc` is renamed to `tag_acc` to be consistent with `token_acc`
and the new `morph_acc`, `pos_acc`, and `lemma_acc`
* Scoring is no longer cumulative: `Scorer.score` scores a list of
examples rather than a single example and does not retain any state
about previously scored examples
* PRF values in the returned scores are no longer multiplied by 100
* Add kwargs to Morphologizer.evaluate
* Create generalized scoring methods in Scorer
* Generalized static scoring methods are added to `Scorer`
* Methods require an attribute (either on Token or Doc) that is
used to key the returned scores
Naming differences:
* `uas`, `las`, and `las_per_type` in the scores dict are renamed to
`dep_uas`, `dep_las`, and `dep_las_per_type`
Scoring differences:
* `Doc.sents` is now scored as spans rather than on sentence-initial
token positions so that `Doc.sents` and `Doc.ents` can be scored with
the same method (this lowers scores since a single incorrect sentence
start results in two incorrect spans)
* Simplify / extend hasattr check for eval method
* Add hasattr check to tokenizer scoring
* Simplify to hasattr check for component scoring
* Reset Example alignment if docs are set
Reset the Example alignment if either doc is set in case the
tokenization has changed.
* Add PRF tokenization scoring for tokens as spans
Add PRF scores for tokens as character spans. The scores are:
* token_acc: # correct tokens / # gold tokens
* token_p/r/f: PRF for (token.idx, token.idx + len(token))
* Add docstring to Scorer.score_tokenization
* Rename component.evaluate() to component.score()
* Update Scorer API docs
* Update scoring for positive_label in textcat
* Fix TextCategorizer.score kwargs
* Update Language.evaluate docs
* Update score names in default config
2020-07-25 10:53:02 +00:00
|
|
|
)
|