diff --git a/spacy/scorer.py b/spacy/scorer.py index fe64c23ad..5cace8fda 100644 --- a/spacy/scorer.py +++ b/spacy/scorer.py @@ -512,7 +512,7 @@ class Scorer: negative_labels (Iterable[str]): The string values that refer to no annotation (e.g. "NIL") RETURNS (Dict[str, Any]): A dictionary containing the scores. - DOCS (TODO): https://nightly.spacy.io/api/scorer#score_links + DOCS: https://nightly.spacy.io/api/scorer#score_links """ f_per_type = {} for example in examples: diff --git a/website/docs/usage/layers-architectures.md b/website/docs/usage/layers-architectures.md index aa62a77d4..641db02f5 100644 --- a/website/docs/usage/layers-architectures.md +++ b/website/docs/usage/layers-architectures.md @@ -843,6 +843,27 @@ def __call__(self, Doc doc): return doc ``` +There is one more optional method to implement: [`score`](/api/pipe#score) +calculates the performance of your component on a set of examples, and +returns the results as a dictionary: + +```python +### The score method +def score(self, examples: Iterable[Example]) -> Dict[str, Any]: + prf = PRFScore() + for example in examples: + ... + + return { + "rel_micro_p": prf.precision, + "rel_micro_r": prf.recall, + "rel_micro_f": prf.fscore, + } +``` + +This is particularly useful to see the scores on the development corpus +when training the component with [`spacy train`](/api/cli#training). + Once our `TrainablePipe` subclass is fully implemented, we can [register](/usage/processing-pipelines#custom-components-factories) the component with the [`@Language.factory`](/api/language#factory) decorator. This @@ -865,6 +886,11 @@ assigns it a name and lets you create the component with > [components.relation_extractor.model.get_candidates] > @misc = "rel_cand_generator.v1" > max_length = 20 +> +> [training.score_weights] +> rel_micro_p = 0.0 +> rel_micro_r = 0.0 +> rel_micro_f = 1.0 > ``` ```python @@ -876,6 +902,28 @@ def make_relation_extractor(nlp, name, model): return RelationExtractor(nlp.vocab, model, name) ``` +You can extend the decorator to include information such as the type of +annotations that are required for this component to run, the type of annotations +it produces, and the scores that can be calculated: + +```python +### Factory annotations {highlight="5-11"} +from spacy.language import Language + +@Language.factory( + "relation_extractor", + requires=["doc.ents", "token.ent_iob", "token.ent_type"], + assigns=["doc._.rel"], + default_score_weights={ + "rel_micro_p": None, + "rel_micro_r": None, + "rel_micro_f": None, + }, +) +def make_relation_extractor(nlp, name, model): + return RelationExtractor(nlp.vocab, model, name) +``` +