From 080066ae74c10d5ef2a61eb123ad608b354a1103 Mon Sep 17 00:00:00 2001 From: svlandeg Date: Mon, 26 Oct 2020 10:37:25 +0100 Subject: [PATCH 1/5] remove TODO note --- spacy/scorer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spacy/scorer.py b/spacy/scorer.py index 273bda898..97f54f4fb 100644 --- a/spacy/scorer.py +++ b/spacy/scorer.py @@ -478,7 +478,7 @@ class Scorer: negative_labels (Iterable[str]): The string values that refer to no annotation (e.g. "NIL") RETURNS (Dict[str, Any]): A dictionary containing the scores. - DOCS (TODO): https://nightly.spacy.io/api/scorer#score_links + DOCS: https://nightly.spacy.io/api/scorer#score_links """ f_per_type = {} for example in examples: From a664994a8143355500335f291ece33e0bc22e35d Mon Sep 17 00:00:00 2001 From: svlandeg Date: Mon, 26 Oct 2020 10:52:47 +0100 Subject: [PATCH 2/5] adding score method to explanation of new component --- website/docs/usage/layers-architectures.md | 52 ++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/website/docs/usage/layers-architectures.md b/website/docs/usage/layers-architectures.md index aa62a77d4..d91fd6bf1 100644 --- a/website/docs/usage/layers-architectures.md +++ b/website/docs/usage/layers-architectures.md @@ -843,6 +843,27 @@ def __call__(self, Doc doc): return doc ``` +There is one more optional method to implement: [`score`](/api/pipe#score) +calculates the performance of your component on a set of examples, and +returns the results as a dictionary: + +```python +### The score method +def score(self, examples: Iterable[Example]) -> Dict[str, Any]: + prf = PRFScore() + for example in examples: + ... + + return { + f"rel_micro_p": prf.precision, + f"rel_micro_r": prf.recall, + f"rel_micro_f": prf.fscore, + } +``` + +This is particularly useful to see the scores on the development corpus +when training the component with [`spacy train`](/api/cli#training). + Once our `TrainablePipe` subclass is fully implemented, we can [register](/usage/processing-pipelines#custom-components-factories) the component with the [`@Language.factory`](/api/language#factory) decorator. This @@ -876,6 +897,37 @@ def make_relation_extractor(nlp, name, model): return RelationExtractor(nlp.vocab, model, name) ``` +You can extend the decorator to include information such as the type of +annotations that are required for this component to run, the type of annotations +it produces, and the scores that can be calculated: + +> #### config.cfg (excerpt) +> +> ```ini +> [training.score_weights] +> rel_micro_p: 0.0 +> rel_micro_r: 0.0 +> rel_micro_f: 1.0 +> ``` + +```python +### Factory annotations +from spacy.language import Language + +@Language.factory( + "relation_extractor", + requires=["doc.ents", "token.ent_iob", "token.ent_type"], + assigns=["doc._.rel"], + default_score_weights={ + "rel_micro_p": None, + "rel_micro_r": None, + "rel_micro_f": None, + }, +) +def make_relation_extractor(nlp, name, model): + return RelationExtractor(nlp.vocab, model, name) +``` + From e95d9caa878a39070ed47af3b390e161d6486f40 Mon Sep 17 00:00:00 2001 From: svlandeg Date: Mon, 26 Oct 2020 11:09:25 +0100 Subject: [PATCH 3/5] small edits --- website/docs/usage/layers-architectures.md | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/website/docs/usage/layers-architectures.md b/website/docs/usage/layers-architectures.md index d91fd6bf1..db0b81112 100644 --- a/website/docs/usage/layers-architectures.md +++ b/website/docs/usage/layers-architectures.md @@ -855,10 +855,10 @@ def score(self, examples: Iterable[Example]) -> Dict[str, Any]: ... return { - f"rel_micro_p": prf.precision, - f"rel_micro_r": prf.recall, - f"rel_micro_f": prf.fscore, - } + "rel_micro_p": prf.precision, + "rel_micro_r": prf.recall, + "rel_micro_f": prf.fscore, + } ``` This is particularly useful to see the scores on the development corpus @@ -886,6 +886,11 @@ assigns it a name and lets you create the component with > [components.relation_extractor.model.get_candidates] > @misc = "rel_cand_generator.v1" > max_length = 20 +> +> [training.score_weights] +> rel_micro_p: 0.0 +> rel_micro_r: 0.0 +> rel_micro_f: 1.0 > ``` ```python @@ -904,14 +909,11 @@ it produces, and the scores that can be calculated: > #### config.cfg (excerpt) > > ```ini -> [training.score_weights] -> rel_micro_p: 0.0 -> rel_micro_r: 0.0 -> rel_micro_f: 1.0 + > ``` ```python -### Factory annotations +### Factory annotations {highlight="5-11"} from spacy.language import Language @Language.factory( From 5878ff6bcdd21847a4f9e0b97464d6dcfdd421c0 Mon Sep 17 00:00:00 2001 From: svlandeg Date: Mon, 26 Oct 2020 11:13:02 +0100 Subject: [PATCH 4/5] cleanup --- website/docs/usage/layers-architectures.md | 6 ------ 1 file changed, 6 deletions(-) diff --git a/website/docs/usage/layers-architectures.md b/website/docs/usage/layers-architectures.md index db0b81112..a1b58f41e 100644 --- a/website/docs/usage/layers-architectures.md +++ b/website/docs/usage/layers-architectures.md @@ -906,12 +906,6 @@ You can extend the decorator to include information such as the type of annotations that are required for this component to run, the type of annotations it produces, and the scores that can be calculated: -> #### config.cfg (excerpt) -> -> ```ini - -> ``` - ```python ### Factory annotations {highlight="5-11"} from spacy.language import Language From 77688b0072ebb4580870ba5b964e60a298ed24e2 Mon Sep 17 00:00:00 2001 From: svlandeg Date: Mon, 26 Oct 2020 11:14:34 +0100 Subject: [PATCH 5/5] fix config --- website/docs/usage/layers-architectures.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/website/docs/usage/layers-architectures.md b/website/docs/usage/layers-architectures.md index a1b58f41e..641db02f5 100644 --- a/website/docs/usage/layers-architectures.md +++ b/website/docs/usage/layers-architectures.md @@ -888,9 +888,9 @@ assigns it a name and lets you create the component with > max_length = 20 > > [training.score_weights] -> rel_micro_p: 0.0 -> rel_micro_r: 0.0 -> rel_micro_f: 1.0 +> rel_micro_p = 0.0 +> rel_micro_r = 0.0 +> rel_micro_f = 1.0 > ``` ```python