2020-06-28 13:34:28 +00:00
|
|
|
from typing import Optional, List, Dict
|
2020-06-21 19:35:01 +00:00
|
|
|
from wasabi import Printer
|
|
|
|
from pathlib import Path
|
2020-06-27 19:13:11 +00:00
|
|
|
import re
|
|
|
|
import srsly
|
2020-09-28 13:09:59 +00:00
|
|
|
from thinc.api import fix_random_seed
|
2017-10-01 19:04:32 +00:00
|
|
|
|
2020-09-09 08:31:03 +00:00
|
|
|
from ..training import Corpus
|
2020-06-21 19:35:01 +00:00
|
|
|
from ..tokens import Doc
|
2020-09-29 19:20:56 +00:00
|
|
|
from ._util import app, Arg, Opt, setup_gpu, import_code
|
2020-06-21 19:35:01 +00:00
|
|
|
from ..scorer import Scorer
|
2017-10-01 19:04:32 +00:00
|
|
|
from .. import util
|
|
|
|
from .. import displacy
|
2017-10-27 12:38:39 +00:00
|
|
|
|
2017-10-01 19:04:32 +00:00
|
|
|
|
2020-06-21 11:44:00 +00:00
|
|
|
@app.command("evaluate")
|
2020-06-21 19:35:01 +00:00
|
|
|
def evaluate_cli(
|
2020-01-01 12:15:46 +00:00
|
|
|
# fmt: off
|
2020-06-21 11:44:00 +00:00
|
|
|
model: str = Arg(..., help="Model name or path"),
|
2020-08-07 12:40:58 +00:00
|
|
|
data_path: Path = Arg(..., help="Location of binary evaluation data in .spacy format", exists=True),
|
2020-06-27 19:13:11 +00:00
|
|
|
output: Optional[Path] = Opt(None, "--output", "-o", help="Output JSON file for metrics", dir_okay=False),
|
2020-09-29 19:20:56 +00:00
|
|
|
code_path: Optional[Path] = Opt(None, "--code", "-c", help="Path to Python file with additional code (registered functions) to be imported"),
|
2020-08-07 12:40:58 +00:00
|
|
|
use_gpu: int = Opt(-1, "--gpu-id", "-g", help="GPU ID or -1 for CPU"),
|
2020-06-21 11:44:00 +00:00
|
|
|
gold_preproc: bool = Opt(False, "--gold-preproc", "-G", help="Use gold preprocessing"),
|
2020-06-21 19:35:01 +00:00
|
|
|
displacy_path: Optional[Path] = Opt(None, "--displacy-path", "-dp", help="Directory to output rendered parses as HTML", exists=True, file_okay=False),
|
2020-06-21 11:44:00 +00:00
|
|
|
displacy_limit: int = Opt(25, "--displacy-limit", "-dl", help="Limit of parses to render as HTML"),
|
2020-06-27 19:13:11 +00:00
|
|
|
# fmt: on
|
2018-11-30 19:16:14 +00:00
|
|
|
):
|
2017-10-01 19:04:32 +00:00
|
|
|
"""
|
2020-09-03 11:13:03 +00:00
|
|
|
Evaluate a trained pipeline. Expects a loadable spaCy pipeline and evaluation
|
2020-09-04 10:58:50 +00:00
|
|
|
data in the binary .spacy format. The --gold-preproc option sets up the
|
|
|
|
evaluation examples with gold-standard sentences and tokens for the
|
|
|
|
predictions. Gold preprocessing helps the annotations align to the
|
|
|
|
tokenization, and may result in sequences of more consistent length. However,
|
|
|
|
it may reduce runtime accuracy due to train/test skew. To render a sample of
|
|
|
|
dependency parses in a HTML file, set as output directory as the
|
|
|
|
displacy_path argument.
|
|
|
|
|
|
|
|
DOCS: https://nightly.spacy.io/api/cli#evaluate
|
2017-10-01 19:04:32 +00:00
|
|
|
"""
|
2020-09-29 19:20:56 +00:00
|
|
|
import_code(code_path)
|
2020-06-21 19:35:01 +00:00
|
|
|
evaluate(
|
|
|
|
model,
|
|
|
|
data_path,
|
2020-06-27 19:13:11 +00:00
|
|
|
output=output,
|
2020-08-07 12:40:58 +00:00
|
|
|
use_gpu=use_gpu,
|
2020-06-21 19:35:01 +00:00
|
|
|
gold_preproc=gold_preproc,
|
|
|
|
displacy_path=displacy_path,
|
|
|
|
displacy_limit=displacy_limit,
|
|
|
|
silent=False,
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
def evaluate(
|
|
|
|
model: str,
|
|
|
|
data_path: Path,
|
2020-07-20 12:42:46 +00:00
|
|
|
output: Optional[Path] = None,
|
2020-08-07 12:40:58 +00:00
|
|
|
use_gpu: int = -1,
|
2020-06-21 19:35:01 +00:00
|
|
|
gold_preproc: bool = False,
|
|
|
|
displacy_path: Optional[Path] = None,
|
|
|
|
displacy_limit: int = 25,
|
|
|
|
silent: bool = True,
|
|
|
|
) -> Scorer:
|
|
|
|
msg = Printer(no_print=silent, pretty=not silent)
|
2020-07-06 11:06:25 +00:00
|
|
|
fix_random_seed()
|
2020-09-28 13:09:59 +00:00
|
|
|
setup_gpu(use_gpu)
|
2017-10-01 19:04:32 +00:00
|
|
|
data_path = util.ensure_path(data_path)
|
2020-06-27 19:13:11 +00:00
|
|
|
output_path = util.ensure_path(output)
|
2017-10-03 22:03:15 +00:00
|
|
|
displacy_path = util.ensure_path(displacy_path)
|
2017-10-01 19:04:32 +00:00
|
|
|
if not data_path.exists():
|
2018-12-08 10:49:43 +00:00
|
|
|
msg.fail("Evaluation data not found", data_path, exits=1)
|
2017-10-03 22:03:15 +00:00
|
|
|
if displacy_path and not displacy_path.exists():
|
2018-12-08 10:49:43 +00:00
|
|
|
msg.fail("Visualization output directory not found", displacy_path, exits=1)
|
2020-08-04 13:09:37 +00:00
|
|
|
corpus = Corpus(data_path, gold_preproc=gold_preproc)
|
2020-06-27 19:16:57 +00:00
|
|
|
nlp = util.load_model(model)
|
2020-08-04 13:09:37 +00:00
|
|
|
dev_dataset = list(corpus(nlp))
|
2020-08-17 14:45:24 +00:00
|
|
|
scores = nlp.evaluate(dev_dataset)
|
Refactor the Scorer to improve flexibility (#5731)
* Refactor the Scorer to improve flexibility
Refactor the `Scorer` to improve flexibility for arbitrary pipeline
components.
* Individual pipeline components provide their own `evaluate` methods
that score a list of `Example`s and return a dictionary of scores
* `Scorer` is initialized either:
* with a provided pipeline containing components to be scored
* with a default pipeline containing the built-in statistical
components (senter, tagger, morphologizer, parser, ner)
* `Scorer.score` evaluates a list of `Example`s and returns a dictionary
of scores referring to the scores provided by the components in the
pipeline
Significant differences:
* `tags_acc` is renamed to `tag_acc` to be consistent with `token_acc`
and the new `morph_acc`, `pos_acc`, and `lemma_acc`
* Scoring is no longer cumulative: `Scorer.score` scores a list of
examples rather than a single example and does not retain any state
about previously scored examples
* PRF values in the returned scores are no longer multiplied by 100
* Add kwargs to Morphologizer.evaluate
* Create generalized scoring methods in Scorer
* Generalized static scoring methods are added to `Scorer`
* Methods require an attribute (either on Token or Doc) that is
used to key the returned scores
Naming differences:
* `uas`, `las`, and `las_per_type` in the scores dict are renamed to
`dep_uas`, `dep_las`, and `dep_las_per_type`
Scoring differences:
* `Doc.sents` is now scored as spans rather than on sentence-initial
token positions so that `Doc.sents` and `Doc.ents` can be scored with
the same method (this lowers scores since a single incorrect sentence
start results in two incorrect spans)
* Simplify / extend hasattr check for eval method
* Add hasattr check to tokenizer scoring
* Simplify to hasattr check for component scoring
* Reset Example alignment if docs are set
Reset the Example alignment if either doc is set in case the
tokenization has changed.
* Add PRF tokenization scoring for tokens as spans
Add PRF scores for tokens as character spans. The scores are:
* token_acc: # correct tokens / # gold tokens
* token_p/r/f: PRF for (token.idx, token.idx + len(token))
* Add docstring to Scorer.score_tokenization
* Rename component.evaluate() to component.score()
* Update Scorer API docs
* Update scoring for positive_label in textcat
* Fix TextCategorizer.score kwargs
* Update Language.evaluate docs
* Update score names in default config
2020-07-25 10:53:02 +00:00
|
|
|
metrics = {
|
|
|
|
"TOK": "token_acc",
|
|
|
|
"TAG": "tag_acc",
|
|
|
|
"POS": "pos_acc",
|
|
|
|
"MORPH": "morph_acc",
|
|
|
|
"LEMMA": "lemma_acc",
|
|
|
|
"UAS": "dep_uas",
|
|
|
|
"LAS": "dep_las",
|
|
|
|
"NER P": "ents_p",
|
|
|
|
"NER R": "ents_r",
|
|
|
|
"NER F": "ents_f",
|
2020-07-29 09:02:31 +00:00
|
|
|
"TEXTCAT": "cats_score",
|
|
|
|
"SENT P": "sents_p",
|
|
|
|
"SENT R": "sents_r",
|
|
|
|
"SENT F": "sents_f",
|
|
|
|
"SPEED": "speed",
|
2018-11-30 19:16:14 +00:00
|
|
|
}
|
Refactor the Scorer to improve flexibility (#5731)
* Refactor the Scorer to improve flexibility
Refactor the `Scorer` to improve flexibility for arbitrary pipeline
components.
* Individual pipeline components provide their own `evaluate` methods
that score a list of `Example`s and return a dictionary of scores
* `Scorer` is initialized either:
* with a provided pipeline containing components to be scored
* with a default pipeline containing the built-in statistical
components (senter, tagger, morphologizer, parser, ner)
* `Scorer.score` evaluates a list of `Example`s and returns a dictionary
of scores referring to the scores provided by the components in the
pipeline
Significant differences:
* `tags_acc` is renamed to `tag_acc` to be consistent with `token_acc`
and the new `morph_acc`, `pos_acc`, and `lemma_acc`
* Scoring is no longer cumulative: `Scorer.score` scores a list of
examples rather than a single example and does not retain any state
about previously scored examples
* PRF values in the returned scores are no longer multiplied by 100
* Add kwargs to Morphologizer.evaluate
* Create generalized scoring methods in Scorer
* Generalized static scoring methods are added to `Scorer`
* Methods require an attribute (either on Token or Doc) that is
used to key the returned scores
Naming differences:
* `uas`, `las`, and `las_per_type` in the scores dict are renamed to
`dep_uas`, `dep_las`, and `dep_las_per_type`
Scoring differences:
* `Doc.sents` is now scored as spans rather than on sentence-initial
token positions so that `Doc.sents` and `Doc.ents` can be scored with
the same method (this lowers scores since a single incorrect sentence
start results in two incorrect spans)
* Simplify / extend hasattr check for eval method
* Add hasattr check to tokenizer scoring
* Simplify to hasattr check for component scoring
* Reset Example alignment if docs are set
Reset the Example alignment if either doc is set in case the
tokenization has changed.
* Add PRF tokenization scoring for tokens as spans
Add PRF scores for tokens as character spans. The scores are:
* token_acc: # correct tokens / # gold tokens
* token_p/r/f: PRF for (token.idx, token.idx + len(token))
* Add docstring to Scorer.score_tokenization
* Rename component.evaluate() to component.score()
* Update Scorer API docs
* Update scoring for positive_label in textcat
* Fix TextCategorizer.score kwargs
* Update Language.evaluate docs
* Update score names in default config
2020-07-25 10:53:02 +00:00
|
|
|
results = {}
|
2020-10-19 13:03:19 +00:00
|
|
|
data = {}
|
Refactor the Scorer to improve flexibility (#5731)
* Refactor the Scorer to improve flexibility
Refactor the `Scorer` to improve flexibility for arbitrary pipeline
components.
* Individual pipeline components provide their own `evaluate` methods
that score a list of `Example`s and return a dictionary of scores
* `Scorer` is initialized either:
* with a provided pipeline containing components to be scored
* with a default pipeline containing the built-in statistical
components (senter, tagger, morphologizer, parser, ner)
* `Scorer.score` evaluates a list of `Example`s and returns a dictionary
of scores referring to the scores provided by the components in the
pipeline
Significant differences:
* `tags_acc` is renamed to `tag_acc` to be consistent with `token_acc`
and the new `morph_acc`, `pos_acc`, and `lemma_acc`
* Scoring is no longer cumulative: `Scorer.score` scores a list of
examples rather than a single example and does not retain any state
about previously scored examples
* PRF values in the returned scores are no longer multiplied by 100
* Add kwargs to Morphologizer.evaluate
* Create generalized scoring methods in Scorer
* Generalized static scoring methods are added to `Scorer`
* Methods require an attribute (either on Token or Doc) that is
used to key the returned scores
Naming differences:
* `uas`, `las`, and `las_per_type` in the scores dict are renamed to
`dep_uas`, `dep_las`, and `dep_las_per_type`
Scoring differences:
* `Doc.sents` is now scored as spans rather than on sentence-initial
token positions so that `Doc.sents` and `Doc.ents` can be scored with
the same method (this lowers scores since a single incorrect sentence
start results in two incorrect spans)
* Simplify / extend hasattr check for eval method
* Add hasattr check to tokenizer scoring
* Simplify to hasattr check for component scoring
* Reset Example alignment if docs are set
Reset the Example alignment if either doc is set in case the
tokenization has changed.
* Add PRF tokenization scoring for tokens as spans
Add PRF scores for tokens as character spans. The scores are:
* token_acc: # correct tokens / # gold tokens
* token_p/r/f: PRF for (token.idx, token.idx + len(token))
* Add docstring to Scorer.score_tokenization
* Rename component.evaluate() to component.score()
* Update Scorer API docs
* Update scoring for positive_label in textcat
* Fix TextCategorizer.score kwargs
* Update Language.evaluate docs
* Update score names in default config
2020-07-25 10:53:02 +00:00
|
|
|
for metric, key in metrics.items():
|
|
|
|
if key in scores:
|
2020-07-27 09:17:52 +00:00
|
|
|
if key == "cats_score":
|
|
|
|
metric = metric + " (" + scores.get("cats_score_desc", "unk") + ")"
|
2020-11-03 14:47:18 +00:00
|
|
|
if isinstance(scores[key], (int, float)):
|
|
|
|
if key == "speed":
|
|
|
|
results[metric] = f"{scores[key]:.0f}"
|
|
|
|
else:
|
|
|
|
results[metric] = f"{scores[key]*100:.2f}"
|
2020-07-29 09:02:31 +00:00
|
|
|
else:
|
2020-11-03 14:47:18 +00:00
|
|
|
results[metric] = "-"
|
2020-10-19 13:03:19 +00:00
|
|
|
data[re.sub(r"[\s/]", "_", key.lower())] = scores[key]
|
2020-06-28 13:34:28 +00:00
|
|
|
|
2018-11-30 19:16:14 +00:00
|
|
|
msg.table(results, title="Results")
|
|
|
|
|
2020-10-19 10:07:46 +00:00
|
|
|
if "morph_per_feat" in scores:
|
|
|
|
if scores["morph_per_feat"]:
|
2020-10-19 11:18:47 +00:00
|
|
|
print_prf_per_type(msg, scores["morph_per_feat"], "MORPH", "feat")
|
2020-10-19 10:07:46 +00:00
|
|
|
data["morph_per_feat"] = scores["morph_per_feat"]
|
2020-10-19 11:18:47 +00:00
|
|
|
if "dep_las_per_type" in scores:
|
|
|
|
if scores["dep_las_per_type"]:
|
|
|
|
print_prf_per_type(msg, scores["dep_las_per_type"], "LAS", "type")
|
|
|
|
data["dep_las_per_type"] = scores["dep_las_per_type"]
|
Refactor the Scorer to improve flexibility (#5731)
* Refactor the Scorer to improve flexibility
Refactor the `Scorer` to improve flexibility for arbitrary pipeline
components.
* Individual pipeline components provide their own `evaluate` methods
that score a list of `Example`s and return a dictionary of scores
* `Scorer` is initialized either:
* with a provided pipeline containing components to be scored
* with a default pipeline containing the built-in statistical
components (senter, tagger, morphologizer, parser, ner)
* `Scorer.score` evaluates a list of `Example`s and returns a dictionary
of scores referring to the scores provided by the components in the
pipeline
Significant differences:
* `tags_acc` is renamed to `tag_acc` to be consistent with `token_acc`
and the new `morph_acc`, `pos_acc`, and `lemma_acc`
* Scoring is no longer cumulative: `Scorer.score` scores a list of
examples rather than a single example and does not retain any state
about previously scored examples
* PRF values in the returned scores are no longer multiplied by 100
* Add kwargs to Morphologizer.evaluate
* Create generalized scoring methods in Scorer
* Generalized static scoring methods are added to `Scorer`
* Methods require an attribute (either on Token or Doc) that is
used to key the returned scores
Naming differences:
* `uas`, `las`, and `las_per_type` in the scores dict are renamed to
`dep_uas`, `dep_las`, and `dep_las_per_type`
Scoring differences:
* `Doc.sents` is now scored as spans rather than on sentence-initial
token positions so that `Doc.sents` and `Doc.ents` can be scored with
the same method (this lowers scores since a single incorrect sentence
start results in two incorrect spans)
* Simplify / extend hasattr check for eval method
* Add hasattr check to tokenizer scoring
* Simplify to hasattr check for component scoring
* Reset Example alignment if docs are set
Reset the Example alignment if either doc is set in case the
tokenization has changed.
* Add PRF tokenization scoring for tokens as spans
Add PRF scores for tokens as character spans. The scores are:
* token_acc: # correct tokens / # gold tokens
* token_p/r/f: PRF for (token.idx, token.idx + len(token))
* Add docstring to Scorer.score_tokenization
* Rename component.evaluate() to component.score()
* Update Scorer API docs
* Update scoring for positive_label in textcat
* Fix TextCategorizer.score kwargs
* Update Language.evaluate docs
* Update score names in default config
2020-07-25 10:53:02 +00:00
|
|
|
if "ents_per_type" in scores:
|
|
|
|
if scores["ents_per_type"]:
|
2020-10-19 11:18:47 +00:00
|
|
|
print_prf_per_type(msg, scores["ents_per_type"], "NER", "type")
|
2020-10-19 10:07:46 +00:00
|
|
|
data["ents_per_type"] = scores["ents_per_type"]
|
2020-07-27 09:17:52 +00:00
|
|
|
if "cats_f_per_type" in scores:
|
|
|
|
if scores["cats_f_per_type"]:
|
2020-10-19 11:18:47 +00:00
|
|
|
print_prf_per_type(msg, scores["cats_f_per_type"], "Textcat F", "label")
|
2020-10-19 10:07:46 +00:00
|
|
|
data["cats_f_per_type"] = scores["cats_f_per_type"]
|
2020-07-27 09:17:52 +00:00
|
|
|
if "cats_auc_per_type" in scores:
|
|
|
|
if scores["cats_auc_per_type"]:
|
|
|
|
print_textcats_auc_per_cat(msg, scores["cats_auc_per_type"])
|
2020-10-19 10:07:46 +00:00
|
|
|
data["cats_auc_per_type"] = scores["cats_auc_per_type"]
|
2020-06-28 13:34:28 +00:00
|
|
|
|
2017-10-03 22:03:15 +00:00
|
|
|
if displacy_path:
|
2020-07-22 11:42:59 +00:00
|
|
|
factory_names = [nlp.get_pipe_meta(pipe).factory for pipe in nlp.pipe_names]
|
2020-06-27 19:15:25 +00:00
|
|
|
docs = [ex.predicted for ex in dev_dataset]
|
2020-07-22 11:42:59 +00:00
|
|
|
render_deps = "parser" in factory_names
|
|
|
|
render_ents = "ner" in factory_names
|
2018-11-30 19:16:14 +00:00
|
|
|
render_parses(
|
|
|
|
docs,
|
|
|
|
displacy_path,
|
|
|
|
model_name=model,
|
|
|
|
limit=displacy_limit,
|
|
|
|
deps=render_deps,
|
|
|
|
ents=render_ents,
|
|
|
|
)
|
2019-12-22 00:53:56 +00:00
|
|
|
msg.good(f"Generated {displacy_limit} parses as HTML", displacy_path)
|
2020-06-27 19:13:11 +00:00
|
|
|
|
|
|
|
if output_path is not None:
|
|
|
|
srsly.write_json(output_path, data)
|
2020-06-27 19:15:13 +00:00
|
|
|
msg.good(f"Saved results to {output_path}")
|
2020-06-27 19:13:11 +00:00
|
|
|
return data
|
2017-10-01 19:04:32 +00:00
|
|
|
|
|
|
|
|
2020-06-21 19:35:01 +00:00
|
|
|
def render_parses(
|
|
|
|
docs: List[Doc],
|
|
|
|
output_path: Path,
|
|
|
|
model_name: str = "",
|
|
|
|
limit: int = 250,
|
|
|
|
deps: bool = True,
|
|
|
|
ents: bool = True,
|
|
|
|
):
|
2018-11-30 19:16:14 +00:00
|
|
|
docs[0].user_data["title"] = model_name
|
2017-10-03 22:03:15 +00:00
|
|
|
if ents:
|
2019-08-18 11:54:26 +00:00
|
|
|
html = displacy.render(docs[:limit], style="ent", page=True)
|
2019-08-18 11:55:34 +00:00
|
|
|
with (output_path / "entities.html").open("w", encoding="utf8") as file_:
|
2017-10-03 22:03:15 +00:00
|
|
|
file_.write(html)
|
|
|
|
if deps:
|
2019-08-18 11:54:26 +00:00
|
|
|
html = displacy.render(
|
|
|
|
docs[:limit], style="dep", page=True, options={"compact": True}
|
|
|
|
)
|
2019-08-18 11:55:34 +00:00
|
|
|
with (output_path / "parses.html").open("w", encoding="utf8") as file_:
|
2017-10-03 22:03:15 +00:00
|
|
|
file_.write(html)
|
2020-06-28 13:34:28 +00:00
|
|
|
|
|
|
|
|
2020-10-19 11:18:47 +00:00
|
|
|
def print_prf_per_type(msg: Printer, scores: Dict[str, Dict[str, float]], name: str, type: str) -> None:
|
2020-10-19 10:07:46 +00:00
|
|
|
data = [
|
|
|
|
(k, f"{v['p']*100:.2f}", f"{v['r']*100:.2f}", f"{v['f']*100:.2f}")
|
|
|
|
for k, v in scores.items()
|
|
|
|
]
|
|
|
|
msg.table(
|
|
|
|
data,
|
|
|
|
header=("", "P", "R", "F"),
|
|
|
|
aligns=("l", "r", "r", "r"),
|
2020-10-19 11:18:47 +00:00
|
|
|
title=f"{name} (per {type})",
|
2020-06-28 13:34:28 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
def print_textcats_auc_per_cat(
|
|
|
|
msg: Printer, scores: Dict[str, Dict[str, float]]
|
|
|
|
) -> None:
|
|
|
|
msg.table(
|
Refactor the Scorer to improve flexibility (#5731)
* Refactor the Scorer to improve flexibility
Refactor the `Scorer` to improve flexibility for arbitrary pipeline
components.
* Individual pipeline components provide their own `evaluate` methods
that score a list of `Example`s and return a dictionary of scores
* `Scorer` is initialized either:
* with a provided pipeline containing components to be scored
* with a default pipeline containing the built-in statistical
components (senter, tagger, morphologizer, parser, ner)
* `Scorer.score` evaluates a list of `Example`s and returns a dictionary
of scores referring to the scores provided by the components in the
pipeline
Significant differences:
* `tags_acc` is renamed to `tag_acc` to be consistent with `token_acc`
and the new `morph_acc`, `pos_acc`, and `lemma_acc`
* Scoring is no longer cumulative: `Scorer.score` scores a list of
examples rather than a single example and does not retain any state
about previously scored examples
* PRF values in the returned scores are no longer multiplied by 100
* Add kwargs to Morphologizer.evaluate
* Create generalized scoring methods in Scorer
* Generalized static scoring methods are added to `Scorer`
* Methods require an attribute (either on Token or Doc) that is
used to key the returned scores
Naming differences:
* `uas`, `las`, and `las_per_type` in the scores dict are renamed to
`dep_uas`, `dep_las`, and `dep_las_per_type`
Scoring differences:
* `Doc.sents` is now scored as spans rather than on sentence-initial
token positions so that `Doc.sents` and `Doc.ents` can be scored with
the same method (this lowers scores since a single incorrect sentence
start results in two incorrect spans)
* Simplify / extend hasattr check for eval method
* Add hasattr check to tokenizer scoring
* Simplify to hasattr check for component scoring
* Reset Example alignment if docs are set
Reset the Example alignment if either doc is set in case the
tokenization has changed.
* Add PRF tokenization scoring for tokens as spans
Add PRF scores for tokens as character spans. The scores are:
* token_acc: # correct tokens / # gold tokens
* token_p/r/f: PRF for (token.idx, token.idx + len(token))
* Add docstring to Scorer.score_tokenization
* Rename component.evaluate() to component.score()
* Update Scorer API docs
* Update scoring for positive_label in textcat
* Fix TextCategorizer.score kwargs
* Update Language.evaluate docs
* Update score names in default config
2020-07-25 10:53:02 +00:00
|
|
|
[(k, f"{v:.2f}") for k, v in scores.items()],
|
2020-06-28 13:34:28 +00:00
|
|
|
header=("", "ROC AUC"),
|
|
|
|
aligns=("l", "r"),
|
|
|
|
title="Textcat ROC AUC (per label)",
|
|
|
|
)
|