From 69c7c642c2c3a9c51d0b7c1c1e5e67848f9b7953 Mon Sep 17 00:00:00 2001
From: Matthew Honnibal <honnibal+gh@gmail.com>
Date: Sun, 1 Oct 2017 14:04:32 -0500
Subject: [PATCH] Add spacy evaluate

---
 spacy/__main__.py     |  3 +-
 spacy/cli/__init__.py |  1 +
 spacy/cli/evaluate.py | 93 +++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 96 insertions(+), 1 deletion(-)
 create mode 100644 spacy/cli/evaluate.py

diff --git a/spacy/__main__.py b/spacy/__main__.py
index d02242d68..0ec96e4a1 100644
--- a/spacy/__main__.py
+++ b/spacy/__main__.py
@@ -7,7 +7,7 @@ if __name__ == '__main__':
     import plac
     import sys
     from spacy.cli import download, link, info, package, train, convert, model
-    from spacy.cli import profile
+    from spacy.cli import profile, evaluate
     from spacy.util import prints
 
     commands = {
@@ -15,6 +15,7 @@ if __name__ == '__main__':
         'link': link,
         'info': info,
         'train': train,
+        'evaluate': evaluate,
         'convert': convert,
         'package': package,
         'model': model,
diff --git a/spacy/cli/__init__.py b/spacy/cli/__init__.py
index e58c94642..ebe185f24 100644
--- a/spacy/cli/__init__.py
+++ b/spacy/cli/__init__.py
@@ -4,5 +4,6 @@ from .link import link
 from .package import package
 from .profile import profile
 from .train import train
+from .evaluate import evaluate
 from .convert import convert
 from .model import model
diff --git a/spacy/cli/evaluate.py b/spacy/cli/evaluate.py
new file mode 100644
index 000000000..209660529
--- /dev/null
+++ b/spacy/cli/evaluate.py
@@ -0,0 +1,93 @@
+# coding: utf8
+from __future__ import unicode_literals, division, print_function
+
+import plac
+import json
+from collections import defaultdict
+import cytoolz
+from pathlib import Path
+import dill
+import tqdm
+from thinc.neural._classes.model import Model
+from thinc.neural.optimizers import linear_decay
+from timeit import default_timer as timer
+import random
+import numpy.random
+
+from ..tokens.doc import Doc
+from ..scorer import Scorer
+from ..gold import GoldParse, merge_sents
+from ..gold import GoldCorpus, minibatch
+from ..util import prints
+from .. import util
+from .. import about
+from .. import displacy
+from ..compat import json_dumps
+
+random.seed(0)
+numpy.random.seed(0)
+
+
+@plac.annotations(
+    model=("Model name or path", "positional", None, str),
+    data_path=("Location of JSON-formatted evaluation data", "positional", None, str),
+    gold_preproc=("Use gold preprocessing", "flag", "G", bool),
+)
+def evaluate(cmd, model, data_path, gold_preproc=False):
+    """
+    Train a model. Expects data in spaCy's JSON format.
+    """
+    util.set_env_log(True)
+    data_path = util.ensure_path(data_path)
+    if not data_path.exists():
+        prints(data_path, title="Evaluation data not found", exits=1)
+    corpus = GoldCorpus(data_path, data_path)
+    nlp = util.load_model(model)
+    scorer = nlp.evaluate(list(corpus.dev_docs(nlp, gold_preproc=gold_preproc)))
+    print_results(scorer)
+
+
+def _render_parses(i, to_render):
+    to_render[0].user_data['title'] = "Batch %d" % i
+    with Path('/tmp/entities.html').open('w') as file_:
+        html = displacy.render(to_render[:5], style='ent', page=True)
+        file_.write(html)
+    with Path('/tmp/parses.html').open('w') as file_:
+        html = displacy.render(to_render[:5], style='dep', page=True)
+        file_.write(html)
+
+
+def print_progress(itn, losses, dev_scores, wps=0.0):
+    scores = {}
+    for col in ['dep_loss', 'tag_loss', 'uas', 'tags_acc', 'token_acc',
+                'ents_p', 'ents_r', 'ents_f', 'wps']:
+        scores[col] = 0.0
+    scores['dep_loss'] = losses.get('parser', 0.0)
+    scores['ner_loss'] = losses.get('ner', 0.0)
+    scores['tag_loss'] = losses.get('tagger', 0.0)
+    scores.update(dev_scores)
+    scores['wps'] = wps
+    tpl = '\t'.join((
+        '{:d}',
+        '{dep_loss:.3f}',
+        '{ner_loss:.3f}',
+        '{uas:.3f}',
+        '{ents_p:.3f}',
+        '{ents_r:.3f}',
+        '{ents_f:.3f}',
+        '{tags_acc:.3f}',
+        '{token_acc:.3f}',
+        '{wps:.1f}'))
+    print(tpl.format(itn, **scores))
+
+
+def print_results(scorer):
+    results = {
+        'TOK': '%.2f' % scorer.token_acc,
+        'POS': '%.2f' % scorer.tags_acc,
+        'UAS': '%.2f' % scorer.uas,
+        'LAS': '%.2f' % scorer.las,
+        'NER P': '%.2f' % scorer.ents_p,
+        'NER R': '%.2f' % scorer.ents_r,
+        'NER F': '%.2f' % scorer.ents_f}
+    util.print_table(results, title="Results")