From 3b81b009547b5c48dea7660e8081f050014f8609 Mon Sep 17 00:00:00 2001 From: svlandeg Date: Mon, 13 May 2019 14:26:04 +0200 Subject: [PATCH] evaluating on dev set during training --- .../pipeline/wiki_entity_linking/run_el.py | 25 +++--- .../pipeline/wiki_entity_linking/train_el.py | 87 ++++++++++++++++--- 2 files changed, 90 insertions(+), 22 deletions(-) diff --git a/examples/pipeline/wiki_entity_linking/run_el.py b/examples/pipeline/wiki_entity_linking/run_el.py index 96fe58740..66ab0385e 100644 --- a/examples/pipeline/wiki_entity_linking/run_el.py +++ b/examples/pipeline/wiki_entity_linking/run_el.py @@ -70,12 +70,10 @@ def is_dev(file_name): return file_name.endswith("3.txt") -def evaluate(predictions, golds): +def evaluate(predictions, golds, to_print=True): if len(predictions) != len(golds): raise ValueError("predictions and gold entities should have the same length") - print("Evaluating", len(golds), "entities") - tp = 0 fp = 0 fn = 0 @@ -89,17 +87,22 @@ def evaluate(predictions, golds): else: fp += 1 - print("tp", tp) - print("fp", fp) - print("fn", fn) + if to_print: + print("Evaluating", len(golds), "entities") + print("tp", tp) + print("fp", fp) + print("fn", fn) - precision = tp / (tp + fp + 0.0000001) - recall = tp / (tp + fn + 0.0000001) + precision = 100 * tp / (tp + fp + 0.0000001) + recall = 100 * tp / (tp + fn + 0.0000001) fscore = 2 * recall * precision / (recall + precision + 0.0000001) - print("precision", round(100 * precision, 1), "%") - print("recall", round(100 * recall, 1), "%") - print("Fscore", round(100 * fscore, 1), "%") + if to_print: + print("precision", round(precision, 1), "%") + print("recall", round(recall, 1), "%") + print("Fscore", round(fscore, 1), "%") + + return precision, recall, fscore def _prepare_pipeline(nlp, kb): diff --git a/examples/pipeline/wiki_entity_linking/train_el.py b/examples/pipeline/wiki_entity_linking/train_el.py index cfd17bd78..7fd301e02 100644 --- a/examples/pipeline/wiki_entity_linking/train_el.py +++ b/examples/pipeline/wiki_entity_linking/train_el.py @@ -5,6 +5,7 @@ import os import datetime from os import listdir import numpy as np +from random import shuffle from examples.pipeline.wiki_entity_linking import run_el, training_set_creator, kb_creator @@ -16,6 +17,8 @@ from thinc.t2v import Pooling, sum_pool, mean_pool from thinc.t2t import ExtractWindow, ParametricAttention from thinc.misc import Residual, LayerNorm as LN +from spacy.tokens import Doc + """ TODO: this code needs to be implemented in pipes.pyx""" @@ -33,34 +36,93 @@ class EL_Model(): self.article_encoder = self._simple_encoder(in_width=300, out_width=96) def train_model(self, training_dir, entity_descr_output, limit=None, to_print=True): - instances, pos_entities, neg_entities, doc_by_article = self._get_training_data(training_dir, - entity_descr_output, - limit, to_print) + Doc.set_extension("entity_id", default=None) + + train_instances, train_pos, train_neg, train_doc = self._get_training_data(training_dir, + entity_descr_output, + False, + limit, to_print) + + dev_instances, dev_pos, dev_neg, dev_doc = self._get_training_data(training_dir, + entity_descr_output, + True, + limit, to_print) if to_print: - print("Training on", len(instances), "instance clusters") + print("Training on", len(train_instances), "instance clusters") + print("Dev test on", len(dev_instances), "instance clusters") print() self.sgd_entity = self.begin_training(self.entity_encoder) self.sgd_article = self.begin_training(self.article_encoder) + self._test_dev(dev_instances, dev_pos, dev_neg, dev_doc) + losses = {} - for inst_cluster in instances: - pos_ex = pos_entities.get(inst_cluster) - neg_exs = neg_entities.get(inst_cluster, []) + for inst_cluster in train_instances: + pos_ex = train_pos.get(inst_cluster) + neg_exs = train_neg.get(inst_cluster, []) if pos_ex and neg_exs: article = inst_cluster.split(sep="_")[0] entity_id = inst_cluster.split(sep="_")[1] - article_doc = doc_by_article[article] + article_doc = train_doc[article] self.update(article_doc, pos_ex, neg_exs, losses=losses) + p, r, fscore = self._test_dev(dev_instances, dev_pos, dev_neg, dev_doc) + print(round(fscore, 1)) # TODO # elif not pos_ex: # print("Weird. Couldn't find pos example for", inst_cluster) # elif not neg_exs: # print("Weird. Couldn't find neg examples for", inst_cluster) + def _test_dev(self, dev_instances, dev_pos, dev_neg, dev_doc): + predictions = list() + golds = list() + + for inst_cluster in dev_instances: + pos_ex = dev_pos.get(inst_cluster) + neg_exs = dev_neg.get(inst_cluster, []) + ex_to_id = dict() + + if pos_ex and neg_exs: + ex_to_id[pos_ex] = pos_ex._.entity_id + for neg_ex in neg_exs: + ex_to_id[neg_ex] = neg_ex._.entity_id + + article = inst_cluster.split(sep="_")[0] + entity_id = inst_cluster.split(sep="_")[1] + article_doc = dev_doc[article] + + examples = list(neg_exs) + examples.append(pos_ex) + shuffle(examples) + + best_entity, lowest_mse = self._predict(examples, article_doc) + predictions.append(ex_to_id[best_entity]) + golds.append(ex_to_id[pos_ex]) + + + # TODO: use lowest_mse and combine with prior probability + p, r, F = run_el.evaluate(predictions, golds, to_print=False) + return p, r, F + + def _predict(self, entities, article_doc): + doc_encoding = self.article_encoder([article_doc]) + + lowest_mse = None + best_entity = None + + for entity in entities: + entity_encoding = self.entity_encoder([entity]) + mse, _ = self._calculate_similarity(doc_encoding, entity_encoding) + if not best_entity or mse < lowest_mse: + lowest_mse = mse + best_entity = entity + + return best_entity, lowest_mse + def _simple_encoder(self, in_width, out_width): conv_depth = 1 cnn_maxout_pieces = 3 @@ -145,7 +207,7 @@ class EL_Model(): # print("true index", true_index) # print("true prob", entity_probs[true_index]) - print(true_mse) + # print("training loss", true_mse) # print() @@ -198,13 +260,14 @@ class EL_Model(): def _get_labels(self): return tuple(self.labels) - def _get_training_data(self, training_dir, entity_descr_output, limit, to_print): + def _get_training_data(self, training_dir, entity_descr_output, dev, limit, to_print): id_to_descr = kb_creator._get_id_to_description(entity_descr_output) correct_entries, incorrect_entries = training_set_creator.read_training_entities(training_output=training_dir, collect_correct=True, collect_incorrect=True) + instances = list() local_vectors = list() # TODO: local vectors doc_by_article = dict() @@ -214,7 +277,7 @@ class EL_Model(): cnt = 0 for f in listdir(training_dir): if not limit or cnt < limit: - if not run_el.is_dev(f): + if dev == run_el.is_dev(f): article_id = f.replace(".txt", "") if cnt % 500 == 0 and to_print: print(datetime.datetime.now(), "processed", cnt, "files in the dev dataset") @@ -230,6 +293,7 @@ class EL_Model(): if descr: instances.append(article_id + "_" + mention) doc_descr = self.nlp(descr) + doc_descr._.entity_id = entity_pos pos_entities[article_id + "_" + mention] = doc_descr for mention, entity_negs in incorrect_entries[article_id].items(): @@ -237,6 +301,7 @@ class EL_Model(): descr = id_to_descr.get(entity_neg) if descr: doc_descr = self.nlp(descr) + doc_descr._.entity_id = entity_neg descr_list = neg_entities.get(article_id + "_" + mention, []) descr_list.append(doc_descr) neg_entities[article_id + "_" + mention] = descr_list