Rooba
/
spaCy
mirror of https://github.com/explosion/spaCy.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

* Fix reading of NER in gold.pyx

This commit is contained in:
Matthew Honnibal 2015-05-27 03:17:50 +02:00
parent 04bda8648d
commit 6016ee83a6
1 changed files with 8 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
spacy/gold.pyx
View File

@ -4,6 +4,7 @@ import json
import random import random
import re import re
from spacy.munge.read_ner import tags_to_entities
from libc.string cimport memset from libc.string cimport memset
@ -97,18 +98,19 @@ def read_json_file(loc):
tags = [] tags = []
heads = [] heads = []
labels = [] labels = []
iob_ents = [] ner = []
for token in paragraph['tokens']: for token in paragraph['tokens']:
words.append(token['orth']) words.append(token['orth'])
ids.append(token['id']) ids.append(token['id'])
tags.append(token['tag']) tags.append(token['tag'])
heads.append(token['head'] if token['head'] >= 0 else token['id']) heads.append(token['head'] if token['head'] >= 0 else token['id'])
labels.append(token['dep']) labels.append(token['dep'])
iob_ents.append(token.get('iob_ent', '-')) ner.append(token.get('ner', '-'))
brackets = [] brackets = []
paragraphs.append((paragraph['raw'], paragraphs.append((
(ids, words, tags, heads, labels, _iob_to_biluo(iob_ents)), paragraph['raw'],
(ids, words, tags, heads, labels, ner),
paragraph.get('brackets', []))) paragraph.get('brackets', [])))
return paragraphs return paragraphs
@ -171,8 +173,6 @@ cdef class GoldParse:
self.orig_annot = zip(*annot_tuples) self.orig_annot = zip(*annot_tuples)
self.ents = []
for i, gold_i in enumerate(self.cand_to_gold): for i, gold_i in enumerate(self.cand_to_gold):
if gold_i is None: if gold_i is None:
# TODO: What do we do for missing values again? # TODO: What do we do for missing values again?
@ -181,15 +181,7 @@ cdef class GoldParse:
self.tags[i] = annot_tuples[2][gold_i] self.tags[i] = annot_tuples[2][gold_i]
self.heads[i] = self.gold_to_cand[annot_tuples[3][gold_i]] self.heads[i] = self.gold_to_cand[annot_tuples[3][gold_i]]
self.labels[i] = annot_tuples[4][gold_i] self.labels[i] = annot_tuples[4][gold_i]
# TODO: Declare NER information MISSING if tokenization incorrect self.ner[i] = annot_tuples[5][gold_i]
for start, end, label in self.ents:
if start == (end - 1):
self.ner[start] = 'U-%s' % label
else:
self.ner[start] = 'B-%s' % label
for i in range(start+1, end-1):
self.ner[i] = 'I-%s' % label
self.ner[end-1] = 'L-%s' % label
self.brackets = {} self.brackets = {}
for (gold_start, gold_end, label_str) in brackets: for (gold_start, gold_end, label_str) in brackets:
@ -197,7 +189,7 @@ cdef class GoldParse:
end = self.gold_to_cand[gold_end] end = self.gold_to_cand[gold_end]
if start is not None and end is not None: if start is not None and end is not None:
self.brackets.setdefault(start, {}).setdefault(end, set()) self.brackets.setdefault(start, {}).setdefault(end, set())
self.brackets[end][start].add(label) self.brackets[end][start].add(label_str)
def __len__(self): def __len__(self):
return self.length return self.length