mirror of https://github.com/explosion/spaCy.git
* Begin revising tagger, focussing on POS tagging
This commit is contained in:
parent
f5c4f2eb52
commit
0c7aeb9de7
|
@ -5,20 +5,17 @@ from thinc.features cimport Extractor
|
||||||
from thinc.typedefs cimport atom_t, feat_t, weight_t, class_t
|
from thinc.typedefs cimport atom_t, feat_t, weight_t, class_t
|
||||||
|
|
||||||
from .typedefs cimport hash_t
|
from .typedefs cimport hash_t
|
||||||
from .context cimport Slots
|
|
||||||
from .tokens cimport Tokens
|
from .tokens cimport Tokens
|
||||||
|
|
||||||
|
|
||||||
cpdef enum TagType:
|
cpdef enum TagType:
|
||||||
POS
|
POS
|
||||||
ENTITY
|
|
||||||
SENSE
|
SENSE
|
||||||
|
|
||||||
|
|
||||||
cdef class Tagger:
|
cdef class Tagger:
|
||||||
cpdef int set_tags(self, Tokens tokens) except -1
|
cpdef int set_tags(self, Tokens tokens) except -1
|
||||||
cpdef class_t predict(self, int i, Tokens tokens) except 0
|
cpdef class_t predict(self, int i, Tokens tokens, object golds=*) except 0
|
||||||
cpdef int tell_answer(self, list gold) except -1
|
|
||||||
|
|
||||||
cpdef readonly Pool mem
|
cpdef readonly Pool mem
|
||||||
cpdef readonly Extractor extractor
|
cpdef readonly Extractor extractor
|
||||||
|
@ -26,9 +23,3 @@ cdef class Tagger:
|
||||||
|
|
||||||
cpdef readonly TagType tag_type
|
cpdef readonly TagType tag_type
|
||||||
cpdef readonly list tag_names
|
cpdef readonly list tag_names
|
||||||
|
|
||||||
cdef class_t _guess
|
|
||||||
cdef atom_t* _context
|
|
||||||
cdef feat_t* _feats
|
|
||||||
cdef weight_t* _values
|
|
||||||
cdef weight_t* _scores
|
|
||||||
|
|
100
spacy/tagger.pyx
100
spacy/tagger.pyx
|
@ -1,8 +1,10 @@
|
||||||
# cython: profile=True
|
# cython: profile=True
|
||||||
from __future__ import print_function
|
|
||||||
from __future__ import unicode_literals
|
from __future__ import unicode_literals
|
||||||
from __future__ import division
|
from __future__ import division
|
||||||
|
|
||||||
|
from .context cimport fill_context
|
||||||
|
from .context cimport N_FIELDS
|
||||||
|
|
||||||
from os import path
|
from os import path
|
||||||
import os
|
import os
|
||||||
import shutil
|
import shutil
|
||||||
|
@ -10,11 +12,7 @@ import random
|
||||||
import json
|
import json
|
||||||
import cython
|
import cython
|
||||||
|
|
||||||
|
from thinc.features cimport Feature, count_feats
|
||||||
from .context cimport fill_context
|
|
||||||
from .context cimport N_FIELDS
|
|
||||||
|
|
||||||
from thinc.features cimport ConjFeat
|
|
||||||
|
|
||||||
|
|
||||||
NULL_TAG = 0
|
NULL_TAG = 0
|
||||||
|
@ -35,7 +33,8 @@ def setup_model_dir(tag_type, tag_names, templates, model_dir):
|
||||||
|
|
||||||
def train(train_sents, model_dir, nr_iter=10):
|
def train(train_sents, model_dir, nr_iter=10):
|
||||||
cdef Tokens tokens
|
cdef Tokens tokens
|
||||||
tagger = Tagger(model_dir)
|
cdef Tagger tagger = Tagger(model_dir)
|
||||||
|
cdef int i
|
||||||
for _ in range(nr_iter):
|
for _ in range(nr_iter):
|
||||||
n_corr = 0
|
n_corr = 0
|
||||||
total = 0
|
total = 0
|
||||||
|
@ -43,9 +42,10 @@ def train(train_sents, model_dir, nr_iter=10):
|
||||||
assert len(tokens) == len(golds), [t.string for t in tokens]
|
assert len(tokens) == len(golds), [t.string for t in tokens]
|
||||||
for i in range(tokens.length):
|
for i in range(tokens.length):
|
||||||
if tagger.tag_type == POS:
|
if tagger.tag_type == POS:
|
||||||
gold = _get_gold_pos(i, golds, tokens.pos)
|
gold = _get_gold_pos(i, golds)
|
||||||
elif tagger.tag_type == ENTITY:
|
else:
|
||||||
gold = _get_gold_ner(i, golds, tokens.ner)
|
raise StandardError
|
||||||
|
|
||||||
guess = tagger.predict(i, tokens)
|
guess = tagger.predict(i, tokens)
|
||||||
tokens.set_tag(i, tagger.tag_type, guess)
|
tokens.set_tag(i, tagger.tag_type, guess)
|
||||||
if gold is not None:
|
if gold is not None:
|
||||||
|
@ -59,7 +59,7 @@ def train(train_sents, model_dir, nr_iter=10):
|
||||||
tagger.model.dump(path.join(model_dir, 'model'))
|
tagger.model.dump(path.join(model_dir, 'model'))
|
||||||
|
|
||||||
|
|
||||||
cdef object _get_gold_pos(i, golds, int* pred):
|
cdef object _get_gold_pos(i, golds):
|
||||||
if golds[i] == 0:
|
if golds[i] == 0:
|
||||||
return None
|
return None
|
||||||
else:
|
else:
|
||||||
|
@ -96,17 +96,11 @@ cdef class Tagger:
|
||||||
templates = cfg['templates']
|
templates = cfg['templates']
|
||||||
self.tag_names = cfg['tag_names']
|
self.tag_names = cfg['tag_names']
|
||||||
self.tag_type = cfg['tag_type']
|
self.tag_type = cfg['tag_type']
|
||||||
self.extractor = Extractor(templates, [ConjFeat] * len(templates))
|
self.extractor = Extractor(templates)
|
||||||
self.model = LinearModel(len(self.tag_names))
|
self.model = LinearModel(len(self.tag_names))
|
||||||
if path.exists(path.join(model_dir, 'model')):
|
if path.exists(path.join(model_dir, 'model')):
|
||||||
self.model.load(path.join(model_dir, 'model'))
|
self.model.load(path.join(model_dir, 'model'))
|
||||||
|
|
||||||
self._context = <atom_t*>self.mem.alloc(N_FIELDS, sizeof(atom_t))
|
|
||||||
self._feats = <feat_t*>self.mem.alloc(self.extractor.n+1, sizeof(feat_t))
|
|
||||||
self._values = <weight_t*>self.mem.alloc(self.extractor.n+1, sizeof(weight_t))
|
|
||||||
self._scores = <weight_t*>self.mem.alloc(self.model.nr_class, sizeof(weight_t))
|
|
||||||
self._guess = NULL_TAG
|
|
||||||
|
|
||||||
cpdef int set_tags(self, Tokens tokens) except -1:
|
cpdef int set_tags(self, Tokens tokens) except -1:
|
||||||
"""Assign tags to a Tokens object.
|
"""Assign tags to a Tokens object.
|
||||||
|
|
||||||
|
@ -119,7 +113,7 @@ cdef class Tagger:
|
||||||
for i in range(tokens.length):
|
for i in range(tokens.length):
|
||||||
tokens.set_tag(i, self.tag_type, self.predict(i, tokens))
|
tokens.set_tag(i, self.tag_type, self.predict(i, tokens))
|
||||||
|
|
||||||
cpdef class_t predict(self, int i, Tokens tokens) except 0:
|
cpdef class_t predict(self, int i, Tokens tokens, object golds=None) except 0:
|
||||||
"""Predict the tag of tokens[i]. The tagger remembers the features and
|
"""Predict the tag of tokens[i]. The tagger remembers the features and
|
||||||
prediction, in case you later call tell_answer.
|
prediction, in case you later call tell_answer.
|
||||||
|
|
||||||
|
@ -127,38 +121,20 @@ cdef class Tagger:
|
||||||
>>> tag = EN.pos_tagger.predict(0, tokens)
|
>>> tag = EN.pos_tagger.predict(0, tokens)
|
||||||
>>> assert tag == EN.pos_tagger.tag_id('DT') == 5
|
>>> assert tag == EN.pos_tagger.tag_id('DT') == 5
|
||||||
"""
|
"""
|
||||||
fill_context(self._context, i, tokens)
|
cdef int n_feats
|
||||||
self.extractor.extract(self._feats, self._values, self._context, NULL)
|
cdef atom_t[N_FIELDS] context
|
||||||
self._guess = self.model.score(self._scores, self._feats, self._values)
|
print sizeof(context)
|
||||||
return self._guess
|
fill_context(context, i, tokens.data)
|
||||||
|
cdef Feature* feats = self.extractor.get_feats(context, &n_feats)
|
||||||
cpdef int tell_answer(self, list golds) except -1:
|
cdef weight_t* scores = self.model.get_scores(feats, n_feats)
|
||||||
"""Provide the correct tag for the word the tagger was last asked to predict.
|
cdef class_t guess = _arg_max(scores, self.nr_class)
|
||||||
During Tagger.predict, the tagger remembers the features and prediction
|
if golds is not None and guess not in golds:
|
||||||
for the example. These are used to calculate a weight update given the
|
best = _arg_max_among(scores, golds)
|
||||||
correct label.
|
counts = {}
|
||||||
|
count_feats(counts[guess], feats, n_feats, -1)
|
||||||
>>> tokens = EN.tokenize('An example sentence.')
|
count_feats(counts[best], feats, n_feats, 1)
|
||||||
>>> guess = EN.pos_tagger.predict(1, tokens)
|
self.model.update(counts)
|
||||||
>>> JJ = EN.pos_tagger.tag_id('JJ')
|
return guess
|
||||||
>>> JJ
|
|
||||||
7
|
|
||||||
>>> EN.pos_tagger.tell_answer(JJ)
|
|
||||||
"""
|
|
||||||
cdef class_t guess = self._guess
|
|
||||||
if guess in golds:
|
|
||||||
self.model.update({})
|
|
||||||
return 0
|
|
||||||
best_gold = golds[0]
|
|
||||||
best_score = self._scores[best_gold-1]
|
|
||||||
for gold in golds[1:]:
|
|
||||||
if self._scores[gold-1] > best_gold:
|
|
||||||
best_score = self._scores[best_gold-1]
|
|
||||||
best_gold = gold
|
|
||||||
counts = {guess: {}, best_gold: {}}
|
|
||||||
self.extractor.count(counts[best_gold], self._feats, 1)
|
|
||||||
self.extractor.count(counts[guess], self._feats, -1)
|
|
||||||
self.model.update(counts)
|
|
||||||
|
|
||||||
def tag_id(self, object tag_name):
|
def tag_id(self, object tag_name):
|
||||||
"""Encode tag_name into a tag ID integer."""
|
"""Encode tag_name into a tag ID integer."""
|
||||||
|
@ -167,3 +143,25 @@ cdef class Tagger:
|
||||||
tag_id = len(self.tag_names)
|
tag_id = len(self.tag_names)
|
||||||
self.tag_names.append(tag_name)
|
self.tag_names.append(tag_name)
|
||||||
return tag_id
|
return tag_id
|
||||||
|
|
||||||
|
|
||||||
|
cdef class_t _arg_max(weight_t* scores, int n_classes):
|
||||||
|
cdef int best = 0
|
||||||
|
cdef weight_t score = scores[best]
|
||||||
|
cdef int i
|
||||||
|
for i in range(1, n_classes):
|
||||||
|
if scores[i] > score:
|
||||||
|
score = scores[i]
|
||||||
|
best = i
|
||||||
|
return best
|
||||||
|
|
||||||
|
|
||||||
|
cdef class_t _arg_max_among(weight_t* scores, list classes):
|
||||||
|
cdef int best = classes[0]
|
||||||
|
cdef weight_t score = scores[best]
|
||||||
|
cdef class_t clas
|
||||||
|
for clas in classes:
|
||||||
|
if scores[clas] > score:
|
||||||
|
score = scores[clas]
|
||||||
|
best = clas
|
||||||
|
return best
|
||||||
|
|
Loading…
Reference in New Issue