From aac5028b6eb538d6de2ee3d8447bb27972e3d0d9 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Tue, 30 Dec 2014 21:21:38 +1100 Subject: [PATCH] * Move tagger to _ml --- spacy/tagger.pxd | 20 ------------ spacy/tagger.pyx | 84 ------------------------------------------------ 2 files changed, 104 deletions(-) delete mode 100644 spacy/tagger.pxd delete mode 100644 spacy/tagger.pyx diff --git a/spacy/tagger.pxd b/spacy/tagger.pxd deleted file mode 100644 index abae428bb..000000000 --- a/spacy/tagger.pxd +++ /dev/null @@ -1,20 +0,0 @@ -from libc.stdint cimport uint8_t - -from cymem.cymem cimport Pool - -from thinc.learner cimport LinearModel -from thinc.features cimport Extractor -from thinc.typedefs cimport atom_t, feat_t, weight_t, class_t - -from preshed.maps cimport PreshMapArray - -from .typedefs cimport hash_t, id_t -from .tokens cimport Tokens - - -cdef class Tagger: - cdef class_t predict(self, const atom_t* context, object golds=*) except * - - cpdef readonly Pool mem - cpdef readonly Extractor extractor - cpdef readonly LinearModel model diff --git a/spacy/tagger.pyx b/spacy/tagger.pyx deleted file mode 100644 index a718d04ec..000000000 --- a/spacy/tagger.pyx +++ /dev/null @@ -1,84 +0,0 @@ -# cython: profile=True -from __future__ import unicode_literals -from __future__ import division - -from os import path -import os -import shutil -import random -import json -import cython - -from thinc.features cimport Feature, count_feats - - -def setup_model_dir(tag_names, tag_map, templates, model_dir): - if path.exists(model_dir): - shutil.rmtree(model_dir) - os.mkdir(model_dir) - config = { - 'templates': templates, - 'tag_names': tag_names, - 'tag_map': tag_map - } - with open(path.join(model_dir, 'config.json'), 'w') as file_: - json.dump(config, file_) - - -cdef class Tagger: - """Predict some type of tag, using greedy decoding. The tagger reads its - model and configuration from disk. - """ - def __init__(self, model_dir): - self.mem = Pool() - cfg = json.load(open(path.join(model_dir, 'config.json'))) - templates = cfg['templates'] - univ_counts = {} - cdef unicode tag - cdef unicode univ_tag - tag_names = cfg['tag_names'] - self.extractor = Extractor(templates) - self.model = LinearModel(len(tag_names) + 1, self.extractor.n_templ+2) # TODO - if path.exists(path.join(model_dir, 'model')): - self.model.load(path.join(model_dir, 'model')) - - cdef class_t predict(self, atom_t* context, object golds=None) except *: - """Predict the tag of tokens[i]. - - >>> tokens = EN.tokenize(u'An example sentence.') - >>> tag = EN.pos_tagger.predict(0, tokens) - >>> assert tag == EN.pos_tagger.tag_id('DT') == 5 - """ - cdef int n_feats - cdef const Feature* feats = self.extractor.get_feats(context, &n_feats) - cdef const weight_t* scores = self.model.get_scores(feats, n_feats) - guess = _arg_max(scores, self.model.nr_class) - if golds is not None and guess not in golds: - best = _arg_max_among(scores, golds) - counts = {guess: {}, best: {}} - count_feats(counts[guess], feats, n_feats, -1) - count_feats(counts[best], feats, n_feats, 1) - self.model.update(counts) - return guess - - -cdef int _arg_max(const weight_t* scores, int n_classes) except -1: - cdef int best = 0 - cdef weight_t score = scores[best] - cdef int i - for i in range(1, n_classes): - if scores[i] >= score: - score = scores[i] - best = i - return best - - -cdef int _arg_max_among(const weight_t* scores, list classes) except -1: - cdef int best = classes[0] - cdef weight_t score = scores[best] - cdef class_t clas - for clas in classes: - if scores[clas] > score: - score = scores[clas] - best = clas - return best