spaCy/spacy/syntax/parser.pyx

121 lines
3.7 KiB
Cython
Raw Normal View History

2015-06-02 16:38:41 +00:00
# cython: profile=True
2014-12-16 11:44:43 +00:00
"""
MALT-style dependency parser
"""
from __future__ import unicode_literals
cimport cython
from cpython.ref cimport PyObject, Py_INCREF, Py_XDECREF
from libc.stdint cimport uint32_t, uint64_t
2015-06-02 16:38:41 +00:00
from libc.string cimport memset, memcpy
2014-12-16 11:44:43 +00:00
import random
import os.path
from os import path
2014-12-16 11:44:43 +00:00
import shutil
import json
import sys
2014-12-16 11:44:43 +00:00
from cymem.cymem cimport Pool, Address
from murmurhash.mrmr cimport hash64
from thinc.typedefs cimport weight_t, class_t, feat_t, atom_t, hash_t
2014-12-16 11:44:43 +00:00
from util import Config
from thinc.api cimport Example
2014-12-16 11:44:43 +00:00
from ..structs cimport TokenC
from ..tokens.doc cimport Doc
from ..strings cimport StringStore
2014-12-16 11:44:43 +00:00
from .transition_system import OracleError
from .transition_system cimport TransitionSystem, Transition
2014-12-16 11:44:43 +00:00
from ..gold cimport GoldParse
2014-12-16 11:44:43 +00:00
from . import _parse_features
from ._parse_features cimport CONTEXT_SIZE
from ._parse_features cimport fill_context
from .stateclass cimport StateClass
2014-12-16 11:44:43 +00:00
2015-04-19 08:31:31 +00:00
DEBUG = False
2014-12-16 11:44:43 +00:00
def set_debug(val):
global DEBUG
DEBUG = val
def get_templates(name):
2014-12-17 10:09:29 +00:00
pf = _parse_features
2015-03-24 04:08:35 +00:00
if name == 'ner':
return pf.ner
elif name == 'debug':
return pf.unigrams
elif name.startswith('embed'):
2015-06-27 02:18:47 +00:00
return (pf.words, pf.tags, pf.labels)
else:
return (pf.unigrams + pf.s0_n0 + pf.s1_n0 + pf.s1_s0 + pf.s0_n1 + pf.n0_n1 + \
pf.tree_shape + pf.trigrams)
2014-12-16 11:44:43 +00:00
def ParserFactory(transition_system):
return lambda strings, dir_: Parser(strings, dir_, transition_system)
cdef class Parser:
def __init__(self, StringStore strings, model_dir, transition_system):
if not os.path.exists(model_dir):
print >> sys.stderr, "Warning: No model found at", model_dir
elif not os.path.isdir(model_dir):
print >> sys.stderr, "Warning: model path:", model_dir, "is not a directory"
else:
self.cfg = Config.read(model_dir, 'config')
self.moves = transition_system(strings, self.cfg.labels)
templates = get_templates(self.cfg.features)
self.model = Model(self.moves.n_moves, templates, model_dir)
2014-12-16 11:44:43 +00:00
2015-07-08 16:53:00 +00:00
def __call__(self, Doc tokens):
cdef StateClass stcls = StateClass.init(tokens.data, tokens.length)
self.moves.initialize_state(stcls)
2015-06-28 20:36:03 +00:00
cdef Example eg = Example(self.model.n_classes, CONTEXT_SIZE,
self.model.n_feats, self.model.n_feats)
while not stcls.is_final():
2015-06-28 20:36:03 +00:00
memset(eg.c.scores, 0, eg.c.nr_class * sizeof(weight_t))
self.moves.set_valid(<bint*>eg.c.is_valid, stcls)
fill_context(eg.c.atoms, stcls)
self.model.predict(eg)
2015-06-28 20:36:03 +00:00
self.moves.c[eg.c.guess].do(stcls, self.moves.c[eg.c.guess].label)
self.moves.finalize_state(stcls)
tokens.set_parse(stcls._sent)
def train(self, Doc tokens, GoldParse gold):
self.moves.preprocess_gold(gold)
cdef StateClass stcls = StateClass.init(tokens.data, tokens.length)
self.moves.initialize_state(stcls)
2015-06-28 20:36:03 +00:00
cdef Example eg = Example(self.model.n_classes, CONTEXT_SIZE,
self.model.n_feats, self.model.n_feats)
2015-06-30 12:26:32 +00:00
cdef weight_t loss = 0
words = [w.orth_ for w in tokens]
2015-06-30 12:26:32 +00:00
cdef Transition G
while not stcls.is_final():
2015-06-28 20:36:03 +00:00
memset(eg.c.scores, 0, eg.c.nr_class * sizeof(weight_t))
self.moves.set_costs(<bint*>eg.c.is_valid, eg.c.costs, stcls, gold)
fill_context(eg.c.atoms, stcls)
self.model.train(eg)
2015-06-30 12:26:32 +00:00
G = self.moves.c[eg.c.guess]
2015-06-28 20:36:03 +00:00
self.moves.c[eg.c.guess].do(stcls, self.moves.c[eg.c.guess].label)
2015-06-30 12:26:32 +00:00
loss += eg.c.loss
return loss