# cython: infer_types=True # coding: utf-8 from __future__ import unicode_literals from cpython.ref cimport PyObject, Py_INCREF, Py_XDECREF from cymem.cymem cimport Pool from thinc.typedefs cimport weight_t from collections import defaultdict, OrderedDict from ..structs cimport TokenC from .stateclass cimport StateClass from ..attrs cimport TAG, HEAD, DEP, ENT_TYPE, ENT_IOB cdef weight_t MIN_SCORE = -90000 class OracleError(Exception): pass cdef void* _init_state(Pool mem, int length, void* tokens) except NULL: cdef StateClass st = StateClass.init(tokens, length) Py_INCREF(st) return st cdef class TransitionSystem: def __init__(self, StringStore string_table, labels_by_action): self.mem = Pool() self.strings = string_table self.n_moves = 0 self._size = 100 self.c = self.mem.alloc(self._size, sizeof(Transition)) for action, label_strs in labels_by_action.items(): for label_str in label_strs: self.add_action(int(action), label_str) self.root_label = self.strings['ROOT'] self.init_beam_state = _init_state def __reduce__(self): labels_by_action = OrderedDict() cdef Transition t for trans in self.c[:self.n_moves]: label_str = self.strings[trans.label] labels_by_action.setdefault(trans.move, []).append(label_str) return (self.__class__, (self.strings, labels_by_action), None, None) def init_batch(self, docs): cdef StateClass state states = [] offset = 0 for doc in docs: state = StateClass(doc, offset=offset) self.initialize_state(state.c) states.append(state) offset += len(doc) return states def get_oracle_sequence(self, doc, GoldParse gold): cdef Pool mem = Pool() costs = mem.alloc(self.n_moves, sizeof(float)) is_valid = mem.alloc(self.n_moves, sizeof(int)) cdef StateClass state = StateClass(doc, offset=0) self.initialize_state(state.c) history = [] while not state.is_final(): self.set_costs(is_valid, costs, state, gold) for i in range(self.n_moves): if is_valid[i] and costs[i] <= 0: action = self.c[i] history.append(i) action.do(state.c, action.label) break else: print(gold.words) print(gold.ner) print(history) raise ValueError("Could not find gold move") return history cdef int initialize_state(self, StateC* state) nogil: pass cdef int finalize_state(self, StateC* state) nogil: pass def finalize_doc(self, doc): pass def preprocess_gold(self, GoldParse gold): raise NotImplementedError cdef Transition lookup_transition(self, object name) except *: raise NotImplementedError cdef Transition init_transition(self, int clas, int move, attr_t label) except *: raise NotImplementedError def is_valid(self, StateClass stcls, move_name): action = self.lookup_transition(move_name) return action.is_valid(stcls.c, action.label) cdef int set_valid(self, int* is_valid, const StateC* st) nogil: cdef int i for i in range(self.n_moves): is_valid[i] = self.c[i].is_valid(st, self.c[i].label) cdef int set_costs(self, int* is_valid, weight_t* costs, StateClass stcls, GoldParse gold) except -1: cdef int i self.set_valid(is_valid, stcls.c) cdef int n_gold = 0 for i in range(self.n_moves): if is_valid[i]: costs[i] = self.c[i].get_cost(stcls, &gold.c, self.c[i].label) n_gold += costs[i] <= 0 else: costs[i] = 9000 if n_gold <= 0: print(gold.words) print(gold.ner) raise ValueError( "Could not find a gold-standard action to supervise " "the entity recognizer\n" "The transition system has %d actions.\n" "%s" % (self.n_moves)) def add_action(self, int action, label): if not isinstance(label, int): label = self.strings[label] # Check we're not creating a move we already have, so that this is # idempotent for trans in self.c[:self.n_moves]: if trans.move == action and trans.label == label: return 0 if self.n_moves >= self._size: self._size *= 2 self.c = self.mem.realloc(self.c, self._size * sizeof(self.c[0])) self.c[self.n_moves] = self.init_transition(self.n_moves, action, label) self.n_moves += 1 return 1