2016-10-27 16:08:13 +00:00
|
|
|
# cython: infer_types=True
|
2017-04-15 11:05:15 +00:00
|
|
|
# coding: utf-8
|
|
|
|
from __future__ import unicode_literals
|
|
|
|
|
|
|
|
from cpython.ref cimport PyObject, Py_INCREF, Py_XDECREF
|
2015-02-21 15:38:18 +00:00
|
|
|
from cymem.cymem cimport Pool
|
|
|
|
from thinc.typedefs cimport weight_t
|
2017-05-27 20:52:20 +00:00
|
|
|
from collections import defaultdict, OrderedDict
|
2017-05-29 09:45:45 +00:00
|
|
|
import ujson
|
2015-02-21 15:38:18 +00:00
|
|
|
|
2017-05-29 09:45:45 +00:00
|
|
|
from .. import util
|
2015-07-19 13:18:17 +00:00
|
|
|
from ..structs cimport TokenC
|
2015-06-09 19:20:14 +00:00
|
|
|
from .stateclass cimport StateClass
|
2015-07-19 13:18:17 +00:00
|
|
|
from ..attrs cimport TAG, HEAD, DEP, ENT_TYPE, ENT_IOB
|
2017-05-28 16:09:27 +00:00
|
|
|
from ..typedefs cimport attr_t
|
2015-06-09 19:20:14 +00:00
|
|
|
|
2015-02-21 15:38:18 +00:00
|
|
|
|
|
|
|
cdef weight_t MIN_SCORE = -90000
|
|
|
|
|
|
|
|
|
2015-02-22 05:32:07 +00:00
|
|
|
class OracleError(Exception):
|
|
|
|
pass
|
|
|
|
|
|
|
|
|
2017-03-11 17:12:01 +00:00
|
|
|
cdef void* _init_state(Pool mem, int length, void* tokens) except NULL:
|
|
|
|
cdef StateClass st = StateClass.init(<const TokenC*>tokens, length)
|
|
|
|
Py_INCREF(st)
|
|
|
|
return <void*>st
|
|
|
|
|
|
|
|
|
2015-02-21 15:38:18 +00:00
|
|
|
cdef class TransitionSystem:
|
2017-05-27 20:52:20 +00:00
|
|
|
def __init__(self, StringStore string_table, labels_by_action):
|
2015-02-21 15:38:18 +00:00
|
|
|
self.mem = Pool()
|
2015-03-14 15:06:35 +00:00
|
|
|
self.strings = string_table
|
2016-01-19 18:09:33 +00:00
|
|
|
self.n_moves = 0
|
|
|
|
self._size = 100
|
2017-03-11 17:12:01 +00:00
|
|
|
|
2016-01-19 18:09:33 +00:00
|
|
|
self.c = <Transition*>self.mem.alloc(self._size, sizeof(Transition))
|
2017-03-11 17:12:01 +00:00
|
|
|
|
2017-05-27 20:52:20 +00:00
|
|
|
for action, label_strs in labels_by_action.items():
|
2017-04-14 21:52:17 +00:00
|
|
|
for label_str in label_strs:
|
2016-01-19 18:09:33 +00:00
|
|
|
self.add_action(int(action), label_str)
|
2017-05-28 16:09:27 +00:00
|
|
|
self.root_label = self.strings.add('ROOT')
|
2017-03-11 17:12:01 +00:00
|
|
|
self.init_beam_state = _init_state
|
2015-02-21 15:38:18 +00:00
|
|
|
|
2015-10-12 08:33:11 +00:00
|
|
|
def __reduce__(self):
|
2017-05-27 20:52:20 +00:00
|
|
|
labels_by_action = OrderedDict()
|
2016-01-19 18:09:33 +00:00
|
|
|
cdef Transition t
|
|
|
|
for trans in self.c[:self.n_moves]:
|
|
|
|
label_str = self.strings[trans.label]
|
|
|
|
labels_by_action.setdefault(trans.move, []).append(label_str)
|
2015-10-12 08:33:11 +00:00
|
|
|
return (self.__class__,
|
2017-05-20 07:20:16 +00:00
|
|
|
(self.strings, labels_by_action),
|
2015-10-12 08:33:11 +00:00
|
|
|
None, None)
|
|
|
|
|
2017-05-15 19:46:08 +00:00
|
|
|
def init_batch(self, docs):
|
|
|
|
cdef StateClass state
|
|
|
|
states = []
|
|
|
|
offset = 0
|
|
|
|
for doc in docs:
|
|
|
|
state = StateClass(doc, offset=offset)
|
|
|
|
self.initialize_state(state.c)
|
|
|
|
states.append(state)
|
|
|
|
offset += len(doc)
|
|
|
|
return states
|
|
|
|
|
2017-05-26 16:31:23 +00:00
|
|
|
def get_oracle_sequence(self, doc, GoldParse gold):
|
|
|
|
cdef Pool mem = Pool()
|
|
|
|
costs = <float*>mem.alloc(self.n_moves, sizeof(float))
|
|
|
|
is_valid = <int*>mem.alloc(self.n_moves, sizeof(int))
|
|
|
|
|
|
|
|
cdef StateClass state = StateClass(doc, offset=0)
|
|
|
|
self.initialize_state(state.c)
|
|
|
|
history = []
|
|
|
|
while not state.is_final():
|
|
|
|
self.set_costs(is_valid, costs, state, gold)
|
|
|
|
for i in range(self.n_moves):
|
|
|
|
if is_valid[i] and costs[i] <= 0:
|
|
|
|
action = self.c[i]
|
|
|
|
history.append(i)
|
|
|
|
action.do(state.c, action.label)
|
|
|
|
break
|
2017-05-27 20:52:20 +00:00
|
|
|
else:
|
|
|
|
print(gold.words)
|
|
|
|
print(gold.ner)
|
|
|
|
print(history)
|
|
|
|
raise ValueError("Could not find gold move")
|
2017-05-26 16:31:23 +00:00
|
|
|
return history
|
|
|
|
|
2016-02-01 07:34:55 +00:00
|
|
|
cdef int initialize_state(self, StateC* state) nogil:
|
2015-04-28 18:45:51 +00:00
|
|
|
pass
|
|
|
|
|
2016-02-01 07:34:55 +00:00
|
|
|
cdef int finalize_state(self, StateC* state) nogil:
|
2015-04-28 18:45:51 +00:00
|
|
|
pass
|
2015-03-10 17:00:23 +00:00
|
|
|
|
2016-05-02 12:25:10 +00:00
|
|
|
def finalize_doc(self, doc):
|
|
|
|
pass
|
|
|
|
|
2017-05-22 15:30:12 +00:00
|
|
|
def preprocess_gold(self, GoldParse gold):
|
2015-03-09 11:06:01 +00:00
|
|
|
raise NotImplementedError
|
|
|
|
|
|
|
|
cdef Transition lookup_transition(self, object name) except *:
|
|
|
|
raise NotImplementedError
|
|
|
|
|
2017-05-28 12:06:40 +00:00
|
|
|
cdef Transition init_transition(self, int clas, int move, attr_t label) except *:
|
2015-02-21 15:38:18 +00:00
|
|
|
raise NotImplementedError
|
|
|
|
|
2015-08-08 21:36:18 +00:00
|
|
|
def is_valid(self, StateClass stcls, move_name):
|
|
|
|
action = self.lookup_transition(move_name)
|
2016-02-01 01:58:14 +00:00
|
|
|
return action.is_valid(stcls.c, action.label)
|
2015-08-08 21:36:18 +00:00
|
|
|
|
2016-02-01 02:00:15 +00:00
|
|
|
cdef int set_valid(self, int* is_valid, const StateC* st) nogil:
|
2015-06-04 17:32:32 +00:00
|
|
|
cdef int i
|
|
|
|
for i in range(self.n_moves):
|
2016-02-01 02:00:15 +00:00
|
|
|
is_valid[i] = self.c[i].is_valid(st, self.c[i].label)
|
2015-06-04 17:32:32 +00:00
|
|
|
|
2016-01-30 13:31:12 +00:00
|
|
|
cdef int set_costs(self, int* is_valid, weight_t* costs,
|
2015-06-26 04:25:36 +00:00
|
|
|
StateClass stcls, GoldParse gold) except -1:
|
2015-02-21 15:38:18 +00:00
|
|
|
cdef int i
|
2016-02-01 02:00:15 +00:00
|
|
|
self.set_valid(is_valid, stcls.c)
|
2017-05-26 16:31:23 +00:00
|
|
|
cdef int n_gold = 0
|
2015-02-21 15:38:18 +00:00
|
|
|
for i in range(self.n_moves):
|
2015-06-26 04:25:36 +00:00
|
|
|
if is_valid[i]:
|
|
|
|
costs[i] = self.c[i].get_cost(stcls, &gold.c, self.c[i].label)
|
2017-05-26 16:31:23 +00:00
|
|
|
n_gold += costs[i] <= 0
|
2015-06-26 04:25:36 +00:00
|
|
|
else:
|
|
|
|
costs[i] = 9000
|
2017-05-26 16:31:23 +00:00
|
|
|
if n_gold <= 0:
|
|
|
|
print(gold.words)
|
|
|
|
print(gold.ner)
|
2017-05-28 16:09:27 +00:00
|
|
|
print([gold.c.ner[i].clas for i in range(gold.length)])
|
|
|
|
print([gold.c.ner[i].move for i in range(gold.length)])
|
|
|
|
print([gold.c.ner[i].label for i in range(gold.length)])
|
|
|
|
print("Self labels", [self.c[i].label for i in range(self.n_moves)])
|
2017-05-26 16:31:23 +00:00
|
|
|
raise ValueError(
|
|
|
|
"Could not find a gold-standard action to supervise "
|
|
|
|
"the entity recognizer\n"
|
2017-05-28 16:09:27 +00:00
|
|
|
"The transition system has %d actions." % (self.n_moves))
|
|
|
|
|
|
|
|
def add_action(self, int action, label_name):
|
|
|
|
cdef attr_t label_id
|
|
|
|
if not isinstance(label_name, int):
|
|
|
|
label_id = self.strings.add(label_name)
|
|
|
|
else:
|
|
|
|
label_id = label_name
|
2016-01-19 19:10:04 +00:00
|
|
|
# Check we're not creating a move we already have, so that this is
|
|
|
|
# idempotent
|
|
|
|
for trans in self.c[:self.n_moves]:
|
2017-05-28 16:09:27 +00:00
|
|
|
if trans.move == action and trans.label == label_id:
|
2016-01-19 19:10:04 +00:00
|
|
|
return 0
|
2016-01-19 18:09:33 +00:00
|
|
|
if self.n_moves >= self._size:
|
|
|
|
self._size *= 2
|
|
|
|
self.c = <Transition*>self.mem.realloc(self.c, self._size * sizeof(self.c[0]))
|
2017-05-28 16:09:27 +00:00
|
|
|
self.c[self.n_moves] = self.init_transition(self.n_moves, action, label_id)
|
|
|
|
assert self.c[self.n_moves].label == label_id
|
2016-01-19 18:09:33 +00:00
|
|
|
self.n_moves += 1
|
2016-01-19 19:10:04 +00:00
|
|
|
return 1
|
2017-05-29 09:45:45 +00:00
|
|
|
|
|
|
|
def to_disk(self, path, **exclude):
|
2017-05-31 11:44:00 +00:00
|
|
|
with path.open('wb') as file_:
|
|
|
|
file_.write(self.to_bytes(**exclude))
|
2017-05-29 09:45:45 +00:00
|
|
|
|
|
|
|
def from_disk(self, path, **exclude):
|
2017-05-31 11:44:00 +00:00
|
|
|
with path.open('rb') as file_:
|
|
|
|
byte_data = file_.read()
|
|
|
|
self.from_bytes(byte_data, **exclude)
|
2017-05-29 09:45:45 +00:00
|
|
|
return self
|
|
|
|
|
|
|
|
def to_bytes(self, **exclude):
|
|
|
|
transitions = []
|
|
|
|
for trans in self.c[:self.n_moves]:
|
|
|
|
transitions.append({
|
|
|
|
'clas': trans.clas,
|
|
|
|
'move': trans.move,
|
|
|
|
'label': self.strings[trans.label],
|
|
|
|
'name': self.move_name(trans.move, trans.label)
|
|
|
|
})
|
|
|
|
serializers = {
|
|
|
|
'transitions': lambda: ujson.dumps(transitions),
|
|
|
|
'strings': lambda: self.strings.to_bytes()
|
|
|
|
}
|
|
|
|
return util.to_bytes(serializers, exclude)
|
|
|
|
|
|
|
|
def from_bytes(self, bytes_data, **exclude):
|
|
|
|
transitions = []
|
|
|
|
deserializers = {
|
|
|
|
'transitions': lambda b: transitions.extend(ujson.loads(b)),
|
|
|
|
'strings': lambda b: self.strings.from_bytes(b)
|
|
|
|
}
|
|
|
|
msg = util.from_bytes(bytes_data, deserializers, exclude)
|
|
|
|
for trans in transitions:
|
|
|
|
self.add_action(trans['move'], trans['label'])
|
|
|
|
return self
|