spaCy/spacy/syntax/orig_arc_eager.pyx

# cython: profile=True
from __future__ import unicode_literals

import ctypes
import os

from ..structs cimport TokenC

from .transition_system cimport do_func_t, get_cost_func_t
from .transition_system cimport move_cost_func_t, label_cost_func_t
from ..gold cimport GoldParse
from ..gold cimport GoldParseC

from libc.stdint cimport uint32_t
from libc.string cimport memcpy

from cymem.cymem cimport Pool
from .stateclass cimport StateClass


cdef weight_t MIN_SCORE = -90000

cdef enum:
    SHIFT
    REDUCE
    LEFT
    RIGHT

    N_MOVES


MOVE_NAMES = [None] * N_MOVES
MOVE_NAMES[SHIFT] = 'S'
MOVE_NAMES[REDUCE] = 'D'
MOVE_NAMES[LEFT] = 'L'
MOVE_NAMES[RIGHT] = 'R'


# Helper functions for the arc-eager oracle

cdef int push_cost(StateClass stcls, const GoldParseC* gold, int target) nogil:
    cdef int cost = 0
    cdef int i, S_i
    for i in range(stcls.stack_depth()):
        S_i = stcls.S(i)
        if gold.heads[target] == S_i:
            cost += 1
        if gold.heads[S_i] == target and not stcls.has_head(S_i):
            cost += 1
    return cost


cdef int pop_cost(StateClass stcls, const GoldParseC* gold, int target) nogil:
    if stcls.buffer_length() == 0:
        return 0
    cdef int cost = 0
    cdef int i, B_i
    for i in range(stcls.buffer_length()):
        B_i = stcls.B(i)
        cost += gold.heads[B_i] == target
        if not stcls.has_head(target):
            cost += gold.heads[target] == B_i
        if gold.heads[B_i] == B_i or gold.heads[B_i] < target:
            break
    return cost


cdef int arc_cost(StateClass stcls, const GoldParseC* gold, int head, int child) nogil:
    if arc_is_gold(gold, head, child):
        return 0
    elif stcls.H(child) == gold.heads[child]:
        return 1
    # Head in buffer
    elif gold.heads[child] >= stcls.B(0) and stcls.B(1) != -1:
        return 1
    else:
        return 0


cdef bint arc_is_gold(const GoldParseC* gold, int head, int child) nogil:
    if gold.labels[child] == -1:
        return True
    elif gold.heads[child] == head:
        return True
    else:
        return False


cdef bint label_is_gold(const GoldParseC* gold, int head, int child, int label) nogil:
    if gold.labels[child] == -1:
        return True
    elif label == -1:
        return True
    elif gold.labels[child] == label:
        return True
    else:
        return False


cdef bint _is_gold_root(const GoldParseC* gold, int word) nogil:
    return gold.labels[word] == -1 or gold.heads[word] == word
 

cdef class Shift:
    @staticmethod
    cdef bint is_valid(StateClass st, int label) nogil:
        return st.buffer_length() >= 1

    @staticmethod
    cdef int transition(StateClass st, int label) nogil:
        st.push()

    @staticmethod
    cdef int cost(StateClass st, const GoldParseC* gold, int label) nogil:
        return Shift.move_cost(st, gold) + Shift.label_cost(st, gold, label)

    @staticmethod
    cdef inline int move_cost(StateClass s, const GoldParseC* gold) nogil:
        return push_cost(s, gold, s.B(0))

    @staticmethod
    cdef inline int label_cost(StateClass s, const GoldParseC* gold, int label) nogil:
        return 0


cdef class Reduce:
    @staticmethod
    cdef bint is_valid(StateClass st, int label) nogil:
        return st.stack_depth() >= 1 and st.has_head(st.S(0))

    @staticmethod
    cdef int transition(StateClass st, int label) nogil:
        st.pop()

    @staticmethod
    cdef int cost(StateClass s, const GoldParseC* gold, int label) nogil:
        return Reduce.move_cost(s, gold) + Reduce.label_cost(s, gold, label)

    @staticmethod
    cdef inline int move_cost(StateClass st, const GoldParseC* gold) nogil:
        return pop_cost(st, gold, st.S(0))

    @staticmethod
    cdef inline int label_cost(StateClass s, const GoldParseC* gold, int label) nogil:
        return 0


cdef class LeftArc:
    @staticmethod
    cdef bint is_valid(StateClass st, int label) nogil:
        return st.stack_depth() >= 1 and not st.has_head(st.S(0))

    @staticmethod
    cdef int transition(StateClass st, int label) nogil:
        if not st.buffer_length():
            st.add_arc(st.S(0), st.S(0), label)
        else:
            st.add_arc(st.B(0), st.S(0), label)
        st.pop()

    @staticmethod
    cdef int cost(StateClass s, const GoldParseC* gold, int label) nogil:
        return LeftArc.move_cost(s, gold) + LeftArc.label_cost(s, gold, label)

    @staticmethod
    cdef inline int move_cost(StateClass s, const GoldParseC* gold) nogil:
        if not s.buffer_length():
            return 0
        elif arc_is_gold(gold, s.B(0), s.S(0)):
            return 0
        else:
            return pop_cost(s, gold, s.S(0)) + arc_cost(s, gold, s.B(0), s.S(0))

    @staticmethod
    cdef inline int label_cost(StateClass s, const GoldParseC* gold, int label) nogil:
        if not s.buffer_length():
            return 0
        return arc_is_gold(gold, s.B(0), s.S(0)) and not label_is_gold(gold, s.B(0), s.S(0), label)


cdef class RightArc:
    @staticmethod
    cdef bint is_valid(StateClass st, int label) nogil:
        return st.stack_depth() >= 1 and st.buffer_length() >= 1

    @staticmethod
    cdef int transition(StateClass st, int label) nogil:
        st.add_arc(st.S(0), st.B(0), label)
        st.push()

    @staticmethod
    cdef inline int cost(StateClass s, const GoldParseC* gold, int label) nogil:
        return RightArc.move_cost(s, gold) + RightArc.label_cost(s, gold, label)

    @staticmethod
    cdef inline int move_cost(StateClass s, const GoldParseC* gold) nogil:
        if arc_is_gold(gold, s.S(0), s.B(0)):
            return 0
        elif s.shifted[s.B(0)]:
            return push_cost(s, gold, s.B(0))
        else:
            return push_cost(s, gold, s.B(0)) + arc_cost(s, gold, s.S(0), s.B(0))

    @staticmethod
    cdef int label_cost(StateClass s, const GoldParseC* gold, int label) nogil:
        return arc_is_gold(gold, s.S(0), s.B(0)) and not label_is_gold(gold, s.S(0), s.B(0), label)


cdef class OrigArcEager(TransitionSystem):
    @classmethod
    def get_labels(cls, gold_parses):
        move_labels = {SHIFT: {'': True}, RIGHT: {'': True},
                       REDUCE: {'': True}, LEFT: {'root': True}}
        for raw_text, sents in gold_parses:
            for (ids, words, tags, heads, labels, iob), ctnts in sents:
                for child, head, label in zip(ids, heads, labels):
                    if label != 'root':
                        if head < child:
                            move_labels[RIGHT][label] = True
                        elif head > child:
                            move_labels[LEFT][label] = True
        return move_labels

    cdef int preprocess_gold(self, GoldParse gold) except -1:
        for i in range(gold.length):
            if gold.heads[i] is None: # Missing values
                gold.c.heads[i] = i
                gold.c.labels[i] = -1
            else:
                gold.c.heads[i] = gold.heads[i]
                gold.c.labels[i] = self.strings[gold.labels[i]]
        for end, brackets in gold.brackets.items():
            for start, label_strs in brackets.items():
                gold.c.brackets[start][end] = 1
                for label_str in label_strs:
                    # Add the encoded label to the set
                    gold.brackets[end][start].add(self.strings[label_str])

    cdef Transition lookup_transition(self, object name) except *:
        if '-' in name:
            move_str, label_str = name.split('-', 1)
            label = self.label_ids[label_str]
        else:
            label = 0
        move = MOVE_NAMES.index(move_str)
        for i in range(self.n_moves):
            if self.c[i].move == move and self.c[i].label == label:
                return self.c[i]

    def move_name(self, int move, int label):
        label_str = self.strings[label]
        if label_str:
            return MOVE_NAMES[move] + '-' + label_str
        else:
            return MOVE_NAMES[move]

    cdef Transition init_transition(self, int clas, int move, int label) except *:
        # TODO: Apparent Cython bug here when we try to use the Transition()
        # constructor with the function pointers
        cdef Transition t
        t.score = 0
        t.clas = clas
        t.move = move
        t.label = label
        if move == SHIFT:
            t.is_valid = Shift.is_valid
            t.do = Shift.transition
            t.get_cost = Shift.cost
        elif move == REDUCE:
            t.is_valid = Reduce.is_valid
            t.do = Reduce.transition
            t.get_cost = Reduce.cost
        elif move == LEFT:
            t.is_valid = LeftArc.is_valid
            t.do = LeftArc.transition
            t.get_cost = LeftArc.cost
        elif move == RIGHT:
            t.is_valid = RightArc.is_valid
            t.do = RightArc.transition
            t.get_cost = RightArc.cost
        else:
            raise Exception(move)
        return t

    cdef int initialize_state(self, StateClass st) except -1:
        # Ensure sent_end is set to 0 throughout
        for i in range(st.length):
            st._sent[i].sent_end = False
        st.push()

    cdef int finalize_state(self, StateClass st) except -1:
        cdef int root_label = self.strings['root']
        for i in range(st.length):
            if st._sent[i].head == 0:
                st._sent[i].dep = root_label

    cdef int set_valid(self, bint* output, StateClass stcls) except -1:
        cdef bint[N_MOVES] is_valid
        is_valid[SHIFT] = Shift.is_valid(stcls, -1)
        is_valid[REDUCE] = Reduce.is_valid(stcls, -1)
        is_valid[LEFT] = LeftArc.is_valid(stcls, -1)
        is_valid[RIGHT] = RightArc.is_valid(stcls, -1)
        cdef int i
        n_valid = 0
        for i in range(self.n_moves):
            output[i] = is_valid[self.c[i].move]
            n_valid += output[i]
        assert n_valid >= 1

    cdef int set_costs(self, int* output, StateClass stcls, GoldParse gold) except -1:
        cdef int i, move, label
        cdef label_cost_func_t[N_MOVES] label_cost_funcs
        cdef move_cost_func_t[N_MOVES] move_cost_funcs
        cdef int[N_MOVES] move_costs
        for i in range(N_MOVES):
            move_costs[i] = -1
        move_cost_funcs[SHIFT] = Shift.move_cost
        move_cost_funcs[REDUCE] = Reduce.move_cost
        move_cost_funcs[LEFT] = LeftArc.move_cost
        move_cost_funcs[RIGHT] = RightArc.move_cost

        label_cost_funcs[SHIFT] = Shift.label_cost
        label_cost_funcs[REDUCE] = Reduce.label_cost
        label_cost_funcs[LEFT] = LeftArc.label_cost
        label_cost_funcs[RIGHT] = RightArc.label_cost

        cdef int* labels = gold.c.labels
        cdef int* heads = gold.c.heads

        n_gold = 0
        for i in range(self.n_moves):
            if self.c[i].is_valid(stcls, self.c[i].label):
                move = self.c[i].move
                label = self.c[i].label
                if move_costs[move] == -1:
                    move_costs[move] = move_cost_funcs[move](stcls, &gold.c)
                output[i] = move_costs[move] + label_cost_funcs[move](stcls, &gold.c, label)
                n_gold += output[i] == 0
            else:
                output[i] = 9000
        assert n_gold >= 1

    cdef Transition best_valid(self, const weight_t* scores, StateClass stcls) except *:
        cdef bint[N_MOVES] is_valid
        is_valid[SHIFT] = Shift.is_valid(stcls, -1)
        is_valid[REDUCE] = Reduce.is_valid(stcls, -1)
        is_valid[LEFT] = LeftArc.is_valid(stcls, -1)
        is_valid[RIGHT] = RightArc.is_valid(stcls, -1)
        cdef Transition best
        cdef weight_t score = MIN_SCORE
        cdef int i
        for i in range(self.n_moves):
            if scores[i] > score and is_valid[self.c[i].move]:
                best = self.c[i]
                score = scores[i]
        assert score > MIN_SCORE, (self.n_moves, stcls.stack_depth(), stcls.buffer_length(), stcls.is_final(), stcls._b_i, stcls.length, stcls.has_head(stcls.S(0)), LeftArc.is_valid(stcls, -1))
        return best