spaCy/spacy/tokens/span.pyx

from __future__ import unicode_literals
from collections import defaultdict
import numpy
import numpy.linalg
cimport numpy as np
import math
import six

from ..structs cimport TokenC, LexemeC
from ..typedefs cimport flags_t, attr_t
from ..attrs cimport attr_id_t
from ..parts_of_speech cimport univ_pos_t
from ..util import normalize_slice


cdef class Span:
    """A slice from a Doc object."""
    def __cinit__(self, Doc tokens, int start, int end, int label=0, vector=None,
                  vector_norm=None):
        if not (0 <= start <= end <= len(tokens)):
            raise IndexError

        self.doc = tokens
        self.start_token = start
        self.start_idx = self.doc[start].idx
        self.end_token = end
        self.end_idx = self.doc[end - 1].idx + len(self.doc[end - 1])
        self.label = label
        self._vector = vector
        self._vector_norm = vector_norm

    def __richcmp__(self, Span other, int op):
        # Eq
        if op == 0:
            return self.start < other.start
        elif op == 1:
            return self.start <= other.start
        elif op == 2:
            return self.start == other.start and self.end == other.end
        elif op == 3:
            return self.start != other.start or self.end != other.end
        elif op == 4:
            return self.start > other.start
        elif op == 5:
            return self.start >= other.start

    def __len__(self):
        if self.end < self.start:
            return 0
        return self.end - self.start

    def __repr__(self):
        if six.PY3:
            return self.text
        return self.text.encode('utf-8')

    def __getitem__(self, object i):
        if isinstance(i, slice):
            start, end = normalize_slice(len(self), i.start, i.stop, i.step)
            start += self.start
            end += self.start
            return Span(self.doc, start, end)

        if i < 0:
            return self.doc[self.end + i]
        else:
            return self.doc[self.start + i]

    def __iter__(self):
        for i in range(self.start, self.end):
            yield self.doc[i]

    def merge(self, unicode tag, unicode lemma, unicode ent_type):
        self.doc.merge(self[0].idx, self[-1].idx + len(self[-1]), tag, lemma, ent_type)

    def similarity(self, other):
        if self.vector_norm == 0.0 or other.vector_norm == 0.0:
            return 0.0
        return numpy.dot(self.vector, other.vector) / (self.vector_norm * other.vector_norm)

    property start:
        """ Get start token index of this span from the Doc."""
        def __get__(self):
            # first is the first token of the span - get it from the doc
            first = None
            if self.start_token < len(self.doc):
                first = self.doc[self.start_token]
            # if we have merged spans in Doc start might have changed.
            # check if token start index is in doc index range and the token
            # index is start_idx (it hasn't changed).
            if first is None or first.idx != self.start_idx:
                # go through tokens in Doc - find index of token equal to start_idx
                new_start = self.doc.token_index_start(self.start_idx)
                if new_start is not None:
                    self.start_token = new_start
                else:
                    raise IndexError('Something went terribly wrong during a merge.'
                                     'No token found with idx %s' % self.start_idx)
            return self.start_token

    property end:
        """ Get end token index of this span from the Doc."""
        def __get__(self):
            # last is the last token of the span - get it from the doc
            last = None
            if self.end_token <= len(self.doc):
                last = self.doc[self.end_token -1]
            # if we have merged spans in Doc end will have changed.
            # check if token end index is in doc index range and the token
            # index is end_idx + len(last_token) (it hasn't changed).
            if last is None or last.idx + len(last) != self.end_idx:
                # go through tokens in Doc - find index of token equal to end_idx
                new_end = self.doc.token_index_end(self.end_idx)
                if new_end is not None:
                    self.end_token = new_end
                else:
                    raise IndexError('Something went terribly wrong during a merge.'
                                     'No token found with idx %s' % self.end_idx)
            return self.end_token

    property vector:
        def __get__(self):
            if self._vector is None:
                self._vector = sum(t.vector for t in self) / len(self)
            return self._vector

    property vector_norm:
        def __get__(self):
            cdef float value
            if self._vector_norm is None:
                self._vector_norm = 1e-20
                for value in self.vector:
                    self._vector_norm += value * value
                self._vector_norm = math.sqrt(self._vector_norm)
            return self._vector_norm

    property text:
        def __get__(self):
            text = self.text_with_ws
            if self[-1].whitespace_:
                text = text[:-1]
            return text

    property text_with_ws:
        def __get__(self):
            return u''.join([t.text_with_ws for t in self])

    property root:
        """The first ancestor of the first word of the span that has its head
        outside the span.
        
        For example:
        
        >>> toks = nlp(u'I like New York in Autumn.')

        Let's name the indices --- easier than writing "toks[4]" etc.

        >>> i, like, new, york, in_, autumn, dot = range(len(toks)) 

        The head of 'new' is 'York', and the head of 'York' is 'like'

        >>> toks[new].head.orth_
        'York'
        >>> toks[york].head.orth_
        'like'

        Create a span for "New York". Its root is "York".

        >>> new_york = toks[new:york+1]
        >>> new_york.root.orth_
        'York'

        When there are multiple words with external dependencies, we take the first:

        >>> toks[autumn].head.orth_, toks[dot].head.orth_
        ('in', like')
        >>> autumn_dot = toks[autumn:]
        >>> autumn_dot.root.orth_
        'Autumn'
        """
        def __get__(self):
            # This should probably be called 'head', and the other one called
            # 'gov'. But we went with 'head' elsehwhere, and now we're stuck =/
            cdef const TokenC* start = &self.doc.c[self.start]
            cdef const TokenC* end = &self.doc.c[self.end]
            head = start
            while start <= (head + head.head) < end and head.head != 0:
                head += head.head
            return self.doc[head - self.doc.c]

    property lefts:
        """Tokens that are to the left of the Span, whose head is within the Span."""
        def __get__(self):
            for token in reversed(self): # Reverse, so we get the tokens in order
                for left in token.lefts:
                    if left.i < self.start:
                        yield left

    property rights:
        """Tokens that are to the right of the Span, whose head is within the Span."""
        def __get__(self):
            for token in self:
                for right in token.rights:
                    if right.i >= self.end:
                        yield right

    property subtree:
        def __get__(self):
            for word in self.lefts:
                yield from word.subtree
            yield from self
            for word in self.rights:
                yield from word.subtree

    property orth_:
        def __get__(self):
            return ''.join([t.string for t in self]).strip()

    property lemma_:
        def __get__(self):
            return ' '.join([t.lemma_ for t in self]).strip()

    property string:
        def __get__(self):
            return ''.join([t.string for t in self])

    property label_:
        def __get__(self):
            return self.doc.vocab.strings[self.label]