spaCy/spacy/tokens/span.pyx

254 lines
8.9 KiB
Cython
Raw Normal View History

from __future__ import unicode_literals
from collections import defaultdict
import numpy
import numpy.linalg
cimport numpy as np
import math
2015-11-02 18:22:18 +00:00
import six
2015-08-26 17:20:46 +00:00
from ..structs cimport TokenC, LexemeC
2015-07-16 17:55:21 +00:00
from ..typedefs cimport flags_t, attr_t
from ..attrs cimport attr_id_t
from ..parts_of_speech cimport univ_pos_t
from ..util import normalize_slice
from .doc cimport token_by_start, token_by_end
from ..attrs cimport IS_PUNCT, IS_SPACE
from ..lexeme cimport Lexeme
cdef class Span:
2015-07-08 16:53:00 +00:00
"""A slice from a Doc object."""
def __cinit__(self, Doc tokens, int start, int end, int label=0, vector=None,
vector_norm=None):
if not (0 <= start <= end <= len(tokens)):
raise IndexError
2015-09-29 13:03:55 +00:00
self.doc = tokens
self.start = start
2015-11-07 06:05:16 +00:00
self.start_char = self.doc[start].idx if start < self.doc.length else 0
self.end = end
2015-11-07 06:05:16 +00:00
if end >= 1:
self.end_char = self.doc[end - 1].idx + len(self.doc[end - 1])
else:
self.end_char = 0
self.label = label
self._vector = vector
self._vector_norm = vector_norm
def __richcmp__(self, Span other, int op):
# Eq
if op == 0:
return self.start_char < other.start_char
elif op == 1:
return self.start_char <= other.start_char
elif op == 2:
2015-11-06 22:54:14 +00:00
return self.start_char == other.start_char and self.end_char == other.end_char
elif op == 3:
return self.start_char != other.start_char or self.end_char != other.end_char
elif op == 4:
return self.start_char > other.start_char
elif op == 5:
return self.start_char >= other.start_char
def __len__(self):
self._recalculate_indices()
if self.end < self.start:
return 0
return self.end - self.start
def __repr__(self):
2015-11-02 18:22:18 +00:00
if six.PY3:
return self.text
return self.text.encode('utf-8')
2015-10-06 09:45:49 +00:00
def __getitem__(self, object i):
self._recalculate_indices()
2015-10-06 09:45:49 +00:00
if isinstance(i, slice):
start, end = normalize_slice(len(self), i.start, i.stop, i.step)
return Span(self.doc, start + self.start, end + self.start)
2015-07-30 00:30:24 +00:00
else:
if i < 0:
return self.doc[self.end + i]
else:
return self.doc[self.start + i]
def __iter__(self):
self._recalculate_indices()
for i in range(self.start, self.end):
2015-09-29 13:03:55 +00:00
yield self.doc[i]
2015-07-30 00:30:24 +00:00
def merge(self, unicode tag, unicode lemma, unicode ent_type):
self.doc.merge(self.start_char, self.end_char, tag, lemma, ent_type)
2015-07-30 00:30:24 +00:00
def similarity(self, other):
2015-09-22 00:10:01 +00:00
if self.vector_norm == 0.0 or other.vector_norm == 0.0:
return 0.0
return numpy.dot(self.vector, other.vector) / (self.vector_norm * other.vector_norm)
cpdef int _recalculate_indices(self) except -1:
2015-11-07 06:05:16 +00:00
if self.end > self.doc.length \
or self.doc.c[self.start].idx != self.start_char \
or (self.doc.c[self.end-1].idx + self.doc.c[self.end-1].lex.length) != self.end_char:
start = token_by_start(self.doc.c, self.doc.length, self.start_char)
if self.start == -1:
raise IndexError("Error calculating span: Can't find start")
end = token_by_end(self.doc.c, self.doc.length, self.end_char)
if end == -1:
raise IndexError("Error calculating span: Can't find end")
self.start = start
self.end = end + 1
property vector:
def __get__(self):
if self._vector is None:
self._vector = sum(t.vector for t in self) / len(self)
return self._vector
property vector_norm:
def __get__(self):
cdef float value
if self._vector_norm is None:
self._vector_norm = 1e-20
for value in self.vector:
self._vector_norm += value * value
self._vector_norm = math.sqrt(self._vector_norm)
return self._vector_norm
property text:
def __get__(self):
text = self.text_with_ws
if self[-1].whitespace_:
text = text[:-1]
return text
property text_with_ws:
def __get__(self):
return u''.join([t.text_with_ws for t in self])
2015-07-09 15:30:58 +00:00
property root:
"""The word of the span that is highest in the parse tree, i.e. has the
shortest path to the root of the sentence (or is the root itself).
If multiple words are equally high in the tree, the first word is taken.
2015-07-09 15:30:58 +00:00
For example:
>>> toks = nlp(u'I like New York in Autumn.')
Let's name the indices --- easier than writing "toks[4]" etc.
>>> i, like, new, york, in_, autumn, dot = range(len(toks))
The head of 'new' is 'York', and the head of 'York' is 'like'
>>> toks[new].head.orth_
'York'
>>> toks[york].head.orth_
'like'
Create a span for "New York". Its root is "York".
>>> new_york = toks[new:york+1]
>>> new_york.root.orth_
'York'
Here's a more complicated case, raise by Issue #214
>>> toks = nlp(u'to, north and south carolina')
>>> to, north, and_, south, carolina = toks
>>> south.head.text, carolina.head.text
('north', 'to')
Here 'south' is a child of 'north', which is a child of 'carolina'.
Carolina is the root of the span:
2015-07-09 15:30:58 +00:00
>>> south_carolina = toks[-2:]
>>> south_carolina.root.text
'carolina'
2015-05-13 19:45:19 +00:00
"""
def __get__(self):
self._recalculate_indices()
2015-07-09 15:30:58 +00:00
# This should probably be called 'head', and the other one called
# 'gov'. But we went with 'head' elsehwhere, and now we're stuck =/
cdef int i
# First, we scan through the Span, and check whether there's a word
# with head==0, i.e. a sentence root. If so, we can return it. The
# longer the span, the more likely it contains a sentence root, and
# in this case we return in linear time.
for i in range(self.start, self.end):
if self.doc.c[i].head == 0:
2016-01-18 14:40:28 +00:00
return self.doc[i]
# If we don't have a sentence root, we do something that's not so
# algorithmically clever, but I think should be quite fast, especially
# for short spans.
# For each word, we count the path length, and arg min this measure.
# We could use better tree logic to save steps here...But I think this
# should be okay.
cdef int current_best = self.doc.length
cdef int root = -1
for i in range(self.start, self.end):
2016-01-18 15:59:38 +00:00
if self.start <= (i+self.doc.c[i].head) < self.end:
continue
words_to_root = _count_words_to_root(&self.doc.c[i], self.doc.length)
if words_to_root < current_best:
current_best = words_to_root
root = i
if root == -1:
return self.doc[self.start]
else:
return self.doc[root]
2015-05-13 19:45:19 +00:00
property lefts:
"""Tokens that are to the left of the Span, whose head is within the Span."""
def __get__(self):
for token in reversed(self): # Reverse, so we get the tokens in order
for left in token.lefts:
if left.i < self.start:
yield left
2015-07-11 20:15:04 +00:00
property rights:
2015-05-13 19:45:19 +00:00
"""Tokens that are to the right of the Span, whose head is within the Span."""
def __get__(self):
for token in self:
for right in token.rights:
if right.i >= self.end:
yield right
2015-07-09 15:30:58 +00:00
property subtree:
def __get__(self):
for word in self.lefts:
yield from word.subtree
yield from self
for word in self.rights:
yield from word.subtree
property orth_:
def __get__(self):
2015-04-07 02:53:40 +00:00
return ''.join([t.string for t in self]).strip()
property lemma_:
def __get__(self):
2015-03-26 02:45:11 +00:00
return ' '.join([t.lemma_ for t in self]).strip()
property string:
def __get__(self):
return ''.join([t.string for t in self])
property label_:
def __get__(self):
2015-09-29 13:03:55 +00:00
return self.doc.vocab.strings[self.label]
cdef int _count_words_to_root(const TokenC* token, int sent_length) except -1:
cdef int n = 0
while token.head != 0:
token += token.head
n += 1
if n >= sent_length:
raise RuntimeError(
"Array bounds exceeded while searching for root word. This likely "
"means the parse tree is in an invalid state. Please report this "
"issue here: http://github.com/honnibal/spaCy/")
return n