mirror of https://github.com/explosion/spaCy.git
211 lines
6.9 KiB
Cython
211 lines
6.9 KiB
Cython
from __future__ import unicode_literals
|
|
from collections import defaultdict
|
|
import numpy
|
|
import numpy.linalg
|
|
cimport numpy as np
|
|
import math
|
|
import six
|
|
|
|
from ..structs cimport TokenC, LexemeC
|
|
from ..typedefs cimport flags_t, attr_t
|
|
from ..attrs cimport attr_id_t
|
|
from ..parts_of_speech cimport univ_pos_t
|
|
from ..util import normalize_slice
|
|
from .doc cimport token_by_start, token_by_end
|
|
|
|
|
|
cdef class Span:
|
|
"""A slice from a Doc object."""
|
|
def __cinit__(self, Doc tokens, int start, int end, int label=0, vector=None,
|
|
vector_norm=None):
|
|
if not (0 <= start <= end <= len(tokens)):
|
|
raise IndexError
|
|
|
|
self.doc = tokens
|
|
self.start = start
|
|
self.start_char = self.doc[start].idx if start < self.doc.length else 0
|
|
self.end = end
|
|
if end >= 1:
|
|
self.end_char = self.doc[end - 1].idx + len(self.doc[end - 1])
|
|
else:
|
|
self.end_char = 0
|
|
self.label = label
|
|
self._vector = vector
|
|
self._vector_norm = vector_norm
|
|
|
|
def __richcmp__(self, Span other, int op):
|
|
# Eq
|
|
if op == 0:
|
|
return self.start_char < other.start_char
|
|
elif op == 1:
|
|
return self.start_char <= other.start_char
|
|
elif op == 2:
|
|
return self.start_char == other.start_char and self.end_char == other.end_char
|
|
elif op == 3:
|
|
return self.start_char != other.start_char or self.end_char != other.end_char
|
|
elif op == 4:
|
|
return self.start_char > other.start_char
|
|
elif op == 5:
|
|
return self.start_char >= other.start_char
|
|
|
|
def __len__(self):
|
|
self._recalculate_indices()
|
|
if self.end < self.start:
|
|
return 0
|
|
return self.end - self.start
|
|
|
|
def __repr__(self):
|
|
if six.PY3:
|
|
return self.text
|
|
return self.text.encode('utf-8')
|
|
|
|
def __getitem__(self, object i):
|
|
self._recalculate_indices()
|
|
if isinstance(i, slice):
|
|
start, end = normalize_slice(len(self), i.start, i.stop, i.step)
|
|
return Span(self.doc, start + self.start, end + self.start)
|
|
else:
|
|
if i < 0:
|
|
return self.doc[self.end + i]
|
|
else:
|
|
return self.doc[self.start + i]
|
|
|
|
def __iter__(self):
|
|
self._recalculate_indices()
|
|
for i in range(self.start, self.end):
|
|
yield self.doc[i]
|
|
|
|
def merge(self, unicode tag, unicode lemma, unicode ent_type):
|
|
self.doc.merge(self.start_char, self.end_char, tag, lemma, ent_type)
|
|
|
|
def similarity(self, other):
|
|
if self.vector_norm == 0.0 or other.vector_norm == 0.0:
|
|
return 0.0
|
|
return numpy.dot(self.vector, other.vector) / (self.vector_norm * other.vector_norm)
|
|
|
|
cpdef int _recalculate_indices(self) except -1:
|
|
if self.end > self.doc.length \
|
|
or self.doc.c[self.start].idx != self.start_char \
|
|
or (self.doc.c[self.end-1].idx + self.doc.c[self.end-1].lex.length) != self.end_char:
|
|
start = token_by_start(self.doc.c, self.doc.length, self.start_char)
|
|
if self.start == -1:
|
|
raise IndexError("Error calculating span: Can't find start")
|
|
end = token_by_end(self.doc.c, self.doc.length, self.end_char)
|
|
if end == -1:
|
|
raise IndexError("Error calculating span: Can't find end")
|
|
|
|
self.start = start
|
|
self.end = end + 1
|
|
|
|
property vector:
|
|
def __get__(self):
|
|
if self._vector is None:
|
|
self._vector = sum(t.vector for t in self) / len(self)
|
|
return self._vector
|
|
|
|
property vector_norm:
|
|
def __get__(self):
|
|
cdef float value
|
|
if self._vector_norm is None:
|
|
self._vector_norm = 1e-20
|
|
for value in self.vector:
|
|
self._vector_norm += value * value
|
|
self._vector_norm = math.sqrt(self._vector_norm)
|
|
return self._vector_norm
|
|
|
|
property text:
|
|
def __get__(self):
|
|
text = self.text_with_ws
|
|
if self[-1].whitespace_:
|
|
text = text[:-1]
|
|
return text
|
|
|
|
property text_with_ws:
|
|
def __get__(self):
|
|
return u''.join([t.text_with_ws for t in self])
|
|
|
|
property root:
|
|
"""The first ancestor of the first word of the span that has its head
|
|
outside the span.
|
|
|
|
For example:
|
|
|
|
>>> toks = nlp(u'I like New York in Autumn.')
|
|
|
|
Let's name the indices --- easier than writing "toks[4]" etc.
|
|
|
|
>>> i, like, new, york, in_, autumn, dot = range(len(toks))
|
|
|
|
The head of 'new' is 'York', and the head of 'York' is 'like'
|
|
|
|
>>> toks[new].head.orth_
|
|
'York'
|
|
>>> toks[york].head.orth_
|
|
'like'
|
|
|
|
Create a span for "New York". Its root is "York".
|
|
|
|
>>> new_york = toks[new:york+1]
|
|
>>> new_york.root.orth_
|
|
'York'
|
|
|
|
When there are multiple words with external dependencies, we take the first:
|
|
|
|
>>> toks[autumn].head.orth_, toks[dot].head.orth_
|
|
('in', like')
|
|
>>> autumn_dot = toks[autumn:]
|
|
>>> autumn_dot.root.orth_
|
|
'Autumn'
|
|
"""
|
|
def __get__(self):
|
|
self._recalculate_indices()
|
|
# This should probably be called 'head', and the other one called
|
|
# 'gov'. But we went with 'head' elsehwhere, and now we're stuck =/
|
|
cdef const TokenC* start = &self.doc.c[self.start]
|
|
cdef const TokenC* end = &self.doc.c[self.end]
|
|
head = start
|
|
while start <= (head + head.head) < end and head.head != 0:
|
|
head += head.head
|
|
return self.doc[head - self.doc.c]
|
|
|
|
property lefts:
|
|
"""Tokens that are to the left of the Span, whose head is within the Span."""
|
|
def __get__(self):
|
|
for token in reversed(self): # Reverse, so we get the tokens in order
|
|
for left in token.lefts:
|
|
if left.i < self.start:
|
|
yield left
|
|
|
|
property rights:
|
|
"""Tokens that are to the right of the Span, whose head is within the Span."""
|
|
def __get__(self):
|
|
for token in self:
|
|
for right in token.rights:
|
|
if right.i >= self.end:
|
|
yield right
|
|
|
|
property subtree:
|
|
def __get__(self):
|
|
for word in self.lefts:
|
|
yield from word.subtree
|
|
yield from self
|
|
for word in self.rights:
|
|
yield from word.subtree
|
|
|
|
property orth_:
|
|
def __get__(self):
|
|
return ''.join([t.string for t in self]).strip()
|
|
|
|
property lemma_:
|
|
def __get__(self):
|
|
return ' '.join([t.lemma_ for t in self]).strip()
|
|
|
|
property string:
|
|
def __get__(self):
|
|
return ''.join([t.string for t in self])
|
|
|
|
property label_:
|
|
def __get__(self):
|
|
return self.doc.vocab.strings[self.label]
|
|
|