2023-09-12 06:49:41 +00:00
|
|
|
|
# cython: profile=False
|
2017-04-15 11:05:15 +00:00
|
|
|
|
cimport numpy as np
|
2019-03-08 10:42:26 +00:00
|
|
|
|
|
2023-06-14 15:48:41 +00:00
|
|
|
|
import copy
|
|
|
|
|
import warnings
|
|
|
|
|
|
2015-09-14 07:49:58 +00:00
|
|
|
|
import numpy
|
2020-02-18 14:38:18 +00:00
|
|
|
|
from thinc.api import get_array_module
|
2015-03-26 02:16:40 +00:00
|
|
|
|
|
2018-12-30 14:17:46 +00:00
|
|
|
|
from ..attrs cimport *
|
2023-06-28 07:43:14 +00:00
|
|
|
|
from ..attrs cimport ORTH, attr_id_t
|
2016-01-18 17:14:09 +00:00
|
|
|
|
from ..lexeme cimport Lexeme
|
2023-07-19 10:03:31 +00:00
|
|
|
|
from ..structs cimport TokenC
|
2019-07-23 16:28:55 +00:00
|
|
|
|
from ..symbols cimport dep
|
2023-07-19 10:03:31 +00:00
|
|
|
|
from ..typedefs cimport attr_t, hash_t
|
|
|
|
|
from .doc cimport _get_lca_matrix, get_token_attr
|
2023-06-28 07:43:14 +00:00
|
|
|
|
from .token cimport Token
|
2019-03-08 10:42:26 +00:00
|
|
|
|
|
2020-10-04 09:16:31 +00:00
|
|
|
|
from ..errors import Errors, Warnings
|
2023-06-14 15:48:41 +00:00
|
|
|
|
from ..util import normalize_slice
|
2018-04-03 16:30:17 +00:00
|
|
|
|
from .underscore import Underscore, get_ext_args
|
2015-07-13 18:20:58 +00:00
|
|
|
|
|
2015-03-26 02:16:40 +00:00
|
|
|
|
|
|
|
|
|
cdef class Span:
|
2019-03-08 10:42:26 +00:00
|
|
|
|
"""A slice from a Doc object.
|
|
|
|
|
|
2021-01-30 09:09:38 +00:00
|
|
|
|
DOCS: https://spacy.io/api/span
|
2019-03-08 10:42:26 +00:00
|
|
|
|
"""
|
2017-10-07 16:56:01 +00:00
|
|
|
|
@classmethod
|
2018-04-03 16:30:17 +00:00
|
|
|
|
def set_extension(cls, name, **kwargs):
|
2019-03-08 10:42:26 +00:00
|
|
|
|
"""Define a custom attribute which becomes available as `Span._`.
|
|
|
|
|
|
2020-05-24 15:20:58 +00:00
|
|
|
|
name (str): Name of the attribute to set.
|
2019-03-08 10:42:26 +00:00
|
|
|
|
default: Optional default value of the attribute.
|
|
|
|
|
getter (callable): Optional getter function.
|
|
|
|
|
setter (callable): Optional setter function.
|
|
|
|
|
method (callable): Optional method for method extension.
|
|
|
|
|
force (bool): Force overwriting existing attribute.
|
|
|
|
|
|
2021-01-30 09:09:38 +00:00
|
|
|
|
DOCS: https://spacy.io/api/span#set_extension
|
|
|
|
|
USAGE: https://spacy.io/usage/processing-pipelines#custom-components-attributes
|
2019-03-08 10:42:26 +00:00
|
|
|
|
"""
|
|
|
|
|
if cls.has_extension(name) and not kwargs.get("force", False):
|
|
|
|
|
raise ValueError(Errors.E090.format(name=name, obj="Span"))
|
2018-04-03 16:30:17 +00:00
|
|
|
|
Underscore.span_extensions[name] = get_ext_args(**kwargs)
|
2017-10-07 16:56:01 +00:00
|
|
|
|
|
|
|
|
|
@classmethod
|
|
|
|
|
def get_extension(cls, name):
|
2019-03-08 10:42:26 +00:00
|
|
|
|
"""Look up a previously registered extension by name.
|
|
|
|
|
|
2020-05-24 15:20:58 +00:00
|
|
|
|
name (str): Name of the extension.
|
2019-03-08 10:42:26 +00:00
|
|
|
|
RETURNS (tuple): A `(default, method, getter, setter)` tuple.
|
|
|
|
|
|
2021-01-30 09:09:38 +00:00
|
|
|
|
DOCS: https://spacy.io/api/span#get_extension
|
2019-03-08 10:42:26 +00:00
|
|
|
|
"""
|
2017-10-07 16:56:01 +00:00
|
|
|
|
return Underscore.span_extensions.get(name)
|
|
|
|
|
|
|
|
|
|
@classmethod
|
|
|
|
|
def has_extension(cls, name):
|
2019-03-08 10:42:26 +00:00
|
|
|
|
"""Check whether an extension has been registered.
|
|
|
|
|
|
2020-05-24 15:20:58 +00:00
|
|
|
|
name (str): Name of the extension.
|
2019-03-08 10:42:26 +00:00
|
|
|
|
RETURNS (bool): Whether the extension has been registered.
|
|
|
|
|
|
2021-01-30 09:09:38 +00:00
|
|
|
|
DOCS: https://spacy.io/api/span#has_extension
|
2019-03-08 10:42:26 +00:00
|
|
|
|
"""
|
2017-10-07 16:56:01 +00:00
|
|
|
|
return name in Underscore.span_extensions
|
|
|
|
|
|
2018-04-28 21:33:09 +00:00
|
|
|
|
@classmethod
|
|
|
|
|
def remove_extension(cls, name):
|
2019-03-08 10:42:26 +00:00
|
|
|
|
"""Remove a previously registered extension.
|
|
|
|
|
|
2020-05-24 15:20:58 +00:00
|
|
|
|
name (str): Name of the extension.
|
2019-03-08 10:42:26 +00:00
|
|
|
|
RETURNS (tuple): A `(default, method, getter, setter)` tuple of the
|
|
|
|
|
removed extension.
|
|
|
|
|
|
2021-01-30 09:09:38 +00:00
|
|
|
|
DOCS: https://spacy.io/api/span#remove_extension
|
2019-03-08 10:42:26 +00:00
|
|
|
|
"""
|
2018-04-28 21:33:09 +00:00
|
|
|
|
if not cls.has_extension(name):
|
|
|
|
|
raise ValueError(Errors.E046.format(name=name))
|
|
|
|
|
return Underscore.span_extensions.pop(name)
|
|
|
|
|
|
2019-03-08 10:42:26 +00:00
|
|
|
|
def __cinit__(self, Doc doc, int start, int end, label=0, vector=None,
|
2022-06-02 11:12:53 +00:00
|
|
|
|
vector_norm=None, kb_id=0, span_id=0):
|
2017-05-18 20:17:24 +00:00
|
|
|
|
"""Create a `Span` object from the slice `doc[start : end]`.
|
|
|
|
|
|
|
|
|
|
doc (Doc): The parent document.
|
|
|
|
|
start (int): The index of the first token of the span.
|
|
|
|
|
end (int): The index of the first token after the span.
|
2022-06-02 11:12:53 +00:00
|
|
|
|
label (Union[int, str]): A label to attach to the Span, e.g. for named
|
|
|
|
|
entities.
|
2017-10-27 13:41:45 +00:00
|
|
|
|
vector (ndarray[ndim=1, dtype='float32']): A meaning representation
|
|
|
|
|
of the span.
|
2021-08-25 14:06:22 +00:00
|
|
|
|
vector_norm (float): The L2 norm of the span's vector representation.
|
2022-06-02 11:12:53 +00:00
|
|
|
|
kb_id (Union[int, str]): An identifier from a Knowledge Base to capture
|
|
|
|
|
the meaning of a named entity.
|
|
|
|
|
span_id (Union[int, str]): An identifier to associate with the span.
|
2019-03-08 10:42:26 +00:00
|
|
|
|
|
2021-01-30 09:09:38 +00:00
|
|
|
|
DOCS: https://spacy.io/api/span#init
|
2017-04-15 11:05:15 +00:00
|
|
|
|
"""
|
2016-11-01 12:27:44 +00:00
|
|
|
|
if not (0 <= start <= end <= len(doc)):
|
2018-04-03 13:50:31 +00:00
|
|
|
|
raise IndexError(Errors.E035.format(start=start, end=end, length=len(doc)))
|
2016-11-01 11:25:36 +00:00
|
|
|
|
self.doc = doc
|
2019-12-22 00:53:56 +00:00
|
|
|
|
if isinstance(label, str):
|
2018-12-08 12:08:41 +00:00
|
|
|
|
label = doc.vocab.strings.add(label)
|
2019-12-22 00:53:56 +00:00
|
|
|
|
if isinstance(kb_id, str):
|
2019-03-22 11:05:35 +00:00
|
|
|
|
kb_id = doc.vocab.strings.add(kb_id)
|
2022-06-02 11:12:53 +00:00
|
|
|
|
if isinstance(span_id, str):
|
|
|
|
|
span_id = doc.vocab.strings.add(span_id)
|
2018-04-03 13:50:31 +00:00
|
|
|
|
if label not in doc.vocab.strings:
|
|
|
|
|
raise ValueError(Errors.E084.format(label=label))
|
2021-01-14 06:30:41 +00:00
|
|
|
|
|
2021-08-02 17:07:19 +00:00
|
|
|
|
start_char = doc[start].idx if start < doc.length else len(doc.text)
|
|
|
|
|
if start == end:
|
|
|
|
|
end_char = start_char
|
|
|
|
|
else:
|
|
|
|
|
end_char = doc[end - 1].idx + len(doc[end - 1])
|
2021-01-14 06:30:41 +00:00
|
|
|
|
self.c = SpanC(
|
|
|
|
|
label=label,
|
|
|
|
|
kb_id=kb_id,
|
2022-06-02 11:12:53 +00:00
|
|
|
|
id=span_id,
|
2021-01-14 06:30:41 +00:00
|
|
|
|
start=start,
|
|
|
|
|
end=end,
|
2021-08-02 17:07:19 +00:00
|
|
|
|
start_char=start_char,
|
|
|
|
|
end_char=end_char,
|
2021-01-14 06:30:41 +00:00
|
|
|
|
)
|
2015-09-21 06:50:40 +00:00
|
|
|
|
self._vector = vector
|
|
|
|
|
self._vector_norm = vector_norm
|
2015-03-26 02:16:40 +00:00
|
|
|
|
|
2023-10-12 09:53:33 +00:00
|
|
|
|
def __richcmp__(self, object other, int op):
|
2018-01-15 14:51:25 +00:00
|
|
|
|
if other is None:
|
|
|
|
|
if op == 0 or op == 1 or op == 2:
|
|
|
|
|
return False
|
|
|
|
|
else:
|
|
|
|
|
return True
|
2023-10-12 09:53:33 +00:00
|
|
|
|
if not isinstance(other, Span):
|
|
|
|
|
return False
|
|
|
|
|
cdef Span other_span = other
|
2022-06-02 11:12:53 +00:00
|
|
|
|
self_tuple = (self.c.start_char, self.c.end_char, self.c.label, self.c.kb_id, self.id, self.doc)
|
2023-10-12 09:53:33 +00:00
|
|
|
|
other_tuple = (other_span.c.start_char, other_span.c.end_char, other_span.c.label, other_span.c.kb_id, other_span.id, other_span.doc)
|
2020-02-16 16:20:36 +00:00
|
|
|
|
# <
|
2015-03-26 02:16:40 +00:00
|
|
|
|
if op == 0:
|
2022-01-17 10:17:49 +00:00
|
|
|
|
return self_tuple < other_tuple
|
2020-02-16 16:20:36 +00:00
|
|
|
|
# <=
|
2015-03-26 02:16:40 +00:00
|
|
|
|
elif op == 1:
|
2022-01-17 10:17:49 +00:00
|
|
|
|
return self_tuple <= other_tuple
|
2020-02-16 16:20:36 +00:00
|
|
|
|
# ==
|
2015-03-26 02:16:40 +00:00
|
|
|
|
elif op == 2:
|
2022-01-17 10:17:49 +00:00
|
|
|
|
return self_tuple == other_tuple
|
2020-02-16 16:20:36 +00:00
|
|
|
|
# !=
|
2015-03-26 02:16:40 +00:00
|
|
|
|
elif op == 3:
|
2022-01-17 10:17:49 +00:00
|
|
|
|
return self_tuple != other_tuple
|
2020-02-16 16:20:36 +00:00
|
|
|
|
# >
|
2015-03-26 02:16:40 +00:00
|
|
|
|
elif op == 4:
|
2022-01-17 10:17:49 +00:00
|
|
|
|
return self_tuple > other_tuple
|
2020-02-16 16:20:36 +00:00
|
|
|
|
# >=
|
2015-03-26 02:16:40 +00:00
|
|
|
|
elif op == 5:
|
2022-01-17 10:17:49 +00:00
|
|
|
|
return self_tuple >= other_tuple
|
2015-03-26 02:16:40 +00:00
|
|
|
|
|
2017-04-26 17:01:05 +00:00
|
|
|
|
def __hash__(self):
|
2022-06-02 11:12:53 +00:00
|
|
|
|
return hash((self.doc, self.c.start_char, self.c.end_char, self.c.label, self.c.kb_id, self.c.id))
|
2017-04-26 17:01:05 +00:00
|
|
|
|
|
2015-03-26 02:16:40 +00:00
|
|
|
|
def __len__(self):
|
2017-05-18 22:31:31 +00:00
|
|
|
|
"""Get the number of tokens in the span.
|
|
|
|
|
|
|
|
|
|
RETURNS (int): The number of tokens in the span.
|
2019-03-08 10:42:26 +00:00
|
|
|
|
|
2021-01-30 09:09:38 +00:00
|
|
|
|
DOCS: https://spacy.io/api/span#len
|
2017-05-18 22:31:31 +00:00
|
|
|
|
"""
|
2021-01-14 06:30:41 +00:00
|
|
|
|
if self.c.end < self.c.start:
|
2015-03-26 02:16:40 +00:00
|
|
|
|
return 0
|
2021-01-14 06:30:41 +00:00
|
|
|
|
return self.c.end - self.c.start
|
2015-03-26 02:16:40 +00:00
|
|
|
|
|
2015-10-21 11:11:46 +00:00
|
|
|
|
def __repr__(self):
|
2019-12-22 00:53:56 +00:00
|
|
|
|
return self.text
|
2015-10-21 11:11:46 +00:00
|
|
|
|
|
2015-10-06 09:45:49 +00:00
|
|
|
|
def __getitem__(self, object i):
|
2017-05-18 22:31:31 +00:00
|
|
|
|
"""Get a `Token` or a `Span` object
|
|
|
|
|
|
|
|
|
|
i (int or tuple): The index of the token within the span, or slice of
|
|
|
|
|
the span to get.
|
|
|
|
|
RETURNS (Token or Span): The token at `span[i]`.
|
|
|
|
|
|
2021-01-30 09:09:38 +00:00
|
|
|
|
DOCS: https://spacy.io/api/span#getitem
|
2017-05-18 22:31:31 +00:00
|
|
|
|
"""
|
2015-10-06 09:45:49 +00:00
|
|
|
|
if isinstance(i, slice):
|
2015-10-07 08:25:35 +00:00
|
|
|
|
start, end = normalize_slice(len(self), i.start, i.stop, i.step)
|
2015-11-06 21:55:34 +00:00
|
|
|
|
return Span(self.doc, start + self.start, end + self.start)
|
2015-07-30 00:30:24 +00:00
|
|
|
|
else:
|
2015-11-06 21:55:34 +00:00
|
|
|
|
if i < 0:
|
2021-01-14 06:30:41 +00:00
|
|
|
|
token_i = self.c.end + i
|
2015-11-06 21:55:34 +00:00
|
|
|
|
else:
|
2021-01-14 06:30:41 +00:00
|
|
|
|
token_i = self.c.start + i
|
|
|
|
|
if self.c.start <= token_i < self.c.end:
|
2020-08-04 11:35:25 +00:00
|
|
|
|
return self.doc[token_i]
|
|
|
|
|
else:
|
2020-08-04 15:02:39 +00:00
|
|
|
|
raise IndexError(Errors.E1002)
|
2015-03-26 02:16:40 +00:00
|
|
|
|
|
|
|
|
|
def __iter__(self):
|
2017-05-18 22:31:31 +00:00
|
|
|
|
"""Iterate over `Token` objects.
|
|
|
|
|
|
|
|
|
|
YIELDS (Token): A `Token` object.
|
2019-03-08 10:42:26 +00:00
|
|
|
|
|
2021-01-30 09:09:38 +00:00
|
|
|
|
DOCS: https://spacy.io/api/span#iter
|
2017-05-18 22:31:31 +00:00
|
|
|
|
"""
|
2021-01-14 06:30:41 +00:00
|
|
|
|
for i in range(self.c.start, self.c.end):
|
2015-09-29 13:03:55 +00:00
|
|
|
|
yield self.doc[i]
|
2015-03-26 02:16:40 +00:00
|
|
|
|
|
2019-02-13 12:22:05 +00:00
|
|
|
|
def __reduce__(self):
|
|
|
|
|
raise NotImplementedError(Errors.E112)
|
|
|
|
|
|
2017-10-07 16:56:01 +00:00
|
|
|
|
@property
|
|
|
|
|
def _(self):
|
2019-03-08 10:42:26 +00:00
|
|
|
|
"""Custom extension attributes registered via `set_extension`."""
|
2017-10-07 16:56:01 +00:00
|
|
|
|
return Underscore(Underscore.span_extensions, self,
|
2021-01-14 06:30:41 +00:00
|
|
|
|
start=self.c.start_char, end=self.c.end_char)
|
2017-10-23 08:38:06 +00:00
|
|
|
|
|
2021-08-10 13:13:53 +00:00
|
|
|
|
def as_doc(self, *, bint copy_user_data=False, array_head=None, array=None):
|
2019-03-08 10:42:26 +00:00
|
|
|
|
"""Create a `Doc` object with a copy of the `Span`'s data.
|
2017-10-08 21:50:20 +00:00
|
|
|
|
|
2019-09-12 15:08:14 +00:00
|
|
|
|
copy_user_data (bool): Whether or not to copy the original doc's user data.
|
2021-08-10 13:13:53 +00:00
|
|
|
|
array_head (tuple): `Doc` array attrs, can be passed in to speed up computation.
|
|
|
|
|
array (ndarray): `Doc` as array, can be passed in to speed up computation.
|
2018-12-30 14:17:46 +00:00
|
|
|
|
RETURNS (Doc): The `Doc` copy of the span.
|
2019-03-08 10:42:26 +00:00
|
|
|
|
|
2021-01-30 09:09:38 +00:00
|
|
|
|
DOCS: https://spacy.io/api/span#as_doc
|
2017-10-27 15:07:26 +00:00
|
|
|
|
"""
|
2019-03-08 10:42:26 +00:00
|
|
|
|
words = [t.text for t in self]
|
|
|
|
|
spaces = [bool(t.whitespace_) for t in self]
|
|
|
|
|
cdef Doc doc = Doc(self.doc.vocab, words=words, spaces=spaces)
|
2021-08-10 13:13:53 +00:00
|
|
|
|
if array_head is None:
|
|
|
|
|
array_head = self.doc._get_array_attrs()
|
|
|
|
|
if array is None:
|
|
|
|
|
array = self.doc.to_array(array_head)
|
2019-07-23 16:28:55 +00:00
|
|
|
|
array = array[self.start : self.end]
|
|
|
|
|
self._fix_dep_copy(array_head, array)
|
2021-05-11 15:10:16 +00:00
|
|
|
|
# Fix initial IOB so the entities are valid for doc.ents below.
|
|
|
|
|
if len(array) > 0 and ENT_IOB in array_head:
|
|
|
|
|
ent_iob_col = array_head.index(ENT_IOB)
|
|
|
|
|
if array[0][ent_iob_col] == 1:
|
|
|
|
|
array[0][ent_iob_col] = 3
|
2019-07-23 16:28:55 +00:00
|
|
|
|
doc.from_array(array_head, array)
|
2021-05-11 15:10:16 +00:00
|
|
|
|
# Set partial entities at the beginning or end of the span to have
|
|
|
|
|
# missing entity annotation. Note: the initial partial entity could be
|
|
|
|
|
# detected from the IOB annotation but the final partial entity can't,
|
|
|
|
|
# so detect and remove both in the same way by checking self.ents.
|
|
|
|
|
span_ents = {(ent.start, ent.end) for ent in self.ents}
|
|
|
|
|
doc_ents = doc.ents
|
|
|
|
|
if len(doc_ents) > 0:
|
|
|
|
|
# Remove initial partial ent
|
|
|
|
|
if (doc_ents[0].start + self.start, doc_ents[0].end + self.start) not in span_ents:
|
|
|
|
|
doc.set_ents([], missing=[doc_ents[0]], default="unmodified")
|
|
|
|
|
# Remove final partial ent
|
|
|
|
|
if (doc_ents[-1].start + self.start, doc_ents[-1].end + self.start) not in span_ents:
|
|
|
|
|
doc.set_ents([], missing=[doc_ents[-1]], default="unmodified")
|
2017-10-08 21:50:20 +00:00
|
|
|
|
doc.noun_chunks_iterator = self.doc.noun_chunks_iterator
|
|
|
|
|
doc.user_hooks = self.doc.user_hooks
|
|
|
|
|
doc.user_span_hooks = self.doc.user_span_hooks
|
|
|
|
|
doc.user_token_hooks = self.doc.user_token_hooks
|
|
|
|
|
doc.vector = self.vector
|
|
|
|
|
doc.vector_norm = self.vector_norm
|
2018-12-30 14:17:46 +00:00
|
|
|
|
doc.tensor = self.doc.tensor[self.start : self.end]
|
2017-10-08 21:50:20 +00:00
|
|
|
|
for key, value in self.doc.cats.items():
|
2019-03-08 10:42:26 +00:00
|
|
|
|
if hasattr(key, "__len__") and len(key) == 3:
|
2017-10-08 21:50:20 +00:00
|
|
|
|
cat_start, cat_end, cat_label = key
|
|
|
|
|
if cat_start == self.start_char and cat_end == self.end_char:
|
|
|
|
|
doc.cats[cat_label] = value
|
2019-09-12 15:08:14 +00:00
|
|
|
|
if copy_user_data:
|
2021-03-30 07:49:12 +00:00
|
|
|
|
user_data = {}
|
|
|
|
|
char_offset = self.start_char
|
|
|
|
|
for key, value in self.doc.user_data.items():
|
|
|
|
|
if isinstance(key, tuple) and len(key) == 4 and key[0] == "._.":
|
|
|
|
|
data_type, name, start, end = key
|
|
|
|
|
if start is not None or end is not None:
|
|
|
|
|
start -= char_offset
|
|
|
|
|
if end is not None:
|
|
|
|
|
end -= char_offset
|
|
|
|
|
user_data[(data_type, name, start, end)] = copy.copy(value)
|
|
|
|
|
else:
|
|
|
|
|
user_data[key] = copy.copy(value)
|
|
|
|
|
doc.user_data = user_data
|
2017-10-08 21:50:20 +00:00
|
|
|
|
return doc
|
2017-10-07 16:56:01 +00:00
|
|
|
|
|
2019-07-23 16:28:55 +00:00
|
|
|
|
def _fix_dep_copy(self, attrs, array):
|
|
|
|
|
""" Rewire dependency links to make sure their heads fall into the span
|
|
|
|
|
while still keeping the correct number of sentences. """
|
|
|
|
|
cdef int length = len(array)
|
|
|
|
|
cdef attr_t value
|
|
|
|
|
cdef int i, head_col, ancestor_i
|
|
|
|
|
old_to_new_root = dict()
|
|
|
|
|
if HEAD in attrs:
|
|
|
|
|
head_col = attrs.index(HEAD)
|
|
|
|
|
for i in range(length):
|
|
|
|
|
# if the HEAD refers to a token outside this span, find a more appropriate ancestor
|
|
|
|
|
token = self[i]
|
2021-01-14 06:30:41 +00:00
|
|
|
|
ancestor_i = token.head.i - self.c.start # span offset
|
2019-07-23 16:28:55 +00:00
|
|
|
|
if ancestor_i not in range(length):
|
|
|
|
|
if DEP in attrs:
|
|
|
|
|
array[i, attrs.index(DEP)] = dep
|
|
|
|
|
|
|
|
|
|
# try finding an ancestor within this span
|
|
|
|
|
ancestors = token.ancestors
|
|
|
|
|
for ancestor in ancestors:
|
2021-01-14 06:30:41 +00:00
|
|
|
|
ancestor_i = ancestor.i - self.c.start
|
2019-07-23 16:28:55 +00:00
|
|
|
|
if ancestor_i in range(length):
|
2022-12-12 07:45:35 +00:00
|
|
|
|
array[i, head_col] = numpy.int32(ancestor_i - i).astype(numpy.uint64)
|
2019-07-23 16:28:55 +00:00
|
|
|
|
|
|
|
|
|
# if there is no appropriate ancestor, define a new artificial root
|
|
|
|
|
value = array[i, head_col]
|
|
|
|
|
if (i+value) not in range(length):
|
|
|
|
|
new_root = old_to_new_root.get(ancestor_i, None)
|
|
|
|
|
if new_root is not None:
|
|
|
|
|
# take the same artificial root as a previous token from the same sentence
|
2022-12-12 07:45:35 +00:00
|
|
|
|
array[i, head_col] = numpy.int32(new_root - i).astype(numpy.uint64)
|
2019-07-23 16:28:55 +00:00
|
|
|
|
else:
|
|
|
|
|
# set this token as the new artificial root
|
|
|
|
|
array[i, head_col] = 0
|
|
|
|
|
old_to_new_root[ancestor_i] = i
|
|
|
|
|
|
|
|
|
|
return array
|
|
|
|
|
|
2018-12-29 17:02:26 +00:00
|
|
|
|
def get_lca_matrix(self):
|
|
|
|
|
"""Calculates a matrix of Lowest Common Ancestors (LCA) for a given
|
|
|
|
|
`Span`, where LCA[i, j] is the index of the lowest common ancestor among
|
|
|
|
|
the tokens span[i] and span[j]. If they have no common ancestor within
|
|
|
|
|
the span, LCA[i, j] will be -1.
|
|
|
|
|
|
|
|
|
|
RETURNS (np.array[ndim=2, dtype=numpy.int32]): LCA matrix with shape
|
|
|
|
|
(n, n), where n = len(self).
|
2019-03-08 10:42:26 +00:00
|
|
|
|
|
2021-01-30 09:09:38 +00:00
|
|
|
|
DOCS: https://spacy.io/api/span#get_lca_matrix
|
2018-12-29 17:02:26 +00:00
|
|
|
|
"""
|
2021-01-14 06:30:41 +00:00
|
|
|
|
return numpy.asarray(_get_lca_matrix(self.doc, self.c.start, self.c.end))
|
2018-12-29 17:02:26 +00:00
|
|
|
|
|
2015-09-14 07:49:58 +00:00
|
|
|
|
def similarity(self, other):
|
2017-05-19 16:47:46 +00:00
|
|
|
|
"""Make a semantic similarity estimate. The default estimate is cosine
|
2016-11-01 11:25:36 +00:00
|
|
|
|
similarity using an average of word vectors.
|
|
|
|
|
|
2017-05-18 20:17:24 +00:00
|
|
|
|
other (object): The object to compare with. By default, accepts `Doc`,
|
|
|
|
|
`Span`, `Token` and `Lexeme` objects.
|
|
|
|
|
RETURNS (float): A scalar similarity score. Higher is more similar.
|
2019-03-08 10:42:26 +00:00
|
|
|
|
|
2021-01-30 09:09:38 +00:00
|
|
|
|
DOCS: https://spacy.io/api/span#similarity
|
2017-04-15 11:05:15 +00:00
|
|
|
|
"""
|
2019-03-08 10:42:26 +00:00
|
|
|
|
if "similarity" in self.doc.user_span_hooks:
|
2019-07-27 13:33:27 +00:00
|
|
|
|
return self.doc.user_span_hooks["similarity"](self, other)
|
2023-06-28 07:43:14 +00:00
|
|
|
|
attr = getattr(self.doc.vocab.vectors, "attr", ORTH)
|
|
|
|
|
cdef Token this_token
|
|
|
|
|
cdef Token other_token
|
|
|
|
|
cdef Lexeme other_lex
|
|
|
|
|
if len(self) == 1 and isinstance(other, Token):
|
|
|
|
|
this_token = self[0]
|
|
|
|
|
other_token = other
|
|
|
|
|
if Token.get_struct_attr(this_token.c, attr) == Token.get_struct_attr(other_token.c, attr):
|
|
|
|
|
return 1.0
|
|
|
|
|
elif len(self) == 1 and isinstance(other, Lexeme):
|
|
|
|
|
this_token = self[0]
|
|
|
|
|
other_lex = other
|
|
|
|
|
if Token.get_struct_attr(this_token.c, attr) == Lexeme.get_struct_attr(other_lex.c, attr):
|
2018-01-15 15:29:48 +00:00
|
|
|
|
return 1.0
|
2020-04-27 14:51:27 +00:00
|
|
|
|
elif isinstance(other, (Doc, Span)) and len(self) == len(other):
|
|
|
|
|
similar = True
|
2018-01-15 15:29:48 +00:00
|
|
|
|
for i in range(len(self)):
|
2023-06-28 07:43:14 +00:00
|
|
|
|
this_token = self[i]
|
|
|
|
|
other_token = other[i]
|
|
|
|
|
if Token.get_struct_attr(this_token.c, attr) != Token.get_struct_attr(other_token.c, attr):
|
2020-04-27 14:51:27 +00:00
|
|
|
|
similar = False
|
2018-01-15 15:29:48 +00:00
|
|
|
|
break
|
2020-04-27 14:51:27 +00:00
|
|
|
|
if similar:
|
2018-01-15 15:29:48 +00:00
|
|
|
|
return 1.0
|
2018-05-20 23:22:38 +00:00
|
|
|
|
if self.vocab.vectors.n_keys == 0:
|
2020-04-28 11:37:37 +00:00
|
|
|
|
warnings.warn(Warnings.W007.format(obj="Span"))
|
2015-09-22 00:10:01 +00:00
|
|
|
|
if self.vector_norm == 0.0 or other.vector_norm == 0.0:
|
2022-06-28 17:50:47 +00:00
|
|
|
|
if not self.has_vector or not other.has_vector:
|
|
|
|
|
warnings.warn(Warnings.W008.format(obj="Span"))
|
2015-09-22 00:10:01 +00:00
|
|
|
|
return 0.0
|
2019-03-06 22:58:38 +00:00
|
|
|
|
vector = self.vector
|
|
|
|
|
xp = get_array_module(vector)
|
2022-01-20 10:40:46 +00:00
|
|
|
|
result = xp.dot(vector, other.vector) / (self.vector_norm * other.vector_norm)
|
|
|
|
|
# ensure we get a scalar back (numpy does this automatically but cupy doesn't)
|
|
|
|
|
return result.item()
|
2023-01-27 10:43:40 +00:00
|
|
|
|
|
2017-08-19 10:20:45 +00:00
|
|
|
|
cpdef np.ndarray to_array(self, object py_attr_ids):
|
|
|
|
|
"""Given a list of M attribute IDs, export the tokens to a numpy
|
|
|
|
|
`ndarray` of shape `(N, M)`, where `N` is the length of the document.
|
|
|
|
|
The values will be 32-bit integers.
|
|
|
|
|
|
|
|
|
|
attr_ids (list[int]): A list of attribute ID ints.
|
|
|
|
|
RETURNS (numpy.ndarray[long, ndim=2]): A feature matrix, with one row
|
|
|
|
|
per word, and one column per attribute indicated in the input
|
|
|
|
|
`attr_ids`.
|
|
|
|
|
"""
|
|
|
|
|
cdef int i, j
|
|
|
|
|
cdef attr_id_t feature
|
|
|
|
|
cdef np.ndarray[attr_t, ndim=2] output
|
2019-03-08 10:42:26 +00:00
|
|
|
|
# Make an array from the attributes - otherwise our inner loop is Python
|
|
|
|
|
# dict iteration
|
2017-08-19 10:20:45 +00:00
|
|
|
|
cdef np.ndarray[attr_t, ndim=1] attr_ids = numpy.asarray(py_attr_ids, dtype=numpy.uint64)
|
2017-08-19 14:24:28 +00:00
|
|
|
|
cdef int length = self.end - self.start
|
|
|
|
|
output = numpy.ndarray(shape=(length, len(attr_ids)), dtype=numpy.uint64)
|
2017-08-19 10:20:45 +00:00
|
|
|
|
for i in range(self.start, self.end):
|
|
|
|
|
for j, feature in enumerate(attr_ids):
|
2017-08-19 14:24:28 +00:00
|
|
|
|
output[i-self.start, j] = get_token_attr(&self.doc.c[i], feature)
|
2017-08-19 10:20:45 +00:00
|
|
|
|
return output
|
|
|
|
|
|
2019-03-11 14:59:09 +00:00
|
|
|
|
@property
|
|
|
|
|
def vocab(self):
|
2018-01-14 14:06:30 +00:00
|
|
|
|
"""RETURNS (Vocab): The Span's Doc's vocab."""
|
2019-03-11 14:59:09 +00:00
|
|
|
|
return self.doc.vocab
|
2018-01-14 14:06:30 +00:00
|
|
|
|
|
2019-03-11 14:59:09 +00:00
|
|
|
|
@property
|
|
|
|
|
def sent(self):
|
2021-02-19 12:02:38 +00:00
|
|
|
|
"""Obtain the sentence that contains this span. If the given span
|
|
|
|
|
crosses sentence boundaries, return only the first sentence
|
|
|
|
|
to which it belongs.
|
|
|
|
|
|
|
|
|
|
RETURNS (Span): The sentence span that the span is a part of.
|
|
|
|
|
"""
|
2019-03-11 14:59:09 +00:00
|
|
|
|
if "sent" in self.doc.user_span_hooks:
|
|
|
|
|
return self.doc.user_span_hooks["sent"](self)
|
2021-12-06 08:58:01 +00:00
|
|
|
|
elif "sents" in self.doc.user_hooks:
|
|
|
|
|
for sentence in self.doc.user_hooks["sents"](self.doc):
|
|
|
|
|
if sentence.start <= self.start < sentence.end:
|
|
|
|
|
return sentence
|
2020-05-14 16:22:51 +00:00
|
|
|
|
# Use `sent_start` token attribute to find sentence boundaries
|
2019-03-11 14:59:09 +00:00
|
|
|
|
cdef int n = 0
|
2020-09-16 22:14:01 +00:00
|
|
|
|
if self.doc.has_annotation("SENT_START"):
|
2019-03-11 14:59:09 +00:00
|
|
|
|
# Find start of the sentence
|
|
|
|
|
start = self.start
|
|
|
|
|
while self.doc.c[start].sent_start != 1 and start > 0:
|
|
|
|
|
start += -1
|
2021-02-19 12:02:38 +00:00
|
|
|
|
# Find end of the sentence - can be within the entity
|
|
|
|
|
end = self.start + 1
|
2019-03-11 14:59:09 +00:00
|
|
|
|
while end < self.doc.length and self.doc.c[end].sent_start != 1:
|
|
|
|
|
end += 1
|
|
|
|
|
n += 1
|
|
|
|
|
if n >= self.doc.length:
|
|
|
|
|
break
|
|
|
|
|
return self.doc[start:end]
|
2020-10-01 12:01:52 +00:00
|
|
|
|
else:
|
|
|
|
|
raise ValueError(Errors.E030)
|
2019-03-11 14:59:09 +00:00
|
|
|
|
|
2021-12-06 08:58:01 +00:00
|
|
|
|
@property
|
|
|
|
|
def sents(self):
|
|
|
|
|
"""Obtain the sentences that contain this span. If the given span
|
|
|
|
|
crosses sentence boundaries, return all sentences it is a part of.
|
|
|
|
|
|
|
|
|
|
RETURNS (Iterable[Span]): All sentences that the span is a part of.
|
|
|
|
|
|
|
|
|
|
DOCS: https://spacy.io/api/span#sents
|
|
|
|
|
"""
|
|
|
|
|
cdef int start
|
|
|
|
|
cdef int i
|
|
|
|
|
|
|
|
|
|
if "sents" in self.doc.user_span_hooks:
|
|
|
|
|
yield from self.doc.user_span_hooks["sents"](self)
|
|
|
|
|
elif "sents" in self.doc.user_hooks:
|
|
|
|
|
for sentence in self.doc.user_hooks["sents"](self.doc):
|
|
|
|
|
if sentence.end > self.start:
|
|
|
|
|
if sentence.start < self.end or sentence.start == self.start == self.end:
|
|
|
|
|
yield sentence
|
|
|
|
|
else:
|
|
|
|
|
break
|
|
|
|
|
else:
|
|
|
|
|
if not self.doc.has_annotation("SENT_START"):
|
|
|
|
|
raise ValueError(Errors.E030)
|
|
|
|
|
# Use `sent_start` token attribute to find sentence boundaries
|
|
|
|
|
# Find start of the 1st sentence of the Span
|
|
|
|
|
start = self.start
|
|
|
|
|
while self.doc.c[start].sent_start != 1 and start > 0:
|
|
|
|
|
start -= 1
|
|
|
|
|
|
|
|
|
|
# Now, find all the sentences in the span
|
|
|
|
|
for i in range(start + 1, self.doc.length):
|
|
|
|
|
if self.doc.c[i].sent_start == 1:
|
|
|
|
|
yield Span(self.doc, start, i)
|
|
|
|
|
start = i
|
|
|
|
|
if start >= self.end:
|
|
|
|
|
break
|
2023-03-14 09:21:53 +00:00
|
|
|
|
elif i == self.doc.length - 1:
|
|
|
|
|
yield Span(self.doc, start, self.doc.length)
|
2021-12-06 08:58:01 +00:00
|
|
|
|
|
2023-03-29 16:54:47 +00:00
|
|
|
|
# Ensure that trailing parts of the Span instance are included in last element of .sents.
|
|
|
|
|
if start == self.doc.length - 1:
|
|
|
|
|
yield Span(self.doc, start, self.doc.length)
|
|
|
|
|
|
2019-03-11 14:59:09 +00:00
|
|
|
|
@property
|
|
|
|
|
def ents(self):
|
2022-01-31 07:41:42 +00:00
|
|
|
|
"""The named entities that fall completely within the span. Returns
|
|
|
|
|
a tuple of `Span` objects.
|
2019-03-08 10:42:26 +00:00
|
|
|
|
|
|
|
|
|
RETURNS (tuple): Entities in the span, one `Span` per entity.
|
|
|
|
|
|
2021-01-30 09:09:38 +00:00
|
|
|
|
DOCS: https://spacy.io/api/span#ents
|
2019-03-08 10:42:26 +00:00
|
|
|
|
"""
|
2021-01-14 06:30:41 +00:00
|
|
|
|
cdef Span ent
|
2019-03-11 14:59:09 +00:00
|
|
|
|
ents = []
|
|
|
|
|
for ent in self.doc.ents:
|
2021-01-14 06:30:41 +00:00
|
|
|
|
if ent.c.start >= self.c.start:
|
|
|
|
|
if ent.c.end <= self.c.end:
|
|
|
|
|
ents.append(ent)
|
|
|
|
|
else:
|
|
|
|
|
break
|
2019-03-11 14:59:09 +00:00
|
|
|
|
return ents
|
2018-08-07 11:52:32 +00:00
|
|
|
|
|
2019-03-11 14:59:09 +00:00
|
|
|
|
@property
|
|
|
|
|
def has_vector(self):
|
2019-03-08 10:42:26 +00:00
|
|
|
|
"""A boolean value indicating whether a word vector is associated with
|
|
|
|
|
the object.
|
|
|
|
|
|
|
|
|
|
RETURNS (bool): Whether a word vector is associated with the object.
|
|
|
|
|
|
2021-01-30 09:09:38 +00:00
|
|
|
|
DOCS: https://spacy.io/api/span#has_vector
|
2017-05-19 16:47:46 +00:00
|
|
|
|
"""
|
2019-03-11 14:59:09 +00:00
|
|
|
|
if "has_vector" in self.doc.user_span_hooks:
|
|
|
|
|
return self.doc.user_span_hooks["has_vector"](self)
|
2022-01-18 16:14:35 +00:00
|
|
|
|
elif self.vocab.vectors.size > 0:
|
2019-03-11 14:59:09 +00:00
|
|
|
|
return any(token.has_vector for token in self)
|
|
|
|
|
elif self.doc.tensor.size > 0:
|
|
|
|
|
return True
|
|
|
|
|
else:
|
|
|
|
|
return False
|
2017-04-01 08:19:01 +00:00
|
|
|
|
|
2019-03-11 14:59:09 +00:00
|
|
|
|
@property
|
|
|
|
|
def vector(self):
|
2017-05-19 16:47:46 +00:00
|
|
|
|
"""A real-valued meaning representation. Defaults to an average of the
|
|
|
|
|
token vectors.
|
|
|
|
|
|
|
|
|
|
RETURNS (numpy.ndarray[ndim=1, dtype='float32']): A 1D numpy array
|
|
|
|
|
representing the span's semantics.
|
2019-03-08 10:42:26 +00:00
|
|
|
|
|
2021-01-30 09:09:38 +00:00
|
|
|
|
DOCS: https://spacy.io/api/span#vector
|
2017-05-19 16:47:46 +00:00
|
|
|
|
"""
|
2019-03-11 14:59:09 +00:00
|
|
|
|
if "vector" in self.doc.user_span_hooks:
|
|
|
|
|
return self.doc.user_span_hooks["vector"](self)
|
|
|
|
|
if self._vector is None:
|
2021-09-20 18:22:49 +00:00
|
|
|
|
if not len(self):
|
|
|
|
|
xp = get_array_module(self.vocab.vectors.data)
|
|
|
|
|
self._vector = xp.zeros((self.vocab.vectors_length,), dtype="f")
|
|
|
|
|
else:
|
|
|
|
|
self._vector = sum(t.vector for t in self) / len(self)
|
2019-03-11 14:59:09 +00:00
|
|
|
|
return self._vector
|
2015-09-17 01:50:11 +00:00
|
|
|
|
|
2019-03-11 14:59:09 +00:00
|
|
|
|
@property
|
|
|
|
|
def vector_norm(self):
|
2019-03-08 10:42:26 +00:00
|
|
|
|
"""The L2 norm of the span's vector representation.
|
|
|
|
|
|
|
|
|
|
RETURNS (float): The L2 norm of the vector representation.
|
|
|
|
|
|
2021-01-30 09:09:38 +00:00
|
|
|
|
DOCS: https://spacy.io/api/span#vector_norm
|
2019-03-08 10:42:26 +00:00
|
|
|
|
"""
|
2019-03-11 14:59:09 +00:00
|
|
|
|
if "vector_norm" in self.doc.user_span_hooks:
|
|
|
|
|
return self.doc.user_span_hooks["vector"](self)
|
|
|
|
|
if self._vector_norm is None:
|
2021-08-25 14:06:22 +00:00
|
|
|
|
vector = self.vector
|
2019-03-20 11:09:59 +00:00
|
|
|
|
total = (vector*vector).sum()
|
2021-08-25 14:06:22 +00:00
|
|
|
|
xp = get_array_module(vector)
|
2019-03-20 11:09:59 +00:00
|
|
|
|
self._vector_norm = xp.sqrt(total) if total != 0. else 0.
|
2019-03-11 14:59:09 +00:00
|
|
|
|
return self._vector_norm
|
|
|
|
|
|
2019-08-01 16:30:50 +00:00
|
|
|
|
@property
|
|
|
|
|
def tensor(self):
|
|
|
|
|
"""The span's slice of the doc's tensor.
|
2019-12-22 00:53:56 +00:00
|
|
|
|
|
2019-08-01 16:30:50 +00:00
|
|
|
|
RETURNS (ndarray[ndim=2, dtype='float32']): A 2D numpy or cupy array
|
|
|
|
|
representing the span's semantics.
|
|
|
|
|
"""
|
|
|
|
|
if self.doc.tensor is None:
|
|
|
|
|
return None
|
|
|
|
|
return self.doc.tensor[self.start : self.end]
|
|
|
|
|
|
2019-03-11 14:59:09 +00:00
|
|
|
|
@property
|
|
|
|
|
def sentiment(self):
|
2017-10-27 15:07:26 +00:00
|
|
|
|
"""RETURNS (float): A scalar value indicating the positivity or
|
|
|
|
|
negativity of the span.
|
|
|
|
|
"""
|
2019-03-11 14:59:09 +00:00
|
|
|
|
if "sentiment" in self.doc.user_span_hooks:
|
|
|
|
|
return self.doc.user_span_hooks["sentiment"](self)
|
|
|
|
|
else:
|
|
|
|
|
return sum([token.sentiment for token in self]) / len(self)
|
2016-12-02 10:05:50 +00:00
|
|
|
|
|
2019-03-11 14:59:09 +00:00
|
|
|
|
@property
|
|
|
|
|
def text(self):
|
2020-05-24 15:20:58 +00:00
|
|
|
|
"""RETURNS (str): The original verbatim text of the span."""
|
2019-03-11 14:59:09 +00:00
|
|
|
|
text = self.text_with_ws
|
2021-01-26 03:52:45 +00:00
|
|
|
|
if len(self) > 0 and self[-1].whitespace_:
|
2019-03-11 14:59:09 +00:00
|
|
|
|
text = text[:-1]
|
|
|
|
|
return text
|
2015-09-13 00:27:42 +00:00
|
|
|
|
|
2019-03-11 14:59:09 +00:00
|
|
|
|
@property
|
|
|
|
|
def text_with_ws(self):
|
2017-05-19 16:47:46 +00:00
|
|
|
|
"""The text content of the span with a trailing whitespace character if
|
|
|
|
|
the last token has one.
|
|
|
|
|
|
2020-05-24 15:20:58 +00:00
|
|
|
|
RETURNS (str): The text content of the span (with trailing
|
2017-10-27 13:41:45 +00:00
|
|
|
|
whitespace).
|
2017-05-19 16:47:46 +00:00
|
|
|
|
"""
|
2019-03-11 14:59:09 +00:00
|
|
|
|
return "".join([t.text_with_ws for t in self])
|
2015-09-13 00:27:42 +00:00
|
|
|
|
|
2019-03-11 14:59:09 +00:00
|
|
|
|
@property
|
|
|
|
|
def noun_chunks(self):
|
2021-01-17 11:56:05 +00:00
|
|
|
|
"""Iterate over the base noun phrases in the span. Yields base
|
|
|
|
|
noun-phrase #[code Span] objects, if the language has a noun chunk iterator.
|
|
|
|
|
Raises a NotImplementedError otherwise.
|
|
|
|
|
|
|
|
|
|
A base noun phrase, or "NP chunk", is a noun
|
2017-05-18 20:17:24 +00:00
|
|
|
|
phrase that does not permit other NPs to be nested within it – so no
|
2017-10-27 13:41:45 +00:00
|
|
|
|
NP-level coordination, no prepositional phrases, and no relative
|
|
|
|
|
clauses.
|
2017-05-18 20:17:24 +00:00
|
|
|
|
|
2021-01-17 11:56:05 +00:00
|
|
|
|
YIELDS (Span): Noun chunks in the span.
|
2019-03-08 10:42:26 +00:00
|
|
|
|
|
2021-01-30 09:09:38 +00:00
|
|
|
|
DOCS: https://spacy.io/api/span#noun_chunks
|
2017-04-15 11:05:15 +00:00
|
|
|
|
"""
|
2021-01-17 11:56:05 +00:00
|
|
|
|
for span in self.doc.noun_chunks:
|
|
|
|
|
if span.start >= self.start and span.end <= self.end:
|
|
|
|
|
yield span
|
2019-03-11 14:59:09 +00:00
|
|
|
|
|
|
|
|
|
@property
|
|
|
|
|
def root(self):
|
2019-03-08 10:42:26 +00:00
|
|
|
|
"""The token with the shortest path to the root of the
|
|
|
|
|
sentence (or the root itself). If multiple tokens are equally
|
|
|
|
|
high in the tree, the first token is taken.
|
2016-11-01 11:25:36 +00:00
|
|
|
|
|
2017-05-18 20:17:24 +00:00
|
|
|
|
RETURNS (Token): The root token.
|
2017-04-01 08:19:01 +00:00
|
|
|
|
|
2021-01-30 09:09:38 +00:00
|
|
|
|
DOCS: https://spacy.io/api/span#root
|
2015-05-13 19:45:19 +00:00
|
|
|
|
"""
|
2019-03-11 14:59:09 +00:00
|
|
|
|
if "root" in self.doc.user_span_hooks:
|
|
|
|
|
return self.doc.user_span_hooks["root"](self)
|
|
|
|
|
# This should probably be called 'head', and the other one called
|
2019-07-23 16:28:55 +00:00
|
|
|
|
# 'gov'. But we went with 'head' elsewhere, and now we're stuck =/
|
2019-03-11 14:59:09 +00:00
|
|
|
|
cdef int i
|
|
|
|
|
# First, we scan through the Span, and check whether there's a word
|
|
|
|
|
# with head==0, i.e. a sentence root. If so, we can return it. The
|
|
|
|
|
# longer the span, the more likely it contains a sentence root, and
|
|
|
|
|
# in this case we return in linear time.
|
2021-01-14 06:30:41 +00:00
|
|
|
|
for i in range(self.c.start, self.c.end):
|
2019-03-11 14:59:09 +00:00
|
|
|
|
if self.doc.c[i].head == 0:
|
|
|
|
|
return self.doc[i]
|
|
|
|
|
# If we don't have a sentence root, we do something that's not so
|
|
|
|
|
# algorithmically clever, but I think should be quite fast,
|
|
|
|
|
# especially for short spans.
|
|
|
|
|
# For each word, we count the path length, and arg min this measure.
|
|
|
|
|
# We could use better tree logic to save steps here...But I
|
|
|
|
|
# think this should be okay.
|
|
|
|
|
cdef int current_best = self.doc.length
|
|
|
|
|
cdef int root = -1
|
2021-01-14 06:30:41 +00:00
|
|
|
|
for i in range(self.c.start, self.c.end):
|
|
|
|
|
if self.c.start <= (i+self.doc.c[i].head) < self.c.end:
|
2019-03-11 14:59:09 +00:00
|
|
|
|
continue
|
|
|
|
|
words_to_root = _count_words_to_root(&self.doc.c[i], self.doc.length)
|
|
|
|
|
if words_to_root < current_best:
|
|
|
|
|
current_best = words_to_root
|
|
|
|
|
root = i
|
|
|
|
|
if root == -1:
|
2021-01-14 06:30:41 +00:00
|
|
|
|
return self.doc[self.c.start]
|
2019-03-11 14:59:09 +00:00
|
|
|
|
else:
|
|
|
|
|
return self.doc[root]
|
2017-04-01 08:19:01 +00:00
|
|
|
|
|
2023-01-27 14:09:17 +00:00
|
|
|
|
def char_span(self, int start_idx, int end_idx, label=0, kb_id=0, vector=None, id=0, alignment_mode="strict", span_id=0):
|
2019-12-13 14:54:58 +00:00
|
|
|
|
"""Create a `Span` object from the slice `span.text[start : end]`.
|
|
|
|
|
|
|
|
|
|
start (int): The index of the first character of the span.
|
|
|
|
|
end (int): The index of the first character after the span.
|
2023-01-27 14:09:17 +00:00
|
|
|
|
label (Union[int, str]): A label to attach to the Span, e.g. for
|
2019-12-13 14:54:58 +00:00
|
|
|
|
named entities.
|
2023-01-27 14:09:17 +00:00
|
|
|
|
kb_id (Union[int, str]): An ID from a KB to capture the meaning of a named entity.
|
2019-12-13 14:54:58 +00:00
|
|
|
|
vector (ndarray[ndim=1, dtype='float32']): A meaning representation of
|
|
|
|
|
the span.
|
2023-01-27 14:09:17 +00:00
|
|
|
|
id (Union[int, str]): Unused.
|
2023-01-27 10:43:40 +00:00
|
|
|
|
alignment_mode (str): How character indices are aligned to token
|
|
|
|
|
boundaries. Options: "strict" (character indices must be aligned
|
|
|
|
|
with token boundaries), "contract" (span of all tokens completely
|
|
|
|
|
within the character span), "expand" (span of all tokens at least
|
|
|
|
|
partially covered by the character span). Defaults to "strict".
|
2023-01-27 14:09:17 +00:00
|
|
|
|
span_id (Union[int, str]): An identifier to associate with the span.
|
2019-12-13 14:54:58 +00:00
|
|
|
|
RETURNS (Span): The newly constructed object.
|
|
|
|
|
"""
|
2021-01-14 06:30:41 +00:00
|
|
|
|
start_idx += self.c.start_char
|
|
|
|
|
end_idx += self.c.start_char
|
2023-01-27 14:09:17 +00:00
|
|
|
|
return self.doc.char_span(start_idx, end_idx, label=label, kb_id=kb_id, vector=vector, alignment_mode=alignment_mode, span_id=span_id)
|
2019-12-13 14:54:58 +00:00
|
|
|
|
|
2019-03-11 16:05:45 +00:00
|
|
|
|
@property
|
|
|
|
|
def conjuncts(self):
|
|
|
|
|
"""Tokens that are conjoined to the span's root.
|
|
|
|
|
|
|
|
|
|
RETURNS (tuple): A tuple of Token objects.
|
|
|
|
|
|
2021-01-30 09:09:38 +00:00
|
|
|
|
DOCS: https://spacy.io/api/span#lefts
|
2019-03-11 16:05:45 +00:00
|
|
|
|
"""
|
|
|
|
|
return self.root.conjuncts
|
|
|
|
|
|
2019-03-11 14:59:09 +00:00
|
|
|
|
@property
|
|
|
|
|
def lefts(self):
|
2019-03-08 10:42:26 +00:00
|
|
|
|
"""Tokens that are to the left of the span, whose head is within the
|
2017-05-18 20:17:24 +00:00
|
|
|
|
`Span`.
|
2017-04-01 08:19:01 +00:00
|
|
|
|
|
2017-05-18 20:17:24 +00:00
|
|
|
|
YIELDS (Token):A left-child of a token of the span.
|
2019-03-08 10:42:26 +00:00
|
|
|
|
|
2021-01-30 09:09:38 +00:00
|
|
|
|
DOCS: https://spacy.io/api/span#lefts
|
2016-11-01 11:25:36 +00:00
|
|
|
|
"""
|
2019-03-11 14:59:09 +00:00
|
|
|
|
for token in reversed(self): # Reverse, so we get tokens in order
|
|
|
|
|
for left in token.lefts:
|
|
|
|
|
if left.i < self.start:
|
|
|
|
|
yield left
|
2015-05-13 19:45:19 +00:00
|
|
|
|
|
2019-03-11 14:59:09 +00:00
|
|
|
|
@property
|
|
|
|
|
def rights(self):
|
2017-05-18 20:17:24 +00:00
|
|
|
|
"""Tokens that are to the right of the Span, whose head is within the
|
|
|
|
|
`Span`.
|
2017-04-01 08:19:01 +00:00
|
|
|
|
|
2017-05-18 20:17:24 +00:00
|
|
|
|
YIELDS (Token): A right-child of a token of the span.
|
2019-03-08 10:42:26 +00:00
|
|
|
|
|
2021-01-30 09:09:38 +00:00
|
|
|
|
DOCS: https://spacy.io/api/span#rights
|
2016-11-01 11:25:36 +00:00
|
|
|
|
"""
|
2019-03-11 14:59:09 +00:00
|
|
|
|
for token in self:
|
|
|
|
|
for right in token.rights:
|
|
|
|
|
if right.i >= self.end:
|
|
|
|
|
yield right
|
2015-05-13 19:45:19 +00:00
|
|
|
|
|
2019-03-11 14:59:09 +00:00
|
|
|
|
@property
|
|
|
|
|
def n_lefts(self):
|
2019-03-08 10:42:26 +00:00
|
|
|
|
"""The number of tokens that are to the left of the span, whose
|
|
|
|
|
heads are within the span.
|
|
|
|
|
|
|
|
|
|
RETURNS (int): The number of leftward immediate children of the
|
2017-10-27 15:07:26 +00:00
|
|
|
|
span, in the syntactic dependency parse.
|
2019-03-08 10:42:26 +00:00
|
|
|
|
|
2021-01-30 09:09:38 +00:00
|
|
|
|
DOCS: https://spacy.io/api/span#n_lefts
|
2017-10-27 15:07:26 +00:00
|
|
|
|
"""
|
2019-03-11 14:59:09 +00:00
|
|
|
|
return len(list(self.lefts))
|
2017-10-27 15:07:26 +00:00
|
|
|
|
|
2019-03-11 14:59:09 +00:00
|
|
|
|
@property
|
|
|
|
|
def n_rights(self):
|
2019-03-08 10:42:26 +00:00
|
|
|
|
"""The number of tokens that are to the right of the span, whose
|
|
|
|
|
heads are within the span.
|
|
|
|
|
|
|
|
|
|
RETURNS (int): The number of rightward immediate children of the
|
2017-10-27 15:07:26 +00:00
|
|
|
|
span, in the syntactic dependency parse.
|
2019-03-08 10:42:26 +00:00
|
|
|
|
|
2021-01-30 09:09:38 +00:00
|
|
|
|
DOCS: https://spacy.io/api/span#n_rights
|
2017-10-27 15:07:26 +00:00
|
|
|
|
"""
|
2019-03-11 14:59:09 +00:00
|
|
|
|
return len(list(self.rights))
|
2017-10-27 15:07:26 +00:00
|
|
|
|
|
2019-03-11 14:59:09 +00:00
|
|
|
|
@property
|
|
|
|
|
def subtree(self):
|
2019-01-09 02:11:15 +00:00
|
|
|
|
"""Tokens within the span and tokens which descend from them.
|
2016-11-01 11:25:36 +00:00
|
|
|
|
|
2019-01-09 02:11:15 +00:00
|
|
|
|
YIELDS (Token): A token within the span, or a descendant from it.
|
2019-03-08 10:42:26 +00:00
|
|
|
|
|
2021-01-30 09:09:38 +00:00
|
|
|
|
DOCS: https://spacy.io/api/span#subtree
|
2016-11-01 11:25:36 +00:00
|
|
|
|
"""
|
2019-03-11 14:59:09 +00:00
|
|
|
|
for word in self.lefts:
|
|
|
|
|
yield from word.subtree
|
|
|
|
|
yield from self
|
|
|
|
|
for word in self.rights:
|
|
|
|
|
yield from word.subtree
|
2015-07-09 15:30:58 +00:00
|
|
|
|
|
2024-04-16 09:51:14 +00:00
|
|
|
|
@property
|
|
|
|
|
def start(self):
|
|
|
|
|
return self.c.start
|
2021-01-14 06:30:41 +00:00
|
|
|
|
|
2024-04-16 09:51:14 +00:00
|
|
|
|
@start.setter
|
|
|
|
|
def start(self, int start):
|
|
|
|
|
if start < 0:
|
|
|
|
|
raise IndexError(Errors.E1032.format(var="start", forbidden="< 0", value=start))
|
|
|
|
|
self.c.start = start
|
2021-01-14 06:30:41 +00:00
|
|
|
|
|
2024-04-16 09:51:14 +00:00
|
|
|
|
@property
|
|
|
|
|
def end(self):
|
|
|
|
|
return self.c.end
|
2021-01-14 06:30:41 +00:00
|
|
|
|
|
2024-04-16 09:51:14 +00:00
|
|
|
|
@end.setter
|
|
|
|
|
def end(self, int end):
|
|
|
|
|
if end < 0:
|
|
|
|
|
raise IndexError(Errors.E1032.format(var="end", forbidden="< 0", value=end))
|
|
|
|
|
self.c.end = end
|
2021-01-14 06:30:41 +00:00
|
|
|
|
|
2024-04-16 09:51:14 +00:00
|
|
|
|
@property
|
|
|
|
|
def start_char(self):
|
|
|
|
|
return self.c.start_char
|
2021-01-14 06:30:41 +00:00
|
|
|
|
|
2024-04-16 09:51:14 +00:00
|
|
|
|
@start_char.setter
|
|
|
|
|
def start_char(self, int start_char):
|
|
|
|
|
if start_char < 0:
|
|
|
|
|
raise IndexError(Errors.E1032.format(var="start_char", forbidden="< 0", value=start_char))
|
|
|
|
|
self.c.start_char = start_char
|
2021-01-14 06:30:41 +00:00
|
|
|
|
|
2024-04-16 09:51:14 +00:00
|
|
|
|
@property
|
|
|
|
|
def end_char(self):
|
|
|
|
|
return self.c.end_char
|
2021-01-14 06:30:41 +00:00
|
|
|
|
|
2024-04-16 09:51:14 +00:00
|
|
|
|
@end_char.setter
|
|
|
|
|
def end_char(self, int end_char):
|
|
|
|
|
if end_char < 0:
|
|
|
|
|
raise IndexError(Errors.E1032.format(var="end_char", forbidden="< 0", value=end_char))
|
|
|
|
|
self.c.end_char = end_char
|
2021-01-14 06:30:41 +00:00
|
|
|
|
|
2024-04-16 09:51:14 +00:00
|
|
|
|
@property
|
|
|
|
|
def label(self):
|
|
|
|
|
return self.c.label
|
2021-01-14 06:30:41 +00:00
|
|
|
|
|
2024-04-16 09:51:14 +00:00
|
|
|
|
@label.setter
|
|
|
|
|
def label(self, attr_t label):
|
|
|
|
|
self.c.label = label
|
2021-01-14 06:30:41 +00:00
|
|
|
|
|
2024-04-16 09:51:14 +00:00
|
|
|
|
@property
|
|
|
|
|
def kb_id(self):
|
|
|
|
|
return self.c.kb_id
|
2021-01-14 06:30:41 +00:00
|
|
|
|
|
2024-04-16 09:51:14 +00:00
|
|
|
|
@kb_id.setter
|
|
|
|
|
def kb_id(self, attr_t kb_id):
|
|
|
|
|
self.c.kb_id = kb_id
|
2021-01-14 06:30:41 +00:00
|
|
|
|
|
2024-04-16 09:51:14 +00:00
|
|
|
|
@property
|
|
|
|
|
def id(self):
|
|
|
|
|
return self.c.id
|
2022-06-02 11:12:53 +00:00
|
|
|
|
|
2024-04-16 09:51:14 +00:00
|
|
|
|
@id.setter
|
|
|
|
|
def id(self, attr_t id):
|
|
|
|
|
self.c.id = id
|
2022-06-02 11:12:53 +00:00
|
|
|
|
|
2024-04-16 09:51:14 +00:00
|
|
|
|
@property
|
|
|
|
|
def ent_id(self):
|
2017-10-27 15:07:26 +00:00
|
|
|
|
"""RETURNS (uint64): The entity ID."""
|
2024-04-16 09:51:14 +00:00
|
|
|
|
return self.root.ent_id
|
2016-09-21 12:54:55 +00:00
|
|
|
|
|
2024-04-16 09:51:14 +00:00
|
|
|
|
@ent_id.setter
|
|
|
|
|
def ent_id(self, hash_t key):
|
|
|
|
|
raise NotImplementedError(Errors.E200.format(attr="ent_id"))
|
2017-05-18 20:17:24 +00:00
|
|
|
|
|
2024-04-16 09:51:14 +00:00
|
|
|
|
@property
|
|
|
|
|
def ent_id_(self):
|
2020-05-24 15:20:58 +00:00
|
|
|
|
"""RETURNS (str): The (string) entity ID."""
|
2024-04-16 09:51:14 +00:00
|
|
|
|
return self.root.ent_id_
|
2016-09-21 12:54:55 +00:00
|
|
|
|
|
2024-04-16 09:51:14 +00:00
|
|
|
|
@ent_id_.setter
|
|
|
|
|
def ent_id_(self, str key):
|
|
|
|
|
raise NotImplementedError(Errors.E200.format(attr="ent_id_"))
|
2016-09-21 12:54:55 +00:00
|
|
|
|
|
2019-03-11 14:59:09 +00:00
|
|
|
|
@property
|
|
|
|
|
def orth_(self):
|
2019-03-08 10:42:26 +00:00
|
|
|
|
"""Verbatim text content (identical to `Span.text`). Exists mostly for
|
2017-10-27 13:41:45 +00:00
|
|
|
|
consistency with other attributes.
|
|
|
|
|
|
2020-05-24 15:20:58 +00:00
|
|
|
|
RETURNS (str): The span's text."""
|
2019-03-11 14:59:09 +00:00
|
|
|
|
return self.text
|
2015-03-26 02:16:40 +00:00
|
|
|
|
|
2019-03-11 14:59:09 +00:00
|
|
|
|
@property
|
|
|
|
|
def lemma_(self):
|
2020-05-24 15:20:58 +00:00
|
|
|
|
"""RETURNS (str): The span's lemma."""
|
2021-06-15 11:24:54 +00:00
|
|
|
|
return "".join([t.lemma_ + t.whitespace_ for t in self]).strip()
|
2017-03-11 00:50:02 +00:00
|
|
|
|
|
2024-04-16 09:51:14 +00:00
|
|
|
|
@property
|
|
|
|
|
def label_(self):
|
2020-05-24 15:20:58 +00:00
|
|
|
|
"""RETURNS (str): The span's label."""
|
2024-04-16 09:51:14 +00:00
|
|
|
|
return self.doc.vocab.strings[self.label]
|
2019-03-08 10:42:26 +00:00
|
|
|
|
|
2024-04-16 09:51:14 +00:00
|
|
|
|
@label_.setter
|
|
|
|
|
def label_(self, str label_):
|
|
|
|
|
self.label = self.doc.vocab.strings.add(label_)
|
2015-03-26 02:16:40 +00:00
|
|
|
|
|
2024-04-16 09:51:14 +00:00
|
|
|
|
@property
|
|
|
|
|
def kb_id_(self):
|
2022-06-02 11:12:53 +00:00
|
|
|
|
"""RETURNS (str): The span's KB ID."""
|
2024-04-16 09:51:14 +00:00
|
|
|
|
return self.doc.vocab.strings[self.kb_id]
|
2019-03-15 14:00:53 +00:00
|
|
|
|
|
2024-04-16 09:51:14 +00:00
|
|
|
|
@kb_id_.setter
|
|
|
|
|
def kb_id_(self, str kb_id_):
|
|
|
|
|
self.kb_id = self.doc.vocab.strings.add(kb_id_)
|
2019-03-14 14:48:40 +00:00
|
|
|
|
|
2024-04-16 09:51:14 +00:00
|
|
|
|
@property
|
|
|
|
|
def id_(self):
|
2022-06-02 11:12:53 +00:00
|
|
|
|
"""RETURNS (str): The span's ID."""
|
2024-04-16 09:51:14 +00:00
|
|
|
|
return self.doc.vocab.strings[self.id]
|
2022-06-02 11:12:53 +00:00
|
|
|
|
|
2024-04-16 09:51:14 +00:00
|
|
|
|
@id_.setter
|
|
|
|
|
def id_(self, str id_):
|
|
|
|
|
self.id = self.doc.vocab.strings.add(id_)
|
2022-06-02 11:12:53 +00:00
|
|
|
|
|
2016-01-16 14:38:50 +00:00
|
|
|
|
|
|
|
|
|
cdef int _count_words_to_root(const TokenC* token, int sent_length) except -1:
|
2016-02-06 12:37:41 +00:00
|
|
|
|
# Don't allow spaces to be the root, if there are
|
|
|
|
|
# better candidates
|
|
|
|
|
if Lexeme.c_check_flag(token.lex, IS_SPACE) and token.l_kids == 0 and token.r_kids == 0:
|
|
|
|
|
return sent_length-1
|
|
|
|
|
if Lexeme.c_check_flag(token.lex, IS_PUNCT) and token.l_kids == 0 and token.r_kids == 0:
|
|
|
|
|
return sent_length-1
|
2016-01-16 14:38:50 +00:00
|
|
|
|
cdef int n = 0
|
|
|
|
|
while token.head != 0:
|
|
|
|
|
token += token.head
|
|
|
|
|
n += 1
|
|
|
|
|
if n >= sent_length:
|
2018-04-03 13:50:31 +00:00
|
|
|
|
raise RuntimeError(Errors.E039)
|
2016-01-16 14:38:50 +00:00
|
|
|
|
return n
|