2016-12-18 21:33:53 +00:00
|
|
|
|
# cython: infer_types=True
|
2015-07-13 17:20:48 +00:00
|
|
|
|
from libc.string cimport memcpy
|
|
|
|
|
from cpython.mem cimport PyMem_Malloc, PyMem_Free
|
|
|
|
|
# Compiler crashes on memory view coercion without this. Should report bug.
|
|
|
|
|
from cython.view cimport array as cvarray
|
2015-07-13 18:20:58 +00:00
|
|
|
|
cimport numpy as np
|
|
|
|
|
np.import_array()
|
2019-03-08 10:42:26 +00:00
|
|
|
|
|
2015-07-13 18:20:58 +00:00
|
|
|
|
import numpy
|
2020-02-18 14:38:18 +00:00
|
|
|
|
from thinc.api import get_array_module
|
2020-04-28 11:37:37 +00:00
|
|
|
|
import warnings
|
2015-07-13 18:20:58 +00:00
|
|
|
|
|
2016-09-23 12:22:06 +00:00
|
|
|
|
from ..typedefs cimport hash_t
|
2015-09-06 17:45:15 +00:00
|
|
|
|
from ..lexeme cimport Lexeme
|
2015-07-26 14:37:16 +00:00
|
|
|
|
from ..attrs cimport IS_ALPHA, IS_ASCII, IS_DIGIT, IS_LOWER, IS_PUNCT, IS_SPACE
|
2017-10-27 13:41:45 +00:00
|
|
|
|
from ..attrs cimport IS_BRACKET, IS_QUOTE, IS_LEFT_PUNCT, IS_RIGHT_PUNCT
|
2020-05-19 13:59:14 +00:00
|
|
|
|
from ..attrs cimport IS_TITLE, IS_UPPER, IS_CURRENCY, LIKE_URL, LIKE_NUM, LIKE_EMAIL
|
2017-10-27 13:41:45 +00:00
|
|
|
|
from ..attrs cimport IS_STOP, ID, ORTH, NORM, LOWER, SHAPE, PREFIX, SUFFIX
|
|
|
|
|
from ..attrs cimport LENGTH, CLUSTER, LEMMA, POS, TAG, DEP
|
2019-03-08 10:42:26 +00:00
|
|
|
|
from ..symbols cimport conj
|
2020-03-02 10:48:10 +00:00
|
|
|
|
from .morphanalysis cimport MorphAnalysis
|
2019-03-08 10:42:26 +00:00
|
|
|
|
|
|
|
|
|
from .. import parts_of_speech
|
|
|
|
|
from .. import util
|
2020-04-28 11:37:37 +00:00
|
|
|
|
from ..errors import Errors, Warnings
|
2018-04-03 16:30:17 +00:00
|
|
|
|
from .underscore import Underscore, get_ext_args
|
2015-08-23 18:49:18 +00:00
|
|
|
|
|
2015-07-13 17:20:48 +00:00
|
|
|
|
|
|
|
|
|
cdef class Token:
|
2017-10-27 13:41:45 +00:00
|
|
|
|
"""An individual token – i.e. a word, punctuation symbol, whitespace,
|
2019-03-08 10:42:26 +00:00
|
|
|
|
etc.
|
|
|
|
|
|
2020-09-04 10:58:50 +00:00
|
|
|
|
DOCS: https://nightly.spacy.io/api/token
|
2019-03-08 10:42:26 +00:00
|
|
|
|
"""
|
2017-10-07 16:56:01 +00:00
|
|
|
|
@classmethod
|
2018-04-03 16:30:17 +00:00
|
|
|
|
def set_extension(cls, name, **kwargs):
|
2019-03-08 10:42:26 +00:00
|
|
|
|
"""Define a custom attribute which becomes available as `Token._`.
|
|
|
|
|
|
2020-05-24 15:20:58 +00:00
|
|
|
|
name (str): Name of the attribute to set.
|
2019-03-08 10:42:26 +00:00
|
|
|
|
default: Optional default value of the attribute.
|
|
|
|
|
getter (callable): Optional getter function.
|
|
|
|
|
setter (callable): Optional setter function.
|
|
|
|
|
method (callable): Optional method for method extension.
|
|
|
|
|
force (bool): Force overwriting existing attribute.
|
|
|
|
|
|
2020-09-04 10:58:50 +00:00
|
|
|
|
DOCS: https://nightly.spacy.io/api/token#set_extension
|
|
|
|
|
USAGE: https://nightly.spacy.io/usage/processing-pipelines#custom-components-attributes
|
2019-03-08 10:42:26 +00:00
|
|
|
|
"""
|
|
|
|
|
if cls.has_extension(name) and not kwargs.get("force", False):
|
|
|
|
|
raise ValueError(Errors.E090.format(name=name, obj="Token"))
|
2018-04-03 16:30:17 +00:00
|
|
|
|
Underscore.token_extensions[name] = get_ext_args(**kwargs)
|
2017-10-07 16:56:01 +00:00
|
|
|
|
|
|
|
|
|
@classmethod
|
|
|
|
|
def get_extension(cls, name):
|
2019-03-08 10:42:26 +00:00
|
|
|
|
"""Look up a previously registered extension by name.
|
|
|
|
|
|
2020-05-24 15:20:58 +00:00
|
|
|
|
name (str): Name of the extension.
|
2019-03-08 10:42:26 +00:00
|
|
|
|
RETURNS (tuple): A `(default, method, getter, setter)` tuple.
|
|
|
|
|
|
2020-09-04 10:58:50 +00:00
|
|
|
|
DOCS: https://nightly.spacy.io/api/token#get_extension
|
2019-03-08 10:42:26 +00:00
|
|
|
|
"""
|
2018-04-29 13:48:19 +00:00
|
|
|
|
return Underscore.token_extensions.get(name)
|
2017-10-07 16:56:01 +00:00
|
|
|
|
|
|
|
|
|
@classmethod
|
|
|
|
|
def has_extension(cls, name):
|
2019-03-08 10:42:26 +00:00
|
|
|
|
"""Check whether an extension has been registered.
|
|
|
|
|
|
2020-05-24 15:20:58 +00:00
|
|
|
|
name (str): Name of the extension.
|
2019-03-08 10:42:26 +00:00
|
|
|
|
RETURNS (bool): Whether the extension has been registered.
|
|
|
|
|
|
2020-09-04 10:58:50 +00:00
|
|
|
|
DOCS: https://nightly.spacy.io/api/token#has_extension
|
2019-03-08 10:42:26 +00:00
|
|
|
|
"""
|
2018-04-29 13:48:19 +00:00
|
|
|
|
return name in Underscore.token_extensions
|
2017-10-07 16:56:01 +00:00
|
|
|
|
|
2018-04-28 21:33:09 +00:00
|
|
|
|
@classmethod
|
|
|
|
|
def remove_extension(cls, name):
|
2019-03-08 10:42:26 +00:00
|
|
|
|
"""Remove a previously registered extension.
|
|
|
|
|
|
2020-05-24 15:20:58 +00:00
|
|
|
|
name (str): Name of the extension.
|
2019-03-08 10:42:26 +00:00
|
|
|
|
RETURNS (tuple): A `(default, method, getter, setter)` tuple of the
|
|
|
|
|
removed extension.
|
|
|
|
|
|
2020-09-04 10:58:50 +00:00
|
|
|
|
DOCS: https://nightly.spacy.io/api/token#remove_extension
|
2019-03-08 10:42:26 +00:00
|
|
|
|
"""
|
2018-04-28 21:33:09 +00:00
|
|
|
|
if not cls.has_extension(name):
|
|
|
|
|
raise ValueError(Errors.E046.format(name=name))
|
|
|
|
|
return Underscore.token_extensions.pop(name)
|
|
|
|
|
|
2015-07-13 22:10:11 +00:00
|
|
|
|
def __cinit__(self, Vocab vocab, Doc doc, int offset):
|
2017-05-19 16:47:56 +00:00
|
|
|
|
"""Construct a `Token` object.
|
|
|
|
|
|
|
|
|
|
vocab (Vocab): A storage container for lexical types.
|
|
|
|
|
doc (Doc): The parent document.
|
|
|
|
|
offset (int): The index of the token within the document.
|
2019-03-08 10:42:26 +00:00
|
|
|
|
|
2020-09-04 10:58:50 +00:00
|
|
|
|
DOCS: https://nightly.spacy.io/api/token#init
|
2017-05-19 16:47:56 +00:00
|
|
|
|
"""
|
2015-07-13 17:20:48 +00:00
|
|
|
|
self.vocab = vocab
|
2015-07-13 22:10:11 +00:00
|
|
|
|
self.doc = doc
|
2015-11-03 13:15:14 +00:00
|
|
|
|
self.c = &self.doc.c[offset]
|
2015-07-13 22:10:11 +00:00
|
|
|
|
self.i = offset
|
2015-07-13 17:20:48 +00:00
|
|
|
|
|
2017-01-16 12:27:57 +00:00
|
|
|
|
def __hash__(self):
|
|
|
|
|
return hash((self.doc, self.i))
|
|
|
|
|
|
2015-07-13 17:20:48 +00:00
|
|
|
|
def __len__(self):
|
2017-05-19 16:47:56 +00:00
|
|
|
|
"""The number of unicode characters in the token, i.e. `token.text`.
|
|
|
|
|
|
|
|
|
|
RETURNS (int): The number of unicode characters in the token.
|
2019-03-08 10:42:26 +00:00
|
|
|
|
|
2020-09-04 10:58:50 +00:00
|
|
|
|
DOCS: https://nightly.spacy.io/api/token#len
|
2017-04-15 11:05:15 +00:00
|
|
|
|
"""
|
2015-07-13 17:20:48 +00:00
|
|
|
|
return self.c.lex.length
|
|
|
|
|
|
|
|
|
|
def __unicode__(self):
|
2016-10-17 12:02:47 +00:00
|
|
|
|
return self.text
|
2015-07-13 17:20:48 +00:00
|
|
|
|
|
2015-11-02 18:22:18 +00:00
|
|
|
|
def __bytes__(self):
|
2016-10-17 12:02:47 +00:00
|
|
|
|
return self.text.encode('utf8')
|
2015-11-02 18:22:18 +00:00
|
|
|
|
|
2015-07-24 01:49:30 +00:00
|
|
|
|
def __str__(self):
|
2019-12-22 00:53:56 +00:00
|
|
|
|
return self.__unicode__()
|
2015-07-24 01:49:30 +00:00
|
|
|
|
|
2015-10-21 11:11:46 +00:00
|
|
|
|
def __repr__(self):
|
2015-11-02 18:22:18 +00:00
|
|
|
|
return self.__str__()
|
2015-10-21 11:11:46 +00:00
|
|
|
|
|
2017-01-11 12:03:32 +00:00
|
|
|
|
def __richcmp__(self, Token other, int op):
|
2017-01-09 18:30:31 +00:00
|
|
|
|
# http://cython.readthedocs.io/en/latest/src/userguide/special_methods.html
|
2018-01-15 14:51:25 +00:00
|
|
|
|
if other is None:
|
|
|
|
|
if op in (0, 1, 2):
|
|
|
|
|
return False
|
|
|
|
|
else:
|
|
|
|
|
return True
|
2017-08-19 14:39:32 +00:00
|
|
|
|
cdef Doc my_doc = self.doc
|
|
|
|
|
cdef Doc other_doc = other.doc
|
2017-01-09 18:30:31 +00:00
|
|
|
|
my = self.idx
|
2018-01-15 14:51:25 +00:00
|
|
|
|
their = other.idx
|
2017-01-09 18:30:31 +00:00
|
|
|
|
if op == 0:
|
|
|
|
|
return my < their
|
|
|
|
|
elif op == 2:
|
2017-08-19 14:39:32 +00:00
|
|
|
|
if my_doc is other_doc:
|
|
|
|
|
return my == their
|
|
|
|
|
else:
|
|
|
|
|
return False
|
2017-01-09 18:30:31 +00:00
|
|
|
|
elif op == 4:
|
|
|
|
|
return my > their
|
|
|
|
|
elif op == 1:
|
|
|
|
|
return my <= their
|
|
|
|
|
elif op == 3:
|
2017-08-19 14:39:32 +00:00
|
|
|
|
if my_doc is other_doc:
|
|
|
|
|
return my != their
|
|
|
|
|
else:
|
|
|
|
|
return True
|
2017-01-09 18:30:31 +00:00
|
|
|
|
elif op == 5:
|
|
|
|
|
return my >= their
|
|
|
|
|
else:
|
2018-04-03 13:50:31 +00:00
|
|
|
|
raise ValueError(Errors.E041.format(op=op))
|
2017-01-09 18:30:31 +00:00
|
|
|
|
|
2019-02-13 10:27:04 +00:00
|
|
|
|
def __reduce__(self):
|
|
|
|
|
raise NotImplementedError(Errors.E111)
|
|
|
|
|
|
2017-10-07 16:56:01 +00:00
|
|
|
|
@property
|
|
|
|
|
def _(self):
|
2019-03-08 10:42:26 +00:00
|
|
|
|
"""Custom extension attributes registered via `set_extension`."""
|
2017-10-07 16:56:01 +00:00
|
|
|
|
return Underscore(Underscore.token_extensions, self,
|
|
|
|
|
start=self.idx, end=None)
|
|
|
|
|
|
2015-07-13 17:20:48 +00:00
|
|
|
|
cpdef bint check_flag(self, attr_id_t flag_id) except -1:
|
2017-05-19 16:47:56 +00:00
|
|
|
|
"""Check the value of a boolean flag.
|
|
|
|
|
|
|
|
|
|
flag_id (int): The ID of the flag attribute.
|
|
|
|
|
RETURNS (bool): Whether the flag is set.
|
2017-02-26 21:27:11 +00:00
|
|
|
|
|
2020-09-04 10:58:50 +00:00
|
|
|
|
DOCS: https://nightly.spacy.io/api/token#check_flag
|
2017-04-15 11:05:15 +00:00
|
|
|
|
"""
|
2015-09-15 03:06:18 +00:00
|
|
|
|
return Lexeme.c_check_flag(self.c.lex, flag_id)
|
2015-07-13 17:20:48 +00:00
|
|
|
|
|
|
|
|
|
def nbor(self, int i=1):
|
2017-05-19 16:47:56 +00:00
|
|
|
|
"""Get a neighboring token.
|
2016-11-01 11:25:36 +00:00
|
|
|
|
|
2017-05-19 16:47:56 +00:00
|
|
|
|
i (int): The relative position of the token to get. Defaults to 1.
|
|
|
|
|
RETURNS (Token): The token at position `self.doc[self.i+i]`.
|
2019-03-08 10:42:26 +00:00
|
|
|
|
|
2020-09-04 10:58:50 +00:00
|
|
|
|
DOCS: https://nightly.spacy.io/api/token#nbor
|
2017-04-15 11:05:15 +00:00
|
|
|
|
"""
|
2017-10-24 10:10:39 +00:00
|
|
|
|
if self.i+i < 0 or (self.i+i >= len(self.doc)):
|
2018-04-03 13:50:31 +00:00
|
|
|
|
raise IndexError(Errors.E042.format(i=self.i, j=i, length=len(self.doc)))
|
2015-07-13 22:10:11 +00:00
|
|
|
|
return self.doc[self.i+i]
|
2015-07-13 17:20:48 +00:00
|
|
|
|
|
2015-09-14 07:49:58 +00:00
|
|
|
|
def similarity(self, other):
|
2017-05-19 16:47:56 +00:00
|
|
|
|
"""Make a semantic similarity estimate. The default estimate is cosine
|
|
|
|
|
similarity using an average of word vectors.
|
|
|
|
|
|
|
|
|
|
other (object): The object to compare with. By default, accepts `Doc`,
|
|
|
|
|
`Span`, `Token` and `Lexeme` objects.
|
|
|
|
|
RETURNS (float): A scalar similarity score. Higher is more similar.
|
2019-03-08 10:42:26 +00:00
|
|
|
|
|
2020-09-04 10:58:50 +00:00
|
|
|
|
DOCS: https://nightly.spacy.io/api/token#similarity
|
2017-04-15 11:05:15 +00:00
|
|
|
|
"""
|
2019-03-08 10:42:26 +00:00
|
|
|
|
if "similarity" in self.doc.user_token_hooks:
|
2019-07-27 13:26:01 +00:00
|
|
|
|
return self.doc.user_token_hooks["similarity"](self, other)
|
2019-03-08 10:42:26 +00:00
|
|
|
|
if hasattr(other, "__len__") and len(other) == 1 and hasattr(other, "__getitem__"):
|
|
|
|
|
if self.c.lex.orth == getattr(other[0], "orth", None):
|
2018-01-15 15:29:48 +00:00
|
|
|
|
return 1.0
|
2019-03-08 10:42:26 +00:00
|
|
|
|
elif hasattr(other, "orth"):
|
2018-01-15 15:29:48 +00:00
|
|
|
|
if self.c.lex.orth == other.orth:
|
|
|
|
|
return 1.0
|
2018-05-20 23:22:38 +00:00
|
|
|
|
if self.vocab.vectors.n_keys == 0:
|
2020-04-28 11:37:37 +00:00
|
|
|
|
warnings.warn(Warnings.W007.format(obj="Token"))
|
2015-09-22 00:10:01 +00:00
|
|
|
|
if self.vector_norm == 0 or other.vector_norm == 0:
|
2020-04-28 11:37:37 +00:00
|
|
|
|
warnings.warn(Warnings.W008.format(obj="Token"))
|
2015-09-22 00:10:01 +00:00
|
|
|
|
return 0.0
|
2019-03-06 22:58:38 +00:00
|
|
|
|
vector = self.vector
|
2019-03-06 23:56:31 +00:00
|
|
|
|
xp = get_array_module(vector)
|
2019-03-07 00:06:12 +00:00
|
|
|
|
return (xp.dot(vector, other.vector) / (self.vector_norm * other.vector_norm))
|
2015-09-14 07:49:58 +00:00
|
|
|
|
|
2019-03-16 12:44:22 +00:00
|
|
|
|
@property
|
|
|
|
|
def morph(self):
|
|
|
|
|
return MorphAnalysis.from_id(self.vocab, self.c.morph)
|
2019-03-07 16:14:57 +00:00
|
|
|
|
|
2020-01-23 21:01:54 +00:00
|
|
|
|
property morph_:
|
|
|
|
|
def __get__(self):
|
|
|
|
|
return str(MorphAnalysis.from_id(self.vocab, self.c.morph))
|
|
|
|
|
|
|
|
|
|
def __set__(self, features):
|
|
|
|
|
cdef hash_t key = self.vocab.morphology.add(features)
|
|
|
|
|
self.c.morph = key
|
|
|
|
|
|
2020-08-10 14:43:52 +00:00
|
|
|
|
@property
|
|
|
|
|
def lex(self):
|
|
|
|
|
"""RETURNS (Lexeme): The underlying lexeme."""
|
|
|
|
|
return self.vocab[self.c.lex.orth]
|
|
|
|
|
|
2019-03-11 14:59:09 +00:00
|
|
|
|
@property
|
|
|
|
|
def lex_id(self):
|
2017-10-27 15:07:26 +00:00
|
|
|
|
"""RETURNS (int): Sequential ID of the token's lexical type."""
|
2019-03-11 14:59:09 +00:00
|
|
|
|
return self.c.lex.id
|
2015-07-13 17:20:48 +00:00
|
|
|
|
|
2019-03-11 14:59:09 +00:00
|
|
|
|
@property
|
|
|
|
|
def rank(self):
|
2017-10-27 15:07:26 +00:00
|
|
|
|
"""RETURNS (int): Sequential ID of the token's lexical type, used to
|
|
|
|
|
index into tables, e.g. for word vectors."""
|
2019-03-11 14:59:09 +00:00
|
|
|
|
return self.c.lex.id
|
2015-11-08 15:18:25 +00:00
|
|
|
|
|
2019-03-11 14:59:09 +00:00
|
|
|
|
@property
|
|
|
|
|
def text(self):
|
2020-05-24 15:20:58 +00:00
|
|
|
|
"""RETURNS (str): The original verbatim text of the token."""
|
2019-03-11 14:59:09 +00:00
|
|
|
|
return self.orth_
|
2015-09-13 00:27:42 +00:00
|
|
|
|
|
2019-03-11 14:59:09 +00:00
|
|
|
|
@property
|
|
|
|
|
def text_with_ws(self):
|
2020-05-24 15:20:58 +00:00
|
|
|
|
"""RETURNS (str): The text content of the span (with trailing
|
2017-10-27 13:41:45 +00:00
|
|
|
|
whitespace).
|
2017-05-19 16:47:56 +00:00
|
|
|
|
"""
|
2019-03-11 14:59:09 +00:00
|
|
|
|
cdef unicode orth = self.vocab.strings[self.c.lex.orth]
|
|
|
|
|
if self.c.spacy:
|
|
|
|
|
return orth + " "
|
|
|
|
|
else:
|
|
|
|
|
return orth
|
2015-09-13 00:27:42 +00:00
|
|
|
|
|
2019-03-11 14:59:09 +00:00
|
|
|
|
@property
|
|
|
|
|
def prob(self):
|
2017-10-27 15:07:26 +00:00
|
|
|
|
"""RETURNS (float): Smoothed log probability estimate of token type."""
|
2020-05-19 13:59:14 +00:00
|
|
|
|
return self.vocab[self.c.lex.orth].prob
|
2015-07-13 17:20:48 +00:00
|
|
|
|
|
2019-03-11 14:59:09 +00:00
|
|
|
|
@property
|
|
|
|
|
def sentiment(self):
|
2017-10-27 15:07:26 +00:00
|
|
|
|
"""RETURNS (float): A scalar value indicating the positivity or
|
|
|
|
|
negativity of the token."""
|
2019-03-11 14:59:09 +00:00
|
|
|
|
if "sentiment" in self.doc.user_token_hooks:
|
|
|
|
|
return self.doc.user_token_hooks["sentiment"](self)
|
2020-05-19 13:59:14 +00:00
|
|
|
|
return self.vocab[self.c.lex.orth].sentiment
|
2016-10-19 18:54:03 +00:00
|
|
|
|
|
2019-03-11 14:59:09 +00:00
|
|
|
|
@property
|
|
|
|
|
def lang(self):
|
2017-10-27 15:07:26 +00:00
|
|
|
|
"""RETURNS (uint64): ID of the language of the parent document's
|
|
|
|
|
vocabulary.
|
|
|
|
|
"""
|
2019-03-11 14:59:09 +00:00
|
|
|
|
return self.c.lex.lang
|
2016-03-10 12:01:34 +00:00
|
|
|
|
|
2019-03-11 14:59:09 +00:00
|
|
|
|
@property
|
|
|
|
|
def idx(self):
|
2017-10-27 15:07:26 +00:00
|
|
|
|
"""RETURNS (int): The character offset of the token within the parent
|
|
|
|
|
document.
|
|
|
|
|
"""
|
2019-03-11 14:59:09 +00:00
|
|
|
|
return self.c.idx
|
2015-07-13 17:20:48 +00:00
|
|
|
|
|
2019-03-11 14:59:09 +00:00
|
|
|
|
@property
|
|
|
|
|
def cluster(self):
|
2017-10-27 15:07:26 +00:00
|
|
|
|
"""RETURNS (int): Brown cluster ID."""
|
2020-05-19 13:59:14 +00:00
|
|
|
|
return self.vocab[self.c.lex.orth].cluster
|
2015-07-13 17:20:48 +00:00
|
|
|
|
|
2019-03-11 14:59:09 +00:00
|
|
|
|
@property
|
|
|
|
|
def orth(self):
|
2017-10-27 15:07:26 +00:00
|
|
|
|
"""RETURNS (uint64): ID of the verbatim text content."""
|
2019-03-11 14:59:09 +00:00
|
|
|
|
return self.c.lex.orth
|
2015-07-13 17:20:48 +00:00
|
|
|
|
|
2019-03-11 14:59:09 +00:00
|
|
|
|
@property
|
|
|
|
|
def lower(self):
|
2017-10-27 15:07:26 +00:00
|
|
|
|
"""RETURNS (uint64): ID of the lowercase token text."""
|
2019-03-11 14:59:09 +00:00
|
|
|
|
return self.c.lex.lower
|
2015-07-13 17:20:48 +00:00
|
|
|
|
|
2019-03-11 14:59:09 +00:00
|
|
|
|
@property
|
|
|
|
|
def norm(self):
|
2017-10-27 15:07:26 +00:00
|
|
|
|
"""RETURNS (uint64): ID of the token's norm, i.e. a normalised form of
|
|
|
|
|
the token text. Usually set in the language's tokenizer exceptions
|
|
|
|
|
or norm exceptions.
|
|
|
|
|
"""
|
2019-03-11 14:59:09 +00:00
|
|
|
|
if self.c.norm == 0:
|
|
|
|
|
return self.c.lex.norm
|
|
|
|
|
else:
|
|
|
|
|
return self.c.norm
|
2015-07-13 17:20:48 +00:00
|
|
|
|
|
2019-03-11 14:59:09 +00:00
|
|
|
|
@property
|
|
|
|
|
def shape(self):
|
2017-10-27 15:07:26 +00:00
|
|
|
|
"""RETURNS (uint64): ID of the token's shape, a transform of the
|
|
|
|
|
tokens's string, to show orthographic features (e.g. "Xxxx", "dd").
|
|
|
|
|
"""
|
2019-03-11 14:59:09 +00:00
|
|
|
|
return self.c.lex.shape
|
2015-07-13 17:20:48 +00:00
|
|
|
|
|
2019-03-11 14:59:09 +00:00
|
|
|
|
@property
|
|
|
|
|
def prefix(self):
|
2017-10-27 15:07:26 +00:00
|
|
|
|
"""RETURNS (uint64): ID of a length-N substring from the start of the
|
|
|
|
|
token. Defaults to `N=1`.
|
|
|
|
|
"""
|
2019-03-11 14:59:09 +00:00
|
|
|
|
return self.c.lex.prefix
|
2015-07-13 17:20:48 +00:00
|
|
|
|
|
2019-03-11 14:59:09 +00:00
|
|
|
|
@property
|
|
|
|
|
def suffix(self):
|
2017-10-27 15:07:26 +00:00
|
|
|
|
"""RETURNS (uint64): ID of a length-N substring from the end of the
|
|
|
|
|
token. Defaults to `N=3`.
|
|
|
|
|
"""
|
2019-03-11 14:59:09 +00:00
|
|
|
|
return self.c.lex.suffix
|
2015-07-13 17:20:48 +00:00
|
|
|
|
|
|
|
|
|
property lemma:
|
2017-10-27 15:07:26 +00:00
|
|
|
|
"""RETURNS (uint64): ID of the base form of the word, with no
|
|
|
|
|
inflectional suffixes.
|
2017-05-19 16:47:56 +00:00
|
|
|
|
"""
|
2015-07-13 17:20:48 +00:00
|
|
|
|
def __get__(self):
|
2020-08-07 13:27:13 +00:00
|
|
|
|
return self.c.lemma
|
2017-10-27 15:07:26 +00:00
|
|
|
|
|
2017-05-28 13:10:22 +00:00
|
|
|
|
def __set__(self, attr_t lemma):
|
2017-04-16 16:07:53 +00:00
|
|
|
|
self.c.lemma = lemma
|
2015-07-13 17:20:48 +00:00
|
|
|
|
|
|
|
|
|
property pos:
|
2017-10-27 15:07:26 +00:00
|
|
|
|
"""RETURNS (uint64): ID of coarse-grained part-of-speech tag."""
|
2015-07-13 17:20:48 +00:00
|
|
|
|
def __get__(self):
|
|
|
|
|
return self.c.pos
|
2019-03-08 10:42:26 +00:00
|
|
|
|
|
2018-03-27 19:21:11 +00:00
|
|
|
|
def __set__(self, pos):
|
|
|
|
|
self.c.pos = pos
|
2015-07-13 17:20:48 +00:00
|
|
|
|
|
|
|
|
|
property tag:
|
2017-10-27 15:07:26 +00:00
|
|
|
|
"""RETURNS (uint64): ID of fine-grained part-of-speech tag."""
|
2015-07-13 17:20:48 +00:00
|
|
|
|
def __get__(self):
|
|
|
|
|
return self.c.tag
|
2017-10-27 15:07:26 +00:00
|
|
|
|
|
2017-05-28 13:10:22 +00:00
|
|
|
|
def __set__(self, attr_t tag):
|
2020-08-07 13:27:13 +00:00
|
|
|
|
self.c.tag = tag
|
2015-07-13 17:20:48 +00:00
|
|
|
|
|
|
|
|
|
property dep:
|
2017-10-27 15:07:26 +00:00
|
|
|
|
"""RETURNS (uint64): ID of syntactic dependency label."""
|
2015-07-13 17:20:48 +00:00
|
|
|
|
def __get__(self):
|
|
|
|
|
return self.c.dep
|
2017-10-27 15:07:26 +00:00
|
|
|
|
|
2017-05-28 13:10:22 +00:00
|
|
|
|
def __set__(self, attr_t label):
|
2016-03-11 16:31:06 +00:00
|
|
|
|
self.c.dep = label
|
2015-07-13 17:20:48 +00:00
|
|
|
|
|
2019-03-11 14:59:09 +00:00
|
|
|
|
@property
|
|
|
|
|
def has_vector(self):
|
2017-05-19 16:47:56 +00:00
|
|
|
|
"""A boolean value indicating whether a word vector is associated with
|
|
|
|
|
the object.
|
|
|
|
|
|
|
|
|
|
RETURNS (bool): Whether a word vector is associated with the object.
|
2019-03-08 10:42:26 +00:00
|
|
|
|
|
2020-09-04 10:58:50 +00:00
|
|
|
|
DOCS: https://nightly.spacy.io/api/token#has_vector
|
2017-04-15 11:05:15 +00:00
|
|
|
|
"""
|
2019-03-11 14:59:09 +00:00
|
|
|
|
if "has_vector" in self.doc.user_token_hooks:
|
|
|
|
|
return self.doc.user_token_hooks["has_vector"](self)
|
|
|
|
|
if self.vocab.vectors.size == 0 and self.doc.tensor.size != 0:
|
|
|
|
|
return True
|
|
|
|
|
return self.vocab.has_vector(self.c.lex.orth)
|
2015-09-21 09:52:43 +00:00
|
|
|
|
|
2019-03-11 14:59:09 +00:00
|
|
|
|
@property
|
|
|
|
|
def vector(self):
|
2017-05-19 16:47:56 +00:00
|
|
|
|
"""A real-valued meaning representation.
|
2017-02-26 21:27:11 +00:00
|
|
|
|
|
2017-05-19 16:47:56 +00:00
|
|
|
|
RETURNS (numpy.ndarray[ndim=1, dtype='float32']): A 1D numpy array
|
|
|
|
|
representing the token's semantics.
|
2019-03-08 10:42:26 +00:00
|
|
|
|
|
2020-09-04 10:58:50 +00:00
|
|
|
|
DOCS: https://nightly.spacy.io/api/token#vector
|
2017-04-15 11:05:15 +00:00
|
|
|
|
"""
|
2019-03-11 14:59:09 +00:00
|
|
|
|
if "vector" in self.doc.user_token_hooks:
|
|
|
|
|
return self.doc.user_token_hooks["vector"](self)
|
|
|
|
|
if self.vocab.vectors.size == 0 and self.doc.tensor.size != 0:
|
|
|
|
|
return self.doc.tensor[self.i]
|
|
|
|
|
else:
|
|
|
|
|
return self.vocab.get_vector(self.c.lex.orth)
|
2015-07-13 17:20:48 +00:00
|
|
|
|
|
2019-03-11 14:59:09 +00:00
|
|
|
|
@property
|
|
|
|
|
def vector_norm(self):
|
2017-05-20 13:13:33 +00:00
|
|
|
|
"""The L2 norm of the token's vector representation.
|
2017-05-19 16:47:56 +00:00
|
|
|
|
|
|
|
|
|
RETURNS (float): The L2 norm of the vector representation.
|
2019-03-08 10:42:26 +00:00
|
|
|
|
|
2020-09-04 10:58:50 +00:00
|
|
|
|
DOCS: https://nightly.spacy.io/api/token#vector_norm
|
2017-05-19 16:47:56 +00:00
|
|
|
|
"""
|
2019-03-11 14:59:09 +00:00
|
|
|
|
if "vector_norm" in self.doc.user_token_hooks:
|
|
|
|
|
return self.doc.user_token_hooks["vector_norm"](self)
|
|
|
|
|
vector = self.vector
|
2019-03-20 11:09:59 +00:00
|
|
|
|
xp = get_array_module(vector)
|
|
|
|
|
total = (vector ** 2).sum()
|
|
|
|
|
return xp.sqrt(total) if total != 0. else 0.
|
2015-09-14 07:49:58 +00:00
|
|
|
|
|
2019-08-01 16:30:50 +00:00
|
|
|
|
@property
|
|
|
|
|
def tensor(self):
|
|
|
|
|
if self.doc.tensor is None:
|
|
|
|
|
return None
|
|
|
|
|
return self.doc.tensor[self.i]
|
2015-09-14 07:49:58 +00:00
|
|
|
|
|
2019-03-11 14:59:09 +00:00
|
|
|
|
@property
|
|
|
|
|
def n_lefts(self):
|
2019-03-08 10:42:26 +00:00
|
|
|
|
"""The number of leftward immediate children of the word, in the
|
|
|
|
|
syntactic dependency parse.
|
|
|
|
|
|
|
|
|
|
RETURNS (int): The number of leftward immediate children of the
|
2017-10-27 15:07:26 +00:00
|
|
|
|
word, in the syntactic dependency parse.
|
2019-03-08 10:42:26 +00:00
|
|
|
|
|
2020-09-04 10:58:50 +00:00
|
|
|
|
DOCS: https://nightly.spacy.io/api/token#n_lefts
|
2017-10-27 15:07:26 +00:00
|
|
|
|
"""
|
2019-03-11 14:59:09 +00:00
|
|
|
|
return self.c.l_kids
|
2015-07-13 17:20:48 +00:00
|
|
|
|
|
2019-03-11 14:59:09 +00:00
|
|
|
|
@property
|
|
|
|
|
def n_rights(self):
|
2019-03-08 10:42:26 +00:00
|
|
|
|
"""The number of rightward immediate children of the word, in the
|
|
|
|
|
syntactic dependency parse.
|
|
|
|
|
|
|
|
|
|
RETURNS (int): The number of rightward immediate children of the
|
2017-10-27 15:07:26 +00:00
|
|
|
|
word, in the syntactic dependency parse.
|
2019-03-08 10:42:26 +00:00
|
|
|
|
|
2020-09-04 10:58:50 +00:00
|
|
|
|
DOCS: https://nightly.spacy.io/api/token#n_rights
|
2017-10-27 15:07:26 +00:00
|
|
|
|
"""
|
2019-03-11 14:59:09 +00:00
|
|
|
|
return self.c.r_kids
|
2015-07-13 17:20:48 +00:00
|
|
|
|
|
2019-03-11 14:59:09 +00:00
|
|
|
|
@property
|
|
|
|
|
def sent(self):
|
2018-07-06 13:54:15 +00:00
|
|
|
|
"""RETURNS (Span): The sentence span that the token is a part of."""
|
2019-03-11 14:59:09 +00:00
|
|
|
|
if 'sent' in self.doc.user_token_hooks:
|
|
|
|
|
return self.doc.user_token_hooks["sent"](self)
|
|
|
|
|
return self.doc[self.i : self.i+1].sent
|
2018-07-06 13:54:15 +00:00
|
|
|
|
|
2016-05-05 09:53:20 +00:00
|
|
|
|
property sent_start:
|
|
|
|
|
def __get__(self):
|
2019-03-23 14:45:02 +00:00
|
|
|
|
"""Deprecated: use Token.is_sent_start instead."""
|
2018-04-03 13:50:31 +00:00
|
|
|
|
# Raising a deprecation warning here causes errors for autocomplete
|
2017-11-01 12:27:14 +00:00
|
|
|
|
# Handle broken backwards compatibility case: doc[0].sent_start
|
|
|
|
|
# was False.
|
|
|
|
|
if self.i == 0:
|
|
|
|
|
return False
|
|
|
|
|
else:
|
2018-01-14 14:02:15 +00:00
|
|
|
|
return self.c.sent_start
|
2017-11-01 12:27:14 +00:00
|
|
|
|
|
|
|
|
|
def __set__(self, value):
|
|
|
|
|
self.is_sent_start = value
|
|
|
|
|
|
|
|
|
|
property is_sent_start:
|
2019-03-08 10:42:26 +00:00
|
|
|
|
"""A boolean value indicating whether the token starts a sentence.
|
|
|
|
|
`None` if unknown. Defaults to `True` for the first token in the `Doc`.
|
|
|
|
|
|
|
|
|
|
RETURNS (bool / None): Whether the token starts a sentence.
|
2017-11-01 12:27:14 +00:00
|
|
|
|
None if unknown.
|
2019-03-08 10:42:26 +00:00
|
|
|
|
|
2020-09-04 10:58:50 +00:00
|
|
|
|
DOCS: https://nightly.spacy.io/api/token#is_sent_start
|
2017-11-01 12:27:14 +00:00
|
|
|
|
"""
|
|
|
|
|
def __get__(self):
|
|
|
|
|
if self.c.sent_start == 0:
|
|
|
|
|
return None
|
|
|
|
|
elif self.c.sent_start < 0:
|
|
|
|
|
return False
|
|
|
|
|
else:
|
|
|
|
|
return True
|
2016-05-05 09:53:20 +00:00
|
|
|
|
|
2017-10-08 21:51:58 +00:00
|
|
|
|
def __set__(self, value):
|
2016-05-05 09:53:20 +00:00
|
|
|
|
if self.doc.is_parsed:
|
2018-04-03 13:50:31 +00:00
|
|
|
|
raise ValueError(Errors.E043)
|
2017-10-08 21:51:58 +00:00
|
|
|
|
if value is None:
|
|
|
|
|
self.c.sent_start = 0
|
|
|
|
|
elif value is True:
|
|
|
|
|
self.c.sent_start = 1
|
|
|
|
|
elif value is False:
|
|
|
|
|
self.c.sent_start = -1
|
|
|
|
|
else:
|
2018-04-03 13:50:31 +00:00
|
|
|
|
raise ValueError(Errors.E044.format(value=value))
|
2016-05-05 09:53:20 +00:00
|
|
|
|
|
2020-04-29 10:53:16 +00:00
|
|
|
|
property is_sent_end:
|
|
|
|
|
"""A boolean value indicating whether the token ends a sentence.
|
|
|
|
|
`None` if unknown. Defaults to `True` for the last token in the `Doc`.
|
|
|
|
|
|
|
|
|
|
RETURNS (bool / None): Whether the token ends a sentence.
|
|
|
|
|
None if unknown.
|
|
|
|
|
|
2020-09-04 10:58:50 +00:00
|
|
|
|
DOCS: https://nightly.spacy.io/api/token#is_sent_end
|
2020-04-29 10:53:16 +00:00
|
|
|
|
"""
|
|
|
|
|
def __get__(self):
|
|
|
|
|
if self.i + 1 == len(self.doc):
|
|
|
|
|
return True
|
|
|
|
|
elif self.doc[self.i+1].is_sent_start == None:
|
|
|
|
|
return None
|
|
|
|
|
elif self.doc[self.i+1].is_sent_start == True:
|
|
|
|
|
return True
|
|
|
|
|
else:
|
|
|
|
|
return False
|
|
|
|
|
|
|
|
|
|
def __set__(self, value):
|
|
|
|
|
raise ValueError(Errors.E196)
|
|
|
|
|
|
2019-03-11 14:59:09 +00:00
|
|
|
|
@property
|
|
|
|
|
def lefts(self):
|
2017-10-27 15:07:26 +00:00
|
|
|
|
"""The leftward immediate children of the word, in the syntactic
|
|
|
|
|
dependency parse.
|
|
|
|
|
|
|
|
|
|
YIELDS (Token): A left-child of the token.
|
2019-03-08 10:42:26 +00:00
|
|
|
|
|
2020-09-04 10:58:50 +00:00
|
|
|
|
DOCS: https://nightly.spacy.io/api/token#lefts
|
2017-10-27 15:07:26 +00:00
|
|
|
|
"""
|
2019-03-11 14:59:09 +00:00
|
|
|
|
cdef int nr_iter = 0
|
|
|
|
|
cdef const TokenC* ptr = self.c - (self.i - self.c.l_edge)
|
|
|
|
|
while ptr < self.c:
|
|
|
|
|
if ptr + ptr.head == self.c:
|
|
|
|
|
yield self.doc[ptr - (self.c - self.i)]
|
|
|
|
|
ptr += 1
|
|
|
|
|
nr_iter += 1
|
|
|
|
|
# This is ugly, but it's a way to guard out infinite loops
|
|
|
|
|
if nr_iter >= 10000000:
|
|
|
|
|
raise RuntimeError(Errors.E045.format(attr="token.lefts"))
|
|
|
|
|
|
|
|
|
|
@property
|
|
|
|
|
def rights(self):
|
2017-10-27 15:07:26 +00:00
|
|
|
|
"""The rightward immediate children of the word, in the syntactic
|
|
|
|
|
dependency parse.
|
|
|
|
|
|
|
|
|
|
YIELDS (Token): A right-child of the token.
|
2019-03-08 10:42:26 +00:00
|
|
|
|
|
2020-09-04 10:58:50 +00:00
|
|
|
|
DOCS: https://nightly.spacy.io/api/token#rights
|
2017-10-27 15:07:26 +00:00
|
|
|
|
"""
|
2019-03-11 14:59:09 +00:00
|
|
|
|
cdef const TokenC* ptr = self.c + (self.c.r_edge - self.i)
|
|
|
|
|
tokens = []
|
|
|
|
|
cdef int nr_iter = 0
|
|
|
|
|
while ptr > self.c:
|
|
|
|
|
if ptr + ptr.head == self.c:
|
|
|
|
|
tokens.append(self.doc[ptr - (self.c - self.i)])
|
|
|
|
|
ptr -= 1
|
|
|
|
|
nr_iter += 1
|
|
|
|
|
if nr_iter >= 10000000:
|
|
|
|
|
raise RuntimeError(Errors.E045.format(attr="token.rights"))
|
|
|
|
|
tokens.reverse()
|
|
|
|
|
for t in tokens:
|
|
|
|
|
yield t
|
|
|
|
|
|
|
|
|
|
@property
|
|
|
|
|
def children(self):
|
2017-10-27 13:41:45 +00:00
|
|
|
|
"""A sequence of the token's immediate syntactic children.
|
2016-11-01 11:25:36 +00:00
|
|
|
|
|
2019-03-08 10:42:26 +00:00
|
|
|
|
YIELDS (Token): A child token such that `child.head==self`.
|
|
|
|
|
|
2020-09-04 10:58:50 +00:00
|
|
|
|
DOCS: https://nightly.spacy.io/api/token#children
|
2017-04-15 11:05:15 +00:00
|
|
|
|
"""
|
2019-03-11 14:59:09 +00:00
|
|
|
|
yield from self.lefts
|
|
|
|
|
yield from self.rights
|
2015-07-13 17:20:48 +00:00
|
|
|
|
|
2019-03-11 14:59:09 +00:00
|
|
|
|
@property
|
|
|
|
|
def subtree(self):
|
2019-01-09 02:11:15 +00:00
|
|
|
|
"""A sequence containing the token and all the token's syntactic
|
|
|
|
|
descendants.
|
2016-11-01 11:25:36 +00:00
|
|
|
|
|
2017-10-27 13:41:45 +00:00
|
|
|
|
YIELDS (Token): A descendent token such that
|
2019-01-09 02:11:15 +00:00
|
|
|
|
`self.is_ancestor(descendent) or token == self`.
|
2019-03-08 10:42:26 +00:00
|
|
|
|
|
2020-09-04 10:58:50 +00:00
|
|
|
|
DOCS: https://nightly.spacy.io/api/token#subtree
|
2017-04-15 11:05:15 +00:00
|
|
|
|
"""
|
2019-03-11 14:59:09 +00:00
|
|
|
|
for word in self.lefts:
|
|
|
|
|
yield from word.subtree
|
|
|
|
|
yield self
|
|
|
|
|
for word in self.rights:
|
|
|
|
|
yield from word.subtree
|
2015-07-13 17:20:48 +00:00
|
|
|
|
|
2019-03-11 14:59:09 +00:00
|
|
|
|
@property
|
|
|
|
|
def left_edge(self):
|
2017-05-19 16:47:56 +00:00
|
|
|
|
"""The leftmost token of this token's syntactic descendents.
|
2016-11-01 11:25:36 +00:00
|
|
|
|
|
2017-05-19 16:47:56 +00:00
|
|
|
|
RETURNS (Token): The first token such that `self.is_ancestor(token)`.
|
2017-04-15 11:05:15 +00:00
|
|
|
|
"""
|
2019-03-11 14:59:09 +00:00
|
|
|
|
return self.doc[self.c.l_edge]
|
2015-08-08 21:37:44 +00:00
|
|
|
|
|
2019-03-11 14:59:09 +00:00
|
|
|
|
@property
|
|
|
|
|
def right_edge(self):
|
2017-05-19 16:47:56 +00:00
|
|
|
|
"""The rightmost token of this token's syntactic descendents.
|
2016-11-01 11:25:36 +00:00
|
|
|
|
|
2017-05-19 16:47:56 +00:00
|
|
|
|
RETURNS (Token): The last token such that `self.is_ancestor(token)`.
|
2017-04-15 11:05:15 +00:00
|
|
|
|
"""
|
2019-03-11 14:59:09 +00:00
|
|
|
|
return self.doc[self.c.r_edge]
|
2015-07-13 17:20:48 +00:00
|
|
|
|
|
2019-03-11 14:59:09 +00:00
|
|
|
|
@property
|
|
|
|
|
def ancestors(self):
|
2017-05-19 16:47:56 +00:00
|
|
|
|
"""A sequence of this token's syntactic ancestors.
|
2016-11-01 11:25:36 +00:00
|
|
|
|
|
2017-05-19 16:47:56 +00:00
|
|
|
|
YIELDS (Token): A sequence of ancestor tokens such that
|
|
|
|
|
`ancestor.is_ancestor(self)`.
|
2019-03-08 10:42:26 +00:00
|
|
|
|
|
2020-09-04 10:58:50 +00:00
|
|
|
|
DOCS: https://nightly.spacy.io/api/token#ancestors
|
2017-04-15 11:05:15 +00:00
|
|
|
|
"""
|
2019-03-11 14:59:09 +00:00
|
|
|
|
cdef const TokenC* head_ptr = self.c
|
|
|
|
|
# Guard against infinite loop, no token can have
|
|
|
|
|
# more ancestors than tokens in the tree.
|
|
|
|
|
cdef int i = 0
|
|
|
|
|
while head_ptr.head != 0 and i < self.doc.length:
|
|
|
|
|
head_ptr += head_ptr.head
|
|
|
|
|
yield self.doc[head_ptr - (self.c - self.i)]
|
|
|
|
|
i += 1
|
2016-03-11 16:31:06 +00:00
|
|
|
|
|
2016-11-01 11:25:36 +00:00
|
|
|
|
def is_ancestor(self, descendant):
|
2017-05-19 16:47:56 +00:00
|
|
|
|
"""Check whether this token is a parent, grandparent, etc. of another
|
2016-11-01 11:25:36 +00:00
|
|
|
|
in the dependency tree.
|
|
|
|
|
|
2017-05-19 16:47:56 +00:00
|
|
|
|
descendant (Token): Another token.
|
|
|
|
|
RETURNS (bool): Whether this token is the ancestor of the descendant.
|
2019-03-08 10:42:26 +00:00
|
|
|
|
|
2020-09-04 10:58:50 +00:00
|
|
|
|
DOCS: https://nightly.spacy.io/api/token#is_ancestor
|
2017-04-15 11:05:15 +00:00
|
|
|
|
"""
|
2016-11-01 12:28:00 +00:00
|
|
|
|
if self.doc is not descendant.doc:
|
2016-11-01 11:25:36 +00:00
|
|
|
|
return False
|
2017-10-27 15:07:26 +00:00
|
|
|
|
return any(ancestor.i == self.i for ancestor in descendant.ancestors)
|
2016-03-11 16:31:06 +00:00
|
|
|
|
|
2015-07-13 17:20:48 +00:00
|
|
|
|
property head:
|
2017-05-19 16:47:56 +00:00
|
|
|
|
"""The syntactic parent, or "governor", of this token.
|
2017-02-26 21:27:11 +00:00
|
|
|
|
|
2017-10-27 15:07:26 +00:00
|
|
|
|
RETURNS (Token): The token predicted by the parser to be the head of
|
|
|
|
|
the current token.
|
2017-04-15 11:05:15 +00:00
|
|
|
|
"""
|
2015-07-13 17:20:48 +00:00
|
|
|
|
def __get__(self):
|
2015-07-13 22:10:11 +00:00
|
|
|
|
return self.doc[self.i + self.c.head]
|
2017-10-27 15:07:26 +00:00
|
|
|
|
|
2016-03-11 16:31:06 +00:00
|
|
|
|
def __set__(self, Token new_head):
|
2019-03-08 10:42:26 +00:00
|
|
|
|
# This function sets the head of self to new_head and updates the
|
|
|
|
|
# counters for left/right dependents and left/right corner for the
|
|
|
|
|
# new and the old head
|
2020-03-03 20:44:51 +00:00
|
|
|
|
# Check that token is from the same document
|
|
|
|
|
if self.doc != new_head.doc:
|
|
|
|
|
raise ValueError(Errors.E191)
|
2019-03-08 10:42:26 +00:00
|
|
|
|
# Do nothing if old head is new head
|
2016-03-11 16:31:06 +00:00
|
|
|
|
if self.i + self.c.head == new_head.i:
|
|
|
|
|
return
|
|
|
|
|
cdef Token old_head = self.head
|
|
|
|
|
cdef int rel_newhead_i = new_head.i - self.i
|
2019-03-08 10:42:26 +00:00
|
|
|
|
# Is the new head a descendant of the old head
|
2017-05-19 18:23:40 +00:00
|
|
|
|
cdef bint is_desc = old_head.is_ancestor(new_head)
|
2016-03-11 16:31:06 +00:00
|
|
|
|
cdef int new_edge
|
2016-03-14 12:43:48 +00:00
|
|
|
|
cdef Token anc, child
|
2019-03-08 10:42:26 +00:00
|
|
|
|
# Update number of deps of old head
|
2017-10-27 15:07:26 +00:00
|
|
|
|
if self.c.head > 0: # left dependent
|
2016-03-11 16:31:06 +00:00
|
|
|
|
old_head.c.l_kids -= 1
|
|
|
|
|
if self.c.l_edge == old_head.c.l_edge:
|
2019-03-08 10:42:26 +00:00
|
|
|
|
# The token dominates the left edge so the left edge of
|
|
|
|
|
# the head may change when the token is reattached, it may
|
2017-10-27 13:41:45 +00:00
|
|
|
|
# not change if the new head is a descendant of the current
|
2019-03-08 10:42:26 +00:00
|
|
|
|
# head.
|
2016-03-11 16:31:06 +00:00
|
|
|
|
new_edge = self.c.l_edge
|
2019-03-08 10:42:26 +00:00
|
|
|
|
# The new l_edge is the left-most l_edge on any of the
|
2017-10-27 13:41:45 +00:00
|
|
|
|
# other dependents where the l_edge is left of the head,
|
|
|
|
|
# otherwise it is the head
|
2016-03-11 16:31:06 +00:00
|
|
|
|
if not is_desc:
|
2016-03-14 12:43:48 +00:00
|
|
|
|
new_edge = old_head.i
|
|
|
|
|
for child in old_head.children:
|
|
|
|
|
if child == self:
|
|
|
|
|
continue
|
|
|
|
|
if child.c.l_edge < new_edge:
|
|
|
|
|
new_edge = child.c.l_edge
|
|
|
|
|
old_head.c.l_edge = new_edge
|
2019-03-08 10:42:26 +00:00
|
|
|
|
# Walk up the tree from old_head and assign new l_edge to
|
2017-10-27 13:41:45 +00:00
|
|
|
|
# ancestors until an ancestor already has an l_edge that's
|
|
|
|
|
# further left
|
2016-03-11 16:31:06 +00:00
|
|
|
|
for anc in old_head.ancestors:
|
|
|
|
|
if anc.c.l_edge <= new_edge:
|
|
|
|
|
break
|
|
|
|
|
anc.c.l_edge = new_edge
|
2017-10-27 13:41:45 +00:00
|
|
|
|
elif self.c.head < 0: # right dependent
|
2016-03-11 16:31:06 +00:00
|
|
|
|
old_head.c.r_kids -= 1
|
2019-03-08 10:42:26 +00:00
|
|
|
|
# Do the same thing as for l_edge
|
2016-03-11 16:31:06 +00:00
|
|
|
|
if self.c.r_edge == old_head.c.r_edge:
|
|
|
|
|
new_edge = self.c.r_edge
|
|
|
|
|
if not is_desc:
|
2016-03-14 12:43:48 +00:00
|
|
|
|
new_edge = old_head.i
|
|
|
|
|
for child in old_head.children:
|
|
|
|
|
if child == self:
|
|
|
|
|
continue
|
|
|
|
|
if child.c.r_edge > new_edge:
|
|
|
|
|
new_edge = child.c.r_edge
|
|
|
|
|
old_head.c.r_edge = new_edge
|
2016-03-11 16:31:06 +00:00
|
|
|
|
for anc in old_head.ancestors:
|
|
|
|
|
if anc.c.r_edge >= new_edge:
|
|
|
|
|
break
|
|
|
|
|
anc.c.r_edge = new_edge
|
2019-03-08 10:42:26 +00:00
|
|
|
|
# Update number of deps of new head
|
2017-10-27 13:41:45 +00:00
|
|
|
|
if rel_newhead_i > 0: # left dependent
|
2016-03-11 16:31:06 +00:00
|
|
|
|
new_head.c.l_kids += 1
|
2019-03-08 10:42:26 +00:00
|
|
|
|
# Walk up the tree from new head and set l_edge to self.l_edge
|
2016-03-11 16:31:06 +00:00
|
|
|
|
# until you hit a token with an l_edge further to the left
|
|
|
|
|
if self.c.l_edge < new_head.c.l_edge:
|
2016-03-14 12:43:48 +00:00
|
|
|
|
new_head.c.l_edge = self.c.l_edge
|
2016-03-11 16:31:06 +00:00
|
|
|
|
for anc in new_head.ancestors:
|
2016-03-14 12:43:48 +00:00
|
|
|
|
if anc.c.l_edge <= self.c.l_edge:
|
2016-03-11 16:31:06 +00:00
|
|
|
|
break
|
2016-03-14 12:43:48 +00:00
|
|
|
|
anc.c.l_edge = self.c.l_edge
|
2017-10-27 13:41:45 +00:00
|
|
|
|
elif rel_newhead_i < 0: # right dependent
|
2016-03-11 16:31:06 +00:00
|
|
|
|
new_head.c.r_kids += 1
|
2019-03-08 10:42:26 +00:00
|
|
|
|
# Do the same as for l_edge
|
2016-03-11 16:31:06 +00:00
|
|
|
|
if self.c.r_edge > new_head.c.r_edge:
|
2016-03-14 12:43:48 +00:00
|
|
|
|
new_head.c.r_edge = self.c.r_edge
|
2016-03-11 16:31:06 +00:00
|
|
|
|
for anc in new_head.ancestors:
|
2016-03-14 12:43:48 +00:00
|
|
|
|
if anc.c.r_edge >= self.c.r_edge:
|
2016-03-11 16:31:06 +00:00
|
|
|
|
break
|
2016-03-14 12:43:48 +00:00
|
|
|
|
anc.c.r_edge = self.c.r_edge
|
2019-03-08 10:42:26 +00:00
|
|
|
|
# Set new head
|
2016-03-11 16:31:06 +00:00
|
|
|
|
self.c.head = rel_newhead_i
|
2015-08-08 21:37:44 +00:00
|
|
|
|
|
2019-03-11 14:59:09 +00:00
|
|
|
|
@property
|
|
|
|
|
def conjuncts(self):
|
2017-05-19 16:47:56 +00:00
|
|
|
|
"""A sequence of coordinated tokens, including the token itself.
|
2016-11-01 11:25:36 +00:00
|
|
|
|
|
2019-03-11 16:05:45 +00:00
|
|
|
|
RETURNS (tuple): The coordinated tokens.
|
2019-03-08 10:42:26 +00:00
|
|
|
|
|
2020-09-04 10:58:50 +00:00
|
|
|
|
DOCS: https://nightly.spacy.io/api/token#conjuncts
|
2017-04-15 11:05:15 +00:00
|
|
|
|
"""
|
2019-03-11 16:05:45 +00:00
|
|
|
|
cdef Token word, child
|
2019-03-11 14:59:09 +00:00
|
|
|
|
if "conjuncts" in self.doc.user_token_hooks:
|
2019-03-11 16:05:45 +00:00
|
|
|
|
return tuple(self.doc.user_token_hooks["conjuncts"](self))
|
|
|
|
|
start = self
|
|
|
|
|
while start.i != start.head.i:
|
|
|
|
|
if start.dep == conj:
|
|
|
|
|
start = start.head
|
2016-10-17 00:44:49 +00:00
|
|
|
|
else:
|
2019-03-11 16:05:45 +00:00
|
|
|
|
break
|
|
|
|
|
queue = [start]
|
|
|
|
|
output = [start]
|
|
|
|
|
for word in queue:
|
|
|
|
|
for child in word.rights:
|
|
|
|
|
if child.c.dep == conj:
|
|
|
|
|
output.append(child)
|
|
|
|
|
queue.append(child)
|
|
|
|
|
return tuple([w for w in output if w.i != self.i])
|
2015-07-13 17:20:48 +00:00
|
|
|
|
|
|
|
|
|
property ent_type:
|
2017-10-27 15:07:26 +00:00
|
|
|
|
"""RETURNS (uint64): Named entity type."""
|
2015-07-13 17:20:48 +00:00
|
|
|
|
def __get__(self):
|
|
|
|
|
return self.c.ent_type
|
2017-10-27 15:07:26 +00:00
|
|
|
|
|
2017-05-28 13:10:22 +00:00
|
|
|
|
def __set__(self, ent_type):
|
|
|
|
|
self.c.ent_type = ent_type
|
2015-07-13 17:20:48 +00:00
|
|
|
|
|
|
|
|
|
property ent_type_:
|
2020-05-24 15:20:58 +00:00
|
|
|
|
"""RETURNS (str): Named entity type."""
|
2015-07-13 17:20:48 +00:00
|
|
|
|
def __get__(self):
|
2016-09-30 18:20:22 +00:00
|
|
|
|
return self.vocab.strings[self.c.ent_type]
|
2017-10-27 15:07:26 +00:00
|
|
|
|
|
2017-05-28 13:10:22 +00:00
|
|
|
|
def __set__(self, ent_type):
|
|
|
|
|
self.c.ent_type = self.vocab.strings.add(ent_type)
|
2015-07-13 17:20:48 +00:00
|
|
|
|
|
2019-03-11 14:59:09 +00:00
|
|
|
|
@property
|
|
|
|
|
def ent_iob(self):
|
|
|
|
|
"""IOB code of named entity tag. `1="I", 2="O", 3="B"`. 0 means no tag
|
|
|
|
|
is assigned.
|
|
|
|
|
|
|
|
|
|
RETURNS (uint64): IOB code of named entity tag.
|
|
|
|
|
"""
|
|
|
|
|
return self.c.ent_iob
|
|
|
|
|
|
2020-06-26 17:34:12 +00:00
|
|
|
|
@classmethod
|
|
|
|
|
def iob_strings(cls):
|
|
|
|
|
return ("", "I", "O", "B")
|
|
|
|
|
|
2019-03-11 14:59:09 +00:00
|
|
|
|
@property
|
|
|
|
|
def ent_iob_(self):
|
2017-05-19 16:47:56 +00:00
|
|
|
|
"""IOB code of named entity tag. "B" means the token begins an entity,
|
2017-10-27 13:41:45 +00:00
|
|
|
|
"I" means it is inside an entity, "O" means it is outside an entity,
|
2019-09-18 19:37:17 +00:00
|
|
|
|
and "" means no entity tag is set. "B" with an empty ent_type
|
|
|
|
|
means that the token is blocked from further processing by NER.
|
2017-05-19 16:47:56 +00:00
|
|
|
|
|
2020-05-24 15:20:58 +00:00
|
|
|
|
RETURNS (str): IOB code of named entity tag.
|
2017-05-19 16:47:56 +00:00
|
|
|
|
"""
|
2020-06-26 17:34:12 +00:00
|
|
|
|
return self.iob_strings()[self.c.ent_iob]
|
2015-07-13 17:20:48 +00:00
|
|
|
|
|
2016-09-21 12:54:55 +00:00
|
|
|
|
property ent_id:
|
2017-10-27 15:07:26 +00:00
|
|
|
|
"""RETURNS (uint64): ID of the entity the token is an instance of,
|
|
|
|
|
if any.
|
2017-04-15 11:05:15 +00:00
|
|
|
|
"""
|
2016-09-21 12:54:55 +00:00
|
|
|
|
def __get__(self):
|
2016-09-23 13:07:07 +00:00
|
|
|
|
return self.c.ent_id
|
2016-09-21 12:54:55 +00:00
|
|
|
|
|
|
|
|
|
def __set__(self, hash_t key):
|
2017-03-31 12:00:14 +00:00
|
|
|
|
self.c.ent_id = key
|
2016-09-21 12:54:55 +00:00
|
|
|
|
|
|
|
|
|
property ent_id_:
|
2020-05-24 15:20:58 +00:00
|
|
|
|
"""RETURNS (str): ID of the entity the token is an instance of,
|
2017-10-27 15:07:26 +00:00
|
|
|
|
if any.
|
2017-04-15 11:05:15 +00:00
|
|
|
|
"""
|
2016-09-21 12:54:55 +00:00
|
|
|
|
def __get__(self):
|
2016-09-30 18:20:22 +00:00
|
|
|
|
return self.vocab.strings[self.c.ent_id]
|
2016-09-21 12:54:55 +00:00
|
|
|
|
|
2017-03-31 12:00:14 +00:00
|
|
|
|
def __set__(self, name):
|
2017-05-28 13:10:22 +00:00
|
|
|
|
self.c.ent_id = self.vocab.strings.add(name)
|
2016-09-21 12:54:55 +00:00
|
|
|
|
|
2019-03-14 14:48:40 +00:00
|
|
|
|
property ent_kb_id:
|
|
|
|
|
"""RETURNS (uint64): Named entity KB ID."""
|
|
|
|
|
def __get__(self):
|
|
|
|
|
return self.c.ent_kb_id
|
|
|
|
|
|
|
|
|
|
def __set__(self, attr_t ent_kb_id):
|
|
|
|
|
self.c.ent_kb_id = ent_kb_id
|
|
|
|
|
|
|
|
|
|
property ent_kb_id_:
|
2020-05-24 15:20:58 +00:00
|
|
|
|
"""RETURNS (str): Named entity KB ID."""
|
2019-03-14 14:48:40 +00:00
|
|
|
|
def __get__(self):
|
|
|
|
|
return self.vocab.strings[self.c.ent_kb_id]
|
|
|
|
|
|
|
|
|
|
def __set__(self, ent_kb_id):
|
|
|
|
|
self.c.ent_kb_id = self.vocab.strings.add(ent_kb_id)
|
|
|
|
|
|
2019-03-11 14:59:09 +00:00
|
|
|
|
@property
|
|
|
|
|
def whitespace_(self):
|
2020-05-24 15:20:58 +00:00
|
|
|
|
"""RETURNS (str): The trailing whitespace character, if present."""
|
2019-03-11 14:59:09 +00:00
|
|
|
|
return " " if self.c.spacy else ""
|
2015-07-13 17:20:48 +00:00
|
|
|
|
|
2019-03-11 14:59:09 +00:00
|
|
|
|
@property
|
|
|
|
|
def orth_(self):
|
2020-05-24 15:20:58 +00:00
|
|
|
|
"""RETURNS (str): Verbatim text content (identical to
|
💫 Port master changes over to develop (#2979)
* Create aryaprabhudesai.md (#2681)
* Update _install.jade (#2688)
Typo fix: "models" -> "model"
* Add FAC to spacy.explain (resolves #2706)
* Remove docstrings for deprecated arguments (see #2703)
* When calling getoption() in conftest.py, pass a default option (#2709)
* When calling getoption() in conftest.py, pass a default option
This is necessary to allow testing an installed spacy by running:
pytest --pyargs spacy
* Add contributor agreement
* update bengali token rules for hyphen and digits (#2731)
* Less norm computations in token similarity (#2730)
* Less norm computations in token similarity
* Contributor agreement
* Remove ')' for clarity (#2737)
Sorry, don't mean to be nitpicky, I just noticed this when going through the CLI and thought it was a quick fix. That said, if this was intention than please let me know.
* added contributor agreement for mbkupfer (#2738)
* Basic support for Telugu language (#2751)
* Lex _attrs for polish language (#2750)
* Signed spaCy contributor agreement
* Added polish version of english lex_attrs
* Introduces a bulk merge function, in order to solve issue #653 (#2696)
* Fix comment
* Introduce bulk merge to increase performance on many span merges
* Sign contributor agreement
* Implement pull request suggestions
* Describe converters more explicitly (see #2643)
* Add multi-threading note to Language.pipe (resolves #2582) [ci skip]
* Fix formatting
* Fix dependency scheme docs (closes #2705) [ci skip]
* Don't set stop word in example (closes #2657) [ci skip]
* Add words to portuguese language _num_words (#2759)
* Add words to portuguese language _num_words
* Add words to portuguese language _num_words
* Update Indonesian model (#2752)
* adding e-KTP in tokenizer exceptions list
* add exception token
* removing lines with containing space as it won't matter since we use .split() method in the end, added new tokens in exception
* add tokenizer exceptions list
* combining base_norms with norm_exceptions
* adding norm_exception
* fix double key in lemmatizer
* remove unused import on punctuation.py
* reformat stop_words to reduce number of lines, improve readibility
* updating tokenizer exception
* implement is_currency for lang/id
* adding orth_first_upper in tokenizer_exceptions
* update the norm_exception list
* remove bunch of abbreviations
* adding contributors file
* Fixed spaCy+Keras example (#2763)
* bug fixes in keras example
* created contributor agreement
* Adding French hyphenated first name (#2786)
* Fix typo (closes #2784)
* Fix typo (#2795) [ci skip]
Fixed typo on line 6 "regcognizer --> recognizer"
* Adding basic support for Sinhala language. (#2788)
* adding Sinhala language package, stop words, examples and lex_attrs.
* Adding contributor agreement
* Updating contributor agreement
* Also include lowercase norm exceptions
* Fix error (#2802)
* Fix error
ValueError: cannot resize an array that references or is referenced
by another array in this way. Use the resize function
* added spaCy Contributor Agreement
* Add charlax's contributor agreement (#2805)
* agreement of contributor, may I introduce a tiny pl languge contribution (#2799)
* Contributors agreement
* Contributors agreement
* Contributors agreement
* Add jupyter=True to displacy.render in documentation (#2806)
* Revert "Also include lowercase norm exceptions"
This reverts commit 70f4e8adf37cfcfab60be2b97d6deae949b30e9e.
* Remove deprecated encoding argument to msgpack
* Set up dependency tree pattern matching skeleton (#2732)
* Fix bug when too many entity types. Fixes #2800
* Fix Python 2 test failure
* Require older msgpack-numpy
* Restore encoding arg on msgpack-numpy
* Try to fix version pin for msgpack-numpy
* Update Portuguese Language (#2790)
* Add words to portuguese language _num_words
* Add words to portuguese language _num_words
* Portuguese - Add/remove stopwords, fix tokenizer, add currency symbols
* Extended punctuation and norm_exceptions in the Portuguese language
* Correct error in spacy universe docs concerning spacy-lookup (#2814)
* Update Keras Example for (Parikh et al, 2016) implementation (#2803)
* bug fixes in keras example
* created contributor agreement
* baseline for Parikh model
* initial version of parikh 2016 implemented
* tested asymmetric models
* fixed grevious error in normalization
* use standard SNLI test file
* begin to rework parikh example
* initial version of running example
* start to document the new version
* start to document the new version
* Update Decompositional Attention.ipynb
* fixed calls to similarity
* updated the README
* import sys package duh
* simplified indexing on mapping word to IDs
* stupid python indent error
* added code from https://github.com/tensorflow/tensorflow/issues/3388 for tf bug workaround
* Fix typo (closes #2815) [ci skip]
* Update regex version dependency
* Set version to 2.0.13.dev3
* Skip seemingly problematic test
* Remove problematic test
* Try previous version of regex
* Revert "Remove problematic test"
This reverts commit bdebbef45552d698d390aa430b527ee27830f11b.
* Unskip test
* Try older version of regex
* 💫 Update training examples and use minibatching (#2830)
<!--- Provide a general summary of your changes in the title. -->
## Description
Update the training examples in `/examples/training` to show usage of spaCy's `minibatch` and `compounding` helpers ([see here](https://spacy.io/usage/training#tips-batch-size) for details). The lack of batching in the examples has caused some confusion in the past, especially for beginners who would copy-paste the examples, update them with large training sets and experienced slow and unsatisfying results.
### Types of change
enhancements
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
* Visual C++ link updated (#2842) (closes #2841) [ci skip]
* New landing page
* Add contribution agreement
* Correcting lang/ru/examples.py (#2845)
* Correct some grammatical inaccuracies in lang\ru\examples.py; filled Contributor Agreement
* Correct some grammatical inaccuracies in lang\ru\examples.py
* Move contributor agreement to separate file
* Set version to 2.0.13.dev4
* Add Persian(Farsi) language support (#2797)
* Also include lowercase norm exceptions
* Remove in favour of https://github.com/explosion/spaCy/graphs/contributors
* Rule-based French Lemmatizer (#2818)
<!--- Provide a general summary of your changes in the title. -->
## Description
<!--- Use this section to describe your changes. If your changes required
testing, include information about the testing environment and the tests you
ran. If your test fixes a bug reported in an issue, don't forget to include the
issue number. If your PR is still a work in progress, that's totally fine – just
include a note to let us know. -->
Add a rule-based French Lemmatizer following the english one and the excellent PR for [greek language optimizations](https://github.com/explosion/spaCy/pull/2558) to adapt the Lemmatizer class.
### Types of change
<!-- What type of change does your PR cover? Is it a bug fix, an enhancement
or new feature, or a change to the documentation? -->
- Lemma dictionary used can be found [here](http://infolingu.univ-mlv.fr/DonneesLinguistiques/Dictionnaires/telechargement.html), I used the XML version.
- Add several files containing exhaustive list of words for each part of speech
- Add some lemma rules
- Add POS that are not checked in the standard Lemmatizer, i.e PRON, DET, ADV and AUX
- Modify the Lemmatizer class to check in lookup table as a last resort if POS not mentionned
- Modify the lemmatize function to check in lookup table as a last resort
- Init files are updated so the model can support all the functionalities mentioned above
- Add words to tokenizer_exceptions_list.py in respect to regex used in tokenizer_exceptions.py
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [X] I have submitted the spaCy Contributor Agreement.
- [X] I ran the tests, and all new and existing tests passed.
- [X] My changes don't require a change to the documentation, or if they do, I've added all required information.
* Set version to 2.0.13
* Fix formatting and consistency
* Update docs for new version [ci skip]
* Increment version [ci skip]
* Add info on wheels [ci skip]
* Adding "This is a sentence" example to Sinhala (#2846)
* Add wheels badge
* Update badge [ci skip]
* Update README.rst [ci skip]
* Update murmurhash pin
* Increment version to 2.0.14.dev0
* Update GPU docs for v2.0.14
* Add wheel to setup_requires
* Import prefer_gpu and require_gpu functions from Thinc
* Add tests for prefer_gpu() and require_gpu()
* Update requirements and setup.py
* Workaround bug in thinc require_gpu
* Set version to v2.0.14
* Update push-tag script
* Unhack prefer_gpu
* Require thinc 6.10.6
* Update prefer_gpu and require_gpu docs [ci skip]
* Fix specifiers for GPU
* Set version to 2.0.14.dev1
* Set version to 2.0.14
* Update Thinc version pin
* Increment version
* Fix msgpack-numpy version pin
* Increment version
* Update version to 2.0.16
* Update version [ci skip]
* Redundant ')' in the Stop words' example (#2856)
<!--- Provide a general summary of your changes in the title. -->
## Description
<!--- Use this section to describe your changes. If your changes required
testing, include information about the testing environment and the tests you
ran. If your test fixes a bug reported in an issue, don't forget to include the
issue number. If your PR is still a work in progress, that's totally fine – just
include a note to let us know. -->
### Types of change
<!-- What type of change does your PR cover? Is it a bug fix, an enhancement
or new feature, or a change to the documentation? -->
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [ ] I have submitted the spaCy Contributor Agreement.
- [ ] I ran the tests, and all new and existing tests passed.
- [ ] My changes don't require a change to the documentation, or if they do, I've added all required information.
* Documentation improvement regarding joblib and SO (#2867)
Some documentation improvements
## Description
1. Fixed the dead URL to joblib
2. Fixed Stack Overflow brand name (with space)
### Types of change
Documentation
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
* raise error when setting overlapping entities as doc.ents (#2880)
* Fix out-of-bounds access in NER training
The helper method state.B(1) gets the index of the first token of the
buffer, or -1 if no such token exists. Normally this is safe because we
pass this to functions like state.safe_get(), which returns an empty
token. Here we used it directly as an array index, which is not okay!
This error may have been the cause of out-of-bounds access errors during
training. Similar errors may still be around, so much be hunted down.
Hunting this one down took a long time...I printed out values across
training runs and diffed, looking for points of divergence between
runs, when no randomness should be allowed.
* Change PyThaiNLP Url (#2876)
* Fix missing comma
* Add example showing a fix-up rule for space entities
* Set version to 2.0.17.dev0
* Update regex version
* Revert "Update regex version"
This reverts commit 62358dd867d15bc6a475942dff34effba69dd70a.
* Try setting older regex version, to align with conda
* Set version to 2.0.17
* Add spacy-js to universe [ci-skip]
* Add spacy-raspberry to universe (closes #2889)
* Add script to validate universe json [ci skip]
* Removed space in docs + added contributor indo (#2909)
* - removed unneeded space in documentation
* - added contributor info
* Allow input text of length up to max_length, inclusive (#2922)
* Include universe spec for spacy-wordnet component (#2919)
* feat: include universe spec for spacy-wordnet component
* chore: include spaCy contributor agreement
* Minor formatting changes [ci skip]
* Fix image [ci skip]
Twitter URL doesn't work on live site
* Check if the word is in one of the regular lists specific to each POS (#2886)
* 💫 Create random IDs for SVGs to prevent ID clashes (#2927)
Resolves #2924.
## Description
Fixes problem where multiple visualizations in Jupyter notebooks would have clashing arc IDs, resulting in weirdly positioned arc labels. Generating a random ID prefix so even identical parses won't receive the same IDs for consistency (even if effect of ID clash isn't noticable here.)
### Types of change
bug fix
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
* Fix typo [ci skip]
* fixes symbolic link on py3 and windows (#2949)
* fixes symbolic link on py3 and windows
during setup of spacy using command
python -m spacy link en_core_web_sm en
closes #2948
* Update spacy/compat.py
Co-Authored-By: cicorias <cicorias@users.noreply.github.com>
* Fix formatting
* Update universe [ci skip]
* Catalan Language Support (#2940)
* Catalan language Support
* Ddding Catalan to documentation
* Sort languages alphabetically [ci skip]
* Update tests for pytest 4.x (#2965)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Replace marks in params for pytest 4.0 compat ([see here](https://docs.pytest.org/en/latest/deprecations.html#marks-in-pytest-mark-parametrize))
- [x] Un-xfail passing tests (some fixes in a recent update resolved a bunch of issues, but tests were apparently never updated here)
### Types of change
<!-- What type of change does your PR cover? Is it a bug fix, an enhancement
or new feature, or a change to the documentation? -->
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
* Fix regex pin to harmonize with conda (#2964)
* Update README.rst
* Fix bug where Vocab.prune_vector did not use 'batch_size' (#2977)
Fixes #2976
* Fix typo
* Fix typo
* Remove duplicate file
* Require thinc 7.0.0.dev2
Fixes bug in gpu_ops that would use cupy instead of numpy on CPU
* Add missing import
* Fix error IDs
* Fix tests
2018-11-29 15:30:29 +00:00
|
|
|
|
`Token.text`). Exists mostly for consistency with the other
|
2017-10-27 15:07:26 +00:00
|
|
|
|
attributes.
|
2017-10-27 13:41:45 +00:00
|
|
|
|
"""
|
2019-03-11 14:59:09 +00:00
|
|
|
|
return self.vocab.strings[self.c.lex.orth]
|
2015-07-13 17:20:48 +00:00
|
|
|
|
|
2019-03-11 14:59:09 +00:00
|
|
|
|
@property
|
|
|
|
|
def lower_(self):
|
2020-05-24 15:20:58 +00:00
|
|
|
|
"""RETURNS (str): The lowercase token text. Equivalent to
|
2017-10-27 15:07:26 +00:00
|
|
|
|
`Token.text.lower()`.
|
2017-10-27 13:41:45 +00:00
|
|
|
|
"""
|
2019-03-11 14:59:09 +00:00
|
|
|
|
return self.vocab.strings[self.c.lex.lower]
|
2015-07-13 17:20:48 +00:00
|
|
|
|
|
|
|
|
|
property norm_:
|
2020-05-24 15:20:58 +00:00
|
|
|
|
"""RETURNS (str): The token's norm, i.e. a normalised form of the
|
2017-10-27 15:07:26 +00:00
|
|
|
|
token text. Usually set in the language's tokenizer exceptions or
|
|
|
|
|
norm exceptions.
|
2017-10-27 13:41:45 +00:00
|
|
|
|
"""
|
2015-07-13 17:20:48 +00:00
|
|
|
|
def __get__(self):
|
2018-12-08 09:49:10 +00:00
|
|
|
|
return self.vocab.strings[self.norm]
|
|
|
|
|
|
|
|
|
|
def __set__(self, unicode norm_):
|
|
|
|
|
self.c.norm = self.vocab.strings.add(norm_)
|
2015-07-13 17:20:48 +00:00
|
|
|
|
|
2019-03-11 14:59:09 +00:00
|
|
|
|
@property
|
|
|
|
|
def shape_(self):
|
2020-05-24 15:20:58 +00:00
|
|
|
|
"""RETURNS (str): Transform of the tokens's string, to show
|
2017-10-27 15:07:26 +00:00
|
|
|
|
orthographic features. For example, "Xxxx" or "dd".
|
2017-10-27 13:41:45 +00:00
|
|
|
|
"""
|
2019-03-11 14:59:09 +00:00
|
|
|
|
return self.vocab.strings[self.c.lex.shape]
|
2015-07-13 17:20:48 +00:00
|
|
|
|
|
2019-03-11 14:59:09 +00:00
|
|
|
|
@property
|
|
|
|
|
def prefix_(self):
|
2020-05-24 15:20:58 +00:00
|
|
|
|
"""RETURNS (str): A length-N substring from the start of the token.
|
2017-10-27 15:07:26 +00:00
|
|
|
|
Defaults to `N=1`.
|
2017-10-27 13:41:45 +00:00
|
|
|
|
"""
|
2019-03-11 14:59:09 +00:00
|
|
|
|
return self.vocab.strings[self.c.lex.prefix]
|
2015-07-13 17:20:48 +00:00
|
|
|
|
|
2019-03-11 14:59:09 +00:00
|
|
|
|
@property
|
|
|
|
|
def suffix_(self):
|
2020-05-24 15:20:58 +00:00
|
|
|
|
"""RETURNS (str): A length-N substring from the end of the token.
|
2017-10-27 15:07:26 +00:00
|
|
|
|
Defaults to `N=3`.
|
2017-10-27 13:41:45 +00:00
|
|
|
|
"""
|
2019-03-11 14:59:09 +00:00
|
|
|
|
return self.vocab.strings[self.c.lex.suffix]
|
2015-07-13 17:20:48 +00:00
|
|
|
|
|
2019-03-11 14:59:09 +00:00
|
|
|
|
@property
|
|
|
|
|
def lang_(self):
|
2020-05-24 15:20:58 +00:00
|
|
|
|
"""RETURNS (str): Language of the parent document's vocabulary,
|
2017-10-27 15:07:26 +00:00
|
|
|
|
e.g. 'en'.
|
2017-10-27 13:41:45 +00:00
|
|
|
|
"""
|
2019-03-11 14:59:09 +00:00
|
|
|
|
return self.vocab.strings[self.c.lex.lang]
|
2016-03-10 12:01:34 +00:00
|
|
|
|
|
2015-07-13 17:20:48 +00:00
|
|
|
|
property lemma_:
|
2020-05-24 15:20:58 +00:00
|
|
|
|
"""RETURNS (str): The token lemma, i.e. the base form of the word,
|
2017-10-27 15:07:26 +00:00
|
|
|
|
with no inflectional suffixes.
|
2017-05-19 16:47:56 +00:00
|
|
|
|
"""
|
2015-07-13 17:20:48 +00:00
|
|
|
|
def __get__(self):
|
2020-08-07 13:27:13 +00:00
|
|
|
|
return self.vocab.strings[self.c.lemma]
|
2017-10-27 15:07:26 +00:00
|
|
|
|
|
2017-04-16 16:07:53 +00:00
|
|
|
|
def __set__(self, unicode lemma_):
|
2017-05-28 13:10:22 +00:00
|
|
|
|
self.c.lemma = self.vocab.strings.add(lemma_)
|
2015-07-13 17:20:48 +00:00
|
|
|
|
|
|
|
|
|
property pos_:
|
2020-05-24 15:20:58 +00:00
|
|
|
|
"""RETURNS (str): Coarse-grained part-of-speech tag."""
|
2015-07-13 17:20:48 +00:00
|
|
|
|
def __get__(self):
|
2015-10-10 06:55:55 +00:00
|
|
|
|
return parts_of_speech.NAMES[self.c.pos]
|
2019-03-08 10:42:26 +00:00
|
|
|
|
|
2018-03-27 19:21:11 +00:00
|
|
|
|
def __set__(self, pos_name):
|
|
|
|
|
self.c.pos = parts_of_speech.IDS[pos_name]
|
2015-07-13 17:20:48 +00:00
|
|
|
|
|
|
|
|
|
property tag_:
|
2020-05-24 15:20:58 +00:00
|
|
|
|
"""RETURNS (str): Fine-grained part-of-speech tag."""
|
2015-07-13 17:20:48 +00:00
|
|
|
|
def __get__(self):
|
2016-09-30 18:20:22 +00:00
|
|
|
|
return self.vocab.strings[self.c.tag]
|
2017-10-27 15:07:26 +00:00
|
|
|
|
|
2016-11-02 22:28:59 +00:00
|
|
|
|
def __set__(self, tag):
|
2017-05-28 13:10:22 +00:00
|
|
|
|
self.tag = self.vocab.strings.add(tag)
|
2015-07-13 17:20:48 +00:00
|
|
|
|
|
|
|
|
|
property dep_:
|
2020-05-24 15:20:58 +00:00
|
|
|
|
"""RETURNS (str): The syntactic dependency label."""
|
2015-07-13 17:20:48 +00:00
|
|
|
|
def __get__(self):
|
2016-09-30 18:20:22 +00:00
|
|
|
|
return self.vocab.strings[self.c.dep]
|
2017-10-27 15:07:26 +00:00
|
|
|
|
|
2016-03-11 16:31:06 +00:00
|
|
|
|
def __set__(self, unicode label):
|
2017-05-28 13:10:22 +00:00
|
|
|
|
self.c.dep = self.vocab.strings.add(label)
|
2015-07-13 18:20:58 +00:00
|
|
|
|
|
2019-03-11 14:59:09 +00:00
|
|
|
|
@property
|
|
|
|
|
def is_oov(self):
|
2017-10-27 15:07:26 +00:00
|
|
|
|
"""RETURNS (bool): Whether the token is out-of-vocabulary."""
|
2020-06-23 11:29:51 +00:00
|
|
|
|
return self.c.lex.orth not in self.vocab.vectors
|
2015-07-26 23:50:06 +00:00
|
|
|
|
|
2019-03-11 14:59:09 +00:00
|
|
|
|
@property
|
|
|
|
|
def is_stop(self):
|
2017-10-27 15:07:26 +00:00
|
|
|
|
"""RETURNS (bool): Whether the token is a stop word, i.e. part of a
|
|
|
|
|
"stop list" defined by the language data.
|
2017-10-27 13:41:45 +00:00
|
|
|
|
"""
|
2019-03-11 14:59:09 +00:00
|
|
|
|
return Lexeme.c_check_flag(self.c.lex, IS_STOP)
|
2015-09-14 07:49:58 +00:00
|
|
|
|
|
2019-03-11 14:59:09 +00:00
|
|
|
|
@property
|
|
|
|
|
def is_alpha(self):
|
2017-10-27 15:07:26 +00:00
|
|
|
|
"""RETURNS (bool): Whether the token consists of alpha characters.
|
|
|
|
|
Equivalent to `token.text.isalpha()`.
|
2017-10-27 13:41:45 +00:00
|
|
|
|
"""
|
2019-03-11 14:59:09 +00:00
|
|
|
|
return Lexeme.c_check_flag(self.c.lex, IS_ALPHA)
|
2015-08-08 21:37:44 +00:00
|
|
|
|
|
2019-03-11 14:59:09 +00:00
|
|
|
|
@property
|
|
|
|
|
def is_ascii(self):
|
2017-10-27 15:07:26 +00:00
|
|
|
|
"""RETURNS (bool): Whether the token consists of ASCII characters.
|
|
|
|
|
Equivalent to `[any(ord(c) >= 128 for c in token.text)]`.
|
2017-10-27 13:41:45 +00:00
|
|
|
|
"""
|
2019-03-11 14:59:09 +00:00
|
|
|
|
return Lexeme.c_check_flag(self.c.lex, IS_ASCII)
|
2015-07-26 14:37:16 +00:00
|
|
|
|
|
2019-03-11 14:59:09 +00:00
|
|
|
|
@property
|
|
|
|
|
def is_digit(self):
|
2017-10-27 15:07:26 +00:00
|
|
|
|
"""RETURNS (bool): Whether the token consists of digits. Equivalent to
|
|
|
|
|
`token.text.isdigit()`.
|
2017-10-27 13:41:45 +00:00
|
|
|
|
"""
|
2019-03-11 14:59:09 +00:00
|
|
|
|
return Lexeme.c_check_flag(self.c.lex, IS_DIGIT)
|
2015-07-26 14:37:16 +00:00
|
|
|
|
|
2019-03-11 14:59:09 +00:00
|
|
|
|
@property
|
|
|
|
|
def is_lower(self):
|
2017-10-27 15:07:26 +00:00
|
|
|
|
"""RETURNS (bool): Whether the token is in lowercase. Equivalent to
|
|
|
|
|
`token.text.islower()`.
|
2017-10-27 13:41:45 +00:00
|
|
|
|
"""
|
2019-03-11 14:59:09 +00:00
|
|
|
|
return Lexeme.c_check_flag(self.c.lex, IS_LOWER)
|
2015-07-26 14:37:16 +00:00
|
|
|
|
|
2019-03-11 14:59:09 +00:00
|
|
|
|
@property
|
|
|
|
|
def is_upper(self):
|
2017-10-27 15:07:26 +00:00
|
|
|
|
"""RETURNS (bool): Whether the token is in uppercase. Equivalent to
|
|
|
|
|
`token.text.isupper()`
|
2017-10-27 13:41:45 +00:00
|
|
|
|
"""
|
2019-03-11 14:59:09 +00:00
|
|
|
|
return Lexeme.c_check_flag(self.c.lex, IS_UPPER)
|
2017-10-27 13:41:45 +00:00
|
|
|
|
|
2019-03-11 14:59:09 +00:00
|
|
|
|
@property
|
|
|
|
|
def is_title(self):
|
2017-10-27 15:07:26 +00:00
|
|
|
|
"""RETURNS (bool): Whether the token is in titlecase. Equivalent to
|
|
|
|
|
`token.text.istitle()`.
|
2017-10-27 13:41:45 +00:00
|
|
|
|
"""
|
2019-03-11 14:59:09 +00:00
|
|
|
|
return Lexeme.c_check_flag(self.c.lex, IS_TITLE)
|
2015-07-26 14:37:16 +00:00
|
|
|
|
|
2019-03-11 14:59:09 +00:00
|
|
|
|
@property
|
|
|
|
|
def is_punct(self):
|
2017-10-27 15:07:26 +00:00
|
|
|
|
"""RETURNS (bool): Whether the token is punctuation."""
|
2019-03-11 14:59:09 +00:00
|
|
|
|
return Lexeme.c_check_flag(self.c.lex, IS_PUNCT)
|
2015-07-26 14:37:16 +00:00
|
|
|
|
|
2019-03-11 14:59:09 +00:00
|
|
|
|
@property
|
|
|
|
|
def is_space(self):
|
2017-10-27 15:07:26 +00:00
|
|
|
|
"""RETURNS (bool): Whether the token consists of whitespace characters.
|
|
|
|
|
Equivalent to `token.text.isspace()`.
|
2017-10-27 13:41:45 +00:00
|
|
|
|
"""
|
2019-03-11 14:59:09 +00:00
|
|
|
|
return Lexeme.c_check_flag(self.c.lex, IS_SPACE)
|
2017-02-26 21:27:11 +00:00
|
|
|
|
|
2019-03-11 14:59:09 +00:00
|
|
|
|
@property
|
|
|
|
|
def is_bracket(self):
|
2017-10-27 15:07:26 +00:00
|
|
|
|
"""RETURNS (bool): Whether the token is a bracket."""
|
2019-03-11 14:59:09 +00:00
|
|
|
|
return Lexeme.c_check_flag(self.c.lex, IS_BRACKET)
|
2016-02-04 12:04:16 +00:00
|
|
|
|
|
2019-03-11 14:59:09 +00:00
|
|
|
|
@property
|
|
|
|
|
def is_quote(self):
|
2017-10-27 15:07:26 +00:00
|
|
|
|
"""RETURNS (bool): Whether the token is a quotation mark."""
|
2019-03-11 14:59:09 +00:00
|
|
|
|
return Lexeme.c_check_flag(self.c.lex, IS_QUOTE)
|
2016-02-04 12:04:16 +00:00
|
|
|
|
|
2019-03-11 14:59:09 +00:00
|
|
|
|
@property
|
|
|
|
|
def is_left_punct(self):
|
2017-10-27 15:07:26 +00:00
|
|
|
|
"""RETURNS (bool): Whether the token is a left punctuation mark."""
|
2019-03-11 14:59:09 +00:00
|
|
|
|
return Lexeme.c_check_flag(self.c.lex, IS_LEFT_PUNCT)
|
2016-02-04 12:04:16 +00:00
|
|
|
|
|
2019-03-11 14:59:09 +00:00
|
|
|
|
@property
|
|
|
|
|
def is_right_punct(self):
|
2018-12-14 09:11:11 +00:00
|
|
|
|
"""RETURNS (bool): Whether the token is a right punctuation mark."""
|
2019-03-11 14:59:09 +00:00
|
|
|
|
return Lexeme.c_check_flag(self.c.lex, IS_RIGHT_PUNCT)
|
2015-07-26 14:37:16 +00:00
|
|
|
|
|
2019-03-11 14:59:09 +00:00
|
|
|
|
@property
|
|
|
|
|
def is_currency(self):
|
2018-02-11 17:55:48 +00:00
|
|
|
|
"""RETURNS (bool): Whether the token is a currency symbol."""
|
2019-03-11 14:59:09 +00:00
|
|
|
|
return Lexeme.c_check_flag(self.c.lex, IS_CURRENCY)
|
2018-02-11 17:55:48 +00:00
|
|
|
|
|
2019-03-11 14:59:09 +00:00
|
|
|
|
@property
|
|
|
|
|
def like_url(self):
|
2017-10-27 15:07:26 +00:00
|
|
|
|
"""RETURNS (bool): Whether the token resembles a URL."""
|
2019-03-11 14:59:09 +00:00
|
|
|
|
return Lexeme.c_check_flag(self.c.lex, LIKE_URL)
|
2015-08-08 21:37:44 +00:00
|
|
|
|
|
2019-03-11 14:59:09 +00:00
|
|
|
|
@property
|
|
|
|
|
def like_num(self):
|
2017-10-27 15:07:26 +00:00
|
|
|
|
"""RETURNS (bool): Whether the token resembles a number, e.g. "10.9",
|
|
|
|
|
"10", "ten", etc.
|
2017-10-27 13:41:45 +00:00
|
|
|
|
"""
|
2019-03-11 14:59:09 +00:00
|
|
|
|
return Lexeme.c_check_flag(self.c.lex, LIKE_NUM)
|
2015-07-26 14:37:16 +00:00
|
|
|
|
|
2019-03-11 14:59:09 +00:00
|
|
|
|
@property
|
|
|
|
|
def like_email(self):
|
2017-10-27 15:07:26 +00:00
|
|
|
|
"""RETURNS (bool): Whether the token resembles an email address."""
|
2019-03-11 14:59:09 +00:00
|
|
|
|
return Lexeme.c_check_flag(self.c.lex, LIKE_EMAIL)
|