spaCy/spacy/lexeme.pyx

518 lines
17 KiB
Cython
Raw Normal View History

# cython: embedsignature=True
# Compiler crashes on memory view coercion without this. Should report bug.
from cython.view cimport array as cvarray
from libc.string cimport memset
cimport numpy as np
np.import_array()
import numpy
2020-02-18 14:38:18 +00:00
from thinc.api import get_array_module
2020-02-28 11:20:23 +00:00
import warnings
from .typedefs cimport attr_t, flags_t
from .attrs cimport IS_ALPHA, IS_ASCII, IS_DIGIT, IS_LOWER, IS_PUNCT, IS_SPACE
from .attrs cimport IS_TITLE, IS_UPPER, LIKE_URL, LIKE_NUM, LIKE_EMAIL, IS_STOP
from .attrs cimport IS_BRACKET, IS_QUOTE, IS_LEFT_PUNCT, IS_RIGHT_PUNCT
from .attrs cimport IS_CURRENCY, IS_OOV, PROB
2017-10-30 10:49:11 +00:00
from .attrs import intify_attrs
2020-02-28 11:20:23 +00:00
from .errors import Errors, Warnings
memset(&EMPTY_LEXEME, 0, sizeof(LexemeC))
cdef class Lexeme:
"""An entry in the vocabulary. A `Lexeme` has no string context it's a
2015-01-24 09:48:34 +00:00
word-type, as opposed to a word token. It therefore has no part-of-speech
2017-10-27 19:07:50 +00:00
tag, dependency parse, or lemma (lemmatization depends on the
part-of-speech tag).
DOCS: https://spacy.io/api/lexeme
2015-01-24 09:48:34 +00:00
"""
def __init__(self, Vocab vocab, attr_t orth):
"""Create a Lexeme object.
2016-11-01 11:25:36 +00:00
vocab (Vocab): The parent vocabulary
orth (uint64): The orth id of the lexeme.
2016-11-01 11:25:36 +00:00
Returns (Lexeme): The newly constructd object.
"""
2015-08-22 20:04:34 +00:00
self.vocab = vocab
self.orth = orth
self.c = <LexemeC*><void*>vocab.get_by_orth(vocab.mem, orth)
if self.c.orth != orth:
raise ValueError(Errors.E071.format(orth=orth, vocab_orth=self.c.orth))
2015-08-22 20:04:34 +00:00
def __richcmp__(self, other, int op):
if other is None:
if op == 0 or op == 1 or op == 2:
return False
else:
return True
if isinstance(other, Lexeme):
a = self.orth
b = other.orth
elif isinstance(other, long):
a = self.orth
b = other
elif isinstance(other, str):
a = self.orth_
b = other
else:
a = 0
b = 1
2017-10-27 19:07:50 +00:00
if op == 2: # ==
return a == b
2017-10-27 19:07:50 +00:00
elif op == 3: # !=
return a != b
2017-10-27 19:07:50 +00:00
elif op == 0: # <
return a < b
2017-10-27 19:07:50 +00:00
elif op == 1: # <=
return a <= b
2017-10-27 19:07:50 +00:00
elif op == 4: # >
return a > b
2017-10-27 19:07:50 +00:00
elif op == 5: # >=
return a >= b
else:
raise NotImplementedError(op)
def __hash__(self):
return self.c.orth
2017-10-30 10:49:11 +00:00
def set_attrs(self, **attrs):
cdef attr_id_t attr
attrs = intify_attrs(attrs)
for attr, value in attrs.items():
if attr == PROB:
self.c.prob = value
2017-10-30 15:07:50 +00:00
elif attr == CLUSTER:
self.c.cluster = int(value)
2017-10-30 10:49:11 +00:00
elif isinstance(value, int) or isinstance(value, long):
Lexeme.set_struct_attr(self.c, attr, value)
else:
Lexeme.set_struct_attr(self.c, attr, self.vocab.strings.add(value))
def set_flag(self, attr_id_t flag_id, bint value):
"""Change the value of a boolean flag.
2016-11-01 11:25:36 +00:00
flag_id (int): The attribute ID of the flag to set.
value (bool): The new value of the flag.
2016-11-01 11:25:36 +00:00
"""
Lexeme.c_set_flag(self.c, flag_id, value)
2017-04-01 08:19:01 +00:00
def check_flag(self, attr_id_t flag_id):
"""Check the value of a boolean flag.
2016-11-01 11:25:36 +00:00
flag_id (int): The attribute ID of the flag to query.
RETURNS (bool): The value of the flag.
2016-11-01 11:25:36 +00:00
"""
return True if Lexeme.c_check_flag(self.c, flag_id) else False
def similarity(self, other):
"""Compute a semantic similarity estimate. Defaults to cosine over
vectors.
other (object): The object to compare with. By default, accepts `Doc`,
`Span`, `Token` and `Lexeme` objects.
RETURNS (float): A scalar similarity score. Higher is more similar.
"""
# Return 1.0 similarity for matches
if hasattr(other, "orth"):
if self.c.orth == other.orth:
return 1.0
elif hasattr(other, "__len__") and len(other) == 1 \
and hasattr(other[0], "orth"):
if self.c.orth == other[0].orth:
return 1.0
2015-09-22 00:10:01 +00:00
if self.vector_norm == 0 or other.vector_norm == 0:
2020-02-28 11:20:23 +00:00
warnings.warn(Warnings.W008.format(obj="Lexeme"))
2015-09-22 00:10:01 +00:00
return 0.0
vector = self.vector
xp = get_array_module(vector)
2019-03-07 00:06:12 +00:00
return (xp.dot(vector, other.vector) / (self.vector_norm * other.vector_norm))
def to_bytes(self):
lex_data = Lexeme.c_to_bytes(self.c)
start = <const char*>&self.c.flags
end = <const char*>&self.c.sentiment + sizeof(self.c.sentiment)
if (end-start) != sizeof(lex_data.data):
raise ValueError(Errors.E072.format(length=end-start,
bad_length=sizeof(lex_data.data)))
byte_string = b"\0" * sizeof(lex_data.data)
byte_chars = <char*>byte_string
for i in range(sizeof(lex_data.data)):
byte_chars[i] = lex_data.data[i]
if len(byte_string) != sizeof(lex_data.data):
raise ValueError(Errors.E072.format(length=len(byte_string),
bad_length=sizeof(lex_data.data)))
return byte_string
def from_bytes(self, bytes byte_string):
# This method doesn't really have a use-case --- wrote it for testing.
# Possibly delete? It puts the Lexeme out of synch with the vocab.
cdef SerializedLexemeC lex_data
if len(byte_string) != sizeof(lex_data.data):
raise ValueError(Errors.E072.format(length=len(byte_string),
bad_length=sizeof(lex_data.data)))
for i in range(len(byte_string)):
lex_data.data[i] = byte_string[i]
Lexeme.c_from_bytes(self.c, lex_data)
self.orth = self.c.orth
@property
def has_vector(self):
2017-10-27 19:07:50 +00:00
"""RETURNS (bool): Whether a word vector is associated with the object.
"""
return self.vocab.has_vector(self.c.orth)
@property
def vector_norm(self):
2017-10-27 19:07:50 +00:00
"""RETURNS (float): The L2 norm of the vector representation."""
vector = self.vector
return numpy.sqrt((vector**2).sum())
property vector:
"""A real-valued meaning representation.
RETURNS (numpy.ndarray[ndim=1, dtype='float32']): A 1D numpy array
representing the lexeme's semantics.
"""
def __get__(self):
2015-09-15 09:05:11 +00:00
cdef int length = self.vocab.vectors_length
if length == 0:
raise ValueError(Errors.E010)
2017-05-28 09:45:48 +00:00
return self.vocab.get_vector(self.c.orth)
def __set__(self, vector):
if len(vector) != self.vocab.vectors_length:
raise ValueError(Errors.E073.format(new_length=len(vector),
length=self.vocab.vectors_length))
2017-05-28 09:45:48 +00:00
self.vocab.set_vector(self.c.orth, vector)
property rank:
2020-05-24 15:20:58 +00:00
"""RETURNS (str): Sequential ID of the lexemes's lexical type, used
2017-10-27 19:07:50 +00:00
to index into tables, e.g. for word vectors."""
def __get__(self):
return self.c.id
2017-10-27 19:07:50 +00:00
2017-08-24 19:43:00 +00:00
def __set__(self, value):
self.c.id = value
property sentiment:
2017-10-27 19:07:50 +00:00
"""RETURNS (float): A scalar value indicating the positivity or
negativity of the lexeme."""
def __get__(self):
return self.c.sentiment
2017-10-27 19:07:50 +00:00
def __set__(self, float sentiment):
self.c.sentiment = sentiment
2017-04-01 08:19:01 +00:00
@property
def orth_(self):
2020-05-24 15:20:58 +00:00
"""RETURNS (str): The original verbatim text of the lexeme
2017-10-27 19:07:50 +00:00
(identical to `Lexeme.text`). Exists mostly for consistency with
the other attributes."""
return self.vocab.strings[self.c.orth]
2015-08-22 20:04:34 +00:00
@property
def text(self):
2020-05-24 15:20:58 +00:00
"""RETURNS (str): The original verbatim text of the lexeme."""
return self.orth_
2015-08-22 20:04:34 +00:00
property lower:
2020-05-24 15:20:58 +00:00
"""RETURNS (str): Lowercase form of the lexeme."""
2017-10-27 19:07:50 +00:00
def __get__(self):
return self.c.lower
def __set__(self, attr_t x):
self.c.lower = x
2017-04-01 08:19:01 +00:00
2015-08-22 20:04:34 +00:00
property norm:
2017-10-27 19:07:50 +00:00
"""RETURNS (uint64): The lexemes's norm, i.e. a normalised form of the
lexeme text.
"""
def __get__(self):
return self.c.norm
def __set__(self, attr_t x):
self.c.norm = x
2015-08-22 20:04:34 +00:00
property shape:
2017-10-27 19:07:50 +00:00
"""RETURNS (uint64): Transform of the word's string, to show
orthographic features.
"""
def __get__(self):
return self.c.shape
def __set__(self, attr_t x):
self.c.shape = x
2015-08-22 20:04:34 +00:00
property prefix:
2017-10-27 19:07:50 +00:00
"""RETURNS (uint64): Length-N substring from the start of the word.
Defaults to `N=1`.
"""
def __get__(self):
return self.c.prefix
def __set__(self, attr_t x):
self.c.prefix = x
2015-08-22 20:04:34 +00:00
property suffix:
2017-10-27 19:07:50 +00:00
"""RETURNS (uint64): Length-N substring from the end of the word.
Defaults to `N=3`.
"""
def __get__(self):
return self.c.suffix
def __set__(self, attr_t x):
self.c.suffix = x
2017-04-01 08:19:01 +00:00
property cluster:
2017-10-27 19:07:50 +00:00
"""RETURNS (int): Brown cluster ID."""
def __get__(self):
return self.c.cluster
def __set__(self, attr_t x):
self.c.cluster = x
2017-04-01 08:19:01 +00:00
property lang:
2017-10-27 19:07:50 +00:00
"""RETURNS (uint64): Language of the parent vocabulary."""
def __get__(self):
return self.c.lang
def __set__(self, attr_t x):
self.c.lang = x
property prob:
2017-10-27 19:07:50 +00:00
"""RETURNS (float): Smoothed log probability estimate of the lexeme's
type."""
def __get__(self):
return self.c.prob
def __set__(self, float x):
self.c.prob = x
2015-08-22 20:04:34 +00:00
property lower_:
2020-05-24 15:20:58 +00:00
"""RETURNS (str): Lowercase form of the word."""
2017-10-27 19:07:50 +00:00
def __get__(self):
return self.vocab.strings[self.c.lower]
def __set__(self, unicode x):
self.c.lower = self.vocab.strings.add(x)
2017-04-01 08:19:01 +00:00
2015-08-22 20:04:34 +00:00
property norm_:
2020-05-24 15:20:58 +00:00
"""RETURNS (str): The lexemes's norm, i.e. a normalised form of the
2017-10-27 19:07:50 +00:00
lexeme text.
"""
def __get__(self):
return self.vocab.strings[self.c.norm]
def __set__(self, unicode x):
self.c.norm = self.vocab.strings.add(x)
2017-04-01 08:19:01 +00:00
2015-08-22 20:04:34 +00:00
property shape_:
2020-05-24 15:20:58 +00:00
"""RETURNS (str): Transform of the word's string, to show
2017-10-27 19:07:50 +00:00
orthographic features.
"""
def __get__(self):
return self.vocab.strings[self.c.shape]
def __set__(self, unicode x):
self.c.shape = self.vocab.strings.add(x)
2015-08-22 20:04:34 +00:00
property prefix_:
2020-05-24 15:20:58 +00:00
"""RETURNS (str): Length-N substring from the start of the word.
2017-10-27 19:07:50 +00:00
Defaults to `N=1`.
"""
def __get__(self):
return self.vocab.strings[self.c.prefix]
def __set__(self, unicode x):
self.c.prefix = self.vocab.strings.add(x)
2015-08-22 20:04:34 +00:00
property suffix_:
2020-05-24 15:20:58 +00:00
"""RETURNS (str): Length-N substring from the end of the word.
2017-10-27 19:07:50 +00:00
Defaults to `N=3`.
"""
def __get__(self):
return self.vocab.strings[self.c.suffix]
def __set__(self, unicode x):
self.c.suffix = self.vocab.strings.add(x)
property lang_:
2020-05-24 15:20:58 +00:00
"""RETURNS (str): Language of the parent vocabulary."""
2017-10-27 19:07:50 +00:00
def __get__(self):
return self.vocab.strings[self.c.lang]
def __set__(self, unicode x):
self.c.lang = self.vocab.strings.add(x)
property flags:
2017-10-27 19:07:50 +00:00
"""RETURNS (uint64): Container of the lexeme's binary flags."""
def __get__(self):
return self.c.flags
def __set__(self, flags_t x):
self.c.flags = x
property is_oov:
2017-10-27 19:07:50 +00:00
"""RETURNS (bool): Whether the lexeme is out-of-vocabulary."""
def __get__(self):
return Lexeme.c_check_flag(self.c, IS_OOV)
def __set__(self, attr_t x):
Lexeme.c_set_flag(self.c, IS_OOV, x)
2015-09-14 08:25:40 +00:00
property is_stop:
2017-10-27 19:07:50 +00:00
"""RETURNS (bool): Whether the lexeme is a stop word."""
def __get__(self):
return Lexeme.c_check_flag(self.c, IS_STOP)
def __set__(self, bint x):
Lexeme.c_set_flag(self.c, IS_STOP, x)
2015-09-14 08:25:40 +00:00
property is_alpha:
"""RETURNS (bool): Whether the lexeme consists of alphabetic
2017-10-27 19:07:50 +00:00
characters. Equivalent to `lexeme.text.isalpha()`.
"""
def __get__(self):
return Lexeme.c_check_flag(self.c, IS_ALPHA)
def __set__(self, bint x):
Lexeme.c_set_flag(self.c, IS_ALPHA, x)
2017-04-01 08:19:01 +00:00
property is_ascii:
2017-10-27 19:07:50 +00:00
"""RETURNS (bool): Whether the lexeme consists of ASCII characters.
Equivalent to `[any(ord(c) >= 128 for c in lexeme.text)]`.
"""
def __get__(self):
return Lexeme.c_check_flag(self.c, IS_ASCII)
def __set__(self, bint x):
Lexeme.c_set_flag(self.c, IS_ASCII, x)
property is_digit:
2017-10-27 19:07:50 +00:00
"""RETURNS (bool): Whether the lexeme consists of digits. Equivalent
to `lexeme.text.isdigit()`.
"""
def __get__(self):
return Lexeme.c_check_flag(self.c, IS_DIGIT)
def __set__(self, bint x):
Lexeme.c_set_flag(self.c, IS_DIGIT, x)
property is_lower:
2017-10-27 19:07:50 +00:00
"""RETURNS (bool): Whether the lexeme is in lowercase. Equivalent to
`lexeme.text.islower()`.
"""
def __get__(self):
return Lexeme.c_check_flag(self.c, IS_LOWER)
def __set__(self, bint x):
Lexeme.c_set_flag(self.c, IS_LOWER, x)
property is_upper:
"""RETURNS (bool): Whether the lexeme is in uppercase. Equivalent to
`lexeme.text.isupper()`.
"""
def __get__(self):
return Lexeme.c_check_flag(self.c, IS_UPPER)
def __set__(self, bint x):
Lexeme.c_set_flag(self.c, IS_UPPER, x)
property is_title:
2017-10-27 19:07:50 +00:00
"""RETURNS (bool): Whether the lexeme is in titlecase. Equivalent to
`lexeme.text.istitle()`.
"""
def __get__(self):
return Lexeme.c_check_flag(self.c, IS_TITLE)
def __set__(self, bint x):
Lexeme.c_set_flag(self.c, IS_TITLE, x)
property is_punct:
2017-10-27 19:07:50 +00:00
"""RETURNS (bool): Whether the lexeme is punctuation."""
def __get__(self):
return Lexeme.c_check_flag(self.c, IS_PUNCT)
def __set__(self, bint x):
Lexeme.c_set_flag(self.c, IS_PUNCT, x)
2017-04-01 08:19:01 +00:00
property is_space:
2017-10-27 19:07:50 +00:00
"""RETURNS (bool): Whether the lexeme consist of whitespace characters.
Equivalent to `lexeme.text.isspace()`.
"""
def __get__(self):
return Lexeme.c_check_flag(self.c, IS_SPACE)
def __set__(self, bint x):
Lexeme.c_set_flag(self.c, IS_SPACE, x)
2017-04-01 08:19:01 +00:00
property is_bracket:
2017-10-27 19:07:50 +00:00
"""RETURNS (bool): Whether the lexeme is a bracket."""
def __get__(self):
return Lexeme.c_check_flag(self.c, IS_BRACKET)
def __set__(self, bint x):
Lexeme.c_set_flag(self.c, IS_BRACKET, x)
2017-04-01 08:19:01 +00:00
property is_quote:
2017-10-27 19:07:50 +00:00
"""RETURNS (bool): Whether the lexeme is a quotation mark."""
def __get__(self):
return Lexeme.c_check_flag(self.c, IS_QUOTE)
def __set__(self, bint x):
Lexeme.c_set_flag(self.c, IS_QUOTE, x)
2017-04-01 08:19:01 +00:00
property is_left_punct:
2017-10-27 19:07:50 +00:00
"""RETURNS (bool): Whether the lexeme is left punctuation, e.g. )."""
def __get__(self):
return Lexeme.c_check_flag(self.c, IS_LEFT_PUNCT)
def __set__(self, bint x):
Lexeme.c_set_flag(self.c, IS_LEFT_PUNCT, x)
2017-04-01 08:19:01 +00:00
property is_right_punct:
2017-10-27 19:07:50 +00:00
"""RETURNS (bool): Whether the lexeme is right punctuation, e.g. )."""
def __get__(self):
return Lexeme.c_check_flag(self.c, IS_RIGHT_PUNCT)
def __set__(self, bint x):
Lexeme.c_set_flag(self.c, IS_RIGHT_PUNCT, x)
2018-02-11 17:51:48 +00:00
property is_currency:
"""RETURNS (bool): Whether the lexeme is a currency symbol, e.g. $, €."""
def __get__(self):
return Lexeme.c_check_flag(self.c, IS_CURRENCY)
def __set__(self, bint x):
Lexeme.c_set_flag(self.c, IS_CURRENCY, x)
property like_url:
2017-10-27 19:07:50 +00:00
"""RETURNS (bool): Whether the lexeme resembles a URL."""
def __get__(self):
return Lexeme.c_check_flag(self.c, LIKE_URL)
def __set__(self, bint x):
Lexeme.c_set_flag(self.c, LIKE_URL, x)
2017-04-01 08:19:01 +00:00
property like_num:
2017-10-27 19:07:50 +00:00
"""RETURNS (bool): Whether the lexeme represents a number, e.g. "10.9",
"10", "ten", etc.
"""
def __get__(self):
return Lexeme.c_check_flag(self.c, LIKE_NUM)
def __set__(self, bint x):
Lexeme.c_set_flag(self.c, LIKE_NUM, x)
property like_email:
2017-10-27 19:07:50 +00:00
"""RETURNS (bool): Whether the lexeme resembles an email address."""
def __get__(self):
return Lexeme.c_check_flag(self.c, LIKE_EMAIL)
def __set__(self, bint x):
Lexeme.c_set_flag(self.c, LIKE_EMAIL, x)