2015-01-14 13:33:16 +00:00
|
|
|
|
# cython: embedsignature=True
|
2015-09-14 07:49:58 +00:00
|
|
|
|
# Compiler crashes on memory view coercion without this. Should report bug.
|
|
|
|
|
from cython.view cimport array as cvarray
|
2019-03-08 10:42:26 +00:00
|
|
|
|
from libc.string cimport memset
|
2015-09-14 07:49:58 +00:00
|
|
|
|
cimport numpy as np
|
|
|
|
|
np.import_array()
|
2019-03-08 10:42:26 +00:00
|
|
|
|
|
2017-04-15 10:05:47 +00:00
|
|
|
|
import numpy
|
2020-02-18 14:38:18 +00:00
|
|
|
|
from thinc.api import get_array_module
|
2020-04-28 11:37:37 +00:00
|
|
|
|
import warnings
|
2014-10-22 14:57:59 +00:00
|
|
|
|
|
2015-07-01 16:50:37 +00:00
|
|
|
|
from .typedefs cimport attr_t, flags_t
|
2015-07-26 14:37:16 +00:00
|
|
|
|
from .attrs cimport IS_ALPHA, IS_ASCII, IS_DIGIT, IS_LOWER, IS_PUNCT, IS_SPACE
|
|
|
|
|
from .attrs cimport IS_TITLE, IS_UPPER, LIKE_URL, LIKE_NUM, LIKE_EMAIL, IS_STOP
|
2019-03-08 10:42:26 +00:00
|
|
|
|
from .attrs cimport IS_BRACKET, IS_QUOTE, IS_LEFT_PUNCT, IS_RIGHT_PUNCT
|
2020-05-19 13:59:14 +00:00
|
|
|
|
from .attrs cimport IS_CURRENCY
|
2019-03-08 10:42:26 +00:00
|
|
|
|
|
2017-10-30 10:49:11 +00:00
|
|
|
|
from .attrs import intify_attrs
|
2020-04-28 11:37:37 +00:00
|
|
|
|
from .errors import Errors, Warnings
|
2015-07-26 14:37:16 +00:00
|
|
|
|
|
2014-10-09 08:53:30 +00:00
|
|
|
|
|
2020-06-04 18:02:05 +00:00
|
|
|
|
OOV_RANK = 0xffffffffffffffff # UINT64_MAX
|
2015-01-11 23:26:22 +00:00
|
|
|
|
memset(&EMPTY_LEXEME, 0, sizeof(LexemeC))
|
2020-04-15 11:49:47 +00:00
|
|
|
|
EMPTY_LEXEME.id = OOV_RANK
|
2014-10-09 08:53:30 +00:00
|
|
|
|
|
|
|
|
|
|
2015-01-12 00:23:44 +00:00
|
|
|
|
cdef class Lexeme:
|
2017-05-20 13:13:42 +00:00
|
|
|
|
"""An entry in the vocabulary. A `Lexeme` has no string context – it's a
|
2015-01-24 09:48:34 +00:00
|
|
|
|
word-type, as opposed to a word token. It therefore has no part-of-speech
|
2017-10-27 19:07:50 +00:00
|
|
|
|
tag, dependency parse, or lemma (lemmatization depends on the
|
|
|
|
|
part-of-speech tag).
|
2019-03-08 10:42:26 +00:00
|
|
|
|
|
2021-01-30 09:09:38 +00:00
|
|
|
|
DOCS: https://spacy.io/api/lexeme
|
2015-01-24 09:48:34 +00:00
|
|
|
|
"""
|
2017-05-28 10:51:09 +00:00
|
|
|
|
def __init__(self, Vocab vocab, attr_t orth):
|
2017-05-20 13:13:42 +00:00
|
|
|
|
"""Create a Lexeme object.
|
2016-11-01 11:25:36 +00:00
|
|
|
|
|
2017-05-20 13:13:42 +00:00
|
|
|
|
vocab (Vocab): The parent vocabulary
|
2017-05-28 10:51:09 +00:00
|
|
|
|
orth (uint64): The orth id of the lexeme.
|
2016-11-01 11:25:36 +00:00
|
|
|
|
Returns (Lexeme): The newly constructd object.
|
|
|
|
|
"""
|
2015-08-22 20:04:34 +00:00
|
|
|
|
self.vocab = vocab
|
|
|
|
|
self.orth = orth
|
2015-09-06 17:45:15 +00:00
|
|
|
|
self.c = <LexemeC*><void*>vocab.get_by_orth(vocab.mem, orth)
|
2018-04-03 13:50:31 +00:00
|
|
|
|
if self.c.orth != orth:
|
|
|
|
|
raise ValueError(Errors.E071.format(orth=orth, vocab_orth=self.c.orth))
|
2015-08-22 20:04:34 +00:00
|
|
|
|
|
2016-05-04 23:32:26 +00:00
|
|
|
|
def __richcmp__(self, other, int op):
|
2018-01-15 14:54:25 +00:00
|
|
|
|
if other is None:
|
|
|
|
|
if op == 0 or op == 1 or op == 2:
|
|
|
|
|
return False
|
|
|
|
|
else:
|
|
|
|
|
return True
|
2016-05-04 23:32:26 +00:00
|
|
|
|
if isinstance(other, Lexeme):
|
|
|
|
|
a = self.orth
|
|
|
|
|
b = other.orth
|
2017-05-28 10:51:09 +00:00
|
|
|
|
elif isinstance(other, long):
|
2016-05-04 23:32:26 +00:00
|
|
|
|
a = self.orth
|
|
|
|
|
b = other
|
|
|
|
|
elif isinstance(other, str):
|
2016-05-12 10:58:57 +00:00
|
|
|
|
a = self.orth_
|
2016-05-04 23:32:26 +00:00
|
|
|
|
b = other
|
|
|
|
|
else:
|
|
|
|
|
a = 0
|
|
|
|
|
b = 1
|
2017-10-27 19:07:50 +00:00
|
|
|
|
if op == 2: # ==
|
2016-05-04 23:32:26 +00:00
|
|
|
|
return a == b
|
2017-10-27 19:07:50 +00:00
|
|
|
|
elif op == 3: # !=
|
2016-05-04 23:32:26 +00:00
|
|
|
|
return a != b
|
2017-10-27 19:07:50 +00:00
|
|
|
|
elif op == 0: # <
|
2016-05-04 23:32:26 +00:00
|
|
|
|
return a < b
|
2017-10-27 19:07:50 +00:00
|
|
|
|
elif op == 1: # <=
|
2016-05-04 23:32:26 +00:00
|
|
|
|
return a <= b
|
2017-10-27 19:07:50 +00:00
|
|
|
|
elif op == 4: # >
|
2016-05-04 23:32:26 +00:00
|
|
|
|
return a > b
|
2017-10-27 19:07:50 +00:00
|
|
|
|
elif op == 5: # >=
|
2016-05-04 23:32:26 +00:00
|
|
|
|
return a >= b
|
|
|
|
|
else:
|
|
|
|
|
raise NotImplementedError(op)
|
|
|
|
|
|
2016-09-27 11:22:30 +00:00
|
|
|
|
def __hash__(self):
|
|
|
|
|
return self.c.orth
|
|
|
|
|
|
2017-10-30 10:49:11 +00:00
|
|
|
|
def set_attrs(self, **attrs):
|
|
|
|
|
cdef attr_id_t attr
|
|
|
|
|
attrs = intify_attrs(attrs)
|
|
|
|
|
for attr, value in attrs.items():
|
2020-05-19 13:59:14 +00:00
|
|
|
|
# skip PROB, e.g. from lexemes.jsonl
|
|
|
|
|
if isinstance(value, float):
|
|
|
|
|
continue
|
|
|
|
|
elif isinstance(value, (int, long)):
|
|
|
|
|
Lexeme.set_struct_attr(self.c, attr, value)
|
2017-10-30 10:49:11 +00:00
|
|
|
|
else:
|
|
|
|
|
Lexeme.set_struct_attr(self.c, attr, self.vocab.strings.add(value))
|
|
|
|
|
|
2015-09-15 03:06:18 +00:00
|
|
|
|
def set_flag(self, attr_id_t flag_id, bint value):
|
2017-05-20 13:13:42 +00:00
|
|
|
|
"""Change the value of a boolean flag.
|
2016-11-01 11:25:36 +00:00
|
|
|
|
|
2017-05-20 13:13:42 +00:00
|
|
|
|
flag_id (int): The attribute ID of the flag to set.
|
|
|
|
|
value (bool): The new value of the flag.
|
2016-11-01 11:25:36 +00:00
|
|
|
|
"""
|
2015-09-15 03:06:18 +00:00
|
|
|
|
Lexeme.c_set_flag(self.c, flag_id, value)
|
2017-04-01 08:19:01 +00:00
|
|
|
|
|
2015-09-15 03:06:18 +00:00
|
|
|
|
def check_flag(self, attr_id_t flag_id):
|
2017-05-20 13:13:42 +00:00
|
|
|
|
"""Check the value of a boolean flag.
|
2016-11-01 11:25:36 +00:00
|
|
|
|
|
2017-05-20 13:13:42 +00:00
|
|
|
|
flag_id (int): The attribute ID of the flag to query.
|
|
|
|
|
RETURNS (bool): The value of the flag.
|
2016-11-01 11:25:36 +00:00
|
|
|
|
"""
|
2015-09-15 03:06:18 +00:00
|
|
|
|
return True if Lexeme.c_check_flag(self.c, flag_id) else False
|
2015-09-06 15:52:51 +00:00
|
|
|
|
|
2015-09-14 07:49:58 +00:00
|
|
|
|
def similarity(self, other):
|
2017-05-20 13:13:42 +00:00
|
|
|
|
"""Compute a semantic similarity estimate. Defaults to cosine over
|
|
|
|
|
vectors.
|
|
|
|
|
|
|
|
|
|
other (object): The object to compare with. By default, accepts `Doc`,
|
|
|
|
|
`Span`, `Token` and `Lexeme` objects.
|
|
|
|
|
RETURNS (float): A scalar similarity score. Higher is more similar.
|
2017-04-15 09:59:21 +00:00
|
|
|
|
"""
|
2018-01-15 15:29:48 +00:00
|
|
|
|
# Return 1.0 similarity for matches
|
2019-03-08 10:42:26 +00:00
|
|
|
|
if hasattr(other, "orth"):
|
2018-01-15 15:29:48 +00:00
|
|
|
|
if self.c.orth == other.orth:
|
|
|
|
|
return 1.0
|
2019-03-08 10:42:26 +00:00
|
|
|
|
elif hasattr(other, "__len__") and len(other) == 1 \
|
|
|
|
|
and hasattr(other[0], "orth"):
|
2018-01-15 15:29:48 +00:00
|
|
|
|
if self.c.orth == other[0].orth:
|
|
|
|
|
return 1.0
|
2015-09-22 00:10:01 +00:00
|
|
|
|
if self.vector_norm == 0 or other.vector_norm == 0:
|
2020-04-28 11:37:37 +00:00
|
|
|
|
warnings.warn(Warnings.W008.format(obj="Lexeme"))
|
2015-09-22 00:10:01 +00:00
|
|
|
|
return 0.0
|
2019-03-06 22:58:38 +00:00
|
|
|
|
vector = self.vector
|
|
|
|
|
xp = get_array_module(vector)
|
2019-03-07 00:06:12 +00:00
|
|
|
|
return (xp.dot(vector, other.vector) / (self.vector_norm * other.vector_norm))
|
2015-09-14 07:49:58 +00:00
|
|
|
|
|
2019-03-11 14:59:09 +00:00
|
|
|
|
@property
|
|
|
|
|
def has_vector(self):
|
2017-10-27 19:07:50 +00:00
|
|
|
|
"""RETURNS (bool): Whether a word vector is associated with the object.
|
2017-05-20 13:13:42 +00:00
|
|
|
|
"""
|
2019-03-11 14:59:09 +00:00
|
|
|
|
return self.vocab.has_vector(self.c.orth)
|
2015-09-21 09:52:43 +00:00
|
|
|
|
|
2019-03-11 14:59:09 +00:00
|
|
|
|
@property
|
|
|
|
|
def vector_norm(self):
|
2017-10-27 19:07:50 +00:00
|
|
|
|
"""RETURNS (float): The L2 norm of the vector representation."""
|
2019-03-11 14:59:09 +00:00
|
|
|
|
vector = self.vector
|
|
|
|
|
return numpy.sqrt((vector**2).sum())
|
2015-09-14 07:49:58 +00:00
|
|
|
|
|
|
|
|
|
property vector:
|
2017-05-20 13:13:42 +00:00
|
|
|
|
"""A real-valued meaning representation.
|
|
|
|
|
|
|
|
|
|
RETURNS (numpy.ndarray[ndim=1, dtype='float32']): A 1D numpy array
|
|
|
|
|
representing the lexeme's semantics.
|
|
|
|
|
"""
|
2015-09-14 07:49:58 +00:00
|
|
|
|
def __get__(self):
|
2015-09-15 09:05:11 +00:00
|
|
|
|
cdef int length = self.vocab.vectors_length
|
2015-09-21 08:35:40 +00:00
|
|
|
|
if length == 0:
|
2018-04-03 13:50:31 +00:00
|
|
|
|
raise ValueError(Errors.E010)
|
2017-05-28 09:45:48 +00:00
|
|
|
|
return self.vocab.get_vector(self.c.orth)
|
2015-09-14 07:49:58 +00:00
|
|
|
|
|
2015-09-15 04:42:27 +00:00
|
|
|
|
def __set__(self, vector):
|
2018-04-03 13:50:31 +00:00
|
|
|
|
if len(vector) != self.vocab.vectors_length:
|
|
|
|
|
raise ValueError(Errors.E073.format(new_length=len(vector),
|
|
|
|
|
length=self.vocab.vectors_length))
|
2017-05-28 09:45:48 +00:00
|
|
|
|
self.vocab.set_vector(self.c.orth, vector)
|
2015-09-15 04:42:27 +00:00
|
|
|
|
|
2015-11-08 15:18:25 +00:00
|
|
|
|
property rank:
|
2021-06-15 08:57:08 +00:00
|
|
|
|
"""RETURNS (str): Sequential ID of the lexeme's lexical type, used
|
2017-10-27 19:07:50 +00:00
|
|
|
|
to index into tables, e.g. for word vectors."""
|
2015-11-08 15:18:25 +00:00
|
|
|
|
def __get__(self):
|
|
|
|
|
return self.c.id
|
2017-10-27 19:07:50 +00:00
|
|
|
|
|
2017-08-24 19:43:00 +00:00
|
|
|
|
def __set__(self, value):
|
|
|
|
|
self.c.id = value
|
2015-11-08 15:18:25 +00:00
|
|
|
|
|
2016-10-19 18:52:52 +00:00
|
|
|
|
property sentiment:
|
2017-10-27 19:07:50 +00:00
|
|
|
|
"""RETURNS (float): A scalar value indicating the positivity or
|
|
|
|
|
negativity of the lexeme."""
|
2016-10-19 18:52:52 +00:00
|
|
|
|
def __get__(self):
|
2020-05-19 13:59:14 +00:00
|
|
|
|
sentiment_table = self.vocab.lookups.get_table("lexeme_sentiment", {})
|
|
|
|
|
return sentiment_table.get(self.c.orth, 0.0)
|
2017-10-27 19:07:50 +00:00
|
|
|
|
|
2020-05-19 13:59:14 +00:00
|
|
|
|
def __set__(self, float x):
|
|
|
|
|
if "lexeme_sentiment" not in self.vocab.lookups:
|
|
|
|
|
self.vocab.lookups.add_table("lexeme_sentiment")
|
|
|
|
|
sentiment_table = self.vocab.lookups.get_table("lexeme_sentiment")
|
|
|
|
|
sentiment_table[self.c.orth] = x
|
2017-04-01 08:19:01 +00:00
|
|
|
|
|
2019-03-11 14:59:09 +00:00
|
|
|
|
@property
|
|
|
|
|
def orth_(self):
|
2020-05-24 15:20:58 +00:00
|
|
|
|
"""RETURNS (str): The original verbatim text of the lexeme
|
2017-10-27 19:07:50 +00:00
|
|
|
|
(identical to `Lexeme.text`). Exists mostly for consistency with
|
|
|
|
|
the other attributes."""
|
2019-03-11 14:59:09 +00:00
|
|
|
|
return self.vocab.strings[self.c.orth]
|
2015-08-22 20:04:34 +00:00
|
|
|
|
|
2019-03-11 14:59:09 +00:00
|
|
|
|
@property
|
|
|
|
|
def text(self):
|
2020-05-24 15:20:58 +00:00
|
|
|
|
"""RETURNS (str): The original verbatim text of the lexeme."""
|
2019-03-11 14:59:09 +00:00
|
|
|
|
return self.orth_
|
2017-05-20 13:13:42 +00:00
|
|
|
|
|
2015-08-22 20:04:34 +00:00
|
|
|
|
property lower:
|
2020-05-24 15:20:58 +00:00
|
|
|
|
"""RETURNS (str): Lowercase form of the lexeme."""
|
2017-10-27 19:07:50 +00:00
|
|
|
|
def __get__(self):
|
|
|
|
|
return self.c.lower
|
|
|
|
|
|
|
|
|
|
def __set__(self, attr_t x):
|
|
|
|
|
self.c.lower = x
|
2017-04-01 08:19:01 +00:00
|
|
|
|
|
2015-08-22 20:04:34 +00:00
|
|
|
|
property norm:
|
2021-06-15 08:57:08 +00:00
|
|
|
|
"""RETURNS (uint64): The lexeme's norm, i.e. a normalised form of the
|
2017-10-27 19:07:50 +00:00
|
|
|
|
lexeme text.
|
|
|
|
|
"""
|
|
|
|
|
def __get__(self):
|
2020-05-19 13:59:14 +00:00
|
|
|
|
return self.c.norm
|
2017-10-27 19:07:50 +00:00
|
|
|
|
|
|
|
|
|
def __set__(self, attr_t x):
|
2020-05-19 13:59:14 +00:00
|
|
|
|
if "lexeme_norm" not in self.vocab.lookups:
|
|
|
|
|
self.vocab.lookups.add_table("lexeme_norm")
|
|
|
|
|
norm_table = self.vocab.lookups.get_table("lexeme_norm")
|
|
|
|
|
norm_table[self.c.orth] = self.vocab.strings[x]
|
2017-10-27 19:07:50 +00:00
|
|
|
|
self.c.norm = x
|
2015-08-22 20:04:34 +00:00
|
|
|
|
|
|
|
|
|
property shape:
|
2017-10-27 19:07:50 +00:00
|
|
|
|
"""RETURNS (uint64): Transform of the word's string, to show
|
|
|
|
|
orthographic features.
|
|
|
|
|
"""
|
|
|
|
|
def __get__(self):
|
|
|
|
|
return self.c.shape
|
|
|
|
|
|
|
|
|
|
def __set__(self, attr_t x):
|
|
|
|
|
self.c.shape = x
|
2015-08-22 20:04:34 +00:00
|
|
|
|
|
|
|
|
|
property prefix:
|
2017-10-27 19:07:50 +00:00
|
|
|
|
"""RETURNS (uint64): Length-N substring from the start of the word.
|
|
|
|
|
Defaults to `N=1`.
|
|
|
|
|
"""
|
|
|
|
|
def __get__(self):
|
|
|
|
|
return self.c.prefix
|
|
|
|
|
|
|
|
|
|
def __set__(self, attr_t x):
|
|
|
|
|
self.c.prefix = x
|
2015-08-22 20:04:34 +00:00
|
|
|
|
|
|
|
|
|
property suffix:
|
2017-10-27 19:07:50 +00:00
|
|
|
|
"""RETURNS (uint64): Length-N substring from the end of the word.
|
|
|
|
|
Defaults to `N=3`.
|
|
|
|
|
"""
|
|
|
|
|
def __get__(self):
|
|
|
|
|
return self.c.suffix
|
|
|
|
|
|
|
|
|
|
def __set__(self, attr_t x):
|
|
|
|
|
self.c.suffix = x
|
2017-04-01 08:19:01 +00:00
|
|
|
|
|
2015-08-26 17:16:28 +00:00
|
|
|
|
property cluster:
|
2017-10-27 19:07:50 +00:00
|
|
|
|
"""RETURNS (int): Brown cluster ID."""
|
|
|
|
|
def __get__(self):
|
2020-07-22 13:59:37 +00:00
|
|
|
|
cluster_table = self.vocab.lookups.get_table("lexeme_cluster", {})
|
2020-05-19 13:59:14 +00:00
|
|
|
|
return cluster_table.get(self.c.orth, 0)
|
2017-10-27 19:07:50 +00:00
|
|
|
|
|
2020-05-19 13:59:14 +00:00
|
|
|
|
def __set__(self, int x):
|
2020-07-22 13:59:37 +00:00
|
|
|
|
cluster_table = self.vocab.lookups.get_table("lexeme_cluster", {})
|
2020-05-19 13:59:14 +00:00
|
|
|
|
cluster_table[self.c.orth] = x
|
2017-04-01 08:19:01 +00:00
|
|
|
|
|
2016-03-10 12:01:34 +00:00
|
|
|
|
property lang:
|
2017-10-27 19:07:50 +00:00
|
|
|
|
"""RETURNS (uint64): Language of the parent vocabulary."""
|
|
|
|
|
def __get__(self):
|
|
|
|
|
return self.c.lang
|
|
|
|
|
|
|
|
|
|
def __set__(self, attr_t x):
|
|
|
|
|
self.c.lang = x
|
2016-03-10 12:01:34 +00:00
|
|
|
|
|
2015-08-26 17:16:28 +00:00
|
|
|
|
property prob:
|
2017-10-27 19:07:50 +00:00
|
|
|
|
"""RETURNS (float): Smoothed log probability estimate of the lexeme's
|
|
|
|
|
type."""
|
|
|
|
|
def __get__(self):
|
2020-07-22 13:59:37 +00:00
|
|
|
|
prob_table = self.vocab.lookups.get_table("lexeme_prob", {})
|
|
|
|
|
settings_table = self.vocab.lookups.get_table("lexeme_settings", {})
|
2020-05-19 13:59:14 +00:00
|
|
|
|
default_oov_prob = settings_table.get("oov_prob", -20.0)
|
|
|
|
|
return prob_table.get(self.c.orth, default_oov_prob)
|
2017-10-27 19:07:50 +00:00
|
|
|
|
|
|
|
|
|
def __set__(self, float x):
|
2020-07-22 13:59:37 +00:00
|
|
|
|
prob_table = self.vocab.lookups.get_table("lexeme_prob", {})
|
2020-05-19 13:59:14 +00:00
|
|
|
|
prob_table[self.c.orth] = x
|
2015-08-22 20:04:34 +00:00
|
|
|
|
|
|
|
|
|
property lower_:
|
2020-05-24 15:20:58 +00:00
|
|
|
|
"""RETURNS (str): Lowercase form of the word."""
|
2017-10-27 19:07:50 +00:00
|
|
|
|
def __get__(self):
|
|
|
|
|
return self.vocab.strings[self.c.lower]
|
|
|
|
|
|
2021-09-13 15:02:17 +00:00
|
|
|
|
def __set__(self, str x):
|
2017-10-27 19:07:50 +00:00
|
|
|
|
self.c.lower = self.vocab.strings.add(x)
|
2017-04-01 08:19:01 +00:00
|
|
|
|
|
2015-08-22 20:04:34 +00:00
|
|
|
|
property norm_:
|
2021-06-15 08:57:08 +00:00
|
|
|
|
"""RETURNS (str): The lexeme's norm, i.e. a normalised form of the
|
2017-10-27 19:07:50 +00:00
|
|
|
|
lexeme text.
|
|
|
|
|
"""
|
|
|
|
|
def __get__(self):
|
|
|
|
|
return self.vocab.strings[self.c.norm]
|
|
|
|
|
|
2021-09-13 15:02:17 +00:00
|
|
|
|
def __set__(self, str x):
|
2020-05-19 13:59:14 +00:00
|
|
|
|
self.norm = self.vocab.strings.add(x)
|
2017-04-01 08:19:01 +00:00
|
|
|
|
|
2015-08-22 20:04:34 +00:00
|
|
|
|
property shape_:
|
2020-05-24 15:20:58 +00:00
|
|
|
|
"""RETURNS (str): Transform of the word's string, to show
|
2017-10-27 19:07:50 +00:00
|
|
|
|
orthographic features.
|
|
|
|
|
"""
|
|
|
|
|
def __get__(self):
|
|
|
|
|
return self.vocab.strings[self.c.shape]
|
|
|
|
|
|
2021-09-13 15:02:17 +00:00
|
|
|
|
def __set__(self, str x):
|
2017-10-27 19:07:50 +00:00
|
|
|
|
self.c.shape = self.vocab.strings.add(x)
|
2015-02-07 13:42:44 +00:00
|
|
|
|
|
2015-08-22 20:04:34 +00:00
|
|
|
|
property prefix_:
|
2020-05-24 15:20:58 +00:00
|
|
|
|
"""RETURNS (str): Length-N substring from the start of the word.
|
2017-10-27 19:07:50 +00:00
|
|
|
|
Defaults to `N=1`.
|
|
|
|
|
"""
|
|
|
|
|
def __get__(self):
|
|
|
|
|
return self.vocab.strings[self.c.prefix]
|
|
|
|
|
|
2021-09-13 15:02:17 +00:00
|
|
|
|
def __set__(self, str x):
|
2017-10-27 19:07:50 +00:00
|
|
|
|
self.c.prefix = self.vocab.strings.add(x)
|
2015-02-07 13:42:44 +00:00
|
|
|
|
|
2015-08-22 20:04:34 +00:00
|
|
|
|
property suffix_:
|
2020-05-24 15:20:58 +00:00
|
|
|
|
"""RETURNS (str): Length-N substring from the end of the word.
|
2017-10-27 19:07:50 +00:00
|
|
|
|
Defaults to `N=3`.
|
|
|
|
|
"""
|
|
|
|
|
def __get__(self):
|
|
|
|
|
return self.vocab.strings[self.c.suffix]
|
|
|
|
|
|
2021-09-13 15:02:17 +00:00
|
|
|
|
def __set__(self, str x):
|
2017-10-27 19:07:50 +00:00
|
|
|
|
self.c.suffix = self.vocab.strings.add(x)
|
2015-07-26 14:37:16 +00:00
|
|
|
|
|
2016-03-10 12:01:34 +00:00
|
|
|
|
property lang_:
|
2020-05-24 15:20:58 +00:00
|
|
|
|
"""RETURNS (str): Language of the parent vocabulary."""
|
2017-10-27 19:07:50 +00:00
|
|
|
|
def __get__(self):
|
|
|
|
|
return self.vocab.strings[self.c.lang]
|
|
|
|
|
|
2021-09-13 15:02:17 +00:00
|
|
|
|
def __set__(self, str x):
|
2017-10-27 19:07:50 +00:00
|
|
|
|
self.c.lang = self.vocab.strings.add(x)
|
2016-03-10 12:01:34 +00:00
|
|
|
|
|
2015-08-26 17:16:28 +00:00
|
|
|
|
property flags:
|
2017-10-27 19:07:50 +00:00
|
|
|
|
"""RETURNS (uint64): Container of the lexeme's binary flags."""
|
|
|
|
|
def __get__(self):
|
|
|
|
|
return self.c.flags
|
|
|
|
|
|
|
|
|
|
def __set__(self, flags_t x):
|
|
|
|
|
self.c.flags = x
|
2015-08-26 17:16:28 +00:00
|
|
|
|
|
2020-05-19 13:59:14 +00:00
|
|
|
|
@property
|
|
|
|
|
def is_oov(self):
|
2017-10-27 19:07:50 +00:00
|
|
|
|
"""RETURNS (bool): Whether the lexeme is out-of-vocabulary."""
|
2020-06-23 11:29:51 +00:00
|
|
|
|
return self.orth not in self.vocab.vectors
|
2015-07-26 23:50:06 +00:00
|
|
|
|
|
2015-09-14 08:25:40 +00:00
|
|
|
|
property is_stop:
|
2017-10-27 19:07:50 +00:00
|
|
|
|
"""RETURNS (bool): Whether the lexeme is a stop word."""
|
|
|
|
|
def __get__(self):
|
|
|
|
|
return Lexeme.c_check_flag(self.c, IS_STOP)
|
|
|
|
|
|
|
|
|
|
def __set__(self, bint x):
|
|
|
|
|
Lexeme.c_set_flag(self.c, IS_STOP, x)
|
2015-09-14 08:25:40 +00:00
|
|
|
|
|
2015-07-26 14:37:16 +00:00
|
|
|
|
property is_alpha:
|
2019-10-06 11:30:01 +00:00
|
|
|
|
"""RETURNS (bool): Whether the lexeme consists of alphabetic
|
2017-10-27 19:07:50 +00:00
|
|
|
|
characters. Equivalent to `lexeme.text.isalpha()`.
|
|
|
|
|
"""
|
|
|
|
|
def __get__(self):
|
|
|
|
|
return Lexeme.c_check_flag(self.c, IS_ALPHA)
|
|
|
|
|
|
|
|
|
|
def __set__(self, bint x):
|
|
|
|
|
Lexeme.c_set_flag(self.c, IS_ALPHA, x)
|
2017-04-01 08:19:01 +00:00
|
|
|
|
|
2015-07-26 14:37:16 +00:00
|
|
|
|
property is_ascii:
|
2017-10-27 19:07:50 +00:00
|
|
|
|
"""RETURNS (bool): Whether the lexeme consists of ASCII characters.
|
|
|
|
|
Equivalent to `[any(ord(c) >= 128 for c in lexeme.text)]`.
|
|
|
|
|
"""
|
|
|
|
|
def __get__(self):
|
|
|
|
|
return Lexeme.c_check_flag(self.c, IS_ASCII)
|
|
|
|
|
|
|
|
|
|
def __set__(self, bint x):
|
|
|
|
|
Lexeme.c_set_flag(self.c, IS_ASCII, x)
|
2015-07-26 14:37:16 +00:00
|
|
|
|
|
|
|
|
|
property is_digit:
|
2017-10-27 19:07:50 +00:00
|
|
|
|
"""RETURNS (bool): Whether the lexeme consists of digits. Equivalent
|
|
|
|
|
to `lexeme.text.isdigit()`.
|
|
|
|
|
"""
|
|
|
|
|
def __get__(self):
|
|
|
|
|
return Lexeme.c_check_flag(self.c, IS_DIGIT)
|
|
|
|
|
|
|
|
|
|
def __set__(self, bint x):
|
|
|
|
|
Lexeme.c_set_flag(self.c, IS_DIGIT, x)
|
2015-07-26 14:37:16 +00:00
|
|
|
|
|
|
|
|
|
property is_lower:
|
2017-10-27 19:07:50 +00:00
|
|
|
|
"""RETURNS (bool): Whether the lexeme is in lowercase. Equivalent to
|
|
|
|
|
`lexeme.text.islower()`.
|
|
|
|
|
"""
|
|
|
|
|
def __get__(self):
|
|
|
|
|
return Lexeme.c_check_flag(self.c, IS_LOWER)
|
|
|
|
|
|
|
|
|
|
def __set__(self, bint x):
|
|
|
|
|
Lexeme.c_set_flag(self.c, IS_LOWER, x)
|
|
|
|
|
|
|
|
|
|
property is_upper:
|
|
|
|
|
"""RETURNS (bool): Whether the lexeme is in uppercase. Equivalent to
|
|
|
|
|
`lexeme.text.isupper()`.
|
|
|
|
|
"""
|
|
|
|
|
def __get__(self):
|
|
|
|
|
return Lexeme.c_check_flag(self.c, IS_UPPER)
|
|
|
|
|
|
|
|
|
|
def __set__(self, bint x):
|
|
|
|
|
Lexeme.c_set_flag(self.c, IS_UPPER, x)
|
2015-07-26 14:37:16 +00:00
|
|
|
|
|
|
|
|
|
property is_title:
|
2017-10-27 19:07:50 +00:00
|
|
|
|
"""RETURNS (bool): Whether the lexeme is in titlecase. Equivalent to
|
|
|
|
|
`lexeme.text.istitle()`.
|
|
|
|
|
"""
|
|
|
|
|
def __get__(self):
|
|
|
|
|
return Lexeme.c_check_flag(self.c, IS_TITLE)
|
|
|
|
|
|
|
|
|
|
def __set__(self, bint x):
|
|
|
|
|
Lexeme.c_set_flag(self.c, IS_TITLE, x)
|
2015-07-26 14:37:16 +00:00
|
|
|
|
|
|
|
|
|
property is_punct:
|
2017-10-27 19:07:50 +00:00
|
|
|
|
"""RETURNS (bool): Whether the lexeme is punctuation."""
|
|
|
|
|
def __get__(self):
|
|
|
|
|
return Lexeme.c_check_flag(self.c, IS_PUNCT)
|
|
|
|
|
|
|
|
|
|
def __set__(self, bint x):
|
|
|
|
|
Lexeme.c_set_flag(self.c, IS_PUNCT, x)
|
2015-07-26 14:37:16 +00:00
|
|
|
|
|
2017-04-01 08:19:01 +00:00
|
|
|
|
property is_space:
|
2017-10-27 19:07:50 +00:00
|
|
|
|
"""RETURNS (bool): Whether the lexeme consist of whitespace characters.
|
|
|
|
|
Equivalent to `lexeme.text.isspace()`.
|
|
|
|
|
"""
|
|
|
|
|
def __get__(self):
|
|
|
|
|
return Lexeme.c_check_flag(self.c, IS_SPACE)
|
|
|
|
|
|
|
|
|
|
def __set__(self, bint x):
|
|
|
|
|
Lexeme.c_set_flag(self.c, IS_SPACE, x)
|
2015-07-26 14:37:16 +00:00
|
|
|
|
|
2017-04-01 08:19:01 +00:00
|
|
|
|
property is_bracket:
|
2017-10-27 19:07:50 +00:00
|
|
|
|
"""RETURNS (bool): Whether the lexeme is a bracket."""
|
|
|
|
|
def __get__(self):
|
|
|
|
|
return Lexeme.c_check_flag(self.c, IS_BRACKET)
|
|
|
|
|
|
|
|
|
|
def __set__(self, bint x):
|
|
|
|
|
Lexeme.c_set_flag(self.c, IS_BRACKET, x)
|
2016-02-04 12:04:16 +00:00
|
|
|
|
|
2017-04-01 08:19:01 +00:00
|
|
|
|
property is_quote:
|
2017-10-27 19:07:50 +00:00
|
|
|
|
"""RETURNS (bool): Whether the lexeme is a quotation mark."""
|
|
|
|
|
def __get__(self):
|
|
|
|
|
return Lexeme.c_check_flag(self.c, IS_QUOTE)
|
|
|
|
|
|
|
|
|
|
def __set__(self, bint x):
|
|
|
|
|
Lexeme.c_set_flag(self.c, IS_QUOTE, x)
|
2016-02-04 12:04:16 +00:00
|
|
|
|
|
2017-04-01 08:19:01 +00:00
|
|
|
|
property is_left_punct:
|
2021-02-07 00:05:43 +00:00
|
|
|
|
"""RETURNS (bool): Whether the lexeme is left punctuation, e.g. (."""
|
2017-10-27 19:07:50 +00:00
|
|
|
|
def __get__(self):
|
|
|
|
|
return Lexeme.c_check_flag(self.c, IS_LEFT_PUNCT)
|
|
|
|
|
|
|
|
|
|
def __set__(self, bint x):
|
|
|
|
|
Lexeme.c_set_flag(self.c, IS_LEFT_PUNCT, x)
|
2016-02-04 12:04:16 +00:00
|
|
|
|
|
2017-04-01 08:19:01 +00:00
|
|
|
|
property is_right_punct:
|
2017-10-27 19:07:50 +00:00
|
|
|
|
"""RETURNS (bool): Whether the lexeme is right punctuation, e.g. )."""
|
|
|
|
|
def __get__(self):
|
|
|
|
|
return Lexeme.c_check_flag(self.c, IS_RIGHT_PUNCT)
|
|
|
|
|
|
|
|
|
|
def __set__(self, bint x):
|
|
|
|
|
Lexeme.c_set_flag(self.c, IS_RIGHT_PUNCT, x)
|
2016-02-04 12:04:16 +00:00
|
|
|
|
|
2018-02-11 17:51:48 +00:00
|
|
|
|
property is_currency:
|
|
|
|
|
"""RETURNS (bool): Whether the lexeme is a currency symbol, e.g. $, €."""
|
|
|
|
|
def __get__(self):
|
|
|
|
|
return Lexeme.c_check_flag(self.c, IS_CURRENCY)
|
|
|
|
|
|
|
|
|
|
def __set__(self, bint x):
|
|
|
|
|
Lexeme.c_set_flag(self.c, IS_CURRENCY, x)
|
|
|
|
|
|
2015-07-26 14:37:16 +00:00
|
|
|
|
property like_url:
|
2017-10-27 19:07:50 +00:00
|
|
|
|
"""RETURNS (bool): Whether the lexeme resembles a URL."""
|
|
|
|
|
def __get__(self):
|
|
|
|
|
return Lexeme.c_check_flag(self.c, LIKE_URL)
|
|
|
|
|
|
|
|
|
|
def __set__(self, bint x):
|
|
|
|
|
Lexeme.c_set_flag(self.c, LIKE_URL, x)
|
2017-04-01 08:19:01 +00:00
|
|
|
|
|
2015-07-26 14:37:16 +00:00
|
|
|
|
property like_num:
|
2017-10-27 19:07:50 +00:00
|
|
|
|
"""RETURNS (bool): Whether the lexeme represents a number, e.g. "10.9",
|
|
|
|
|
"10", "ten", etc.
|
|
|
|
|
"""
|
|
|
|
|
def __get__(self):
|
|
|
|
|
return Lexeme.c_check_flag(self.c, LIKE_NUM)
|
|
|
|
|
|
|
|
|
|
def __set__(self, bint x):
|
|
|
|
|
Lexeme.c_set_flag(self.c, LIKE_NUM, x)
|
2015-07-26 14:37:16 +00:00
|
|
|
|
|
|
|
|
|
property like_email:
|
2017-10-27 19:07:50 +00:00
|
|
|
|
"""RETURNS (bool): Whether the lexeme resembles an email address."""
|
|
|
|
|
def __get__(self):
|
|
|
|
|
return Lexeme.c_check_flag(self.c, LIKE_EMAIL)
|
|
|
|
|
|
|
|
|
|
def __set__(self, bint x):
|
|
|
|
|
Lexeme.c_set_flag(self.c, LIKE_EMAIL, x)
|