spaCy/spacy/lexeme.pyx

55 lines
1.6 KiB
Cython
Raw Normal View History

# cython: embedsignature=True
from cpython.ref cimport Py_INCREF
from cymem.cymem cimport Pool
2014-10-29 12:19:38 +00:00
from murmurhash.mrmr cimport hash64
from libc.string cimport memset
2015-01-05 07:49:19 +00:00
from .orth cimport word_shape
from .typedefs cimport attr_t, flags_t
import numpy
2014-09-10 18:41:37 +00:00
memset(&EMPTY_LEXEME, 0, sizeof(LexemeC))
cdef int set_lex_struct_props(LexemeC* lex, dict props, StringStore string_store,
const float* empty_vec) except -1:
lex.length = props['length']
2015-01-22 15:08:25 +00:00
lex.orth = string_store[props['orth']]
2015-04-19 08:31:31 +00:00
lex.lower = string_store[props['lower']]
lex.norm = string_store[props['norm']]
lex.shape = string_store[props['shape']]
lex.prefix = string_store[props['prefix']]
lex.suffix = string_store[props['suffix']]
2015-04-19 08:31:31 +00:00
lex.cluster = props['cluster']
lex.prob = props['prob']
lex.sentiment = props['sentiment']
lex.flags = props['flags']
cdef flags_t sense_id
for sense_id in props.get('senses', []):
lex.senses |= 1 << sense_id
2015-01-21 15:03:54 +00:00
lex.repvec = empty_vec
cdef class Lexeme:
2015-01-24 09:48:34 +00:00
"""An entry in the vocabulary. A Lexeme has no string context --- it's a
word-type, as opposed to a word token. It therefore has no part-of-speech
tag, dependency parse, or lemma (lemmatization depends on the part-of-speech
tag).
"""
def __cinit__(self, int vec_size):
2015-01-22 15:08:25 +00:00
self.repvec = numpy.ndarray(shape=(vec_size,), dtype=numpy.float32)
@property
def has_repvec(self):
return self.l2_norm != 0
cpdef bint check(self, attr_id_t flag_id) except -1:
return self.flags & (1 << flag_id)
cpdef bint has_sense(self, flags_t flag_id) except -1:
return self.senses & (1 << flag_id)