spaCy/spacy/lexeme.pyx

105 lines
3.2 KiB
Cython

from cpython.ref cimport Py_INCREF
from cymem.cymem cimport Pool
import orth
OOV_DIST_FLAGS = 0
cpdef dict get_lexeme_dict(size_t i, unicode string):
ints = [None for _ in range(LexInt_N)]
ints[<int>LexInt_i] = i
ints[<int>LexInt_length] = len(string)
ints[<int>LexInt_cluster] = 0
ints[<int>LexInt_pos] = 0
ints[<int>LexInt_supersense] = 0
floats = [None for _ in range(LexFloat_N)]
floats[<int>LexFloat_prob] = 0
floats[<int>LexFloat_sentiment] = 0
strings = [None for _ in range(LexStr_N)]
strings[<int>LexStr_key] = string
strings[<int>LexStr_casefix] = strings[<int>LexStr_key]
strings[<int>LexStr_shape] = orth.word_shape(string)
strings[<int>LexStr_unsparse] = strings[<int>LexStr_shape]
strings[<int>LexStr_asciied] = orth.asciied(string)
orth_flags = get_orth_flags(string)
dist_flags = OOV_DIST_FLAGS
return {'ints': ints, 'floats': floats, 'strings': strings,
'orth_flags': orth_flags, 'dist_flags': dist_flags}
def get_orth_flags(unicode string):
cdef flag_t flags = 0
flags |= orth.is_ascii(string) << LexOrth_ascii
flags |= orth.is_alpha(string) << LexOrth_alpha
flags |= orth.is_digit(string) << LexOrth_digit
flags |= orth.is_lower(string) << LexOrth_lower
flags |= orth.is_punct(string) << LexOrth_punct
flags |= orth.is_space(string) << LexOrth_space
flags |= orth.is_title(string) << LexOrth_title
flags |= orth.is_upper(string) << LexOrth_upper
return flags
def get_dist_flags(unicode string):
return 0
cdef char* intern_and_encode(unicode string, size_t* length) except NULL:
cdef bytes byte_string = string.encode('utf8')
cdef bytes utf8_string = intern(byte_string)
Py_INCREF(utf8_string)
length[0] = len(utf8_string)
return <char*>utf8_string
cdef int lexeme_get_int(LexemeC* lexeme, size_t i) except *:
return lexeme.ints[i]
cdef float lexeme_get_float(LexemeC* lexeme, size_t i) except *:
return lexeme.floats[i]
cdef unicode lexeme_get_string(LexemeC* lexeme, size_t i):
cdef bytes byte_string = lexeme.strings[i]
return byte_string.decode('utf8')
cdef bint lexeme_check_orth_flag(LexemeC* lexeme, size_t flag_id) except *:
return lexeme.orth_flags & (1 << flag_id)
cdef bint lexeme_check_dist_flag(LexemeC* lexeme, size_t flag_id) except *:
return lexeme.dist_flags & (1 << flag_id)
cdef dict lexeme_pack(LexemeC* lex):
cdef dict packed = {}
packed['ints'] = [lex.ints[i] for i in range(LexInt_N)]
packed['floats'] = [lex.floats[i] for i in range(LexFloat_N)]
packed['strings'] = [lex.strings[i].decode('utf8') for i in range(LexStr_N)]
packed['orth_flags'] = lex.orth_flags
packed['dist_flags'] = lex.orth_flags
return packed
cdef int lexeme_unpack(LexemeC* lex, dict p) except -1:
cdef size_t i
cdef int lex_int
cdef float lex_float
cdef unicode string
for i, lex_int in enumerate(p['ints']):
lex.ints[i] = lex_int
for i, lex_float in enumerate(p['floats']):
lex.floats[i] = lex_float
cdef size_t _
for i, lex_string in enumerate(p['strings']):
lex.strings[i] = intern_and_encode(lex_string, &_)
lex.orth_flags = p['orth_flags']
lex.dist_flags = p['dist_flags']