2014-09-15 01:22:40 +00:00
|
|
|
from cpython.ref cimport Py_INCREF
|
2014-09-17 21:09:24 +00:00
|
|
|
from cymem.cymem cimport Pool
|
2014-10-29 12:19:38 +00:00
|
|
|
from murmurhash.mrmr cimport hash64
|
2014-09-15 01:22:40 +00:00
|
|
|
|
2014-10-22 14:57:59 +00:00
|
|
|
from libc.string cimport memset
|
|
|
|
|
|
2015-01-05 07:49:19 +00:00
|
|
|
from .orth cimport word_shape
|
2014-09-10 18:41:37 +00:00
|
|
|
|
2014-10-09 08:53:30 +00:00
|
|
|
|
2014-10-23 13:59:17 +00:00
|
|
|
memset(&EMPTY_LEXEME, 0, sizeof(Lexeme))
|
2014-10-09 08:53:30 +00:00
|
|
|
|
|
|
|
|
|
2014-10-31 06:43:00 +00:00
|
|
|
cpdef Lexeme init(id_t i, unicode string, hash_t hashed,
|
2014-12-03 00:04:00 +00:00
|
|
|
StringStore string_store, dict props) except *:
|
2014-10-29 12:19:38 +00:00
|
|
|
cdef Lexeme lex
|
2014-10-31 06:43:00 +00:00
|
|
|
lex.id = i
|
2014-10-29 12:19:38 +00:00
|
|
|
lex.length = len(string)
|
2014-12-03 00:04:00 +00:00
|
|
|
lex.sic = string_store[string]
|
2014-10-29 12:19:38 +00:00
|
|
|
|
|
|
|
|
lex.cluster = props.get('cluster', 0)
|
2014-12-03 00:04:00 +00:00
|
|
|
lex.pos_type = props.get('pos_type', 0)
|
2014-10-29 12:19:38 +00:00
|
|
|
lex.prob = props.get('prob', 0)
|
|
|
|
|
|
2014-12-03 00:04:00 +00:00
|
|
|
lex.prefix = string_store[string[:1]]
|
|
|
|
|
lex.suffix = string_store[string[-3:]]
|
2015-01-05 07:49:19 +00:00
|
|
|
lex.shape = string_store[word_shape(string)]
|
2014-12-03 00:04:00 +00:00
|
|
|
|
|
|
|
|
lex.flags = props.get('flags', 0)
|
2014-10-29 12:19:38 +00:00
|
|
|
return lex
|
