2014-08-02 20:51:52 +00:00
|
|
|
from libc.stdint cimport uint32_t
|
2014-07-05 18:51:42 +00:00
|
|
|
from libc.stdint cimport uint64_t
|
|
|
|
|
2014-07-07 02:21:06 +00:00
|
|
|
# Put these above import to avoid circular import problem
|
2014-07-05 18:51:42 +00:00
|
|
|
ctypedef int ClusterID
|
2014-08-02 20:51:52 +00:00
|
|
|
ctypedef uint32_t StringHash
|
2014-07-07 02:21:06 +00:00
|
|
|
ctypedef size_t Lexeme_addr
|
2014-07-07 14:58:48 +00:00
|
|
|
ctypedef char Bits8
|
|
|
|
ctypedef uint64_t Bits64
|
|
|
|
|
2014-07-05 18:51:42 +00:00
|
|
|
|
2014-07-07 14:58:48 +00:00
|
|
|
cdef struct Orthography:
|
|
|
|
StringHash last3
|
|
|
|
StringHash shape
|
|
|
|
StringHash norm
|
|
|
|
|
2014-08-02 20:26:44 +00:00
|
|
|
size_t length
|
2014-08-16 14:09:24 +00:00
|
|
|
unsigned char first
|
2014-07-07 14:58:48 +00:00
|
|
|
Bits8 flags
|
|
|
|
|
|
|
|
|
|
|
|
cdef struct Distribution:
|
|
|
|
double prob
|
|
|
|
ClusterID cluster
|
|
|
|
Bits64 tagdict
|
|
|
|
Bits8 flags
|
|
|
|
|
|
|
|
|
2014-07-05 18:51:42 +00:00
|
|
|
cdef struct Lexeme:
|
|
|
|
StringHash sic # Hash of the original string
|
|
|
|
StringHash lex # Hash of the word, with punctuation and clitics split off
|
|
|
|
|
2014-07-07 14:58:48 +00:00
|
|
|
Distribution* dist # Distribution info, lazy loaded
|
|
|
|
Orthography* orth # Extra orthographic views
|
2014-07-05 18:51:42 +00:00
|
|
|
Lexeme* tail # Lexemes are linked lists, to deal with sub-tokens
|
|
|
|
|
|
|
|
|
2014-07-07 14:58:48 +00:00
|
|
|
cdef Lexeme BLANK_WORD = Lexeme(0, 0, NULL, NULL, NULL)
|
2014-07-07 02:21:06 +00:00
|
|
|
|
2014-07-07 14:58:48 +00:00
|
|
|
|
2014-07-07 18:27:02 +00:00
|
|
|
cdef enum StringAttr:
|
|
|
|
SIC
|
|
|
|
LEX
|
|
|
|
NORM
|
|
|
|
SHAPE
|
|
|
|
LAST3
|
2014-08-02 20:26:44 +00:00
|
|
|
LENGTH
|
2014-07-07 18:27:02 +00:00
|
|
|
|
|
|
|
|
|
|
|
cpdef StringHash attr_of(size_t lex_id, StringAttr attr) except 0
|
|
|
|
|
|
|
|
cpdef StringHash sic_of(size_t lex_id) except 0
|
2014-07-07 14:58:48 +00:00
|
|
|
cpdef StringHash lex_of(size_t lex_id) except 0
|
|
|
|
cpdef StringHash norm_of(size_t lex_id) except 0
|
2014-07-07 17:12:19 +00:00
|
|
|
cpdef StringHash shape_of(size_t lex_id) except 0
|
2014-07-07 18:27:02 +00:00
|
|
|
cpdef StringHash last3_of(size_t lex_id) except 0
|
2014-08-02 20:26:44 +00:00
|
|
|
cpdef StringHash length_of(size_t lex_id)
|