2014-12-09 21:09:32 +00:00
|
|
|
|
|
|
|
|
from .tokens cimport TokenC
|
2014-12-09 10:16:17 +00:00
|
|
|
from .lexeme cimport Lexeme
|
|
|
|
|
from .utf8string cimport StringStore
|
2014-12-09 21:09:32 +00:00
|
|
|
from .typedefs cimport id_t, Morphology
|
2014-12-09 10:16:17 +00:00
|
|
|
|
|
|
|
|
from preshed.maps cimport PreshMapArray
|
|
|
|
|
from cymem.cymem cimport Pool
|
|
|
|
|
|
2014-12-09 21:09:32 +00:00
|
|
|
|
2014-12-09 10:16:17 +00:00
|
|
|
# Google universal tag set
|
|
|
|
|
cpdef enum univ_tag_t:
|
|
|
|
|
NO_TAG
|
|
|
|
|
ADJ
|
|
|
|
|
ADV
|
|
|
|
|
ADP
|
|
|
|
|
CONJ
|
|
|
|
|
DET
|
|
|
|
|
NOUN
|
|
|
|
|
NUM
|
|
|
|
|
PRON
|
|
|
|
|
PRT
|
|
|
|
|
VERB
|
|
|
|
|
X
|
|
|
|
|
PUNCT
|
|
|
|
|
EOL
|
|
|
|
|
N_UNIV_TAGS
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
cdef struct PosTag:
|
|
|
|
|
Morphology morph
|
|
|
|
|
int id
|
|
|
|
|
univ_tag_t pos
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
cdef class Morphologizer:
|
|
|
|
|
cdef Pool mem
|
|
|
|
|
cdef StringStore strings
|
|
|
|
|
cdef object lemmatizer
|
|
|
|
|
cdef PosTag* tags
|
2014-12-09 14:02:04 +00:00
|
|
|
cdef readonly list tag_names
|
2014-12-09 10:16:17 +00:00
|
|
|
|
2014-12-09 14:02:04 +00:00
|
|
|
cdef PreshMapArray _cache
|
2014-12-09 10:16:17 +00:00
|
|
|
cdef int lemmatize(self, const univ_tag_t pos, const Lexeme* lex) except -1
|
|
|
|
|
cdef int set_morph(self, const int i, TokenC* tokens) except -1
|
