spaCy/spacy/lang.pxd

63 lines
1.8 KiB
Cython
Raw Normal View History

from libcpp.vector cimport vector
2014-09-10 16:11:13 +00:00
from preshed.maps cimport PreshMap
from cymem.cymem cimport Pool
from .typedefs cimport hash_t
from .tokens cimport Tokens
from .lexeme cimport Lexeme
2014-11-02 03:21:43 +00:00
from .pos cimport Tagger as PosTagger
from .utf8string cimport StringStore
2014-09-10 16:11:13 +00:00
cdef extern from "Python.h":
cdef bint Py_UNICODE_ISSPACE(Py_UNICODE ch)
cdef bint Py_UNICODE_ISALNUM(Py_UNICODE ch)
cdef bint Py_UNICODE_ISALPHA(Py_UNICODE ch)
cdef bint Py_UNICODE_ISUPPER(Py_UNICODE ch)
cdef struct String:
Py_UNICODE* chars
size_t n
hash_t key
cdef class Lexicon:
cdef Pool mem
2014-09-10 16:11:13 +00:00
cpdef readonly size_t size
cpdef readonly StringStore strings
2014-10-31 06:43:25 +00:00
cdef vector[Lexeme*] lexemes
2014-09-10 16:11:13 +00:00
cdef Lexeme* get(self, String* s) except NULL
2014-08-28 23:59:23 +00:00
cdef PreshMap _dict
2014-08-28 23:59:23 +00:00
cdef class Language:
2014-10-30 07:01:27 +00:00
cdef Pool mem
cdef readonly unicode name
cdef PreshMap _cache
cdef PreshMap _specials
cpdef readonly Lexicon lexicon
2014-11-02 03:21:43 +00:00
cpdef readonly PosTagger pos_tagger
2014-10-30 07:01:27 +00:00
cdef object _prefix_re
cdef object _suffix_re
cdef object _infix_re
2014-09-11 19:37:32 +00:00
cpdef Tokens tokenize(self, unicode text)
2014-11-02 03:21:43 +00:00
cpdef Tokens pos_tag(self, Tokens t)
2014-10-14 07:22:41 +00:00
cdef int _tokenize(self, Tokens tokens, String* span, int start, int end) except -1
cdef String* _split_affixes(self, String* string, vector[Lexeme*] *prefixes,
vector[Lexeme*] *suffixes) except NULL
cdef int _attach_tokens(self, Tokens tokens, int idx, String* string,
vector[Lexeme*] *prefixes, vector[Lexeme*] *suffixes) except -1
cdef int _find_prefix(self, Py_UNICODE* characters, size_t length) except -1
cdef int _find_suffix(self, Py_UNICODE* characters, size_t length) except -1
cdef int _find_infix(self, Py_UNICODE* characters, size_t length) except -1
cdef int _save_cached(self, Lexeme** tokens, hash_t key, int n) except -1