spaCy/spacy/tokens.pxd

from libc.stdint cimport uint32_t

from cython.view cimport array as cvarray

from cymem.cymem cimport Pool
from thinc.typedefs cimport atom_t

from .typedefs cimport flags_t, attr_id_t, attr_t
from .structs cimport Morphology, TokenC, Lexeme
from .vocab cimport Vocab
from .strings cimport StringStore


ctypedef const Lexeme* const_Lexeme_ptr
ctypedef TokenC* TokenC_ptr

ctypedef fused LexemeOrToken:
    const_Lexeme_ptr
    TokenC_ptr


cdef attr_t get_lex_attr(const Lexeme* lex, attr_id_t feat_name) nogil
cdef attr_t get_token_attr(const TokenC* lex, attr_id_t feat_name) nogil

cdef inline bint check_flag(const Lexeme* lexeme, attr_id_t flag_id) nogil:
    return lexeme.flags & (1 << flag_id)


cdef class Tokens:
    cdef Pool mem
    cdef Vocab vocab
    cdef list tag_names

    cdef TokenC* data

    cdef int length
    cdef int max_length

    cdef int push_back(self, int i, LexemeOrToken lex_or_tok) except -1

    cpdef long[:,:] to_array(self, object features)


cdef class Token:
    cdef Tokens _seq
    cdef readonly int i
* Work on greedy parser 2014-12-16 11:44:43 +00:00			`from libc.stdint cimport uint32_t`

* Make PyPy work 2015-01-05 06:54:13 +00:00			`from cython.view cimport array as cvarray`
* Work on get_array method of Tokens 2014-12-02 12:48:05 +00:00
* Remove the feature array stuff from Tokens class, and replace vector with array-based implementation, with padding. 2014-10-22 14:57:59 +00:00			`from cymem.cymem cimport Pool`
* Work on train 2014-12-21 20:25:43 +00:00			`from thinc.typedefs cimport atom_t`
* Remove the feature array stuff from Tokens class, and replace vector with array-based implementation, with padding. 2014-10-22 14:57:59 +00:00
* Tmp 2014-12-24 06:42:00 +00:00			`from .typedefs cimport flags_t, attr_id_t, attr_t`
* Work on train 2014-12-21 20:25:43 +00:00			`from .structs cimport Morphology, TokenC, Lexeme`
			`from .vocab cimport Vocab`
* Update tokens.pyx as part of reorg 2014-12-19 20:03:26 +00:00			`from .strings cimport StringStore`
* Introduce a TokenC struct, to handle token indices, pos tags and sense tags 2014-12-05 04:56:14 +00:00

* Use fused type in Tokens.push_back, simplifying the use of the cache 2014-12-09 05:50:01 +00:00			`ctypedef const Lexeme* const_Lexeme_ptr`
			`ctypedef TokenC* TokenC_ptr`

			`ctypedef fused LexemeOrToken:`
			`const_Lexeme_ptr`
			`TokenC_ptr`


* Tmp 2014-12-24 06:42:00 +00:00			`cdef attr_t get_lex_attr(const Lexeme* lex, attr_id_t feat_name) nogil`
			`cdef attr_t get_token_attr(const TokenC* lex, attr_id_t feat_name) nogil`

			`cdef inline bint check_flag(const Lexeme* lexeme, attr_id_t flag_id) nogil:`
			`return lexeme.flags & (1 << flag_id)`


* Upd Tokens to use vector, with bounds checking. 2014-09-15 01:22:40 +00:00			`cdef class Tokens:`
* Remove the feature array stuff from Tokens class, and replace vector with array-based implementation, with padding. 2014-10-22 14:57:59 +00:00			`cdef Pool mem`
* Work on train 2014-12-21 20:25:43 +00:00			`cdef Vocab vocab`
* Working morphology and lemmatisation. POS tagging quite fast. 2014-12-09 21:09:32 +00:00			`cdef list tag_names`
* Remove the feature array stuff from Tokens class, and replace vector with array-based implementation, with padding. 2014-10-22 14:57:59 +00:00
* Introduce a TokenC struct, to handle token indices, pos tags and sense tags 2014-12-05 04:56:14 +00:00			`cdef TokenC* data`
* Remove the feature array stuff from Tokens class, and replace vector with array-based implementation, with padding. 2014-10-22 14:57:59 +00:00
			`cdef int length`
			`cdef int max_length`
* Have tokens track tuples that record the start offset and pos tag as well as a lexeme pointer 2014-10-14 04:21:03 +00:00
* Use fused type in Tokens.push_back, simplifying the use of the cache 2014-12-09 05:50:01 +00:00			`cdef int push_back(self, int i, LexemeOrToken lex_or_tok) except -1`
* Large refactor, particularly to Python API 2014-10-23 13:59:17 +00:00
* Make PyPy work 2015-01-05 06:54:13 +00:00			`cpdef long[:,:] to_array(self, object features)`
* Work on get_array method of Tokens 2014-12-02 12:48:05 +00:00
* Large refactor, particularly to Python API 2014-10-23 13:59:17 +00:00
			`cdef class Token:`
* Tmp 2014-12-24 06:42:00 +00:00			`cdef Tokens _seq`
			`cdef readonly int i`