spaCy/spacy/tokens.pxd

from libc.stdint cimport uint32_t

import numpy as np
cimport numpy as np

from cymem.cymem cimport Pool
from thinc.typedefs cimport atom_t

from .lexeme cimport Lexeme

from .typedefs cimport flags_t
from .typedefs cimport Morphology
from .lang cimport Language


cdef struct TokenC:
    const Lexeme* lex
    Morphology morph
    int idx
    int pos
    int lemma
    int sense
    int head
    int dep_tag
    uint32_t l_kids
    uint32_t r_kids


ctypedef const Lexeme* const_Lexeme_ptr
ctypedef TokenC* TokenC_ptr

ctypedef fused LexemeOrToken:
    const_Lexeme_ptr
    TokenC_ptr


cdef class Tokens:
    cdef Pool mem
    cdef Language lang
    cdef list tag_names

    cdef TokenC* data

    cdef int length
    cdef int max_length

    cdef int push_back(self, int i, LexemeOrToken lex_or_tok) except -1

    cpdef np.ndarray[long, ndim=2] get_array(self, list features)


cdef class Token:
    cdef public Language lang
    cdef public int i
    cdef public int idx
    cdef int pos
    cdef int lemma
    cdef public int head
    cdef public int dep_tag

    cdef public atom_t id
    cdef public atom_t cluster
    cdef public atom_t length
    cdef public atom_t postype
    cdef public atom_t sensetype

    cdef public atom_t sic
    cdef public atom_t norm
    cdef public atom_t shape
    cdef public atom_t asciied
    cdef public atom_t prefix
    cdef public atom_t suffix

    cdef public float prob

    cdef public flags_t flags
* Work on greedy parser 2014-12-16 11:44:43 +00:00			`from libc.stdint cimport uint32_t`

* Work on get_array method of Tokens 2014-12-02 12:48:05 +00:00			`import numpy as np`
			`cimport numpy as np`

* Remove the feature array stuff from Tokens class, and replace vector with array-based implementation, with padding. 2014-10-22 14:57:59 +00:00			`from cymem.cymem cimport Pool`
* Add as_array and count_by method 2014-12-04 09:46:55 +00:00			`from thinc.typedefs cimport atom_t`
* Remove the feature array stuff from Tokens class, and replace vector with array-based implementation, with padding. 2014-10-22 14:57:59 +00:00
* Large refactor, particularly to Python API 2014-10-23 13:59:17 +00:00			`from .lexeme cimport Lexeme`
* Work on morphological processing 2014-12-08 10:12:15 +00:00
* Hack Tokens to work without tagger.pyx 2014-12-03 00:05:15 +00:00			`from .typedefs cimport flags_t`
* Working morphology and lemmatisation. POS tagging quite fast. 2014-12-09 21:09:32 +00:00			`from .typedefs cimport Morphology`
			`from .lang cimport Language`
* Work on morphological processing 2014-12-08 10:12:15 +00:00

* Large refactor, particularly to Python API 2014-10-23 13:59:17 +00:00
* Introduce a TokenC struct, to handle token indices, pos tags and sense tags 2014-12-05 04:56:14 +00:00			`cdef struct TokenC:`
			`const Lexeme* lex`
* Work on morphological processing 2014-12-08 10:12:15 +00:00			`Morphology morph`
* Introduce a TokenC struct, to handle token indices, pos tags and sense tags 2014-12-05 04:56:14 +00:00			`int idx`
			`int pos`
* Work on morphological processing 2014-12-08 10:12:15 +00:00			`int lemma`
* Introduce a TokenC struct, to handle token indices, pos tags and sense tags 2014-12-05 04:56:14 +00:00			`int sense`
* Work on greedy parser 2014-12-16 11:44:43 +00:00			`int head`
			`int dep_tag`
			`uint32_t l_kids`
			`uint32_t r_kids`
* Introduce a TokenC struct, to handle token indices, pos tags and sense tags 2014-12-05 04:56:14 +00:00

* Use fused type in Tokens.push_back, simplifying the use of the cache 2014-12-09 05:50:01 +00:00			`ctypedef const Lexeme* const_Lexeme_ptr`
			`ctypedef TokenC* TokenC_ptr`

			`ctypedef fused LexemeOrToken:`
			`const_Lexeme_ptr`
			`TokenC_ptr`


* Upd Tokens to use vector, with bounds checking. 2014-09-15 01:22:40 +00:00			`cdef class Tokens:`
* Remove the feature array stuff from Tokens class, and replace vector with array-based implementation, with padding. 2014-10-22 14:57:59 +00:00			`cdef Pool mem`
* Working morphology and lemmatisation. POS tagging quite fast. 2014-12-09 21:09:32 +00:00			`cdef Language lang`
			`cdef list tag_names`
* Remove the feature array stuff from Tokens class, and replace vector with array-based implementation, with padding. 2014-10-22 14:57:59 +00:00
* Introduce a TokenC struct, to handle token indices, pos tags and sense tags 2014-12-05 04:56:14 +00:00			`cdef TokenC* data`
* Remove the feature array stuff from Tokens class, and replace vector with array-based implementation, with padding. 2014-10-22 14:57:59 +00:00
			`cdef int length`
			`cdef int max_length`
* Have tokens track tuples that record the start offset and pos tag as well as a lexeme pointer 2014-10-14 04:21:03 +00:00
* Use fused type in Tokens.push_back, simplifying the use of the cache 2014-12-09 05:50:01 +00:00			`cdef int push_back(self, int i, LexemeOrToken lex_or_tok) except -1`
* Large refactor, particularly to Python API 2014-10-23 13:59:17 +00:00
* Add as_array and count_by method 2014-12-04 09:46:55 +00:00			`cpdef np.ndarray[long, ndim=2] get_array(self, list features)`
* Work on get_array method of Tokens 2014-12-02 12:48:05 +00:00
* Large refactor, particularly to Python API 2014-10-23 13:59:17 +00:00
			`cdef class Token:`
* Working morphology and lemmatisation. POS tagging quite fast. 2014-12-09 21:09:32 +00:00			`cdef public Language lang`
* Large refactor, particularly to Python API 2014-10-23 13:59:17 +00:00			`cdef public int i`
			`cdef public int idx`
* Working morphology and lemmatisation. POS tagging quite fast. 2014-12-09 21:09:32 +00:00			`cdef int pos`
* Work on morphological processing 2014-12-08 10:12:15 +00:00			`cdef int lemma`
* Work on parser 2014-12-17 10:10:12 +00:00			`cdef public int head`
			`cdef public int dep_tag`
* Large refactor, particularly to Python API 2014-10-23 13:59:17 +00:00
* Rename Token.lex_pos to Token.postype, and Token.lex_supersense to Token.sensetype 2014-10-31 06:44:39 +00:00			`cdef public atom_t id`
* Large refactor, particularly to Python API 2014-10-23 13:59:17 +00:00			`cdef public atom_t cluster`
			`cdef public atom_t length`
* Rename Token.lex_pos to Token.postype, and Token.lex_supersense to Token.sensetype 2014-10-31 06:44:39 +00:00			`cdef public atom_t postype`
			`cdef public atom_t sensetype`
* Large refactor, particularly to Python API 2014-10-23 13:59:17 +00:00
* Rewriting Lexeme serialization. 2014-10-29 12:19:38 +00:00			`cdef public atom_t sic`
* Large refactor, particularly to Python API 2014-10-23 13:59:17 +00:00			`cdef public atom_t norm`
			`cdef public atom_t shape`
			`cdef public atom_t asciied`
			`cdef public atom_t prefix`
			`cdef public atom_t suffix`

			`cdef public float prob`

* Hack Tokens to work without tagger.pyx 2014-12-03 00:05:15 +00:00			`cdef public flags_t flags`