Rooba
/
spaCy
mirror of https://github.com/explosion/spaCy.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity
spaCy/spacy/tokens.pxd

63 lines
1.2 KiB
Cython
Raw Normal View History

* Work on greedy parser
2014-12-16 11:44:43 +00:00
from libc.stdint cimport uint32_t
* Work on get_array method of Tokens
2014-12-02 12:48:05 +00:00
import numpy as np
cimport numpy as np
* Remove the feature array stuff from Tokens class, and replace vector with array-based implementation, with padding.
2014-10-22 14:57:59 +00:00
from cymem.cymem cimport Pool
* Update tokens.pyx as part of reorg
2014-12-19 20:03:26 +00:00
from .structs cimport Lexeme, TokenC, Morphology
* Work on morphological processing
2014-12-08 10:12:15 +00:00
* Update tokens.pyx as part of reorg
2014-12-19 20:03:26 +00:00
from .typedefs cimport flags_t, attr_t, flags_t
* Work on morphological processing
2014-12-08 10:12:15 +00:00
* Update tokens.pyx as part of reorg
2014-12-19 20:03:26 +00:00
from .strings cimport StringStore
* Introduce a TokenC struct, to handle token indices, pos tags and sense tags
2014-12-05 04:56:14 +00:00
* Use fused type in Tokens.push_back, simplifying the use of the cache
2014-12-09 05:50:01 +00:00
ctypedef const Lexeme* const_Lexeme_ptr
ctypedef TokenC* TokenC_ptr
ctypedef fused LexemeOrToken:
const_Lexeme_ptr
TokenC_ptr
* Upd Tokens to use vector, with bounds checking.
2014-09-15 01:22:40 +00:00
cdef class Tokens:
* Remove the feature array stuff from Tokens class, and replace vector with array-based implementation, with padding.
2014-10-22 14:57:59 +00:00
cdef Pool mem
* Update tokens.pyx as part of reorg
2014-12-19 20:03:26 +00:00
cdef StringStore strings
* Working morphology and lemmatisation. POS tagging quite fast.
2014-12-09 21:09:32 +00:00
cdef list tag_names
* Remove the feature array stuff from Tokens class, and replace vector with array-based implementation, with padding.
2014-10-22 14:57:59 +00:00
* Introduce a TokenC struct, to handle token indices, pos tags and sense tags
2014-12-05 04:56:14 +00:00
cdef TokenC* data
* Remove the feature array stuff from Tokens class, and replace vector with array-based implementation, with padding.
2014-10-22 14:57:59 +00:00
cdef int length
cdef int max_length
* Have tokens track tuples that record the start offset and pos tag as well as a lexeme pointer
2014-10-14 04:21:03 +00:00
* Use fused type in Tokens.push_back, simplifying the use of the cache
2014-12-09 05:50:01 +00:00
cdef int push_back(self, int i, LexemeOrToken lex_or_tok) except -1
* Large refactor, particularly to Python API
2014-10-23 13:59:17 +00:00
* Add as_array and count_by method
2014-12-04 09:46:55 +00:00
cpdef np.ndarray[long, ndim=2] get_array(self, list features)
* Work on get_array method of Tokens
2014-12-02 12:48:05 +00:00
* Large refactor, particularly to Python API
2014-10-23 13:59:17 +00:00
cdef class Token:
* Update tokens.pyx as part of reorg
2014-12-19 20:03:26 +00:00
cdef public StringStore strings
* Large refactor, particularly to Python API
2014-10-23 13:59:17 +00:00
cdef public int i
cdef public int idx
* Working morphology and lemmatisation. POS tagging quite fast.
2014-12-09 21:09:32 +00:00
cdef int pos
* Work on morphological processing
2014-12-08 10:12:15 +00:00
cdef int lemma
* Work on parser
2014-12-17 10:10:12 +00:00
cdef public int head
cdef public int dep_tag
* Large refactor, particularly to Python API
2014-10-23 13:59:17 +00:00
* Update tokens.pyx as part of reorg
2014-12-19 20:03:26 +00:00
cdef public attr_t id
cdef public attr_t cluster
cdef public attr_t length
cdef public attr_t postype
cdef public attr_t sensetype
cdef public attr_t sic
cdef public attr_t norm
cdef public attr_t shape
cdef public attr_t asciied
cdef public attr_t prefix
cdef public attr_t suffix
* Large refactor, particularly to Python API
2014-10-23 13:59:17 +00:00
cdef public float prob
* Hack Tokens to work without tagger.pyx
2014-12-03 00:05:15 +00:00
cdef public flags_t flags