Rooba
/
spaCy
mirror of https://github.com/explosion/spaCy.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity
spaCy/spacy/tokenizer.pxd

34 lines
1.1 KiB
Cython
Raw Normal View History

* Move tokenizer to its own file, and refactor
2014-12-19 20:29:16 +00:00
from libcpp.vector cimport vector
from preshed.maps cimport PreshMap
from cymem.cymem cimport Pool
from .typedefs cimport hash_t
* Hack out morphology stuff from tokenizer, while morphology being reimplemented.
2015-08-26 17:20:11 +00:00
from .structs cimport LexemeC, TokenC
* Move tokenizer to its own file, and refactor
2014-12-19 20:29:16 +00:00
from .strings cimport StringStore
* Break up tokens.pyx into tokens/doc.pyx, tokens/token.pyx, tokens/spans.pyx
2015-07-13 18:20:58 +00:00
from .tokens.doc cimport Doc
* More work on language-generic parsing
2015-08-28 00:02:33 +00:00
from .vocab cimport Vocab, LexemesOrTokens, _Cached
* Move tokenizer to its own file, and refactor
2014-12-19 20:29:16 +00:00
cdef class Tokenizer:
cdef Pool mem
cdef PreshMap _cache
cdef PreshMap _specials
cpdef readonly Vocab vocab
cdef object _prefix_re
cdef object _suffix_re
cdef object _infix_re
Add __reduce__ to Tokenizer so that English pickles. - Add tests to test_pickle and test_tokenizer that save to tempfiles.
2015-10-24 05:18:47 +00:00
cdef object _rules
* Move tokenizer to its own file, and refactor
2014-12-19 20:29:16 +00:00
* Rename Tokens to Doc
2015-07-08 16:53:00 +00:00
cpdef Doc tokens_from_list(self, list strings)
* Move tokenizer to its own file, and refactor
2014-12-19 20:29:16 +00:00
* Refactor tokenizer, to set the 'spacy' field on TokenC instead of passing a string
2015-07-13 19:46:02 +00:00
cdef int _try_cache(self, hash_t key, Doc tokens) except -1
* Replace UniStr, using unicode objects instead
2015-07-22 02:49:39 +00:00
cdef int _tokenize(self, Doc tokens, unicode span, hash_t key) except -1
* Fix bug in tokenizer that caused new tokens to be added for affixes
2016-02-21 23:17:47 +00:00
cdef unicode _split_affixes(self, Pool mem, unicode string, vector[LexemeC*] *prefixes,
* Replace UniStr, using unicode objects instead
2015-07-22 02:49:39 +00:00
vector[LexemeC*] *suffixes)
cdef int _attach_tokens(self, Doc tokens, unicode string,
* Tmp commit. Refactoring to create a Python Lexeme class.
2015-01-11 23:26:22 +00:00
vector[LexemeC*] *prefixes, vector[LexemeC*] *suffixes) except -1
* Replace UniStr, using unicode objects instead
2015-07-22 02:49:39 +00:00
* Move tokenizer to its own file, and refactor
2014-12-19 20:29:16 +00:00
cdef int _save_cached(self, const TokenC* tokens, hash_t key, int n) except -1