mirror of https://github.com/explosion/spaCy.git
29 lines
720 B
Cython
29 lines
720 B
Cython
from cymem.cymem cimport Pool
|
|
from preshed.maps cimport PreshMap
|
|
from murmurhash.mrmr cimport hash64
|
|
from .typedefs cimport attr_t
|
|
|
|
from libc.stdint cimport int64_t
|
|
|
|
from .structs cimport Utf8Str, UniStr
|
|
from .typedefs cimport hash_t
|
|
|
|
cpdef hash_t hash_string(unicode string) except 0
|
|
|
|
|
|
cdef inline void slice_unicode(UniStr* s, Py_UNICODE* chars, int start, int end) nogil:
|
|
s.chars = &chars[start]
|
|
s.n = end - start
|
|
s.key = hash64(s.chars, <int>(s.n * sizeof(Py_UNICODE)), 0)
|
|
|
|
|
|
cdef class StringStore:
|
|
cdef Pool mem
|
|
cdef Utf8Str* strings
|
|
cdef int64_t size
|
|
|
|
cdef PreshMap _map
|
|
cdef size_t _resize_at
|
|
|
|
cdef const Utf8Str* intern(self, char* chars, int length, attr_t* id_) except NULL
|