mirror of https://github.com/explosion/spaCy.git
* Remove unnecessary key and id properties from Utf8String.
This commit is contained in:
parent
5b0a7190c9
commit
d1cb30dbc4
|
@ -332,8 +332,7 @@ cdef class EnPosTagger:
|
||||||
cdef unicode lemma_string
|
cdef unicode lemma_string
|
||||||
lemma_strings = self.lemmatizer(py_string, pos)
|
lemma_strings = self.lemmatizer(py_string, pos)
|
||||||
lemma_string = sorted(lemma_strings)[0]
|
lemma_string = sorted(lemma_strings)[0]
|
||||||
bytes_string = lemma_string.encode('utf8')
|
lemma = self.strings[lemma_string]
|
||||||
lemma = self.strings.intern(bytes_string, len(bytes_string)).i
|
|
||||||
return lemma
|
return lemma
|
||||||
|
|
||||||
def load_morph_exceptions(self, dict exc):
|
def load_morph_exceptions(self, dict exc):
|
||||||
|
|
|
@ -22,4 +22,4 @@ cdef class StringStore:
|
||||||
cdef PreshMap _map
|
cdef PreshMap _map
|
||||||
cdef size_t _resize_at
|
cdef size_t _resize_at
|
||||||
|
|
||||||
cdef const Utf8Str* intern(self, char* chars, int length) except NULL
|
cdef const Utf8Str* intern(self, char* chars, int length, int* id_) except NULL
|
||||||
|
|
|
@ -67,6 +67,7 @@ cdef class StringStore:
|
||||||
def __getitem__(self, object string_or_id):
|
def __getitem__(self, object string_or_id):
|
||||||
cdef bytes byte_string
|
cdef bytes byte_string
|
||||||
cdef const Utf8Str* utf8str
|
cdef const Utf8Str* utf8str
|
||||||
|
cdef int id_
|
||||||
if isinstance(string_or_id, int) or isinstance(string_or_id, long):
|
if isinstance(string_or_id, int) or isinstance(string_or_id, long):
|
||||||
if string_or_id == 0:
|
if string_or_id == 0:
|
||||||
return u''
|
return u''
|
||||||
|
@ -75,16 +76,16 @@ cdef class StringStore:
|
||||||
utf8str = &self.strings[<int>string_or_id]
|
utf8str = &self.strings[<int>string_or_id]
|
||||||
return utf8str.chars[:utf8str.length].decode('utf8')
|
return utf8str.chars[:utf8str.length].decode('utf8')
|
||||||
elif isinstance(string_or_id, bytes):
|
elif isinstance(string_or_id, bytes):
|
||||||
utf8str = self.intern(<char*>string_or_id, len(string_or_id))
|
utf8str = self.intern(<char*>string_or_id, len(string_or_id), &id_)
|
||||||
return utf8str.i
|
return id_
|
||||||
elif isinstance(string_or_id, unicode):
|
elif isinstance(string_or_id, unicode):
|
||||||
byte_string = string_or_id.encode('utf8')
|
byte_string = string_or_id.encode('utf8')
|
||||||
utf8str = self.intern(<char*>byte_string, len(byte_string))
|
utf8str = self.intern(<char*>byte_string, len(byte_string), &id_)
|
||||||
return utf8str.i
|
return id_
|
||||||
else:
|
else:
|
||||||
raise TypeError(type(string_or_id))
|
raise TypeError(type(string_or_id))
|
||||||
|
|
||||||
cdef const Utf8Str* intern(self, char* chars, int length) except NULL:
|
cdef const Utf8Str* intern(self, char* chars, int length, int* id_) except NULL:
|
||||||
# 0 means missing, but we don't bother offsetting the index. We waste
|
# 0 means missing, but we don't bother offsetting the index. We waste
|
||||||
# slot 0 to simplify the code, because it doesn't matter.
|
# slot 0 to simplify the code, because it doesn't matter.
|
||||||
assert length != 0
|
assert length != 0
|
||||||
|
@ -97,7 +98,6 @@ cdef class StringStore:
|
||||||
self.strings = <Utf8Str*>self.mem.realloc(self.strings, self._resize_at * sizeof(Utf8Str))
|
self.strings = <Utf8Str*>self.mem.realloc(self.strings, self._resize_at * sizeof(Utf8Str))
|
||||||
i = self.size
|
i = self.size
|
||||||
self.strings[i].i = self.size
|
self.strings[i].i = self.size
|
||||||
self.strings[i].key = key
|
|
||||||
self.strings[i].chars = <unsigned char*>self.mem.alloc(length, sizeof(char))
|
self.strings[i].chars = <unsigned char*>self.mem.alloc(length, sizeof(char))
|
||||||
memcpy(self.strings[i].chars, chars, length)
|
memcpy(self.strings[i].chars, chars, length)
|
||||||
self.strings[i].length = length
|
self.strings[i].length = length
|
||||||
|
@ -123,6 +123,7 @@ cdef class StringStore:
|
||||||
strings = file_.read().split(SEPARATOR)
|
strings = file_.read().split(SEPARATOR)
|
||||||
cdef unicode string
|
cdef unicode string
|
||||||
cdef bytes byte_string
|
cdef bytes byte_string
|
||||||
|
cdef int id_
|
||||||
for string in strings[1:]:
|
for string in strings[1:]:
|
||||||
byte_string = string.encode('utf8')
|
byte_string = string.encode('utf8')
|
||||||
self.intern(byte_string, len(byte_string))
|
self.intern(byte_string, len(byte_string), &id_)
|
||||||
|
|
|
@ -80,8 +80,6 @@ cdef struct TokenC:
|
||||||
|
|
||||||
|
|
||||||
cdef struct Utf8Str:
|
cdef struct Utf8Str:
|
||||||
id_t i
|
|
||||||
hash_t key
|
|
||||||
unsigned char* chars
|
unsigned char* chars
|
||||||
int length
|
int length
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue