Rooba
/
spaCy
mirror of https://github.com/explosion/spaCy.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

* Remove need for confusing _data pointer to be stored on Tokens

This commit is contained in:
Matthew Honnibal 2014-12-05 16:31:30 +11:00
parent 1c9253701d
commit e27b912ef9
2 changed files with 6 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
spacy/tokens.pxd
View File

@ -23,7 +23,6 @@ cdef class Tokens:
cdef Pool mem
cdef StringStore _string_store
cdef TokenC* _data
cdef TokenC* data
cdef int length

11
spacy/tokens.pyx
View File

@ -40,11 +40,11 @@ cdef class Tokens:
# Guarantee self.lex[i-x], for any i >= 0 and x < padding is in bounds
# However, we need to remember the true starting places, so that we can
# realloc.
self._data = <TokenC*>self.mem.alloc(size + (PADDING*2), sizeof(TokenC))
data_start = <TokenC*>self.mem.alloc(size + (PADDING*2), sizeof(TokenC))
cdef int i
for i in range(size + (PADDING*2)):
self._data[i] = EMPTY_TOKEN
self.data = self._data + PADDING
data_start[i] = EMPTY_TOKEN
self.data = data_start + PADDING
self.max_length = size
self.length = 0
@ -116,8 +116,9 @@ cdef class Tokens:
def _realloc(self, new_size):
self.max_length = new_size
n = new_size + (PADDING * 2)
self._data = <TokenC*>self.mem.realloc(self._data, n * sizeof(TokenC))
self.data = self._data + PADDING
cdef TokenC* data_start = self.data - PADDING
data_start = <TokenC*>self.mem.realloc(data_start, n * sizeof(TokenC))
self.data = data_start + PADDING
cdef int i
for i in range(self.length, self.max_length + PADDING):
self.data[i] = EMPTY_TOKEN