diff --git a/spacy/tokens.pxd b/spacy/tokens.pxd index 09c11aa54..8f8564a83 100644 --- a/spacy/tokens.pxd +++ b/spacy/tokens.pxd @@ -53,7 +53,6 @@ cdef class Tokens: cdef class Token: cdef Vocab vocab - cdef Pool mem cdef unicode _string cdef const TokenC* c @@ -66,14 +65,14 @@ cdef class Token: cdef tuple _dep_strings @staticmethod - cdef inline Token cinit(Pool mem, Vocab vocab, unicode string, + cdef inline Token cinit(Vocab vocab, unicode string, const TokenC* token, int offset, int array_len, list py_tokens, tuple tag_strings, tuple dep_strings): assert offset >= 0 and offset < array_len if py_tokens[offset] is not None: return py_tokens[offset] - cdef Token self = Token.__new__(Token, mem, vocab, string) + cdef Token self = Token.__new__(Token, vocab, string) self.c = token self.i = offset diff --git a/spacy/tokens.pyx b/spacy/tokens.pyx index e594cf895..1463be17b 100644 --- a/spacy/tokens.pyx +++ b/spacy/tokens.pyx @@ -104,7 +104,7 @@ cdef class Tokens: if i < 0: i = self.length - i bounds_check(i, self.length, PADDING) - return Token.cinit(self.mem, self.vocab, self._string, + return Token.cinit(self.vocab, self._string, &self.data[i], i, self.length, self._py_tokens, self._tag_strings, self._dep_strings) @@ -115,7 +115,7 @@ cdef class Tokens: token (Token): """ for i in range(self.length): - yield Token.cinit(self.mem, self.vocab, self._string, + yield Token.cinit(self.vocab, self._string, &self.data[i], i, self.length, self._py_tokens, self._tag_strings, self._dep_strings) @@ -233,8 +233,7 @@ cdef class Tokens: cdef class Token: """An individual token.""" - def __cinit__(self, Pool mem, Vocab vocab, unicode string): - self.mem = mem + def __cinit__(self, Vocab vocab, unicode string): self.vocab = vocab self._string = string @@ -242,7 +241,7 @@ cdef class Token: return self.c.lex.length def nbor(self, int i=1): - return Token.cinit(self.mem, self.vocab, self._string, + return Token.cinit(self.vocab, self._string, self.c, self.i, self.array_len, self._py, self._tag_strings, self._dep_strings) @@ -343,7 +342,7 @@ cdef class Token: ptr += ptr.head elif ptr + ptr.head == self.c: - yield Token.cinit(self.mem, self.vocab, self._string, + yield Token.cinit(self.vocab, self._string, ptr, ptr - (self.c - self.i), self.array_len, self._py, self._tag_strings, self._dep_strings) ptr += 1 @@ -362,7 +361,7 @@ cdef class Token: if (ptr.head < 0) and ((ptr + ptr.head) > self.c): ptr += ptr.head elif ptr + ptr.head == self.c: - yield Token.cinit(self.mem, self.vocab, self._string, + yield Token.cinit(self.vocab, self._string, ptr, ptr - (self.c - self.i), self.array_len, self._py, self._tag_strings, self._dep_strings) ptr -= 1 @@ -372,7 +371,7 @@ cdef class Token: @property def head(self): """The token predicted by the parser to be the head of the current token.""" - return Token.cinit(self.mem, self.vocab, self._string, + return Token.cinit(self.vocab, self._string, self.c + self.c.head, self.i + self.c.head, self.array_len, self._py, self._tag_strings, self._dep_strings)