diff --git a/spacy/tokens.pxd b/spacy/tokens.pxd index 616353e8f..0f88008cb 100644 --- a/spacy/tokens.pxd +++ b/spacy/tokens.pxd @@ -31,10 +31,11 @@ cdef class Token: cdef public int idx cdef public int pos + cdef public atom_t id cdef public atom_t cluster cdef public atom_t length - cdef public atom_t lex_pos - cdef public atom_t lex_supersense + cdef public atom_t postype + cdef public atom_t sensetype cdef public atom_t sic cdef public atom_t norm diff --git a/spacy/tokens.pyx b/spacy/tokens.pyx index fe7badbeb..4b18bfc00 100644 --- a/spacy/tokens.pyx +++ b/spacy/tokens.pyx @@ -105,11 +105,12 @@ cdef class Token: self.idx = idx self.pos = pos self.i = i + self.id = lex['id'] self.cluster = lex['cluster'] self.length = lex['length'] - self.lex_pos = lex['pos'] - self.lex_supersense = lex['supersense'] + self.postype = lex['postype'] + self.sensetype = lex['supersense'] self.sic = lex['sic'] self.norm = lex['norm'] self.shape = lex['shape'] @@ -122,6 +123,8 @@ cdef class Token: property string: def __get__(self): + if self.sic == 0: + return '' cdef bytes utf8string = self._string_store[self.sic] return utf8string.decode('utf8')