spaCy/spacy/tokens.pyx

157 lines
4.9 KiB
Cython
Raw Normal View History

2014-09-14 23:31:44 +00:00
# cython: profile=True
2014-12-04 09:46:55 +00:00
from preshed.maps cimport PreshMap
from preshed.counter cimport PreshCounter
from .lexeme cimport *
2014-10-22 01:55:42 +00:00
cimport cython
2014-12-04 09:46:55 +00:00
import numpy as np
cimport numpy as np
POS = 0
ENTITY = 0
2014-09-14 23:31:44 +00:00
DEF PADDING = 5
cdef int bounds_check(int i, int length, int padding) except -1:
if (i + padding) < 0:
raise IndexError
if (i - padding) >= length:
raise IndexError
2014-09-14 23:31:44 +00:00
2014-09-10 16:11:13 +00:00
cdef class Tokens:
"""A sequence of references to Lexeme objects.
The Tokens class provides fast and memory-efficient access to lexical features,
2014-12-04 09:46:55 +00:00
and can efficiently export the data to a numpy array.
2014-09-10 16:11:13 +00:00
>>> from spacy.en import EN
>>> tokens = EN.tokenize('An example sentence.')
"""
def __init__(self, Language lang, string_length=0):
self.lang = lang
if string_length >= 3:
size = int(string_length / 3.0)
else:
size = 5
self.mem = Pool()
# Guarantee self.lex[i-x], for any i >= 0 and x < padding is in bounds
# However, we need to remember the true starting places, so that we can
# realloc.
data_start = <TokenC*>self.mem.alloc(size + (PADDING*2), sizeof(TokenC))
2014-10-22 17:01:17 +00:00
cdef int i
for i in range(size + (PADDING*2)):
2014-12-07 11:07:41 +00:00
data_start[i].lex = &EMPTY_LEXEME
self.data = data_start + PADDING
self.max_length = size
self.length = 0
2014-10-14 07:22:41 +00:00
2014-09-11 19:37:32 +00:00
def __getitem__(self, i):
bounds_check(i, self.length, PADDING)
return Token(self.lang, i, self.data[i].idx, self.data[i].pos,
2014-12-08 10:12:15 +00:00
self.data[i].lemma, self.data[i].lex[0])
2014-09-11 19:37:32 +00:00
2014-11-03 14:07:08 +00:00
def __iter__(self):
for i in range(self.length):
yield self[i]
2014-09-11 19:37:32 +00:00
def __len__(self):
return self.length
2014-09-11 19:37:32 +00:00
cdef int push_back(self, int idx, LexemeOrToken lex_or_tok) except -1:
if self.length == self.max_length:
self._realloc(self.length * 2)
cdef TokenC* t = &self.data[self.length]
if LexemeOrToken is TokenC_ptr:
t[0] = lex_or_tok[0]
else:
t.lex = lex_or_tok
self.length += 1
return idx + t.lex.length
2014-12-04 09:46:55 +00:00
@cython.boundscheck(False)
cpdef np.ndarray[long, ndim=2] get_array(self, list attr_ids):
2014-12-02 12:48:05 +00:00
cdef int i, j
2014-12-04 09:46:55 +00:00
cdef attr_id_t feature
cdef np.ndarray[long, ndim=2] output
output = np.ndarray(shape=(self.length, len(attr_ids)), dtype=int)
2014-12-02 12:48:05 +00:00
for i in range(self.length):
2014-12-04 09:46:55 +00:00
for j, feature in enumerate(attr_ids):
output[i, j] = get_attr(self.data[i].lex, feature)
2014-12-02 12:48:05 +00:00
return output
2014-12-04 09:46:55 +00:00
def count_by(self, attr_id_t attr_id):
cdef int i
cdef attr_t attr
cdef size_t count
cdef PreshCounter counts = PreshCounter(2 ** 8)
for i in range(self.length):
if attr_id == LEMMA:
attr = self.data[i].lemma
else:
attr = get_attr(self.data[i].lex, attr_id)
2014-12-04 09:46:55 +00:00
counts.inc(attr, 1)
return dict(counts)
def _realloc(self, new_size):
self.max_length = new_size
n = new_size + (PADDING * 2)
2014-12-07 11:07:41 +00:00
# What we're storing is a "padded" array. We've jumped forward PADDING
# places, and are storing the pointer to that. This way, we can access
# words out-of-bounds, and get out-of-bounds markers.
# Now that we want to realloc, we need the address of the true start,
# so we jump the pointer back PADDING places.
cdef TokenC* data_start = self.data - PADDING
data_start = <TokenC*>self.mem.realloc(data_start, n * sizeof(TokenC))
self.data = data_start + PADDING
cdef int i
for i in range(self.length, self.max_length + PADDING):
2014-12-07 11:07:41 +00:00
self.data[i].lex = &EMPTY_LEXEME
2014-09-14 23:31:44 +00:00
@cython.freelist(64)
cdef class Token:
def __init__(self, Language lang, int i, int idx,
int pos, int lemma, dict lex):
self.lang = lang
self.idx = idx
self.pos = pos
2014-10-30 04:30:52 +00:00
self.i = i
self.id = lex['id']
2014-12-08 10:12:15 +00:00
self.lemma = lemma
self.cluster = lex['cluster']
self.length = lex['length']
self.postype = lex['pos_type']
self.sensetype = 0
2014-10-29 12:19:38 +00:00
self.sic = lex['sic']
self.norm = lex['dense']
self.shape = lex['shape']
self.suffix = lex['suffix']
self.prefix = lex['prefix']
self.prob = lex['prob']
self.flags = lex['flags']
property string:
def __get__(self):
if self.sic == 0:
return ''
cdef bytes utf8string = self.lang.lexicon.strings[self.sic]
return utf8string.decode('utf8')
2014-12-08 10:12:15 +00:00
property lemma:
def __get__(self):
if self.lemma == 0:
return self.string
cdef bytes utf8string = self.lang.lexicon.strings[self.lemma]
2014-12-08 10:12:15 +00:00
return utf8string.decode('utf8')
property pos:
def __get__(self):
return self.lang.pos_tagger.tag_names[self.pos]