2014-09-14 23:31:44 +00:00
|
|
|
# cython: profile=True
|
2014-12-04 09:46:55 +00:00
|
|
|
from preshed.maps cimport PreshMap
|
|
|
|
from preshed.counter cimport PreshCounter
|
|
|
|
|
2014-10-09 21:11:31 +00:00
|
|
|
from .lexeme cimport *
|
2014-10-22 01:55:42 +00:00
|
|
|
cimport cython
|
2014-12-03 00:05:15 +00:00
|
|
|
|
2014-12-04 09:46:55 +00:00
|
|
|
import numpy as np
|
|
|
|
cimport numpy as np
|
|
|
|
|
2014-12-03 00:05:15 +00:00
|
|
|
POS = 0
|
|
|
|
ENTITY = 0
|
2014-09-14 23:31:44 +00:00
|
|
|
|
2014-10-22 14:57:59 +00:00
|
|
|
DEF PADDING = 5
|
|
|
|
|
2014-11-04 16:42:14 +00:00
|
|
|
|
2014-10-22 14:57:59 +00:00
|
|
|
cdef int bounds_check(int i, int length, int padding) except -1:
|
|
|
|
if (i + padding) < 0:
|
|
|
|
raise IndexError
|
|
|
|
if (i - padding) >= length:
|
|
|
|
raise IndexError
|
|
|
|
|
2014-09-14 23:31:44 +00:00
|
|
|
|
2014-09-10 16:11:13 +00:00
|
|
|
cdef class Tokens:
|
|
|
|
"""A sequence of references to Lexeme objects.
|
|
|
|
|
|
|
|
The Tokens class provides fast and memory-efficient access to lexical features,
|
2014-12-04 09:46:55 +00:00
|
|
|
and can efficiently export the data to a numpy array.
|
2014-09-10 16:11:13 +00:00
|
|
|
|
|
|
|
>>> from spacy.en import EN
|
|
|
|
>>> tokens = EN.tokenize('An example sentence.')
|
|
|
|
"""
|
2014-10-23 13:59:17 +00:00
|
|
|
def __init__(self, StringStore string_store, string_length=0):
|
|
|
|
self._string_store = string_store
|
2014-10-22 14:57:59 +00:00
|
|
|
if string_length >= 3:
|
|
|
|
size = int(string_length / 3.0)
|
|
|
|
else:
|
|
|
|
size = 5
|
|
|
|
self.mem = Pool()
|
|
|
|
# Guarantee self.lex[i-x], for any i >= 0 and x < padding is in bounds
|
|
|
|
# However, we need to remember the true starting places, so that we can
|
|
|
|
# realloc.
|
2014-12-03 04:44:25 +00:00
|
|
|
self._lex_ptr = <const Lexeme**>self.mem.alloc(size + (PADDING*2), sizeof(Lexeme*))
|
2014-10-22 14:57:59 +00:00
|
|
|
self._idx_ptr = <int*>self.mem.alloc(size + (PADDING*2), sizeof(int))
|
|
|
|
self._pos_ptr = <int*>self.mem.alloc(size + (PADDING*2), sizeof(int))
|
2014-11-05 09:45:29 +00:00
|
|
|
self._ner_ptr = <int*>self.mem.alloc(size + (PADDING*2), sizeof(int))
|
2014-10-22 14:57:59 +00:00
|
|
|
self.lex = self._lex_ptr
|
|
|
|
self.idx = self._idx_ptr
|
|
|
|
self.pos = self._pos_ptr
|
2014-11-05 09:45:29 +00:00
|
|
|
self.ner = self._ner_ptr
|
2014-10-22 17:01:17 +00:00
|
|
|
cdef int i
|
|
|
|
for i in range(size + (PADDING*2)):
|
2014-10-22 14:57:59 +00:00
|
|
|
self.lex[i] = &EMPTY_LEXEME
|
|
|
|
self.lex += PADDING
|
|
|
|
self.idx += PADDING
|
|
|
|
self.pos += PADDING
|
2014-11-05 09:45:29 +00:00
|
|
|
self.ner += PADDING
|
2014-10-22 14:57:59 +00:00
|
|
|
self.max_length = size
|
|
|
|
self.length = 0
|
2014-10-14 07:22:41 +00:00
|
|
|
|
2014-09-11 19:37:32 +00:00
|
|
|
def __getitem__(self, i):
|
2014-10-22 14:57:59 +00:00
|
|
|
bounds_check(i, self.length, PADDING)
|
2014-11-05 09:45:29 +00:00
|
|
|
return Token(self._string_store, i, self.idx[i], self.pos[i], self.ner[i],
|
|
|
|
self.lex[i][0])
|
2014-09-11 19:37:32 +00:00
|
|
|
|
2014-11-03 14:07:08 +00:00
|
|
|
def __iter__(self):
|
|
|
|
for i in range(self.length):
|
|
|
|
yield self[i]
|
|
|
|
|
2014-09-11 19:37:32 +00:00
|
|
|
def __len__(self):
|
2014-10-22 14:57:59 +00:00
|
|
|
return self.length
|
2014-09-11 19:37:32 +00:00
|
|
|
|
2014-12-03 04:44:25 +00:00
|
|
|
cdef int push_back(self, int idx, const Lexeme* lexeme) except -1:
|
2014-10-22 14:57:59 +00:00
|
|
|
if self.length == self.max_length:
|
|
|
|
self._realloc(self.length * 2)
|
|
|
|
self.lex[self.length] = lexeme
|
|
|
|
self.idx[self.length] = idx
|
|
|
|
self.pos[self.length] = 0
|
2014-11-05 09:45:29 +00:00
|
|
|
self.ner[self.length] = 0
|
2014-10-22 14:57:59 +00:00
|
|
|
self.length += 1
|
2014-10-23 13:59:17 +00:00
|
|
|
return idx + lexeme.length
|
2014-10-14 04:21:03 +00:00
|
|
|
|
2014-12-03 04:44:25 +00:00
|
|
|
cdef int extend(self, int idx, const Lexeme* const* lexemes, int n) except -1:
|
2014-10-14 04:21:03 +00:00
|
|
|
cdef int i
|
|
|
|
if lexemes == NULL:
|
|
|
|
return idx
|
|
|
|
elif n == 0:
|
|
|
|
i = 0
|
|
|
|
while lexemes[i] != NULL:
|
2014-10-22 14:57:59 +00:00
|
|
|
idx = self.push_back(idx, lexemes[i])
|
2014-10-14 04:47:06 +00:00
|
|
|
i += 1
|
2014-10-14 04:21:03 +00:00
|
|
|
else:
|
|
|
|
for i in range(n):
|
2014-10-22 14:57:59 +00:00
|
|
|
idx = self.push_back(idx, lexemes[i])
|
2014-10-14 04:21:03 +00:00
|
|
|
return idx
|
2014-09-10 16:11:13 +00:00
|
|
|
|
2014-12-03 00:05:15 +00:00
|
|
|
cpdef int set_tag(self, int i, int tag_type, int tag) except -1:
|
2014-11-05 09:45:29 +00:00
|
|
|
if tag_type == POS:
|
|
|
|
self.pos[i] = tag
|
|
|
|
elif tag_type == ENTITY:
|
|
|
|
self.ner[i] = tag
|
2014-11-04 16:42:14 +00:00
|
|
|
|
2014-12-04 09:46:55 +00:00
|
|
|
@cython.boundscheck(False)
|
|
|
|
cpdef np.ndarray[long, ndim=2] get_array(self, list attr_ids):
|
2014-12-02 12:48:05 +00:00
|
|
|
cdef int i, j
|
2014-12-04 09:46:55 +00:00
|
|
|
cdef attr_id_t feature
|
|
|
|
cdef np.ndarray[long, ndim=2] output
|
|
|
|
output = np.ndarray(shape=(self.length, len(attr_ids)), dtype=int)
|
2014-12-02 12:48:05 +00:00
|
|
|
for i in range(self.length):
|
2014-12-04 09:46:55 +00:00
|
|
|
for j, feature in enumerate(attr_ids):
|
2014-12-03 00:05:15 +00:00
|
|
|
output[i, j] = get_attr(self.lex[i], feature)
|
2014-12-02 12:48:05 +00:00
|
|
|
return output
|
|
|
|
|
2014-12-04 09:46:55 +00:00
|
|
|
def count_by(self, attr_id_t attr_id):
|
|
|
|
cdef int i
|
|
|
|
cdef attr_t attr
|
|
|
|
cdef size_t count
|
|
|
|
|
|
|
|
cdef PreshCounter counts = PreshCounter(2 ** 8)
|
|
|
|
for i in range(self.length):
|
|
|
|
attr = get_attr(self.lex[i], attr_id)
|
|
|
|
counts.inc(attr, 1)
|
|
|
|
return dict(counts)
|
|
|
|
|
2014-10-23 13:59:17 +00:00
|
|
|
def _realloc(self, new_size):
|
|
|
|
self.max_length = new_size
|
|
|
|
n = new_size + (PADDING * 2)
|
2014-12-03 04:44:25 +00:00
|
|
|
self._lex_ptr = <const Lexeme**>self.mem.realloc(self._lex_ptr, n * sizeof(Lexeme*))
|
2014-10-23 13:59:17 +00:00
|
|
|
self._idx_ptr = <int*>self.mem.realloc(self._idx_ptr, n * sizeof(int))
|
|
|
|
self._pos_ptr = <int*>self.mem.realloc(self._pos_ptr, n * sizeof(int))
|
2014-11-05 09:45:29 +00:00
|
|
|
self._ner_ptr = <int*>self.mem.realloc(self._ner_ptr, n * sizeof(int))
|
2014-10-23 13:59:17 +00:00
|
|
|
self.lex = self._lex_ptr + PADDING
|
|
|
|
self.idx = self._idx_ptr + PADDING
|
|
|
|
self.pos = self._pos_ptr + PADDING
|
2014-11-05 09:45:29 +00:00
|
|
|
self.ner = self._ner_ptr + PADDING
|
2014-10-23 13:59:17 +00:00
|
|
|
for i in range(self.length, self.max_length + PADDING):
|
|
|
|
self.lex[i] = &EMPTY_LEXEME
|
2014-09-14 23:31:44 +00:00
|
|
|
|
|
|
|
|
2014-10-23 13:59:17 +00:00
|
|
|
@cython.freelist(64)
|
|
|
|
cdef class Token:
|
2014-11-05 09:45:29 +00:00
|
|
|
def __init__(self, StringStore string_store, int i, int idx, int pos, int ner,
|
|
|
|
dict lex):
|
2014-10-23 13:59:17 +00:00
|
|
|
self._string_store = string_store
|
|
|
|
self.idx = idx
|
|
|
|
self.pos = pos
|
2014-11-05 09:45:29 +00:00
|
|
|
self.ner = ner
|
2014-10-30 04:30:52 +00:00
|
|
|
self.i = i
|
2014-10-31 06:44:39 +00:00
|
|
|
self.id = lex['id']
|
2014-10-23 13:59:17 +00:00
|
|
|
|
|
|
|
self.cluster = lex['cluster']
|
|
|
|
self.length = lex['length']
|
2014-12-03 00:05:15 +00:00
|
|
|
self.postype = lex['pos_type']
|
|
|
|
self.sensetype = lex['sense_type']
|
2014-10-29 12:19:38 +00:00
|
|
|
self.sic = lex['sic']
|
2014-10-23 13:59:17 +00:00
|
|
|
self.norm = lex['norm']
|
|
|
|
self.shape = lex['shape']
|
|
|
|
self.suffix = lex['asciied']
|
|
|
|
self.prefix = lex['prefix']
|
|
|
|
|
|
|
|
self.prob = lex['prob']
|
|
|
|
self.flags = lex['flags']
|
|
|
|
|
|
|
|
property string:
|
|
|
|
def __get__(self):
|
2014-10-31 06:44:39 +00:00
|
|
|
if self.sic == 0:
|
|
|
|
return ''
|
2014-10-29 12:19:38 +00:00
|
|
|
cdef bytes utf8string = self._string_store[self.sic]
|
2014-10-23 13:59:17 +00:00
|
|
|
return utf8string.decode('utf8')
|