Rooba
/
spaCy
mirror of https://github.com/explosion/spaCy.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Check for out-of-memory when calling calloc. Closes #1446

This commit is contained in:
Matthew Honnibal 2017-10-24 12:40:47 +02:00
parent 0f9d966317
commit dd5b2d8fa3
2 changed files with 13 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
spacy/syntax/_state.pxd
View File

@ -2,6 +2,8 @@ from libc.string cimport memcpy, memset, memmove
from libc.stdlib cimport malloc, calloc, free from libc.stdlib cimport malloc, calloc, free
from libc.stdint cimport uint32_t, uint64_t from libc.stdint cimport uint32_t, uint64_t
from cpython.exc cimport PyErr_CheckSignals, PyErr_SetFromErrno
from murmurhash.mrmr cimport hash64 from murmurhash.mrmr cimport hash64
from ..vocab cimport EMPTY_LEXEME from ..vocab cimport EMPTY_LEXEME
@ -55,6 +57,11 @@ cdef cppclass StateC:
this.shifted = <bint*>calloc(length + (PADDING * 2), sizeof(bint)) this.shifted = <bint*>calloc(length + (PADDING * 2), sizeof(bint))
this._sent = <TokenC*>calloc(length + (PADDING * 2), sizeof(TokenC)) this._sent = <TokenC*>calloc(length + (PADDING * 2), sizeof(TokenC))
this._ents = <Entity*>calloc(length + (PADDING * 2), sizeof(Entity)) this._ents = <Entity*>calloc(length + (PADDING * 2), sizeof(Entity))
if not (this._buffer and this._stack and this.shifted
and this._sent and this._ents):
with gil:
PyErr_SetFromErrno(MemoryError)
PyErr_CheckSignals()
memset(&this._hist, 0, sizeof(this._hist)) memset(&this._hist, 0, sizeof(this._hist))
this.offset = 0 this.offset = 0
cdef int i cdef int i

7
spacy/syntax/nn_parser.pyx
View File

@ -22,7 +22,7 @@ cimport numpy as np
from libcpp.vector cimport vector from libcpp.vector cimport vector
from cpython.ref cimport PyObject, Py_INCREF, Py_XDECREF from cpython.ref cimport PyObject, Py_INCREF, Py_XDECREF
from cpython.exc cimport PyErr_CheckSignals from cpython.exc cimport PyErr_CheckSignals, PyErr_SetFromErrno
from libc.stdint cimport uint32_t, uint64_t from libc.stdint cimport uint32_t, uint64_t
from libc.string cimport memset, memcpy from libc.string cimport memset, memcpy
from libc.stdlib cimport malloc, calloc, free from libc.stdlib cimport malloc, calloc, free
@ -429,6 +429,7 @@ cdef class Parser:
self._parseC(states[i], self._parseC(states[i],
feat_weights, hW, hb, feat_weights, hW, hb,
nr_class, nr_hidden, nr_feat, nr_piece) nr_class, nr_hidden, nr_feat, nr_piece)
PyErr_CheckSignals()
return state_objs return state_objs
cdef void _parseC(self, StateC* state, cdef void _parseC(self, StateC* state,
@ -438,6 +439,10 @@ cdef class Parser:
is_valid = <int*>calloc(nr_class, sizeof(int)) is_valid = <int*>calloc(nr_class, sizeof(int))
vectors = <float*>calloc(nr_hidden * nr_piece, sizeof(float)) vectors = <float*>calloc(nr_hidden * nr_piece, sizeof(float))
scores = <float*>calloc(nr_class, sizeof(float)) scores = <float*>calloc(nr_class, sizeof(float))
if not (token_ids and is_valid and vectors and scores):
with gil:
PyErr_SetFromErrno(MemoryError)
PyErr_CheckSignals()
while not state.is_final(): while not state.is_final():
state.set_context_tokens(token_ids, nr_feat) state.set_context_tokens(token_ids, nr_feat)