From 7ab03050d450b7ed029d816735c751507bd4c396 Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Fri, 21 Oct 2016 01:44:50 +0200 Subject: [PATCH] Add resize_vectors method to Vocab --- spacy/vocab.pyx | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/spacy/vocab.pyx b/spacy/vocab.pyx index 999f9608c..f852e67da 100644 --- a/spacy/vocab.pyx +++ b/spacy/vocab.pyx @@ -9,7 +9,7 @@ import bz2 from os import path import io import math -import json +import ujson as json import tempfile from .lexeme cimport EMPTY_LEXEME @@ -129,6 +129,20 @@ cdef class Vocab: """The current number of lexemes stored.""" return self.length + def resize_vectors(self, int new_size): + ''' + Set vectors_length to a new size, and allocate more memory for the Lexeme + vectors if necessary. The memory will be zeroed. + ''' + cdef hash_t key + cdef size_t addr + if new_size > self.vectors_length: + for key, addr in self._by_hash.items(): + lex = addr + lex.vector = self.mem.realloc(lex.vector, + new_size * sizeof(lex.vector[0])) + self.vectors_length = new_size + def add_flag(self, flag_getter, int flag_id=-1): '''Set a new boolean flag to words in the vocabulary. The flag_setter function will be called over the words currently in the vocab, and then