From 9692c98f5715f91e1e10fdae1f218035557c6c95 Mon Sep 17 00:00:00 2001 From: ines Date: Fri, 2 Jun 2017 10:56:09 +0200 Subject: [PATCH 1/5] Add test utils for temp file and temp dir --- spacy/tests/util.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/spacy/tests/util.py b/spacy/tests/util.py index 476ddb993..7f8884235 100644 --- a/spacy/tests/util.py +++ b/spacy/tests/util.py @@ -3,9 +3,14 @@ from __future__ import unicode_literals from ..tokens import Doc from ..attrs import ORTH, POS, HEAD, DEP +from ..compat import path2str import pytest import numpy +import tempfile +import shutil +import contextlib +from pathlib import Path MODELS = {} @@ -19,6 +24,20 @@ def load_test_model(model): return MODELS[model] +@contextlib.contextmanager +def make_tempfile(mode='r'): + f = tempfile.TemporaryFile(mode=mode) + yield f + f.close() + + +@contextlib.contextmanager +def make_tempdir(): + d = Path(tempfile.mkdtemp()) + yield d + shutil.rmtree(path2str(d)) + + def get_doc(vocab, words=[], pos=None, heads=None, deps=None, tags=None, ents=None): """Create Doc object from given vocab, words and annotations.""" pos = pos or [''] * len(words) From 023f38bdd4c25d55c4be1157e6ded75d86235ebb Mon Sep 17 00:00:00 2001 From: ines Date: Fri, 2 Jun 2017 10:56:40 +0200 Subject: [PATCH 2/5] Fix return value of Vocab.from_bytes --- spacy/vocab.pyx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/spacy/vocab.pyx b/spacy/vocab.pyx index d42e8951b..2f1822c26 100644 --- a/spacy/vocab.pyx +++ b/spacy/vocab.pyx @@ -329,7 +329,8 @@ cdef class Vocab: ('strings', lambda b: self.strings.from_bytes(b)), ('lexemes', lambda b: self.lexemes_from_bytes(b)), )) - return util.from_bytes(bytes_data, setters, exclude) + util.from_bytes(bytes_data, setters, exclude) + return self def lexemes_to_bytes(self): cdef hash_t key From 53b82f972a59710ed28b01508ab0b2c5ede1eaa1 Mon Sep 17 00:00:00 2001 From: ines Date: Fri, 2 Jun 2017 10:57:06 +0200 Subject: [PATCH 3/5] Add strings to Vocab in init, instead of StringStore --- spacy/vocab.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spacy/vocab.pyx b/spacy/vocab.pyx index 2f1822c26..57518f3aa 100644 --- a/spacy/vocab.pyx +++ b/spacy/vocab.pyx @@ -56,7 +56,7 @@ cdef class Vocab: self.strings = StringStore() if strings: for string in strings: - self.strings.add(string) + _ = self[string] for name in tag_map.keys(): if name: self.strings.add(name) From 41a6adf1f688a6eec523091002f5cfe4910339fc Mon Sep 17 00:00:00 2001 From: ines Date: Fri, 2 Jun 2017 10:57:25 +0200 Subject: [PATCH 4/5] Initialise Vocab length correctly --- spacy/vocab.pyx | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/spacy/vocab.pyx b/spacy/vocab.pyx index 57518f3aa..d3aa426cd 100644 --- a/spacy/vocab.pyx +++ b/spacy/vocab.pyx @@ -54,6 +54,7 @@ cdef class Vocab: self._by_hash = PreshMap() self._by_orth = PreshMap() self.strings = StringStore() + self.length = 0 if strings: for string in strings: _ = self[string] @@ -63,8 +64,6 @@ cdef class Vocab: self.lex_attr_getters = lex_attr_getters self.morphology = Morphology(self.strings, tag_map, lemmatizer) - self.length = 1 - property lang: def __get__(self): langfunc = None From acd65c00f62bd3e77a87efc199ec29255118dfc9 Mon Sep 17 00:00:00 2001 From: ines Date: Fri, 2 Jun 2017 10:57:42 +0200 Subject: [PATCH 5/5] Add serialization tests for StringStore and Vocab --- .../serialize/test_serialize_stringstore.py | 46 ++++++++++++ spacy/tests/serialize/test_serialize_vocab.py | 73 +++++++++++++++++++ 2 files changed, 119 insertions(+) create mode 100644 spacy/tests/serialize/test_serialize_stringstore.py create mode 100644 spacy/tests/serialize/test_serialize_vocab.py diff --git a/spacy/tests/serialize/test_serialize_stringstore.py b/spacy/tests/serialize/test_serialize_stringstore.py new file mode 100644 index 000000000..594413922 --- /dev/null +++ b/spacy/tests/serialize/test_serialize_stringstore.py @@ -0,0 +1,46 @@ +# coding: utf-8 +from __future__ import unicode_literals + +from ..util import make_tempdir +from ...strings import StringStore + +import pytest + + +test_strings = [([], []), (['rats', 'are', 'cute'], ['i', 'like', 'rats'])] + + +@pytest.mark.parametrize('strings1,strings2', test_strings) +def test_serialize_stringstore_roundtrip_bytes(strings1,strings2): + sstore1 = StringStore(strings=strings1) + sstore2 = StringStore(strings=strings2) + sstore1_b = sstore1.to_bytes() + sstore2_b = sstore2.to_bytes() + if strings1 == strings2: + assert sstore1_b == sstore2_b + else: + assert sstore1_b != sstore2_b + sstore1 = sstore1.from_bytes(sstore1_b) + assert sstore1.to_bytes() == sstore1_b + new_sstore1 = StringStore().from_bytes(sstore1_b) + assert new_sstore1.to_bytes() == sstore1_b + assert list(new_sstore1) == strings1 + + +@pytest.mark.parametrize('strings1,strings2', test_strings) +def test_serialize_stringstore_roundtrip_disk(strings1,strings2): + sstore1 = StringStore(strings=strings1) + sstore2 = StringStore(strings=strings2) + with make_tempdir() as d: + file_path1 = d / 'strings1' + file_path2 = d / 'strings2' + sstore1.to_disk(file_path1) + sstore2.to_disk(file_path2) + sstore1_d = StringStore().from_disk(file_path1) + sstore2_d = StringStore().from_disk(file_path2) + assert list(sstore1_d) == list(sstore1) + assert list(sstore2_d) == list(sstore2) + if strings1 == strings2: + assert list(sstore1_d) == list(sstore2_d) + else: + assert list(sstore1_d) != list(sstore2_d) diff --git a/spacy/tests/serialize/test_serialize_vocab.py b/spacy/tests/serialize/test_serialize_vocab.py new file mode 100644 index 000000000..47749e69f --- /dev/null +++ b/spacy/tests/serialize/test_serialize_vocab.py @@ -0,0 +1,73 @@ +# coding: utf-8 +from __future__ import unicode_literals + +from ..util import make_tempdir +from ...vocab import Vocab + +import pytest + + +test_strings = [([], []), (['rats', 'are', 'cute'], ['i', 'like', 'rats'])] +test_strings_attrs = [(['rats', 'are', 'cute'], 'Hello')] + + +@pytest.mark.parametrize('strings1,strings2', test_strings) +def test_serialize_vocab_roundtrip_bytes(strings1,strings2): + vocab1 = Vocab(strings=strings1) + vocab2 = Vocab(strings=strings2) + vocab1_b = vocab1.to_bytes() + vocab2_b = vocab2.to_bytes() + if strings1 == strings2: + assert vocab1_b == vocab2_b + else: + assert vocab1_b != vocab2_b + vocab1 = vocab1.from_bytes(vocab1_b) + assert vocab1.to_bytes() == vocab1_b + new_vocab1 = Vocab().from_bytes(vocab1_b) + assert new_vocab1.to_bytes() == vocab1_b + assert len(new_vocab1) == len(strings1) + assert sorted([lex.text for lex in new_vocab1]) == sorted(strings1) + + +@pytest.mark.parametrize('strings1,strings2', test_strings) +def test_serialize_vocab_roundtrip_disk(strings1,strings2): + vocab1 = Vocab(strings=strings1) + vocab2 = Vocab(strings=strings2) + with make_tempdir() as d: + file_path1 = d / 'vocab1' + file_path2 = d / 'vocab2' + vocab1.to_disk(file_path1) + vocab2.to_disk(file_path2) + vocab1_d = Vocab().from_disk(file_path1) + vocab2_d = Vocab().from_disk(file_path2) + assert list(vocab1_d) == list(vocab1) + assert list(vocab2_d) == list(vocab2) + if strings1 == strings2: + assert list(vocab1_d) == list(vocab2_d) + else: + assert list(vocab1_d) != list(vocab2_d) + + +@pytest.mark.parametrize('strings,lex_attr', test_strings_attrs) +def test_serialize_vocab_lex_attrs_bytes(strings, lex_attr): + vocab1 = Vocab(strings=strings) + vocab2 = Vocab() + vocab1[strings[0]].norm_ = lex_attr + assert vocab1[strings[0]].norm_ == lex_attr + assert vocab2[strings[0]].norm_ != lex_attr + vocab2 = vocab2.from_bytes(vocab1.to_bytes()) + assert vocab2[strings[0]].norm_ == lex_attr + + +@pytest.mark.parametrize('strings,lex_attr', test_strings_attrs) +def test_serialize_vocab_lex_attrs_disk(strings, lex_attr): + vocab1 = Vocab(strings=strings) + vocab2 = Vocab() + vocab1[strings[0]].norm_ = lex_attr + assert vocab1[strings[0]].norm_ == lex_attr + assert vocab2[strings[0]].norm_ != lex_attr + with make_tempdir() as d: + file_path = d / 'vocab' + vocab1.to_disk(file_path) + vocab2 = vocab2.from_disk(file_path) + assert vocab2[strings[0]].norm_ == lex_attr