mirror of https://github.com/explosion/spaCy.git
120 lines
3.8 KiB
Python
120 lines
3.8 KiB
Python
# coding: utf-8
|
|
from __future__ import unicode_literals
|
|
|
|
import pytest
|
|
from spacy.strings import StringStore
|
|
|
|
|
|
@pytest.fixture
|
|
def stringstore():
|
|
return StringStore()
|
|
|
|
|
|
def test_string_hash(stringstore):
|
|
"""Test that string hashing is stable across platforms"""
|
|
assert stringstore.add('apple') == 8566208034543834098
|
|
heart = '\U0001f499'
|
|
h = stringstore.add(heart)
|
|
assert h == 11841826740069053588
|
|
|
|
|
|
def test_stringstore_from_api_docs(stringstore):
|
|
apple_hash = stringstore.add('apple')
|
|
assert apple_hash == 8566208034543834098
|
|
assert stringstore[apple_hash] == 'apple'
|
|
assert 'apple' in stringstore
|
|
assert 'cherry' not in stringstore
|
|
orange_hash = stringstore.add('orange')
|
|
all_strings = [s for s in stringstore]
|
|
assert all_strings == ['apple', 'orange']
|
|
banana_hash = stringstore.add('banana')
|
|
assert len(stringstore) == 3
|
|
assert banana_hash == 2525716904149915114
|
|
assert stringstore[banana_hash] == 'banana'
|
|
assert stringstore['banana'] == banana_hash
|
|
|
|
|
|
@pytest.mark.parametrize('text1,text2,text3', [(b'Hello', b'goodbye', b'hello')])
|
|
def test_stringstore_save_bytes(stringstore, text1, text2, text3):
|
|
key = stringstore.add(text1)
|
|
assert stringstore[text1] == key
|
|
assert stringstore[text2] != key
|
|
assert stringstore[text3] != key
|
|
|
|
|
|
@pytest.mark.parametrize('text1,text2,text3', [('Hello', 'goodbye', 'hello')])
|
|
def test_stringstore_save_unicode(stringstore, text1, text2, text3):
|
|
key = stringstore.add(text1)
|
|
assert stringstore[text1] == key
|
|
assert stringstore[text2] != key
|
|
assert stringstore[text3] != key
|
|
|
|
|
|
@pytest.mark.parametrize('text', [b'A'])
|
|
def test_stringstore_retrieve_id(stringstore, text):
|
|
key = stringstore.add(text)
|
|
assert len(stringstore) == 1
|
|
assert stringstore[key] == text.decode('utf8')
|
|
with pytest.raises(KeyError):
|
|
stringstore[20000]
|
|
|
|
|
|
@pytest.mark.parametrize('text1,text2', [(b'0123456789', b'A')])
|
|
def test_stringstore_med_string(stringstore, text1, text2):
|
|
store = stringstore.add(text1)
|
|
assert stringstore[store] == text1.decode('utf8')
|
|
dummy = stringstore.add(text2)
|
|
assert stringstore[text1] == store
|
|
|
|
|
|
def test_stringstore_long_string(stringstore):
|
|
text = "INFORMATIVE](http://www.google.com/search?as_q=RedditMonkey&hl=en&num=50&btnG=Google+Search&as_epq=&as_oq=&as_eq=&lr=&as_ft=i&as_filetype=&as_qdr=all&as_nlo=&as_nhi=&as_occt=any&as_dt=i&as_sitesearch=&as_rights=&safe=off"
|
|
store = stringstore.add(text)
|
|
assert stringstore[store] == text
|
|
|
|
|
|
@pytest.mark.parametrize('factor', [254, 255, 256])
|
|
def test_stringstore_multiply(stringstore, factor):
|
|
text = 'a' * factor
|
|
store = stringstore.add(text)
|
|
assert stringstore[store] == text
|
|
|
|
|
|
def test_stringstore_massive_strings(stringstore):
|
|
text = 'a' * 511
|
|
store = stringstore.add(text)
|
|
assert stringstore[store] == text
|
|
text2 = 'z' * 512
|
|
store = stringstore.add(text2)
|
|
assert stringstore[store] == text2
|
|
text3 = '1' * 513
|
|
store = stringstore.add(text3)
|
|
assert stringstore[store] == text3
|
|
|
|
|
|
@pytest.mark.parametrize('text', ["qqqqq"])
|
|
def test_stringstore_to_bytes(stringstore, text):
|
|
store = stringstore.add(text)
|
|
serialized = stringstore.to_bytes()
|
|
new_stringstore = StringStore().from_bytes(serialized)
|
|
assert new_stringstore[store] == text
|
|
|
|
|
|
@pytest.mark.xfail
|
|
@pytest.mark.parametrize('text', [["a", "b", "c"]])
|
|
def test_stringstore_freeze_oov(stringstore, text):
|
|
"""Test the possibly temporary workaround of flushing the stringstore of
|
|
OOV words."""
|
|
assert stringstore[text[0]] == 1
|
|
assert stringstore[text[1]] == 2
|
|
|
|
stringstore.set_frozen(True)
|
|
s = stringstore[text[2]]
|
|
assert s >= 4
|
|
s_ = stringstore[s]
|
|
assert s_ == text[2]
|
|
|
|
stringstore.flush_oov()
|
|
with pytest.raises(IndexError):
|
|
s_ = stringstore[s]
|