spaCy/tests/tokens/test_tokens_api.py

55 lines
1.7 KiB
Python
Raw Normal View History

2015-02-07 18:14:07 +00:00
from __future__ import unicode_literals
from spacy.tokens import Doc
2015-02-07 18:14:07 +00:00
import pytest
2015-07-26 14:23:41 +00:00
@pytest.mark.models
def test_getitem(EN):
tokens = EN(u'Give it back! He pleaded.')
2015-02-07 18:14:07 +00:00
assert tokens[0].orth_ == 'Give'
assert tokens[-1].orth_ == '.'
with pytest.raises(IndexError):
tokens[len(tokens)]
2015-10-06 07:51:25 +00:00
span = tokens[1:1]
assert not '/'.join(token.orth_ for token in span)
span = tokens[1:4]
assert '/'.join(token.orth_ for token in span) == 'it/back/!'
with pytest.raises(ValueError):
tokens[1:4:2]
with pytest.raises(ValueError):
tokens[1:4:-1]
2015-07-26 14:23:41 +00:00
@pytest.mark.models
def test_serialize(EN):
2015-07-22 23:19:11 +00:00
tokens = EN(u'Give it back! He pleaded.')
packed = tokens.to_bytes()
new_tokens = Doc(EN.vocab).from_bytes(packed)
assert tokens.string == new_tokens.string
assert [t.orth_ for t in tokens] == [t.orth_ for t in new_tokens]
assert [t.orth for t in tokens] == [t.orth for t in new_tokens]
2015-07-26 14:23:41 +00:00
@pytest.mark.models
2015-07-22 23:19:11 +00:00
def test_serialize_whitespace(EN):
tokens = EN(u' Give it back! He pleaded. ')
2015-07-22 23:19:11 +00:00
packed = tokens.to_bytes()
new_tokens = Doc(EN.vocab).from_bytes(packed)
assert tokens.string == new_tokens.string
assert [t.orth_ for t in tokens] == [t.orth_ for t in new_tokens]
assert [t.orth for t in tokens] == [t.orth for t in new_tokens]
def test_set_ents(EN):
tokens = EN.tokenizer(u'I use goggle chrone to surf the web')
assert len(tokens.ents) == 0
tokens.ents = [(EN.vocab.strings['PRODUCT'], 2, 4)]
assert len(list(tokens.ents)) == 1
assert [t.ent_iob for t in tokens] == [0, 0, 3, 1, 0, 0, 0, 0]
ent = tokens.ents[0]
assert ent.label_ == 'PRODUCT'
assert ent.start == 2
assert ent.end == 4