spaCy/spacy/tests/spans/test_merge.py

90 lines
3.1 KiB
Python
Raw Normal View History

2015-03-29 23:37:57 +00:00
from __future__ import unicode_literals
from spacy.attrs import HEAD
2015-07-22 23:19:11 +00:00
import pytest
import numpy
2015-03-29 23:37:57 +00:00
2015-06-07 16:44:18 +00:00
def test_merge_tokens(EN):
2015-06-07 16:45:19 +00:00
tokens = EN(u'Los Angeles start.')
tokens.from_array([HEAD], numpy.asarray([[1, 1, 0, -1]], dtype='int32').T)
2015-03-29 23:37:57 +00:00
assert len(tokens) == 4
2015-04-07 02:51:31 +00:00
assert tokens[0].head.orth_ == 'Angeles'
2015-03-29 23:37:57 +00:00
assert tokens[1].head.orth_ == 'start'
2015-04-07 02:51:31 +00:00
tokens.merge(0, len('Los Angeles'), 'NNP', 'Los Angeles', 'GPE')
2015-03-29 23:37:57 +00:00
assert len(tokens) == 3
2015-04-07 02:51:31 +00:00
assert tokens[0].orth_ == 'Los Angeles'
2015-03-29 23:37:57 +00:00
assert tokens[0].head.orth_ == 'start'
2015-11-07 08:10:56 +00:00
@pytest.mark.models
2015-06-07 16:44:18 +00:00
def test_merge_heads(EN):
2015-06-07 16:45:19 +00:00
tokens = EN(u'I found a pilates class near work.')
2015-03-29 23:37:57 +00:00
assert len(tokens) == 8
tokens.merge(tokens[3].idx, tokens[4].idx + len(tokens[4]), tokens[4].tag_,
'pilates class', 'O')
assert len(tokens) == 7
assert tokens[0].head.i == 1
assert tokens[1].head.i == 1
assert tokens[2].head.i == 3
assert tokens[3].head.i == 1
assert tokens[4].head.i in [1, 3]
assert tokens[5].head.i == 4
2015-11-07 08:27:38 +00:00
@pytest.mark.models
def test_np_merges(EN):
text = u'displaCy is a parse tool built with Javascript'
tokens = EN(text)
assert tokens[4].head.i == 1
tokens.merge(tokens[2].idx, tokens[4].idx + len(tokens[4]), u'NP', u'tool', u'O')
assert tokens[2].head.i == 1
tokens = EN('displaCy is a lightweight and modern dependency parse tree visualization tool built with CSS3 and JavaScript.')
ents = [(e[0].idx, e[-1].idx + len(e[-1]), e.label_, e.lemma_)
for e in tokens.ents]
for start, end, label, lemma in ents:
merged = tokens.merge(start, end, label, lemma, label)
assert merged != None, (start, end, label, lemma)
tokens = EN(u'One test with entities like New York City so the ents list is not void')
for span in tokens.ents:
merged = span.merge()
assert merged != None, (span.start, span.end, span.label_, span.lemma_)
2015-11-07 08:10:56 +00:00
@pytest.mark.models
2015-11-05 16:02:23 +00:00
def test_entity_merge(EN):
tokens = EN(u'Stewart Lee is a stand up comedian who lives in England and loves Joe Pasquale.\n')
assert(len(tokens) == 17)
for ent in tokens.ents:
label, lemma, type_ = (ent.root.tag_, ent.root.lemma_, max(w.ent_type_ for w in ent))
ent.merge(label, lemma, type_)
# check looping is ok
assert(len(tokens) == 15)
2015-11-07 08:27:38 +00:00
2015-11-07 08:10:56 +00:00
@pytest.mark.models
2015-11-05 16:02:23 +00:00
def test_sentence_update_after_merge(EN):
tokens = EN(u'Stewart Lee is a stand up comedian. He lives in England and loves Joe Pasquale.')
sent1, sent2 = list(tokens.sents)
init_len = len(sent1)
init_len2 = len(sent2)
merge_me = tokens[0:2]
merge_me.merge(u'none', u'none', u'none')
merge_me2 = tokens[-2:]
merge_me2.merge(u'none', u'none', u'none')
assert(len(sent1) == init_len - 1)
assert(len(sent2) == init_len2 - 1)
2015-11-07 08:27:38 +00:00
2015-11-07 08:10:56 +00:00
@pytest.mark.models
2015-11-05 16:02:23 +00:00
def test_subtree_size_check(EN):
tokens = EN(u'Stewart Lee is a stand up comedian who lives in England and loves Joe Pasquale')
sent1 = list(tokens.sents)[0]
init_len = len(list(sent1.root.subtree))
merge_me = tokens[0:2]
merge_me.merge(u'none', u'none', u'none')
assert(len(list(sent1.root.subtree)) == init_len - 1)