mirror of https://github.com/explosion/spaCy.git
121 lines
3.5 KiB
Python
121 lines
3.5 KiB
Python
import pytest
|
|
import numpy
|
|
|
|
from spacy.matcher import Matcher
|
|
from spacy.attrs import ORTH, LOWER, ENT_IOB, ENT_TYPE
|
|
from spacy.symbols import DATE
|
|
|
|
|
|
def test_overlap_issue118(EN):
|
|
'''Test a bug that arose from having overlapping matches'''
|
|
doc = EN.tokenizer(u'how many points did lebron james score against the boston celtics last night')
|
|
ORG = doc.vocab.strings['ORG']
|
|
matcher = Matcher(EN.vocab,
|
|
{'BostonCeltics':
|
|
('ORG', {},
|
|
[
|
|
[{LOWER: 'celtics'}],
|
|
[{LOWER: 'boston'}, {LOWER: 'celtics'}],
|
|
]
|
|
)
|
|
}
|
|
)
|
|
|
|
assert len(list(doc.ents)) == 0
|
|
matches = matcher(doc)
|
|
assert matches == [(ORG, 9, 11), (ORG, 10, 11)]
|
|
ents = list(doc.ents)
|
|
assert len(ents) == 1
|
|
assert ents[0].label == ORG
|
|
assert ents[0].start == 9
|
|
assert ents[0].end == 11
|
|
|
|
|
|
def test_overlap_reorder(EN):
|
|
'''Test order dependence'''
|
|
doc = EN.tokenizer(u'how many points did lebron james score against the boston celtics last night')
|
|
ORG = doc.vocab.strings['ORG']
|
|
matcher = Matcher(EN.vocab,
|
|
{'BostonCeltics':
|
|
('ORG', {},
|
|
[
|
|
[{LOWER: 'boston'}, {LOWER: 'celtics'}],
|
|
[{LOWER: 'celtics'}],
|
|
]
|
|
)
|
|
}
|
|
)
|
|
|
|
assert len(list(doc.ents)) == 0
|
|
matches = matcher(doc)
|
|
assert matches == [(ORG, 9, 11), (ORG, 10, 11)]
|
|
ents = list(doc.ents)
|
|
assert len(ents) == 1
|
|
assert ents[0].label == ORG
|
|
assert ents[0].start == 9
|
|
assert ents[0].end == 11
|
|
|
|
|
|
def test_overlap_prefix(EN):
|
|
'''Test order dependence'''
|
|
doc = EN.tokenizer(u'how many points did lebron james score against the boston celtics last night')
|
|
ORG = doc.vocab.strings['ORG']
|
|
matcher = Matcher(EN.vocab,
|
|
{'BostonCeltics':
|
|
('ORG', {},
|
|
[
|
|
[{LOWER: 'boston'}],
|
|
[{LOWER: 'boston'}, {LOWER: 'celtics'}],
|
|
]
|
|
)
|
|
}
|
|
)
|
|
|
|
assert len(list(doc.ents)) == 0
|
|
matches = matcher(doc)
|
|
assert matches == [(ORG, 9, 10), (ORG, 9, 11)]
|
|
ents = list(doc.ents)
|
|
assert len(ents) == 1
|
|
assert ents[0].label == ORG
|
|
assert ents[0].start == 9
|
|
assert ents[0].end == 11
|
|
|
|
|
|
def test_overlap_prefix_reorder(EN):
|
|
'''Test order dependence'''
|
|
doc = EN.tokenizer(u'how many points did lebron james score against the boston celtics last night')
|
|
ORG = doc.vocab.strings['ORG']
|
|
matcher = Matcher(EN.vocab,
|
|
{'BostonCeltics':
|
|
('ORG', {},
|
|
[
|
|
[{LOWER: 'boston'}, {LOWER: 'celtics'}],
|
|
[{LOWER: 'boston'}],
|
|
]
|
|
)
|
|
}
|
|
)
|
|
|
|
assert len(list(doc.ents)) == 0
|
|
matches = matcher(doc)
|
|
assert matches == [(ORG, 9, 10), (ORG, 9, 11)]
|
|
ents = list(doc.ents)
|
|
assert len(ents) == 1
|
|
assert ents[0].label == ORG
|
|
assert ents[0].start == 9
|
|
assert ents[0].end == 11
|
|
|
|
|
|
@pytest.mark.models
|
|
def test_ner_interaction(EN):
|
|
EN.matcher.add('LAX_Airport', 'AIRPORT', {}, [[{ORTH: 'LAX'}]])
|
|
EN.matcher.add('SFO_Airport', 'AIRPORT', {}, [[{ORTH: 'SFO'}]])
|
|
doc = EN(u'get me a flight from SFO to LAX leaving 20 December and arriving on January 5th')
|
|
|
|
ents = [(ent.label_, ent.text) for ent in doc.ents]
|
|
assert ents[0] == ('AIRPORT', 'SFO')
|
|
assert ents[1] == ('AIRPORT', 'LAX')
|
|
assert ents[2] == ('DATE', '20 December')
|
|
assert ents[3] == ('DATE', 'January 5th')
|
|
|