2018-07-24 21:38:44 +00:00
|
|
|
# coding: utf-8
|
|
|
|
from __future__ import unicode_literals
|
|
|
|
|
|
|
|
import pytest
|
|
|
|
import re
|
2019-10-16 11:34:58 +00:00
|
|
|
|
|
|
|
from spacy.lang.en import English
|
2018-07-24 21:38:44 +00:00
|
|
|
from spacy.matcher import Matcher
|
2019-02-27 09:25:56 +00:00
|
|
|
from spacy.tokens import Doc, Span
|
2018-07-24 21:38:44 +00:00
|
|
|
|
|
|
|
|
2018-11-27 00:09:36 +00:00
|
|
|
pattern1 = [{"ORTH": "A", "OP": "1"}, {"ORTH": "A", "OP": "*"}]
|
|
|
|
pattern2 = [{"ORTH": "A", "OP": "*"}, {"ORTH": "A", "OP": "1"}]
|
|
|
|
pattern3 = [{"ORTH": "A", "OP": "1"}, {"ORTH": "A", "OP": "1"}]
|
|
|
|
pattern4 = [
|
|
|
|
{"ORTH": "B", "OP": "1"},
|
|
|
|
{"ORTH": "A", "OP": "*"},
|
|
|
|
{"ORTH": "B", "OP": "1"},
|
|
|
|
]
|
|
|
|
pattern5 = [
|
|
|
|
{"ORTH": "B", "OP": "*"},
|
|
|
|
{"ORTH": "A", "OP": "*"},
|
|
|
|
{"ORTH": "B", "OP": "1"},
|
|
|
|
]
|
|
|
|
|
|
|
|
re_pattern1 = "AA*"
|
|
|
|
re_pattern2 = "A*A"
|
|
|
|
re_pattern3 = "AA"
|
|
|
|
re_pattern4 = "BA*B"
|
|
|
|
re_pattern5 = "B*A*B"
|
2018-07-24 21:38:44 +00:00
|
|
|
|
|
|
|
|
|
|
|
@pytest.fixture
|
|
|
|
def text():
|
|
|
|
return "(ABBAAAAAB)."
|
|
|
|
|
|
|
|
|
|
|
|
@pytest.fixture
|
|
|
|
def doc(en_tokenizer, text):
|
2018-11-27 00:09:36 +00:00
|
|
|
doc = en_tokenizer(" ".join(text))
|
2018-07-24 21:38:44 +00:00
|
|
|
return doc
|
|
|
|
|
|
|
|
|
2018-11-27 00:09:36 +00:00
|
|
|
@pytest.mark.parametrize(
|
|
|
|
"pattern,re_pattern",
|
|
|
|
[
|
|
|
|
pytest.param(pattern1, re_pattern1, marks=pytest.mark.xfail()),
|
|
|
|
pytest.param(pattern2, re_pattern2, marks=pytest.mark.xfail()),
|
|
|
|
pytest.param(pattern3, re_pattern3, marks=pytest.mark.xfail()),
|
|
|
|
(pattern4, re_pattern4),
|
|
|
|
pytest.param(pattern5, re_pattern5, marks=pytest.mark.xfail()),
|
|
|
|
],
|
|
|
|
)
|
2018-07-24 21:38:44 +00:00
|
|
|
def test_greedy_matching(doc, text, pattern, re_pattern):
|
|
|
|
"""Test that the greedy matching behavior of the * op is consistant with
|
|
|
|
other re implementations."""
|
|
|
|
matcher = Matcher(doc.vocab)
|
|
|
|
matcher.add(re_pattern, None, pattern)
|
|
|
|
matches = matcher(doc)
|
|
|
|
re_matches = [m.span() for m in re.finditer(re_pattern, text)]
|
|
|
|
for match, re_match in zip(matches, re_matches):
|
|
|
|
assert match[1:] == re_match
|
|
|
|
|
|
|
|
|
|
|
|
@pytest.mark.xfail
|
2018-11-27 00:09:36 +00:00
|
|
|
@pytest.mark.parametrize(
|
|
|
|
"pattern,re_pattern",
|
|
|
|
[
|
|
|
|
(pattern1, re_pattern1),
|
|
|
|
(pattern2, re_pattern2),
|
|
|
|
(pattern3, re_pattern3),
|
|
|
|
(pattern4, re_pattern4),
|
|
|
|
(pattern5, re_pattern5),
|
|
|
|
],
|
|
|
|
)
|
2018-07-24 21:38:44 +00:00
|
|
|
def test_match_consuming(doc, text, pattern, re_pattern):
|
|
|
|
"""Test that matcher.__call__ consumes tokens on a match similar to
|
|
|
|
re.findall."""
|
|
|
|
matcher = Matcher(doc.vocab)
|
|
|
|
matcher.add(re_pattern, None, pattern)
|
|
|
|
matches = matcher(doc)
|
|
|
|
re_matches = [m.span() for m in re.finditer(re_pattern, text)]
|
|
|
|
assert len(matches) == len(re_matches)
|
|
|
|
|
|
|
|
|
|
|
|
def test_operator_combos(en_vocab):
|
|
|
|
cases = [
|
2018-11-27 00:09:36 +00:00
|
|
|
("aaab", "a a a b", True),
|
|
|
|
("aaab", "a+ b", True),
|
|
|
|
("aaab", "a+ a+ b", True),
|
|
|
|
("aaab", "a+ a+ a b", True),
|
|
|
|
("aaab", "a+ a+ a+ b", True),
|
|
|
|
("aaab", "a+ a a b", True),
|
|
|
|
("aaab", "a+ a a", True),
|
|
|
|
("aaab", "a+", True),
|
|
|
|
("aaa", "a+ b", False),
|
|
|
|
("aaa", "a+ a+ b", False),
|
|
|
|
("aaa", "a+ a+ a+ b", False),
|
|
|
|
("aaa", "a+ a b", False),
|
|
|
|
("aaa", "a+ a a b", False),
|
|
|
|
("aaab", "a+ a a", True),
|
|
|
|
("aaab", "a+", True),
|
|
|
|
("aaab", "a+ a b", True),
|
2018-07-24 21:38:44 +00:00
|
|
|
]
|
|
|
|
for string, pattern_str, result in cases:
|
|
|
|
matcher = Matcher(en_vocab)
|
|
|
|
doc = Doc(matcher.vocab, words=list(string))
|
|
|
|
pattern = []
|
|
|
|
for part in pattern_str.split():
|
2018-11-27 00:09:36 +00:00
|
|
|
if part.endswith("+"):
|
|
|
|
pattern.append({"ORTH": part[0], "OP": "+"})
|
2018-07-24 21:38:44 +00:00
|
|
|
else:
|
2018-11-27 00:09:36 +00:00
|
|
|
pattern.append({"ORTH": part})
|
|
|
|
matcher.add("PATTERN", None, pattern)
|
2018-07-24 21:38:44 +00:00
|
|
|
matches = matcher(doc)
|
|
|
|
if result:
|
|
|
|
assert matches, (string, pattern_str)
|
|
|
|
else:
|
|
|
|
assert not matches, (string, pattern_str)
|
|
|
|
|
|
|
|
|
|
|
|
def test_matcher_end_zero_plus(en_vocab):
|
|
|
|
"""Test matcher works when patterns end with * operator. (issue 1450)"""
|
|
|
|
matcher = Matcher(en_vocab)
|
2018-11-27 00:09:36 +00:00
|
|
|
pattern = [{"ORTH": "a"}, {"ORTH": "b", "OP": "*"}]
|
|
|
|
matcher.add("TSTEND", None, pattern)
|
2018-07-24 21:38:44 +00:00
|
|
|
nlp = lambda string: Doc(matcher.vocab, words=string.split())
|
2018-11-27 00:09:36 +00:00
|
|
|
assert len(matcher(nlp("a"))) == 1
|
|
|
|
assert len(matcher(nlp("a b"))) == 2
|
|
|
|
assert len(matcher(nlp("a c"))) == 1
|
|
|
|
assert len(matcher(nlp("a b c"))) == 2
|
|
|
|
assert len(matcher(nlp("a b b c"))) == 3
|
|
|
|
assert len(matcher(nlp("a b b"))) == 3
|
2019-02-27 09:25:56 +00:00
|
|
|
|
|
|
|
|
|
|
|
def test_matcher_sets_return_correct_tokens(en_vocab):
|
|
|
|
matcher = Matcher(en_vocab)
|
|
|
|
patterns = [
|
2019-08-20 15:36:34 +00:00
|
|
|
[{"LOWER": {"IN": ["zero"]}}],
|
|
|
|
[{"LOWER": {"IN": ["one"]}}],
|
|
|
|
[{"LOWER": {"IN": ["two"]}}],
|
2019-02-27 09:25:56 +00:00
|
|
|
]
|
2019-08-20 15:36:34 +00:00
|
|
|
matcher.add("TEST", None, *patterns)
|
2019-02-27 09:25:56 +00:00
|
|
|
doc = Doc(en_vocab, words="zero one two three".split())
|
|
|
|
matches = matcher(doc)
|
|
|
|
texts = [Span(doc, s, e, label=L).text for L, s, e in matches]
|
2019-08-20 15:36:34 +00:00
|
|
|
assert texts == ["zero", "one", "two"]
|
2019-10-10 12:01:53 +00:00
|
|
|
|
|
|
|
|
2019-10-16 11:34:58 +00:00
|
|
|
def test_matcher_remove():
|
|
|
|
nlp = English()
|
|
|
|
matcher = Matcher(nlp.vocab)
|
|
|
|
text = "This is a test case."
|
|
|
|
|
2019-10-10 12:01:53 +00:00
|
|
|
pattern = [{"ORTH": "test"}, {"OP": "?"}]
|
|
|
|
assert len(matcher) == 0
|
|
|
|
matcher.add("Rule", None, pattern)
|
|
|
|
assert "Rule" in matcher
|
|
|
|
|
2019-10-16 11:34:58 +00:00
|
|
|
# should give two matches
|
|
|
|
results1 = matcher(nlp(text))
|
2019-10-18 09:27:38 +00:00
|
|
|
assert len(results1) == 2
|
2019-10-16 11:34:58 +00:00
|
|
|
|
2019-10-10 12:01:53 +00:00
|
|
|
# removing once should work
|
|
|
|
matcher.remove("Rule")
|
|
|
|
|
2019-10-16 11:34:58 +00:00
|
|
|
# should not return any maches anymore
|
|
|
|
results2 = matcher(nlp(text))
|
2019-10-18 09:27:38 +00:00
|
|
|
assert len(results2) == 0
|
2019-10-16 11:34:58 +00:00
|
|
|
|
2019-10-10 12:01:53 +00:00
|
|
|
# removing again should throw an error
|
|
|
|
with pytest.raises(ValueError):
|
|
|
|
matcher.remove("Rule")
|