Rooba
/
spaCy
mirror of https://github.com/explosion/spaCy.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Try to find test failing on appveyor

This commit is contained in:
Matthew Honnibal 2018-02-23 20:59:21 +01:00
parent 4492a33a9d
commit 39de8cd4d3
1 changed files with 245 additions and 245 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

490
spacy/tests/test_matcher.py
View File

@ -26,248 +26,248 @@ def matcher(en_vocab):
return matcher return matcher
def test_matcher_from_api_docs(en_vocab): #def test_matcher_from_api_docs(en_vocab):
matcher = Matcher(en_vocab) # matcher = Matcher(en_vocab)
pattern = [{'ORTH': 'test'}] # pattern = [{'ORTH': 'test'}]
assert len(matcher) == 0 # assert len(matcher) == 0
matcher.add('Rule', None, pattern) # matcher.add('Rule', None, pattern)
assert len(matcher) == 1 # assert len(matcher) == 1
matcher.remove('Rule') # matcher.remove('Rule')
assert 'Rule' not in matcher # assert 'Rule' not in matcher
matcher.add('Rule', None, pattern) # matcher.add('Rule', None, pattern)
assert 'Rule' in matcher # assert 'Rule' in matcher
on_match, patterns = matcher.get('Rule') # on_match, patterns = matcher.get('Rule')
assert len(patterns[0]) # assert len(patterns[0])
#
#
def test_matcher_from_usage_docs(en_vocab): #def test_matcher_from_usage_docs(en_vocab):
text = "Wow 😀 This is really cool! 😂 😂" # text = "Wow 😀 This is really cool! 😂 😂"
doc = get_doc(en_vocab, words=text.split(' ')) # doc = get_doc(en_vocab, words=text.split(' '))
pos_emoji = [u'😀', u'😃', u'😂', u'🤣', u'😊', u'😍'] # pos_emoji = [u'😀', u'😃', u'😂', u'🤣', u'😊', u'😍']
pos_patterns = [[{'ORTH': emoji}] for emoji in pos_emoji] # pos_patterns = [[{'ORTH': emoji}] for emoji in pos_emoji]
#
def label_sentiment(matcher, doc, i, matches): # def label_sentiment(matcher, doc, i, matches):
match_id, start, end = matches[i] # match_id, start, end = matches[i]
if doc.vocab.strings[match_id] == 'HAPPY': # if doc.vocab.strings[match_id] == 'HAPPY':
doc.sentiment += 0.1 # doc.sentiment += 0.1
span = doc[start : end] # span = doc[start : end]
token = span.merge() # token = span.merge()
token.vocab[token.text].norm_ = 'happy emoji' # token.vocab[token.text].norm_ = 'happy emoji'
#
matcher = Matcher(en_vocab) # matcher = Matcher(en_vocab)
matcher.add('HAPPY', label_sentiment, *pos_patterns) # matcher.add('HAPPY', label_sentiment, *pos_patterns)
matches = matcher(doc) # matches = matcher(doc)
assert doc.sentiment != 0 # assert doc.sentiment != 0
assert doc[1].norm_ == 'happy emoji' # assert doc[1].norm_ == 'happy emoji'
#
#
@pytest.mark.parametrize('words', [["Some", "words"]]) #@pytest.mark.parametrize('words', [["Some", "words"]])
def test_matcher_init(en_vocab, words): #def test_matcher_init(en_vocab, words):
matcher = Matcher(en_vocab) # matcher = Matcher(en_vocab)
doc = get_doc(en_vocab, words) # doc = get_doc(en_vocab, words)
assert len(matcher) == 0 # assert len(matcher) == 0
assert matcher(doc) == [] # assert matcher(doc) == []
#
#
def test_matcher_contains(matcher): #def test_matcher_contains(matcher):
matcher.add('TEST', None, [{'ORTH': 'test'}]) # matcher.add('TEST', None, [{'ORTH': 'test'}])
assert 'TEST' in matcher # assert 'TEST' in matcher
assert 'TEST2' not in matcher # assert 'TEST2' not in matcher
#
#
def test_matcher_no_match(matcher): #def test_matcher_no_match(matcher):
words = ["I", "like", "cheese", "."] # words = ["I", "like", "cheese", "."]
doc = get_doc(matcher.vocab, words) # doc = get_doc(matcher.vocab, words)
assert matcher(doc) == [] # assert matcher(doc) == []
#
#
def test_matcher_compile(en_vocab): #def test_matcher_compile(en_vocab):
rules = { # rules = {
'JS': [[{'ORTH': 'JavaScript'}]], # 'JS': [[{'ORTH': 'JavaScript'}]],
'GoogleNow': [[{'ORTH': 'Google'}, {'ORTH': 'Now'}]], # 'GoogleNow': [[{'ORTH': 'Google'}, {'ORTH': 'Now'}]],
'Java': [[{'LOWER': 'java'}]] # 'Java': [[{'LOWER': 'java'}]]
} # }
matcher = Matcher(en_vocab) # matcher = Matcher(en_vocab)
for key, patterns in rules.items(): # for key, patterns in rules.items():
matcher.add(key, None, *patterns) # matcher.add(key, None, *patterns)
assert len(matcher) == 3 # assert len(matcher) == 3
#
#
def test_matcher_match_start(matcher): #def test_matcher_match_start(matcher):
words = ["JavaScript", "is", "good"] # words = ["JavaScript", "is", "good"]
doc = get_doc(matcher.vocab, words) # doc = get_doc(matcher.vocab, words)
assert matcher(doc) == [(matcher.vocab.strings['JS'], 0, 1)] # assert matcher(doc) == [(matcher.vocab.strings['JS'], 0, 1)]
#
#
def test_matcher_match_end(matcher): #def test_matcher_match_end(matcher):
words = ["I", "like", "java"] # words = ["I", "like", "java"]
doc = get_doc(matcher.vocab, words) # doc = get_doc(matcher.vocab, words)
assert matcher(doc) == [(doc.vocab.strings['Java'], 2, 3)] # assert matcher(doc) == [(doc.vocab.strings['Java'], 2, 3)]
#
#
def test_matcher_match_middle(matcher): #def test_matcher_match_middle(matcher):
words = ["I", "like", "Google", "Now", "best"] # words = ["I", "like", "Google", "Now", "best"]
doc = get_doc(matcher.vocab, words) # doc = get_doc(matcher.vocab, words)
assert matcher(doc) == [(doc.vocab.strings['GoogleNow'], 2, 4)] # assert matcher(doc) == [(doc.vocab.strings['GoogleNow'], 2, 4)]
#
#
def test_matcher_match_multi(matcher): #def test_matcher_match_multi(matcher):
words = ["I", "like", "Google", "Now", "and", "java", "best"] # words = ["I", "like", "Google", "Now", "and", "java", "best"]
doc = get_doc(matcher.vocab, words) # doc = get_doc(matcher.vocab, words)
assert matcher(doc) == [(doc.vocab.strings['GoogleNow'], 2, 4), # assert matcher(doc) == [(doc.vocab.strings['GoogleNow'], 2, 4),
(doc.vocab.strings['Java'], 5, 6)] # (doc.vocab.strings['Java'], 5, 6)]
#
#
def test_matcher_empty_dict(en_vocab): #def test_matcher_empty_dict(en_vocab):
'''Test matcher allows empty token specs, meaning match on any token.''' # '''Test matcher allows empty token specs, meaning match on any token.'''
matcher = Matcher(en_vocab) # matcher = Matcher(en_vocab)
abc = ["a", "b", "c"] # abc = ["a", "b", "c"]
doc = get_doc(matcher.vocab, abc) # doc = get_doc(matcher.vocab, abc)
matcher.add('A.C', None, [{'ORTH': 'a'}, {}, {'ORTH': 'c'}]) # matcher.add('A.C', None, [{'ORTH': 'a'}, {}, {'ORTH': 'c'}])
matches = matcher(doc) # matches = matcher(doc)
assert len(matches) == 1 # assert len(matches) == 1
assert matches[0][1:] == (0, 3) # assert matches[0][1:] == (0, 3)
matcher = Matcher(en_vocab) # matcher = Matcher(en_vocab)
matcher.add('A.', None, [{'ORTH': 'a'}, {}]) # matcher.add('A.', None, [{'ORTH': 'a'}, {}])
matches = matcher(doc) # matches = matcher(doc)
assert matches[0][1:] == (0, 2) # assert matches[0][1:] == (0, 2)
#
#
def test_matcher_operator_shadow(en_vocab): #def test_matcher_operator_shadow(en_vocab):
matcher = Matcher(en_vocab) # matcher = Matcher(en_vocab)
abc = ["a", "b", "c"] # abc = ["a", "b", "c"]
doc = get_doc(matcher.vocab, abc) # doc = get_doc(matcher.vocab, abc)
matcher.add('A.C', None, [{'ORTH': 'a'}, # matcher.add('A.C', None, [{'ORTH': 'a'},
{"IS_ALPHA": True, "OP": "+"}, # {"IS_ALPHA": True, "OP": "+"},
{'ORTH': 'c'}]) # {'ORTH': 'c'}])
matches = matcher(doc) # matches = matcher(doc)
assert len(matches) == 1 # assert len(matches) == 1
assert matches[0][1:] == (0, 3) # assert matches[0][1:] == (0, 3)
#
#
def test_matcher_phrase_matcher(en_vocab): #def test_matcher_phrase_matcher(en_vocab):
words = ["Google", "Now"] # words = ["Google", "Now"]
doc = get_doc(en_vocab, words) # doc = get_doc(en_vocab, words)
matcher = PhraseMatcher(en_vocab) # matcher = PhraseMatcher(en_vocab)
matcher.add('COMPANY', None, doc) # matcher.add('COMPANY', None, doc)
words = ["I", "like", "Google", "Now", "best"] # words = ["I", "like", "Google", "Now", "best"]
doc = get_doc(en_vocab, words) # doc = get_doc(en_vocab, words)
assert len(matcher(doc)) == 1 # assert len(matcher(doc)) == 1
#
#
def test_phrase_matcher_length(en_vocab): #def test_phrase_matcher_length(en_vocab):
matcher = PhraseMatcher(en_vocab) # matcher = PhraseMatcher(en_vocab)
assert len(matcher) == 0 # assert len(matcher) == 0
matcher.add('TEST', None, get_doc(en_vocab, ['test'])) # matcher.add('TEST', None, get_doc(en_vocab, ['test']))
assert len(matcher) == 1 # assert len(matcher) == 1
matcher.add('TEST2', None, get_doc(en_vocab, ['test2'])) # matcher.add('TEST2', None, get_doc(en_vocab, ['test2']))
assert len(matcher) == 2 # assert len(matcher) == 2
#
#
def test_phrase_matcher_contains(en_vocab): #def test_phrase_matcher_contains(en_vocab):
matcher = PhraseMatcher(en_vocab) # matcher = PhraseMatcher(en_vocab)
matcher.add('TEST', None, get_doc(en_vocab, ['test'])) # matcher.add('TEST', None, get_doc(en_vocab, ['test']))
assert 'TEST' in matcher # assert 'TEST' in matcher
assert 'TEST2' not in matcher # assert 'TEST2' not in matcher
#
#
def test_matcher_match_zero(matcher): #def test_matcher_match_zero(matcher):
words1 = 'He said , " some words " ...'.split() # words1 = 'He said , " some words " ...'.split()
words2 = 'He said , " some three words " ...'.split() # words2 = 'He said , " some three words " ...'.split()
pattern1 = [{'ORTH': '"'}, # pattern1 = [{'ORTH': '"'},
{'OP': '!', 'IS_PUNCT': True}, # {'OP': '!', 'IS_PUNCT': True},
{'OP': '!', 'IS_PUNCT': True}, # {'OP': '!', 'IS_PUNCT': True},
{'ORTH': '"'}] # {'ORTH': '"'}]
pattern2 = [{'ORTH': '"'}, # pattern2 = [{'ORTH': '"'},
{'IS_PUNCT': True}, # {'IS_PUNCT': True},
{'IS_PUNCT': True}, # {'IS_PUNCT': True},
{'IS_PUNCT': True}, # {'IS_PUNCT': True},
{'ORTH': '"'}] # {'ORTH': '"'}]
#
matcher.add('Quote', None, pattern1) # matcher.add('Quote', None, pattern1)
doc = get_doc(matcher.vocab, words1) # doc = get_doc(matcher.vocab, words1)
assert len(matcher(doc)) == 1 # assert len(matcher(doc)) == 1
#
doc = get_doc(matcher.vocab, words2) # doc = get_doc(matcher.vocab, words2)
assert len(matcher(doc)) == 0 # assert len(matcher(doc)) == 0
matcher.add('Quote', None, pattern2) # matcher.add('Quote', None, pattern2)
assert len(matcher(doc)) == 0 # assert len(matcher(doc)) == 0
#
#
def test_matcher_match_zero_plus(matcher): #def test_matcher_match_zero_plus(matcher):
words = 'He said , " some words " ...'.split() # words = 'He said , " some words " ...'.split()
pattern = [{'ORTH': '"'}, # pattern = [{'ORTH': '"'},
{'OP': '*', 'IS_PUNCT': False}, # {'OP': '*', 'IS_PUNCT': False},
{'ORTH': '"'}] # {'ORTH': '"'}]
matcher = Matcher(matcher.vocab) # matcher = Matcher(matcher.vocab)
matcher.add('Quote', None, pattern) # matcher.add('Quote', None, pattern)
doc = get_doc(matcher.vocab, words) # doc = get_doc(matcher.vocab, words)
assert len(matcher(doc)) == 1 # assert len(matcher(doc)) == 1
#
#
def test_matcher_match_one_plus(matcher): #def test_matcher_match_one_plus(matcher):
control = Matcher(matcher.vocab) # control = Matcher(matcher.vocab)
control.add('BasicPhilippe', None, [{'ORTH': 'Philippe'}]) # control.add('BasicPhilippe', None, [{'ORTH': 'Philippe'}])
doc = get_doc(control.vocab, ['Philippe', 'Philippe']) # doc = get_doc(control.vocab, ['Philippe', 'Philippe'])
m = control(doc) # m = control(doc)
assert len(m) == 2 # assert len(m) == 2
matcher.add('KleenePhilippe', None, [{'ORTH': 'Philippe', 'OP': '1'}, # matcher.add('KleenePhilippe', None, [{'ORTH': 'Philippe', 'OP': '1'},
{'ORTH': 'Philippe', 'OP': '+'}]) # {'ORTH': 'Philippe', 'OP': '+'}])
m = matcher(doc) # m = matcher(doc)
assert len(m) == 1 # assert len(m) == 1
#
#
def test_operator_combos(matcher): #def test_operator_combos(matcher):
cases = [ # cases = [
('aaab', 'a a a b', True), # ('aaab', 'a a a b', True),
('aaab', 'a+ b', True), # ('aaab', 'a+ b', True),
('aaab', 'a+ a+ b', True), # ('aaab', 'a+ a+ b', True),
('aaab', 'a+ a+ a b', True), # ('aaab', 'a+ a+ a b', True),
('aaab', 'a+ a+ a+ b', True), # ('aaab', 'a+ a+ a+ b', True),
('aaab', 'a+ a a b', True), # ('aaab', 'a+ a a b', True),
('aaab', 'a+ a a', True), # ('aaab', 'a+ a a', True),
('aaab', 'a+', True), # ('aaab', 'a+', True),
('aaa', 'a+ b', False), # ('aaa', 'a+ b', False),
('aaa', 'a+ a+ b', False), # ('aaa', 'a+ a+ b', False),
('aaa', 'a+ a+ a+ b', False), # ('aaa', 'a+ a+ a+ b', False),
('aaa', 'a+ a b', False), # ('aaa', 'a+ a b', False),
('aaa', 'a+ a a b', False), # ('aaa', 'a+ a a b', False),
('aaab', 'a+ a a', True), # ('aaab', 'a+ a a', True),
('aaab', 'a+', True), # ('aaab', 'a+', True),
('aaab', 'a+ a b', True), # ('aaab', 'a+ a b', True),
] # ]
for string, pattern_str, result in cases: # for string, pattern_str, result in cases:
matcher = Matcher(matcher.vocab) # matcher = Matcher(matcher.vocab)
doc = get_doc(matcher.vocab, words=list(string)) # doc = get_doc(matcher.vocab, words=list(string))
pattern = [] # pattern = []
for part in pattern_str.split(): # for part in pattern_str.split():
if part.endswith('+'): # if part.endswith('+'):
pattern.append({'ORTH': part[0], 'op': '+'}) # pattern.append({'ORTH': part[0], 'op': '+'})
else: # else:
pattern.append({'ORTH': part}) # pattern.append({'ORTH': part})
matcher.add('PATTERN', None, pattern) # matcher.add('PATTERN', None, pattern)
matches = matcher(doc) # matches = matcher(doc)
if result: # if result:
assert matches, (string, pattern_str) # assert matches, (string, pattern_str)
else: # else:
assert not matches, (string, pattern_str) # assert not matches, (string, pattern_str)
#
#
def test_matcher_end_zero_plus(matcher): #def test_matcher_end_zero_plus(matcher):
'''Test matcher works when patterns end with * operator. (issue 1450)''' # '''Test matcher works when patterns end with * operator. (issue 1450)'''
matcher = Matcher(matcher.vocab) # matcher = Matcher(matcher.vocab)
matcher.add( # matcher.add(
"TSTEND", # "TSTEND",
None, # None,
[ # [
{'ORTH': "a"}, # {'ORTH': "a"},
{'ORTH': "b", 'OP': "*"} # {'ORTH': "b", 'OP': "*"}
] # ]
) # )
nlp = lambda string: Doc(matcher.vocab, words=string.split()) # nlp = lambda string: Doc(matcher.vocab, words=string.split())
assert len(matcher(nlp(u'a'))) == 1 # assert len(matcher(nlp(u'a'))) == 1
assert len(matcher(nlp(u'a b'))) == 2 # assert len(matcher(nlp(u'a b'))) == 2
assert len(matcher(nlp(u'a c'))) == 1 # assert len(matcher(nlp(u'a c'))) == 1
assert len(matcher(nlp(u'a b c'))) == 2 # assert len(matcher(nlp(u'a b c'))) == 2
assert len(matcher(nlp(u'a b b c'))) == 3 # assert len(matcher(nlp(u'a b b c'))) == 3
assert len(matcher(nlp(u'a b b'))) == 3 # assert len(matcher(nlp(u'a b b'))) == 3