Load token_match regex with .match, not .search

This commit is contained in:
Matthew Honnibal 2019-02-21 09:09:03 +01:00
parent 9a478b6db8
commit b21481eeca
1 changed files with 1 additions and 1 deletions

View File

@ -407,7 +407,7 @@ cdef class Tokenizer:
if data.get('infix_finditer'):
self.infix_finditer = re.compile(data['infix_finditer']).finditer
if data.get('token_match'):
self.token_match = re.compile(data['token_match']).search
self.token_match = re.compile(data['token_match']).match
for string, substrings in data.get('rules', {}).items():
self.add_special_case(string, substrings)
return self