Rooba
/
spaCy
mirror of https://github.com/explosion/spaCy.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix tokenizer deserialization

This commit is contained in:
Matthew Honnibal 2018-07-06 12:36:33 +02:00
parent 63f5651f8d
commit 6c8d627733
1 changed files with 3 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
spacy/tokenizer.pyx
View File

@ -400,11 +400,11 @@ cdef class Tokenizer:
('exceptions', lambda b: data.setdefault('rules', b)) ('exceptions', lambda b: data.setdefault('rules', b))
)) ))
msg = util.from_bytes(bytes_data, deserializers, exclude) msg = util.from_bytes(bytes_data, deserializers, exclude)
if 'prefix_search' in data: if data.get('prefix_search'):
self.prefix_search = re.compile(data['prefix_search']).search self.prefix_search = re.compile(data['prefix_search']).search
if 'suffix_search' in data: if data.get('suffix_search'):
self.suffix_search = re.compile(data['suffix_search']).search self.suffix_search = re.compile(data['suffix_search']).search
if 'infix_finditer' in data: if data.get('infix_finditer'):
self.infix_finditer = re.compile(data['infix_finditer']).finditer self.infix_finditer = re.compile(data['infix_finditer']).finditer
if data.get('token_match'): if data.get('token_match'):
self.token_match = re.compile(data['token_match']).search self.token_match = re.compile(data['token_match']).search