Rooba
/
spaCy
mirror of https://github.com/explosion/spaCy.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix tokenizer serialization

This commit is contained in:
Matthew Honnibal 2018-07-06 12:32:11 +02:00
parent e1569fda4e
commit 63f5651f8d
1 changed files with 4 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
spacy/tokenizer.pyx
View File

@ -375,10 +375,10 @@ cdef class Tokenizer:
""" """
serializers = OrderedDict(( serializers = OrderedDict((
('vocab', lambda: self.vocab.to_bytes()), ('vocab', lambda: self.vocab.to_bytes()),
('prefix_search', _get_regex_pattern(self.prefix_search)), ('prefix_search', lambda: _get_regex_pattern(self.prefix_search)),
('suffix_search', _get_regex_pattern(self.suffix_search)), ('suffix_search', lambda: _get_regex_pattern(self.suffix_search)),
('infix_finditer', _get_regex_pattern(self.infix_finditer)), ('infix_finditer', lambda: _get_regex_pattern(self.infix_finditer)),
('token_match', _get_regex_pattern(self.token_match)), ('token_match', lambda: _get_regex_pattern(self.token_match)),
('exceptions', lambda: OrderedDict(sorted(self._rules.items()))) ('exceptions', lambda: OrderedDict(sorted(self._rules.items())))
)) ))
return util.to_bytes(serializers, exclude) return util.to_bytes(serializers, exclude)