From 7bec0ebbcb9c5f0e1434c43e7c19ef2420e4fd23 Mon Sep 17 00:00:00 2001 From: svlandeg Date: Wed, 28 Aug 2019 14:16:34 +0200 Subject: [PATCH 1/2] failing unit test for Issue 4190 --- spacy/tests/regression/test_issue4190.py | 57 ++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 spacy/tests/regression/test_issue4190.py diff --git a/spacy/tests/regression/test_issue4190.py b/spacy/tests/regression/test_issue4190.py new file mode 100644 index 000000000..464996705 --- /dev/null +++ b/spacy/tests/regression/test_issue4190.py @@ -0,0 +1,57 @@ +# coding: utf8 +from __future__ import unicode_literals + +from spacy.lang.en import English + +import spacy +from spacy.tokenizer import Tokenizer + +from spacy.tests.util import make_tempdir + + +def test_issue4190(): + test_string = "Test c." + + # Load default language + nlp_1 = English() + doc_1a = nlp_1(test_string) + result_1a = [token.text for token in doc_1a] + + # Modify tokenizer + customize_tokenizer(nlp_1) + doc_1b = nlp_1(test_string) + result_1b = [token.text for token in doc_1b] + + # Save and Reload + with make_tempdir() as model_dir: + nlp_1.to_disk(model_dir) + nlp_2 = spacy.load(model_dir) + + # This should be the modified tokenizer + doc_2 = nlp_2(test_string) + result_2 = [token.text for token in doc_2] + + assert result_1b == result_2 + + +def customize_tokenizer(nlp): + prefix_re = spacy.util.compile_prefix_regex(nlp.Defaults.prefixes) + suffix_re = spacy.util.compile_suffix_regex(nlp.Defaults.suffixes) + infix_re = spacy.util.compile_infix_regex(nlp.Defaults.infixes) + + # remove all exceptions where a single letter is followed by a period (e.g. 'h.') + exceptions = { + k: v + for k, v in dict(nlp.Defaults.tokenizer_exceptions).items() + if not (len(k) == 2 and k[1] == ".") + } + new_tokenizer = Tokenizer( + nlp.vocab, + exceptions, + prefix_search=prefix_re.search, + suffix_search=suffix_re.search, + infix_finditer=infix_re.finditer, + token_match=nlp.tokenizer.token_match, + ) + + nlp.tokenizer = new_tokenizer From c54aabc3cdc1381db256c9e29bb97d43554b76c4 Mon Sep 17 00:00:00 2001 From: svlandeg Date: Wed, 28 Aug 2019 14:17:44 +0200 Subject: [PATCH 2/2] fix loading custom tokenizer rules/exceptions from file --- spacy/tokenizer.pyx | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/spacy/tokenizer.pyx b/spacy/tokenizer.pyx index f19f851c7..19029ec05 100644 --- a/spacy/tokenizer.pyx +++ b/spacy/tokenizer.pyx @@ -441,8 +441,13 @@ cdef class Tokenizer: self.infix_finditer = re.compile(data["infix_finditer"]).finditer if data.get("token_match"): self.token_match = re.compile(data["token_match"]).match - for string, substrings in data.get("rules", {}).items(): - self.add_special_case(string, substrings) + if data.get("rules"): + # make sure to hard reset the cache to remove data from the default exceptions + self._rules = {} + self._cache = PreshMap() + for string, substrings in data.get("rules", {}).items(): + self.add_special_case(string, substrings) + return self