From 5445074cbdf962afa1f2705e9c99949465bf364e Mon Sep 17 00:00:00 2001 From: Ines Montani Date: Sat, 17 Dec 2016 12:34:08 +0100 Subject: [PATCH] Expand tokenizer exceptions with unicode apostrophe (fixes #685) --- spacy/en/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/spacy/en/__init__.py b/spacy/en/__init__.py index 8979cc8fb..623585f7d 100644 --- a/spacy/en/__init__.py +++ b/spacy/en/__init__.py @@ -13,6 +13,7 @@ from ..attrs import LANG from ..language_data import update_exc from ..language_data import strings_to_exc +from ..language_data import expand_exc from ..language_data import EMOTICONS from .language_data import ORTH_ONLY @@ -30,6 +31,7 @@ STOP_WORDS = set(language_data.STOP_WORDS) update_exc(TOKENIZER_EXCEPTIONS, strings_to_exc(EMOTICONS)) update_exc(TOKENIZER_EXCEPTIONS, strings_to_exc(ORTH_ONLY)) update_exc(TOKENIZER_EXCEPTIONS, get_time_exc(range(1, 12 + 1))) +update_exc(TOKENIZER_EXCEPTIONS, expand_exc(TOKENIZER_EXCEPTIONS, "'", "’")) class English(Language):