enable tokenizer exceptions

2017-07-24 14:11:10 +07:00 · 2017-07-24 14:11:10 +07:00 · 7aad6718bc
parent ad56c9179a
commit 7aad6718bc
1 changed files with 8 additions and 1 deletions
--- a/spacy/lang/id/tokenizer_exceptions.py
+++ b/spacy/lang/id/tokenizer_exceptions.py
@ -1,4 +1,11 @@
 # coding: utf8
 from __future__ import unicode_literals

-TOKENIZER_EXCEPTIONS = {}
+from ._tokenizer_exceptions_list import FR_BASE_EXCEPTIONS
+
+_exc = {}
+
+for orth in FR_BASE_EXCEPTIONS + ["etc."]:
+    _exc[orth] = [{ORTH: orth}]
+
+TOKENIZER_EXCEPTIONS = dict(_exc)