Fix infixes in Italian

2016-11-02 20:43:52 +01:00 · 2016-11-02 20:43:52 +01:00 · 8f24dc1982
parent 41a4766c1c
commit 8f24dc1982
1 changed files with 4 additions and 1 deletions
--- a/spacy/it/language_data.py
+++ b/spacy/it/language_data.py
@ -114,7 +114,10 @@ _
 '''.strip().split('\n')


-TOKENIZER_INFIXES = tuple()
+TOKENIZER_INFIXES = (r'''\.\.\.+ (?<=[a-z])\.(?=[A-Z]) (?<=[a-zA-Z])-(?=[a-zA-z]) '''
+                     r'''(?<=[a-zA-Z])--(?=[a-zA-z]) (?<=[0-9])-(?=[0-9]) '''
+                     r'''(?<=[A-Za-z]),(?=[A-Za-z])''').split()
+


 TOKENIZER_EXCEPTIONS = {