* Adjust hyphenation rule in tokenizer

This commit is contained in:
Matthew Honnibal 2015-06-28 06:18:58 +02:00
parent 2b5421e60c
commit b5b869366b
1 changed files with 2 additions and 1 deletions

View File

@ -1,2 +1,3 @@
(?<=[a-z])\.(?=[A-Z]) (?<=[a-z])\.(?=[A-Z])
(?<=[a-zA-Z])-(?=[a-zA-z]) (?<=[a-zA-Z0-9])-(?=[a-zA-z])
(?<=[a-zA-Z])-(?=[0-9a-zA-z])