mirror of https://github.com/explosion/spaCy.git
11 lines
176 B
Python
11 lines
176 B
Python
|
# coding: utf8
|
|||
|
|
from __future__ import unicode_literals
|
|||
|
|
|
|||
|
|
from ..punctuation import TOKENIZER_SUFFIXES
|
|||
|
|
|
|||
|
|
|
|||
|
|
_suffixes = TOKENIZER_SUFFIXES + ["۔"]
|
|||
|
|
|
|||
|
|
|
|||
|
|
TOKENIZER_SUFFIXES = _suffixes
|