# coding: utf8 from __future__ import unicode_literals from ..char_classes import LIST_PUNCT, LIST_ELLIPSES, LIST_QUOTES, CURRENCY from ..char_classes import UNITS, ALPHA_UPPER _suffixes = ( LIST_PUNCT + LIST_ELLIPSES + LIST_QUOTES + [ r"(?<=[0-9])\+", r"(?<=[0-9])%", # 4% -> ["4", "%"] # Persian is written from Right-To-Left r"(?<=[0-9])(?:{})".format(CURRENCY), r"(?<=[0-9])(?:{})".format(UNITS), r"(?<=[{au}][{au}])\.".format(au=ALPHA_UPPER), ] ) TOKENIZER_SUFFIXES = _suffixes