Rooba
/
spaCy
mirror of https://github.com/explosion/spaCy.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Reorganise French punctuation rules

This commit is contained in:
ines 2017-05-09 00:00:54 +02:00
parent 014bda0ae3
commit e895d1afd7
1 changed files with 19 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
spacy/lang/fr/punctuation.py
View File

@ -1,29 +1,29 @@
# coding: utf8 # coding: utf8
from __future__ import unicode_literals from __future__ import unicode_literals
from ..char_classes import TOKENIZER_INFIXES, LIST_PUNCT LIST_ELLIPSES from ..punctuation import TOKENIZER_INFIXES
from ..char_classes import LIST_QUOTES, CURRENCY, QUOTES, UNITS from ..char_classes import LIST_PUNCT, LIST_ELLIPSES, LIST_QUOTES, CURRENCY
from ..char_classes import ALPHA, ALPHA_LOWER, ALPHA_UPPER from ..char_classes import QUOTES, UNITS, ALPHA, ALPHA_LOWER, ALPHA_UPPER
ELISION = " ' ’ ".strip().replace(' ', '').replace('\n', '') ELISION = " ' ’ ".strip().replace(' ', '').replace('\n', '')
HYPHENS = r"""- – — ‐ ‑""".strip().replace(' ', '').replace('\n', '') HYPHENS = r"- – — ‐ ‑".strip().replace(' ', '').replace('\n', '')
TOKENIZER_SUFFIXES = ( _suffixes = (LIST_PUNCT + LIST_ELLIPSES + LIST_QUOTES +
LIST_PUNCT + [r'(?<=[0-9])\+',
LIST_ELLIPSES +
LIST_QUOTES +
[
r'(?<=[0-9])\+',
r'(?<=°[FfCcKk])\.', # 4°C. -> ["4°C", "."] r'(?<=°[FfCcKk])\.', # 4°C. -> ["4°C", "."]
r'(?<=[0-9])°[FfCcKk]', # 4°C -> ["4", "°C"] r'(?<=[0-9])°[FfCcKk]', # 4°C -> ["4", "°C"]
r'(?<=[0-9])%', # 4% -> ["4", "%"] r'(?<=[0-9])%', # 4% -> ["4", "%"]
r'(?<=[0-9])(?:{c})'.format(c=CURRENCY), r'(?<=[0-9])(?:{})'.format(CURRENCY),
r'(?<=[0-9])(?:{u})'.format(u=UNITS), r'(?<=[0-9])(?:{})'.format(UNITS),
r'(?<=[0-9{al}{p}(?:{q})])\.'.format(al=ALPHA_LOWER, p=r'%²\-\)\]\+', q=QUOTES), r'(?<=[0-9{}{}(?:{})])\.'.format(ALPHA_LOWER, r'%²\-\)\]\+', QUOTES),
r'(?<=[{au}][{au}])\.'.format(au=ALPHA_UPPER)]) r'(?<=[{au}][{au}])\.'.format(au=ALPHA_UPPER)])
TOKENIZER_INFIXES += [ _infixes = (TOKENIZER_INFIXES +
r'(?<=[{a}][{el}])(?=[{a}])'.format(a=ALPHA, el=ELISION)] [r'(?<=[{a}][{el}])(?=[{a}])'.format(a=ALPHA, el=ELISION)])
TOKENIZER_SUFFIXES = _suffixes
TOKENIZER_INFIXES = _infixes