spaCy/spacy/lang/pl/tokenizer_exceptions.py

24 lines
580 B
Python
Raw Normal View History

2017-06-26 21:40:04 +00:00
# encoding: utf8
from __future__ import unicode_literals
from ...symbols import ORTH, LEMMA, POS, ADV, ADJ, NOUN
2017-06-26 21:40:04 +00:00
_exc = {}
for exc_data in [
{ORTH: "m.in.", LEMMA: "między innymi", POS: ADV},
{ORTH: "inż.", LEMMA: "inżynier", POS: NOUN},
{ORTH: "mgr.", LEMMA: "magister", POS: NOUN},
{ORTH: "tzn.", LEMMA: "to znaczy", POS: ADV},
{ORTH: "tj.", LEMMA: "to jest", POS: ADV},
{ORTH: "tzw.", LEMMA: "tak zwany", POS: ADJ},
]:
2017-11-01 22:02:45 +00:00
_exc[exc_data[ORTH]] = [exc_data]
2017-06-26 21:40:04 +00:00
for orth in ["w.", "r."]:
2017-06-26 21:40:04 +00:00
_exc[orth] = [{ORTH: orth}]
TOKENIZER_EXCEPTIONS = _exc