Rooba
/
spaCy
mirror of https://github.com/explosion/spaCy.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Move abbreviations below other exceptions

This commit is contained in:
Ines Montani 2017-01-05 19:58:07 +01:00
parent 7d2cf934b9
commit a23504fe07
1 changed files with 1 additions and 134 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

135
spacy/en/tokenizer_exceptions.py
View File

@ -505,142 +505,9 @@ ABBREVIATIONS = {
}
# Other exceptions
OTHER = {
" ": [
{ORTH: " ", TAG: "SP"}
],
"\u00a0": [
{ORTH: "\u00a0", TAG: "SP", LEMMA: " "}
],
"and/or": [
{ORTH: "and/or", LEMMA: "and/or", TAG: "CC"}
],
"'cause": [
{ORTH: "'cause", LEMMA: "because"}
],
"y'all": [
{ORTH: "y'", LEMMA: PRON_LEMMA, NORM: "you"},
{ORTH: "all"}
],
"yall": [
{ORTH: "y", LEMMA: PRON_LEMMA, NORM: "you"},
{ORTH: "all"}
],
"'em": [
{ORTH: "'em", LEMMA: PRON_LEMMA, NORM: "them"}
],
"em": [
{ORTH: "em", LEMMA: PRON_LEMMA, NORM: "them"}
],
"nothin'": [
{ORTH: "nothin'", LEMMA: "nothing"}
],
"nuthin'": [
{ORTH: "nuthin'", LEMMA: "nothing"}
],
"'nuff": [
{ORTH: "'nuff", LEMMA: "enough"}
],
"ol'": [
{ORTH: "ol'", LEMMA: "old"}
],
"not've": [
{ORTH: "not", LEMMA: "not", TAG: "RB"},
{ORTH: "'ve", LEMMA: "have", TAG: "VB"}
],
"notve": [
{ORTH: "not", LEMMA: "not", TAG: "RB"},
{ORTH: "ve", LEMMA: "have", TAG: "VB"}
],
"Not've": [
{ORTH: "Not", LEMMA: "not", TAG: "RB"},
{ORTH: "'ve", LEMMA: "have", TAG: "VB"}
],
"Notve": [
{ORTH: "Not", LEMMA: "not", TAG: "RB"},
{ORTH: "ve", LEMMA: "have", TAG: "VB"}
],
"cannot": [
{ORTH: "can", LEMMA: "can", TAG: "MD"},
{ORTH: "not", LEMMA: "not", TAG: "RB"}
],
"Cannot": [
{ORTH: "Can", LEMMA: "can", TAG: "MD"},
{ORTH: "not", LEMMA: "not", TAG: "RB"}
],
"gonna": [
{ORTH: "gon", LEMMA: "go", NORM: "going"},
{ORTH: "na", LEMMA: "to"}
],
"Gonna": [
{ORTH: "Gon", LEMMA: "go", NORM: "going"},
{ORTH: "na", LEMMA: "to"}
],
"let's": [
{ORTH: "let"},
{ORTH: "'s", LEMMA: PRON_LEMMA, NORM: "us"}
],
"Let's": [
{ORTH: "Let"},
{ORTH: "'s", LEMMA: PRON_LEMMA, NORM: "us"}
],
"'S": [
{ORTH: "'S", LEMMA: "'s"}
],
"'s": [
{ORTH: "'s", LEMMA: "'s"}
],
"\u2018S": [
{ORTH: "\u2018S", LEMMA: "'s"}
],
"\u2018s": [
{ORTH: "\u2018s", LEMMA: "'s"}
],
"\u2014": [
{ORTH: "\u2014", TAG: ":", LEMMA: "--"}
],
"\n": [
{ORTH: "\n", TAG: "SP"}
],
"\t": [
{ORTH: "\t", TAG: "SP"}
]
}
TOKENIZER_EXCEPTIONS = dict(EXC)
TOKENIZER_EXCEPTIONS.update(ABBREVIATIONS)
TOKENIZER_EXCEPTIONS.update(OTHER)
TOKENIZER_EXCEPTIONS.update(ABBREVIATIONS)
# Remove EXCLUDE_EXC if in exceptions