Rooba
/
spaCy
mirror of https://github.com/explosion/spaCy.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1566 from MathiasDesch/master (resolves #1248) 

Add exceptions to tokenizer and norm
This commit is contained in:
Ines Montani 2017-11-13 19:05:22 +01:00 committed by GitHub
parent 1b348389bb d82f868e1c
commit ea6c85c67a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 26 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@ -97,3 +97,6 @@ Desktop.ini
# Other
*.tgz
# Pycharm project files
*.idea

8
spacy/lang/en/norm_exceptions.py
View File

@ -459,6 +459,8 @@ _exc = {
"disorganised": "disorganized",
"distil": "distill",
"distils": "distills",
"doin": "doing",
"doin'": "doing",
"dramatisation": "dramatization",
"dramatisations": "dramatizations",
"dramatise": "dramatize",
@ -687,6 +689,8 @@ _exc = {
"globalises": "globalizes",
"globalising": "globalizing",
"glueing ": "gluing ",
"goin": "going",
"goin'":"going",
"goitre": "goiter",
"goitres": "goiters",
"gonorrhoea": "gonorrhea",
@ -733,6 +737,8 @@ _exc = {
"harmonised": "harmonized",
"harmonises": "harmonizes",
"harmonising": "harmonizing",
"havin": "having",
"havin'": "having",
"homoeopath": "homeopath",
"homoeopathic": "homeopathic",
"homoeopaths": "homeopaths",
@ -924,6 +930,8 @@ _exc = {
"localised": "localized",
"localises": "localizes",
"localising": "localizing",
"lovin": "loving",
"lovin'": "loving",
"louvre": "louver",
"louvred": "louvered",
"louvres": "louvers ",

15
spacy/lang/en/tokenizer_exceptions.py
View File

@ -387,6 +387,21 @@ for exc_data in [
{ORTH: "O'clock", LEMMA: "o'clock", NORM: "o'clock"},
{ORTH: "lovin'", LEMMA: "love", NORM: "loving"},
{ORTH: "Lovin'", LEMMA: "love", NORM: "loving"},
{ORTH: "lovin", LEMMA: "love", NORM: "loving"},
{ORTH: "Lovin", LEMMA: "love", NORM: "loving"},
{ORTH: "havin'", LEMMA: "have", NORM: "having"},
{ORTH: "Havin'", LEMMA: "have", NORM: "having"},
{ORTH: "havin", LEMMA: "have", NORM: "having"},
{ORTH: "Havin", LEMMA: "have", NORM: "having"},
{ORTH: "doin'", LEMMA: "do", NORM: "doing"},
{ORTH: "Doin'", LEMMA: "do", NORM: "doing"},
{ORTH: "doin", LEMMA: "do", NORM: "doing"},
{ORTH: "Doin", LEMMA: "do", NORM: "doing"},
{ORTH: "goin'", LEMMA: "go", NORM: "going"},
{ORTH: "Goin'", LEMMA: "go", NORM: "going"},
{ORTH: "goin", LEMMA: "go", NORM: "going"},
{ORTH: "Goin", LEMMA: "go", NORM: "going"},
{ORTH: "Mt.", LEMMA: "Mount", NORM: "Mount"},
{ORTH: "Ak.", LEMMA: "Alaska", NORM: "Alaska"},