spaCy/spacy/lemmatizer.py

98 lines
2.5 KiB
Python
Raw Normal View History

from __future__ import unicode_literals, print_function
2014-12-23 04:16:57 +00:00
from os import path
2015-01-04 19:01:32 +00:00
import codecs
2014-12-23 04:16:57 +00:00
try:
import ujson as json
except ImportError:
import json
from .parts_of_speech import NOUN, VERB, ADJ, PUNCT
2014-12-23 04:16:57 +00:00
class Lemmatizer(object):
@classmethod
2015-12-07 05:01:28 +00:00
def from_package(cls, package):
index = {}
exc = {}
for pos in ['adj', 'noun', 'verb']:
2015-12-07 05:01:28 +00:00
index[pos] = package.load_utf8(read_index,
'wordnet', 'index.%s' % pos,
2015-12-07 05:01:28 +00:00
default=set()) # TODO: really optional?
exc[pos] = package.load_utf8(read_exc,
'wordnet', '%s.exc' % pos,
2015-12-07 05:01:28 +00:00
default={}) # TODO: really optional?
rules = package.load_utf8(json.load,
'vocab', 'lemma_rules.json',
2015-12-07 05:01:28 +00:00
default={}) # TODO: really optional?
return cls(index, exc, rules)
2014-12-23 04:16:57 +00:00
def __init__(self, index, exceptions, rules):
self.index = index
self.exc = exceptions
self.rules = rules
2015-08-25 13:46:19 +00:00
def __call__(self, string, pos):
if pos == NOUN:
pos = 'noun'
elif pos == VERB:
pos = 'verb'
elif pos == ADJ:
pos = 'adj'
elif pos == PUNCT:
pos = 'punct'
2015-09-10 12:51:39 +00:00
lemmas = lemmatize(string, self.index.get(pos, {}), self.exc.get(pos, {}), self.rules.get(pos, []))
2015-09-08 13:38:03 +00:00
return lemmas
2014-12-23 04:16:57 +00:00
def noun(self, string):
2015-08-25 13:46:19 +00:00
return self(string, 'noun')
2014-12-23 04:16:57 +00:00
def verb(self, string):
2015-08-25 13:46:19 +00:00
return self(string, 'verb')
2014-12-23 04:16:57 +00:00
def adj(self, string):
2015-08-25 13:46:19 +00:00
return self(string, 'adj')
2014-12-23 04:16:57 +00:00
def punct(self, string):
return self(string, 'punct')
2014-12-23 04:16:57 +00:00
def lemmatize(string, index, exceptions, rules):
string = string.lower()
forms = []
if string in index:
forms.append(string)
forms.extend(exceptions.get(string, []))
for old, new in rules:
if string.endswith(old):
form = string[:len(string) - len(old)] + new
if form in index or not form.isalpha():
2014-12-23 04:16:57 +00:00
forms.append(form)
if not forms:
forms.append(string)
return set(forms)
2015-12-07 05:01:28 +00:00
def read_index(fileobj):
2014-12-23 04:16:57 +00:00
index = set()
2015-12-07 05:01:28 +00:00
for line in fileobj:
2014-12-23 04:16:57 +00:00
if line.startswith(' '):
continue
pieces = line.split()
word = pieces[0]
if word.count('_') == 0:
index.add(word)
return index
2015-12-07 05:01:28 +00:00
def read_exc(fileobj):
2014-12-23 04:16:57 +00:00
exceptions = {}
2015-12-07 05:01:28 +00:00
for line in fileobj:
2014-12-23 04:16:57 +00:00
if line.startswith(' '):
continue
pieces = line.split()
exceptions[pieces[0]] = tuple(pieces[1:])
return exceptions