2017-04-15 10:05:47 +00:00
|
|
|
# coding: utf8
|
|
|
|
from __future__ import unicode_literals
|
2015-08-27 07:16:11 +00:00
|
|
|
|
2016-12-18 12:28:20 +00:00
|
|
|
from .symbols import POS, NOUN, VERB, ADJ, PUNCT
|
2017-04-15 10:05:47 +00:00
|
|
|
from .symbols import VerbForm_inf, VerbForm_none, Number_sing, Degree_pos
|
2015-08-27 07:16:11 +00:00
|
|
|
|
2014-12-23 04:16:57 +00:00
|
|
|
|
|
|
|
class Lemmatizer(object):
|
2015-08-27 07:16:11 +00:00
|
|
|
@classmethod
|
2017-10-24 14:00:33 +00:00
|
|
|
def load(cls, path, index=None, exc=None, rules=None, lookup=None):
|
|
|
|
return cls(index or {}, exc or {}, rules or {}, lookup or {})
|
2014-12-23 04:16:57 +00:00
|
|
|
|
2017-10-11 11:25:51 +00:00
|
|
|
def __init__(self, index=None, exceptions=None, rules=None, lookup=None):
|
|
|
|
self.index = index if index is not None else {}
|
|
|
|
self.exc = exceptions if exceptions is not None else {}
|
|
|
|
self.rules = rules if rules is not None else {}
|
|
|
|
self.lookup_table = lookup if lookup is not None else {}
|
2015-08-25 13:46:19 +00:00
|
|
|
|
2016-12-07 20:12:49 +00:00
|
|
|
def __call__(self, string, univ_pos, morphology=None):
|
2017-10-12 20:50:43 +00:00
|
|
|
if univ_pos in (NOUN, 'NOUN', 'noun'):
|
2016-09-27 12:16:22 +00:00
|
|
|
univ_pos = 'noun'
|
2017-10-12 20:50:43 +00:00
|
|
|
elif univ_pos in (VERB, 'VERB', 'verb'):
|
2016-09-27 12:16:22 +00:00
|
|
|
univ_pos = 'verb'
|
2017-10-12 20:50:43 +00:00
|
|
|
elif univ_pos in (ADJ, 'ADJ', 'adj'):
|
2016-09-27 12:16:22 +00:00
|
|
|
univ_pos = 'adj'
|
2017-10-12 20:50:43 +00:00
|
|
|
elif univ_pos in (PUNCT, 'PUNCT', 'punct'):
|
2016-09-27 12:16:22 +00:00
|
|
|
univ_pos = 'punct'
|
2017-10-11 01:23:23 +00:00
|
|
|
else:
|
2017-10-24 14:00:54 +00:00
|
|
|
return list(set([string.lower()]))
|
2016-09-27 11:52:11 +00:00
|
|
|
# See Issue #435 for example of where this logic is requied.
|
2016-12-07 20:12:49 +00:00
|
|
|
if self.is_base_form(univ_pos, morphology):
|
2017-10-24 14:00:54 +00:00
|
|
|
return list(set([string.lower()]))
|
2016-09-27 15:47:05 +00:00
|
|
|
lemmas = lemmatize(string, self.index.get(univ_pos, {}),
|
|
|
|
self.exc.get(univ_pos, {}),
|
|
|
|
self.rules.get(univ_pos, []))
|
2015-09-08 13:38:03 +00:00
|
|
|
return lemmas
|
2014-12-23 04:16:57 +00:00
|
|
|
|
2016-12-07 20:12:49 +00:00
|
|
|
def is_base_form(self, univ_pos, morphology=None):
|
2017-04-15 09:59:21 +00:00
|
|
|
"""
|
|
|
|
Check whether we're dealing with an uninflected paradigm, so we can
|
|
|
|
avoid lemmatization entirely.
|
|
|
|
"""
|
2016-12-07 20:12:49 +00:00
|
|
|
morphology = {} if morphology is None else morphology
|
2017-09-06 17:13:24 +00:00
|
|
|
others = [key for key in morphology
|
|
|
|
if key not in (POS, 'Number', 'POS', 'VerbForm', 'Tense')]
|
2017-03-16 22:59:31 +00:00
|
|
|
true_morph_key = morphology.get('morph', 0)
|
2017-03-25 20:56:41 +00:00
|
|
|
if univ_pos == 'noun' and morphology.get('Number') == 'sing':
|
2016-09-27 11:52:11 +00:00
|
|
|
return True
|
2017-03-25 20:56:41 +00:00
|
|
|
elif univ_pos == 'verb' and morphology.get('VerbForm') == 'inf':
|
2016-09-27 11:52:11 +00:00
|
|
|
return True
|
2017-09-04 13:18:41 +00:00
|
|
|
# This maps 'VBP' to base form -- probably just need 'IS_BASE'
|
|
|
|
# morphology
|
|
|
|
elif univ_pos == 'verb' and (morphology.get('VerbForm') == 'fin' and \
|
2017-09-06 17:13:24 +00:00
|
|
|
morphology.get('Tense') == 'pres' and \
|
|
|
|
morphology.get('Number') is None and \
|
|
|
|
not others):
|
2017-09-04 13:18:41 +00:00
|
|
|
return True
|
2017-03-25 20:29:57 +00:00
|
|
|
elif univ_pos == 'adj' and morphology.get('Degree') == 'pos':
|
|
|
|
return True
|
2017-03-25 20:56:41 +00:00
|
|
|
elif VerbForm_inf in morphology:
|
|
|
|
return True
|
|
|
|
elif VerbForm_none in morphology:
|
|
|
|
return True
|
|
|
|
elif Number_sing in morphology:
|
|
|
|
return True
|
|
|
|
elif Degree_pos in morphology:
|
2017-03-16 22:59:31 +00:00
|
|
|
return True
|
2016-09-27 11:52:11 +00:00
|
|
|
else:
|
|
|
|
return False
|
2014-12-23 04:16:57 +00:00
|
|
|
|
2016-12-07 20:12:49 +00:00
|
|
|
def noun(self, string, morphology=None):
|
|
|
|
return self(string, 'noun', morphology)
|
2014-12-23 04:16:57 +00:00
|
|
|
|
2016-12-07 20:12:49 +00:00
|
|
|
def verb(self, string, morphology=None):
|
|
|
|
return self(string, 'verb', morphology)
|
2014-12-23 04:16:57 +00:00
|
|
|
|
2016-12-07 20:12:49 +00:00
|
|
|
def adj(self, string, morphology=None):
|
|
|
|
return self(string, 'adj', morphology)
|
2016-09-27 11:52:11 +00:00
|
|
|
|
2016-12-07 20:12:49 +00:00
|
|
|
def punct(self, string, morphology=None):
|
|
|
|
return self(string, 'punct', morphology)
|
2015-10-09 07:44:21 +00:00
|
|
|
|
2017-10-11 11:25:51 +00:00
|
|
|
def lookup(self, string):
|
|
|
|
if string in self.lookup_table:
|
|
|
|
return self.lookup_table[string]
|
|
|
|
return string
|
|
|
|
|
2014-12-23 04:16:57 +00:00
|
|
|
|
|
|
|
def lemmatize(string, index, exceptions, rules):
|
|
|
|
string = string.lower()
|
|
|
|
forms = []
|
2016-09-27 11:52:11 +00:00
|
|
|
# TODO: Is this correct? See discussion in Issue #435.
|
|
|
|
#if string in index:
|
|
|
|
# forms.append(string)
|
2014-12-23 04:16:57 +00:00
|
|
|
forms.extend(exceptions.get(string, []))
|
2017-03-01 20:44:17 +00:00
|
|
|
oov_forms = []
|
2017-10-14 11:32:55 +00:00
|
|
|
if not forms:
|
|
|
|
for old, new in rules:
|
|
|
|
if string.endswith(old):
|
|
|
|
form = string[:len(string) - len(old)] + new
|
|
|
|
if not form:
|
|
|
|
pass
|
|
|
|
elif form in index or not form.isalpha():
|
|
|
|
forms.append(form)
|
|
|
|
else:
|
|
|
|
oov_forms.append(form)
|
2014-12-23 04:16:57 +00:00
|
|
|
if not forms:
|
2017-03-01 20:44:17 +00:00
|
|
|
forms.extend(oov_forms)
|
2017-03-16 22:59:31 +00:00
|
|
|
if not forms:
|
|
|
|
forms.append(string)
|
2017-10-24 14:00:54 +00:00
|
|
|
return list(set(forms))
|