Pass lemmatizer morphological features, so that rules are sensitive to base/inflected distinction, which is how the WordNet data is designed. See Issue #435

This commit is contained in:
Matthew Honnibal 2016-09-27 13:52:11 +02:00
parent e233328d38
commit 3cb4d455d2
1 changed files with 25 additions and 11 deletions

View File

@ -37,7 +37,7 @@ class Lemmatizer(object):
self.exc = exceptions self.exc = exceptions
self.rules = rules self.rules = rules
def __call__(self, string, pos): def __call__(self, string, pos, **morphology):
if pos == NOUN: if pos == NOUN:
pos = 'noun' pos = 'noun'
elif pos == VERB: elif pos == VERB:
@ -46,27 +46,41 @@ class Lemmatizer(object):
pos = 'adj' pos = 'adj'
elif pos == PUNCT: elif pos == PUNCT:
pos = 'punct' pos = 'punct'
# See Issue #435 for example of where this logic is requied.
if self.is_base_form(pos, **morphology):
return set([string.lower()])
lemmas = lemmatize(string, self.index.get(pos, {}), self.exc.get(pos, {}), self.rules.get(pos, [])) lemmas = lemmatize(string, self.index.get(pos, {}), self.exc.get(pos, {}), self.rules.get(pos, []))
return lemmas return lemmas
def noun(self, string): def is_base_form(self, pos, **morphology):
return self(string, 'noun') '''Check whether we're dealing with an uninflected paradigm, so we can
avoid lemmatization entirely.'''
if pos == 'noun' and morphology.get('number') == 'sing' and len(morphology) == 1:
return True
elif pos == 'verb' and morphology.get('verbform') == 'inf' and len(morphology) == 1:
return True
else:
return False
def verb(self, string): def noun(self, string, **morphology):
return self(string, 'verb') return self(string, 'noun', **morphology)
def adj(self, string): def verb(self, string, **morphology):
return self(string, 'adj') return self(string, 'verb', **morphology)
def punct(self, string): def adj(self, string, **morphology):
return self(string, 'punct') return self(string, 'adj', **morphology)
def punct(self, string, **morphology):
return self(string, 'punct', **morphology)
def lemmatize(string, index, exceptions, rules): def lemmatize(string, index, exceptions, rules):
string = string.lower() string = string.lower()
forms = [] forms = []
if string in index: # TODO: Is this correct? See discussion in Issue #435.
forms.append(string) #if string in index:
# forms.append(string)
forms.extend(exceptions.get(string, [])) forms.extend(exceptions.get(string, []))
for old, new in rules: for old, new in rules:
if string.endswith(old): if string.endswith(old):