mirror of https://github.com/explosion/spaCy.git
Pass lemmatizer morphological features, so that rules are sensitive to base/inflected distinction, which is how the WordNet data is designed. See Issue #435
This commit is contained in:
parent
e233328d38
commit
3cb4d455d2
|
@ -37,7 +37,7 @@ class Lemmatizer(object):
|
||||||
self.exc = exceptions
|
self.exc = exceptions
|
||||||
self.rules = rules
|
self.rules = rules
|
||||||
|
|
||||||
def __call__(self, string, pos):
|
def __call__(self, string, pos, **morphology):
|
||||||
if pos == NOUN:
|
if pos == NOUN:
|
||||||
pos = 'noun'
|
pos = 'noun'
|
||||||
elif pos == VERB:
|
elif pos == VERB:
|
||||||
|
@ -46,27 +46,41 @@ class Lemmatizer(object):
|
||||||
pos = 'adj'
|
pos = 'adj'
|
||||||
elif pos == PUNCT:
|
elif pos == PUNCT:
|
||||||
pos = 'punct'
|
pos = 'punct'
|
||||||
|
# See Issue #435 for example of where this logic is requied.
|
||||||
|
if self.is_base_form(pos, **morphology):
|
||||||
|
return set([string.lower()])
|
||||||
lemmas = lemmatize(string, self.index.get(pos, {}), self.exc.get(pos, {}), self.rules.get(pos, []))
|
lemmas = lemmatize(string, self.index.get(pos, {}), self.exc.get(pos, {}), self.rules.get(pos, []))
|
||||||
return lemmas
|
return lemmas
|
||||||
|
|
||||||
def noun(self, string):
|
def is_base_form(self, pos, **morphology):
|
||||||
return self(string, 'noun')
|
'''Check whether we're dealing with an uninflected paradigm, so we can
|
||||||
|
avoid lemmatization entirely.'''
|
||||||
|
if pos == 'noun' and morphology.get('number') == 'sing' and len(morphology) == 1:
|
||||||
|
return True
|
||||||
|
elif pos == 'verb' and morphology.get('verbform') == 'inf' and len(morphology) == 1:
|
||||||
|
return True
|
||||||
|
else:
|
||||||
|
return False
|
||||||
|
|
||||||
def verb(self, string):
|
def noun(self, string, **morphology):
|
||||||
return self(string, 'verb')
|
return self(string, 'noun', **morphology)
|
||||||
|
|
||||||
def adj(self, string):
|
def verb(self, string, **morphology):
|
||||||
return self(string, 'adj')
|
return self(string, 'verb', **morphology)
|
||||||
|
|
||||||
def punct(self, string):
|
def adj(self, string, **morphology):
|
||||||
return self(string, 'punct')
|
return self(string, 'adj', **morphology)
|
||||||
|
|
||||||
|
def punct(self, string, **morphology):
|
||||||
|
return self(string, 'punct', **morphology)
|
||||||
|
|
||||||
|
|
||||||
def lemmatize(string, index, exceptions, rules):
|
def lemmatize(string, index, exceptions, rules):
|
||||||
string = string.lower()
|
string = string.lower()
|
||||||
forms = []
|
forms = []
|
||||||
if string in index:
|
# TODO: Is this correct? See discussion in Issue #435.
|
||||||
forms.append(string)
|
#if string in index:
|
||||||
|
# forms.append(string)
|
||||||
forms.extend(exceptions.get(string, []))
|
forms.extend(exceptions.get(string, []))
|
||||||
for old, new in rules:
|
for old, new in rules:
|
||||||
if string.endswith(old):
|
if string.endswith(old):
|
||||||
|
|
Loading…
Reference in New Issue