Rooba
/
spaCy
mirror of https://github.com/explosion/spaCy.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add lookup lemmas to tokens without POS tags

This commit is contained in:
ines 2017-10-11 13:27:10 +02:00
parent 9620c1a640
commit 6dd14dc342
1 changed files with 6 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
spacy/morphology.pyx
View File

@ -67,9 +67,13 @@ cdef class Morphology:
self.exc), None, None)
cdef int assign_untagged(self, TokenC* token) except -1:
'''Set morphological attributes on a token without a POS tag.'''
"""Set morphological attributes on a token without a POS tag. Uses
the lemmatizer's lookup() method, which looks up the string in the
table provided by the language data as lemma_lookup (if available)."""
if token.lemma == 0:
token.lemma = self.lemmatize(0, token.lex.orth, {})
orth_str = self.strings[token.lex.orth]
lemma = self.lemmatizer.lookup(orth_str)
token.lemma = self.strings.add(lemma)
cdef int assign_tag(self, TokenC* token, tag) except -1:
if isinstance(tag, basestring):