Rooba
/
spaCy
mirror of https://github.com/explosion/spaCy.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

* Add comment to tokenizer, explaining the spacy attr

This commit is contained in:
Matthew Honnibal 2015-07-13 22:29:13 +02:00
parent 8214b74eec
commit 24d6ce99ec
1 changed files with 0 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
spacy/tokenizer.pyx
View File

@ -185,9 +185,6 @@ cdef class Tokenizer:
cdef const LexemeC* lexeme cdef const LexemeC* lexeme
cdef UniStr span cdef UniStr span
cdef int i cdef int i
# Have to calculate is_spacy here, i.e. does the token have a trailing
# space. There are no spaces *between* the tokens we attach
# here, and there *is* a space after the last token.
if prefixes.size(): if prefixes.size():
for i in range(prefixes.size()): for i in range(prefixes.size()):
tokens.push_back(prefixes[0][i], False) tokens.push_back(prefixes[0][i], False)