* Add comment to tokenizer, explaining the spacy attr

2015-07-13 22:29:13 +02:00 · 2015-07-13 22:29:13 +02:00 · 24d6ce99ec
parent 8214b74eec
commit 24d6ce99ec
1 changed files with 0 additions and 3 deletions
--- a/spacy/tokenizer.pyx
+++ b/spacy/tokenizer.pyx
@ -185,9 +185,6 @@ cdef class Tokenizer:
        cdef const LexemeC* lexeme
        cdef UniStr span
        cdef int i
-        # Have to calculate is_spacy here, i.e. does the token have a trailing
-        # space. There are no spaces *between* the tokens we attach
-        # here, and there *is* a space after the last token.
        if prefixes.size():
            for i in range(prefixes.size()):
                tokens.push_back(prefixes[0][i], False)