From 24d6ce99ecdf812c53119adaa98d90ddff31fbc3 Mon Sep 17 00:00:00 2001
From: Matthew Honnibal <honnibal@gmail.com>
Date: Mon, 13 Jul 2015 22:29:13 +0200
Subject: [PATCH] * Add comment to tokenizer, explaining the spacy attr

---
 spacy/tokenizer.pyx | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/spacy/tokenizer.pyx b/spacy/tokenizer.pyx
index cd9dd722f..df10c35df 100644
--- a/spacy/tokenizer.pyx
+++ b/spacy/tokenizer.pyx
@@ -185,9 +185,6 @@ cdef class Tokenizer:
         cdef const LexemeC* lexeme
         cdef UniStr span
         cdef int i
-        # Have to calculate is_spacy here, i.e. does the token have a trailing
-        # space. There are no spaces *between* the tokens we attach
-        # here, and there *is* a space after the last token.
         if prefixes.size():
             for i in range(prefixes.size()):
                 tokens.push_back(prefixes[0][i], False)