diff --git a/spacy/language.py b/spacy/language.py
index 4a489387a..3079249f6 100644
--- a/spacy/language.py
+++ b/spacy/language.py
@@ -382,9 +382,13 @@ class Language(object):
         return optimizer
 
     def evaluate(self, docs_golds):
-        docs, golds = zip(*docs_golds)
         scorer = Scorer()
-        for doc, gold in zip(self.pipe(docs, batch_size=32), golds):
+        docs, golds = zip(*docs_golds)
+        docs = list(docs)
+        golds = list(golds)
+        for pipe in self.pipeline:
+            docs = pipe.pipe(docs)
+        for doc, gold in zip(docs, golds):
             scorer.score(doc, gold)
             doc.tensor = None
         return scorer