diff --git a/spacy/cli/train.py b/spacy/cli/train.py
index af028dae5..9ed621c12 100644
--- a/spacy/cli/train.py
+++ b/spacy/cli/train.py
@@ -91,7 +91,8 @@ def train(cmd, lang, output_dir, train_data, dev_data, n_iter=20, n_sents=0,
                 for batch in minibatch(train_docs, size=batch_sizes):
                     docs, golds = zip(*batch)
                     nlp.update(docs, golds, sgd=optimizer,
-                               drop=next(dropout_rates), losses=losses)
+                               drop=next(dropout_rates), losses=losses,
+                               update_tensors=True)
                     pbar.update(sum(len(doc) for doc in docs))
 
             with nlp.use_params(optimizer.averages):