diff --git a/spacy/_ml.py b/spacy/_ml.py
index ce818c8e1..dde78af8a 100644
--- a/spacy/_ml.py
+++ b/spacy/_ml.py
@@ -251,7 +251,8 @@ def Tok2Vec(width, embed_size, pretrained_dims=0):
         tok2vec = (
             embed
             >> with_flatten(
-                Affine(width, width+pretrained_dims)
+                (Affine(width, width+pretrained_dims)
+                 if pretrained_dims else noop())
                 >> convolution ** 4,
                 pad=4)
         )