diff --git a/spacy/ml/models/textcat.py b/spacy/ml/models/textcat.py
index 139917581..53200c165 100644
--- a/spacy/ml/models/textcat.py
+++ b/spacy/ml/models/textcat.py
@@ -158,7 +158,6 @@ def build_text_classifier_lowdata(width, pretrained_vectors, dropout, nO=None):
         model = (
             StaticVectors(width)
             >> list2ragged()
-            >> with_ragged(0, Linear(width, vector_dim))
             >> ParametricAttention(width)
             >> reduce_sum()
             >> residual(Relu(width, width)) ** 2