diff --git a/spacy/_ml.py b/spacy/_ml.py
index bbd3e2b3c..e882c954e 100644
--- a/spacy/_ml.py
+++ b/spacy/_ml.py
@@ -622,7 +622,7 @@ def build_text_classifier(nr_class, width=64, **cfg):
             >> with_flatten(
                 LN(Maxout(width, vectors_width))
                 >> Residual(
-                    (ExtractWindow(nW=1) >> zero_init(Maxout(width, width*3)))
+                    (ExtractWindow(nW=1) >> LN(Maxout(width, width*3)))
                 ) ** 2, pad=2
             )
             >> flatten_add_lengths