diff --git a/spacy/pipeline.pyx b/spacy/pipeline.pyx
index ad646f25f..81cf1ea69 100644
--- a/spacy/pipeline.pyx
+++ b/spacy/pipeline.pyx
@@ -711,6 +711,9 @@ class Tagger(Pipe):
                                           exc=vocab.morphology.exc)
         self.cfg['pretrained_vectors'] = kwargs.get('pretrained_vectors')
         if self.model is True:
+            for hp in ['token_vector_width', 'conv_depth']:
+                if hp in kwargs:
+                    self.cfg[hp] = kwargs[hp]
             self.model = self.Model(self.vocab.morphology.n_tags, **self.cfg)
         link_vectors_to_models(self.vocab)
         if sgd is None: