From 2527ba68e56c744ee898ae7aea6d92d11d332a18 Mon Sep 17 00:00:00 2001
From: Matthew Honnibal <honnibal+gh@gmail.com>
Date: Fri, 2 Nov 2018 23:51:37 +0100
Subject: [PATCH] Fix tensorizer

---
 spacy/_ml.py       |  6 +++++-
 spacy/pipeline.pyx | 25 ++++++++++++++-----------
 2 files changed, 19 insertions(+), 12 deletions(-)

diff --git a/spacy/_ml.py b/spacy/_ml.py
index 964b1fa7a..679b1aef6 100644
--- a/spacy/_ml.py
+++ b/spacy/_ml.py
@@ -532,7 +532,7 @@ def build_text_classifier(nr_class, width=64, **cfg):
             vectors = trained_vectors
             vectors_width = width
             static_vectors = None
-        cnn_model = (
+        tok2vec = (
             vectors
             >> with_flatten(
                 LN(Maxout(width, vectors_width))
@@ -540,6 +540,9 @@ def build_text_classifier(nr_class, width=64, **cfg):
                     (ExtractWindow(nW=1) >> LN(Maxout(width, width*3)))
                 ) ** depth, pad=depth
             )
+        )
+        cnn_model = (
+            tok2vec
             >> flatten_add_lengths
             >> ParametricAttention(width)
             >> Pooling(sum_pool)
@@ -556,6 +559,7 @@ def build_text_classifier(nr_class, width=64, **cfg):
             >> zero_init(Affine(nr_class, nr_class*2, drop_factor=0.0))
             >> logistic
         )
+        model.tok2vec = tok2vec
     model.nO = nr_class
     model.lsuv = False
     return model
diff --git a/spacy/pipeline.pyx b/spacy/pipeline.pyx
index 17d002c43..42a0f07e0 100644
--- a/spacy/pipeline.pyx
+++ b/spacy/pipeline.pyx
@@ -434,7 +434,7 @@ class Tensorizer(Pipe):
     name = 'tensorizer'
 
     @classmethod
-    def Model(cls, output_size=300, input_size=384, **cfg):
+    def Model(cls, output_size=300, input_size=128, **cfg):
         """Create a new statistical model for the class.
 
         width (int): Output size of the model.
@@ -442,11 +442,7 @@ class Tensorizer(Pipe):
         **cfg: Config parameters.
         RETURNS (Model): A `thinc.neural.Model` or similar instance.
         """
-        model = chain(
-                    SELU(output_size, input_size),
-                    SELU(output_size, output_size),
-                    zero_init(Affine(output_size, output_size)))
-        return model
+        return zero_init(Affine(output_size, input_size))
 
     def __init__(self, vocab, model=True, **cfg):
         """Construct a new statistical model. Weights are not allocated on
@@ -562,12 +558,11 @@ class Tensorizer(Pipe):
         gold_tuples (iterable): Gold-standard training data.
         pipeline (list): The pipeline the model is part of.
         """
-        for name, model in pipeline:
-            if getattr(model, 'tok2vec', None):
-                self.input_models.append(model.tok2vec)
+        if pipeline is not None:
+            for name, model in pipeline:
+                if getattr(model, 'tok2vec', None):
+                    self.input_models.append(model.tok2vec)
         if self.model is True:
-            self.cfg['input_size'] = 384
-            self.cfg['output_size'] = 300
             self.model = self.Model(**self.cfg)
         link_vectors_to_models(self.vocab)
         if sgd is None:
@@ -1061,6 +1056,14 @@ class TextCategorizer(Pipe):
     def Model(cls, nr_class, **cfg):
         return build_text_classifier(nr_class, **cfg)
 
+    @property
+    def tok2vec(self):
+        if self.model in (None, True, False):
+            return None
+        else:
+            return chain(self.model.tok2vec, flatten)
+
+
     def __init__(self, vocab, model=True, **cfg):
         self.vocab = vocab
         self.model = model