From d163115e91aaa6a0f73b05b05bcca9774d76bf7c Mon Sep 17 00:00:00 2001
From: Matthew Honnibal <honnibal+gh@gmail.com>
Date: Sat, 7 Oct 2017 21:00:43 -0500
Subject: [PATCH] Add non-linearity after history features

---
 spacy/_ml.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/spacy/_ml.py b/spacy/_ml.py
index 898d6ab49..23facb9fb 100644
--- a/spacy/_ml.py
+++ b/spacy/_ml.py
@@ -264,7 +264,8 @@ def HistoryFeatures(nr_class, hist_size=8, nr_dim=8):
         return layerize(noop())
     embed_tables = [Embed(nr_dim, nr_class, column=i, name='embed%d')
                     for i in range(hist_size)]
-    embed = concatenate(*embed_tables)
+    embed = chain(concatenate(*embed_tables),
+                  LN(Maxout(hist_size*nr_dim, hist_size*nr_dim)))
     ops = embed.ops
     def add_history_fwd(vectors_hists, drop=0.):
         vectors, hist_ids = vectors_hists