From 09cebc62a352ba04791571300f3ec9ae968f0192 Mon Sep 17 00:00:00 2001
From: Bryan Marcus McCann <bryan.mccann.is@gmail.com>
Date: Tue, 4 Sep 2018 15:43:12 +0000
Subject: [PATCH] enumerating predictions; exp with intermediate cove

---
 arguments.py                                   | 1 +
 dockerfiles/Dockerfile                         | 3 ++-
 models/multitask_question_answering_network.py | 7 ++++---
 predict.py                                     | 4 ++--
 4 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/arguments.py b/arguments.py
index fc70fd47..d24c785a 100644
--- a/arguments.py
+++ b/arguments.py
@@ -63,6 +63,7 @@ def parse():
     parser.add_argument('--dropout_ratio', default=0.2, type=float, help='dropout for the model')
     parser.add_argument('--no_transformer_lr', action='store_false', dest='transformer_lr', help='turns off the transformer learning rate strategy') 
     parser.add_argument('--cove', action='store_true', help='whether to use contextualized word vectors (McCann et al. 2017)')
+    parser.add_argument('--intermediate_cove', action='store_true', help='whether to use the intermediate layers of contextualized word vectors (McCann et al. 2017)')
 
     parser.add_argument('--warmup', default=800, type=int, help='warmup for learning rate')
     parser.add_argument('--grad_clip', default=1.0, type=float, help='gradient clipping')
diff --git a/dockerfiles/Dockerfile b/dockerfiles/Dockerfile
index 12731daf..05a5353f 100644
--- a/dockerfiles/Dockerfile
+++ b/dockerfiles/Dockerfile
@@ -61,9 +61,10 @@ RUN apt-get install --yes \
   python-lxml
 
 # WikISQL evaluation
-RUN pip install -e git+git://github.com/salesforce/cove.git#egg=cove
 RUN pip install records
 RUN pip install babel
 RUN pip install tabulate
 
+RUN pip install -e git+git://github.com/salesforce/cove.git#egg=cove
+
 CMD bash 
diff --git a/models/multitask_question_answering_network.py b/models/multitask_question_answering_network.py
index b7c74721..f0ee3375 100644
--- a/models/multitask_question_answering_network.py
+++ b/models/multitask_question_answering_network.py
@@ -25,9 +25,10 @@ class MultitaskQuestionAnsweringNetwork(nn.Module):
         self.decoder_embeddings = Embedding(field, args.dimension, 
             dropout=args.dropout_ratio, project=True)
 
-        if self.args.cove:
-            self.cove = MTLSTM(model_cache=args.embeddings)
-            self.project_cove = Feedforward(1000, args.dimension)
+        if self.args.cove or self.args.intermediate_cove:
+            self.cove = MTLSTM(model_cache=args.embeddings, layer0=args.intermediate_cove, layer1=args.cove)
+            cove_dim = int(args.intermediate_cove) * 600 + int(args.cove) * 600 + 400 # the last 400 is for GloVe and char n-gram embeddings
+            self.project_cove = Feedforward(cove_dim, args.dimension)
      
         self.bilstm_before_coattention = PackedLSTM(args.dimension,  args.dimension,
             batch_first=True, dropout=args.dropout_ratio, bidirectional=True, num_layers=1)
diff --git a/predict.py b/predict.py
index ca608a02..07b1b50f 100644
--- a/predict.py
+++ b/predict.py
@@ -157,8 +157,8 @@ def run(args, field, val_sets, model):
 
             print(metrics)
             if not args.silent:
-                for p, a in zip(predictions, answers):
-                    print(f'Prediction: {p}\nAnswer: {a}\n')
+                for i, (p, a) in enumerate(zip(predictions, answers)):
+                    print(f'Prediction {i+1}: {p}\nAnswer {i+1}: {a}\n')
 
 
 def get_args():