From bc82aeb4b6d6ede1dd0813366a23aa701993039d Mon Sep 17 00:00:00 2001
From: Sina <s.j.semnani@gmail.com>
Date: Thu, 25 Aug 2022 20:29:37 -0700
Subject: [PATCH] Add test to check the outputs of --is_hf_model

---
 tests/test_e2e_dialogues.sh |  3 ++-
 tests/test_paraphrasing.sh  | 23 +++++++++++++++++++++--
 2 files changed, 23 insertions(+), 3 deletions(-)

diff --git a/tests/test_e2e_dialogues.sh b/tests/test_e2e_dialogues.sh
index 4f60f05a..61b43b83 100755
--- a/tests/test_e2e_dialogues.sh
+++ b/tests/test_e2e_dialogues.sh
@@ -53,7 +53,8 @@ do
     --e2e_dialogue_evaluation
 
   # check if result file exists
-  if ! [[ -f $workdir/model_$i/eval_results/test/${tasks[i]}.tsv || -f $workdir/model_$i/e2e_eval_results/test/e2e_dialogue_preds.json ]] ; then
+  if ! [[ -f $workdir/model_$i/eval_results/test/${tasks[i]}.tsv || \
+          -f $workdir/model_$i/e2e_eval_results/test/e2e_dialogue_preds.json ]] ; then
       echo "File not found!"
       exit 1
   fi
diff --git a/tests/test_paraphrasing.sh b/tests/test_paraphrasing.sh
index bfa24ea6..760f165d 100755
--- a/tests/test_paraphrasing.sh
+++ b/tests/test_paraphrasing.sh
@@ -20,6 +20,19 @@ for model in \
     --model TransformerSeq2Seq \
     --pretrained_model $model
 
+  # train for 0 iterations
+  genienlp train \
+  $SHARED_TRAIN_HPARAMS \
+  --train_tasks almond_natural_seq2seq \
+  --train_batch_tokens 100 \
+  --val_batch_size 100 \
+  --train_iterations 0 \
+  --save $workdir/model_$i \
+  --data $SRCDIR/dataset/ \
+  --model TransformerSeq2Seq \
+  --override_question "" \
+  --pretrained_model $model
+
   # greedy prediction
   genienlp predict \
     --tasks almond_paraphrase \
@@ -31,7 +44,7 @@ for model in \
     --embeddings $EMBEDDING_DIR \
     --extra_metrics rouge1 rougeL
 
-  # use a HuggingFace model directly in genienlp predict
+  # use as a HuggingFace model directly in genienlp predict
   genienlp predict \
     --tasks almond_paraphrase \
     --evaluate test \
@@ -48,11 +61,17 @@ for model in \
     --is_hf_model
 
   # check if result file exists
-  if test ! -f $workdir/model_$i/eval_results/test/almond_paraphrase.tsv || test ! -f $workdir/model_$i/eval_results/test/almond_paraphrase.results.json || test ! -f $workdir/model_$i/hf_results/test/almond_paraphrase.tsv ; then
+  if test ! -f $workdir/model_$i/eval_results/test/almond_paraphrase.tsv || \
+     test ! -f $workdir/model_$i/eval_results/test/almond_paraphrase.results.json || \
+     test ! -f $workdir/model_$i/hf_results/test/almond_paraphrase.tsv || \
+     test ! -f $workdir/model_$i/hf_results/test/almond_paraphrase.results.json; then
     echo "File not found!"
     exit 1
   fi
 
+  # check if eval_results matche hf_results
+  diff -u $workdir/model_$i/hf_results/test/almond_paraphrase.tsv $workdir/model_$i/eval_results/test/almond_paraphrase.tsv
+
   rm -rf $workdir/model_$i
   i=$((i+1))
 done