Add test to check the outputs of --is_hf_model

tests/test_e2e_dialogues.sh

@@ -53,7 +53,8 @@ do
     --e2e_dialogue_evaluation
 
   # check if result file exists
-  if ! [[ -f $workdir/model_$i/eval_results/test/${tasks[i]}.tsv || -f $workdir/model_$i/e2e_eval_results/test/e2e_dialogue_preds.json ]] ; then
+  if ! [[ -f $workdir/model_$i/eval_results/test/${tasks[i]}.tsv || \
+          -f $workdir/model_$i/e2e_eval_results/test/e2e_dialogue_preds.json ]] ; then
       echo "File not found!"
       exit 1
   fi

tests/test_paraphrasing.sh

@@ -20,6 +20,19 @@ for model in \
     --model TransformerSeq2Seq \
     --pretrained_model $model
 
+  # train for 0 iterations
+  genienlp train \
+  $SHARED_TRAIN_HPARAMS \
+  --train_tasks almond_natural_seq2seq \
+  --train_batch_tokens 100 \
+  --val_batch_size 100 \
+  --train_iterations 0 \
+  --save $workdir/model_$i \
+  --data $SRCDIR/dataset/ \
+  --model TransformerSeq2Seq \
+  --override_question "" \
+  --pretrained_model $model
+
   # greedy prediction
   genienlp predict \
     --tasks almond_paraphrase \
@@ -31,7 +44,7 @@ for model in \
     --embeddings $EMBEDDING_DIR \
     --extra_metrics rouge1 rougeL
 
-  # use a HuggingFace model directly in genienlp predict
+  # use as a HuggingFace model directly in genienlp predict
   genienlp predict \
     --tasks almond_paraphrase \
     --evaluate test \
@@ -48,11 +61,17 @@ for model in \
     --is_hf_model
 
   # check if result file exists
-  if test ! -f $workdir/model_$i/eval_results/test/almond_paraphrase.tsv || test ! -f $workdir/model_$i/eval_results/test/almond_paraphrase.results.json || test ! -f $workdir/model_$i/hf_results/test/almond_paraphrase.tsv ; then
+  if test ! -f $workdir/model_$i/eval_results/test/almond_paraphrase.tsv || \
+     test ! -f $workdir/model_$i/eval_results/test/almond_paraphrase.results.json || \
+     test ! -f $workdir/model_$i/hf_results/test/almond_paraphrase.tsv || \
+     test ! -f $workdir/model_$i/hf_results/test/almond_paraphrase.results.json; then
     echo "File not found!"
     exit 1
   fi
 
+  # check if eval_results matche hf_results
+  diff -u $workdir/model_$i/hf_results/test/almond_paraphrase.tsv $workdir/model_$i/eval_results/test/almond_paraphrase.tsv
+
   rm -rf $workdir/model_$i
   i=$((i+1))
 done