From 941881fb5964fc750ca2f21dc05138d494806ea0 Mon Sep 17 00:00:00 2001 From: mehrad Date: Wed, 2 Mar 2022 14:57:25 -0800 Subject: [PATCH] test_e2e_dialogues: test e2e prediction too --- .../bitod/e2e_dialogue_preds.json | 63 +++++++++++++++++++ tests/test_e2e_dialogues.sh | 7 ++- 2 files changed, 69 insertions(+), 1 deletion(-) create mode 100644 tests/expected_results/bitod/e2e_dialogue_preds.json diff --git a/tests/expected_results/bitod/e2e_dialogue_preds.json b/tests/expected_results/bitod/e2e_dialogue_preds.json new file mode 100644 index 00000000..ce9a1a04 --- /dev/null +++ b/tests/expected_results/bitod/e2e_dialogue_preds.json @@ -0,0 +1,63 @@ +{ + "ec909032-d5fd-4dee-8dd2-53a895a98824": { + "turns": { + "1": { + "state": {}, + "api": "null", + "actions": " up Trend Trend Trend Trend Trend Trend Trend Trend Trend sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metroEarnEarnEarn", + "response": [ + " up Trend Trend Trend Trend Trend Trend Trend Trend Trend sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metroEarnEarnEarn" + ] + }, + "2": { + "state": {}, + "api": "null", + "actions": " up Trend Trend Trend Trend Trend Trend Trend Trend Trend sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metroEarnEarnEarn", + "response": [ + " up Trend Trend Trend Trend Trend Trend Trend Trend Trend sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metroEarnEarnEarn" + ] + }, + "3": { + "state": {}, + "api": "null", + "actions": " up Trend Trend Trend Trend Trend Trend Trend Trend Trend sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metroEarnEarnEarn", + "response": [ + " up Trend Trend Trend Trend Trend Trend Trend Trend Trend sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metroEarnEarnEarn" + ] + }, + "4": { + "state": {}, + "api": "null", + "actions": " up Trend Trend Trend Trend Trend Trend Trend Trend Trend sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metroEarnEarnEarn", + "response": [ + " up Trend Trend Trend Trend Trend Trend Trend Trend Trend sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metroEarnEarnEarn" + ] + }, + "5": { + "state": {}, + "api": "null", + "actions": " up Trend Trend Trend Trend Trend Trend Trend Trend Trend sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metroEarnEarnEarn", + "response": [ + " up Trend Trend Trend Trend Trend Trend Trend Trend Trend sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metroEarnEarnEarn" + ] + }, + "6": { + "state": {}, + "api": "null", + "actions": " up Trend Trend Trend Trend Trend Trend Trend Trend Trend sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metroEarnEarnEarn", + "response": [ + " up Trend Trend Trend Trend Trend Trend Trend Trend Trend sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metroEarnEarnEarn" + ] + }, + "7": { + "state": {}, + "api": "null", + "actions": " up Trend Trend Trend Trend Trend Trend Trend Trend Trend sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metroEarnEarnEarn", + "response": [ + " up Trend Trend Trend Trend Trend Trend Trend Trend Trend sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metroEarnEarnEarn" + ] + } + }, + "API": {} + } +} \ No newline at end of file diff --git a/tests/test_e2e_dialogues.sh b/tests/test_e2e_dialogues.sh index 7c88c017..666555f7 100755 --- a/tests/test_e2e_dialogues.sh +++ b/tests/test_e2e_dialogues.sh @@ -21,8 +21,12 @@ do # greedy prediction genienlp predict --tasks ${tasks[i]} --evaluate test --path $workdir/model_$i --overwrite --eval_dir $workdir/model_$i/eval_results/ --data $SRCDIR/dataset/bitod --embeddings $EMBEDDING_DIR --extra_metrics e2e_dialogue_score + # e2e prediction + genienlp predict --tasks ${tasks[i]} --evaluate test --path $workdir/model_$i --overwrite --eval_dir $workdir/model_$i/e2e_eval_results/ --data $SRCDIR/dataset/bitod --embeddings $EMBEDDING_DIR --extra_metrics e2e_dialogue_score --e2e_dialogue_evaluation + + # check if result file exists - if test ! -f $workdir/model_$i/eval_results/test/${tasks[i]}.tsv ; then + if ! [[ -f $workdir/model_$i/eval_results/test/${tasks[i]}.tsv || -f $workdir/model_$i/e2e_eval_results/test/e2e_dialogue_preds.json ]] ; then echo "File not found!" exit 1 fi @@ -39,6 +43,7 @@ do if [ $i == 0 ] ; then # check if predictions matches expected_results diff -u $SRCDIR/expected_results/bitod/bitod.tsv $workdir/model_$i/eval_results/test/bitod.tsv + diff -u $SRCDIR/expected_results/bitod/e2e_dialogue_preds.json $workdir/model_$i/eval_results/test/e2e_dialogue_preds.json fi rm -rf $workdir/model_$i $workdir/model_"$i"_exported