test_e2e_dialogues: test e2e prediction too
This commit is contained in:
mehrad
parent
cbedde6f63
commit
941881fb59
|
|
@ -0,0 +1,63 @@
|
|||
{
|
||||
"ec909032-d5fd-4dee-8dd2-53a895a98824": {
|
||||
"turns": {
|
||||
"1": {
|
||||
"state": {},
|
||||
"api": "null",
|
||||
"actions": " up Trend Trend Trend Trend Trend Trend Trend Trend Trend sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metroEarnEarnEarn",
|
||||
"response": [
|
||||
" up Trend Trend Trend Trend Trend Trend Trend Trend Trend sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metroEarnEarnEarn"
|
||||
]
|
||||
},
|
||||
"2": {
|
||||
"state": {},
|
||||
"api": "null",
|
||||
"actions": " up Trend Trend Trend Trend Trend Trend Trend Trend Trend sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metroEarnEarnEarn",
|
||||
"response": [
|
||||
" up Trend Trend Trend Trend Trend Trend Trend Trend Trend sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metroEarnEarnEarn"
|
||||
]
|
||||
},
|
||||
"3": {
|
||||
"state": {},
|
||||
"api": "null",
|
||||
"actions": " up Trend Trend Trend Trend Trend Trend Trend Trend Trend sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metroEarnEarnEarn",
|
||||
"response": [
|
||||
" up Trend Trend Trend Trend Trend Trend Trend Trend Trend sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metroEarnEarnEarn"
|
||||
]
|
||||
},
|
||||
"4": {
|
||||
"state": {},
|
||||
"api": "null",
|
||||
"actions": " up Trend Trend Trend Trend Trend Trend Trend Trend Trend sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metroEarnEarnEarn",
|
||||
"response": [
|
||||
" up Trend Trend Trend Trend Trend Trend Trend Trend Trend sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metroEarnEarnEarn"
|
||||
]
|
||||
},
|
||||
"5": {
|
||||
"state": {},
|
||||
"api": "null",
|
||||
"actions": " up Trend Trend Trend Trend Trend Trend Trend Trend Trend sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metroEarnEarnEarn",
|
||||
"response": [
|
||||
" up Trend Trend Trend Trend Trend Trend Trend Trend Trend sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metroEarnEarnEarn"
|
||||
]
|
||||
},
|
||||
"6": {
|
||||
"state": {},
|
||||
"api": "null",
|
||||
"actions": " up Trend Trend Trend Trend Trend Trend Trend Trend Trend sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metroEarnEarnEarn",
|
||||
"response": [
|
||||
" up Trend Trend Trend Trend Trend Trend Trend Trend Trend sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metroEarnEarnEarn"
|
||||
]
|
||||
},
|
||||
"7": {
|
||||
"state": {},
|
||||
"api": "null",
|
||||
"actions": " up Trend Trend Trend Trend Trend Trend Trend Trend Trend sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metroEarnEarnEarn",
|
||||
"response": [
|
||||
" up Trend Trend Trend Trend Trend Trend Trend Trend Trend sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship sponsorship metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metro metroEarnEarnEarn"
|
||||
]
|
||||
}
|
||||
},
|
||||
"API": {}
|
||||
}
|
||||
}
|
||||
|
|
@ -21,8 +21,12 @@ do
|
|||
# greedy prediction
|
||||
genienlp predict --tasks ${tasks[i]} --evaluate test --path $workdir/model_$i --overwrite --eval_dir $workdir/model_$i/eval_results/ --data $SRCDIR/dataset/bitod --embeddings $EMBEDDING_DIR --extra_metrics e2e_dialogue_score
|
||||
|
||||
# e2e prediction
|
||||
genienlp predict --tasks ${tasks[i]} --evaluate test --path $workdir/model_$i --overwrite --eval_dir $workdir/model_$i/e2e_eval_results/ --data $SRCDIR/dataset/bitod --embeddings $EMBEDDING_DIR --extra_metrics e2e_dialogue_score --e2e_dialogue_evaluation
|
||||
|
||||
|
||||
# check if result file exists
|
||||
if test ! -f $workdir/model_$i/eval_results/test/${tasks[i]}.tsv ; then
|
||||
if ! [[ -f $workdir/model_$i/eval_results/test/${tasks[i]}.tsv || -f $workdir/model_$i/e2e_eval_results/test/e2e_dialogue_preds.json ]] ; then
|
||||
echo "File not found!"
|
||||
exit 1
|
||||
fi
|
||||
|
|
@ -39,6 +43,7 @@ do
|
|||
if [ $i == 0 ] ; then
|
||||
# check if predictions matches expected_results
|
||||
diff -u $SRCDIR/expected_results/bitod/bitod.tsv $workdir/model_$i/eval_results/test/bitod.tsv
|
||||
diff -u $SRCDIR/expected_results/bitod/e2e_dialogue_preds.json $workdir/model_$i/eval_results/test/e2e_dialogue_preds.json
|
||||
fi
|
||||
|
||||
rm -rf $workdir/model_$i $workdir/model_"$i"_exported
|
||||
|
|
|
|||
Loading…
Reference in New Issue