Add test for bitod_error_cls task
This commit is contained in:
mehrad
parent
c1c38298ec
commit
996b406184
|
|
@ -0,0 +1,58 @@
|
|||
{
|
||||
"data": [
|
||||
{
|
||||
"dial_id": "d4da198a-a529-4d26-8554-9672333f6d5f",
|
||||
"task": "restaurants search",
|
||||
"turn_id": 2,
|
||||
"input_text": "DST: <state> ( restaurants search ) dietary_restrictions equal_to \" Vegan Options \" <endofstate> <history> AGENT_ACTS: ( restaurants search ) request rating USER: I do not care about the rating. I don't want a place with Steakhouse food, but anything else is good. <endofhistory>",
|
||||
"output_text": "( restaurants search ) cuisine not \" Steakhouse \" , rating equal_to \" don't care \"",
|
||||
"train_target": "dst",
|
||||
"category": "positive"
|
||||
},
|
||||
{
|
||||
"dial_id": "d4da198a-a529-4d26-8554-9672333f6d5f",
|
||||
"task": "restaurants search",
|
||||
"turn_id": 2,
|
||||
"input_text": "DST: <state> ( restaurants search ) dietary_restrictions equal_to \" Vegan Options \" <endofstate> <history> AGENT_ACTS: ( restaurants search ) request rating USER: I do not care about the rating. I don't want a place with Steakhouse food, but anything else is good. <endofhistory>",
|
||||
"output_text": "( restaurants search ) rating equal_to \" don't care \"",
|
||||
"train_target": "dst",
|
||||
"category": "remove_slot"
|
||||
},
|
||||
{
|
||||
"dial_id": "d4da198a-a529-4d26-8554-9672333f6d5f",
|
||||
"task": "restaurants search",
|
||||
"turn_id": 4,
|
||||
"input_text": "DST: <state> ( restaurants search ) cuisine not \" Steakhouse \" , dietary_restrictions equal_to \" Vegan Options \" , location equal_to \" Victoria Park \" , rating equal_to \" don't care \" <endofstate> <history> AGENT_ACTS_PREV: ( restaurants search ) offer available_options equal_to \" 292 \" , offer cuisine equal_to \" Italian \" , offer name equal_to \" ChocoDuck Bistro \" , offer rating equal_to \" 10 \" AGENT_ACTS: ( restaurants search ) offer available_options equal_to \" 1 \" , offer cuisine equal_to \" Diner \" , offer name equal_to \" Little Bao Diner \" , offer rating equal_to \" 9 \" USER: Can I have the address and phone number of the restaurant? <endofhistory>",
|
||||
"output_text": "( restaurants search )",
|
||||
"train_target": "dst",
|
||||
"category": "positive"
|
||||
},
|
||||
{
|
||||
"dial_id": "d4da198a-a529-4d26-8554-9672333f6d5f",
|
||||
"task": "restaurants search",
|
||||
"turn_id": 4,
|
||||
"input_text": "DST: <state> ( restaurants search ) cuisine not \" Steakhouse \" , dietary_restrictions equal_to \" Vegan Options \" , location equal_to \" Victoria Park \" , rating equal_to \" don't care \" <endofstate> <history> AGENT_ACTS_PREV: ( restaurants search ) offer available_options equal_to \" 292 \" , offer cuisine equal_to \" Italian \" , offer name equal_to \" ChocoDuck Bistro \" , offer rating equal_to \" 10 \" AGENT_ACTS: ( restaurants search ) offer available_options equal_to \" 1 \" , offer cuisine equal_to \" Diner \" , offer name equal_to \" Little Bao Diner \" , offer rating equal_to \" 9 \" USER: Can I have the address and phone number of the restaurant? <endofhistory>",
|
||||
"output_text": "( restaurants search ) cuisine equal_to \" Steakhouse \"",
|
||||
"train_target": "dst",
|
||||
"category": "change_relation"
|
||||
},
|
||||
{
|
||||
"dial_id": "d4da198a-a529-4d26-8554-9672333f6d5f",
|
||||
"task": "restaurants booking",
|
||||
"turn_id": 6,
|
||||
"input_text": "DST: <state> ( restaurants booking ) date #unknown , name equal_to \" Little Bao Diner \" , number_of_people #unknown , time #unknown , user_name #unknown , ( restaurants search ) cuisine not \" Steakhouse \" , dietary_restrictions equal_to \" Vegan Options \" , location equal_to \" Victoria Park \" , rating equal_to \" don't care \" <endofstate> <history> AGENT_ACTS_PREV: ( restaurants search ) inform address equal_to \" 9 Kingston Street, Causeway Bay Shop H1, Fashion Walk \" , inform phone_number equal_to \" +852 2555 0600 \" AGENT_ACTS: ( restaurants booking ) request time , request user_name USER: Reserve it under Paul, for 4:50 pm on April 2. <endofhistory>",
|
||||
"output_text": "( restaurants booking ) date equal_to \" April 2 \" , time equal_to \" 4:50 pm \" , user_name equal_to \" Paul \"",
|
||||
"train_target": "dst",
|
||||
"category": "positive"
|
||||
},
|
||||
{
|
||||
"dial_id": "d4da198a-a529-4d26-8554-9672333f6d5f",
|
||||
"task": "restaurants booking",
|
||||
"turn_id": 6,
|
||||
"input_text": "DST: <state> ( restaurants booking ) date #unknown , name equal_to \" Little Bao Diner \" , number_of_people #unknown , time #unknown , user_name #unknown , ( restaurants search ) cuisine not \" Steakhouse \" , dietary_restrictions equal_to \" Vegan Options \" , location equal_to \" Victoria Park \" , rating equal_to \" don't care \" <endofstate> <history> AGENT_ACTS_PREV: ( restaurants search ) inform address equal_to \" 9 Kingston Street, Causeway Bay Shop H1, Fashion Walk \" , inform phone_number equal_to \" +852 2555 0600 \" AGENT_ACTS: ( restaurants booking ) request time , request user_name USER: Reserve it under Paul, for 4:50 pm on April 2. <endofhistory>",
|
||||
"output_text": "( restaurants booking ) date equal_to \" April 2 \" , time equal_to \" 4:50 pm \" , user_name equal_to \" Paul \" , number_of_people equal_to \" 12 \"",
|
||||
"train_target": "dst",
|
||||
"category": "add_slot"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -0,0 +1,58 @@
|
|||
{
|
||||
"data": [
|
||||
{
|
||||
"dial_id": "d4da198a-a529-4d26-8554-9672333f6d5f",
|
||||
"task": "restaurants search",
|
||||
"turn_id": 2,
|
||||
"input_text": "DST: <state> ( restaurants search ) dietary_restrictions equal_to \" Vegan Options \" <endofstate> <history> AGENT_ACTS: ( restaurants search ) request rating USER: I do not care about the rating. I don't want a place with Steakhouse food, but anything else is good. <endofhistory>",
|
||||
"output_text": "( restaurants search ) cuisine not \" Steakhouse \" , rating equal_to \" don't care \"",
|
||||
"train_target": "dst",
|
||||
"category": "positive"
|
||||
},
|
||||
{
|
||||
"dial_id": "d4da198a-a529-4d26-8554-9672333f6d5f",
|
||||
"task": "restaurants search",
|
||||
"turn_id": 2,
|
||||
"input_text": "DST: <state> ( restaurants search ) dietary_restrictions equal_to \" Vegan Options \" <endofstate> <history> AGENT_ACTS: ( restaurants search ) request rating USER: I do not care about the rating. I don't want a place with Steakhouse food, but anything else is good. <endofhistory>",
|
||||
"output_text": "( restaurants search ) rating equal_to \" don't care \"",
|
||||
"train_target": "dst",
|
||||
"category": "remove_slot"
|
||||
},
|
||||
{
|
||||
"dial_id": "d4da198a-a529-4d26-8554-9672333f6d5f",
|
||||
"task": "restaurants search",
|
||||
"turn_id": 4,
|
||||
"input_text": "DST: <state> ( restaurants search ) cuisine not \" Steakhouse \" , dietary_restrictions equal_to \" Vegan Options \" , location equal_to \" Victoria Park \" , rating equal_to \" don't care \" <endofstate> <history> AGENT_ACTS_PREV: ( restaurants search ) offer available_options equal_to \" 292 \" , offer cuisine equal_to \" Italian \" , offer name equal_to \" ChocoDuck Bistro \" , offer rating equal_to \" 10 \" AGENT_ACTS: ( restaurants search ) offer available_options equal_to \" 1 \" , offer cuisine equal_to \" Diner \" , offer name equal_to \" Little Bao Diner \" , offer rating equal_to \" 9 \" USER: Can I have the address and phone number of the restaurant? <endofhistory>",
|
||||
"output_text": "( restaurants search )",
|
||||
"train_target": "dst",
|
||||
"category": "positive"
|
||||
},
|
||||
{
|
||||
"dial_id": "d4da198a-a529-4d26-8554-9672333f6d5f",
|
||||
"task": "restaurants search",
|
||||
"turn_id": 4,
|
||||
"input_text": "DST: <state> ( restaurants search ) cuisine not \" Steakhouse \" , dietary_restrictions equal_to \" Vegan Options \" , location equal_to \" Victoria Park \" , rating equal_to \" don't care \" <endofstate> <history> AGENT_ACTS_PREV: ( restaurants search ) offer available_options equal_to \" 292 \" , offer cuisine equal_to \" Italian \" , offer name equal_to \" ChocoDuck Bistro \" , offer rating equal_to \" 10 \" AGENT_ACTS: ( restaurants search ) offer available_options equal_to \" 1 \" , offer cuisine equal_to \" Diner \" , offer name equal_to \" Little Bao Diner \" , offer rating equal_to \" 9 \" USER: Can I have the address and phone number of the restaurant? <endofhistory>",
|
||||
"output_text": "( restaurants search ) cuisine equal_to \" Steakhouse \"",
|
||||
"train_target": "dst",
|
||||
"category": "change_relation"
|
||||
},
|
||||
{
|
||||
"dial_id": "d4da198a-a529-4d26-8554-9672333f6d5f",
|
||||
"task": "restaurants booking",
|
||||
"turn_id": 6,
|
||||
"input_text": "DST: <state> ( restaurants booking ) date #unknown , name equal_to \" Little Bao Diner \" , number_of_people #unknown , time #unknown , user_name #unknown , ( restaurants search ) cuisine not \" Steakhouse \" , dietary_restrictions equal_to \" Vegan Options \" , location equal_to \" Victoria Park \" , rating equal_to \" don't care \" <endofstate> <history> AGENT_ACTS_PREV: ( restaurants search ) inform address equal_to \" 9 Kingston Street, Causeway Bay Shop H1, Fashion Walk \" , inform phone_number equal_to \" +852 2555 0600 \" AGENT_ACTS: ( restaurants booking ) request time , request user_name USER: Reserve it under Paul, for 4:50 pm on April 2. <endofhistory>",
|
||||
"output_text": "( restaurants booking ) date equal_to \" April 2 \" , time equal_to \" 4:50 pm \" , user_name equal_to \" Paul \"",
|
||||
"train_target": "dst",
|
||||
"category": "positive"
|
||||
},
|
||||
{
|
||||
"dial_id": "d4da198a-a529-4d26-8554-9672333f6d5f",
|
||||
"task": "restaurants booking",
|
||||
"turn_id": 6,
|
||||
"input_text": "DST: <state> ( restaurants booking ) date #unknown , name equal_to \" Little Bao Diner \" , number_of_people #unknown , time #unknown , user_name #unknown , ( restaurants search ) cuisine not \" Steakhouse \" , dietary_restrictions equal_to \" Vegan Options \" , location equal_to \" Victoria Park \" , rating equal_to \" don't care \" <endofstate> <history> AGENT_ACTS_PREV: ( restaurants search ) inform address equal_to \" 9 Kingston Street, Causeway Bay Shop H1, Fashion Walk \" , inform phone_number equal_to \" +852 2555 0600 \" AGENT_ACTS: ( restaurants booking ) request time , request user_name USER: Reserve it under Paul, for 4:50 pm on April 2. <endofhistory>",
|
||||
"output_text": "( restaurants booking ) date equal_to \" April 2 \" , time equal_to \" 4:50 pm \" , user_name equal_to \" Paul \" , number_of_people equal_to \" 12 \"",
|
||||
"train_target": "dst",
|
||||
"category": "add_slot"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
bitod_error_cls/d4da198a-a529-4d26-8554-9672333f6d5f/2/dst positive positive DST: <state> ( restaurants search ) dietary _ restrictions equal _ to " " <endofstate> <history> AGENT_ACTS: ( restaurants search ) request rating USER: don't care about the rating. don't want a place with food, but anything else is good. <endofhistory> ( restaurants search ) cuisine not " ", rating equal _ to " don't care "
|
||||
bitod_error_cls/d4da198a-a529-4d26-8554-9672333f6d5f/2/dst negative negative DST: <state> ( restaurants search ) dietary _ restrictions equal _ to " " <endofstate> <history> AGENT_ACTS: ( restaurants search ) request rating USER: don't care about the rating. don't want a place with food, but anything else is good. <endofhistory> ( restaurants search ) rating equal _ to " don't care "
|
||||
bitod_error_cls/d4da198a-a529-4d26-8554-9672333f6d5f/4/dst positive positive DST: <state> ( restaurants search ) cuisine not " ", dietary _ restrictions equal _ to " ", location equal _ to " ", rating equal _ to " don't care " <endofstate> <history> AGENT_ACTS_PREV: ( restaurants search ) offer available _ options equal _ to " 292 ", offer cuisine equal _ to " ", offer name equal _ to " ", offer rating equal _ to " 10 " AGENT_ACTS: ( restaurants search ) offer available _ options equal _ to " 1 ", offer cuisine equal _ to " ", offer name equal _ to " ", offer rating equal _ to " 9 " USER: have the address and phone number of the restaurant? <endofhistory> ( restaurants search )
|
||||
bitod_error_cls/d4da198a-a529-4d26-8554-9672333f6d5f/4/dst negative negative DST: <state> ( restaurants search ) cuisine not " ", dietary _ restrictions equal _ to " ", location equal _ to " ", rating equal _ to " don't care " <endofstate> <history> AGENT_ACTS_PREV: ( restaurants search ) offer available _ options equal _ to " 292 ", offer cuisine equal _ to " ", offer name equal _ to " ", offer rating equal _ to " 10 " AGENT_ACTS: ( restaurants search ) offer available _ options equal _ to " 1 ", offer cuisine equal _ to " ", offer name equal _ to " ", offer rating equal _ to " 9 " USER: have the address and phone number of the restaurant? <endofhistory> ( restaurants search ) cuisine equal _ to " "
|
||||
bitod_error_cls/d4da198a-a529-4d26-8554-9672333f6d5f/6/dst positive positive DST: <state> ( restaurants booking ) date #unknown, name equal _ to " ", number _ of _ people #unknown, time #unknown, user _ name #unknown, ( restaurants search ) cuisine not " ", dietary _ restrictions equal _ to " ", location equal _ to " ", rating equal _ to " don't care " <endofstate> <history> AGENT_ACTS_PREV: ( restaurants search ) inform address equal _ to " 9,, ", inform phone _ number equal _ to " + 852 2555 0600 " AGENT_ACTS: ( restaurants booking ) request time, request user _ name USER: it under, for 4 : 50 pm on 2. <endofhistory> ( restaurants booking ) date equal _ to " 2 ", time equal _ to " 4 : 50 pm ", user _ name equal _ to " "
|
||||
bitod_error_cls/d4da198a-a529-4d26-8554-9672333f6d5f/6/dst negative negative DST: <state> ( restaurants booking ) date #unknown, name equal _ to " ", number _ of _ people #unknown, time #unknown, user _ name #unknown, ( restaurants search ) cuisine not " ", dietary _ restrictions equal _ to " ", location equal _ to " ", rating equal _ to " don't care " <endofstate> <history> AGENT_ACTS_PREV: ( restaurants search ) inform address equal _ to " 9,, ", inform phone _ number equal _ to " + 852 2555 0600 " AGENT_ACTS: ( restaurants booking ) request time, request user _ name USER: it under, for 4 : 50 pm on 2. <endofhistory> ( restaurants booking ) date equal _ to " 2 ", time equal _ to " 4 : 50 pm ", user _ name equal _ to " ", number _ of _ people equal _ to " 12 "
|
||||
|
Can't render this file because it contains an unexpected character in line 1 and column 148.
|
|
|
@ -2,6 +2,7 @@
|
|||
|
||||
. ./tests/lib.sh
|
||||
|
||||
# Test ood task
|
||||
# train
|
||||
genienlp train --train_tasks ood_task --model TransformerForSequenceClassification --pretrained_model distilbert-base-uncased --min_output_length 1 --save $workdir/model --train_iterations 20 --save_every 10 --log_every 10 --val_every 10 --data $SRCDIR/dataset/ood/ --force_fast_tokenizer --train_batch_tokens 200 --num_print 0
|
||||
|
||||
|
|
@ -18,3 +19,22 @@ fi
|
|||
diff -u $SRCDIR/expected_results/sequence_classification/ood_task.tsv $workdir/model/eval_results/valid/ood_task.tsv
|
||||
|
||||
rm -rf $workdir/model
|
||||
|
||||
|
||||
# Test bitod_error_cls task
|
||||
# train
|
||||
genienlp train --train_tasks bitod_error_cls --model TransformerForSequenceClassification --pretrained_model distilbert-base-uncased --min_output_length 1 --save $workdir/model_error/ --train_iterations 100 --save_every 50 --log_every 50 --val_every 50 --data $SRCDIR/dataset/bitod_error/ --force_fast_tokenizer --train_batch_tokens 200 --num_print 0
|
||||
|
||||
# greedy prediction
|
||||
genienlp predict --tasks bitod_error_cls --evaluate valid --pred_set_name valid --path $workdir/model_error --overwrite --eval_dir $workdir/model_error/eval_results/ --data $SRCDIR/dataset/bitod_error/ --embeddings $EMBEDDING_DIR --val_batch_size 200
|
||||
|
||||
# check if result file exists
|
||||
if test ! -f $workdir/model_error/eval_results/valid/bitod_error_cls.tsv ; then
|
||||
echo "File not found!"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# check if predictions matches expected_results
|
||||
diff -u $SRCDIR/expected_results/sequence_classification/bitod_error_cls.tsv $workdir/model_error/eval_results/valid/bitod_error_cls.tsv
|
||||
|
||||
rm -rf $workdir/model_error
|
||||
|
|
|
|||
Loading…
Reference in New Issue