diff --git a/tests/dataset/bitod_error/train.json b/tests/dataset/bitod_error/train.json new file mode 100644 index 00000000..3c62792f --- /dev/null +++ b/tests/dataset/bitod_error/train.json @@ -0,0 +1,58 @@ +{ + "data": [ + { + "dial_id": "d4da198a-a529-4d26-8554-9672333f6d5f", + "task": "restaurants search", + "turn_id": 2, + "input_text": "DST: ( restaurants search ) dietary_restrictions equal_to \" Vegan Options \" AGENT_ACTS: ( restaurants search ) request rating USER: I do not care about the rating. I don't want a place with Steakhouse food, but anything else is good. ", + "output_text": "( restaurants search ) cuisine not \" Steakhouse \" , rating equal_to \" don't care \"", + "train_target": "dst", + "category": "positive" + }, + { + "dial_id": "d4da198a-a529-4d26-8554-9672333f6d5f", + "task": "restaurants search", + "turn_id": 2, + "input_text": "DST: ( restaurants search ) dietary_restrictions equal_to \" Vegan Options \" AGENT_ACTS: ( restaurants search ) request rating USER: I do not care about the rating. I don't want a place with Steakhouse food, but anything else is good. ", + "output_text": "( restaurants search ) rating equal_to \" don't care \"", + "train_target": "dst", + "category": "remove_slot" + }, + { + "dial_id": "d4da198a-a529-4d26-8554-9672333f6d5f", + "task": "restaurants search", + "turn_id": 4, + "input_text": "DST: ( restaurants search ) cuisine not \" Steakhouse \" , dietary_restrictions equal_to \" Vegan Options \" , location equal_to \" Victoria Park \" , rating equal_to \" don't care \" AGENT_ACTS_PREV: ( restaurants search ) offer available_options equal_to \" 292 \" , offer cuisine equal_to \" Italian \" , offer name equal_to \" ChocoDuck Bistro \" , offer rating equal_to \" 10 \" AGENT_ACTS: ( restaurants search ) offer available_options equal_to \" 1 \" , offer cuisine equal_to \" Diner \" , offer name equal_to \" Little Bao Diner \" , offer rating equal_to \" 9 \" USER: Can I have the address and phone number of the restaurant? ", + "output_text": "( restaurants search )", + "train_target": "dst", + "category": "positive" + }, + { + "dial_id": "d4da198a-a529-4d26-8554-9672333f6d5f", + "task": "restaurants search", + "turn_id": 4, + "input_text": "DST: ( restaurants search ) cuisine not \" Steakhouse \" , dietary_restrictions equal_to \" Vegan Options \" , location equal_to \" Victoria Park \" , rating equal_to \" don't care \" AGENT_ACTS_PREV: ( restaurants search ) offer available_options equal_to \" 292 \" , offer cuisine equal_to \" Italian \" , offer name equal_to \" ChocoDuck Bistro \" , offer rating equal_to \" 10 \" AGENT_ACTS: ( restaurants search ) offer available_options equal_to \" 1 \" , offer cuisine equal_to \" Diner \" , offer name equal_to \" Little Bao Diner \" , offer rating equal_to \" 9 \" USER: Can I have the address and phone number of the restaurant? ", + "output_text": "( restaurants search ) cuisine equal_to \" Steakhouse \"", + "train_target": "dst", + "category": "change_relation" + }, + { + "dial_id": "d4da198a-a529-4d26-8554-9672333f6d5f", + "task": "restaurants booking", + "turn_id": 6, + "input_text": "DST: ( restaurants booking ) date #unknown , name equal_to \" Little Bao Diner \" , number_of_people #unknown , time #unknown , user_name #unknown , ( restaurants search ) cuisine not \" Steakhouse \" , dietary_restrictions equal_to \" Vegan Options \" , location equal_to \" Victoria Park \" , rating equal_to \" don't care \" AGENT_ACTS_PREV: ( restaurants search ) inform address equal_to \" 9 Kingston Street, Causeway Bay Shop H1, Fashion Walk \" , inform phone_number equal_to \" +852 2555 0600 \" AGENT_ACTS: ( restaurants booking ) request time , request user_name USER: Reserve it under Paul, for 4:50 pm on April 2. ", + "output_text": "( restaurants booking ) date equal_to \" April 2 \" , time equal_to \" 4:50 pm \" , user_name equal_to \" Paul \"", + "train_target": "dst", + "category": "positive" + }, + { + "dial_id": "d4da198a-a529-4d26-8554-9672333f6d5f", + "task": "restaurants booking", + "turn_id": 6, + "input_text": "DST: ( restaurants booking ) date #unknown , name equal_to \" Little Bao Diner \" , number_of_people #unknown , time #unknown , user_name #unknown , ( restaurants search ) cuisine not \" Steakhouse \" , dietary_restrictions equal_to \" Vegan Options \" , location equal_to \" Victoria Park \" , rating equal_to \" don't care \" AGENT_ACTS_PREV: ( restaurants search ) inform address equal_to \" 9 Kingston Street, Causeway Bay Shop H1, Fashion Walk \" , inform phone_number equal_to \" +852 2555 0600 \" AGENT_ACTS: ( restaurants booking ) request time , request user_name USER: Reserve it under Paul, for 4:50 pm on April 2. ", + "output_text": "( restaurants booking ) date equal_to \" April 2 \" , time equal_to \" 4:50 pm \" , user_name equal_to \" Paul \" , number_of_people equal_to \" 12 \"", + "train_target": "dst", + "category": "add_slot" + } + ] +} \ No newline at end of file diff --git a/tests/dataset/bitod_error/valid.json b/tests/dataset/bitod_error/valid.json new file mode 100644 index 00000000..3c62792f --- /dev/null +++ b/tests/dataset/bitod_error/valid.json @@ -0,0 +1,58 @@ +{ + "data": [ + { + "dial_id": "d4da198a-a529-4d26-8554-9672333f6d5f", + "task": "restaurants search", + "turn_id": 2, + "input_text": "DST: ( restaurants search ) dietary_restrictions equal_to \" Vegan Options \" AGENT_ACTS: ( restaurants search ) request rating USER: I do not care about the rating. I don't want a place with Steakhouse food, but anything else is good. ", + "output_text": "( restaurants search ) cuisine not \" Steakhouse \" , rating equal_to \" don't care \"", + "train_target": "dst", + "category": "positive" + }, + { + "dial_id": "d4da198a-a529-4d26-8554-9672333f6d5f", + "task": "restaurants search", + "turn_id": 2, + "input_text": "DST: ( restaurants search ) dietary_restrictions equal_to \" Vegan Options \" AGENT_ACTS: ( restaurants search ) request rating USER: I do not care about the rating. I don't want a place with Steakhouse food, but anything else is good. ", + "output_text": "( restaurants search ) rating equal_to \" don't care \"", + "train_target": "dst", + "category": "remove_slot" + }, + { + "dial_id": "d4da198a-a529-4d26-8554-9672333f6d5f", + "task": "restaurants search", + "turn_id": 4, + "input_text": "DST: ( restaurants search ) cuisine not \" Steakhouse \" , dietary_restrictions equal_to \" Vegan Options \" , location equal_to \" Victoria Park \" , rating equal_to \" don't care \" AGENT_ACTS_PREV: ( restaurants search ) offer available_options equal_to \" 292 \" , offer cuisine equal_to \" Italian \" , offer name equal_to \" ChocoDuck Bistro \" , offer rating equal_to \" 10 \" AGENT_ACTS: ( restaurants search ) offer available_options equal_to \" 1 \" , offer cuisine equal_to \" Diner \" , offer name equal_to \" Little Bao Diner \" , offer rating equal_to \" 9 \" USER: Can I have the address and phone number of the restaurant? ", + "output_text": "( restaurants search )", + "train_target": "dst", + "category": "positive" + }, + { + "dial_id": "d4da198a-a529-4d26-8554-9672333f6d5f", + "task": "restaurants search", + "turn_id": 4, + "input_text": "DST: ( restaurants search ) cuisine not \" Steakhouse \" , dietary_restrictions equal_to \" Vegan Options \" , location equal_to \" Victoria Park \" , rating equal_to \" don't care \" AGENT_ACTS_PREV: ( restaurants search ) offer available_options equal_to \" 292 \" , offer cuisine equal_to \" Italian \" , offer name equal_to \" ChocoDuck Bistro \" , offer rating equal_to \" 10 \" AGENT_ACTS: ( restaurants search ) offer available_options equal_to \" 1 \" , offer cuisine equal_to \" Diner \" , offer name equal_to \" Little Bao Diner \" , offer rating equal_to \" 9 \" USER: Can I have the address and phone number of the restaurant? ", + "output_text": "( restaurants search ) cuisine equal_to \" Steakhouse \"", + "train_target": "dst", + "category": "change_relation" + }, + { + "dial_id": "d4da198a-a529-4d26-8554-9672333f6d5f", + "task": "restaurants booking", + "turn_id": 6, + "input_text": "DST: ( restaurants booking ) date #unknown , name equal_to \" Little Bao Diner \" , number_of_people #unknown , time #unknown , user_name #unknown , ( restaurants search ) cuisine not \" Steakhouse \" , dietary_restrictions equal_to \" Vegan Options \" , location equal_to \" Victoria Park \" , rating equal_to \" don't care \" AGENT_ACTS_PREV: ( restaurants search ) inform address equal_to \" 9 Kingston Street, Causeway Bay Shop H1, Fashion Walk \" , inform phone_number equal_to \" +852 2555 0600 \" AGENT_ACTS: ( restaurants booking ) request time , request user_name USER: Reserve it under Paul, for 4:50 pm on April 2. ", + "output_text": "( restaurants booking ) date equal_to \" April 2 \" , time equal_to \" 4:50 pm \" , user_name equal_to \" Paul \"", + "train_target": "dst", + "category": "positive" + }, + { + "dial_id": "d4da198a-a529-4d26-8554-9672333f6d5f", + "task": "restaurants booking", + "turn_id": 6, + "input_text": "DST: ( restaurants booking ) date #unknown , name equal_to \" Little Bao Diner \" , number_of_people #unknown , time #unknown , user_name #unknown , ( restaurants search ) cuisine not \" Steakhouse \" , dietary_restrictions equal_to \" Vegan Options \" , location equal_to \" Victoria Park \" , rating equal_to \" don't care \" AGENT_ACTS_PREV: ( restaurants search ) inform address equal_to \" 9 Kingston Street, Causeway Bay Shop H1, Fashion Walk \" , inform phone_number equal_to \" +852 2555 0600 \" AGENT_ACTS: ( restaurants booking ) request time , request user_name USER: Reserve it under Paul, for 4:50 pm on April 2. ", + "output_text": "( restaurants booking ) date equal_to \" April 2 \" , time equal_to \" 4:50 pm \" , user_name equal_to \" Paul \" , number_of_people equal_to \" 12 \"", + "train_target": "dst", + "category": "add_slot" + } + ] +} \ No newline at end of file diff --git a/tests/expected_results/sequence_classification/bitod_error_cls.tsv b/tests/expected_results/sequence_classification/bitod_error_cls.tsv new file mode 100644 index 00000000..3583a41e --- /dev/null +++ b/tests/expected_results/sequence_classification/bitod_error_cls.tsv @@ -0,0 +1,6 @@ +bitod_error_cls/d4da198a-a529-4d26-8554-9672333f6d5f/2/dst positive positive DST: ( restaurants search ) dietary _ restrictions equal _ to " " AGENT_ACTS: ( restaurants search ) request rating USER: don't care about the rating. don't want a place with food, but anything else is good. ( restaurants search ) cuisine not " ", rating equal _ to " don't care " +bitod_error_cls/d4da198a-a529-4d26-8554-9672333f6d5f/2/dst negative negative DST: ( restaurants search ) dietary _ restrictions equal _ to " " AGENT_ACTS: ( restaurants search ) request rating USER: don't care about the rating. don't want a place with food, but anything else is good. ( restaurants search ) rating equal _ to " don't care " +bitod_error_cls/d4da198a-a529-4d26-8554-9672333f6d5f/4/dst positive positive DST: ( restaurants search ) cuisine not " ", dietary _ restrictions equal _ to " ", location equal _ to " ", rating equal _ to " don't care " AGENT_ACTS_PREV: ( restaurants search ) offer available _ options equal _ to " 292 ", offer cuisine equal _ to " ", offer name equal _ to " ", offer rating equal _ to " 10 " AGENT_ACTS: ( restaurants search ) offer available _ options equal _ to " 1 ", offer cuisine equal _ to " ", offer name equal _ to " ", offer rating equal _ to " 9 " USER: have the address and phone number of the restaurant? ( restaurants search ) +bitod_error_cls/d4da198a-a529-4d26-8554-9672333f6d5f/4/dst negative negative DST: ( restaurants search ) cuisine not " ", dietary _ restrictions equal _ to " ", location equal _ to " ", rating equal _ to " don't care " AGENT_ACTS_PREV: ( restaurants search ) offer available _ options equal _ to " 292 ", offer cuisine equal _ to " ", offer name equal _ to " ", offer rating equal _ to " 10 " AGENT_ACTS: ( restaurants search ) offer available _ options equal _ to " 1 ", offer cuisine equal _ to " ", offer name equal _ to " ", offer rating equal _ to " 9 " USER: have the address and phone number of the restaurant? ( restaurants search ) cuisine equal _ to " " +bitod_error_cls/d4da198a-a529-4d26-8554-9672333f6d5f/6/dst positive positive DST: ( restaurants booking ) date #unknown, name equal _ to " ", number _ of _ people #unknown, time #unknown, user _ name #unknown, ( restaurants search ) cuisine not " ", dietary _ restrictions equal _ to " ", location equal _ to " ", rating equal _ to " don't care " AGENT_ACTS_PREV: ( restaurants search ) inform address equal _ to " 9,, ", inform phone _ number equal _ to " + 852 2555 0600 " AGENT_ACTS: ( restaurants booking ) request time, request user _ name USER: it under, for 4 : 50 pm on 2. ( restaurants booking ) date equal _ to " 2 ", time equal _ to " 4 : 50 pm ", user _ name equal _ to " " +bitod_error_cls/d4da198a-a529-4d26-8554-9672333f6d5f/6/dst negative negative DST: ( restaurants booking ) date #unknown, name equal _ to " ", number _ of _ people #unknown, time #unknown, user _ name #unknown, ( restaurants search ) cuisine not " ", dietary _ restrictions equal _ to " ", location equal _ to " ", rating equal _ to " don't care " AGENT_ACTS_PREV: ( restaurants search ) inform address equal _ to " 9,, ", inform phone _ number equal _ to " + 852 2555 0600 " AGENT_ACTS: ( restaurants booking ) request time, request user _ name USER: it under, for 4 : 50 pm on 2. ( restaurants booking ) date equal _ to " 2 ", time equal _ to " 4 : 50 pm ", user _ name equal _ to " ", number _ of _ people equal _ to " 12 " diff --git a/tests/test_sequence_classification.sh b/tests/test_sequence_classification.sh index 1db24719..06a0e235 100755 --- a/tests/test_sequence_classification.sh +++ b/tests/test_sequence_classification.sh @@ -2,6 +2,7 @@ . ./tests/lib.sh +# Test ood task # train genienlp train --train_tasks ood_task --model TransformerForSequenceClassification --pretrained_model distilbert-base-uncased --min_output_length 1 --save $workdir/model --train_iterations 20 --save_every 10 --log_every 10 --val_every 10 --data $SRCDIR/dataset/ood/ --force_fast_tokenizer --train_batch_tokens 200 --num_print 0 @@ -18,3 +19,22 @@ fi diff -u $SRCDIR/expected_results/sequence_classification/ood_task.tsv $workdir/model/eval_results/valid/ood_task.tsv rm -rf $workdir/model + + +# Test bitod_error_cls task +# train +genienlp train --train_tasks bitod_error_cls --model TransformerForSequenceClassification --pretrained_model distilbert-base-uncased --min_output_length 1 --save $workdir/model_error/ --train_iterations 100 --save_every 50 --log_every 50 --val_every 50 --data $SRCDIR/dataset/bitod_error/ --force_fast_tokenizer --train_batch_tokens 200 --num_print 0 + +# greedy prediction +genienlp predict --tasks bitod_error_cls --evaluate valid --pred_set_name valid --path $workdir/model_error --overwrite --eval_dir $workdir/model_error/eval_results/ --data $SRCDIR/dataset/bitod_error/ --embeddings $EMBEDDING_DIR --val_batch_size 200 + +# check if result file exists +if test ! -f $workdir/model_error/eval_results/valid/bitod_error_cls.tsv ; then + echo "File not found!" + exit 1 +fi + +# check if predictions matches expected_results +diff -u $SRCDIR/expected_results/sequence_classification/bitod_error_cls.tsv $workdir/model_error/eval_results/valid/bitod_error_cls.tsv + +rm -rf $workdir/model_error