From d6c92d870c17ccde32b0b888e5e4111a90eb11fe Mon Sep 17 00:00:00 2001 From: David Anderson Date: Wed, 20 Feb 2013 16:20:00 -0800 Subject: [PATCH] - Code cleanup for remote job submission - Add abort_jobs operation to BOINC GAHP - Change batch-related RPCs so that you can identify batch either by database ID or name --- html/inc/common_defs.inc | 29 ++++++++++ html/inc/submit_db.inc | 14 ++--- html/user/get_output.php | 20 +++++-- html/user/job_file.php | 6 +- html/user/submit_example.php | 10 +--- html/user/submit_rpc_handler.php | 57 ++++++++++++++----- .../job_rpc.cpp => lib/remote_submit.cpp | 45 +++++++++++++-- .../condor/job_rpc.h => lib/remote_submit.h | 11 +++- samples/condor/Makefile | 8 +-- samples/condor/boinc_gahp.cpp | 23 +++++++- 10 files changed, 173 insertions(+), 50 deletions(-) create mode 100644 html/inc/common_defs.inc rename samples/condor/job_rpc.cpp => lib/remote_submit.cpp (88%) rename samples/condor/job_rpc.h => lib/remote_submit.h (91%) diff --git a/html/inc/common_defs.inc b/html/inc/common_defs.inc new file mode 100644 index 0000000000..6b5c5cda5f --- /dev/null +++ b/html/inc/common_defs.inc @@ -0,0 +1,29 @@ +. + +// defines needed by multiple pieces of code + +// see db/boinc_db.h +// +define('BATCH_STATE_INIT', 0); +define('BATCH_STATE_IN_PROGRESS', 1); +define('BATCH_STATE_COMPLETE', 2); +define('BATCH_STATE_ABORTED', 3); +define('BATCH_STATE_RETIRED', 4); + +?> diff --git a/html/inc/submit_db.inc b/html/inc/submit_db.inc index 0687ddf785..bb2bd797c8 100644 --- a/html/inc/submit_db.inc +++ b/html/inc/submit_db.inc @@ -18,11 +18,17 @@ // Tables related to job submission +require_once("../inc/common_defs.inc"); + class BoincBatch { static function lookup_id($id) { $db = BoincDb::get(); return $db->lookup_id($id, 'batch', 'BoincBatch'); } + static function lookup_name($name) { + $db = BoincDb::get(); + return $db->lookup('batch', 'BoincBatch', "name='$name'"); + } static function enum($clause) { $db = BoincDb::get(); return $db->enum('batch', 'BoincBatch', $clause); @@ -43,14 +49,6 @@ class BoincBatch { } } -// see db/boinc_db.h -// -define('BATCH_STATE_INIT', 0); -define('BATCH_STATE_IN_PROGRESS', 1); -define('BATCH_STATE_COMPLETE', 2); -define('BATCH_STATE_ABORTED', 3); -define('BATCH_STATE_RETIRED', 4); - class BoincUserSubmit { static function enum($clause) { $db = BoincDb::get(); diff --git a/html/user/get_output.php b/html/user/get_output.php index 83aa38c1e6..e2d4a353d7 100644 --- a/html/user/get_output.php +++ b/html/user/get_output.php @@ -55,12 +55,21 @@ function get_output_file($instance_name, $file_num, $auth_str) { // get all the output files of a batch (canonical instances only) // and make a zip of all of them // -function get_batch_output_files($batch_id, $auth_str) { - $batch = BoincBatch::lookup_id($batch_id); - if (!$batch) die("no batch $batch_id"); +function get_batch_output_files($auth_str) { + $batch_id = get_int('batch_id', true); + if ($batch_id) { + $batch = BoincBatch::lookup_id($batch_id); + if (!$batch) die("no batch $batch_id"); + } else { + $batch_name = get_int('batch_name'); + $batch_name = BoincDb::escape_string($batch_name); + $batch = BoincBatch::lookup("name='$batch_name'"); + if (!$batch) die("no batch $batch_name"); + } + $user = BoincUser::lookup_id($batch->user_id); if (!$user) die("no user $batch->user_id"); - $x = md5($user->authenticator.$batch_id); + $x = md5($user->authenticator.$batch->id); if ($x != $auth_str) die("bad auth str"); $zip_basename = tempnam("/tmp", "boinc_batch_"); @@ -68,7 +77,7 @@ function get_batch_output_files($batch_id, $auth_str) { $fanout = parse_config(get_config(), ""); $upload_dir = parse_config(get_config(), ""); - $wus = BoincWorkunit::enum("batch=$batch_id"); + $wus = BoincWorkunit::enum("batch=$batch->id"); foreach ($wus as $wu) { if (!$wu->canonical_resultid) continue; $result = BoincResult::lookup_id($wu->canonical_resultid); @@ -149,7 +158,6 @@ case 'result_file'; get_output_file($result_name, $file_num, $auth_str); break; case 'batch_files': - $batch_id = get_int('batch_id'); get_batch_output_files($batch_id, $auth_str); break; case 'workunit_file': diff --git a/html/user/job_file.php b/html/user/job_file.php index ec91f59a7a..62f150aee8 100644 --- a/html/user/job_file.php +++ b/html/user/job_file.php @@ -25,12 +25,12 @@ // This eliminates issues related to file immutability // // 2) how do we keep track of the files? -// In the MySQL database, in a table called job_files. +// In the MySQL database, in a table called "job_file". // Each row describes a file currently on the server. -// In addition, we maintain a table batch_file_assoc to record +// In addition, we maintain a table "batch_file_assoc" to record // that a file is used by a particular batch. // (Note: the association could be at the job level instead. -// but this way is more efficient if all the jobs in a batch use +// but this way is more efficient if many jobs in a batch use // a particular file.) // // 3) how do we clean up unused files? diff --git a/html/user/submit_example.php b/html/user/submit_example.php index b1480f4138..5b0998defc 100644 --- a/html/user/submit_example.php +++ b/html/user/submit_example.php @@ -31,8 +31,9 @@ // you can strip out this stuff if the web site doesn't use BOINC require_once("../inc/submit.inc"); -require_once("../inc/submit_util.inc"); +require_once("../inc/common_defs.inc"); require_once("../inc/submit_db.inc"); + // needed for access control stuff require_once("../inc/util.inc"); require_once("../project/project.inc"); @@ -74,7 +75,6 @@ function handle_main() { with permission to submit jobs.

"; - show_button("submit_example.php?action=manage_files", "Manage files"); show_button("submit_example.php?action=create_form", "Create new batch"); $first = true; @@ -455,19 +455,13 @@ function handle_retire_batch() { page_tail(); } -function manage_files() { - $files = submit_get_file_list(); -} - $action = get_str('action', true); - switch ($action) { case '': handle_main(); break; case 'abort_batch': handle_abort_batch(); break; case 'abort_batch_confirm': handle_abort_batch_confirm(); break; case 'create_action': handle_create_action(); break; case 'create_form': handle_create_form(); break; -case 'manage_files': manage_files(); break; case 'query_batch': handle_query_batch(); break; case 'query_job': handle_query_job(); break; case 'retire_batch': handle_retire_batch(); break; diff --git a/html/user/submit_rpc_handler.php b/html/user/submit_rpc_handler.php index 2f163958a3..d7c30c59e7 100644 --- a/html/user/submit_rpc_handler.php +++ b/html/user/submit_rpc_handler.php @@ -281,11 +281,26 @@ function n_outfiles($wu) { return count($r->file_info); } +// return a batch specified by the command, using either ID or name +// +function get_batch($r) { + if (!empty($r->batch_id)) { + $batch_id = (int)($r->batch_id); + $batch = BoincBatch::lookup_id($batch_id); + } else if (!empty($r->batch_name)) { + $batch_name = (string)($r->batch_name); + $batch_name = BoincDb::escape_string($batch_name); + $batch = BoincBatch::lookup("name='$batch_name'"); + } else { + xml_error(-1, "batch not specified"); + } + if (!$batch) xml_error(-1, "no such batch"); + return $batch; +} + function query_batch($r) { list($user, $user_submit) = authenticate_user($r, null); - $batch_id = (int)($r->batch_id); - $batch = BoincBatch::lookup_id($batch_id); - if (!$batch) xml_error(-1, "no such batch"); + $batch = get_batch($r); if ($batch->user_id != $user->id) xml_error(-1, "not owner"); $wus = BoincWorkunit::enum("batch = $batch_id"); @@ -305,14 +320,13 @@ function query_batch($r) { } // variant for Condor, which doesn't care about job instances +// and refers to batches by name // function query_batch2($r) { list($user, $user_submit) = authenticate_user($r, null); - $batch_id = (int)($r->batch_id); - $batch = BoincBatch::lookup_id($batch_id); - if (!$batch) xml_error(-1, "no such batch"); + $batch = get_batch($r); if ($batch->user_id != $user->id) xml_error(-1, "not owner"); - $wus = BoincWorkunit::enum("batch = $batch_id"); + $wus = BoincWorkunit::enum("batch = $batch->id"); echo "\n"; foreach ($wus as $wu) { if ($wu->canonical_resultid) { @@ -368,9 +382,7 @@ function query_job($r) { function handle_abort_batch($r) { list($user, $user_submit) = authenticate_user($r, null); - $batch_id = (int)($r->batch_id); - $batch = BoincBatch::lookup_id($batch_id); - if (!$batch) xml_error(-1, "no such batch"); + $batch = get_batch($r); if ($batch->user_id != $user->id) { xml_error(-1, "not owner"); } @@ -378,11 +390,29 @@ function handle_abort_batch($r) { echo "1"; } +function handle_abort_jobs($r) { + list($user, $user_submit) = authenticate_user($r, null); + $batch = get_batch($r); + if ($batch->user_id != $user->id) { + xml_error(-1, "not owner"); + } + foreach ($r->job_names as $job_name) { + $job_name = BoincDb::escape_string($job_name); + $wu = BoincWorkunit::lookup("name='$job_name'"); + if (!$wu) { + xml_error(-1, "No job $job_name"); + } + if ($wu->batch != $batch_id) { + xml_error(-1, "Not owner of job $job_name"); + } + abort_workunit($wu); + } + echo "1"; +} + function handle_retire_batch($r) { list($user, $user_submit) = authenticate_user($r, null); - $batch_id = (int)($r->batch_id); - $batch = BoincBatch::lookup_id($batch_id); - if (!$batch) xml_error(-1, "no such batch"); + $batch = get_batch($r); if ($batch->user_id != $user->id) { xml_error(-1, "not owner"); } @@ -442,6 +472,7 @@ if (!$r) { switch ($r->getName()) { case 'abort_batch': handle_abort_batch($r); break; + case 'abort_jobs': handle_abort_jobs($r); break; case 'estimate_batch': estimate_batch($r); break; case 'query_batch': query_batch($r); break; case 'query_batch2': query_batch2($r); break; diff --git a/samples/condor/job_rpc.cpp b/lib/remote_submit.cpp similarity index 88% rename from samples/condor/job_rpc.cpp rename to lib/remote_submit.cpp index 0fb6990903..b69ff10260 100644 --- a/samples/condor/job_rpc.cpp +++ b/lib/remote_submit.cpp @@ -23,7 +23,7 @@ #include "parse.h" -#include "job_rpc.h" +#include "remote_submit.h" using std::vector; using std::string; @@ -41,7 +41,7 @@ static int do_http_get( return -1; } curl_easy_setopt(curl, CURLOPT_URL, url); - curl_easy_setopt(curl, CURLOPT_USERAGENT, "BOINC Condor adapter"); + curl_easy_setopt(curl, CURLOPT_USERAGENT, "BOINC remote job submission"); curl_easy_setopt(curl, CURLOPT_WRITEDATA, reply); CURLcode res = curl_easy_perform(curl); @@ -305,13 +305,13 @@ int submit_jobs( int query_batch( const char* project_url, const char* authenticator, - int batch_id, + string batch_name, QUERY_BATCH_REPLY& qb_reply ) { string request; char url[1024], buf[256]; request = "\n"; - sprintf(buf, "%d\n", batch_id); + sprintf(buf, "%s\n", batch_name.c_str()); request += string(buf); sprintf(buf, "%s\n", authenticator); request += string(buf); @@ -346,6 +346,43 @@ int query_batch( return retval; } +int abort_jobs( + const char* project_url, + const char* authenticator, + string batch_name, + vector &job_names +) { + string request; + char url[1024], buf[256]; + request = "\n"; + sprintf(buf, "%s\n", authenticator); + request += string(buf); + sprintf(buf, "%s\n", batch_name.c_str()); + request += string(buf); + for (unsigned int i=0; i%s\n", job_names[i].c_str()); + request += string(buf); + } + request += "\n"; + sprintf(url, "%ssubmit_rpc_handler.php", project_url); + FILE* reply = tmpfile(); + vector x; + int retval = do_http_post(url, request.c_str(), reply, x); + if (retval) { + fclose(reply); + return retval; + } + fseek(reply, 0, SEEK_SET); + retval = 0; + while (fgets(buf, 256, reply)) { + if (strstr(buf, "error")) { + retval = -1; + } + } + fclose(reply); + return retval; +} + int get_output_file( const char* project_url, const char* authenticator, diff --git a/samples/condor/job_rpc.h b/lib/remote_submit.h similarity index 91% rename from samples/condor/job_rpc.h rename to lib/remote_submit.h index 5644fe1792..9248f0c644 100644 --- a/samples/condor/job_rpc.h +++ b/lib/remote_submit.h @@ -15,6 +15,8 @@ // You should have received a copy of the GNU Lesser General Public License // along with BOINC. If not, see . +// C++ interfaces to remote job submissions and file management RPCs + #include #include #include @@ -104,7 +106,7 @@ extern int submit_jobs( extern int query_batch( const char* project_url, const char* authenticator, - int batch_id, + string batch_name, QUERY_BATCH_REPLY& reply ); @@ -115,3 +117,10 @@ extern int get_output_file( int file_num, const char* dst_path ); + +extern int abort_jobs( + const char* project_url, + const char* authenticator, + string batch_name, + vector &job_names +); diff --git a/samples/condor/Makefile b/samples/condor/Makefile index 4d2a595b47..dc8fa97895 100644 --- a/samples/condor/Makefile +++ b/samples/condor/Makefile @@ -1,11 +1,11 @@ all: boinc_gahp -boinc_gahp: boinc_gahp.cpp job_rpc.cpp job_rpc.h +boinc_gahp: boinc_gahp.cpp ../../lib/remote_submit.h ../../lib/remote_submit.cpp g++ -g -O0 -I../../lib \ - -o boinc_gahp boinc_gahp.cpp job_rpc.cpp \ + -o boinc_gahp boinc_gahp.cpp ../../lib/remote_submit.cpp \ -L../../lib -lboinc -lpthread -lcurl -test: test.cpp job_rpc.cpp +test: test.cpp ../../lib/remote_submit.cpp ../../lib/remote_submit.h g++ -g -o test -I../../lib \ - test.cpp job_rpc.cpp \ + test.cpp ../../lib/remote_submit.cpp \ -L../../lib -lboinc -lcurl diff --git a/samples/condor/boinc_gahp.cpp b/samples/condor/boinc_gahp.cpp index 48f788debf..1542248199 100644 --- a/samples/condor/boinc_gahp.cpp +++ b/samples/condor/boinc_gahp.cpp @@ -33,7 +33,7 @@ #include "md5_file.h" #include "parse.h" -#include "job_rpc.h" +#include "remote_submit.h" using std::map; using std::pair; @@ -208,9 +208,9 @@ void handle_submit(COMMAND& c, char* p) { } void handle_query_batch(COMMAND&c, char* p) { - int batch_id = atoi(strtok_r(NULL, " ", &p)); + char* batch_name = strtok_r(NULL, " ", &p); QUERY_BATCH_REPLY reply; - query_batch(project_url, authenticator, batch_id, reply); + query_batch(project_url, authenticator, batch_name, reply); for (unsigned int i=0; i job_names; + char* batch_name = strtok_r(NULL, " ", &p); + while (1) { + char* job_name = strtok_r(NULL, " ", &p); + if (!job_name) break; + job_names.push_back(string(job_name)); + } + int retval = abort_jobs(project_url, authenticator, string(batch_name), job_names); + if (retval) { + printf("abort_jobs() returned %d\n", retval); + } +} + void* handle_command_aux(void* q) { COMMAND &c = *((COMMAND*)q); char *p; @@ -257,6 +271,8 @@ void* handle_command_aux(void* q) { handle_query_batch(c, p); } else if (!strcmp(cmd, "BOINC_FETCH_OUTPUT")) { handle_fetch_output(c, p); + } else if (!strcmp(cmd, "BOINC_ABORT_JOBS")) { + handle_abort_jobs(c, p); } else { sleep(10); char buf[256]; @@ -384,5 +400,6 @@ int main() { COMMAND c; c.in = p; handle_command(c); + fflush(stdout); } }