diff --git a/checkin_notes b/checkin_notes index e8be9f013c..68ea05bd9c 100644 --- a/checkin_notes +++ b/checkin_notes @@ -4291,3 +4291,15 @@ David 20 July 2011 client_types.cpp,h cs_statefile.cpp scheduler_op.cpp + +David 22 July 2011 + - server: some remote job submission code. Not finished. + + tools/ + create_work.cpp + html/ + inc/ + submit.inc + user/ + submit.php + submit_example.php diff --git a/doc/download_all.php b/doc/download_all.php index 5c8198727e..d04b3b2218 100644 --- a/doc/download_all.php +++ b/doc/download_all.php @@ -117,7 +117,7 @@ function show_platform($short_name, $p, $dev) { global $max_version; $long_name = $p["name"]; $description = $p["description"]; - if ($p["url"]) { + if (array_key_exists('url', $p)) { $url = $p["url"]; $long_name .= " details"; } diff --git a/html/inc/submit.inc b/html/inc/submit.inc new file mode 100644 index 0000000000..f877e82e86 --- /dev/null +++ b/html/inc/submit.inc @@ -0,0 +1,202 @@ +. + +// remote job submission API + +//// Implementation stuff follows + +function req_to_xml($req, $op) { + $x = "<$op> + $req->authenticator + + $req->app_name +"; + foreach ($req->jobs as $job) { + $x .= " + $job->rsc_fpops_est + $job->command_line +"; + foreach ($job->input_files as $file) { + $x .= " + $file->source + $file->name + +"; + } + $x .= " +"; + } + $x .= " + +"; + return $x; +} + +function validate_request($req) { + if (!is_object($req)) return "req is not an object"; + if (!array_key_exists('project', $req)) return "missing req->project"; + if (!array_key_exists('authenticator', $req)) return "missing req->authenticator"; + if (!array_key_exists('app_name', $req)) return "missing req->app_name"; + if (!array_key_exists('jobs', $req)) return "missing req->jobs"; + if (!is_array($req->jobs)) return "req->jobs is not an array"; + foreach ($req->jobs as $job) { + // other checks + } + return null; +} + +function do_http_op($project, $xml) { + $ch = curl_init("$project/submit.php"); + curl_setopt($ch, CURLOPT_POST, 1); + curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1); + curl_setopt($ch, CURLOPT_POSTFIELDS, "request=$xml"); + $reply = curl_exec($ch); + if (!$reply) return array(null, "HTTP error"); + $r = simplexml_load_string($reply); + if (!$r) return array(null, "Can't parse reply XML"); + return array($r, null); +} + +function do_batch_op($req, $op) { + $retval = validate_request($req); + if ($retval) return array(null, $retval); + $xml = req_to_xml($req, $op); + return do_http_op($req->project, $xml); +} + +//// Interface functions follow + +function boinc_estimate_batch($req) { + list($reply, $errmsg) = do_batch_op($req, "estimate_batch"); + if ($errmsg) return array(0, $errmsg); + $name = $reply->getName(); + if ($name == 'estimate') { + return array((string)$reply->seconds, null); + } else if ($name = 'error') { + return array(null, (string)$reply->message); + } else { + return array(null, "Bad reply message"); + } +} + +function boinc_submit_batch($req) { + list($reply, $errmsg) = do_batch_op($req, "submit_batch"); + if ($errmsg) return array(0, $errmsg); + $name = $reply->getName(); + if ($name == 'batch_id') { + return array((int)$reply, null); + } else if ($name == 'error') { + return array(null, (string)$reply->message); + } else { + return array(null, "Bad reply message"); + } +} + +function boinc_query_batches($req) { + $req_xml = " + $req->authenticator + +"; + list($reply, $errmsg) = do_http_op($req->project, $req_xml); + if ($errmsg) return array(null, $errmsg); + $batches = array(); + foreach ($reply->batch as $batch) { + $b = null; + $b->id = (int)($batch->id); + $b->completed = (int)($batch->completed); + $b->njobs = (int)($batch->njobs); + $b->create_time = (double)($batch->create_time); + if (!$batch->completed) { + $b->fraction_done = (double) $batch->fraction_done; + } else { + $b->completed_time = (double)($batch->completed_time); + } + $batches[] = $b; + } + return array($batches, null); +} + +function boinc_query_batch() { + $req_xml = " + $req->authenticator + $req->batch_id + +"; + list($reply, $errmsg) = do_http_op($req->project, $req_xml); + if ($errmsg) return array(null, $errmsg); + $jobs = array(); + foreach ($reply->job as $job) { + $j = null; + $j->id = (int)($job->id); + $jobs[] = $j; + } + $r = null; + $r->jobs = $jobs; + return array($r, null); +} + +function boinc_abort_batch() { +} + +//// example usage follows + +if (0) { + $req->project = "http://foo.edu/test/"; + $req->authenticator = "xxx"; + $req->app_name = "uppercase"; + $req->jobs = array(); + + $f->source = "http://foo.edu/index.php"; + $f->name = "in"; + $job->input_files = array($f); + + for ($i=10; $i<20; $i++) { + $job->rsc_fpops_est = $i*1e9; + $job->command_line = "--t $i"; + $req->jobs[] = $job; + } + + if (0) { + list($e, $errmsg) = boinc_estimate_batch($req); + if ($errmsg) { + echo "Error from server: $errmsg\n"; + } else { + echo "Batch completion estimate: $e seconds\n"; + } + } else { + list($id, $errmsg) = boinc_submit_batch($req); + if ($errmsg) { + echo "Error from server: $errmsg\n"; + } else { + echo "Batch ID: $id\n"; + } + } +} + +if (0) { + $req->project = "http://foo.edu/test/"; + $req->authenticator = "xxx"; + list($batches, $errmsg) = boinc_query_batches($req); + if ($errmsg) { + echo "Error: $errmsg\n"; + } else { + print_r($batches); + } +} + +?> diff --git a/html/ops/submit_example.php b/html/ops/submit_example.php deleted file mode 100644 index f6ae61dd33..0000000000 --- a/html/ops/submit_example.php +++ /dev/null @@ -1,57 +0,0 @@ -. - -// example code for making a remote job submission - -$ch = curl_init("http://foo.edu/test/submit.php"); -curl_setopt($ch, CURLOPT_POST, 1); -curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1); -curl_setopt($ch, CURLOPT_POSTFIELDS, "request= - - xxx - - uppercase - - 100e9 - --t ALPHA - - http://foo.edu/index.php - name - - - - -"); -$reply = curl_exec($ch); -if (!$reply) die("HTTP op failed\n"); -echo "reply: $reply\n"; exit; - -$r = simplexml_load_string($reply); -if (!$r) die("bad reply: $reply\n"); -$name = $r->getName(); -if ($name == 'estimate') { - echo "estimate: ".(string)$r->seconds."\n"; -} else if ($name == 'error') { - foreach ($r->message as $msg) { - echo "$msg\n"; - } -} else { - die("bad reply\n"); -} - -?> diff --git a/html/user/submit.php b/html/user/submit.php index 1160b2416a..65cb71faa7 100644 --- a/html/user/submit.php +++ b/html/user/submit.php @@ -36,7 +36,7 @@ function authenticate_user($r, $app) { if (!$user) error("bad authenticator"); $user_submit = BoincUserSubmit::lookup_userid($user->id); if (!$user_submit) error("no submit access"); - if (!$user_submit->all_apps) { + if ($app && !$user_submit->all_apps) { $usa = BoincUserSubmitApp::lookup("user_id=$user->id and app_id=$app->id"); if (!$usa) { error("no submit access"); @@ -68,7 +68,7 @@ function project_flops() { return $y; } -function batch_estimate($r) { +function estimate_batch($r) { $app = get_app($r); list($user, $user_submit) = authenticate_user($r, $app); @@ -99,9 +99,9 @@ $fanout = parse_config(get_config(), ""); function stage_file($file) { global $fanout; - $physical_name = (string)$file->physical_name; + $name = (string)$file->name; $source = (string)$file->source; - $path = dir_hier_path($physical_name, "../../download", $fanout); + $path = dir_hier_path($name, "../../download", $fanout); if (file_exists($path)) return; if (!copy($source, $path)) { error("can't copy file from $source"); @@ -120,14 +120,16 @@ function submit_job($job, $template, $app, $batch_id, $i) { $cmd = "cd ../..; ./bin/create_work --appname $app->name --batch $batch_id"; $cmd .= " --wu_name batch_".$batch_id."_".$i; foreach ($job->input_file as $file) { - $name = (string)$file->physical_name; + $name = (string)$file->name; $cmd .= " $name"; - system($cmd); } - + $ret = system($cmd); + if ($ret === FALSE) { + error("can't create job"); + } } -function batch_submit($r) { +function submit_batch($r) { $app = get_app($r); list($user, $user_submit) = authenticate_user($r, $app); $template = read_input_template($app); @@ -135,7 +137,9 @@ function batch_submit($r) { stage_files($r, $template); $njobs = count($r->batch->job); $now = time(); - $batch_id = BoincBatch::insert("(user_id, create_time, njobs) values ($user->id, $now, $njobs)"); + $batch_id = BoincBatch::insert( + "(user_id, create_time, njobs) values ($user->id, $now, $njobs)" + ); $i = 0; foreach($r->batch->job as $job) { submit_job($job, $template, $app, $batch_id, $i++); @@ -143,10 +147,54 @@ function batch_submit($r) { echo "$batch_id\n"; } -function query_job($r) { +function fraction_done($batch) { + $wus = BoincWorkunit::enum("batch = $batch->id"); + $fp_total = 0; + $fp_done = 0; + foreach ($wus as $wu) { + $fp_total += $wu->fpops_est; + if ($wu->canonical_resultid) { + $fp_done += $wu->fpops_est; + } + } + if (!$fp_total) return 1; + return $fp_done / $fp_total; +} + +function query_batches($r) { + list($user, $user_submit) = authenticate_user($r, null); + $batches = BoincBatch::enum("user_id = $user->id"); + echo "\n"; + foreach ($batches as $b) { + $fd = fraction_done($b); + echo " + $b->id + $fd + $b->create_time + $b->est_completion_time + $b->njobs + +"; + } + echo "\n"; } function query_batch($r) { + list($user, $user_submit) = authenticate_user($r, null); + $batch_id = (int)($r->batch_id); + $batch = BoincBatch::lookup_id($batch_id); + if ($batch->user_id != $user->id) { + error("not owner"); + } + echo "\n"; + $wus = BoincWorkunit::enum("batch = $batch_id"); + foreach ($wus as $wu) { + echo " + $wu->id + +"; + } + echo "\n"; } function abort_batch($r) { @@ -160,10 +208,10 @@ if (!$r) { } switch ($r->getName()) { - case 'batch_estimate': batch_estimate($r); break; - case 'batch_submit': batch_submit($r); break; - case 'query_job': query_job($r); break; + case 'estimate_batch': estimate_batch($r); break; + case 'submit_batch': submit_batch($r); break; case 'query_batch': query_batch($r); break; + case 'query_batches': query_batches($r); break; case 'abort_batch': abort_batch($r); break; default: error("bad command"); } diff --git a/html/user/submit_example.php b/html/user/submit_example.php new file mode 100644 index 0000000000..e89b4920f2 --- /dev/null +++ b/html/user/submit_example.php @@ -0,0 +1,201 @@ +. + +// example of a web interface to remote job submission +// +// Notes: +// - You'll need to adapt/extend this considerably; +// e.g. the project URL, app name and user authenticator are hardwired here. +// - This can run on any host, not just the project server +// (that's the "remote" part). +// - For convenience, this uses some functions from BOINC +// (page_head() etc.). +// When you adapt this to your own purposes, +// you can strip out this stuff if the web site doesn't use BOINC + +require_once("../inc/submit.inc"); +require_once("../inc/util.inc"); + +$project = "http://foo.edu/test/"; +$auth = "xxx"; +$app_name = "uppercase"; + +function handle_main() { + global $project, $auth, $app_name; + $req->project = $project; + $req->authenticator = $auth; + list($batches, $errmsg) = boinc_query_batches($req); + if ($errmsg) error_page($errmsg); + + page_head("Remote job interface"); + + echo "

Batches in progress

\n"; + start_table(); + table_header("ID", "# jobs", "progress", "submitted"); + foreach ($batches as $batch) { + if ($batch->completed) continue; + $pct_done = (int)($batch->fraction_done*100); + table_row( + "id>$batch->id", + $batch->njobs, + "$pct_done%", + time_str($batch->create_time) + ); + } + end_table(); + + echo "

Batches completed

\n"; + start_table(); + table_header("ID", "# jobs", "submitted", "completed"); + foreach ($batches as $batch) { + if (!$batch->completed) continue; + table_row( + "id>$batch->id", + $batch->njobs, + time_str($batch->submitted_time), + time_str($batch->completed_time) + ); + } + end_table(); + + echo " + Create new batch + "; + page_tail(); +} + +function handle_create_form() { + global $project, $auth, $app_name; + + page_head("Create batch"); + echo " +
+ + "; + start_table(); + row2("Input file URL", ""); + row2("Parameter low value", ""); + row2("Parameter high value", ""); + row2("Parameter increment", ""); + row2("", + "" + ); + row2("", + "" + ); + end_table(); + echo "
\n"; + page_tail(); +} + +// build a request object for boinc_*_batch() from form variables +// +function form_to_request() { + global $project, $auth, $app_name; + + $input_url = get_str('input_url'); + if (!$input_url) error_page("missing input URL"); + $param_lo = get_str('param_lo'); + if (!$param_lo) error_page("missing param lo"); + $param_hi = get_str('param_hi'); + if (!$param_hi) error_page("missing param hi"); + $param_inc = get_str('param_inc'); + if (!$param_inc) error_page("missing param inc"); + + $req->project = $project; + $req->authenticator = $auth; + $req->app_name = $app_name; + $req->jobs = Array(); + + $f->source = $input_url; + $f->name = "in"; + $job->input_files = Array($f); + + for ($x=(double)$param_lo; $x<(double)$param_hi; $x += (double)$param_inc) { + $job->rsc_fpops_est = $x*1e9; + $job->command_line = "--t $x"; + $req->jobs[] = $job; + } + + return $req; +} + +function handle_create_action() { + global $project, $auth, $app_name; + + $get_estimate = get_str('get_estimate', true); + if ($get_estimate) { + $req = form_to_request($project, $auth); + list($e, $errmsg) = boinc_estimate_batch($req); + if ($errmsg) error_page($errmsg); + page_head("Batch estimate"); + echo "Estimate: $e seconds"; + page_tail(); + } else { + $req = form_to_request($project, $auth); + list($id, $errmsg) = boinc_submit_batch($req); + if ($errmsg) error_page($errmsg); + page_head("Batch submitted"); + echo "Batch ID: $id"; + page_tail(); + } +} + +function handle_query_batch() { + global $project, $auth, $app_name; + $req->project = $project; + $req->authenticator = $auth; + $req->batch_id = get_int('batch_id'); + list($reply, $errmsg) = boinc_query_batch($req); + if ($errmsg) error_page($errmsg); + + page_head("Batch $req->batch_id"); + start_table(); + table_header("ID"); + foreach($reply->jobs as $job) { + $id = (int)$job->id; + echo " + $id + + "; + } + end_table(); + page_tail(); +} + +$action = get_str('action', true); + +switch ($action) { +case '': + handle_main(); + break; +case 'create_form': + handle_create_form(); + break; +case 'create_action': + handle_create_action(); + break; +case 'query_batch': + handle_query_batch(); + break; +default: + error_page('no such action'); +} + +?> diff --git a/tools/create_work.cpp b/tools/create_work.cpp index aef4b017fc..13decf3237 100644 --- a/tools/create_work.cpp +++ b/tools/create_work.cpp @@ -18,43 +18,8 @@ // Create a workunit. // Input files must be in the download dir. // See the docs for a description of WU and result template files -// This program must be run in the project's root directory, -// and there must be a valid config.xml file there +// This program must be run in the project's root directory // -// create_work -// --appname name -// [ --wu_name name ] -// // default: generate a name based on app name -// [ --wu_template filename ] -// relative to project root; usually in templates/ -// default: appname_in -// [ --result_template filename ] -// relative to project root; usually in templates/ -// default: appname_out -// [ --config_dir path ] -// [ --batch n ] -// the following can be supplied in WU template; see defaults below -// [ --rsc_fpops_est n ] -// [ --rsc_fpops_bound n ] -// [ --rsc_memory_bound n ] -// [ --rsc_disk_bound n ] -// [ --delay_bound x ] -// [ --min_quorum x ] -// [ --target_nresults x ] -// [ --max_error_results x ] -// [ --max_total_results x ] -// [ --max_success_results x ] -// [ --additional_xml x ] -// [ --assign_all ] -// [ --assign_host ID ] -// [ --assign_user_one ID ] -// [ --assign_user_all ID ] -// [ --assign_team_one ID ] -// [ --assign_team_all ID ] -// [ --wu_id N ] Pass this if you've already created the workunit record -// (used by boinc_submit) -// infile1 infile2 ... - #include "config.h" #include #include @@ -69,6 +34,39 @@ #include "sched_config.h" #include "util.h" +void usage() { + fprintf(stderr, + "usage: create_work [options] infile1 infile2 ...\n" + "\n" + "Options:\n" + " --appname name\n" + " [ --wu_name name ] default: generate a name based on app name\n" + " [ --wu_template filename ] default: appname_in\n" + " [ --result_template filename ] default: appname_out\n" + " [ --config_dir path ]\n" + " [ --batch n ]\n" + " [ --rsc_fpops_est n ]\n" + " [ --rsc_fpops_bound n ]\n" + " [ --rsc_memory_bound n ]\n" + " [ --rsc_disk_bound n ]\n" + " [ --delay_bound x ]\n" + " [ --min_quorum x ]\n" + " [ --target_nresults x ]\n" + " [ --max_error_results x ]\n" + " [ --max_total_results x ]\n" + " [ --max_success_results x ]\n" + " [ --additional_xml x ]\n" + " [ --assign_all ]\n" + " [ --assign_host ID ]\n" + " [ --assign_user_one ID ]\n" + " [ --assign_user_all ID ]\n" + " [ --assign_team_one ID ]\n" + " [ --assign_team_all ID ]\n" + " [ --wu_id N ] ID of existing the workunit record (used by boinc_submit)\n" + ); + exit(1); +} + bool arg(const char** argv, int i, const char* name) { char buf[256]; sprintf(buf, "-%s", name); @@ -211,8 +209,7 @@ int main(int argc, const char** argv) { } if (!strlen(app.name)) { - fprintf(stderr, "create_work: missing --appname\n"); - exit(1); + usage(); } if (!strlen(wu.name)) { sprintf(wu.name, "%s_%d_%f", app.name, getpid(), dtime());