Rooba
/
boinc
mirror of https://github.com/BOINC/boinc.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

- server: some remote job submission code. Not finished. 

svn path=/trunk/boinc/; revision=23868
This commit is contained in:
David Anderson 2011-07-22 22:47:41 +00:00
parent 880d01af58
commit 5ad821df21
7 changed files with 512 additions and 109 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
checkin_notes
View File

@ -4291,3 +4291,15 @@ David 20 July 2011
client_types.cpp,h
cs_statefile.cpp
scheduler_op.cpp
David 22 July 2011
- server: some remote job submission code. Not finished.
tools/
create_work.cpp
html/
inc/
submit.inc
user/
submit.php
submit_example.php

2
doc/download_all.php
View File

@ -117,7 +117,7 @@ function show_platform($short_name, $p, $dev) {
global $max_version;
$long_name = $p["name"];
$description = $p["description"];
if ($p["url"]) {
if (array_key_exists('url', $p)) {
$url = $p["url"];
$long_name .= " <a href=$url><span class=description>details</span></a>";
}

202
html/inc/submit.inc Normal file
View File

@ -0,0 +1,202 @@
<?php
// This file is part of BOINC.
// http://boinc.berkeley.edu
// Copyright (C) 2011 University of California
//
// BOINC is free software; you can redistribute it and/or modify it
// under the terms of the GNU Lesser General Public License
// as published by the Free Software Foundation,
// either version 3 of the License, or (at your option) any later version.
//
// BOINC is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
// See the GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with BOINC. If not, see <http://www.gnu.org/licenses/>.
// remote job submission API
//// Implementation stuff follows
function req_to_xml($req, $op) {
$x = "<$op>
<authenticator>$req->authenticator</authenticator>
<batch>
<app_name>$req->app_name</app_name>
";
foreach ($req->jobs as $job) {
$x .= " <job>
<rsc_fpops_est>$job->rsc_fpops_est</rsc_fpops_est>
<command_line>$job->command_line</command_line>
";
foreach ($job->input_files as $file) {
$x .= " <input_file>
<source>$file->source</source>
<name>$file->name</name>
</input_file>
";
}
$x .= " </job>
";
}
$x .= " </batch>
</$op>
";
return $x;
}
function validate_request($req) {
if (!is_object($req)) return "req is not an object";
if (!array_key_exists('project', $req)) return "missing req->project";
if (!array_key_exists('authenticator', $req)) return "missing req->authenticator";
if (!array_key_exists('app_name', $req)) return "missing req->app_name";
if (!array_key_exists('jobs', $req)) return "missing req->jobs";
if (!is_array($req->jobs)) return "req->jobs is not an array";
foreach ($req->jobs as $job) {
// other checks
}
return null;
}
function do_http_op($project, $xml) {
$ch = curl_init("$project/submit.php");
curl_setopt($ch, CURLOPT_POST, 1);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_POSTFIELDS, "request=$xml");
$reply = curl_exec($ch);
if (!$reply) return array(null, "HTTP error");
$r = simplexml_load_string($reply);
if (!$r) return array(null, "Can't parse reply XML");
return array($r, null);
}
function do_batch_op($req, $op) {
$retval = validate_request($req);
if ($retval) return array(null, $retval);
$xml = req_to_xml($req, $op);
return do_http_op($req->project, $xml);
}
//// Interface functions follow
function boinc_estimate_batch($req) {
list($reply, $errmsg) = do_batch_op($req, "estimate_batch");
if ($errmsg) return array(0, $errmsg);
$name = $reply->getName();
if ($name == 'estimate') {
return array((string)$reply->seconds, null);
} else if ($name = 'error') {
return array(null, (string)$reply->message);
} else {
return array(null, "Bad reply message");
}
}
function boinc_submit_batch($req) {
list($reply, $errmsg) = do_batch_op($req, "submit_batch");
if ($errmsg) return array(0, $errmsg);
$name = $reply->getName();
if ($name == 'batch_id') {
return array((int)$reply, null);
} else if ($name == 'error') {
return array(null, (string)$reply->message);
} else {
return array(null, "Bad reply message");
}
}
function boinc_query_batches($req) {
$req_xml = "<query_batches>
<authenticator>$req->authenticator</authenticator>
</query_batches>
";
list($reply, $errmsg) = do_http_op($req->project, $req_xml);
if ($errmsg) return array(null, $errmsg);
$batches = array();
foreach ($reply->batch as $batch) {
$b = null;
$b->id = (int)($batch->id);
$b->completed = (int)($batch->completed);
$b->njobs = (int)($batch->njobs);
$b->create_time = (double)($batch->create_time);
if (!$batch->completed) {
$b->fraction_done = (double) $batch->fraction_done;
} else {
$b->completed_time = (double)($batch->completed_time);
}
$batches[] = $b;
}
return array($batches, null);
}
function boinc_query_batch() {
$req_xml = "<query_batch>
<authenticator>$req->authenticator</authenticator>
<batch_id>$req->batch_id</batch_id>
</query_batch>
";
list($reply, $errmsg) = do_http_op($req->project, $req_xml);
if ($errmsg) return array(null, $errmsg);
$jobs = array();
foreach ($reply->job as $job) {
$j = null;
$j->id = (int)($job->id);
$jobs[] = $j;
}
$r = null;
$r->jobs = $jobs;
return array($r, null);
}
function boinc_abort_batch() {
}
//// example usage follows
if (0) {
$req->project = "http://foo.edu/test/";
$req->authenticator = "xxx";
$req->app_name = "uppercase";
$req->jobs = array();
$f->source = "http://foo.edu/index.php";
$f->name = "in";
$job->input_files = array($f);
for ($i=10; $i<20; $i++) {
$job->rsc_fpops_est = $i*1e9;
$job->command_line = "--t $i";
$req->jobs[] = $job;
}
if (0) {
list($e, $errmsg) = boinc_estimate_batch($req);
if ($errmsg) {
echo "Error from server: $errmsg\n";
} else {
echo "Batch completion estimate: $e seconds\n";
}
} else {
list($id, $errmsg) = boinc_submit_batch($req);
if ($errmsg) {
echo "Error from server: $errmsg\n";
} else {
echo "Batch ID: $id\n";
}
}
}
if (0) {
$req->project = "http://foo.edu/test/";
$req->authenticator = "xxx";
list($batches, $errmsg) = boinc_query_batches($req);
if ($errmsg) {
echo "Error: $errmsg\n";
} else {
print_r($batches);
}
}
?>

57
html/ops/submit_example.php
View File

@ -1,57 +0,0 @@
<?php
// This file is part of BOINC.
// http://boinc.berkeley.edu
// Copyright (C) 2011 University of California
//
// BOINC is free software; you can redistribute it and/or modify it
// under the terms of the GNU Lesser General Public License
// as published by the Free Software Foundation,
// either version 3 of the License, or (at your option) any later version.
//
// BOINC is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
// See the GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with BOINC. If not, see <http://www.gnu.org/licenses/>.
// example code for making a remote job submission
$ch = curl_init("http://foo.edu/test/submit.php");
curl_setopt($ch, CURLOPT_POST, 1);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_POSTFIELDS, "request=
<batch_submit>
<authenticator>xxx</authenticator>
<batch>
<app_name>uppercase</app_name>
<job>
<rsc_fpops_est>100e9</rsc_fpops_est>
<command_line>--t ALPHA</command_line>
<input_file>
<source>http://foo.edu/index.php</source>
<physical_name>name</physical_name>
</input_file>
</job>
</batch>
</batch_submit>
");
$reply = curl_exec($ch);
if (!$reply) die("HTTP op failed\n");
echo "reply: $reply\n"; exit;
$r = simplexml_load_string($reply);
if (!$r) die("bad reply: $reply\n");
$name = $r->getName();
if ($name == 'estimate') {
echo "estimate: ".(string)$r->seconds."\n";
} else if ($name == 'error') {
foreach ($r->message as $msg) {
echo "$msg\n";
}
} else {
die("bad reply\n");
}
?>

74
html/user/submit.php
View File

@ -36,7 +36,7 @@ function authenticate_user($r, $app) {
if (!$user) error("bad authenticator");
$user_submit = BoincUserSubmit::lookup_userid($user->id);
if (!$user_submit) error("no submit access");
if (!$user_submit->all_apps) {
if ($app && !$user_submit->all_apps) {
$usa = BoincUserSubmitApp::lookup("user_id=$user->id and app_id=$app->id");
if (!$usa) {
error("no submit access");
@ -68,7 +68,7 @@ function project_flops() {
return $y;
}
function batch_estimate($r) {
function estimate_batch($r) {
$app = get_app($r);
list($user, $user_submit) = authenticate_user($r, $app);
@ -99,9 +99,9 @@ $fanout = parse_config(get_config(), "<uldl_dir_fanout>");
function stage_file($file) {
global $fanout;
$physical_name = (string)$file->physical_name;
$name = (string)$file->name;
$source = (string)$file->source;
$path = dir_hier_path($physical_name, "../../download", $fanout);
$path = dir_hier_path($name, "../../download", $fanout);
if (file_exists($path)) return;
if (!copy($source, $path)) {
error("can't copy file from $source");
@ -120,14 +120,16 @@ function submit_job($job, $template, $app, $batch_id, $i) {
$cmd = "cd ../..; ./bin/create_work --appname $app->name --batch $batch_id";
$cmd .= " --wu_name batch_".$batch_id."_".$i;
foreach ($job->input_file as $file) {
$name = (string)$file->physical_name;
$name = (string)$file->name;
$cmd .= " $name";
system($cmd);
}
$ret = system($cmd);
if ($ret === FALSE) {
error("can't create job");
}
}
function batch_submit($r) {
function submit_batch($r) {
$app = get_app($r);
list($user, $user_submit) = authenticate_user($r, $app);
$template = read_input_template($app);
@ -135,7 +137,9 @@ function batch_submit($r) {
stage_files($r, $template);
$njobs = count($r->batch->job);
$now = time();
$batch_id = BoincBatch::insert("(user_id, create_time, njobs) values ($user->id, $now, $njobs)");
$batch_id = BoincBatch::insert(
"(user_id, create_time, njobs) values ($user->id, $now, $njobs)"
);
$i = 0;
foreach($r->batch->job as $job) {
submit_job($job, $template, $app, $batch_id, $i++);
@ -143,10 +147,54 @@ function batch_submit($r) {
echo "<batch_id>$batch_id</batch_id>\n";
}
function query_job($r) {
function fraction_done($batch) {
$wus = BoincWorkunit::enum("batch = $batch->id");
$fp_total = 0;
$fp_done = 0;
foreach ($wus as $wu) {
$fp_total += $wu->fpops_est;
if ($wu->canonical_resultid) {
$fp_done += $wu->fpops_est;
}
}
if (!$fp_total) return 1;
return $fp_done / $fp_total;
}
function query_batches($r) {
list($user, $user_submit) = authenticate_user($r, null);
$batches = BoincBatch::enum("user_id = $user->id");
echo "<batches>\n";
foreach ($batches as $b) {
$fd = fraction_done($b);
echo " <batch>
<id>$b->id</id>
<fraction_done>$fd</fraction_done>
<create_time>$b->create_time</create_time>
<est_completion_time>$b->est_completion_time</est_completion_time>
<njobs>$b->njobs</njobs>
</batch>
";
}
echo "</batches>\n";
}
function query_batch($r) {
list($user, $user_submit) = authenticate_user($r, null);
$batch_id = (int)($r->batch_id);
$batch = BoincBatch::lookup_id($batch_id);
if ($batch->user_id != $user->id) {
error("not owner");
}
echo "<batch>\n";
$wus = BoincWorkunit::enum("batch = $batch_id");
foreach ($wus as $wu) {
echo " <job>
<id>$wu->id</id>
</job>
";
}
echo "</batch>\n";
}
function abort_batch($r) {
@ -160,10 +208,10 @@ if (!$r) {
}
switch ($r->getName()) {
case 'batch_estimate': batch_estimate($r); break;
case 'batch_submit': batch_submit($r); break;
case 'query_job': query_job($r); break;
case 'estimate_batch': estimate_batch($r); break;
case 'submit_batch': submit_batch($r); break;
case 'query_batch': query_batch($r); break;
case 'query_batches': query_batches($r); break;
case 'abort_batch': abort_batch($r); break;
default: error("bad command");
}

201
html/user/submit_example.php Normal file
View File

@ -0,0 +1,201 @@
<?php
// This file is part of BOINC.
// http://boinc.berkeley.edu
// Copyright (C) 2011 University of California
//
// BOINC is free software; you can redistribute it and/or modify it
// under the terms of the GNU Lesser General Public License
// as published by the Free Software Foundation,
// either version 3 of the License, or (at your option) any later version.
//
// BOINC is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
// See the GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with BOINC. If not, see <http://www.gnu.org/licenses/>.
// example of a web interface to remote job submission
//
// Notes:
// - You'll need to adapt/extend this considerably;
// e.g. the project URL, app name and user authenticator are hardwired here.
// - This can run on any host, not just the project server
// (that's the "remote" part).
// - For convenience, this uses some functions from BOINC
// (page_head() etc.).
// When you adapt this to your own purposes,
// you can strip out this stuff if the web site doesn't use BOINC
require_once("../inc/submit.inc");
require_once("../inc/util.inc");
$project = "http://foo.edu/test/";
$auth = "xxx";
$app_name = "uppercase";
function handle_main() {
global $project, $auth, $app_name;
$req->project = $project;
$req->authenticator = $auth;
list($batches, $errmsg) = boinc_query_batches($req);
if ($errmsg) error_page($errmsg);
page_head("Remote job interface");
echo "<h2>Batches in progress</h2>\n";
start_table();
table_header("ID", "# jobs", "progress", "submitted");
foreach ($batches as $batch) {
if ($batch->completed) continue;
$pct_done = (int)($batch->fraction_done*100);
table_row(
"<a href=submit_remote.php?action=query_batch&batch_id=$batch->id>$batch->id</a>",
$batch->njobs,
"$pct_done%",
time_str($batch->create_time)
);
}
end_table();
echo "<h2>Batches completed</h2>\n";
start_table();
table_header("ID", "# jobs", "submitted", "completed");
foreach ($batches as $batch) {
if (!$batch->completed) continue;
table_row(
"<a href=submit_remote.php?action=query_batch&batch_id=$batch->id>$batch->id</a>",
$batch->njobs,
time_str($batch->submitted_time),
time_str($batch->completed_time)
);
}
end_table();
echo "
<a href=submit_remote.php?action=create_form>Create new batch</a>
";
page_tail();
}
function handle_create_form() {
global $project, $auth, $app_name;
page_head("Create batch");
echo "
<form action=submit_remote.php>
<input type=hidden name=action value=create_action>
";
start_table();
row2("Input file URL", "<input name=input_url size=60>");
row2("Parameter low value", "<input name=param_lo>");
row2("Parameter high value", "<input name=param_hi>");
row2("Parameter increment", "<input name=param_inc>");
row2("",
"<input type=submit name=get_estimate value=\"Get completion time estimate\">"
);
row2("",
"<input type=submit name=submit value=Submit>"
);
end_table();
echo "</form>\n";
page_tail();
}
// build a request object for boinc_*_batch() from form variables
//
function form_to_request() {
global $project, $auth, $app_name;
$input_url = get_str('input_url');
if (!$input_url) error_page("missing input URL");
$param_lo = get_str('param_lo');
if (!$param_lo) error_page("missing param lo");
$param_hi = get_str('param_hi');
if (!$param_hi) error_page("missing param hi");
$param_inc = get_str('param_inc');
if (!$param_inc) error_page("missing param inc");
$req->project = $project;
$req->authenticator = $auth;
$req->app_name = $app_name;
$req->jobs = Array();
$f->source = $input_url;
$f->name = "in";
$job->input_files = Array($f);
for ($x=(double)$param_lo; $x<(double)$param_hi; $x += (double)$param_inc) {
$job->rsc_fpops_est = $x*1e9;
$job->command_line = "--t $x";
$req->jobs[] = $job;
}
return $req;
}
function handle_create_action() {
global $project, $auth, $app_name;
$get_estimate = get_str('get_estimate', true);
if ($get_estimate) {
$req = form_to_request($project, $auth);
list($e, $errmsg) = boinc_estimate_batch($req);
if ($errmsg) error_page($errmsg);
page_head("Batch estimate");
echo "Estimate: $e seconds";
page_tail();
} else {
$req = form_to_request($project, $auth);
list($id, $errmsg) = boinc_submit_batch($req);
if ($errmsg) error_page($errmsg);
page_head("Batch submitted");
echo "Batch ID: $id";
page_tail();
}
}
function handle_query_batch() {
global $project, $auth, $app_name;
$req->project = $project;
$req->authenticator = $auth;
$req->batch_id = get_int('batch_id');
list($reply, $errmsg) = boinc_query_batch($req);
if ($errmsg) error_page($errmsg);
page_head("Batch $req->batch_id");
start_table();
table_header("ID");
foreach($reply->jobs as $job) {
$id = (int)$job->id;
echo "<tr>
<td><a href=workunit.php?wuid=$id>$id</a></td>
</tr>
";
}
end_table();
page_tail();
}
$action = get_str('action', true);
switch ($action) {
case '':
handle_main();
break;
case 'create_form':
handle_create_form();
break;
case 'create_action':
handle_create_action();
break;
case 'query_batch':
handle_query_batch();
break;
default:
error_page('no such action');
}
?>

73
tools/create_work.cpp
View File

@ -18,43 +18,8 @@
// Create a workunit.
// Input files must be in the download dir.
// See the docs for a description of WU and result template files
// This program must be run in the project's root directory,
// and there must be a valid config.xml file there
// This program must be run in the project's root directory
//
// create_work
// --appname name
// [ --wu_name name ]
// // default: generate a name based on app name
// [ --wu_template filename ]
// relative to project root; usually in templates/
// default: appname_in
// [ --result_template filename ]
// relative to project root; usually in templates/
// default: appname_out
// [ --config_dir path ]
// [ --batch n ]
// the following can be supplied in WU template; see defaults below
// [ --rsc_fpops_est n ]
// [ --rsc_fpops_bound n ]
// [ --rsc_memory_bound n ]
// [ --rsc_disk_bound n ]
// [ --delay_bound x ]
// [ --min_quorum x ]
// [ --target_nresults x ]
// [ --max_error_results x ]
// [ --max_total_results x ]
// [ --max_success_results x ]
// [ --additional_xml x ]
// [ --assign_all ]
// [ --assign_host ID ]
// [ --assign_user_one ID ]
// [ --assign_user_all ID ]
// [ --assign_team_one ID ]
// [ --assign_team_all ID ]
// [ --wu_id N ] Pass this if you've already created the workunit record
// (used by boinc_submit)
// infile1 infile2 ...
#include "config.h"
#include <cstdio>
#include <cstdlib>
@ -69,6 +34,39 @@
#include "sched_config.h"
#include "util.h"
void usage() {
fprintf(stderr,
"usage: create_work [options] infile1 infile2 ...\n"
"\n"
"Options:\n"
" --appname name\n"
" [ --wu_name name ] default: generate a name based on app name\n"
" [ --wu_template filename ] default: appname_in\n"
" [ --result_template filename ] default: appname_out\n"
" [ --config_dir path ]\n"
" [ --batch n ]\n"
" [ --rsc_fpops_est n ]\n"
" [ --rsc_fpops_bound n ]\n"
" [ --rsc_memory_bound n ]\n"
" [ --rsc_disk_bound n ]\n"
" [ --delay_bound x ]\n"
" [ --min_quorum x ]\n"
" [ --target_nresults x ]\n"
" [ --max_error_results x ]\n"
" [ --max_total_results x ]\n"
" [ --max_success_results x ]\n"
" [ --additional_xml x ]\n"
" [ --assign_all ]\n"
" [ --assign_host ID ]\n"
" [ --assign_user_one ID ]\n"
" [ --assign_user_all ID ]\n"
" [ --assign_team_one ID ]\n"
" [ --assign_team_all ID ]\n"
" [ --wu_id N ] ID of existing the workunit record (used by boinc_submit)\n"
);
exit(1);
}
bool arg(const char** argv, int i, const char* name) {
char buf[256];
sprintf(buf, "-%s", name);
@ -211,8 +209,7 @@ int main(int argc, const char** argv) {
}
if (!strlen(app.name)) {
fprintf(stderr, "create_work: missing --appname\n");
exit(1);
usage();
}
if (!strlen(wu.name)) {
sprintf(wu.name, "%s_%d_%f", app.name, getpid(), dtime());