- web: more remote job submission code. Not finished.

svn path=/trunk/boinc/; revision=23871
This commit is contained in:
David Anderson 2011-07-25 21:45:53 +00:00
parent 8d602f1bbc
commit 83965db576
9 changed files with 407 additions and 35 deletions

View File

@ -4316,3 +4316,18 @@ Charlie 25 July 11
MacBitmapComboBox.h
lib/
hostinfo.h
David 25 July 2011
- web: more remote job submission code. Not finished.
db/
boinc_db.h
html/
inc/
submit.inc
boinc_db.inc
result.inc
user/
submit_example.php
get_output.php
submit.php

View File

@ -447,10 +447,12 @@ struct CREDITED_JOB {
void clear();
};
// WARNING: be Very careful about changing any values,
// WARNING: be very careful about changing any values,
// especially for a project already running -
// the database will become inconsistent
// values of result.server_state
//
//#define RESULT_SERVER_STATE_INACTIVE 1
#define RESULT_SERVER_STATE_UNSENT 2
#define RESULT_SERVER_STATE_IN_PROGRESS 4
@ -458,6 +460,8 @@ struct CREDITED_JOB {
// we received a reply, timed out, or decided not to send.
// Note: we could get a reply even after timing out.
// values of result.outcome
//
#define RESULT_OUTCOME_INIT 0
#define RESULT_OUTCOME_SUCCESS 1
#define RESULT_OUTCOME_COULDNT_SEND 2
@ -475,6 +479,8 @@ struct CREDITED_JOB {
#define RESULT_OUTCOME_CLIENT_DETACHED 7
// we believe that the client detached
// values of result.validate_state
//
#define VALIDATE_STATE_INIT 0
#define VALIDATE_STATE_VALID 1
#define VALIDATE_STATE_INVALID 2

View File

@ -402,6 +402,14 @@ $math = array(
"Sudoku@vtaiwan seeks to solve the minimum Sudoku problem, which is a well-known problem in mathematics and computer science.",
"sudoku.png"
),
array(
"Surveill@Home",
"http://surveill.dei.uc.pt/surveill",
"University of Coimbra, Portugal",
"Web performance",
tra("Surveill@Home is a research project that conducts end-to-end fine-grained monitoring of web sites. The project will deploy thousands of probes, each of which repeatedly performs transactions on web sites. This approach provides end-user failure and performance statistics."),
"surveill.png"
),
),
);

View File

@ -238,6 +238,10 @@ class BoincResult {
$db = BoincDb::get();
return $db->lookup_id($id, 'result', 'BoincResult');
}
static function lookup_name($name) {
$db = BoincDb::get();
return $db->lookup('result', 'BoincResult', "name='$name'");
}
function delete() {
$db = BoincDb::get();
return $db->delete($this, 'result');

View File

@ -17,6 +17,7 @@
// along with BOINC. If not, see <http://www.gnu.org/licenses/>.
require_once("../inc/translation.inc");
require_once("../inc/dir_hier.inc");
$apps = array();
$app_versions = array();
@ -413,7 +414,7 @@ function exit_status_string($result) {
return sprintf("%d (0x%x)", $x, $x);
}
function show_result($result) {
function show_result($result, $show_outfile_links) {
start_table();
row2(tra("Name"), $result->name);
row2(tra("Workunit"), "<a href=\"workunit.php?wuid=$result->workunitid\">$result->workunitid</a>");
@ -431,6 +432,20 @@ function show_result($result) {
row2(tra("Validate state"), validate_state_str($result));
row2(tra("Credit"), number_format($result->granted_credit, 2));
row2(tra("Application version"), app_version_string($result));
if ($show_outfile_links && $result->outcome == 1) {
$fanout = parse_config(get_config(), "<uldl_dir_fanout>");
$names = get_outfile_names($result);
$i = 0;
$x = "";
foreach ($names as $name) {
if ($i) $x .= " | ";
$url = dir_hier_url($name, "upload", $fanout);
echo $name;
$x .= " <a href=$url>$i</a> ";
$i++;
}
row2(tra("Output files"), $x);
}
end_table();
echo "<h3>".tra("Stderr output")."</h3> <pre>".htmlspecialchars($result->stderr_out)."</pre>";
}
@ -524,6 +539,32 @@ function result_navigation($info, $where_clause) {
return $x;
}
function get_outfile_names($result) {
$names = array();
$xml = "<a>".$result->xml_doc_out."</a>";
$r = simplexml_load_string($xml);
if (!$r) return $names;
foreach ($r->file_info as $fi) {
$names[] = (string)($fi->name);
}
return $names;
}
function get_outfile_paths($result) {
$fanout = parse_config(get_config(), "<uldl_dir_fanout>");
$upload_dir = parse_config(get_config(), "<upload_dir>");
$paths = array();
$xml = "<a>".$result->xml_doc_out."</a>";
$r = simplexml_load_string($xml);
if (!$r) return $names;
foreach ($r->file_info as $fi) {
$path = dir_hier_path((string)($fi->name), $upload_dir, $fanout);
$paths[] = $path;
}
return $paths;
}
$cvs_version_tracker[]="\$Id$"; //Generated automatically - do not edit
?>

View File

@ -68,7 +68,7 @@ function do_http_op($project, $xml) {
$reply = curl_exec($ch);
if (!$reply) return array(null, "HTTP error");
$r = simplexml_load_string($reply);
if (!$r) return array(null, "Can't parse reply XML");
if (!$r) return array(null, "Can't parse reply XML: <pre>".htmlentities($reply)."</pre>");
return array($r, null);
}
@ -121,17 +121,15 @@ function boinc_query_batches($req) {
$b->completed = (int)($batch->completed);
$b->njobs = (int)($batch->njobs);
$b->create_time = (double)($batch->create_time);
if (!$batch->completed) {
if (!$b->completed) {
$b->fraction_done = (double) $batch->fraction_done;
} else {
$b->completed_time = (double)($batch->completed_time);
}
$batches[] = $b;
}
return array($batches, null);
}
function boinc_query_batch() {
function boinc_query_batch($req) {
$req_xml = "<query_batch>
<authenticator>$req->authenticator</authenticator>
<batch_id>$req->batch_id</batch_id>
@ -143,6 +141,7 @@ function boinc_query_batch() {
foreach ($reply->job as $job) {
$j = null;
$j->id = (int)($job->id);
$j->canonical_instance_id = (int)($job->canonical_instance_id);
$jobs[] = $j;
}
$r = null;
@ -150,18 +149,85 @@ function boinc_query_batch() {
return array($r, null);
}
function boinc_abort_batch() {
function boinc_query_job($req) {
$req_xml = "<query_job>
<authenticator>$req->authenticator</authenticator>
<job_id>$req->job_id</job_id>
</query_job>
";
list($reply, $errmsg) = do_http_op($req->project, $req_xml);
if ($errmsg) return array(null, $errmsg);
$instances = array();
foreach ($reply->instance as $instance) {
$i = null;
$i->name = (string)($instance->name);
$i->id = (int)($instance->id);
$i->state = (string)($instance->state);
$i->outfiles = array();
foreach ($instance->outfile as $outfile) {
$f = null;
$f->size = (double)$outfile->size;
$i->outfiles[] = $f;
}
$instances[] = $i;
}
$r = null;
$r->instances = $instances;
return array($r, null);
}
function boinc_abort_batch($req) {
$req_xml = "<abort_batch>
<authenticator>$req->authenticator</authenticator>
<batch_id>$req->batch_id</batch_id>
</abort_batch>
";
list($reply, $errmsg) = do_http_op($req->project, $req_xml);
if ($errmsg) return $errmsg;
$name = $reply->getName();
if ($name == 'error') {
return (string)($reply->message);
}
return null;
}
function boinc_get_output_file($req) {
$auth_str = md5($req->authenticator.$req->instance_name);
$name = $req->instance_name;
$file_num = $req->file_num;
return $req->project."/get_output.php?instance_name=$name&file_num=$file_num&auth_str=$auth_str";
}
function boinc_get_output_files($req) {
$auth_str = md5($req->authenticator.$req->batch_id);
$batch_id = $req->batch_id;
return $req->project."/get_output.php?batch_id=$batch_id&auth_str=$auth_str";
}
function boinc_cleanup_batch($req) {
$req_xml = "<cleanup_batch>
<authenticator>$req->authenticator</authenticator>
<batch_id>$req->batch_id</batch_id>
</cleanup_batch>
";
list($reply, $errmsg) = do_http_op($req->project, $req_xml);
if ($errmsg) return $errmsg;
$name = $reply->getName();
if ($name == 'error') {
return (string)($reply->message);
}
return null;
}
//// example usage follows
$req->project = "http://isaac.ssl.berkeley.edu/test/";
$req->authenticator = "x";
if (0) {
$req->project = "http://foo.edu/test/";
$req->authenticator = "xxx";
$req->app_name = "uppercase";
$req->jobs = array();
$f->source = "http://foo.edu/index.php";
$f->source = "http://isaac.ssl.berkeley.edu/index.php";
$f->name = "in";
$job->input_files = array($f);
@ -189,8 +255,6 @@ if (0) {
}
if (0) {
$req->project = "http://foo.edu/test/";
$req->authenticator = "xxx";
list($batches, $errmsg) = boinc_query_batches($req);
if ($errmsg) {
echo "Error: $errmsg\n";
@ -199,4 +263,15 @@ if (0) {
}
}
if (0) {
$req->batch_id = 20;
list($jobs, $errmsg) = boinc_query_batch($req);
if ($errmsg) {
echo "Error: $errmsg\n";
} else {
print_r($jobs);
}
}
?>

99
html/user/get_output.php Normal file
View File

@ -0,0 +1,99 @@
<?php
// This file is part of BOINC.
// http://boinc.berkeley.edu
// Copyright (C) 2011 University of California
//
// BOINC is free software; you can redistribute it and/or modify it
// under the terms of the GNU Lesser General Public License
// as published by the Free Software Foundation,
// either version 3 of the License, or (at your option) any later version.
//
// BOINC is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
// See the GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with BOINC. If not, see <http://www.gnu.org/licenses/>.
// handler for output file requests from remote job submission.
// See http://boinc.berkeley.edu/trac/wiki/RemoteJobs
require_once("../inc/util.inc");
require_once("../inc/result.inc");
require_once("../inc/submit_db.inc");
// get a single output file
//
function get_output_file($instance_name, $file_num, $auth_str) {
$result = BoincResult::lookup_name(BoincDb::escape_string($instance_name));
if (!$result) die("no job instance $instance_name");
$workunit = BoincWorkunit::lookup_id($result->workunitid);
if (!$workunit) die("no job $result->workunitid");
$batch = BoincBatch::lookup_id($workunit->batch);
if (!$batch) die("no batch $workunit->batch");
$user = BoincUser::lookup_id($batch->user_id);
if (!$user) die("no user $batch->user_id");
$x = md5($user->authenticator.$result->name);
if ($x != $auth_str) die("bad auth str");
$names = get_outfile_names($result);
if ($file_num >= count($names)) {
die("bad file num: $file_num > ".count($names));
}
$name = $names[$file_num];
$fanout = parse_config(get_config(), "<uldl_dir_fanout>");
$upload_dir = parse_config(get_config(), "<upload_dir>");
$path = dir_hier_path($name, $upload_dir, $fanout);
if (!is_file($path)) die("no such file $path");
readfile($path);
}
// get all the output files of a batch (canonical instances only)
// and make a zip of all of them
//
function get_output_files($batch_id, $auth_str) {
$batch = BoincBatch::lookup_id($batch_id);
if (!$batch) die("no batch $batch_id");
$user = BoincUser::lookup_id($batch->user_id);
if (!$user) die("no user $batch->user_id");
$x = md5($user->authenticator.$result->name);
if ($x != $auth_str) die("bad auth str");
$zip_filename = tempnam("/tmp", "boinc_batch_");
$fanout = parse_config(get_config(), "<uldl_dir_fanout>");
$download_dir = parse_config(get_config(), "<download_dir>");
$wus = BoincWorkunit::enum("batch=$batch_id");
foreach ($wus as $wu) {
if (!$wu->canonical_resultid) continue;
$result = BoincResult::lookup_id($wu->canonical_resultid);
$names = get_outfile_names($result);
foreach ($names as $name) {
$path = dir_hier_path($name, $upload_dir, $fanout);
if (is_file($path)) {
system("zip -D $zip_filename $path");
}
}
}
readfile($zip_filename);
unlink($zip_filename);
}
get_output_file("batch_23_2_0", 0, "65748bfc28e6c605fcf774f36bbd9842");
exit;
$auth_str = get_str('auth_str');
$instance_name = get_str('instance_name', true);
if ($instance_name) {
$file_num = get_int('file_num');
get_output_file($instance_name, $file_num, $auth_str);
} else {
$batch_id = get_int('batch_id');
get_output_files($batch_id, $auth_str);
}
?>

View File

@ -23,6 +23,11 @@ require_once("../inc/boinc_db.inc");
require_once("../inc/submit_db.inc");
require_once("../inc/xml.inc");
require_once("../inc/dir_hier.inc");
require_once("../inc/result.inc");
error_reporting(E_ALL);
ini_set('display_errors', true);
ini_set('display_startup_errors', true);
function error($s) {
echo "<error>\n<message>$s</message>\n</error>\n";
@ -104,7 +109,7 @@ function stage_file($file) {
$path = dir_hier_path($name, "../../download", $fanout);
if (file_exists($path)) return;
if (!copy($source, $path)) {
error("can't copy file from $source");
error("can't copy file from $source to $path");
}
}
@ -151,14 +156,18 @@ function fraction_done($batch) {
$wus = BoincWorkunit::enum("batch = $batch->id");
$fp_total = 0;
$fp_done = 0;
$completed = 1;
foreach ($wus as $wu) {
$fp_total += $wu->fpops_est;
$fp_total += $wu->rsc_fpops_est;
if ($wu->canonical_resultid) {
$fp_done += $wu->fpops_est;
$fp_done += $wu->rsc_fpops_est;
} else {
$completed = 0;
}
}
if (!$fp_total) return 1;
return $fp_done / $fp_total;
if (!$fp_total) return array(1, true);;
$fd= $fp_done / $fp_total;
return array($fd, $completed);
}
function query_batches($r) {
@ -166,10 +175,11 @@ function query_batches($r) {
$batches = BoincBatch::enum("user_id = $user->id");
echo "<batches>\n";
foreach ($batches as $b) {
$fd = fraction_done($b);
list($fd, $completed) = fraction_done($b);
echo " <batch>
<id>$b->id</id>
<fraction_done>$fd</fraction_done>
<completed>$completed</completed>
<create_time>$b->create_time</create_time>
<est_completion_time>$b->est_completion_time</est_completion_time>
<njobs>$b->njobs</njobs>
@ -179,6 +189,12 @@ function query_batches($r) {
echo "</batches>\n";
}
function n_outfiles($wu) {
$path = "../../$wu->result_template_file";
$r = simplexml_load_file($path);
return count($r->file_info);
}
function query_batch($r) {
list($user, $user_submit) = authenticate_user($r, null);
$batch_id = (int)($r->batch_id);
@ -191,13 +207,74 @@ function query_batch($r) {
foreach ($wus as $wu) {
echo " <job>
<id>$wu->id</id>
</job>
<canonical_instance_id>$wu->canonical_resultid</canonical_instance_id>
<n_outfiles>".n_outfiles($wu)."</n_outfiles>
</job>
";
}
echo "</batch>\n";
}
function query_job($r) {
list($user, $user_submit) = authenticate_user($r, null);
$job_id = (int)($r->job_id);
$wu = BoincWorkunit::lookup_id($job_id);
if (!$wu) error("no such job");
$batch = BoincBatch::lookup_id($wu->batch);
if ($batch->user_id != $user->id) {
error("not owner");
}
echo "<job>\n";
$results = BoincResult::enum("workunitid=$job_id");
foreach ($results as $result) {
echo " <instance>
<name>$result->name</name>
<id>$result->id</id>
<state>".state_string($result)."</state>
";
if ($result->server_state == 5) { // over?
$paths = get_outfile_paths($result);
foreach($paths as $path) {
if (is_file($path)) {
$size = filesize($path);
echo " <outfile>
<size>$size</size>
</outfile>
";
}
}
}
echo "</instance>\n";
}
echo "</job>\n";
}
function abort_batch($r) {
list($user, $user_submit) = authenticate_user($r, null);
$batch_id = (int)($r->batch_id);
$batch = BoincBatch::lookup_id($batch_id);
if ($batch->user_id != $user->id) {
error("not owner");
}
}
function cleanup_batch($r) {
list($user, $user_submit) = authenticate_user($r, null);
$batch_id = (int)($r->batch_id);
$batch = BoincBatch::lookup_id($batch_id);
if ($batch->user_id != $user->id) {
error("not owner");
}
}
if (0) {
$r = simplexml_load_string(
"<query_job>
<authenticator>x</authenticator>
<job_id>305613</job_id>
</query_job>");
query_job($r);
exit;
}
xml_header();
@ -210,9 +287,11 @@ if (!$r) {
switch ($r->getName()) {
case 'estimate_batch': estimate_batch($r); break;
case 'submit_batch': submit_batch($r); break;
case 'query_batch': query_batch($r); break;
case 'query_batches': query_batches($r); break;
case 'query_batch': query_batch($r); break;
case 'query_job': query_job($r); break;
case 'abort_batch': abort_batch($r); break;
case 'cleanup_batch': cleanup_batch($r); break;
default: error("bad command");
}

View File

@ -32,8 +32,12 @@
require_once("../inc/submit.inc");
require_once("../inc/util.inc");
$project = "http://foo.edu/test/";
$auth = "xxx";
error_reporting(E_ALL);
ini_set('display_errors', true);
ini_set('display_startup_errors', true);
$project = "http://isaac.ssl.berkeley.edu/test/";
$auth = "x";
$app_name = "uppercase";
function handle_main() {
@ -43,7 +47,11 @@ function handle_main() {
list($batches, $errmsg) = boinc_query_batches($req);
if ($errmsg) error_page($errmsg);
page_head("Remote job interface");
page_head("Job submission and control");
echo "
<a href=submit_example.php?action=create_form><b>Create new batch</b></a>
";
echo "<h2>Batches in progress</h2>\n";
start_table();
@ -52,7 +60,7 @@ function handle_main() {
if ($batch->completed) continue;
$pct_done = (int)($batch->fraction_done*100);
table_row(
"<a href=submit_remote.php?action=query_batch&batch_id=$batch->id>$batch->id</a>",
"<a href=submit_example.php?action=query_batch&batch_id=$batch->id>$batch->id</a>",
$batch->njobs,
"$pct_done%",
time_str($batch->create_time)
@ -62,21 +70,17 @@ function handle_main() {
echo "<h2>Batches completed</h2>\n";
start_table();
table_header("ID", "# jobs", "submitted", "completed");
table_header("ID", "# jobs", "submitted");
foreach ($batches as $batch) {
if (!$batch->completed) continue;
table_row(
"<a href=submit_remote.php?action=query_batch&batch_id=$batch->id>$batch->id</a>",
"<a href=submit_example.php?action=query_batch&batch_id=$batch->id>$batch->id</a>",
$batch->njobs,
time_str($batch->submitted_time),
time_str($batch->completed_time)
time_str($batch->create_time)
);
}
end_table();
echo "
<a href=submit_remote.php?action=create_form>Create new batch</a>
";
page_tail();
}
@ -85,7 +89,7 @@ function handle_create_form() {
page_head("Create batch");
echo "
<form action=submit_remote.php>
<form action=submit_example.php>
<input type=hidden name=action value=create_action>
";
start_table();
@ -167,11 +171,18 @@ function handle_query_batch() {
page_head("Batch $req->batch_id");
start_table();
table_header("ID");
table_header("Job ID", "Canonical instance");
foreach($reply->jobs as $job) {
$id = (int)$job->id;
$resultid = (int)$job->canonical_instance_id;
if ($resultid) {
$x = "<a href=result.php?resultid=$resultid>$resultid</a>";
} else {
$x = "---";
}
echo "<tr>
<td><a href=workunit.php?wuid=$id>$id</a></td>
<td><a href=submit_example.php?action=query_job&job_id=$id>$id</a></td>
<td>$x</td>
</tr>
";
}
@ -179,6 +190,37 @@ function handle_query_batch() {
page_tail();
}
function handle_query_job() {
global $project, $auth, $app_name;
$req->project = $project;
$req->authenticator = $auth;
$req->job_id = get_int('job_id');
list($reply, $errmsg) = boinc_query_job($req);
if ($errmsg) error_page($errmsg);
page_head("Job $req->job_id");
start_table();
table_header("Instance ID", "State", "Output files");
foreach($reply->instances as $inst) {
echo "<tr>
<td><a href=result.php?resultid=$inst->id>$inst->id</a></td>
<td>$inst->state</td>
<td>
";
$i = 0;
foreach ($inst->outfiles as $outfile) {
$req->instance_name = $inst->name;
$req->file_num = $i;
$url = boinc_get_output_file($req);
echo "<a href=$url>$outfile->size bytes</a>";
$i++;
}
echo "</td></tr>\n";
}
end_table();
page_tail();
}
$action = get_str('action', true);
switch ($action) {
@ -194,6 +236,9 @@ case 'create_action':
case 'query_batch':
handle_query_batch();
break;
case 'query_job':
handle_query_job();
break;
default:
error_page('no such action');
}