From 01b78c714a154e792275d7ac9e11453b10304fb7 Mon Sep 17 00:00:00 2001 From: David Anderson Date: Thu, 16 Jan 2014 10:24:10 -0800 Subject: [PATCH] Remote job submission: allow efficient batch query The batch query call used by Condor (query_batch_set(), in the C++ API) returned info about all the jobs in the set of batches, even those that hadn't changed. This is potentially inefficient - a query might return info about 10,000 jobs, only a few (or none) of which have changed state since the last call. Solution: add a "min_mod_time" parameter to the call. Only jobs that have changed state since that time are reported. Also, add a "server_time" field to the return, giving the current time on the server (in case there's clock skew between client and server) Also, fix some text scrambling introduced in previous checkin; there must have been a gremlin in my vim. --- html/inc/util.inc | 4 ++++ html/user/submit_rpc_handler.php | 25 +++++++++++++++++-------- lib/remote_submit.cpp | 5 +++++ lib/remote_submit.h | 4 +++- samples/condor/boinc_gahp.cpp | 8 ++++++-- samples/condor/request_gen | 2 +- 6 files changed, 36 insertions(+), 12 deletions(-) diff --git a/html/inc/util.inc b/html/inc/util.inc index 110be82646..5cd9a8273d 100644 --- a/html/inc/util.inc +++ b/html/inc/util.inc @@ -941,6 +941,10 @@ function show_badges($is_user, $item) { row2("Badges", $x); } +function dtime() { + return microtime(true); +} + $cvs_version_tracker[]="\$Id$"; //Generated automatically - do not edit ?> diff --git a/html/user/submit_rpc_handler.php b/html/user/submit_rpc_handler.php index 99ce8c5f34..7c717b00d1 100644 --- a/html/user/submit_rpc_handler.php +++ b/html/user/submit_rpc_handler.php @@ -89,7 +89,7 @@ function check_max_jobs_in_progress($r, $user_submit) { $n = $db->get_int($query); if ($n === false) return; if ($n + count($r->batch->job) > $user_submit->max_jobs_in_progress) { - xml_error(,1 "BOINC server: limit on jobs in progress exceeded"); + xml_error(-1, "BOINC server: limit on jobs in progress exceeded"); } } @@ -121,6 +121,12 @@ $fanout = parse_config(get_config(), ""); function stage_file($file) { global $fanout; + switch ($file->mode) { + case "semilocal": + case "local": + $md5 = md5_file($file->source); + if (!$md5) { + xml_error(-1, "BOINC server: Can't get MD5 of file $file->source"); } $name = "jf_$md5"; $path = dir_hier_path($name, "../../download", $fanout); @@ -230,12 +236,6 @@ function submit_batch($r) { xml_error(-1, "BOINC server: no rsc_fpops_est given"); } } - if ($x) { - $total_flops += $x; - } else { - xml_error(-1, "BOINC server: no rsc_fpops_est given"); - } - } } $cmd = "cd ../../bin; ./adjust_user_priority --user $user->id --flops $total_flops --app $app->name"; $x = exec($cmd); @@ -390,9 +390,18 @@ function query_batch2($r) { $batches[] = $batch; } + $min_mod_time = (double)$r->min_mod_time; + if ($min_mod_time) { + $mod_time_clause = "and mod_time > FROM_UNIXTIME($min_mod_time)"; + } else { + $mod_time_clause = ""; + } + + $t = dtime(); + echo "$t\n"; echo "\n"; foreach ($batches as $batch) { - $wus = BoincWorkunit::enum("batch = $batch->id"); + $wus = BoincWorkunit::enum("batch = $batch->id $mod_time_clause"); echo " ".count($wus)."\n"; foreach ($wus as $wu) { if ($wu->canonical_resultid) { diff --git a/lib/remote_submit.cpp b/lib/remote_submit.cpp index e118734d07..efb050847e 100644 --- a/lib/remote_submit.cpp +++ b/lib/remote_submit.cpp @@ -397,6 +397,7 @@ int submit_jobs( int query_batch_set( const char* project_url, const char* authenticator, + double min_mod_time, vector &batch_names, QUERY_BATCH_SET_REPLY& qb_reply, string& error_msg @@ -408,6 +409,8 @@ int query_batch_set( request = "\n"; sprintf(buf, "%s\n", authenticator); request += string(buf); + sprintf(buf, "%f\n", min_mod_time); + request += string(buf); for (unsigned int i=0; i%s\n", batch_names[i].c_str()); request += string(buf); @@ -423,6 +426,7 @@ int query_batch_set( } fseek(reply, 0, SEEK_SET); retval = -1; + qb_reply.server_time = 0; error_msg = ""; while (fgets(buf, 256, reply)) { #ifdef SHOW_REPLY @@ -434,6 +438,7 @@ int query_batch_set( } if (parse_int(buf, "", retval)) continue; if (parse_str(buf, "", error_msg)) continue; + if (parse_double(buf, "", qb_reply.server_time)) continue; if (parse_int(buf, "", batch_size)) { qb_reply.batch_sizes.push_back(batch_size); continue; diff --git a/lib/remote_submit.h b/lib/remote_submit.h index d6bf903344..799055260e 100644 --- a/lib/remote_submit.h +++ b/lib/remote_submit.h @@ -58,8 +58,9 @@ struct JOB_STATUS { }; struct QUERY_BATCH_SET_REPLY { + double server_time; // server time at start of query vector batch_sizes; // how many jobs in each of the queried batches - vector jobs; // the jobs, sequentially + vector jobs; // the jobs, sequentially }; struct OUTFILE { @@ -153,6 +154,7 @@ extern int estimate_batch( extern int query_batch_set( const char* project_url, const char* authenticator, + double min_mod_time, vector &batch_names, QUERY_BATCH_SET_REPLY& reply, string& error_msg diff --git a/samples/condor/boinc_gahp.cpp b/samples/condor/boinc_gahp.cpp index fba3fd8e35..162aaf6355 100644 --- a/samples/condor/boinc_gahp.cpp +++ b/samples/condor/boinc_gahp.cpp @@ -83,6 +83,7 @@ struct COMMAND { vector batch_names; char batch_name[256]; double lease_end_time; + double min_mod_time; COMMAND(char* _in) { in = _in; @@ -296,6 +297,7 @@ void handle_submit(COMMAND& c) { } int COMMAND::parse_query_batches(char* p) { + min_mod_time = atof(strtok_r(NULL, " ", &p)); int n = atoi(strtok_r(NULL, " ", &p)); for (int i=0; i