diff --git a/html/inc/submit.inc b/html/inc/submit.inc index 83d9f406ae..aac37b9fe8 100644 --- a/html/inc/submit.inc +++ b/html/inc/submit.inc @@ -1,7 +1,7 @@ batch_name)) { $req->batch_name = "batch_".time(); @@ -54,6 +57,14 @@ function req_to_xml($req, $op) { } if ((isset($req->app_version_num)) && ($req->app_version_num)) { $x .= " $req->app_version_num +"; + } + if (!empty($req->allocation_priority)) { + $x .= " +"; + } + if (isset($req->priority)) { + $x .= " $req->priority "; } foreach ($req->jobs as $job) { @@ -79,6 +90,10 @@ function req_to_xml($req, $op) { "; } elseif (!empty($job->target_host)) { $x .= " $job->target_host +"; + } + if (isset($job->priority)) { + $x .= " $job->priority "; } foreach ($job->input_files as $file) { @@ -108,6 +123,8 @@ function req_to_xml($req, $op) { return $x; } +// check whether the PHP structure looks like a batch request object +// function validate_request($req) { if (!is_object($req)) return "req is not an object"; if (!array_key_exists('project', $req)) return "missing req->project"; @@ -123,6 +140,8 @@ function validate_request($req) { $rpc_timeout = 0; +// Given a request object and XML string, issue the HTTP POST request +// function do_http_op($req, $xml, $op) { global $rpc_timeout; @@ -133,24 +152,24 @@ function do_http_op($req, $xml, $op) { curl_setopt($ch, CURLOPT_TIMEOUT, $rpc_timeout); } - // see if we need to send any files - // - $nfiles = 0; - $post = array(); - $post["request"] = $xml; - $cwd = getcwd(); - if ($op == "submit_batch") { - foreach ($req->jobs as $job) { - foreach ($job->input_files as $file) { - if ($file->mode == "inline") { - $path = realpath("$cwd/$file->path"); - $post["file$nfiles"] = $path; - $nfiles++; - } +// see if we need to send any files +// +$nfiles = 0; +$post = array(); +$post["request"] = $xml; +$cwd = getcwd(); +if ($op == "submit_batch") { + foreach ($req->jobs as $job) { + foreach ($job->input_files as $file) { + if ($file->mode == "inline") { + $path = realpath("$cwd/$file->path"); + $post["file$nfiles"] = $path; + $nfiles++; } } } - curl_setopt($ch, CURLOPT_POSTFIELDS, $post); +} +curl_setopt($ch, CURLOPT_POSTFIELDS, $post); $reply = curl_exec($ch); curl_close($ch); if (!$reply) return array(null, "HTTP error"); @@ -166,6 +185,8 @@ function do_http_op($req, $xml, $op) { } } +// do a batch op (estimate or submit) +// function do_batch_op($req, $op) { $retval = validate_request($req); if ($retval) return array(null, $retval); @@ -173,6 +194,8 @@ function do_batch_op($req, $op) { return do_http_op($req, $xml, $op); } +// convert a batch description from XML string to object +// function batch_xml_to_object($batch) { $b = new StdClass; $b->id = (int)($batch->id); diff --git a/html/user/submit_rpc_handler.php b/html/user/submit_rpc_handler.php index 845aa6dbd8..bae51231cf 100644 --- a/html/user/submit_rpc_handler.php +++ b/html/user/submit_rpc_handler.php @@ -256,11 +256,18 @@ function submit_jobs( $f = $output_templates[$job->output_template_xml]; $x .= " --result_template $f"; } + if (isset($job->priority)) { + $x .= " --priority $job->priority"; + } $x .= "\n"; } $errfile = "/tmp/create_work_" . getmypid() . ".err"; - $cmd = "cd " . project_dir() . "; ./bin/create_work --appname $app->name --batch $batch_id --priority $priority"; + $cmd = "cd " . project_dir() . "; ./bin/create_work --appname $app->name --batch $batch_id"; + if ($priority !== null) { + $cmd .= " --priority $priority"; + } + if ($input_template_filename) { $cmd .= " --wu_template templates/$input_template_filename"; } @@ -387,6 +394,9 @@ function xml_get_jobs($r) { } $job->input_files[] = $file; } + if (isset($j->priority)) { + $job->priority = (int)$j->priority; + } $jobs[] = $job; if ($job->input_template) { make_input_template($job); @@ -398,6 +408,37 @@ function xml_get_jobs($r) { return $jobs; } +// - compute batch FLOP count +// - run adjust_user_priorities to increment user_submit.logical_start_time +// - return that (use as batch logical end time and job priority) +// +function logical_end_time($r, $jobs, $user, $app) { + $total_flops = 0; + foreach($jobs as $job) { + //print_r($job); + if ($job->rsc_fpops_est) { + $total_flops += $job->rsc_fpops_est; + } else if ($job->input_template && $job->input_template->workunit->rsc_fpops_est) { + $total_flops += (double) $job->input_template->workunit->rsc_fpops_est; + } else if ($r->batch->job_params->rsc_fpops_est) { + $total_flops += (double) $r->batch->job_params->rsc_fpops_est; + } else { + $x = (double) $template->workunit->rsc_fpops_est; + if ($x) { + $total_flops += $x; + } else { + xml_error(-1, "no rsc_fpops_est given"); + } + } + } + $cmd = "cd " . project_dir() . "/bin; ./adjust_user_priority --user $user->id --flops $total_flops --app $app->name"; + $x = exec($cmd); + if (!is_numeric($x) || (double)$x == 0) { + xml_error(-1, "$cmd returned $x"); + } + return (double)$x; +} + // $r is a simplexml object encoding the request message // function submit_batch($r) { @@ -410,6 +451,7 @@ function submit_batch($r) { validate_batch($jobs, $template); } stage_files($jobs); + $njobs = count($jobs); $now = time(); $app_version_num = (int)($r->batch->app_version_num); @@ -433,38 +475,17 @@ function submit_batch($r) { } } - // - compute batch FLOP count - // - run adjust_user_priorities to increment user_submit.logical_start_time - // - use that for batch logical end time and job priority + // compute a priority for the jobs // - $total_flops = 0; - foreach($jobs as $job) { - //print_r($job); - if ($job->rsc_fpops_est) { - $total_flops += $job->rsc_fpops_est; - } else if ($job->input_template && $job->input_template->workunit->rsc_fpops_est) { - $total_flops += (double) $job->input_template->workunit->rsc_fpops_est; - } else if ($r->batch->job_params->rsc_fpops_est) { - $total_flops += (double) $r->batch->job_params->rsc_fpops_est; - } else { - $x = (double) $template->workunit->rsc_fpops_est; - if ($x) { - $total_flops += $x; - } else { - log_write("no rsc_fpops_est given"); - xml_error(-1, "no rsc_fpops_est given"); - } - } + $priority = null; + $let = 0; + if ($r->batch->allocation_priority) { + $let = logical_end_time($r, $jobs, $user, $app); + $priority = -(int)$let; + } else if (isset($r->batch->priority)) { + $priority = (int)$r->batch->priority; } - $cmd = "cd " . project_dir() . "/bin; ./adjust_user_priority --user $user->id --flops $total_flops --app $app->name"; - $x = exec($cmd); - if (!is_numeric($x) || (double)$x == 0) { - log_write("$cmd returned $x"); - xml_error(-1, "$cmd returned $x"); - } - $let = (double)$x; - $njobs = count($jobs); if ($batch_id) { $ret = $batch->update("njobs=$njobs, logical_end_time=$let"); if (!$ret) { @@ -499,7 +520,7 @@ function submit_batch($r) { // possibly empty submit_jobs( - $jobs, $job_params, $app, $batch_id, $let, $app_version_num, + $jobs, $job_params, $app, $batch_id, $priority, $app_version_num, $input_template_filename, $output_template_filename ); diff --git a/html/user/submit_test.php b/html/user/submit_test.php index a058d76cbd..3f79b6ee6f 100644 --- a/html/user/submit_test.php +++ b/html/user/submit_test.php @@ -1,5 +1,20 @@ . // tests for remote job submission interfaces // @@ -7,10 +22,12 @@ require_once("submit.inc"); // line 0: project URL // line 1: authenticator // -// you must run this in a dir with a link to submit.inc +// you must run this in a dir with a copy of or link to html/inc/submit.inc // TODO: add more tests +require_once("submit.inc"); + // for this test, you must have // - an app "uppercase" // - templates uppercase_in and uppercase_out @@ -24,10 +41,9 @@ function test_submit_batch($req) { $f->mode = "local_staged"; $f->source = "input"; - $job = new StdClass; - $job->input_files = array($f); - for ($i=10; $i<20; $i++) { + $job = new StdClass; + $job->input_files = array($f); $job->rsc_fpops_est = $i*1e9; $job->command_line = "--t $i"; $req->jobs[] = $job; diff --git a/lib/submit_api.py b/lib/submit_api.py index 42509eec65..bff5e4dc62 100644 --- a/lib/submit_api.py +++ b/lib/submit_api.py @@ -1,6 +1,6 @@ # This file is part of BOINC. # http://boinc.berkeley.edu -# Copyright (C) 2016 University of California +# Copyright (C) 2020 University of California # # BOINC is free software; you can redistribute it and/or modify it # under the terms of the GNU Lesser General Public License @@ -17,6 +17,7 @@ # Python bindings of the remote job submission and file management APIs +# See https://boinc.berkeley.edu/trac/wiki/RemoteJobs#Pythonbinding import urllib import urllib2 @@ -62,6 +63,8 @@ class JOB_DESC: xml += '%s\n'%self.input_template if hasattr(self, 'output_template'): xml += '%s\n'%self.output_template + if hasattr(self, 'priority'): + xml += '%d\n'%(self.priority) if hasattr(self, 'files'): for file in self.files: xml += file.to_xml() @@ -89,6 +92,11 @@ class BATCH_DESC: if hasattr(self, 'app_version_num'): xml += '%d\n'%(self.app_version_num) + if hasattr(self, 'allocation_priority'): + if self.allocation_priority: + xml += '\n' + if hasattr(self, 'priority'): + xml += '%d\n'%(self.priority) for job in self.jobs: xml += job.to_xml() xml += '\n\n' %(op) @@ -122,6 +130,7 @@ def do_http_post(req, project_url, handler='submit_rpc_handler.php'): f = urllib2.urlopen(url, params, rpc_timeout) else: f = urllib2.urlopen(url, params) + reply = f.read() #print "REPLY:", reply return ET.fromstring(reply) diff --git a/tools/create_work.cpp b/tools/create_work.cpp index c04ffe0757..049a087b09 100644 --- a/tools/create_work.cpp +++ b/tools/create_work.cpp @@ -1,6 +1,6 @@ // This file is part of BOINC. // http://boinc.berkeley.edu -// Copyright (C) 2019 University of California +// Copyright (C) 2020 University of California // // BOINC is free software; you can redistribute it and/or modify it // under the terms of the GNU Lesser General Public License @@ -20,6 +20,11 @@ // run from PHP script for remote job submission. // // see http://boinc.berkeley.edu/trac/wiki/JobSubmission +// +// This program can be used in two ways: +// - to create a single job, with everything passed on the cmdline +// - to create multiple jobs, where per-job info is passed via stdin, +// one line per job #include "config.h" @@ -187,6 +192,8 @@ void JOB_DESC::parse_cmdline(int argc, char** argv) { assign_type = ASSIGN_USER; assign_id = atoi(argv[++i]); check_assign_id(assign_id); + } else if (arg(argv, i, (char*)"priority")) { + wu.priority = atoi(argv[++i]); } else { if (!strncmp("-", argv[i], 1)) { fprintf(stderr, "create_work: bad stdin argument '%s'\n", argv[i]); @@ -433,9 +440,11 @@ int main(int argc, char** argv) { char* p = fgets(buf, sizeof(buf), stdin); if (p == NULL) break; JOB_DESC jd2 = jd; + // things default to what was passed on cmdline strcpy(jd2.wu.name, ""); _argc = parse_command_line(buf, _argv); jd2.parse_cmdline(_argc, _argv); + // get info from stdin line if (!strlen(jd2.wu.name)) { sprintf(jd2.wu.name, "%s_%d", jd.wu.name, j); }