Rooba
/
boinc
mirror of https://github.com/BOINC/boinc.git
1
0
Fork 0
Code Issues 1 Packages Projects Releases Wiki Activity

Remote job submission was always using allocation-based prioritization. 

This is not necessarily the right thing.
For example, nanoHUB submits batches of jobs that are mixtures
of speculative and user-submitted;
these simply need to be given low and high priorities.

The way things work now:
- if you want allocation-based prioritization,
    set the "allocation_priority" flag in the request object
    (Python or PHP API)
- or set the "priority" field in the request object;
    that sets the priority of all the jobs in the batch
- or set the "priority" field of jobs in the batch.

Updated wiki docs accordingly.

Also fixed a bug in the test script
(note to self: adding an object to an array adds a reference, not a copy).
This commit is contained in:
David Anderson 2020-07-24 00:36:12 -07:00
parent 2abd5d7a95
commit 3e903d0e36
5 changed files with 133 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

57
html/inc/submit.inc
View File

@ -1,7 +1,7 @@
<?php
// This file is part of BOINC.
// http://boinc.berkeley.edu
// Copyright (C) 2013 University of California
// Copyright (C) 2020 University of California
//
// BOINC is free software; you can redistribute it and/or modify it
// under the terms of the GNU Lesser General Public License
@ -29,11 +29,14 @@
// boinc_query_job(): get details of a job
// boinc_retire_batch(): retire a batch; delete output files
// boinc_submit_batch(): submit a batch
//
// boinc_set_timeout($x): set RPC timeout to X seconds
//
// See https://boinc.berkeley.edu/trac/wiki/RemoteJobs#PHPinterface
//// Implementation stuff follows
// Convert a request message from PHP object to XML string
//
function req_to_xml($req, $op) {
if (!isset($req->batch_name)) {
$req->batch_name = "batch_".time();
@ -54,6 +57,14 @@ function req_to_xml($req, $op) {
}
if ((isset($req->app_version_num)) && ($req->app_version_num)) {
$x .= " <app_version_num>$req->app_version_num</app_version_num>
";
}
if (!empty($req->allocation_priority)) {
$x .= " <allocation_priority/>
";
}
if (isset($req->priority)) {
$x .= " <priority>$req->priority</priority>
";
}
foreach ($req->jobs as $job) {
@ -79,6 +90,10 @@ function req_to_xml($req, $op) {
";
} elseif (!empty($job->target_host)) {
$x .= " <target_host>$job->target_host</target_host>
";
}
if (isset($job->priority)) {
$x .= " <priority>$job->priority</priority>
";
}
foreach ($job->input_files as $file) {
@ -108,6 +123,8 @@ function req_to_xml($req, $op) {
return $x;
}
// check whether the PHP structure looks like a batch request object
//
function validate_request($req) {
if (!is_object($req)) return "req is not an object";
if (!array_key_exists('project', $req)) return "missing req->project";
@ -123,6 +140,8 @@ function validate_request($req) {
$rpc_timeout = 0;
// Given a request object and XML string, issue the HTTP POST request
//
function do_http_op($req, $xml, $op) {
global $rpc_timeout;
@ -133,24 +152,24 @@ function do_http_op($req, $xml, $op) {
curl_setopt($ch, CURLOPT_TIMEOUT, $rpc_timeout);
}
// see if we need to send any files
//
$nfiles = 0;
$post = array();
$post["request"] = $xml;
$cwd = getcwd();
if ($op == "submit_batch") {
foreach ($req->jobs as $job) {
foreach ($job->input_files as $file) {
if ($file->mode == "inline") {
$path = realpath("$cwd/$file->path");
$post["file$nfiles"] = $path;
$nfiles++;
}
// see if we need to send any files
//
$nfiles = 0;
$post = array();
$post["request"] = $xml;
$cwd = getcwd();
if ($op == "submit_batch") {
foreach ($req->jobs as $job) {
foreach ($job->input_files as $file) {
if ($file->mode == "inline") {
$path = realpath("$cwd/$file->path");
$post["file$nfiles"] = $path;
$nfiles++;
}
}
}
curl_setopt($ch, CURLOPT_POSTFIELDS, $post);
}
curl_setopt($ch, CURLOPT_POSTFIELDS, $post);
$reply = curl_exec($ch);
curl_close($ch);
if (!$reply) return array(null, "HTTP error");
@ -166,6 +185,8 @@ function do_http_op($req, $xml, $op) {
}
}
// do a batch op (estimate or submit)
//
function do_batch_op($req, $op) {
$retval = validate_request($req);
if ($retval) return array(null, $retval);
@ -173,6 +194,8 @@ function do_batch_op($req, $op) {
return do_http_op($req, $xml, $op);
}
// convert a batch description from XML string to object
//
function batch_xml_to_object($batch) {
$b = new StdClass;
$b->id = (int)($batch->id);

83
html/user/submit_rpc_handler.php
View File

@ -256,11 +256,18 @@ function submit_jobs(
$f = $output_templates[$job->output_template_xml];
$x .= " --result_template $f";
}
if (isset($job->priority)) {
$x .= " --priority $job->priority";
}
$x .= "\n";
}
$errfile = "/tmp/create_work_" . getmypid() . ".err";
$cmd = "cd " . project_dir() . "; ./bin/create_work --appname $app->name --batch $batch_id --priority $priority";
$cmd = "cd " . project_dir() . "; ./bin/create_work --appname $app->name --batch $batch_id";
if ($priority !== null) {
$cmd .= " --priority $priority";
}
if ($input_template_filename) {
$cmd .= " --wu_template templates/$input_template_filename";
}
@ -387,6 +394,9 @@ function xml_get_jobs($r) {
}
$job->input_files[] = $file;
}
if (isset($j->priority)) {
$job->priority = (int)$j->priority;
}
$jobs[] = $job;
if ($job->input_template) {
make_input_template($job);
@ -398,6 +408,37 @@ function xml_get_jobs($r) {
return $jobs;
}
// - compute batch FLOP count
// - run adjust_user_priorities to increment user_submit.logical_start_time
// - return that (use as batch logical end time and job priority)
//
function logical_end_time($r, $jobs, $user, $app) {
$total_flops = 0;
foreach($jobs as $job) {
//print_r($job);
if ($job->rsc_fpops_est) {
$total_flops += $job->rsc_fpops_est;
} else if ($job->input_template && $job->input_template->workunit->rsc_fpops_est) {
$total_flops += (double) $job->input_template->workunit->rsc_fpops_est;
} else if ($r->batch->job_params->rsc_fpops_est) {
$total_flops += (double) $r->batch->job_params->rsc_fpops_est;
} else {
$x = (double) $template->workunit->rsc_fpops_est;
if ($x) {
$total_flops += $x;
} else {
xml_error(-1, "no rsc_fpops_est given");
}
}
}
$cmd = "cd " . project_dir() . "/bin; ./adjust_user_priority --user $user->id --flops $total_flops --app $app->name";
$x = exec($cmd);
if (!is_numeric($x) || (double)$x == 0) {
xml_error(-1, "$cmd returned $x");
}
return (double)$x;
}
// $r is a simplexml object encoding the request message
//
function submit_batch($r) {
@ -410,6 +451,7 @@ function submit_batch($r) {
validate_batch($jobs, $template);
}
stage_files($jobs);
$njobs = count($jobs);
$now = time();
$app_version_num = (int)($r->batch->app_version_num);
@ -433,38 +475,17 @@ function submit_batch($r) {
}
}
// - compute batch FLOP count
// - run adjust_user_priorities to increment user_submit.logical_start_time
// - use that for batch logical end time and job priority
// compute a priority for the jobs
//
$total_flops = 0;
foreach($jobs as $job) {
//print_r($job);
if ($job->rsc_fpops_est) {
$total_flops += $job->rsc_fpops_est;
} else if ($job->input_template && $job->input_template->workunit->rsc_fpops_est) {
$total_flops += (double) $job->input_template->workunit->rsc_fpops_est;
} else if ($r->batch->job_params->rsc_fpops_est) {
$total_flops += (double) $r->batch->job_params->rsc_fpops_est;
} else {
$x = (double) $template->workunit->rsc_fpops_est;
if ($x) {
$total_flops += $x;
} else {
log_write("no rsc_fpops_est given");
xml_error(-1, "no rsc_fpops_est given");
}
}
$priority = null;
$let = 0;
if ($r->batch->allocation_priority) {
$let = logical_end_time($r, $jobs, $user, $app);
$priority = -(int)$let;
} else if (isset($r->batch->priority)) {
$priority = (int)$r->batch->priority;
}
$cmd = "cd " . project_dir() . "/bin; ./adjust_user_priority --user $user->id --flops $total_flops --app $app->name";
$x = exec($cmd);
if (!is_numeric($x) || (double)$x == 0) {
log_write("$cmd returned $x");
xml_error(-1, "$cmd returned $x");
}
$let = (double)$x;
$njobs = count($jobs);
if ($batch_id) {
$ret = $batch->update("njobs=$njobs, logical_end_time=$let");
if (!$ret) {
@ -499,7 +520,7 @@ function submit_batch($r) {
// possibly empty
submit_jobs(
$jobs, $job_params, $app, $batch_id, $let, $app_version_num,
$jobs, $job_params, $app, $batch_id, $priority, $app_version_num,
$input_template_filename,
$output_template_filename
);

26
html/user/submit_test.php
View File

@ -1,5 +1,20 @@
<?php
require_once("submit.inc");
// This file is part of BOINC.
// http://boinc.berkeley.edu
// Copyright (C) 2018 University of California
//
// BOINC is free software; you can redistribute it and/or modify it
// under the terms of the GNU Lesser General Public License
// as published by the Free Software Foundation,
// either version 3 of the License, or (at your option) any later version.
//
// BOINC is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
// See the GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with BOINC. If not, see <http://www.gnu.org/licenses/>.
// tests for remote job submission interfaces
//
@ -7,10 +22,12 @@ require_once("submit.inc");
// line 0: project URL
// line 1: authenticator
//
// you must run this in a dir with a link to submit.inc
// you must run this in a dir with a copy of or link to html/inc/submit.inc
// TODO: add more tests
require_once("submit.inc");
// for this test, you must have
// - an app "uppercase"
// - templates uppercase_in and uppercase_out
@ -24,10 +41,9 @@ function test_submit_batch($req) {
$f->mode = "local_staged";
$f->source = "input";
$job = new StdClass;
$job->input_files = array($f);
for ($i=10; $i<20; $i++) {
$job = new StdClass;
$job->input_files = array($f);
$job->rsc_fpops_est = $i*1e9;
$job->command_line = "--t $i";
$req->jobs[] = $job;

11
lib/submit_api.py
View File

@ -1,6 +1,6 @@
# This file is part of BOINC.
# http://boinc.berkeley.edu
# Copyright (C) 2016 University of California
# Copyright (C) 2020 University of California
#
# BOINC is free software; you can redistribute it and/or modify it
# under the terms of the GNU Lesser General Public License
@ -17,6 +17,7 @@
# Python bindings of the remote job submission and file management APIs
# See https://boinc.berkeley.edu/trac/wiki/RemoteJobs#Pythonbinding
import urllib
import urllib2
@ -62,6 +63,8 @@ class JOB_DESC:
xml += '%s\n'%self.input_template
if hasattr(self, 'output_template'):
xml += '%s\n'%self.output_template
if hasattr(self, 'priority'):
xml += '<priority>%d</priority>\n'%(self.priority)
if hasattr(self, 'files'):
for file in self.files:
xml += file.to_xml()
@ -89,6 +92,11 @@ class BATCH_DESC:
if hasattr(self, 'app_version_num'):
xml += '<app_version_num>%d</app_version_num>\n'%(self.app_version_num)
if hasattr(self, 'allocation_priority'):
if self.allocation_priority:
xml += '<allocation_priority/>\n'
if hasattr(self, 'priority'):
xml += '<priority>%d</priority>\n'%(self.priority)
for job in self.jobs:
xml += job.to_xml()
xml += '</batch>\n</%s>\n' %(op)
@ -122,6 +130,7 @@ def do_http_post(req, project_url, handler='submit_rpc_handler.php'):
f = urllib2.urlopen(url, params, rpc_timeout)
else:
f = urllib2.urlopen(url, params)
reply = f.read()
#print "REPLY:", reply
return ET.fromstring(reply)

11
tools/create_work.cpp
View File

@ -1,6 +1,6 @@
// This file is part of BOINC.
// http://boinc.berkeley.edu
// Copyright (C) 2019 University of California
// Copyright (C) 2020 University of California
//
// BOINC is free software; you can redistribute it and/or modify it
// under the terms of the GNU Lesser General Public License
@ -20,6 +20,11 @@
// run from PHP script for remote job submission.
//
// see http://boinc.berkeley.edu/trac/wiki/JobSubmission
//
// This program can be used in two ways:
// - to create a single job, with everything passed on the cmdline
// - to create multiple jobs, where per-job info is passed via stdin,
// one line per job
#include "config.h"
@ -187,6 +192,8 @@ void JOB_DESC::parse_cmdline(int argc, char** argv) {
assign_type = ASSIGN_USER;
assign_id = atoi(argv[++i]);
check_assign_id(assign_id);
} else if (arg(argv, i, (char*)"priority")) {
wu.priority = atoi(argv[++i]);
} else {
if (!strncmp("-", argv[i], 1)) {
fprintf(stderr, "create_work: bad stdin argument '%s'\n", argv[i]);
@ -433,9 +440,11 @@ int main(int argc, char** argv) {
char* p = fgets(buf, sizeof(buf), stdin);
if (p == NULL) break;
JOB_DESC jd2 = jd;
// things default to what was passed on cmdline
strcpy(jd2.wu.name, "");
_argc = parse_command_line(buf, _argv);
jd2.parse_cmdline(_argc, _argv);
// get info from stdin line
if (!strlen(jd2.wu.name)) {
sprintf(jd2.wu.name, "%s_%d", jd.wu.name, j);
}