From a46a5926ae32b384bca7efcd5014b75270644550 Mon Sep 17 00:00:00 2001 From: David Anderson Date: Fri, 25 Jan 2013 00:35:00 -0800 Subject: [PATCH 1/9] - remote file management and job submission stuff for Condor --- html/inc/submit_util.inc | 23 ++++ html/user/submit_rpc_handler.php | 34 +----- samples/condor/Makefile | 7 +- samples/condor/boinc_gahp.cpp | 198 +++++-------------------------- samples/condor/boinc_gahp.h | 51 ++++++++ samples/condor/curl.cpp | 187 ++++++++++++++++++++++++++--- 6 files changed, 284 insertions(+), 216 deletions(-) create mode 100644 samples/condor/boinc_gahp.h diff --git a/html/inc/submit_util.inc b/html/inc/submit_util.inc index 2e51905067..70ab8deda5 100644 --- a/html/inc/submit_util.inc +++ b/html/inc/submit_util.inc @@ -21,6 +21,29 @@ require_once("../inc/submit_db.inc"); +function error($s) { + echo "error: $s\n"; + echo "\n$s\n\n"; + exit; +} + +function authenticate_user($r, $app) { + $auth = (string)$r->authenticator; + if (!$auth) error("no authenticator"); + $auth = BoincDb::escape_string($auth); + $user = BoincUser::lookup("authenticator='$auth'"); + if (!$user) error("bad authenticator"); + $user_submit = BoincUserSubmit::lookup_userid($user->id); + if (!$user_submit) error("no submit access"); + if ($app && !$user_submit->submit_all) { + $usa = BoincUserSubmitApp::lookup("user_id=$user->id and app_id=$app->id"); + if (!$usa) { + error("no submit access"); + } + } + return array($user, $user_submit); +} + // given its WUs, compute params of a batch // NOTE: eventually this should be done by server components // (transitioner, validator etc.) as jobs complete or time out diff --git a/html/user/submit_rpc_handler.php b/html/user/submit_rpc_handler.php index 2abeec8d44..76ed358645 100644 --- a/html/user/submit_rpc_handler.php +++ b/html/user/submit_rpc_handler.php @@ -30,28 +30,6 @@ error_reporting(E_ALL); ini_set('display_errors', true); ini_set('display_startup_errors', true); -function error($s) { - echo "\n$s\n\n"; - exit; -} - -function authenticate_user($r, $app) { - $auth = (string)$r->authenticator; - if (!$auth) error("no authenticator"); - $auth = BoincDb::escape_string($auth); - $user = BoincUser::lookup("authenticator='$auth'"); - if (!$user) error("bad authenticator"); - $user_submit = BoincUserSubmit::lookup_userid($user->id); - if (!$user_submit) error("no submit access"); - if ($app && !$user_submit->submit_all) { - $usa = BoincUserSubmitApp::lookup("user_id=$user->id and app_id=$app->id"); - if (!$usa) { - error("no submit access"); - } - } - return array($user, $user_submit); -} - function get_app($r) { $name = (string)($r->batch->app_name); $name = BoincDb::escape_string($name); @@ -183,16 +161,13 @@ function submit_batch($r) { if ($batch_id) { $batch = BoincBatch::lookup_id($batch_id); if (!$batch) { - echo "no batch $batch_id\n"; - exit; + error("no batch $batch_id"); } if ($batch->user_id != $user->id) { - echo "not owner\n"; - exit; + error("not owner"); } if ($batch->state != BATCH_STATE_INIT) { - echo "batch not in init state\n"; - exit; + error("batch not in init state"); } } @@ -208,8 +183,7 @@ function submit_batch($r) { $cmd = "cd ../../bin; ./adjust_user_priority --user $user->id --flops $total_flops --app $app->name"; $x = system($cmd); if (!is_numeric($x) || (double)$x == 0) { - echo "adjust_user_priority returned $x\n"; - exit; + error("adjust_user_priority returned $x"); } $let = (double)$x; diff --git a/samples/condor/Makefile b/samples/condor/Makefile index 8bf801bbe5..1256089580 100644 --- a/samples/condor/Makefile +++ b/samples/condor/Makefile @@ -1,9 +1,10 @@ all: boinc_gahp -boinc_gahp: boinc_gahp.cpp curl.cpp +boinc_gahp: boinc_gahp.cpp boinc_gahp.h curl.cpp curl.h g++ -g -O0 -I../../lib \ -o boinc_gahp boinc_gahp.cpp curl.cpp \ -L../../lib -lboinc -lpthread -lcurl -curl_test: curl.cpp - g++ -o curl_test curl.cpp -lcurl +test: test.cpp curl.cpp + g++ -g -o test -I../../lib \ + test.cpp curl.cpp -lcurl diff --git a/samples/condor/boinc_gahp.cpp b/samples/condor/boinc_gahp.cpp index 650a979712..7d3cb63735 100644 --- a/samples/condor/boinc_gahp.cpp +++ b/samples/condor/boinc_gahp.cpp @@ -58,74 +58,10 @@ struct COMMAND { typedef map COMMANDS; COMMANDS commands; -struct INFILE { - char src_path[256]; - char dst_path[256]; -}; - -struct JOB { - char job_name[256]; - string cmdline_args; - vector infiles; - bool all_output_files; - vector outfiles; -}; - -struct LOCAL_FILE { - char md5[64]; - double nbytes; -}; - -struct SUBMIT_REQ { - char batch_name[256]; - char app_name[256]; - vector jobs; - map local_files; - // maps local path to info about file - int batch_id; -}; - int compute_md5(string path, LOCAL_FILE& f) { return md5_file(path.c_str(), f.md5, f.nbytes); } -int create_batch(SUBMIT_REQ& sr) { - char request[1024]; - char url[1024]; - sprintf(request, - "\n" - " %s\n" - " \n" - " %s\n" - " %s\n" - " \n" - "\n", - authenticator, - sr.batch_name, - sr.app_name - ); - sprintf(url, "%ssubmit_rpc_handler.php", project_url); - FILE* reply = tmpfile(); - vector x; - int retval = do_http_post(url, request, reply, x); - if (retval) { - fclose(reply); - return retval; - } - char buf[256]; - sr.batch_id = 0; - fseek(reply, 0, SEEK_SET); - while (fgets(buf, 256, reply)) { - if (parse_int(buf, "", sr.batch_id)) break; - } - fclose(reply); - if (sr.batch_id == 0) { - return -1; - } - return 0; -} - - // Get a list of the input files used by the batch. // Get their MD5s. // See if they're already on the server. @@ -148,7 +84,7 @@ int process_input_files(SUBMIT_REQ& req) { } // compute the MD5s of these files, - // and make a map from filename to MD5 and other info (LOCAL_FILE) + // and make a map from path to MD5 and size (LOCAL_FILE) // set::iterator iter = unique_paths.begin(); while (iter != unique_paths.end()) { @@ -161,69 +97,43 @@ int process_input_files(SUBMIT_REQ& req) { } // ask the server which files it doesn't already have. - // We send it the batch ID and a list of (filename, MD5) pairs. - // It - // - creates batch_file_assoc records for all the files - // (to avoid race condition w/ file deletion) - // - returns the list of filenames it doesn't have. // - string req_msg; - req_msg = "\n"; map::iterator map_iter; map_iter = req.local_files.begin(); - sprintf(buf, "%d\n", req.batch_id); - req_msg += string(buf); + vector md5s, paths; + vector absent_files; while (map_iter != req.local_files.end()) { LOCAL_FILE lf = map_iter->second; - string name = map_iter->first; - sprintf(buf, - "\n" - " %s\n" - " %s\n" - "\n", - name.c_str(), - lf.md5 - ); - req_msg += string(buf); + paths.push_back(map_iter->first); + md5s.push_back(lf.md5); iter++; } - req_msg += "\n"; - vector send_files; - FILE* reply = tmpfile(); - retval = do_http_post(project_url, req_msg.c_str(), reply, send_files); - fseek(reply, 0, SEEK_SET); - vector missing_files; - string missing_file; - while (fgets(buf, 256, reply)) { - if (parse_str(buf, "", missing_file)) { - missing_files.push_back(missing_file); - continue; - } - } - fclose(reply); + retval = query_files( + project_url, + authenticator, + req.batch_id, + md5s, + paths, + absent_files + ); + if (retval) return retval; // upload the missing files. - // Send a list of the MD5s so the server doesn't have to compute them. // - req_msg = "\n"; - for (i=0; isecond; - sprintf(buf, "%s\n", lf.md5); - req_msg += string(buf); + vector upload_md5s, upload_paths; + for (unsigned int i=0; i\n" - "%s\n" - "%d\n", - authenticator, - req.batch_id - ); - string request = buf; - for (unsigned int i=0; i\n"; - } - for (unsigned int j=0; j::iterator iter = req.local_files.find(infile.src_path); - LOCAL_FILE& lf = iter->second; - sprintf(buf, - "\n" - "local\n" - "%s\n" - "\n", - lf.md5 - ); - request += buf; - } - request += "\n"; - } - request += "\n"; - sprintf(url, "%ssubmit_rpc_handler.php", project_url); - FILE* reply = tmpfile(); - vector x; - int retval = do_http_post(url, request.c_str(), reply, x); - if (retval) { - fclose(reply); - return retval; - } - fseek(reply, 0, SEEK_SET); - while (fgets(buf, 256, reply)) { - } - fclose(reply); - return 0; -} - // To avoid a race condition with file deletion: // - create a batch record // - create batch/file associations, and upload files @@ -327,7 +187,7 @@ void handle_boinc_submit(COMMAND& c, char* p) { printf("error parsing request: %d\n", retval); return; } - retval = create_batch(req); + retval = create_batch(project_url, authenticator, req); if (retval) { printf("error creating batch: %d\n", retval); return; @@ -337,7 +197,7 @@ void handle_boinc_submit(COMMAND& c, char* p) { printf("error processing input files: %d\n", retval); return; } - retval = submit_jobs(req); + retval = submit_jobs(project_url, authenticator, req); if (retval) { printf("error submitting jobs: %d\n", retval); return; diff --git a/samples/condor/boinc_gahp.h b/samples/condor/boinc_gahp.h new file mode 100644 index 0000000000..c98e9a1ba0 --- /dev/null +++ b/samples/condor/boinc_gahp.h @@ -0,0 +1,51 @@ +// This file is part of BOINC. +// http://boinc.berkeley.edu +// Copyright (C) 2013 University of California +// +// BOINC is free software; you can redistribute it and/or modify it +// under the terms of the GNU Lesser General Public License +// as published by the Free Software Foundation, +// either version 3 of the License, or (at your option) any later version. +// +// BOINC is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +// See the GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with BOINC. If not, see . + +#include +#include +#include + +using std::map; +using std::string; +using std::vector; + +struct INFILE { + char src_path[256]; + char dst_path[256]; +}; + +struct JOB { + char job_name[256]; + string cmdline_args; + vector infiles; + bool all_output_files; + vector outfiles; +}; + +struct LOCAL_FILE { + char md5[64]; + double nbytes; +}; + +struct SUBMIT_REQ { + char batch_name[256]; + char app_name[256]; + vector jobs; + map local_files; + // maps local path to info about file + int batch_id; +}; diff --git a/samples/condor/curl.cpp b/samples/condor/curl.cpp index f6f6d96e9b..1725afcaeb 100644 --- a/samples/condor/curl.cpp +++ b/samples/condor/curl.cpp @@ -19,6 +19,9 @@ #include #include #include +#include + +#include "parse.h" #include "curl.h" @@ -28,7 +31,7 @@ using std::string; // send an HTTP POST request, // with an optional set of multi-part file attachments // -int do_http_post( +static int do_http_post( const char* url, const char* request, FILE* reply, @@ -80,17 +83,173 @@ int do_http_post( return 0; } -#if 0 -int main() { - FILE* reply = fopen("reply", "w"); - vector send_files; - send_files.push_back("curl.cpp"); - send_files.push_back("boinc_gahp.cpp"); - do_http_post( - "http://isaac.ssl.berkeley.edu/foobar.php", - "foo", - reply, - send_files - ); +int query_files( + const char* project_url, + const char* authenticator, + int batch_id, + vector &md5s, + vector &paths, + vector &absent_files +) { + string req_msg; + char buf[256]; + req_msg = "\n"; + sprintf(buf, "%s\n", authenticator); + req_msg += string(buf); + if (batch_id) { + sprintf(buf, "%d\n", batch_id); + req_msg += string(buf); + } + for (unsigned int i=0; i%s\n", md5s[i].c_str()); + req_msg += string(buf); + } + req_msg += "\n"; + FILE* reply = tmpfile(); + char url[256]; + sprintf(url, "%sjob_file.php", project_url); + int retval = do_http_post(url, req_msg.c_str(), reply, paths); + if (retval) { + fclose(reply); + return retval; + } + fseek(reply, 0, SEEK_SET); + int x; + while (fgets(buf, 256, reply)) { + printf("reply: %s", buf); + if (strstr(buf, "error")) { + retval = -1; + } + if (parse_int(buf, "", x)) { + absent_files.push_back(x); + continue; + } + } + fclose(reply); + return retval; +} + +int upload_files ( + const char* project_url, + const char* authenticator, + int batch_id, + vector &md5s, + vector &paths +) { + char buf[1024]; + string req_msg = "\n"; + for (unsigned int i=0; i%s\n", md5s[i].c_str()); + req_msg += string(buf); + } + req_msg = "\n"; + FILE* reply = tmpfile(); + char url[256]; + sprintf(url, "%sjob_file.php", project_url); + int retval = do_http_post(url, req_msg.c_str(), reply, paths); + if (retval) { + fclose(reply); + return retval; + } + bool success = false; + while (fgets(buf, 256, reply)) { + if (strstr(buf, "success")) { + success = true; + break; + } + } + fclose(reply); + if (!success) return -1; + return 0; +} + +int create_batch( + const char* project_url, + const char* authenticator, + SUBMIT_REQ& sr +) { + char request[1024]; + char url[1024]; + sprintf(request, + "\n" + " %s\n" + " \n" + " %s\n" + " %s\n" + " \n" + "\n", + authenticator, + sr.batch_name, + sr.app_name + ); + sprintf(url, "%ssubmit_rpc_handler.php", project_url); + FILE* reply = tmpfile(); + vector x; + int retval = do_http_post(url, request, reply, x); + if (retval) { + fclose(reply); + return retval; + } + char buf[256]; + sr.batch_id = 0; + fseek(reply, 0, SEEK_SET); + while (fgets(buf, 256, reply)) { + if (parse_int(buf, "", sr.batch_id)) break; + } + fclose(reply); + if (sr.batch_id == 0) { + return -1; + } + return 0; +} + +int submit_jobs( + const char* project_url, + const char* authenticator, + SUBMIT_REQ req +) { + char buf[1024], url[1024]; + sprintf(buf, + "\n" + "%s\n" + "%d\n", + authenticator, + req.batch_id + ); + string request = buf; + for (unsigned int i=0; i\n"; + } + for (unsigned int j=0; j::iterator iter = req.local_files.find(infile.src_path); + LOCAL_FILE& lf = iter->second; + sprintf(buf, + "\n" + "local\n" + "%s\n" + "\n", + lf.md5 + ); + request += buf; + } + request += "\n"; + } + request += "\n"; + sprintf(url, "%ssubmit_rpc_handler.php", project_url); + FILE* reply = tmpfile(); + vector x; + int retval = do_http_post(url, request.c_str(), reply, x); + if (retval) { + fclose(reply); + return retval; + } + fseek(reply, 0, SEEK_SET); + while (fgets(buf, 256, reply)) { + } + fclose(reply); + return 0; } -#endif From 2a73dc0e014cc9815e3d90e6857002a8002c5ec7 Mon Sep 17 00:00:00 2001 From: David Anderson Date: Fri, 25 Jan 2013 11:30:50 -0800 Subject: [PATCH 2/9] - remote file management stuff for Condor --- db/constraints.sql | 2 +- html/user/job_file.php | 196 ++++++++++++++++++++++++++++++++++++++++ samples/condor/curl.cpp | 10 +- 3 files changed, 206 insertions(+), 2 deletions(-) create mode 100644 html/user/job_file.php diff --git a/db/constraints.sql b/db/constraints.sql index 948de8fe37..e32daffb21 100644 --- a/db/constraints.sql +++ b/db/constraints.sql @@ -131,4 +131,4 @@ alter table assignment add index asgn_target(target_type, target_id); alter table job_file - add index md5(md5); + add unique jf_md5(md5); diff --git a/html/user/job_file.php b/html/user/job_file.php new file mode 100644 index 0000000000..9e01ad5d17 --- /dev/null +++ b/html/user/job_file.php @@ -0,0 +1,196 @@ +. + +// Web RPCs for managing job input files on the server. +// +// Issues: +// +// 1) how are files named? +// Their name is a function of their MD5. +// This eliminates issues related to file immutability +// +// 2) how do we keep track of the files? +// In the MySQL database, in a table called job_files. +// Each row describes a file currently on the server. +// In addition, we maintain a table batch_file_assoc to record +// that a file is used by a particular batch. +// (Note: the association could be at the job level instead. +// but this way is more efficient if all the jobs in a batch use +// a particular file.) +// +// 3) how do we clean up unused files? +// A daemon (job_file_deleter) deletes files for which +// - the delete date (if given) is in the past, and +// - there are no associations to active batches +// +// 4) what are the RPC operations? +// query_files +// in: +// authenticator +// list of MD5s +// batch ID (optional) +// new delete time (optional) +// out: +// error message, +// or list of files (indices in the MD5 list) not present on server +// action: for each MD5 in in the input list: +// if present on server +// update delete time +// create batch/file association +// add MD5 to output list +// upload_files +// in: +// authenticator +// delete time (optional) +// batch ID (optional) +// list of MD5s +// files (as multipart attachments) +// out: +// error message, or success +// action: +// for each file in list +// move to project download dir w/ appropriate name +// create job_files record +// create batch_file_assoc record if needed + +error_reporting(E_ALL); +ini_set('display_errors', true); +ini_set('display_startup_errors', true); + +require_once("../inc/boinc_db.inc"); +require_once("../inc/dir_hier.inc"); +require_once("../inc/xml.inc"); +require_once("../inc/submit_util.inc"); + +// the physical name of a file is jf_(md5). +// Prepend the jf_ to make the source of the file clear +// +function job_file_name($md5) { + return "jf_$md5"; +} + +function query_files($r) { + list($user, $user_submit) = authenticate_user($r, null); + $absent_files = array(); + $now = time(); + $delete_time = (int)$r->delete_time; + $batch_id = (int)$r->delete_time; + $fanout = parse_config(get_config(), ""); + $i = 0; + foreach($r->md5 as $f) { + $md5 = (string)$f; + echo "processing $md5\n"; + $fname = job_file_name($md5); + $path = dir_hier_path($fname, "../../download", $fanout); + + // if the job_file record is there, + // update the delete time first to avoid race condition + // with job file deleter + // + $job_file = BoincJobFile::lookup_md5($md5); + if ($job_file && $job_file->delete_time < $delete_time) { + $retval = $job_file::update("delete_time=$delete_time"); + if ($retval) { + xml_error(-1, "job_file::update() failed"); + } + } + if (file_exists($path)) { + // create the DB record if needed + // + if (!$job_file) { + BoincJobFile::insert( + "(md5, create_time, delete_time) values ('$md5', $now, $delete_time)" + ); + } + } else { + if ($job_file) { + $job_file->delete(); + } + $absent_files[] = $i; + } + $i++; + } + echo "\n"; + foreach ($absent_files as $i) { + echo "$i\n"; + } + echo "\n"; +} + +// upload_files +// in: list of MD5s, and the files themselves as multipart attachment +// out: error code +// +function upload_files($r) { + list($user, $user_submit) = authenticate_user($r, null); + $fanout = parse_config(get_config(), ""); + $delete_time = (int)$r->delete_time; + print_r($_FILES); + $i = 0; + foreach ($r->md5 as $f) { + $md5 = (string)$f; + $name = "file_$i"; + $tmp_name = $_FILES[$name]['tmp_name']; + if (!is_uploaded_file($tmp_name)) { + xml_error(-1, "$tmp_name is not an uploaded file"); + } + $fname = job_file_name($md5); + $path = dir_hier_path($fname, "../../download", $fanout); + rename($tmp_name, $path); + $now = time(); + $id = BoincJobFile::insert( + "(md5, create_time, delete_time) values ('$md5', $now, $delete_time)" + ); + if (!$id) { + xml_error(-1, "BoincJobFile::insert() failed"); + } + $i++; + } + echo "\n"; +} + +if (0) { +$r = simplexml_load_string("\n0\n 80bf244b43fb5d39541ea7011883b7e0\n a6037b05afb05f36e6a85a7c5138cbc1\n\n "); +submit_batch($r); +exit; +} +if (0) { + $r = simplexml_load_string("\n157f96a018b0b2f2b466e2ce3c7f54db\n1\n80bf244b43fb5d39541ea7011883b7e0\na6037b05afb05f36e6a85a7c5138cbc1\n"); + upload_files($r); + exit; +} + +xml_header(); +$r = simplexml_load_string($_POST['request']); +if (!$r) { + xml_error(-1, "can't parse request message"); +} + +switch($r->getName()) { +case 'query_files': + query_files($r); + break; +case 'upload_files': + echo "foo\n"; + upload_files($r); + break; +default: + xml_error(-1, "no such action"); +} + +?> diff --git a/samples/condor/curl.cpp b/samples/condor/curl.cpp index 1725afcaeb..a0635cce1f 100644 --- a/samples/condor/curl.cpp +++ b/samples/condor/curl.cpp @@ -138,11 +138,17 @@ int upload_files ( ) { char buf[1024]; string req_msg = "\n"; + sprintf(buf, "%s\n", authenticator); + req_msg += string(buf); + if (batch_id) { + sprintf(buf, "%d\n", batch_id); + req_msg += string(buf); + } for (unsigned int i=0; i%s\n", md5s[i].c_str()); req_msg += string(buf); } - req_msg = "\n"; + req_msg += "\n"; FILE* reply = tmpfile(); char url[256]; sprintf(url, "%sjob_file.php", project_url); @@ -151,8 +157,10 @@ int upload_files ( fclose(reply); return retval; } + fseek(reply, 0, SEEK_SET); bool success = false; while (fgets(buf, 256, reply)) { + printf("reply: %s", buf); if (strstr(buf, "success")) { success = true; break; From 6108deb7b0c2aadabab0cea8d871355a1423cbf0 Mon Sep 17 00:00:00 2001 From: David Anderson Date: Fri, 25 Jan 2013 11:37:33 -0800 Subject: [PATCH 3/9] - wrapper: use waitpid() instead of wait4(), which is missing on Android. --- samples/wrapper/wrapper.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/wrapper/wrapper.cpp b/samples/wrapper/wrapper.cpp index 00a68d1ade..fad7c29d6e 100644 --- a/samples/wrapper/wrapper.cpp +++ b/samples/wrapper/wrapper.cpp @@ -786,7 +786,7 @@ bool TASK::poll(int& status) { int wpid; struct rusage ru; - wpid = wait4(pid, &status, WNOHANG, &ru); + wpid = waitpid(pid, &status, WNOHANG); if (wpid) { getrusage(RUSAGE_CHILDREN, &ru); final_cpu_time = (float)ru.ru_utime.tv_sec + ((float)ru.ru_utime.tv_usec)/1e+6; From 986ea8707c84a4a8179ed8f5db4b84360f3c629e Mon Sep 17 00:00:00 2001 From: David Anderson Date: Fri, 25 Jan 2013 12:52:38 -0800 Subject: [PATCH 4/9] - client: if app does temporary exit, don't print premature exit warning --- client/app_control.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/client/app_control.cpp b/client/app_control.cpp index 196efe1753..03432607c8 100644 --- a/client/app_control.cpp +++ b/client/app_control.cpp @@ -420,8 +420,9 @@ void ACTIVE_TASK::handle_exited_app(int stat) { strcpy(buf, ""); if (temporary_exit_file_present(x, buf)) { handle_temporary_exit(will_restart, x, buf); + } else { + handle_premature_exit(will_restart); } - handle_premature_exit(will_restart); break; case 0xc000013a: // control-C?? case 0x40010004: // vista shutdown?? can someone explain this? From c17d20ccd908295809c4ea9a28cdc603c0ea908f Mon Sep 17 00:00:00 2001 From: David Anderson Date: Fri, 25 Jan 2013 14:30:34 -0800 Subject: [PATCH 5/9] - client: show sysmon messages correctly. This was supposed to be in my 507cd79 commit, but it got botched somehow. - client: the debug flag enables suspend/resume messages for both CPU and GPU. Previously CPU messages were always shown, and GPU messages were shown if was set. - client: fix bug where reschedule wasn't being done on GPU suspend or resume. --- client/client_state.cpp | 12 +++++++++++- client/client_state.h | 3 --- client/cs_prefs.cpp | 33 ++++++++++++++++++--------------- lib/cc_config.h | 2 +- 4 files changed, 30 insertions(+), 20 deletions(-) diff --git a/client/client_state.cpp b/client/client_state.cpp index 0ff7e6ff31..6e32dcf329 100644 --- a/client/client_state.cpp +++ b/client/client_state.cpp @@ -768,6 +768,13 @@ bool CLIENT_STATE::poll_slow_events() { start_cpu_benchmarks(); } +#ifdef _WIN32 + if (have_sysmon_msg) { + msg_printf(NULL, MSG_INFO, sysmon_msg); + have_sysmon_msg = false; + } +#endif + bool old_user_active = user_active; user_active = !host_info.users_idle( check_all_logins, global_prefs.idle_time_to_run @@ -836,7 +843,10 @@ bool CLIENT_STATE::poll_slow_events() { // first = false; if (suspend_reason) { - print_suspend_tasks_message(suspend_reason); + msg_printf(NULL, MSG_INFO, + "Suspending computation - %s", + suspend_reason_string(suspend_reason) + ); } } tasks_suspended = (suspend_reason != 0); diff --git a/client/client_state.h b/client/client_state.h index 822a6fb906..d27ffebd70 100644 --- a/client/client_state.h +++ b/client/client_state.h @@ -501,9 +501,6 @@ extern double calculate_exponential_backoff( int n, double MIN, double MAX ); -extern void print_suspend_tasks_message(int); - - //////// TIME-RELATED CONSTANTS //////////// //////// CLIENT INTERNAL diff --git a/client/cs_prefs.cpp b/client/cs_prefs.cpp index 0b0099b483..feb6235536 100644 --- a/client/cs_prefs.cpp +++ b/client/cs_prefs.cpp @@ -285,33 +285,34 @@ int CLIENT_STATE::check_suspend_processing() { } } - if (log_flags.cpu_sched) { - if (old_gpu_suspend_reason && !gpu_suspend_reason) { - msg_printf(NULL, MSG_INFO, "[cpu_sched] resuming GPU activity"); - request_schedule_cpus("GPU resumption"); - } else if (!old_gpu_suspend_reason && gpu_suspend_reason) { - msg_printf(NULL, MSG_INFO, "[cpu_sched] suspending GPU activity"); - request_schedule_cpus("GPU suspension"); + if (old_gpu_suspend_reason && !gpu_suspend_reason) { + if (log_flags.task) { + msg_printf(NULL, MSG_INFO, "[task] resuming GPU activity"); } + request_schedule_cpus("GPU resumption"); + } else if (!old_gpu_suspend_reason && gpu_suspend_reason) { + if (log_flags.task) { + msg_printf(NULL, MSG_INFO, "[task] suspending GPU activity"); + } + request_schedule_cpus("GPU suspension"); } } return 0; } - -void print_suspend_tasks_message(int reason) { - msg_printf(NULL, MSG_INFO, "Suspending computation - %s", suspend_reason_string(reason)); -} - - int CLIENT_STATE::suspend_tasks(int reason) { if (reason == SUSPEND_REASON_CPU_THROTTLE) { if (log_flags.cpu_sched) { msg_printf(NULL, MSG_INFO, "[cpu_sched] Suspending - CPU throttle"); } } else { - print_suspend_tasks_message(reason); + if (log_flags.task) { + msg_printf(NULL, MSG_INFO, + "[task] Suspending computation - %s", + suspend_reason_string(reason) + ); + } } active_tasks.suspend_all(reason); return 0; @@ -324,7 +325,9 @@ int CLIENT_STATE::resume_tasks(int reason) { } active_tasks.unsuspend_all(); } else { - msg_printf(NULL, MSG_INFO, "Resuming computation"); + if (log_flags.task) { + msg_printf(NULL, MSG_INFO, "[task] Resuming computation"); + } active_tasks.unsuspend_all(); request_schedule_cpus("Resuming computation"); } diff --git a/lib/cc_config.h b/lib/cc_config.h index dbdbe087b1..161cf2e1eb 100644 --- a/lib/cc_config.h +++ b/lib/cc_config.h @@ -46,7 +46,7 @@ struct LOG_FLAGS { bool sched_ops; // interactions with schedulers bool task; - // task start and finish + // task start and finish, and suspend/resume // off by default; intended for developers and testers // From 427176174b6dd2ca9dbe4871e7128ca1a5bde88f Mon Sep 17 00:00:00 2001 From: David Anderson Date: Fri, 25 Jan 2013 15:14:23 -0800 Subject: [PATCH 6/9] - user web: remove dead stats site links --- doc/links.php | 39 +++++++++++++++++++++------------------ html/inc/stats_sites.inc | 20 ++++++++++---------- 2 files changed, 31 insertions(+), 28 deletions(-) diff --git a/doc/links.php b/doc/links.php index 64b25acaa7..343cc925ea 100644 --- a/doc/links.php +++ b/doc/links.php @@ -34,7 +34,12 @@ $info_sites = array( "(survey of volunteer computing, including non-BOINC projects)" ), array( - "http://www.kd-web.info/clanky.php", + "http://www.rechenkraft.net/wiki/", + "Rechenkraft.net wiki", + "(German, English, Portuguese)" + ), + array( + "http://www.kd-web.info/#%21/boinc", "Flash-based BOINC tutorials", "(in Czech, English, and Slovak)" ), //array( @@ -56,21 +61,21 @@ $info_sites = array( "BOINC Argentina", "(in Spanish)", ), - array( - "http://faq.boinc.de/", - "Deutsche BOINC FAQ", - "(in German)", - ), - array( - "http://www.boincfrance.org/", - "BOINCFrance.org", - "(in French)", - ), - array( - "http://www.crunching-family.at/wiki/", - "Crunching Family Wiki", - "(In German)", - ), + //array( + // "http://faq.boinc.de/", + // "Deutsche BOINC FAQ", + // "(in German)", + //), + //array( + // "http://www.boincfrance.org/", + // "BOINCFrance.org", + // "(in French)", + //), + //array( + // "http://www.crunching-family.at/wiki/", + // "Crunching Family Wiki", + // "(In German)", + //), array( "http://www.angelfire.com/jkoulouris-boinc/", "The Big BOINC! Projects and Chronology Page", @@ -356,9 +361,7 @@ If you'd like to add a web site to this list, please

BOINC-related videos

"; diff --git a/html/inc/stats_sites.inc b/html/inc/stats_sites.inc index 39908930c2..697fd5c09e 100644 --- a/html/inc/stats_sites.inc +++ b/html/inc/stats_sites.inc @@ -98,15 +98,15 @@ $stats_sites = array( $team_stats_sites = array( array("http://stats.czechnationalteam.cz/", "Czech National Team", "(in Czech)"), array("http://www.boincitaly.org/", "BOINC.Italy"), - array("http://www.spacepage.be/component/option,com_boinc/", "Spacepage"), + //array("http://www.spacepage.be/component/option,com_boinc/", "Spacepage"), array("http://boinc.radax.net/de_boinc.htm", "BOINC@Austria"), - array("http://www.myboinc.com/scores/", "L'Alliance Francophone"), + //array("http://www.myboinc.com/scores/", "L'Alliance Francophone"), array("http://boincdenmark.dk/", "BOINC@Denmark", "(Danish)"), array("http://boincdenmark.dk/default_en.html", "BOINC@Denmark", "(English)"), - array("http://www.bigbee.be/comp/boinc/index.php", - "Boinc.be team stats", - "" - ), + //array("http://www.bigbee.be/comp/boinc/index.php", + // "Boinc.be team stats", + // "" + //), array("http://www.seti-teamartbell.com/", "Team Art Bell", ""), array("http://www.crunchers-freiburg.de/", "crunchers@freiburg", "(German)"), ); @@ -131,10 +131,10 @@ $sig_sites = array( "http://boinc.mundayweb.com", "(User-configurable stats counters. Cool!)" ), - array("http://www.bigbee.be/comp/boinc/", - "boinc.be", - "" - ), + //array("http://www.bigbee.be/comp/boinc/", + // "boinc.be", + // "" + //), array("http://www.boincstats.com/page/faq.php#3", "BOINCstats", "by Willy de Zutter" From 3321837b01844b3d1c876fc1586f9a8fd2804a37 Mon Sep 17 00:00:00 2001 From: David Anderson Date: Fri, 25 Jan 2013 22:43:57 -0800 Subject: [PATCH 7/9] - wrapper: fix CPU time accounting on Unix --- samples/condor/curl.cpp | 15 ++++++++------ samples/wrapper/job.xml | 13 ++++++------- samples/wrapper/wrapper.cpp | 39 ++++++++++++++++++++++++++++++++++--- 3 files changed, 51 insertions(+), 16 deletions(-) diff --git a/samples/condor/curl.cpp b/samples/condor/curl.cpp index a0635cce1f..ea858ff52a 100644 --- a/samples/condor/curl.cpp +++ b/samples/condor/curl.cpp @@ -174,7 +174,9 @@ int upload_files ( int create_batch( const char* project_url, const char* authenticator, - SUBMIT_REQ& sr + const char* batch_name, + const char* app_name, + int& batch_id ) { char request[1024]; char url[1024]; @@ -187,8 +189,8 @@ int create_batch( " \n" "\n", authenticator, - sr.batch_name, - sr.app_name + batch_name, + app_name ); sprintf(url, "%ssubmit_rpc_handler.php", project_url); FILE* reply = tmpfile(); @@ -199,13 +201,14 @@ int create_batch( return retval; } char buf[256]; - sr.batch_id = 0; + batch_id = 0; fseek(reply, 0, SEEK_SET); while (fgets(buf, 256, reply)) { - if (parse_int(buf, "", sr.batch_id)) break; + printf("reply: %s", buf); + if (parse_int(buf, "", batch_id)) break; } fclose(reply); - if (sr.batch_id == 0) { + if (batch_id == 0) { return -1; } return 0; diff --git a/samples/wrapper/job.xml b/samples/wrapper/job.xml index fcac9f18c1..4bc0cdfa24 100644 --- a/samples/wrapper/job.xml +++ b/samples/wrapper/job.xml @@ -5,11 +5,10 @@ stdout 10 - - in.zip - - - foo.zip - out - + + worker + stdin + stdout + 10 + diff --git a/samples/wrapper/wrapper.cpp b/samples/wrapper/wrapper.cpp index fad7c29d6e..81644f06be 100644 --- a/samples/wrapper/wrapper.cpp +++ b/samples/wrapper/wrapper.cpp @@ -66,11 +66,14 @@ #include "regexp.h" +//#define DEBUG +#if 1 +#define debug_msg(x) +#else inline void debug_msg(const char* x) { -#if 0 fprintf(stderr, "%s\n", x); -#endif } +#endif #define JOB_FILENAME "job.xml" #define CHECKPOINT_FILENAME "wrapper_checkpoint.txt" @@ -116,10 +119,11 @@ struct TASK { HANDLE pid_handle; DWORD pid; HANDLE thread_handle; - struct _stat last_stat; // mod time of checkpoint file + struct _stat last_stat; // mod time of checkpoint file #else int pid; struct stat last_stat; + double start_rusage; // getrusage() CPU time at start of task #endif bool stat_first; @@ -699,6 +703,10 @@ int TASK::run(int argct, char** argvt) { FILE* stdin_file; FILE* stderr_file; + struct rusage ru; + getrusage(RUSAGE_CHILDREN, &ru); + start_rusage = (float)ru.ru_utime.tv_sec + ((float)ru.ru_utime.tv_usec)/1e+6; + pid = fork(); if (pid == -1) { perror("fork(): "); @@ -790,6 +798,12 @@ bool TASK::poll(int& status) { if (wpid) { getrusage(RUSAGE_CHILDREN, &ru); final_cpu_time = (float)ru.ru_utime.tv_sec + ((float)ru.ru_utime.tv_usec)/1e+6; + final_cpu_time -= start_rusage; +#ifdef DEBUG + printf("process exited; current CPU %f final CPU %f\n", + current_cpu_time, final_cpu_time + ); +#endif if (final_cpu_time < current_cpu_time) { final_cpu_time = current_cpu_time; } @@ -1005,6 +1019,13 @@ int main(int argc, char** argv) { if (counter%10 == 0) { cpu_time = task.cpu_time(); } +#ifdef DEBUG + printf("cpu time %f, checkpoint CPU time %f frac done %f\n", + task.starting_cpu + cpu_time, + checkpoint_cpu_time, + frac_done + delta + ); +#endif boinc_report_app_status( task.starting_cpu + cpu_time, checkpoint_cpu_time, @@ -1019,6 +1040,18 @@ int main(int argc, char** argv) { counter++; } checkpoint_cpu_time = task.starting_cpu + task.final_cpu_time; +#ifdef DEBUG + printf("cpu time %f, checkpoint CPU time %f frac done %f\n", + task.starting_cpu + task.final_cpu_time, + checkpoint_cpu_time, + frac_done + task.weight/total_weight + ); +#endif + boinc_report_app_status( + task.starting_cpu + task.final_cpu_time, + checkpoint_cpu_time, + frac_done + task.weight/total_weight + ); write_checkpoint(i+1, checkpoint_cpu_time); weight_completed += task.weight; } From ad05a8b0b994868ac1157299071d125f6e326208 Mon Sep 17 00:00:00 2001 From: Joachim Fritzsch Date: Sun, 27 Jan 2013 13:53:38 +0100 Subject: [PATCH 8/9] - client: (android) add wifi location of Android 4 devices to detect wifi state properly. --- client/hostinfo_network.cpp | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/client/hostinfo_network.cpp b/client/hostinfo_network.cpp index 7d1178020a..692dd404d0 100644 --- a/client/hostinfo_network.cpp +++ b/client/hostinfo_network.cpp @@ -63,21 +63,27 @@ // if value cant be read, default return false // bool HOST_INFO::host_wifi_online() { - char wifipath[1024]; - snprintf(wifipath, sizeof(wifipath), "/sys/class/net/eth0/operstate"); + char wifipath_pri[1024]; + snprintf(wifipath_pri, sizeof(wifipath_pri), "/sys/class/net/eth0/operstate"); //location in Android 2.3 + char wifipath_sec[1024]; + snprintf(wifipath_sec, sizeof(wifipath_sec), "/sys/class/net/wlan0/operstate"); //location in Android 4 + + FILE *fsyswifi = fopen(wifipath_pri, "r"); + if(!fsyswifi) { //primary location not available, try _sec + fsyswifi = fopen(wifipath_sec, "r"); + } - FILE *fsyswifi = fopen(wifipath, "r"); char wifi_state[64]; - bool wifi_online = false; if (fsyswifi) { (void) fscanf(fsyswifi, "%s", &wifi_state); fclose(fsyswifi); + } else { + LOGD("wifi adapter not found!"); } if ((strcmp(wifi_state,"up")) == 0) { //operstate = up - LOGD("wifi is online"); wifi_online = true; } From f9d022788d4e1d35acbc00125bee4335d086f882 Mon Sep 17 00:00:00 2001 From: Joachim Fritzsch Date: Sun, 27 Jan 2013 18:23:01 +0100 Subject: [PATCH 9/9] - android: kill client process on startup to allow update - android: bugfix in preferences tab - android: rename of preferences - android: deletion of unused components --- .../BOINC/res/layout/status_layout_error.xml | 55 ------ .../res/layout/status_layout_launching.xml | 40 ---- .../res/layout/status_layout_noproject.xml | 108 ----------- android/BOINC/res/values/configuration.xml | 2 +- android/BOINC/res/values/strings.xml | 21 +-- .../berkeley/boinc/AndroidBOINCActivity.java | 174 ------------------ .../src/edu/berkeley/boinc/PrefsActivity.java | 12 +- .../edu/berkeley/boinc/client/Monitor.java | 91 ++++++--- .../edu/berkeley/boinc/debug/Debugging.java | 2 +- .../src/edu/berkeley/boinc/debug/Logging.java | 4 +- .../receiver/ClientStatusChangeReceiver.java | 41 ----- 11 files changed, 79 insertions(+), 471 deletions(-) delete mode 100644 android/BOINC/res/layout/status_layout_error.xml delete mode 100644 android/BOINC/res/layout/status_layout_launching.xml delete mode 100644 android/BOINC/res/layout/status_layout_noproject.xml delete mode 100644 android/BOINC/src/edu/berkeley/boinc/AndroidBOINCActivity.java delete mode 100644 android/BOINC/src/edu/berkeley/boinc/receiver/ClientStatusChangeReceiver.java diff --git a/android/BOINC/res/layout/status_layout_error.xml b/android/BOINC/res/layout/status_layout_error.xml deleted file mode 100644 index 95a311d510..0000000000 --- a/android/BOINC/res/layout/status_layout_error.xml +++ /dev/null @@ -1,55 +0,0 @@ - - - - - - - - - - - diff --git a/android/BOINC/res/layout/status_layout_launching.xml b/android/BOINC/res/layout/status_layout_launching.xml deleted file mode 100644 index d6665cb368..0000000000 --- a/android/BOINC/res/layout/status_layout_launching.xml +++ /dev/null @@ -1,40 +0,0 @@ - - - - - - - - - diff --git a/android/BOINC/res/layout/status_layout_noproject.xml b/android/BOINC/res/layout/status_layout_noproject.xml deleted file mode 100644 index cf9c485fc4..0000000000 --- a/android/BOINC/res/layout/status_layout_noproject.xml +++ /dev/null @@ -1,108 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - -