From 27e05a3da999876676b61360e2f1bd3aca5e577e Mon Sep 17 00:00:00 2001 From: David Anderson Date: Tue, 19 Jul 2011 20:52:41 +0000 Subject: [PATCH] - server: some stuff to prepare for distributed storage - don't create result records for uploads and downloads. Just create a msg_to_client record. - the scheduler handles file-transfer results specially; it makes a vector of them, then calls a project-supplied function handle_file_xfer_results() - change the interface and implementation of put_file and get_file - client write project sched priority in GUI RPC replies, but not to the state file svn path=/trunk/boinc/; revision=23857 --- checkin_notes | 28 ++++++ client/boinc_cmd.cpp | 2 +- client/client_state.cpp | 2 +- client/client_types.cpp | 7 +- py/Boinc/setup_project.py | 2 +- sched/credit_test.cpp | 1 - sched/delete_file.cpp | 7 +- sched/get_file.cpp | 30 +++--- sched/handle_request.cpp | 1 + sched/put_file.cpp | 73 ++++++++++----- sched/sched_customize.cpp | 9 ++ sched/sched_customize.h | 1 + sched/sched_types.cpp | 7 +- sched/sched_types.h | 1 + tools/backend_lib.cpp | 190 ++++++++++++-------------------------- tools/backend_lib.h | 10 +- 16 files changed, 194 insertions(+), 177 deletions(-) diff --git a/checkin_notes b/checkin_notes index 184adca446..33f0afb39f 100644 --- a/checkin_notes +++ b/checkin_notes @@ -4182,3 +4182,31 @@ David 19 July 2011 sched/ credit.cpp + +David 19 July 2011 + - server: some stuff to prepare for distributed storage + - don't create result records for uploads and downloads. + Just create a msg_to_client record. + - the scheduler handles file-transfer results specially; + it makes a vector of them, then calls a project-supplied function + handle_file_xfer_results() + - change the interface and implementation of put_file and get_file + - client write project sched priority in GUI RPC replies, + but not to the state file + + sched/ + delete_file.cpp + put_file.cpp + get_file.cpp + sched_types.cpp,h + sched_customize.cpp,h + handle_request.cpp + credit_test.cpp + tools/ + backend_lib.cpp,h + py/Boinc/ + setup_project.py + client/ + client_state.cpp + boinc_cmd.cpp + client_types.cpp diff --git a/client/boinc_cmd.cpp b/client/boinc_cmd.cpp index cd39027595..e0a2b261a0 100644 --- a/client/boinc_cmd.cpp +++ b/client/boinc_cmd.cpp @@ -74,7 +74,7 @@ Commands:\n\ --get_tasks show tasks\n\ --join_acct_mgr URL name passwd attach account manager\n\ --lookup_account URL email passwd\n\ - --network_available\n\ + --network_available retry deferred network communication\n\ --project URL op project operation\n\ op = reset | detach | update | suspend | resume | nomorework | allowmorework\n\ --project_attach URL auth attach to project\n\ diff --git a/client/client_state.cpp b/client/client_state.cpp index 2773cf8a80..f765fbee6a 100644 --- a/client/client_state.cpp +++ b/client/client_state.cpp @@ -1532,7 +1532,6 @@ bool CLIENT_STATE::update_results() { case RESULT_FILES_DOWNLOADING: retval = input_files_available(rp, false); if (!retval) { - rp->set_state(RESULT_FILES_DOWNLOADED, "CS::update_results"); if (rp->avp->app_files.size()==0) { // if this is a file-transfer app, start the upload phase // @@ -1541,6 +1540,7 @@ bool CLIENT_STATE::update_results() { } else { // else try to start the computation // + rp->set_state(RESULT_FILES_DOWNLOADED, "CS::update_results"); request_schedule_cpus("files downloaded"); } action = true; diff --git a/client/client_types.cpp b/client/client_types.cpp index a1b5336f82..c2ff467b29 100644 --- a/client/client_types.cpp +++ b/client/client_types.cpp @@ -388,7 +388,6 @@ int PROJECT::write_state(MIOFILE& out, bool gui_rpc) { " %d\n" " %d\n" " %d\n" - " %f\n" "%s%s%s%s%s%s%s%s%s%s%s%s%s", master_url, project_name, @@ -422,7 +421,6 @@ int PROJECT::write_state(MIOFILE& out, bool gui_rpc) { sched_rpc_pending, send_time_stats_log, send_job_log, - project_priority(this), anonymous_platform?" \n":"", master_url_fetch_pending?" \n":"", trickle_up_pending?" \n":"", @@ -480,10 +478,13 @@ int PROJECT::write_state(MIOFILE& out, bool gui_rpc) { ); } if (gui_rpc) { - out.printf("%s", gui_urls.c_str()); out.printf( + "%s" + " %f\n" " %f\n" " %f\n", + gui_urls.c_str(), + project_priority(this), last_rpc_time, project_files_downloaded_time ); diff --git a/py/Boinc/setup_project.py b/py/Boinc/setup_project.py index 1db2768ada..9170136c08 100644 --- a/py/Boinc/setup_project.py +++ b/py/Boinc/setup_project.py @@ -379,7 +379,7 @@ sys.path.insert(0, os.path.join('%s', 'py')) 'single_job_assimilator', 'assimilator.py', 'pymw_assimilator.py', 'update_stats', 'db_dump', 'db_purge', 'show_shmem', 'census', - 'delete_file', 'request_file_list', 'get_file', 'send_file' ]) + 'delete_file', 'request_file_list', 'get_file', 'put_file' ]) map(lambda (s): install(srcdir('tools',s), dir('bin',s)), [ 'appmgr', 'create_work', 'xadd', 'dbcheck_files_exist', 'run_in_ops', 'update_versions', 'parse_config', 'grep_logs', 'db_query', diff --git a/sched/credit_test.cpp b/sched/credit_test.cpp index 1dc1af7e08..85b275e0f3 100644 --- a/sched/credit_test.cpp +++ b/sched/credit_test.cpp @@ -15,7 +15,6 @@ #include "boinc_db.h" #define MAX_JOBS 100000 -#define COBBLESTONE_SCALE 100/86400e9 #define PRINT_AV_PERIOD 100 #define SCALE_AV_PERIOD 20 diff --git a/sched/delete_file.cpp b/sched/delete_file.cpp index e9ba753428..bc7a38ef45 100644 --- a/sched/delete_file.cpp +++ b/sched/delete_file.cpp @@ -42,11 +42,10 @@ void usage(char* name) { fprintf(stderr, - "Arrange to delete a file from a host.\n\n" - "Usage: %s OPTION...\n\n" + "Usage: delete_file [options] : delete a file from a host \n\n" "Options:\n" - " --file_name F Specify te file to delete.\n" - " --host_id H Specify the coresponding host\n" + " --file_name F file name\n" + " --host_id H host DB ID\n" " [-h | --help] Show this help text.\n" " [-v | --version] Show version information.\n", name diff --git a/sched/get_file.cpp b/sched/get_file.cpp index d06e40dbaa..1417fdcb93 100644 --- a/sched/get_file.cpp +++ b/sched/get_file.cpp @@ -15,12 +15,12 @@ // You should have received a copy of the GNU Lesser General Public License // along with BOINC. If not, see . +// upload a file from a host +// // get_file [options] // --host_id N ID of host to upload from -// --file_name name name of specific file, dominates workunit -// -// Create a result entries, initialized to sent, and corresponding -// messages to the host that is assumed to have the file. +// --file_name name file name +// [ --url x ] URL of upload server (can specify several) // // Run from the project root dir. @@ -40,12 +40,11 @@ #include "svn_version.h" void usage() { - fprintf(stderr, "Gets a file from a specific host.\n" + fprintf(stderr, "Gets a file from a host.\n" "Usage: get_file [options]\n\n" - "Retrieve a file from a host.\n" - "Options:\n" - " --host_id id Host to get file from\n" - " --file_name name Name of filE\n" + " --host_id id host DB ID\n" + " --file_name name Name of file\n" + " [-- url X] URL of upload server\n" " [ -v | --version ] Show version\n" " [ -h | --help ] Show help\n" ); @@ -55,6 +54,7 @@ int main(int argc, char** argv) { int i, retval; char file_name[256]; int host_id; + vector urls; strcpy(file_name, ""); host_id = 0; @@ -80,6 +80,8 @@ int main(int argc, char** argv) { } else if (is_arg(argv[i], "v") || is_arg(argv[i], "version")) { printf("%s\n", SVN_VERSION); exit(0); + } else if (is_arg(argv[i], "url")) { + urls.push_back(argv[++i]); } else { usage(); exit(1); @@ -105,9 +107,15 @@ int main(int argc, char** argv) { exit(1); } - retval = get_file(host_id, file_name); + if (urls.size() == 0) { + urls.push_back(config.upload_url); + } + + retval = get_file(host_id, file_name, urls); + if (retval) { + fprintf(stderr, "get_file() failed: %s\n", boincerror(retval)); + } boinc_db.close(); - return retval; } const char *BOINC_RCSID_37238a0141 = "$Id$"; diff --git a/sched/handle_request.cpp b/sched/handle_request.cpp index 6004ed984a..16d436dab5 100644 --- a/sched/handle_request.cpp +++ b/sched/handle_request.cpp @@ -1232,6 +1232,7 @@ void process_request(char* code_sign_key) { update_n_jobs_today(); handle_results(); + handle_file_xfer_results(); // Do this before resending lost jobs // diff --git a/sched/put_file.cpp b/sched/put_file.cpp index 573d38ee13..8b96f7475d 100644 --- a/sched/put_file.cpp +++ b/sched/put_file.cpp @@ -28,63 +28,66 @@ #include #include "backend_lib.h" +#include "md5_file.h" #include "svn_version.h" +#include "filesys.h" #include "sched_config.h" #include "sched_util.h" void usage() { fprintf(stderr, - "Usage: put_file [options]\n\n" - "Arrange to send a file to a host.\n" + "put_file [options]: send a file to a host\n\n" "Options:\n" - " --host_id id ID of host\n" - " --file_name name name of file to send\n" - " [ -h | --help ] Show this help text.\n" - " [ -v | --version ] Show version information.\n" + " --host_id id host DB ID\n" + " --file_name name file name\n" + " [--url X] file URL (can specify several)\n" + " [--md5 X] file MD5 (must specify if nonlocal)\n" + " [--nbytes X] file size (must specify if nonlocal)\n" + " [ -h | --help ] Show this help text.\n" + " [ -v | --version ] Show version information.\n" ); + exit(1); } int main(int argc, char** argv) { int i, retval; - char file_name[256]; + char file_name[256], url[1024], path[1024]; int host_id; + vector urls; + double nbytes = -1; + char md5[256]; strcpy(file_name, ""); + strcpy(md5, ""); host_id = 0; check_stop_daemons(); for (i=1; ifile_xfer_results.size(); i++) { + RESULT& r = g_request->file_xfer_results[i]; + log_messages.printf(MSG_NORMAL, + "completed file xfer %s\n", r.name + ); + } +} diff --git a/sched/sched_customize.h b/sched/sched_customize.h index 3c89a6fa0b..1b73caf26a 100644 --- a/sched/sched_customize.h +++ b/sched/sched_customize.h @@ -58,3 +58,4 @@ extern GPU_REQUIREMENTS ati_requirements; extern bool wu_is_infeasible_custom(WORKUNIT&, APP&, BEST_APP_VERSION&); extern bool app_plan(SCHEDULER_REQUEST&, char* plan_class, HOST_USAGE&); extern bool app_plan_uses_gpu(const char* plan_class); +extern void handle_file_xfer_results(); diff --git a/sched/sched_types.cpp b/sched/sched_types.cpp index 109f118ec2..3ada5aa88e 100644 --- a/sched/sched_types.cpp +++ b/sched/sched_types.cpp @@ -336,7 +336,12 @@ const char* SCHEDULER_REQUEST::parse(FILE* fin) { continue; } if (match_tag(buf, "")) { - result.parse_from_client(fin); + retval = result.parse_from_client(fin); + if (retval) continue; + if (strstr(result.name, "download") || strstr(result.name, "upload")) { + file_xfer_results.push_back(result); + continue; + } #if 0 // enable if you need to limit CGI memory size if (results.size() >= 1024) { continue; diff --git a/sched/sched_types.h b/sched/sched_types.h index cb27cf5ce5..dcccad3c3a 100644 --- a/sched/sched_types.h +++ b/sched/sched_types.h @@ -306,6 +306,7 @@ struct SCHEDULER_REQUEST { COPROCS coprocs; std::vector results; // completed results being reported + std::vector file_xfer_results; std::vector msgs_from_host; std::vector file_infos; // sticky files reported by host for locality scheduling diff --git a/tools/backend_lib.cpp b/tools/backend_lib.cpp index 538d183181..79a6a56171 100644 --- a/tools/backend_lib.cpp +++ b/tools/backend_lib.cpp @@ -703,48 +703,13 @@ int create_work( // STUFF RELATED TO FILE UPLOAD/DOWNLOAD -int create_upload_result( - DB_RESULT& result, int host_id, const char * file_name -) { - int retval; - char result_xml[BLOB_SIZE]; - - result.clear(); - sprintf(result.name, "get_%s_%d_%ld", file_name, host_id, time(0)); - result.create_time = time(0); - result.server_state = RESULT_SERVER_STATE_IN_PROGRESS; - result.hostid = host_id; - result.outcome = RESULT_OUTCOME_INIT; - result.file_delete_state = ASSIMILATE_DONE; - result.validate_state = VALIDATE_STATE_NO_CHECK; - - sprintf(result_xml, - "\n" - " %s\n" - " %s\n" - " \n" - " %s\n" - " \n" - "\n", - result.name, result.name, file_name - ); - strcpy(result.xml_doc_in, result_xml); - result.sent_time = time(0); - result.report_deadline = 0; - result.hostid = host_id; - retval = result.insert(); - if (retval) { - fprintf(stderr, "result.insert(): %s\n", boincerror(retval)); - return retval; - } - return 0; -} - -int create_upload_message( - DB_RESULT& result, int host_id, const char* file_name +int get_file( + int host_id, const char* file_name, vector urls ) {; + char buf[8192]; DB_MSG_TO_HOST mth; int retval; + mth.clear(); mth.create_time = time(0); mth.hostid = host_id; @@ -752,27 +717,41 @@ int create_upload_message( mth.handled = false; sprintf(mth.xml, "\n" - " %s\n" + " file_xfer\n" "\n" "\n" - " %s\n" - " %d00\n" + " file_xfer\n" + " 0\n" "\n" "\n" - " %s\n" - " %s\n" - " %.0f\n" + " %s\n", + file_name + ); + for (unsigned int i=0; i%s\n", urls[i]); + strcat(mth.xml, buf); + } + sprintf(buf, " \n" "\n" - "%s" "\n" - " %s\n" - " %s\n" - "", - FILE_MOVER, FILE_MOVER, BOINC_MAJOR_VERSION, - file_name, config.upload_url, - 1e10, result.xml_doc_in, result.name, FILE_MOVER + " upload_%s\n" + " file_xfer\n" + "\n" + "\n" + " upload_%s\n" + " upload_%s\n" + " \n" + " %s\n" + " \n" + "\n", + file_name, + file_name, + file_name, + file_name ); + strcat(mth.xml, buf); + retval = mth.insert(); if (retval) { fprintf(stderr, "msg_to_host.insert(): %s\n", boincerror(retval)); @@ -781,96 +760,58 @@ int create_upload_message( return 0; } -int get_file(int host_id, const char* file_name) { - DB_RESULT result; - int retval; - retval = create_upload_result(result, host_id, file_name); - if (retval) return retval; - retval = create_upload_message(result, host_id, file_name); - return retval; -} - -int create_download_result( - DB_RESULT& result, int host_id, const char* file_name +int put_file( + int host_id, const char* file_name, + vector urls, const char* md5, double nbytes ) { - int retval; - char result_xml[BLOB_SIZE]; - - result.clear(); - sprintf(result.name, "put_%s_%d_%ld", file_name, host_id, time(0)); - result.create_time = time(0); - result.server_state = RESULT_SERVER_STATE_IN_PROGRESS; - result.hostid = host_id; - result.outcome = RESULT_OUTCOME_INIT; - result.file_delete_state = ASSIMILATE_DONE; - result.validate_state = VALIDATE_STATE_NO_CHECK; - - sprintf(result_xml, - "\n" - " %s\n" - " %s\n" - "\n", - result.name, result.name - ); - strcpy(result.xml_doc_in, result_xml); - result.sent_time = time(0); - result.report_deadline = 0; - result.hostid = host_id; - retval = result.insert(); - if (retval) { - fprintf(stderr, "result.insert(): %s\n", boincerror(retval)); - return retval; - } - return 0; -} - -int create_download_message( - DB_RESULT& result, int host_id, const char* file_name -) {; + char buf[8192]; DB_MSG_TO_HOST mth; int retval; - double nbytes; - char dirpath[256], urlpath[256], path[256], md5[33]; - strcpy(dirpath, config.download_dir); - strcpy(urlpath, config.download_url); mth.clear(); mth.create_time = time(0); mth.hostid = host_id; strcpy(mth.variety, "file_xfer"); mth.handled = false; - sprintf(path, "%s/%s", dirpath, file_name); - retval = md5_file(path, md5, nbytes); - if (retval) { - fprintf(stderr, "md5_file() error: %s\n", boincerror(retval)); - return retval; - } sprintf(mth.xml, "\n" - " %s\n" + " file_xfer\n" "\n" "\n" - " %s\n" - " %d00\n" + " file_xfer\n" + " 0\n" "\n" - "%s" "\n" - " %s\n" - " %s/%s\n" + " %s\n", + file_name + ); + for (unsigned int i=0; i%s\n", urls[i]); + strcat(mth.xml, buf); + } + sprintf(buf, " %s\n" " %.0f\n" " \n" "\n" "\n" - " %s\n" - " %s\n" + " download_%s\n" + " file_xfer\n" " \n" - " %s\n" + " %s\n" " \n" - "", - FILE_MOVER, FILE_MOVER, BOINC_MAJOR_VERSION, result.xml_doc_in, - file_name, urlpath, file_name, md5, - nbytes, result.name, FILE_MOVER, file_name + "\n" + "\n" + " download_%s\n" + " download_%s\n" + "\n", + md5, + nbytes, + file_name, + file_name, + file_name, + file_name ); + strcat(mth.xml, buf); retval = mth.insert(); if (retval) { fprintf(stderr, "msg_to_host.insert(): %s\n", boincerror(retval)); @@ -879,13 +820,4 @@ int create_download_message( return 0; } -int put_file(int host_id, const char* file_name) { - DB_RESULT result; - int retval; - retval = create_download_result(result, host_id, file_name); - if (retval) return retval; - retval = create_download_message(result, host_id, file_name); - return retval; -} - const char *BOINC_RCSID_b5f8b10eb5 = "$Id$"; diff --git a/tools/backend_lib.h b/tools/backend_lib.h index 7c70db21c5..7a5e0b232c 100644 --- a/tools/backend_lib.h +++ b/tools/backend_lib.h @@ -71,7 +71,13 @@ extern int create_work( const char* additional_xml = NULL ); -extern int put_file(int host_id, const char* file_name); -extern int get_file(int host_id, const char* file_name); +extern int put_file( + int host_id, const char* file_name, vector urls, + const char* md5, double nbytes +); + +extern int get_file( + int host_id, const char* file_name, vector urls +); #endif