diff --git a/checkin_notes b/checkin_notes
index 184adca446..33f0afb39f 100644
--- a/checkin_notes
+++ b/checkin_notes
@@ -4182,3 +4182,31 @@ David 19 July 2011
sched/
credit.cpp
+
+David 19 July 2011
+ - server: some stuff to prepare for distributed storage
+ - don't create result records for uploads and downloads.
+ Just create a msg_to_client record.
+ - the scheduler handles file-transfer results specially;
+ it makes a vector of them, then calls a project-supplied function
+ handle_file_xfer_results()
+ - change the interface and implementation of put_file and get_file
+ - client write project sched priority in GUI RPC replies,
+ but not to the state file
+
+ sched/
+ delete_file.cpp
+ put_file.cpp
+ get_file.cpp
+ sched_types.cpp,h
+ sched_customize.cpp,h
+ handle_request.cpp
+ credit_test.cpp
+ tools/
+ backend_lib.cpp,h
+ py/Boinc/
+ setup_project.py
+ client/
+ client_state.cpp
+ boinc_cmd.cpp
+ client_types.cpp
diff --git a/client/boinc_cmd.cpp b/client/boinc_cmd.cpp
index cd39027595..e0a2b261a0 100644
--- a/client/boinc_cmd.cpp
+++ b/client/boinc_cmd.cpp
@@ -74,7 +74,7 @@ Commands:\n\
--get_tasks show tasks\n\
--join_acct_mgr URL name passwd attach account manager\n\
--lookup_account URL email passwd\n\
- --network_available\n\
+ --network_available retry deferred network communication\n\
--project URL op project operation\n\
op = reset | detach | update | suspend | resume | nomorework | allowmorework\n\
--project_attach URL auth attach to project\n\
diff --git a/client/client_state.cpp b/client/client_state.cpp
index 2773cf8a80..f765fbee6a 100644
--- a/client/client_state.cpp
+++ b/client/client_state.cpp
@@ -1532,7 +1532,6 @@ bool CLIENT_STATE::update_results() {
case RESULT_FILES_DOWNLOADING:
retval = input_files_available(rp, false);
if (!retval) {
- rp->set_state(RESULT_FILES_DOWNLOADED, "CS::update_results");
if (rp->avp->app_files.size()==0) {
// if this is a file-transfer app, start the upload phase
//
@@ -1541,6 +1540,7 @@ bool CLIENT_STATE::update_results() {
} else {
// else try to start the computation
//
+ rp->set_state(RESULT_FILES_DOWNLOADED, "CS::update_results");
request_schedule_cpus("files downloaded");
}
action = true;
diff --git a/client/client_types.cpp b/client/client_types.cpp
index a1b5336f82..c2ff467b29 100644
--- a/client/client_types.cpp
+++ b/client/client_types.cpp
@@ -388,7 +388,6 @@ int PROJECT::write_state(MIOFILE& out, bool gui_rpc) {
" %d\n"
" %d\n"
" %d\n"
- " %f\n"
"%s%s%s%s%s%s%s%s%s%s%s%s%s",
master_url,
project_name,
@@ -422,7 +421,6 @@ int PROJECT::write_state(MIOFILE& out, bool gui_rpc) {
sched_rpc_pending,
send_time_stats_log,
send_job_log,
- project_priority(this),
anonymous_platform?" \n":"",
master_url_fetch_pending?" \n":"",
trickle_up_pending?" \n":"",
@@ -480,10 +478,13 @@ int PROJECT::write_state(MIOFILE& out, bool gui_rpc) {
);
}
if (gui_rpc) {
- out.printf("%s", gui_urls.c_str());
out.printf(
+ "%s"
+ " %f\n"
" %f\n"
" %f\n",
+ gui_urls.c_str(),
+ project_priority(this),
last_rpc_time,
project_files_downloaded_time
);
diff --git a/py/Boinc/setup_project.py b/py/Boinc/setup_project.py
index 1db2768ada..9170136c08 100644
--- a/py/Boinc/setup_project.py
+++ b/py/Boinc/setup_project.py
@@ -379,7 +379,7 @@ sys.path.insert(0, os.path.join('%s', 'py'))
'single_job_assimilator',
'assimilator.py', 'pymw_assimilator.py',
'update_stats', 'db_dump', 'db_purge', 'show_shmem', 'census',
- 'delete_file', 'request_file_list', 'get_file', 'send_file' ])
+ 'delete_file', 'request_file_list', 'get_file', 'put_file' ])
map(lambda (s): install(srcdir('tools',s), dir('bin',s)),
[ 'appmgr', 'create_work', 'xadd', 'dbcheck_files_exist', 'run_in_ops',
'update_versions', 'parse_config', 'grep_logs', 'db_query',
diff --git a/sched/credit_test.cpp b/sched/credit_test.cpp
index 1dc1af7e08..85b275e0f3 100644
--- a/sched/credit_test.cpp
+++ b/sched/credit_test.cpp
@@ -15,7 +15,6 @@
#include "boinc_db.h"
#define MAX_JOBS 100000
-#define COBBLESTONE_SCALE 100/86400e9
#define PRINT_AV_PERIOD 100
#define SCALE_AV_PERIOD 20
diff --git a/sched/delete_file.cpp b/sched/delete_file.cpp
index e9ba753428..bc7a38ef45 100644
--- a/sched/delete_file.cpp
+++ b/sched/delete_file.cpp
@@ -42,11 +42,10 @@
void usage(char* name) {
fprintf(stderr,
- "Arrange to delete a file from a host.\n\n"
- "Usage: %s OPTION...\n\n"
+ "Usage: delete_file [options] : delete a file from a host \n\n"
"Options:\n"
- " --file_name F Specify te file to delete.\n"
- " --host_id H Specify the coresponding host\n"
+ " --file_name F file name\n"
+ " --host_id H host DB ID\n"
" [-h | --help] Show this help text.\n"
" [-v | --version] Show version information.\n",
name
diff --git a/sched/get_file.cpp b/sched/get_file.cpp
index d06e40dbaa..1417fdcb93 100644
--- a/sched/get_file.cpp
+++ b/sched/get_file.cpp
@@ -15,12 +15,12 @@
// You should have received a copy of the GNU Lesser General Public License
// along with BOINC. If not, see .
+// upload a file from a host
+//
// get_file [options]
// --host_id N ID of host to upload from
-// --file_name name name of specific file, dominates workunit
-//
-// Create a result entries, initialized to sent, and corresponding
-// messages to the host that is assumed to have the file.
+// --file_name name file name
+// [ --url x ] URL of upload server (can specify several)
//
// Run from the project root dir.
@@ -40,12 +40,11 @@
#include "svn_version.h"
void usage() {
- fprintf(stderr, "Gets a file from a specific host.\n"
+ fprintf(stderr, "Gets a file from a host.\n"
"Usage: get_file [options]\n\n"
- "Retrieve a file from a host.\n"
- "Options:\n"
- " --host_id id Host to get file from\n"
- " --file_name name Name of filE\n"
+ " --host_id id host DB ID\n"
+ " --file_name name Name of file\n"
+ " [-- url X] URL of upload server\n"
" [ -v | --version ] Show version\n"
" [ -h | --help ] Show help\n"
);
@@ -55,6 +54,7 @@ int main(int argc, char** argv) {
int i, retval;
char file_name[256];
int host_id;
+ vector urls;
strcpy(file_name, "");
host_id = 0;
@@ -80,6 +80,8 @@ int main(int argc, char** argv) {
} else if (is_arg(argv[i], "v") || is_arg(argv[i], "version")) {
printf("%s\n", SVN_VERSION);
exit(0);
+ } else if (is_arg(argv[i], "url")) {
+ urls.push_back(argv[++i]);
} else {
usage();
exit(1);
@@ -105,9 +107,15 @@ int main(int argc, char** argv) {
exit(1);
}
- retval = get_file(host_id, file_name);
+ if (urls.size() == 0) {
+ urls.push_back(config.upload_url);
+ }
+
+ retval = get_file(host_id, file_name, urls);
+ if (retval) {
+ fprintf(stderr, "get_file() failed: %s\n", boincerror(retval));
+ }
boinc_db.close();
- return retval;
}
const char *BOINC_RCSID_37238a0141 = "$Id$";
diff --git a/sched/handle_request.cpp b/sched/handle_request.cpp
index 6004ed984a..16d436dab5 100644
--- a/sched/handle_request.cpp
+++ b/sched/handle_request.cpp
@@ -1232,6 +1232,7 @@ void process_request(char* code_sign_key) {
update_n_jobs_today();
handle_results();
+ handle_file_xfer_results();
// Do this before resending lost jobs
//
diff --git a/sched/put_file.cpp b/sched/put_file.cpp
index 573d38ee13..8b96f7475d 100644
--- a/sched/put_file.cpp
+++ b/sched/put_file.cpp
@@ -28,63 +28,66 @@
#include
#include "backend_lib.h"
+#include "md5_file.h"
#include "svn_version.h"
+#include "filesys.h"
#include "sched_config.h"
#include "sched_util.h"
void usage() {
fprintf(stderr,
- "Usage: put_file [options]\n\n"
- "Arrange to send a file to a host.\n"
+ "put_file [options]: send a file to a host\n\n"
"Options:\n"
- " --host_id id ID of host\n"
- " --file_name name name of file to send\n"
- " [ -h | --help ] Show this help text.\n"
- " [ -v | --version ] Show version information.\n"
+ " --host_id id host DB ID\n"
+ " --file_name name file name\n"
+ " [--url X] file URL (can specify several)\n"
+ " [--md5 X] file MD5 (must specify if nonlocal)\n"
+ " [--nbytes X] file size (must specify if nonlocal)\n"
+ " [ -h | --help ] Show this help text.\n"
+ " [ -v | --version ] Show version information.\n"
);
+ exit(1);
}
int main(int argc, char** argv) {
int i, retval;
- char file_name[256];
+ char file_name[256], url[1024], path[1024];
int host_id;
+ vector urls;
+ double nbytes = -1;
+ char md5[256];
strcpy(file_name, "");
+ strcpy(md5, "");
host_id = 0;
check_stop_daemons();
for (i=1; ifile_xfer_results.size(); i++) {
+ RESULT& r = g_request->file_xfer_results[i];
+ log_messages.printf(MSG_NORMAL,
+ "completed file xfer %s\n", r.name
+ );
+ }
+}
diff --git a/sched/sched_customize.h b/sched/sched_customize.h
index 3c89a6fa0b..1b73caf26a 100644
--- a/sched/sched_customize.h
+++ b/sched/sched_customize.h
@@ -58,3 +58,4 @@ extern GPU_REQUIREMENTS ati_requirements;
extern bool wu_is_infeasible_custom(WORKUNIT&, APP&, BEST_APP_VERSION&);
extern bool app_plan(SCHEDULER_REQUEST&, char* plan_class, HOST_USAGE&);
extern bool app_plan_uses_gpu(const char* plan_class);
+extern void handle_file_xfer_results();
diff --git a/sched/sched_types.cpp b/sched/sched_types.cpp
index 109f118ec2..3ada5aa88e 100644
--- a/sched/sched_types.cpp
+++ b/sched/sched_types.cpp
@@ -336,7 +336,12 @@ const char* SCHEDULER_REQUEST::parse(FILE* fin) {
continue;
}
if (match_tag(buf, "")) {
- result.parse_from_client(fin);
+ retval = result.parse_from_client(fin);
+ if (retval) continue;
+ if (strstr(result.name, "download") || strstr(result.name, "upload")) {
+ file_xfer_results.push_back(result);
+ continue;
+ }
#if 0 // enable if you need to limit CGI memory size
if (results.size() >= 1024) {
continue;
diff --git a/sched/sched_types.h b/sched/sched_types.h
index cb27cf5ce5..dcccad3c3a 100644
--- a/sched/sched_types.h
+++ b/sched/sched_types.h
@@ -306,6 +306,7 @@ struct SCHEDULER_REQUEST {
COPROCS coprocs;
std::vector results;
// completed results being reported
+ std::vector file_xfer_results;
std::vector msgs_from_host;
std::vector file_infos;
// sticky files reported by host for locality scheduling
diff --git a/tools/backend_lib.cpp b/tools/backend_lib.cpp
index 538d183181..79a6a56171 100644
--- a/tools/backend_lib.cpp
+++ b/tools/backend_lib.cpp
@@ -703,48 +703,13 @@ int create_work(
// STUFF RELATED TO FILE UPLOAD/DOWNLOAD
-int create_upload_result(
- DB_RESULT& result, int host_id, const char * file_name
-) {
- int retval;
- char result_xml[BLOB_SIZE];
-
- result.clear();
- sprintf(result.name, "get_%s_%d_%ld", file_name, host_id, time(0));
- result.create_time = time(0);
- result.server_state = RESULT_SERVER_STATE_IN_PROGRESS;
- result.hostid = host_id;
- result.outcome = RESULT_OUTCOME_INIT;
- result.file_delete_state = ASSIMILATE_DONE;
- result.validate_state = VALIDATE_STATE_NO_CHECK;
-
- sprintf(result_xml,
- "\n"
- " %s\n"
- " %s\n"
- " \n"
- " %s\n"
- " \n"
- "\n",
- result.name, result.name, file_name
- );
- strcpy(result.xml_doc_in, result_xml);
- result.sent_time = time(0);
- result.report_deadline = 0;
- result.hostid = host_id;
- retval = result.insert();
- if (retval) {
- fprintf(stderr, "result.insert(): %s\n", boincerror(retval));
- return retval;
- }
- return 0;
-}
-
-int create_upload_message(
- DB_RESULT& result, int host_id, const char* file_name
+int get_file(
+ int host_id, const char* file_name, vector urls
) {;
+ char buf[8192];
DB_MSG_TO_HOST mth;
int retval;
+
mth.clear();
mth.create_time = time(0);
mth.hostid = host_id;
@@ -752,27 +717,41 @@ int create_upload_message(
mth.handled = false;
sprintf(mth.xml,
"\n"
- " %s\n"
+ " file_xfer\n"
"\n"
"\n"
- " %s\n"
- " %d00\n"
+ " file_xfer\n"
+ " 0\n"
"\n"
"\n"
- " %s\n"
- " %s\n"
- " %.0f\n"
+ " %s\n",
+ file_name
+ );
+ for (unsigned int i=0; i%s\n", urls[i]);
+ strcat(mth.xml, buf);
+ }
+ sprintf(buf,
" \n"
"\n"
- "%s"
"\n"
- " %s\n"
- " %s\n"
- "",
- FILE_MOVER, FILE_MOVER, BOINC_MAJOR_VERSION,
- file_name, config.upload_url,
- 1e10, result.xml_doc_in, result.name, FILE_MOVER
+ " upload_%s\n"
+ " file_xfer\n"
+ "\n"
+ "\n"
+ " upload_%s\n"
+ " upload_%s\n"
+ " \n"
+ " %s\n"
+ " \n"
+ "\n",
+ file_name,
+ file_name,
+ file_name,
+ file_name
);
+ strcat(mth.xml, buf);
+
retval = mth.insert();
if (retval) {
fprintf(stderr, "msg_to_host.insert(): %s\n", boincerror(retval));
@@ -781,96 +760,58 @@ int create_upload_message(
return 0;
}
-int get_file(int host_id, const char* file_name) {
- DB_RESULT result;
- int retval;
- retval = create_upload_result(result, host_id, file_name);
- if (retval) return retval;
- retval = create_upload_message(result, host_id, file_name);
- return retval;
-}
-
-int create_download_result(
- DB_RESULT& result, int host_id, const char* file_name
+int put_file(
+ int host_id, const char* file_name,
+ vector urls, const char* md5, double nbytes
) {
- int retval;
- char result_xml[BLOB_SIZE];
-
- result.clear();
- sprintf(result.name, "put_%s_%d_%ld", file_name, host_id, time(0));
- result.create_time = time(0);
- result.server_state = RESULT_SERVER_STATE_IN_PROGRESS;
- result.hostid = host_id;
- result.outcome = RESULT_OUTCOME_INIT;
- result.file_delete_state = ASSIMILATE_DONE;
- result.validate_state = VALIDATE_STATE_NO_CHECK;
-
- sprintf(result_xml,
- "\n"
- " %s\n"
- " %s\n"
- "\n",
- result.name, result.name
- );
- strcpy(result.xml_doc_in, result_xml);
- result.sent_time = time(0);
- result.report_deadline = 0;
- result.hostid = host_id;
- retval = result.insert();
- if (retval) {
- fprintf(stderr, "result.insert(): %s\n", boincerror(retval));
- return retval;
- }
- return 0;
-}
-
-int create_download_message(
- DB_RESULT& result, int host_id, const char* file_name
-) {;
+ char buf[8192];
DB_MSG_TO_HOST mth;
int retval;
- double nbytes;
- char dirpath[256], urlpath[256], path[256], md5[33];
- strcpy(dirpath, config.download_dir);
- strcpy(urlpath, config.download_url);
mth.clear();
mth.create_time = time(0);
mth.hostid = host_id;
strcpy(mth.variety, "file_xfer");
mth.handled = false;
- sprintf(path, "%s/%s", dirpath, file_name);
- retval = md5_file(path, md5, nbytes);
- if (retval) {
- fprintf(stderr, "md5_file() error: %s\n", boincerror(retval));
- return retval;
- }
sprintf(mth.xml,
"\n"
- " %s\n"
+ " file_xfer\n"
"\n"
"\n"
- " %s\n"
- " %d00\n"
+ " file_xfer\n"
+ " 0\n"
"\n"
- "%s"
"\n"
- " %s\n"
- " %s/%s\n"
+ " %s\n",
+ file_name
+ );
+ for (unsigned int i=0; i%s\n", urls[i]);
+ strcat(mth.xml, buf);
+ }
+ sprintf(buf,
" %s\n"
" %.0f\n"
" \n"
"\n"
"\n"
- " %s\n"
- " %s\n"
+ " download_%s\n"
+ " file_xfer\n"
" \n"
- " %s\n"
+ " %s\n"
" \n"
- "",
- FILE_MOVER, FILE_MOVER, BOINC_MAJOR_VERSION, result.xml_doc_in,
- file_name, urlpath, file_name, md5,
- nbytes, result.name, FILE_MOVER, file_name
+ "\n"
+ "\n"
+ " download_%s\n"
+ " download_%s\n"
+ "\n",
+ md5,
+ nbytes,
+ file_name,
+ file_name,
+ file_name,
+ file_name
);
+ strcat(mth.xml, buf);
retval = mth.insert();
if (retval) {
fprintf(stderr, "msg_to_host.insert(): %s\n", boincerror(retval));
@@ -879,13 +820,4 @@ int create_download_message(
return 0;
}
-int put_file(int host_id, const char* file_name) {
- DB_RESULT result;
- int retval;
- retval = create_download_result(result, host_id, file_name);
- if (retval) return retval;
- retval = create_download_message(result, host_id, file_name);
- return retval;
-}
-
const char *BOINC_RCSID_b5f8b10eb5 = "$Id$";
diff --git a/tools/backend_lib.h b/tools/backend_lib.h
index 7c70db21c5..7a5e0b232c 100644
--- a/tools/backend_lib.h
+++ b/tools/backend_lib.h
@@ -71,7 +71,13 @@ extern int create_work(
const char* additional_xml = NULL
);
-extern int put_file(int host_id, const char* file_name);
-extern int get_file(int host_id, const char* file_name);
+extern int put_file(
+ int host_id, const char* file_name, vector urls,
+ const char* md5, double nbytes
+);
+
+extern int get_file(
+ int host_id, const char* file_name, vector urls
+);
#endif