Rooba
/
boinc
mirror of https://github.com/BOINC/boinc.git
1
0
Fork 0
Code Issues 1 Packages Projects Releases Wiki Activity

- server: some stuff to prepare for distributed storage 

- don't create result records for uploads and downloads.
        Just create a msg_to_client record.
    - the scheduler handles file-transfer results specially;
        it makes a vector of them, then calls a project-supplied function
        handle_file_xfer_results()
    - change the interface and implementation of put_file and get_file
- client write project sched priority in GUI RPC replies,
    but not to the state file


svn path=/trunk/boinc/; revision=23857
This commit is contained in:
David Anderson 2011-07-19 20:52:41 +00:00
parent fd944052f2
commit 27e05a3da9
16 changed files with 194 additions and 177 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
checkin_notes
View File

@ -4182,3 +4182,31 @@ David 19 July 2011
sched/
credit.cpp
David 19 July 2011
- server: some stuff to prepare for distributed storage
- don't create result records for uploads and downloads.
Just create a msg_to_client record.
- the scheduler handles file-transfer results specially;
it makes a vector of them, then calls a project-supplied function
handle_file_xfer_results()
- change the interface and implementation of put_file and get_file
- client write project sched priority in GUI RPC replies,
but not to the state file
sched/
delete_file.cpp
put_file.cpp
get_file.cpp
sched_types.cpp,h
sched_customize.cpp,h
handle_request.cpp
credit_test.cpp
tools/
backend_lib.cpp,h
py/Boinc/
setup_project.py
client/
client_state.cpp
boinc_cmd.cpp
client_types.cpp

2
client/boinc_cmd.cpp
View File

@ -74,7 +74,7 @@ Commands:\n\
--get_tasks show tasks\n\
--join_acct_mgr URL name passwd attach account manager\n\
--lookup_account URL email passwd\n\
--network_available\n\
--network_available retry deferred network communication\n\
--project URL op project operation\n\
op = reset | detach | update | suspend | resume | nomorework | allowmorework\n\
--project_attach URL auth attach to project\n\

2
client/client_state.cpp
View File

@ -1532,7 +1532,6 @@ bool CLIENT_STATE::update_results() {
case RESULT_FILES_DOWNLOADING:
retval = input_files_available(rp, false);
if (!retval) {
rp->set_state(RESULT_FILES_DOWNLOADED, "CS::update_results");
if (rp->avp->app_files.size()==0) {
// if this is a file-transfer app, start the upload phase
//
@ -1541,6 +1540,7 @@ bool CLIENT_STATE::update_results() {
} else {
// else try to start the computation
//
rp->set_state(RESULT_FILES_DOWNLOADED, "CS::update_results");
request_schedule_cpus("files downloaded");
}
action = true;

7
client/client_types.cpp
View File

@ -388,7 +388,6 @@ int PROJECT::write_state(MIOFILE& out, bool gui_rpc) {
" <sched_rpc_pending>%d</sched_rpc_pending>\n"
" <send_time_stats_log>%d</send_time_stats_log>\n"
" <send_job_log>%d</send_job_log>\n"
" <sched_priority>%f</sched_priority>\n"
"%s%s%s%s%s%s%s%s%s%s%s%s%s",
master_url,
project_name,
@ -422,7 +421,6 @@ int PROJECT::write_state(MIOFILE& out, bool gui_rpc) {
sched_rpc_pending,
send_time_stats_log,
send_job_log,
project_priority(this),
anonymous_platform?" <anonymous_platform/>\n":"",
master_url_fetch_pending?" <master_url_fetch_pending/>\n":"",
trickle_up_pending?" <trickle_up_pending/>\n":"",
@ -480,10 +478,13 @@ int PROJECT::write_state(MIOFILE& out, bool gui_rpc) {
);
}
if (gui_rpc) {
out.printf("%s", gui_urls.c_str());
out.printf(
"%s"
" <sched_priority>%f</sched_priority>\n"
" <last_rpc_time>%f</last_rpc_time>\n"
" <project_files_downloaded_time>%f</project_files_downloaded_time>\n",
gui_urls.c_str(),
project_priority(this),
last_rpc_time,
project_files_downloaded_time
);

2
py/Boinc/setup_project.py
View File

@ -379,7 +379,7 @@ sys.path.insert(0, os.path.join('%s', 'py'))
'single_job_assimilator',
'assimilator.py', 'pymw_assimilator.py',
'update_stats', 'db_dump', 'db_purge', 'show_shmem', 'census',
'delete_file', 'request_file_list', 'get_file', 'send_file' ])
'delete_file', 'request_file_list', 'get_file', 'put_file' ])
map(lambda (s): install(srcdir('tools',s), dir('bin',s)),
[ 'appmgr', 'create_work', 'xadd', 'dbcheck_files_exist', 'run_in_ops',
'update_versions', 'parse_config', 'grep_logs', 'db_query',

1
sched/credit_test.cpp
View File

@ -15,7 +15,6 @@
#include "boinc_db.h"
#define MAX_JOBS 100000
#define COBBLESTONE_SCALE 100/86400e9
#define PRINT_AV_PERIOD 100
#define SCALE_AV_PERIOD 20

7
sched/delete_file.cpp
View File

@ -42,11 +42,10 @@
void usage(char* name) {
fprintf(stderr,
"Arrange to delete a file from a host.\n\n"
"Usage: %s OPTION...\n\n"
"Usage: delete_file [options] : delete a file from a host \n\n"
"Options:\n"
" --file_name F Specify te file to delete.\n"
" --host_id H Specify the coresponding host\n"
" --file_name F file name\n"
" --host_id H host DB ID\n"
" [-h | --help] Show this help text.\n"
" [-v | --version] Show version information.\n",
name

30
sched/get_file.cpp
View File

@ -15,12 +15,12 @@
// You should have received a copy of the GNU Lesser General Public License
// along with BOINC. If not, see <http://www.gnu.org/licenses/>.
// upload a file from a host
//
// get_file [options]
// --host_id N ID of host to upload from
// --file_name name name of specific file, dominates workunit
//
// Create a result entries, initialized to sent, and corresponding
// messages to the host that is assumed to have the file.
// --file_name name file name
// [ --url x ] URL of upload server (can specify several)
//
// Run from the project root dir.
@ -40,12 +40,11 @@
#include "svn_version.h"
void usage() {
fprintf(stderr, "Gets a file from a specific host.\n"
fprintf(stderr, "Gets a file from a host.\n"
"Usage: get_file [options]\n\n"
"Retrieve a file from a host.\n"
"Options:\n"
" --host_id id Host to get file from\n"
" --file_name name Name of filE\n"
" --host_id id host DB ID\n"
" --file_name name Name of file\n"
" [-- url X] URL of upload server\n"
" [ -v | --version ] Show version\n"
" [ -h | --help ] Show help\n"
);
@ -55,6 +54,7 @@ int main(int argc, char** argv) {
int i, retval;
char file_name[256];
int host_id;
vector<const char*> urls;
strcpy(file_name, "");
host_id = 0;
@ -80,6 +80,8 @@ int main(int argc, char** argv) {
} else if (is_arg(argv[i], "v") || is_arg(argv[i], "version")) {
printf("%s\n", SVN_VERSION);
exit(0);
} else if (is_arg(argv[i], "url")) {
urls.push_back(argv[++i]);
} else {
usage();
exit(1);
@ -105,9 +107,15 @@ int main(int argc, char** argv) {
exit(1);
}
retval = get_file(host_id, file_name);
if (urls.size() == 0) {
urls.push_back(config.upload_url);
}
retval = get_file(host_id, file_name, urls);
if (retval) {
fprintf(stderr, "get_file() failed: %s\n", boincerror(retval));
}
boinc_db.close();
return retval;
}
const char *BOINC_RCSID_37238a0141 = "$Id$";

1
sched/handle_request.cpp
View File

@ -1232,6 +1232,7 @@ void process_request(char* code_sign_key) {
update_n_jobs_today();
handle_results();
handle_file_xfer_results();
// Do this before resending lost jobs
//

73
sched/put_file.cpp
View File

@ -28,63 +28,66 @@
#include <time.h>
#include "backend_lib.h"
#include "md5_file.h"
#include "svn_version.h"
#include "filesys.h"
#include "sched_config.h"
#include "sched_util.h"
void usage() {
fprintf(stderr,
"Usage: put_file [options]\n\n"
"Arrange to send a file to a host.\n"
"put_file [options]: send a file to a host\n\n"
"Options:\n"
" --host_id id ID of host\n"
" --file_name name name of file to send\n"
" [ -h | --help ] Show this help text.\n"
" [ -v | --version ] Show version information.\n"
" --host_id id host DB ID\n"
" --file_name name file name\n"
" [--url X] file URL (can specify several)\n"
" [--md5 X] file MD5 (must specify if nonlocal)\n"
" [--nbytes X] file size (must specify if nonlocal)\n"
" [ -h | --help ] Show this help text.\n"
" [ -v | --version ] Show version information.\n"
);
exit(1);
}
int main(int argc, char** argv) {
int i, retval;
char file_name[256];
char file_name[256], url[1024], path[1024];
int host_id;
vector<const char*> urls;
double nbytes = -1;
char md5[256];
strcpy(file_name, "");
strcpy(md5, "");
host_id = 0;
check_stop_daemons();
for (i=1; i<argc; i++) {
if (is_arg(argv[i], "host_id")) {
if (!argv[++i]) {
fprintf(stderr, "%s requires an argument\n\n", argv[--i]);
usage();
exit(1);
}
if (!argv[++i]) usage();
host_id = atoi(argv[i]);
} else if (is_arg(argv[i], "file_name")) {
if (!argv[++i]) {
fprintf(stderr, "%s requires an argument\n\n", argv[--i]);
usage();
exit(1);
}
if (!argv[++i]) usage();
strcpy(file_name, argv[i]);
} else if (!strcmp(argv[i], "-h") || !strcmp(argv[i], "--help")) {
usage();
exit(0);
} else if (!strcmp(argv[i], "-v") || !strcmp(argv[i], "--version")) {
printf("%s\n", SVN_VERSION);
exit(0);
} else if (is_arg(argv[i], "url")) {
urls.push_back(argv[++i]);
} else if (is_arg(argv[i], "md5")) {
strcpy(md5, argv[++i]);
} else if (is_arg(argv[i], "nbytes")) {
nbytes = atof(argv[++i]);
} else {
usage();
exit(1);
}
}
if (!strlen(file_name)) {
usage();
exit(1);
}
retval = config.parse_file();
if (retval) {
@ -92,13 +95,37 @@ int main(int argc, char** argv) {
exit(1);
}
retval = boinc_db.open(config.db_name, config.db_host, config.db_user, config.db_passwd);
retval = boinc_db.open(
config.db_name, config.db_host, config.db_user, config.db_passwd
);
if (retval) {
fprintf(stderr, "boinc_db.open failed: %s\n", boincerror(retval));
exit(1);
}
retval = put_file(host_id, file_name);
if (urls.size() == 0) {
// The file is local.
// Make sure it's there, and compute its MD5
//
dir_hier_path(file_name, config.download_dir, config.uldl_dir_fanout, path);
if (!boinc_file_exists(path)) {
fprintf(stderr, "file not found: %s\n", path);
exit(1);
}
dir_hier_url(file_name, config.download_url, config.uldl_dir_fanout, url);
urls.push_back(url);
retval = md5_file(path, md5, nbytes);
if (retval) {
fprintf(stderr, "md5_file() failed: %s\n", boincerror(retval));
exit(1);
}
} else {
if (nbytes == -1 || !strlen(md5)) {
usage();
}
}
retval = put_file(host_id, file_name, urls, md5, nbytes);
boinc_db.close();
return retval;

9
sched/sched_customize.cpp
View File

@ -600,3 +600,12 @@ bool JOB::get_score() {
disk_usage = wu.rsc_disk_bound;
return true;
}
void handle_file_xfer_results() {
for (unsigned int i=0; i<g_request->file_xfer_results.size(); i++) {
RESULT& r = g_request->file_xfer_results[i];
log_messages.printf(MSG_NORMAL,
"completed file xfer %s\n", r.name
);
}
}

1
sched/sched_customize.h
View File

@ -58,3 +58,4 @@ extern GPU_REQUIREMENTS ati_requirements;
extern bool wu_is_infeasible_custom(WORKUNIT&, APP&, BEST_APP_VERSION&);
extern bool app_plan(SCHEDULER_REQUEST&, char* plan_class, HOST_USAGE&);
extern bool app_plan_uses_gpu(const char* plan_class);
extern void handle_file_xfer_results();

7
sched/sched_types.cpp
View File

@ -336,7 +336,12 @@ const char* SCHEDULER_REQUEST::parse(FILE* fin) {
continue;
}
if (match_tag(buf, "<result>")) {
result.parse_from_client(fin);
retval = result.parse_from_client(fin);
if (retval) continue;
if (strstr(result.name, "download") || strstr(result.name, "upload")) {
file_xfer_results.push_back(result);
continue;
}
#if 0 // enable if you need to limit CGI memory size
if (results.size() >= 1024) {
continue;

1
sched/sched_types.h
View File

@ -306,6 +306,7 @@ struct SCHEDULER_REQUEST {
COPROCS coprocs;
std::vector<SCHED_DB_RESULT> results;
// completed results being reported
std::vector<RESULT> file_xfer_results;
std::vector<MSG_FROM_HOST_DESC> msgs_from_host;
std::vector<FILE_INFO> file_infos;
// sticky files reported by host for locality scheduling

190
tools/backend_lib.cpp
View File

@ -703,48 +703,13 @@ int create_work(
// STUFF RELATED TO FILE UPLOAD/DOWNLOAD
int create_upload_result(
DB_RESULT& result, int host_id, const char * file_name
) {
int retval;
char result_xml[BLOB_SIZE];
result.clear();
sprintf(result.name, "get_%s_%d_%ld", file_name, host_id, time(0));
result.create_time = time(0);
result.server_state = RESULT_SERVER_STATE_IN_PROGRESS;
result.hostid = host_id;
result.outcome = RESULT_OUTCOME_INIT;
result.file_delete_state = ASSIMILATE_DONE;
result.validate_state = VALIDATE_STATE_NO_CHECK;
sprintf(result_xml,
"<result>\n"
" <wu_name>%s</wu_name>\n"
" <name>%s</wu_name>\n"
" <file_ref>\n"
" <file_name>%s</file_name>\n"
" </file_ref>\n"
"</result>\n",
result.name, result.name, file_name
);
strcpy(result.xml_doc_in, result_xml);
result.sent_time = time(0);
result.report_deadline = 0;
result.hostid = host_id;
retval = result.insert();
if (retval) {
fprintf(stderr, "result.insert(): %s\n", boincerror(retval));
return retval;
}
return 0;
}
int create_upload_message(
DB_RESULT& result, int host_id, const char* file_name
int get_file(
int host_id, const char* file_name, vector<const char*> urls
) {;
char buf[8192];
DB_MSG_TO_HOST mth;
int retval;
mth.clear();
mth.create_time = time(0);
mth.hostid = host_id;
@ -752,27 +717,41 @@ int create_upload_message(
mth.handled = false;
sprintf(mth.xml,
"<app>\n"
" <name>%s</name>\n"
" <name>file_xfer</name>\n"
"</app>\n"
"<app_version>\n"
" <app_name>%s</app_name>\n"
" <version_num>%d00</version_num>\n"
" <app_name>file_xfer</app_name>\n"
" <version_num>0</version_num>\n"
"</app_version>\n"
"<file_info>\n"
" <name>%s</name>\n"
" <url>%s</url>\n"
" <max_nbytes>%.0f</max_nbytes>\n"
" <name>%s</name>\n",
file_name
);
for (unsigned int i=0; i<urls.size(); i++) {
sprintf(buf, " <url>%s</url>\n", urls[i]);
strcat(mth.xml, buf);
}
sprintf(buf,
" <upload_when_present/>\n"
"</file_info>\n"
"%s"
"<workunit>\n"
" <name>%s</name>\n"
" <app_name>%s</app_name>\n"
"</workunit>",
FILE_MOVER, FILE_MOVER, BOINC_MAJOR_VERSION,
file_name, config.upload_url,
1e10, result.xml_doc_in, result.name, FILE_MOVER
" <name>upload_%s</name>\n"
" <app_name>file_xfer</app_name>\n"
"</workunit>\n"
"<result>\n"
" <wu_name>upload_%s</wu_name>\n"
" <name>upload_%s</name>\n"
" <file_ref>\n"
" <file_name>%s</file_name>\n"
" </file_ref>\n"
"</result>\n",
file_name,
file_name,
file_name,
file_name
);
strcat(mth.xml, buf);
retval = mth.insert();
if (retval) {
fprintf(stderr, "msg_to_host.insert(): %s\n", boincerror(retval));
@ -781,96 +760,58 @@ int create_upload_message(
return 0;
}
int get_file(int host_id, const char* file_name) {
DB_RESULT result;
int retval;
retval = create_upload_result(result, host_id, file_name);
if (retval) return retval;
retval = create_upload_message(result, host_id, file_name);
return retval;
}
int create_download_result(
DB_RESULT& result, int host_id, const char* file_name
int put_file(
int host_id, const char* file_name,
vector<const char*> urls, const char* md5, double nbytes
) {
int retval;
char result_xml[BLOB_SIZE];
result.clear();
sprintf(result.name, "put_%s_%d_%ld", file_name, host_id, time(0));
result.create_time = time(0);
result.server_state = RESULT_SERVER_STATE_IN_PROGRESS;
result.hostid = host_id;
result.outcome = RESULT_OUTCOME_INIT;
result.file_delete_state = ASSIMILATE_DONE;
result.validate_state = VALIDATE_STATE_NO_CHECK;
sprintf(result_xml,
"<result>\n"
" <wu_name>%s</wu_name>\n"
" <name>%s</name>\n"
"</result>\n",
result.name, result.name
);
strcpy(result.xml_doc_in, result_xml);
result.sent_time = time(0);
result.report_deadline = 0;
result.hostid = host_id;
retval = result.insert();
if (retval) {
fprintf(stderr, "result.insert(): %s\n", boincerror(retval));
return retval;
}
return 0;
}
int create_download_message(
DB_RESULT& result, int host_id, const char* file_name
) {;
char buf[8192];
DB_MSG_TO_HOST mth;
int retval;
double nbytes;
char dirpath[256], urlpath[256], path[256], md5[33];
strcpy(dirpath, config.download_dir);
strcpy(urlpath, config.download_url);
mth.clear();
mth.create_time = time(0);
mth.hostid = host_id;
strcpy(mth.variety, "file_xfer");
mth.handled = false;
sprintf(path, "%s/%s", dirpath, file_name);
retval = md5_file(path, md5, nbytes);
if (retval) {
fprintf(stderr, "md5_file() error: %s\n", boincerror(retval));
return retval;
}
sprintf(mth.xml,
"<app>\n"
" <name>%s</name>\n"
" <name>file_xfer</name>\n"
"</app>\n"
"<app_version>\n"
" <app_name>%s</app_name>\n"
" <version_num>%d00</version_num>\n"
" <app_name>file_xfer</app_name>\n"
" <version_num>0</version_num>\n"
"</app_version>\n"
"%s"
"<file_info>\n"
" <name>%s</name>\n"
" <url>%s/%s</url>\n"
" <name>%s</name>\n",
file_name
);
for (unsigned int i=0; i<urls.size(); i++) {
sprintf(buf, " <url>%s</url>\n", urls[i]);
strcat(mth.xml, buf);
}
sprintf(buf,
" <md5_cksum>%s</md5_cksum>\n"
" <nbytes>%.0f</nbytes>\n"
" <sticky/>\n"
"</file_info>\n"
"<workunit>\n"
" <name>%s</name>\n"
" <app_name>%s</app_name>\n"
" <name>download_%s</name>\n"
" <app_name>file_xfer</app_name>\n"
" <file_ref>\n"
" <file_name>%s</file_name>\n"
" <file_name>%s</file_name>\n"
" </file_ref>\n"
"</workunit>",
FILE_MOVER, FILE_MOVER, BOINC_MAJOR_VERSION, result.xml_doc_in,
file_name, urlpath, file_name, md5,
nbytes, result.name, FILE_MOVER, file_name
"</workunit>\n"
"<result>\n"
" <wu_name>download_%s</wu_name>\n"
" <name>download_%s</name>\n"
"</result>\n",
md5,
nbytes,
file_name,
file_name,
file_name,
file_name
);
strcat(mth.xml, buf);
retval = mth.insert();
if (retval) {
fprintf(stderr, "msg_to_host.insert(): %s\n", boincerror(retval));
@ -879,13 +820,4 @@ int create_download_message(
return 0;
}
int put_file(int host_id, const char* file_name) {
DB_RESULT result;
int retval;
retval = create_download_result(result, host_id, file_name);
if (retval) return retval;
retval = create_download_message(result, host_id, file_name);
return retval;
}
const char *BOINC_RCSID_b5f8b10eb5 = "$Id$";

10
tools/backend_lib.h
View File

@ -71,7 +71,13 @@ extern int create_work(
const char* additional_xml = NULL
);
extern int put_file(int host_id, const char* file_name);
extern int get_file(int host_id, const char* file_name);
extern int put_file(
int host_id, const char* file_name, vector<const char*> urls,
const char* md5, double nbytes
);
extern int get_file(
int host_id, const char* file_name, vector<const char*> urls
);
#endif