From 6ffee81198125adf77109ba7fdc3e03b5ceaec82 Mon Sep 17 00:00:00 2001 From: Hamid Aghdaee Date: Fri, 20 Dec 2002 02:12:27 +0000 Subject: [PATCH] *** empty log message *** svn path=/trunk/boinc/; revision=781 --- checkin_notes | 12 ++++++ client/pers_file_xfer.h | 10 ++++- client/scheduler_op.C | 2 +- client/scheduler_op.h | 16 +++++++- lib/parse.C | 4 +- lib/parse.h | 2 + sched/file_deleter.C | 11 ++++-- sched/make_work.C | 84 ++++++++++++++++++++++++++++++++--------- test/test.inc | 8 ++-- test/test_loop.php | 1 + test/test_uc.php | 2 +- tools/country_select.C | 1 + 12 files changed, 120 insertions(+), 33 deletions(-) diff --git a/checkin_notes b/checkin_notes index 9a921aee5c..c1345c3a37 100755 --- a/checkin_notes +++ b/checkin_notes @@ -2745,6 +2745,17 @@ David Dec 19 2002 tools/ country_select.C +Hamid Dec 19 2002 + + -Edited make_work.C, now when a new work_unit is created, it's input files are copied along with it, with some random numbers added to the end, so for each new work_unit there are a set of associated new input files + + -fixed a little bug in file_deleter + + -note: strtok() changes the original string passed to it, kind of dangerous to use + + -from parse.h : replace_element() is now in use ( I used it for make_work.C) + + David Dec 19 2002 - added -add_new_project option to command line version changed func name from get_initial_project() to add_new_project() @@ -2756,3 +2767,4 @@ David Dec 19 2002 scheduler_op.h win/ wingui.cpp,h + diff --git a/client/pers_file_xfer.h b/client/pers_file_xfer.h index 6fe60c1552..712094e060 100644 --- a/client/pers_file_xfer.h +++ b/client/pers_file_xfer.h @@ -21,6 +21,8 @@ #include "client_types.h" #include "file_xfer.h" + +#define DEBUG // PERS_FILE_XFER represents a persistent file transfer. // A set of URL is given in the FILE_INFO. @@ -32,12 +34,16 @@ // For upload, try to upload the file to the first URL; // if that fails try the others. -/*#define PERS_RETRY_DELAY_MIN 60 // 1 minute +#define PERS_RETRY_DELAY_MIN 60 // 1 minute #define PERS_RETRY_DELAY_MAX (60*60*4) // 4 hours -#define PERS_GIVEUP (60*60*24*7*2) // 2 weeks */ +#define PERS_GIVEUP (60*60*24*7*2) // 2 weeks + +#ifdef DEBUG #define PERS_RETRY_DELAY_MIN 1 #define PERS_RETRY_DELAY_MAX 30 #define PERS_GIVEUP 30 +#endif + // give up on xfer if this time elapses since last byte xferred class PERS_FILE_XFER { diff --git a/client/scheduler_op.C b/client/scheduler_op.C index c89473d80a..5b0b398393 100644 --- a/client/scheduler_op.C +++ b/client/scheduler_op.C @@ -123,7 +123,7 @@ int SCHEDULER_OP::set_min_rpc_time(PROJECT* p) { else { //backoff RETRY_BASE_PERIOD * e^nrpc_failures * random x = RETRY_BASE_PERIOD * exp(((double)rand()/(double)RAND_MAX) * n); - exp_backoff = (int)max(PERS_RETRY_DELAY_MIN,min(PERS_RETRY_DELAY_MAX,(int) x)); + exp_backoff = (int)max(SCHED_RETRY_DELAY_MIN,min(SCHED_RETRY_DELAY_MAX,(int) x)); p->min_rpc_time = time(0) + exp_backoff; } diff --git a/client/scheduler_op.h b/client/scheduler_op.h index db3dca8654..90c90240a0 100644 --- a/client/scheduler_op.h +++ b/client/scheduler_op.h @@ -17,6 +17,8 @@ // Contributor(s): // +#define DEBUG + #ifndef _SCHEDULER_OP_ #define _SCHEDULER_OP_ @@ -51,8 +53,18 @@ // This is the Max on the time to wait after we've contacted the Master URL MASTER_FETCH_RETRY_CAP times. //The next two constants are used to bound RPC exponential waiting. -#define PERS_RETRY_DELAY_MIN 60 // 1 minute -#define PERS_RETRY_DELAY_MAX (60*60*4) // 4 hours +#define SCHED_RETRY_DELAY_MIN 60 // 1 minute +#define SCHED_RETRY_DELAY_MAX (60*60*4) // 4 hours + +#ifdef DEBUG +#define MASTER_FETCH_PERIOD 5 +#define RETRY_BASE_PERIOD 1 +#define RETRY_CAP 5 +#define MASTER_FETCH_RETRY_CAP 3 +#define MASTER_FETCH_INTERVAL 5 +#define SCHED_RETRY_DELAY_MIN 1 +#define SCHED_RETRY_DELAY_MAX 30 +#endif #define SCHEDULER_OP_STATE_IDLE 0 #define SCHEDULER_OP_STATE_GET_MASTER 1 diff --git a/lib/parse.C b/lib/parse.C index 26be75030a..33e189adc9 100644 --- a/lib/parse.C +++ b/lib/parse.C @@ -147,7 +147,7 @@ int read_file_malloc(char* pathname, char*& str) { return 0; } -#if 0 + // replace XML element contents. not currently used // void replace_element(char* buf, char* start, char* end, char* replacement) { @@ -160,4 +160,4 @@ void replace_element(char* buf, char* start, char* end, char* replacement) { strcpy(p, replacement); strcat(p, temp); } -#endif + diff --git a/lib/parse.h b/lib/parse.h index 57e672f974..d4ef0d371a 100644 --- a/lib/parse.h +++ b/lib/parse.h @@ -18,6 +18,7 @@ // #include +#include extern bool parse(char*, char*); extern bool parse_int(char*, char*, int&); @@ -29,3 +30,4 @@ extern void copy_stream(FILE* in, FILE* out); extern void strcatdup(char*& p, char* buf); extern int dup_element_contents(FILE* in, char* end_tag, char** pp); extern int read_file_malloc(char* pathname, char*& str); +extern void replace_element(char* buf, char* start, char* end, char* replacement); diff --git a/sched/file_deleter.C b/sched/file_deleter.C index 0e8be69dc4..f76f274fa3 100644 --- a/sched/file_deleter.C +++ b/sched/file_deleter.C @@ -10,10 +10,12 @@ CONFIG config; int wu_delete_files(WORKUNIT& wu) { char* p; - char filename[256], pathname[256]; + char filename[256], pathname[256], buf[MAX_BLOB_SIZE]; bool no_delete; - p = strtok(wu.xml_doc, "\n"); + strcpy(buf,wu.xml_doc); + + p = strtok(buf, "\n"); strcpy(filename, ""); while (p) { if (parse_str(p, "", filename, sizeof(filename))) { @@ -36,10 +38,11 @@ int wu_delete_files(WORKUNIT& wu) { int result_delete_files(RESULT& result) { char* p; - char filename[256], pathname[256]; + char filename[256], pathname[256], buf[MAX_BLOB_SIZE]; bool no_delete; - p = strtok(result.xml_doc_in, "\n"); + strcpy(buf,result.xml_doc_in); + p = strtok(buf,"\n"); while (p) { if (parse_str(p, "", filename, sizeof(filename))) { } else if (match_tag(p, "")) { diff --git a/sched/make_work.C b/sched/make_work.C index 8f71b9370d..afbb97b7f4 100644 --- a/sched/make_work.C +++ b/sched/make_work.C @@ -38,6 +38,7 @@ #include "crypt.h" #include "backend_lib.h" #include "config.h" +#include "parse.h" #define TRIGGER_FILENAME "stop_server" @@ -45,6 +46,39 @@ int cushion = 10; int redundancy = 10; char wu_name[256], result_template_file[256]; +void replace_file_name(char * xml_doc, char * filename, char * new_filename,char * download_url) +{ + char buf[MAX_BLOB_SIZE], temp[256], download_path[256], new_download_path[256]; + char * p; + + sprintf(download_path,"%s/%s",download_url,filename); + sprintf(new_download_path,"%s/%s",download_url,new_filename); + strcpy(buf,xml_doc); + p = strtok(buf,"\n"); + while (p) { + if (parse_str(p, "", temp, sizeof(temp))) { + if(!strcmp(filename, temp)) + { + replace_element(xml_doc + (p - buf),"","",new_filename); + } + } + else if (parse_str(p, "", temp, sizeof(temp))) { + if(!strcmp(filename, temp)) + { + replace_element(xml_doc + (p - buf),"","",new_filename); + } + } + else if (parse_str(p, "", temp, sizeof(temp))) { + if(!strcmp(temp, download_path)) + { + replace_element(xml_doc + (p - buf),"","",new_download_path); + } + } + p = strtok(0, "\n"); + } + +} + void check_trigger() { FILE* f = fopen(TRIGGER_FILENAME, "r"); if (!f) return; @@ -53,49 +87,49 @@ void check_trigger() { void make_work() { CONFIG config; + char * p; int retval, i, start_time=time(0), n, nresults_left; - char keypath[256], suffix[256], result_template[MAX_BLOB_SIZE]; + char keypath[256], suffix[256], result_template[MAX_BLOB_SIZE], file_name[256], buf[MAX_BLOB_SIZE],pathname[256],new_file_name[256],new_pathname[256],command[256]; R_RSA_PRIVATE_KEY key; WORKUNIT wu; retval = config.parse_file(); if (retval) { - fprintf(stderr, "make_work: can't read config file\n"); + fprintf(stderr,"make_work: can't read config file\n"); exit(1); } retval = db_open(config.db_name, config.db_passwd); if (retval) { - fprintf(stderr, "make_work: can't open db\n"); + fprintf(stderr,"make_work: can't open db\n"); exit(1); } strcpy(wu.name, wu_name); retval = db_workunit_lookup_name(wu); if (retval) { - fprintf(stderr, "make_work: can't find wu %s\n", wu_name); + fprintf(stderr,"make_work: can't find wu %s\n", wu_name); exit(1); } sprintf(keypath, "%s/upload_private", config.key_dir); retval = read_key_file(keypath, key); if (retval) { - fprintf(stderr, "make_work: can't read key\n"); + fprintf(stderr,"make_work: can't read key\n"); exit(1); } - + retval = read_filename(result_template_file, result_template); if (retval) { - fprintf(stderr, "make_work: can't open result template\n"); + fprintf(stderr,"make_work: can't open result template\n"); exit(1); } - nresults_left = 0; while (true) { fflush(stdout); retval = db_result_count_server_state(RESULT_SERVER_STATE_UNSENT, n); if (retval) { - fprintf(stderr, "make_work: can't counts results\n"); + fprintf(stderr,"make_work: can't counts results\n"); exit(1); } printf("make_work: %d results\n", n); @@ -105,12 +139,28 @@ void make_work() { } if (nresults_left == 0) { - nresults_left = redundancy; - sprintf(wu.name, "wu_%d_%d", start_time, i++); - wu.id = 0; - wu.create_time = time(0); - retval = db_workunit_new(wu); - wu.id = db_insert_id(); + strcpy(buf,wu.xml_doc); + p = strtok(buf, "\n"); + strcpy(file_name, ""); + + while (p) { + if (parse_str(p, "", file_name, sizeof(file_name))) { + sprintf(new_file_name,"%s_%d_%d",file_name,start_time,i++); + sprintf(pathname, "%s/%s", config.download_dir, file_name); + sprintf(new_pathname,"%s/%s",config.download_dir, new_file_name); + sprintf(command,"cp %s %s",pathname,new_pathname); + system(command); + replace_file_name(wu.xml_doc,file_name,new_file_name,config.download_url); + } + p = strtok(0, "\n"); + } + nresults_left = redundancy; + sprintf(wu.name, "wu_%d_%d", start_time, i++); + wu.id = 0; + wu.create_time = time(0); + retval = db_workunit_new(wu); + wu.id = db_insert_id(); + } sprintf(suffix, "%d_%d", start_time, i++); create_result( @@ -141,11 +191,11 @@ int main(int argc, char** argv) { } if (!strlen(result_template_file)) { - fprintf(stderr, "make_work: missing -result_template\n"); + fprintf(stderr,"make_work: missing -result_template\n"); exit(1); } if (!strlen(wu_name)) { - fprintf(stderr, "make_work: missing -wu_name\n"); + fprintf(stderr,"make_work: missing -wu_name\n"); exit(1); } diff --git a/test/test.inc b/test/test.inc index 111a8da591..97addcd589 100644 --- a/test/test.inc +++ b/test/test.inc @@ -400,12 +400,12 @@ class Project { PassThru("cd $this->project_dir/cgi; ./feeder -asynch > feeder_out"); } - function result_retry($app){ - PassThru("cd $this->project_dir/cgi; ./result_retry -app $app->name -nerror 10 -ndet 10 -nredundancy 10 > result_retry_out"); + function result_retry($app, $nerror = 5, $ndet = 5, $nredundancy = 5){ + PassThru("cd $this->project_dir/cgi; ./result_retry -app $app->name -nerror $nerror -ndet $ndet -nredundancy $nredundancy > result_retry_out"); } - function start_result_retry($app){ - PassThru("cd $this->project_dir/cgi; ./result_retry -app $app->name -nerror 10 -ndet 10 -nredundancy 10 -asynch > result_retry_out"); + function start_result_retry($app, $nerror = 5,$ndet = 5, $nredundancy = 5){ + PassThru("cd $this->project_dir/cgi; ./result_retry -app $app->name -nerror $nerror -ndet $ndet -nredundancy $nredundancy -asynch > result_retry_out"); } diff --git a/test/test_loop.php b/test/test_loop.php index 2eddc2dea1..647076da09 100644 --- a/test/test_loop.php +++ b/test/test_loop.php @@ -26,6 +26,7 @@ $work->wu_template = "uc_wu"; $work->result_template = "uc_result"; $work->redundancy = 2; + array_push($work->input_files, "input"); $work->install($project); diff --git a/test/test_uc.php b/test/test_uc.php index ff2cb8e0fe..22bc6199f4 100644 --- a/test/test_uc.php +++ b/test/test_uc.php @@ -40,9 +40,9 @@ $project->start_feeder(); $host->run("-exit_when_idle -no_time_test"); + $project->stop(); $project->validate($app, 2); - $result->state = RESULT_STATE_DONE; $result->stderr_out = "APP: upper_case: starting, argc 1"; $result->exit_status = 0; diff --git a/tools/country_select.C b/tools/country_select.C index 1b73c25fc6..083ed22451 100644 --- a/tools/country_select.C +++ b/tools/country_select.C @@ -20,6 +20,7 @@ #include #include + #include "countries.h" int main(int argc, char** argv) {