From 7f3da8dcbfd1a47f84bd670617861ef9367ab489 Mon Sep 17 00:00:00 2001 From: David Anderson Date: Mon, 14 Oct 2002 23:10:12 +0000 Subject: [PATCH] make_work svn path=/trunk/boinc/; revision=496 --- checkin_notes | 25 +++++++++++ lib/parse.C | 15 +++++++ sched/Makefile.in | 8 +++- sched/config.C | 2 + sched/config.h | 2 + sched/make_work.C | 75 +++++++++++++++++++-------------- test/test.inc | 23 +++++----- test/test_loop.php | 2 +- tools/add.C | 10 +---- tools/backend_lib.C | 26 ++++++++++-- tools/backend_lib.h | 9 +++- tools/create_work.C | 46 ++++++-------------- tools/process_result_template.C | 6 +-- 13 files changed, 155 insertions(+), 94 deletions(-) diff --git a/checkin_notes b/checkin_notes index b7f1794528..b634c56fdd 100755 --- a/checkin_notes +++ b/checkin_notes @@ -2140,3 +2140,28 @@ Eric October 14, 2002 boinc.pbproj/ project.pbxproj +David Oct 14 2002 + - Change make_work so that it generates results with + valid file upload signatures. + Instead of directly creating new DB.result records, + it now calls create_result() to do the work. + - Changed create_result() to take a name suffix + - Factored key parsing into a function + - Removed random-WU-name feature from create_work; + names should be generated at a higher level + - added a key-generation function to test.inc + + lib/ + parse.C + sched/ + Makefile.in + config.C,h + make_work.C + test/ + test.inc + test_loop.php + tools/ + add.C + backend_lib.C,h + create_work.C + process_result_template.C diff --git a/lib/parse.C b/lib/parse.C index 1474577b23..535b6c97b3 100644 --- a/lib/parse.C +++ b/lib/parse.C @@ -144,3 +144,18 @@ int read_file_malloc(char* pathname, char*& str) { fclose(f); return 0; } + +#if 0 +// replace XML element contents. not currently used +// +void replace_element(char* buf, char* start, char* end, char* replacement) { + char temp[MAX_BLOB_SIZE], *p, *q; + + p = strstr(buf, start); + p += strlen(start); + q = strstr(p, end); + strcpy(temp, q); + strcpy(p, replacement); + strcat(p, temp); +} +#endif diff --git a/sched/Makefile.in b/sched/Makefile.in index 0bfe5312ce..cd71d616eb 100644 --- a/sched/Makefile.in +++ b/sched/Makefile.in @@ -71,7 +71,13 @@ MAKE_WORK_OBJS = \ config.o \ ../db/db_mysql.o \ ../db/mysql_util.o \ - ../lib/parse.o + ../tools/backend_lib.o \ + ../tools/process_result_template.o \ + ../lib/parse.o \ + ../lib/md5_file.o \ + ../lib/md5.o \ + ../lib/crypt.o \ + ../RSAEuro/source/rsaeuro.a FCGI_OBJS = \ handle_request.fcgi.o \ diff --git a/sched/config.C b/sched/config.C index 9bc2de0110..a5a0c011f3 100644 --- a/sched/config.C +++ b/sched/config.C @@ -16,6 +16,8 @@ int CONFIG::parse(FILE* in) { else if (parse_str(buf, "", db_passwd, sizeof(db_passwd))) continue; else if (parse_int(buf, "", shmem_key)) continue; else if (parse_str(buf, "", key_dir, sizeof(key_dir))) continue; + else if (parse_str(buf, "", download_url, sizeof(download_url))) continue; + else if (parse_str(buf, "", upload_url, sizeof(upload_url))) continue; else if (parse_str(buf, "", upload_dir, sizeof(upload_dir))) continue; else if (parse_str(buf, "", user_name, sizeof(user_name))) continue; } diff --git a/sched/config.h b/sched/config.h index b267c9bfcc..1efdb101bc 100644 --- a/sched/config.h +++ b/sched/config.h @@ -9,6 +9,8 @@ public: char db_passwd[256]; int shmem_key; char key_dir[256]; + char download_url[256]; + char upload_url[256]; char upload_dir[256]; char user_name[256]; diff --git a/sched/make_work.C b/sched/make_work.C index 7907be66b8..157b2ed30c 100644 --- a/sched/make_work.C +++ b/sched/make_work.C @@ -1,10 +1,9 @@ // make_work.C // +// make_work -wu_name name -result_template filename [ -cushion n ] +// // Create result records as needed to maintain a pool to send // -// This reads a result record from the DB, then makes clones of it. -// Assumes the result has a single output file, -// so overwrites the first element with a new name #include #include @@ -13,32 +12,27 @@ #include #include "db.h" +#include "crypt.h" +#include "backend_lib.h" #include "config.h" #define TRIGGER_FILENAME "stop_server" +int cushion = 10; +char wu_name[256], result_template_file[256]; + void check_trigger() { FILE* f = fopen(TRIGGER_FILENAME, "r"); if (!f) return; exit(0); } -void replace_element(char* buf, char* start, char* end, char* replacement) { - char temp[MAX_BLOB_SIZE], *p, *q; - - p = strstr(buf, start); - p += strlen(start); - q = strstr(p, end); - strcpy(temp, q); - strcpy(p, replacement); - strcat(p, temp); -} - void make_work() { CONFIG config; - RESULT result; - int retval, i=time(0), n; - char buf[256]; + int retval, i, start_time=time(0), n; + char keypath[256], suffix[256]; + R_RSA_PRIVATE_KEY key; + WORKUNIT wu; retval = config.parse_file(); if (retval) { @@ -52,9 +46,17 @@ void make_work() { exit(1); } - retval = db_result(1, result); + strcpy(wu.name, wu_name); + retval = db_workunit_lookup_name(wu); if (retval) { - fprintf(stderr, "make_work: can't read result\n"); + fprintf(stderr, "make_work: can't find wu %s\n", wu_name); + exit(1); + } + + sprintf(keypath, "%s/upload_private", config.key_dir); + retval = read_key_file(keypath, key); + if (retval) { + fprintf(stderr, "make_work: can't read key\n"); exit(1); } @@ -66,22 +68,16 @@ void make_work() { exit(1); } printf("make_work: %d results\n", n); - if (n > 10) { + if (n > cushion) { sleep(1); continue; } - result.id = 0; - result.create_time = time(0); - sprintf(result.name, "result_%d", i++); - result.state = RESULT_STATE_UNSENT; - result.validate_state = VALIDATE_STATE_INITIAL; - replace_element(result.xml_doc_in, "", "", result.name); - replace_element(result.xml_doc_in, "", "", result.name); - retval = db_result_new(result); - if (retval) { - fprintf(stderr, "make_work: can't create result\n"); - exit(1); - } + + sprintf(suffix, "%d_%d", start_time, i++); + create_result( + wu, result_template_file, suffix, key, + config.upload_url, config.download_url + ); printf("make_work: added a result\n"); } } @@ -94,9 +90,24 @@ int main(int argc, char** argv) { for (i=1; i$this->db_passwd\n"); fputs($f, " $this->shmem_key\n"); fputs($f, " $this->key_dir\n"); + fputs($f, " $this->download_url\n"); + fputs($f, " $this->upload_url\n"); fputs($f, " $this->project_dir/upload\n"); fputs($f, " $this->user_name\n"); fputs($f, "\n"); @@ -254,8 +263,9 @@ class Project { PassThru("cd $this->project_dir/cgi; feeder -asynch > feeder_out"); } - function start_make_work(){ - PassThru("cd $this->project_dir/cgi; make_work -asynch > make_work_out"); + function start_make_work($work){ + $result_template_path = realpath($work->result_template); + PassThru("cd $this->project_dir/cgi; make_work -asynch -result_template $result_template_path -wu_name $work->wu_template > make_work_out"); } function stop() { @@ -418,7 +428,6 @@ class Work { var $result_template; var $nresults; var $input_files; - var $randomize_name; var $rsc_iops; var $rsc_fpops; var $rsc_disk; @@ -429,7 +438,6 @@ class Work { $this->rcs_iops = 180000000000; $this->rcs_fpops = 100000000000; $this->rcs_disk = 1000000; - $this->randomize_name = false; } function install($project) { @@ -438,12 +446,7 @@ class Work { $x = $this->input_files[$i]; PassThru("cp $x $project->project_dir/download"); } - $cmd = "create_work -db_name $project->db_name -download_dir $project->project_dir/download -upload_url $project->upload_url -download_url $project->download_url/ -keyfile $project->key_dir/upload_private -appname $app->name -rsc_iops $this->rcs_iops -rsc_fpops $this->rsc_fpops -rsc_disk $this->rsc_disk -wu_template $this->wu_template -result_template $this->result_template -nresults $this->nresults "; - if ($this->randomize_name) { - $cmd = $cmd." -wu_name_rand ".$this->wu_template; - } else { - $cmd = $cmd." -wu_name ".$this->wu_template; - } + $cmd = "create_work -db_name $project->db_name -download_dir $project->project_dir/download -upload_url $project->upload_url -download_url $project->download_url/ -keyfile $project->key_dir/upload_private -appname $app->name -rsc_iops $this->rcs_iops -rsc_fpops $this->rsc_fpops -rsc_disk $this->rsc_disk -wu_template $this->wu_template -result_template $this->result_template -nresults $this->nresults -wu_name $this->wu_template"; for ($i=0; $iinput_files); $i++) { $x = $this->input_files[$i]; $cmd = $cmd." ".$x; diff --git a/test/test_loop.php b/test/test_loop.php index c59348133a..9c590740ff 100644 --- a/test/test_loop.php +++ b/test/test_loop.php @@ -30,7 +30,7 @@ $work->install($project); $project->start_feeder(); - $project->start_make_work(); + $project->start_make_work($work); $host->run(""); //$project->stop(); diff --git a/tools/add.C b/tools/add.C index 3ad7893dd3..66d52c97da 100644 --- a/tools/add.C +++ b/tools/add.C @@ -104,15 +104,9 @@ int sign_executable(char* path, char* signature_text) { DATA_BLOCK signature; unsigned char signature_buf[SIGNATURE_SIZE_BINARY]; R_RSA_PRIVATE_KEY code_sign_key; - FILE* fkey = fopen(code_sign_keyfile, "r"); - if (!fkey) { - fprintf(stderr, "add: can't open key file (%s)\n", code_sign_keyfile); - exit(1); - } - retval = scan_key_hex(fkey, (KEY*)&code_sign_key, sizeof(code_sign_key)); - fclose(fkey); + retval = read_key_file(code_sign_keyfile, code_sign_key); if (retval) { - fprintf(stderr, "add: can't parse key\n"); + fprintf(stderr, "add: can't read key\n"); exit(1); } signature.data = signature_buf; diff --git a/tools/backend_lib.C b/tools/backend_lib.C index 715a695a24..e93f6dcade 100644 --- a/tools/backend_lib.C +++ b/tools/backend_lib.C @@ -53,6 +53,22 @@ int read_filename(char* path, char* buf) { return retval; } +int read_key_file(char* keyfile, R_RSA_PRIVATE_KEY& key) { + int retval; + FILE* fkey = fopen(keyfile, "r"); + if (!fkey) { + fprintf(stderr, "can't open key file (%s)\n", keyfile); + return -1; + } + retval = scan_key_hex(fkey, (KEY*)&key, sizeof(key)); + fclose(fkey); + if (retval) { + fprintf(stderr, "can't parse key\n"); + return -1; + } + return 0; +} + // replace INFILE_x with filename from array, // MD5_x with checksum of file, // @@ -125,7 +141,8 @@ static int process_wu_template( } int create_result( - WORKUNIT& wu, char* result_template_filename, int i, R_RSA_PRIVATE_KEY& key, + WORKUNIT& wu, char* result_template_filename, + char* result_name_suffix, R_RSA_PRIVATE_KEY& key, char* upload_url, char* download_url ) { RESULT r; @@ -142,7 +159,7 @@ int create_result( r.workunitid = wu.id; r.state = RESULT_STATE_UNSENT; r.validate_state = VALIDATE_STATE_INITIAL; - sprintf(r.name, "%s_%d", wu.name, i); + sprintf(r.name, "%s_%s", wu.name, result_name_suffix); sprintf(base_outfile_name, "%s_", r.name); result_template_file = fopen(result_template_filename, "r"); @@ -177,6 +194,7 @@ int create_work( char* upload_url, char* download_url ) { int i, retval; + char suffix[256]; assert(wu_template!=NULL); assert(result_template_file!=NULL); assert(nresults>=0); @@ -207,8 +225,10 @@ int create_work( if (!wu.dynamic_results) { for (i=0; i element, add a signature // of its contents up to that point. +// +// TODO - have this work in memory instead of using disk files #include #include