From 6af9f66b4e39ffc3427d1d3c2a59984b68934a1b Mon Sep 17 00:00:00 2001 From: David Anderson Date: Mon, 31 Mar 2008 16:19:45 +0000 Subject: [PATCH] - DB/feeder/scheduler: change app_version.xml_doc from blob to mediumblob, and change the correspending structure field from 64KB to 256KB (could increase this if needed). This is needed to handle app versions with lots (> 100) of files - change LARGE_BLOB_SIZE to BLOB_SIZE a bunch of places - Change COPROCS from vector to vector. Otherwise the right virtual functions of COPROCs don't get called svn path=/trunk/boinc/; revision=14986 --- checkin_notes | 19 +++++++++++++++++++ db/boinc_db.C | 2 +- db/boinc_db.h | 33 +++++++++++++++++++-------------- db/schema.sql | 2 +- html/ops/db_update.php | 4 ++++ lib/coproc.h | 5 +++++ sched/db_purge.C | 2 +- sched/file_deleter.C | 4 ++-- sched/get_file.C | 2 +- sched/handle_request.C | 2 +- sched/make_work.C | 8 ++++---- sched/sched_plan.C | 10 +++++----- sched/sched_send.C | 6 +++--- sched/send_file.C | 2 +- sched/server_types.C | 14 +++++++------- sched/server_types.h | 5 +++-- tools/backend_lib.C | 8 ++++---- tools/create_work.C | 2 +- tools/process_result_template.C | 8 ++++---- 19 files changed, 86 insertions(+), 52 deletions(-) diff --git a/checkin_notes b/checkin_notes index 195043d963..afbb0dd977 100644 --- a/checkin_notes +++ b/checkin_notes @@ -2826,3 +2826,22 @@ Janus Mar 30 2008 sched/ sched_send.C +David Mar 31 2008 + - DB/feeder/scheduler: change app_version.xml_doc from blob to mediumblob, + and change the correspending structure field from 64KB to 256KB + (could increase this if needed). + This is needed to handle app versions with lots (> 100) of files + - change LARGE_BLOB_SIZE to BLOB_SIZE a bunch of places + - Change COPROCS from vector to vector. + Otherwise the right virtual functions of COPROCs don't get called + + db/ + boinc_db.C,h + schema.sql + html/ops/ + db_update.php + lib/ + coproc.h + sched/ + *.C + server_types.h diff --git a/db/boinc_db.C b/db/boinc_db.C index be1430527e..515cdacd1f 100644 --- a/db/boinc_db.C +++ b/db/boinc_db.C @@ -457,7 +457,7 @@ void DB_HOST::db_parse(MYSQL_ROW &r) { // so only include fields modified by the scheduler. // int DB_HOST::update_diff(HOST& h) { - char buf[LARGE_BLOB_SIZE], updates[LARGE_BLOB_SIZE], query[LARGE_BLOB_SIZE]; + char buf[BLOB_SIZE], updates[BLOB_SIZE], query[BLOB_SIZE]; strcpy(updates, ""); if (rpc_seqno != h.rpc_seqno) { sprintf(buf, " rpc_seqno=%d,", rpc_seqno); diff --git a/db/boinc_db.h b/db/boinc_db.h index 7a58513b69..d92d533c4a 100644 --- a/db/boinc_db.h +++ b/db/boinc_db.h @@ -35,11 +35,16 @@ extern DB_CONN boinc_db; // Sizes of text buffers in memory, corresponding to database BLOBs. -// Large is for fields with user-supplied text, and preferences +// The following is for regular blobs, 64KB -#define LARGE_BLOB_SIZE 65536 -#define MSG_FROM_HOST_BLOB_SIZE 262144 -#define MSG_TO_HOST_BLOB_SIZE 262144 +#define BLOB_SIZE 65536 + +// The following are for "medium blobs", +// which are 16MB in the DB +// +#define APP_VERSION_XML_BLOB_SIZE 262144 +#define MSG_FROM_HOST_BLOB_SIZE 262144 +#define MSG_TO_HOST_BLOB_SIZE 262144 // Dummy name for file xfers #define FILE_MOVER "move_file" @@ -84,7 +89,7 @@ struct APP_VERSION { int appid; int version_num; int platformid; - char xml_doc[LARGE_BLOB_SIZE]; + char xml_doc[APP_VERSION_XML_BLOB_SIZE]; // describes app files. format: // ... // ... @@ -127,9 +132,9 @@ struct USER { double total_credit; double expavg_credit; // credit per second, recent average double expavg_time; // when the above was computed - char global_prefs[LARGE_BLOB_SIZE]; + char global_prefs[BLOB_SIZE]; // global preferences, within tag - char project_prefs[LARGE_BLOB_SIZE]; + char project_prefs[BLOB_SIZE]; // project preferences; format: // // X @@ -190,7 +195,7 @@ struct TEAM { char url[256]; int type; // Team type (see above) char name_html[256]; - char description[LARGE_BLOB_SIZE]; + char description[BLOB_SIZE]; int nusers; // UNDEFINED BY DEFAULT char country[256]; double total_credit; @@ -337,7 +342,7 @@ struct WORKUNIT { int create_time; int appid; // associated app char name[256]; - char xml_doc[LARGE_BLOB_SIZE]; + char xml_doc[BLOB_SIZE]; int batch; double rsc_fpops_est; // estimated # of FP operations // used to estimate how long a result will take on a host @@ -462,9 +467,9 @@ struct RESULT { int received_time; // when result was received from host char name[256]; double cpu_time; // CPU time used to complete result - char xml_doc_in[LARGE_BLOB_SIZE]; // descriptions of output files - char xml_doc_out[LARGE_BLOB_SIZE]; // MD5s of output files - char stderr_out[LARGE_BLOB_SIZE]; // stderr output, if any + char xml_doc_in[BLOB_SIZE]; // descriptions of output files + char xml_doc_out[BLOB_SIZE]; // MD5s of output files + char stderr_out[BLOB_SIZE]; // stderr output, if any int batch; int file_delete_state; // see above; values for file_delete_state int validate_state; @@ -778,8 +783,8 @@ struct SCHED_RESULT_ITEM { int received_time; double cpu_time; double claimed_credit; - char xml_doc_out[LARGE_BLOB_SIZE]; - char stderr_out[LARGE_BLOB_SIZE]; + char xml_doc_out[BLOB_SIZE]; + char stderr_out[BLOB_SIZE]; int app_version_num; int exit_status; int file_delete_state; diff --git a/db/schema.sql b/db/schema.sql index 26a5360b24..67bbec6409 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -57,7 +57,7 @@ create table app_version ( appid integer not null, version_num integer not null, platformid integer not null, - xml_doc blob, + xml_doc mediumblob, min_core_version integer not null default 0, max_core_version integer not null default 0, deprecated tinyint not null default 0, diff --git a/html/ops/db_update.php b/html/ops/db_update.php index 6f8f010197..955a16c285 100755 --- a/html/ops/db_update.php +++ b/html/ops/db_update.php @@ -581,6 +581,10 @@ function update_3_27_2008() { do_query("update team set ping_user=0, ping_time=0 where ping_user=userid"); } +function update_3_31_2008() { + do_query("alter table app_version change column xml_doc xml_doc mediumblob"); +} + // modify the following to call the function you want. // Make sure you do all needed functions, in order. // (Look at your DB structure using "explain" queries to see diff --git a/lib/coproc.h b/lib/coproc.h index 37b8f3fece..dcf2694e1d 100644 --- a/lib/coproc.h +++ b/lib/coproc.h @@ -35,6 +35,11 @@ struct COPROCS { std::vector coprocs; COPROCS(){} + ~COPROCS() { + for (unsigned int i=0; i &#NNN; // - char buf[LARGE_BLOB_SIZE*6]; + char buf[BLOB_SIZE*6]; xml_escape(result.stderr_out, buf); fprintf( diff --git a/sched/file_deleter.C b/sched/file_deleter.C index 57710fa70b..67a4ed504c 100644 --- a/sched/file_deleter.C +++ b/sched/file_deleter.C @@ -126,7 +126,7 @@ int get_file_path( int wu_delete_files(WORKUNIT& wu) { char* p; - char filename[256], pathname[256], buf[LARGE_BLOB_SIZE]; + char filename[256], pathname[256], buf[BLOB_SIZE]; bool no_delete=false; int count_deleted = 0, retval, mthd_retval = 0; @@ -180,7 +180,7 @@ int wu_delete_files(WORKUNIT& wu) { int result_delete_files(RESULT& result) { char* p; - char filename[256], pathname[256], buf[LARGE_BLOB_SIZE]; + char filename[256], pathname[256], buf[BLOB_SIZE]; bool no_delete=false; int count_deleted = 0, retval, mthd_retval = 0; diff --git a/sched/get_file.C b/sched/get_file.C index 8476f3c7bf..0ea581645d 100644 --- a/sched/get_file.C +++ b/sched/get_file.C @@ -66,7 +66,7 @@ void init_xfer_result(DB_RESULT& result) { int create_upload_result(DB_RESULT& result, int host_id, const char * file_name) { int retval; - char result_xml[LARGE_BLOB_SIZE]; + char result_xml[BLOB_SIZE]; sprintf(result_xml, "\n" " %s\n" diff --git a/sched/handle_request.C b/sched/handle_request.C index a7f3cc3224..13b190cd87 100644 --- a/sched/handle_request.C +++ b/sched/handle_request.C @@ -786,7 +786,7 @@ int send_result_abort( SCHEDULER_REQUEST& sreq, SCHEDULER_REPLY& reply) { // 2) send global prefs in reply msg if needed // int handle_global_prefs(SCHEDULER_REQUEST& sreq, SCHEDULER_REPLY& reply) { - char buf[LARGE_BLOB_SIZE]; + char buf[BLOB_SIZE]; reply.send_global_prefs = false; bool have_working_prefs = (strlen(sreq.working_global_prefs_xml)>0); bool have_master_prefs = (strlen(sreq.global_prefs_xml)>0); diff --git a/sched/make_work.C b/sched/make_work.C index c289019324..e7a41c12a7 100644 --- a/sched/make_work.C +++ b/sched/make_work.C @@ -63,7 +63,7 @@ bool one_pass = false; // Don't patch the URL; we'll download the same file // void replace_file_name(char* xml_doc, char* filename, char* new_filename) { - char buf[LARGE_BLOB_SIZE], temp[256]; + char buf[BLOB_SIZE], temp[256]; char * p; strcpy(buf, xml_doc); @@ -87,8 +87,8 @@ void replace_file_name(char* xml_doc, char* filename, char* new_filename) { } void make_new_wu(DB_WORKUNIT& original_wu, char* starting_xml, int start_time) { - char file_name[256], buf[LARGE_BLOB_SIZE], new_file_name[256]; - char new_buf[LARGE_BLOB_SIZE]; + char file_name[256], buf[BLOB_SIZE], new_file_name[256]; + char new_buf[BLOB_SIZE]; char * p; int retval; DB_WORKUNIT wu = original_wu; @@ -172,7 +172,7 @@ void wait_for_results(int wu_id) { void make_work(vector &wu_names) { int retval, start_time=time(0); char keypath[256]; - char buf[LARGE_BLOB_SIZE]; + char buf[BLOB_SIZE]; R_RSA_PRIVATE_KEY key; int nwu_names = wu_names.size(); DB_WORKUNIT wus[nwu_names]; diff --git a/sched/sched_plan.C b/sched/sched_plan.C index f18e8ac28e..fa4d732d12 100644 --- a/sched/sched_plan.C +++ b/sched/sched_plan.C @@ -72,11 +72,11 @@ bool app_plan(SCHEDULER_REQUEST& sreq, char* plan_class, HOST_USAGE& hu) { // and some CPU also, and gets 50 GFLOPS total // for (unsigned int i=0; iname, "CUDA")) { + COPROC* cu = new COPROC; + strcpy(cu->name, cp->name); + cu->count = 1; hu.coprocs.coprocs.push_back(cu); double x = 1e9/sreq.host.p_fpops; if (x > 1) x = 1; diff --git a/sched/sched_send.C b/sched/sched_send.C index 066a31c5d9..5e6a11bb27 100644 --- a/sched/sched_send.C +++ b/sched/sched_send.C @@ -644,9 +644,9 @@ int wu_is_infeasible( // int insert_after(char* buffer, const char* after, const char* text) { char* p; - char temp[LARGE_BLOB_SIZE]; + char temp[BLOB_SIZE]; - if (strlen(buffer) + strlen(text) > LARGE_BLOB_SIZE-1) { + if (strlen(buffer) + strlen(text) > BLOB_SIZE-1) { log_messages.printf(MSG_CRITICAL, "insert_after: overflow: %d %d\n", strlen(buffer), strlen(text) @@ -671,7 +671,7 @@ int insert_after(char* buffer, const char* after, const char* text) { // in preparation for sending it to a client // int insert_wu_tags(WORKUNIT& wu, APP& app) { - char buf[LARGE_BLOB_SIZE]; + char buf[BLOB_SIZE]; sprintf(buf, " %f\n" diff --git a/sched/send_file.C b/sched/send_file.C index d67cbd4f61..65ee4d819e 100644 --- a/sched/send_file.C +++ b/sched/send_file.C @@ -65,7 +65,7 @@ void init_xfer_result(DB_RESULT& result) { int create_download_result(DB_RESULT& result, int host_id) { int retval; - char result_xml[LARGE_BLOB_SIZE]; + char result_xml[BLOB_SIZE]; sprintf(result_xml, "\n" " %s\n" diff --git a/sched/server_types.C b/sched/server_types.C index 743a6bffec..642f706ff5 100644 --- a/sched/server_types.C +++ b/sched/server_types.C @@ -504,7 +504,7 @@ SCHEDULER_REPLY::~SCHEDULER_REPLY() { int SCHEDULER_REPLY::write(FILE* fout) { unsigned int i; - char buf[LARGE_BLOB_SIZE]; + char buf[BLOB_SIZE]; // Note: at one point we had // "\n" @@ -831,7 +831,7 @@ int APP::write(FILE* fout) { } int APP_VERSION::write(FILE* fout) { - char buf[LARGE_BLOB_SIZE]; + char buf[BLOB_SIZE]; unsigned int i; strcpy(buf, xml_doc); @@ -862,14 +862,14 @@ int APP_VERSION::write(FILE* fout) { ); } for (i=0; ihost_usage.coprocs.coprocs.size(); i++) { - COPROC& cp = bavp->host_usage.coprocs.coprocs[i]; + COPROC* cp = bavp->host_usage.coprocs.coprocs[i]; fprintf(fout, " \n" " %s\n" " %d\n" " \n", - cp.name, - cp.count + cp->name, + cp->count ); } fputs("\n", fout); @@ -877,7 +877,7 @@ int APP_VERSION::write(FILE* fout) { } int RESULT::write_to_client(FILE* fout) { - char buf[LARGE_BLOB_SIZE]; + char buf[BLOB_SIZE]; unsigned int i; strcpy(buf, xml_doc_in); @@ -1074,7 +1074,7 @@ int HOST::parse_disk_usage(FILE* fin) { } void GLOBAL_PREFS::parse(const char* buf, const char* venue) { - char buf2[LARGE_BLOB_SIZE]; + char buf2[BLOB_SIZE]; double dtemp; defaults(); diff --git a/sched/server_types.h b/sched/server_types.h index 23e8725b13..626df7b933 100644 --- a/sched/server_types.h +++ b/sched/server_types.h @@ -85,6 +85,7 @@ struct HOST_USAGE { flops = x; strcpy(cmdline, ""); } + ~HOST_USAGE(){} }; // keep track of the best app_version for each app for this host @@ -241,8 +242,8 @@ struct SCHEDULER_REQUEST { // how many wall-clock seconds will elapse before // host will begin any new work for this project double duration_correction_factor; - char global_prefs_xml[LARGE_BLOB_SIZE]; - char working_global_prefs_xml[LARGE_BLOB_SIZE]; + char global_prefs_xml[BLOB_SIZE]; + char working_global_prefs_xml[BLOB_SIZE]; char code_sign_key[4096]; bool anonymous_platform; diff --git a/tools/backend_lib.C b/tools/backend_lib.C index 548700d49c..19fa193c2b 100644 --- a/tools/backend_lib.C +++ b/tools/backend_lib.C @@ -174,7 +174,7 @@ static int process_wu_template( const char* additional_xml ) { char* p; - char buf[LARGE_BLOB_SIZE], md5[33], path[256], url[256], top_download_path[256]; + char buf[BLOB_SIZE], md5[33], path[256], url[256], top_download_path[256]; string out, cmdline; int retval, file_number; double nbytes; @@ -443,7 +443,7 @@ int create_result( ) { DB_RESULT result; char base_outfile_name[256]; - char result_template[LARGE_BLOB_SIZE]; + char result_template[BLOB_SIZE]; int retval; result.clear(); @@ -540,8 +540,8 @@ int create_work( const char* additional_xml ) { int retval; - char _result_template[LARGE_BLOB_SIZE]; - char wu_template[LARGE_BLOB_SIZE]; + char _result_template[BLOB_SIZE]; + char wu_template[BLOB_SIZE]; #if 0 retval = check_files(infiles, ninfiles, config); diff --git a/tools/create_work.C b/tools/create_work.C index 9906840509..56838c40b8 100644 --- a/tools/create_work.C +++ b/tools/create_work.C @@ -76,7 +76,7 @@ int main(int argc, const char** argv) { DB_APP app; DB_WORKUNIT wu; int retval; - char wu_template[LARGE_BLOB_SIZE]; + char wu_template[BLOB_SIZE]; char wu_template_file[256], result_template_file[256], result_template_path[1024]; const char* command_line=NULL; const char** infiles = NULL; diff --git a/tools/process_result_template.C b/tools/process_result_template.C index b921b9ed0d..e8daf6e35d 100644 --- a/tools/process_result_template.C +++ b/tools/process_result_template.C @@ -40,9 +40,9 @@ // add a signature of its contents up to that point. // int add_signatures(char* xml, R_RSA_PRIVATE_KEY& key) { - char* p = xml, *q1, *q2, buf[LARGE_BLOB_SIZE], buf2[LARGE_BLOB_SIZE];; - char signature_hex[LARGE_BLOB_SIZE]; - char signature_xml[LARGE_BLOB_SIZE]; + char* p = xml, *q1, *q2, buf[BLOB_SIZE], buf2[BLOB_SIZE];; + char signature_hex[BLOB_SIZE]; + char signature_xml[BLOB_SIZE]; int retval, len; while (1) { @@ -108,7 +108,7 @@ int process_result_template( SCHED_CONFIG& config ) { char* p,*q; - char temp[LARGE_BLOB_SIZE], buf[256]; + char temp[BLOB_SIZE], buf[256]; int retval; while (1) {