diff --git a/checkin_notes b/checkin_notes index 972553f7da..aaec7a5145 100755 --- a/checkin_notes +++ b/checkin_notes @@ -18422,3 +18422,9 @@ David 14 Oct 2004 makefile.gui_test sched/ sched_send.C + +Lana 14 Oct 2004 + - improved db_purge to output to multiple archives + + sched/ + db_purge.C diff --git a/sched/db_purge.C b/sched/db_purge.C index 333b5db9d7..7d2d713fc0 100644 --- a/sched/db_purge.C +++ b/sched/db_purge.C @@ -46,39 +46,49 @@ using namespace std; #define WU_FILENAME_PREFIX "wuarchive" #define RESULT_FILENAME_PREFIX "resultarchive" +#define DB_QUERY_LIMIT 1000 +#define NUMBER_RECORDS_PER_ARCHIVE_FILE 100000 +#define MAX_WORKUNITS_PURGED_IN_ONE_RUN 100000 SCHED_CONFIG config; FILE *wu_stream; FILE *re_stream; -//open archive files for purged workunits with the name- wuarchive_TIMESTAMP.xml -//open archive files for purged results with the name- resultarchive_TIMESTAMP.xml -int open_archive(char* filename_prefix, FILE*& f){ +//open archive files for purged workunits with the name- wuarchive_.xml +//open archive files for purged results with the name- resultarchive_.xml +int open_archive(char* filename_prefix, FILE*& f, int id){ int retval=0; char path[256]; - sprintf(path,"../archives/%s_%d.xml", filename_prefix, (int)time(0)); + sprintf(path,"../archives/%s_%d.xml", filename_prefix, + (id - (id % NUMBER_RECORDS_PER_ARCHIVE_FILE)) + ); log_messages.printf(SCHED_MSG_LOG::NORMAL, "Opening archive %s\n", path); - if ((f = fopen( path,"a+")) == NULL) { - log_messages.printf(SCHED_MSG_LOG::CRITICAL,"Can't open archive file %s\n", path); - return ERR_FOPEN; + if ((f = fopen( path,"a")) == NULL) { + log_messages.printf(SCHED_MSG_LOG::CRITICAL, + "Can't open archive file %s\n", path + ); + return ERR_FOPEN; } fprintf(f, "\n" ); - fprintf(f, - "<%s>\n",filename_prefix - ); - return retval; } int archive_result(DB_RESULT& result) { + int retval = open_archive(RESULT_FILENAME_PREFIX, re_stream, result.id); + if (retval) { + log_messages.printf(SCHED_MSG_LOG::CRITICAL, + "Can't open result archive for result %d\n", result.id + ); + exit(1); + } fprintf(re_stream, "\n" " %d\n", @@ -148,10 +158,18 @@ int archive_result(DB_RESULT& result) { fprintf(re_stream, "\n" ); + fclose(re_stream); return 0; } int archive_wu(DB_WORKUNIT& wu) { + int retval = open_archive(WU_FILENAME_PREFIX, wu_stream, wu.id); + if (retval) { + log_messages.printf(SCHED_MSG_LOG::CRITICAL, + "Can't open archive for workunit %d\n", wu.id + ); + exit(1); + } fprintf(wu_stream, "\n" " %d\n", @@ -213,24 +231,11 @@ int archive_wu(DB_WORKUNIT& wu) { fprintf(wu_stream, "\n" ); + + fclose(wu_stream); return 0; } -void cleanUpResources() { - //close archive files - fprintf(wu_stream, - "\n",WU_FILENAME_PREFIX - ); - fclose(wu_stream); - - fprintf(re_stream, - "\n",RESULT_FILENAME_PREFIX - ); - fclose(re_stream); - - //close DB connection - boinc_db.close(); -} int purge_and_archive_results(DB_WORKUNIT& wu, int& number_results) { int retval= 0; @@ -239,15 +244,24 @@ int purge_and_archive_results(DB_WORKUNIT& wu, int& number_results) { number_results=0; - sprintf(buf, "where workunitid=%d", wu.id); + sprintf(buf, "where workunitid=%d order by id", wu.id); while (!result.enumerate(buf)) { retval= archive_result(result); - if (retval) return retval; - log_messages.printf(SCHED_MSG_LOG::NORMAL,"Archived result [%d] to a file\n", result.id); + if (retval) { + log_messages.printf(SCHED_MSG_LOG::CRITICAL, + "Failed to archive result [%d] to a file\n", result.id + ); + return retval; + } + log_messages.printf(SCHED_MSG_LOG::NORMAL, + "Archived result [%d] to a file\n", result.id + ); retval= result.delete_from_db(); if (retval) return retval; - log_messages.printf(SCHED_MSG_LOG::NORMAL,"Purged result [%d] from database\n", result.id); + log_messages.printf(SCHED_MSG_LOG::NORMAL, + "Purged result [%d] from database\n", result.id + ); number_results++; } @@ -261,37 +275,29 @@ int main(int argc, char** argv) { retval= config.parse_file(".."); if (retval) { - log_messages.printf(SCHED_MSG_LOG::CRITICAL, "Can't parse config file\n"); + log_messages.printf(SCHED_MSG_LOG::CRITICAL, + "Can't parse config file\n"); exit(1); } log_messages.printf(SCHED_MSG_LOG::NORMAL, "Starting DB Purger\n"); retval = boinc_db.open(config.db_name, config.db_host, config.db_user, - config.db_passwd); + config.db_passwd); if (retval) { log_messages.printf(SCHED_MSG_LOG::CRITICAL, "Can't open DB\n"); exit(1); } - log_messages.printf(SCHED_MSG_LOG::NORMAL, "Opened DB\n"); mkdir("../archives", 0777); - retval = open_archive(WU_FILENAME_PREFIX,wu_stream); - retval = open_archive(RESULT_FILENAME_PREFIX,re_stream); - if (retval) { - log_messages.printf(SCHED_MSG_LOG::CRITICAL, "Can't open archives\n"); - exit(1); - } - bool did_something = false; DB_WORKUNIT wu; char buf[256]; - // select all workunits with file_delete_state='DONE' - // - sprintf(buf, "where file_delete_state=%d limit 1000", FILE_DELETE_DONE); - int n=0; + sprintf(buf, "where file_delete_state=%d order by id limit %d", + FILE_DELETE_DONE, DB_QUERY_LIMIT); + int n= 0; while (!wu.enumerate(buf)) { did_something = true; @@ -305,18 +311,25 @@ int main(int argc, char** argv) { ); exit(1); } - log_messages.printf(SCHED_MSG_LOG::NORMAL,"Archived workunit [%d] to a file\n", wu.id); - - //purge workunit from DB + log_messages.printf(SCHED_MSG_LOG::NORMAL, + "Archived workunit [%d] to a file\n", wu.id); + retval= wu.delete_from_db(); if (retval) { - log_messages.printf(SCHED_MSG_LOG::CRITICAL,"Can't delete workunit [%d] from database:%d\n", wu.id, retval); + log_messages.printf(SCHED_MSG_LOG::CRITICAL, + "Can't delete workunit [%d] from database:%d\n", wu.id, retval + ); exit(1); } - log_messages.printf(SCHED_MSG_LOG::NORMAL,"Purged workunit [%d] from database\n", wu.id); + log_messages.printf(SCHED_MSG_LOG::NORMAL, + "Purged workunit [%d] from database\n", wu.id + ); purged_workunits++; + if (purged_workunits >= MAX_WORKUNITS_PURGED_IN_ONE_RUN) + break; } + boinc_db.close(); if (!did_something) { log_messages.printf(SCHED_MSG_LOG::CRITICAL, "Did not do anything\n");