From c417859785bcbae3fa16943c4077a6b642abee40 Mon Sep 17 00:00:00 2001 From: David Anderson Date: Mon, 27 Dec 2004 21:14:18 +0000 Subject: [PATCH] *** empty log message *** svn path=/trunk/boinc/; revision=4944 --- checkin_notes | 21 +++++++++++++++++++++ db/constraints.sql | 4 ++-- html/ops/db_update.php | 6 ++++++ lib/util.C | 10 ++++++++++ lib/util.h | 4 ++++ py/Boinc/setup_project.py | 2 +- sched/db_purge.C | 28 ++++++++++++++++++---------- 7 files changed, 62 insertions(+), 13 deletions(-) diff --git a/checkin_notes b/checkin_notes index 79e0c3037c..1aa8b44342 100755 --- a/checkin_notes +++ b/checkin_notes @@ -21849,3 +21849,24 @@ David 26 Dec 2004 sched/ db_purge.C + +David 27 Dec 2004 + - Add a -min_age_days N option to db_purge. + It will only purge WUs whose mod_time is at least N days ago. + - Change an index on workunit to make this efficient + - have setup_project.py copy db_purge + + Note: if you use this version of db_purge you should modify + the index workunit.wu_filedel. + You can do this manually or using html/ops/db_update.php + + db/ + constraints.sql + html/ops/ + db_update.php + lib/ + util.C,h + py/Boinc/ + setup_project.py + sched/ + db_purge.C diff --git a/db/constraints.sql b/db/constraints.sql index d921832f46..a223c263d4 100644 --- a/db/constraints.sql +++ b/db/constraints.sql @@ -32,8 +32,8 @@ alter table workunit -- validator add index wu_timeout (transition_time), -- transitioner - add index wu_filedel (file_delete_state), - -- file_deleter + add index wu_filedel (file_delete_state, mod_time), + -- file_deleter, db_purge add index wu_assim (appid, assimilate_state); -- assimilator diff --git a/html/ops/db_update.php b/html/ops/db_update.php index 10262286e7..69eccb4b5d 100644 --- a/html/ops/db_update.php +++ b/html/ops/db_update.php @@ -198,6 +198,12 @@ function update_11_24_2004_host() { ." add avg_turnaround double not null" ); } + +function update_12_27_2004() { + mysql_query("alter table workunit drop index wu_filedel"); + mysql_query("alter table workunit add index wu_filedel (file_delete_state, mod_time)"); +} + //update_10_25_2004(); ?> diff --git a/lib/util.C b/lib/util.C index 060c156fa0..1761b156a1 100755 --- a/lib/util.C +++ b/lib/util.C @@ -797,6 +797,16 @@ int dir_hier_url( return 0; } +void mysql_timestamp(double dt, char* p) { + struct tm* tmp; + time_t t = (time_t)dt; + tmp = localtime(&t); // MySQL timestamps are in local time + sprintf(p, "%4d%02d%02d%02d%02d%02d", + tmp->tm_year+1900, tmp->tm_mon+1, tmp->tm_mday, + tmp->tm_hour, tmp->tm_min, tmp->tm_sec + ); +} + #ifdef __GNUC__ static volatile const char __attribute__((unused)) *BOINCrcsid="$Id$"; #else diff --git a/lib/util.h b/lib/util.h index 7dd60d0ad6..3750cb2963 100755 --- a/lib/util.h +++ b/lib/util.h @@ -134,4 +134,8 @@ extern int dir_hier_url( extern int boinc_calling_thread_cpu_time(double&); +// convert UNIX time to MySQL timestamp (yyyymmddhhmmss) +// +extern void mysql_timestamp(double, char*); + #endif diff --git a/py/Boinc/setup_project.py b/py/Boinc/setup_project.py index efe4d97d30..485295e640 100644 --- a/py/Boinc/setup_project.py +++ b/py/Boinc/setup_project.py @@ -280,7 +280,7 @@ def install_boinc_files(dest_dir): [ 'make_work', 'feeder', 'transitioner', 'sample_bitwise_validator', 'sample_trivial_validator', 'file_deleter', 'sample_dummy_assimilator', - 'update_stats', 'db_dump' ]) + 'update_stats', 'db_dump', 'db_purge' ]) map(lambda (s): install(srcdir('sched',s), dir('bin',s)), [ 'start', 'show_shmem' ]) force_symlink(dir('bin', 'start'), dir('bin', 'stop')) diff --git a/sched/db_purge.C b/sched/db_purge.C index 89442a90a1..9c46edeebc 100644 --- a/sched/db_purge.C +++ b/sched/db_purge.C @@ -34,7 +34,7 @@ // // Options: // -// -days n purge WUs with mod_time at least N days in the past +// -min_age_days n purge WUs with mod_time at least N days in the past // -max n purge at most N WUs // -one_pass purge a few (~1000) WUs, then exit // default: keep scanning indefinitely @@ -75,9 +75,6 @@ using namespace std; #include "error_numbers.h" -#define LOCKFILE "db_purge.out" -#define PIDFILE "db_purge.pid" - #define WU_FILENAME_PREFIX "wu_archive" #define RESULT_FILENAME_PREFIX "result_archive" #define WU_INDEX_FILENAME_PREFIX "wu_index" @@ -91,6 +88,7 @@ FILE *re_stream=NULL; FILE *wu_index_stream=NULL; FILE *re_index_stream=NULL; int time_int=0; +int min_age_days=0; // These is used if limiting the total number of workunits to eliminate int purged_workunits= 0; @@ -456,12 +454,19 @@ bool do_pass() { DB_WORKUNIT wu; char buf[256]; - // select all workunits with file_delete_state=DONE - // - sprintf(buf, - "where file_delete_state=%d limit %d", - FILE_DELETE_DONE, DB_QUERY_LIMIT - ); + if (min_age_days) { + char timestamp[15]; + mysql_timestamp(dtime()-min_age_days*86400, timestamp); + sprintf(buf, + "where file_delete_state=%d and mod_time<'%s' limit %d", + FILE_DELETE_DONE, timestamp, DB_QUERY_LIMIT + ); + } else { + sprintf(buf, + "where file_delete_state=%d limit %d", + FILE_DELETE_DONE, DB_QUERY_LIMIT + ); + } int n=0; while (!wu.enumerate(buf)) { @@ -539,6 +544,8 @@ int main(int argc, char** argv) { one_pass = true; } else if (!strcmp(argv[i], "-d")) { log_messages.set_debug_level(atoi(argv[++i])); + } else if (!strcmp(argv[i], "-min_age_days")) { + min_age_days = atoi(argv[++i]); } else if (!strcmp(argv[i], "-max")) { max_number_workunits_to_purge= atoi(argv[++i]); } else if (!strcmp(argv[i], "-zip")) { @@ -552,6 +559,7 @@ int main(int argc, char** argv) { "Unrecognized arg: %s\n", argv[i] ); + exit(1); } }