diff --git a/checkin_notes b/checkin_notes index 69da77f3cd..52c5fa05b8 100755 --- a/checkin_notes +++ b/checkin_notes @@ -3732,3 +3732,27 @@ David Mar 6 2003 util.C,h sched/ *.C + +Eric March 7, 2003 + - Moved common functions (write_log, check_trigger, update_average) + into sched_util.C + - Added stop_server checking to more server programs + - Added update_stats program to periodically update exponential average + credit for users and hosts, and to update team credit by summing member credit + - Added db_sum, db_query_double to retrieve sums of columns (used by update_stats) + - Added db_user_sum_team_expavg_credit, db_user_sum_team_total_credit, + db_user_count_team for updating team statistics + - Confirmed that current exponential average algorithm is correct + + db/ + mysql_util.C,h + db_mysql.C + db.h + html_user/ + top_teams.php + sched/ + Makefile.in + *.C + sched_util.C,h (added) + update_stats.C (added) + diff --git a/db/db.h b/db/db.h index a8784f5b3b..d7521772ca 100644 --- a/db/db.h +++ b/db/db.h @@ -345,12 +345,15 @@ extern int db_user_new(USER&); extern int db_user(int, USER&); extern int db_user_update(USER&); extern int db_user_count(int&); +extern int db_user_count_team(TEAM&, int&); extern int db_user_lookup_auth(USER&); extern int db_user_lookup_email_addr(USER&); extern int db_user_enum_id(USER&); extern int db_user_enum_total_credit(USER&); extern int db_user_enum_expavg_credit(USER&); extern int db_user_enum_teamid(USER&); +extern int db_user_sum_team_expavg_credit(TEAM&,double&); +extern int db_user_sum_team_total_credit(TEAM&,double&); extern int db_team(int, TEAM&); extern int db_team_new(TEAM&); diff --git a/db/db_mysql.C b/db/db_mysql.C index 4a23f408bb..aa589f1886 100644 --- a/db/db_mysql.C +++ b/db/db_mysql.C @@ -619,6 +619,13 @@ int db_user_count(int& n) { return boinc_db.db_count(&n, "*", TYPE_USER); } +int db_user_count_team(TEAM& p, int& n) { + char buf[256]; + + sprintf(buf, "teamid=%d", p.id); + return boinc_db.db_count(&n, "*", TYPE_USER, buf); +} + int db_user_lookup_email_addr(USER& p) { char buf[256]; @@ -650,6 +657,20 @@ int db_user_enum_teamid(USER& p) { return boinc_db.db_enum(e, &p, TYPE_USER, buf); } +int db_user_sum_team_expavg_credit(TEAM& p, double& n) { + char buf[256]; + + sprintf(buf, "teamid=%d", p.id); + return boinc_db.db_sum(&n, "expavg_credit", TYPE_USER, buf); +} + +int db_user_sum_team_total_credit(TEAM& p, double& n) { + char buf[256]; + + sprintf(buf, "teamid=%d", p.id); + return boinc_db.db_sum(&n, "total_credit", TYPE_USER, buf); +} + ////////// TEAM ///////// int db_team(int i, TEAM& p) { diff --git a/db/mysql_util.C b/db/mysql_util.C index 67e46fe9f2..19a413919d 100644 --- a/db/mysql_util.C +++ b/db/mysql_util.C @@ -194,6 +194,17 @@ int MYSQL_DB::db_query_int(int* ip, char* query) { return 0; } +int MYSQL_DB::db_query_double(double* dp, char* query) { + mysql_query(mp, query); + rp = mysql_store_result(mp); + if (!rp) return -1; + row = mysql_fetch_row(rp); + if (!row) return -1; + *dp = atof(row[0]); + mysql_free_result(rp); + return 0; +} + int MYSQL_DB::db_count(int* np, char* what, int type, char* clause) { char buf[MAX_QUERY_LEN]; sprintf(buf, "select count(%s) from %s ", what, table_name[type]); @@ -201,6 +212,13 @@ int MYSQL_DB::db_count(int* np, char* what, int type, char* clause) { return db_query_int(np, buf); } +int MYSQL_DB::db_sum(double* np, char* what, int type, char* clause) { + char buf[MAX_QUERY_LEN]; + sprintf(buf, "select sum(%s) from %s ", what, table_name[type]); + if (clause) strcat(buf, clause); + return db_query_double(np, buf); +} + int MYSQL_DB::db_query(char* p) { return mysql_query(mp, p); } diff --git a/db/mysql_util.h b/db/mysql_util.h index a32c10bb4f..d7870df177 100644 --- a/db/mysql_util.h +++ b/db/mysql_util.h @@ -50,6 +50,8 @@ public: int db_enum(ENUM&, void*, int, char* clause=0, int limit=0); int db_enum_field(ENUM&, int, char*, char*); int db_query_int(int*, char*); + int db_query_double(double*, char*); int db_count(int*, char*, int, char* clause=0); + int db_sum(double*, char*, int, char* clause=0); int db_query(char*); }; diff --git a/html/user/top_teams.php b/html/user/top_teams.php index 650a6ce832..87ce24c911 100644 --- a/html/user/top_teams.php +++ b/html/user/top_teams.php @@ -20,30 +20,6 @@ function show_team_row($team) { db_init(); - // update team stats; eventually this must be moved - // to a separate program - // - $result = mysql_query("select * from team"); - while ($team = mysql_fetch_object($result)) { - $query = "select sum(total_credit) from user where teamid = $team->id"; - $result2 = mysql_query($query); - $total_credit_sum = mysql_result($result2, 0); - - $query = "select sum(expavg_credit) from user where teamid = $team->id"; - $result2 = mysql_query($query); - $expavg_credit_sum = mysql_result($result2, 0); - - $query = "select count(*) from user where teamid = $team->id"; - $result2 = mysql_query($query); - $nmembers = mysql_result($result2, 0); - - $total_credit = $total_credit_sum; - $expavg_credit = $expavg_credit_sum; - $query = "update team set nusers=$nmembers, total_credit=$total_credit, expavg_credit=$expavg_credit where id=$team->id"; - $result2 = mysql_query($query); - } - mysql_free_result($result); - page_head("Top teams"); $result = mysql_query("select * from team order by expavg_credit desc, total_credit desc"); team_table_start(); diff --git a/sched/Makefile.in b/sched/Makefile.in index 3385bec8e7..5bb9dee392 100644 --- a/sched/Makefile.in +++ b/sched/Makefile.in @@ -18,7 +18,7 @@ CC = g++ $(CFLAGS) CLIBS = @LIBS@ -PROGS = cgi feeder show_shmem file_upload_handler validate_test make_work result_retry file_deleter assimilator db_dump start_servers +PROGS = cgi feeder show_shmem file_upload_handler validate_test make_work result_retry file_deleter assimilator db_dump update_stats start_servers all: $(PROGS) @@ -27,6 +27,7 @@ CGI_OBJS = \ main.o \ sched_shmem.o \ server_types.o \ + sched_util.o \ config.o \ ../db/db_mysql.o \ ../db/mysql_util.o \ @@ -37,6 +38,7 @@ CGI_OBJS = \ FEEDER_OBJS = \ feeder.o \ sched_shmem.o \ + sched_util.o \ config.o \ ../db/db_mysql.o \ ../db/mysql_util.o \ @@ -47,6 +49,7 @@ FEEDER_OBJS = \ SHOW_SHMEM_OBJS = \ show_shmem.o \ sched_shmem.o \ + sched_util.o \ config.o \ ../db/db_mysql.o \ ../db/mysql_util.o \ @@ -67,6 +70,7 @@ FILE_UPLOAD_OBJS = \ VALIDATE_OBJS = \ validate.o \ validate_test.o \ + sched_util.o \ config.o \ ../db/db_mysql.o \ ../db/mysql_util.o \ @@ -75,6 +79,7 @@ VALIDATE_OBJS = \ FILE_DELETER_OBJS = \ file_deleter.o \ + sched_util.o \ config.o \ ../db/db_mysql.o \ ../db/mysql_util.o \ @@ -84,6 +89,7 @@ FILE_DELETER_OBJS = \ ASSIMILATOR_OBJS = \ assimilator.o \ assimilate_handler.o \ + sched_util.o \ config.o \ ../db/db_mysql.o \ ../db/mysql_util.o \ @@ -92,6 +98,7 @@ ASSIMILATOR_OBJS = \ MAKE_WORK_OBJS = \ make_work.o \ + sched_util.o \ config.o \ ../db/db_mysql.o \ ../db/mysql_util.o \ @@ -106,6 +113,7 @@ MAKE_WORK_OBJS = \ RESULT_RETRY_OBJS = \ result_retry.o \ + sched_util.o \ config.o \ ../db/db_mysql.o \ ../db/mysql_util.o \ @@ -123,9 +131,19 @@ DB_DUMP_OBJS = \ ../db/mysql_util.o \ ../lib/parse.o \ ../lib/util.o \ + sched_util.o \ config.o \ db_dump.o +UPDATE_STATS_OBJS = \ + ../db/db_mysql.o \ + ../db/mysql_util.o \ + ../lib/parse.o \ + ../lib/util.o \ + sched_util.o \ + config.o \ + update_stats.o + START_SERVERS_OBJS = \ ../lib/util.o \ ../lib/parse.o \ @@ -197,6 +215,9 @@ assimilator: $(ASSIMILATOR_OBJS) db_dump: $(DB_DUMP_OBJS) $(CC) $(DB_DUMP_OBJS) $(MYSQL_LIBS) $(CLIBS) -o db_dump +update_stats: $(UPDATE_STATS_OBJS) + $(CC) $(UPDATE_STATS_OBJS) $(MYSQL_LIBS) $(CLIBS) -o update_stats + start_servers: $(START_SERVERS_OBJS) $(CC) $(START_SERVERS_OBJS) $(CLIBS) -o start_servers diff --git a/sched/assimilator.C b/sched/assimilator.C index aec3574214..bb9832016a 100644 --- a/sched/assimilator.C +++ b/sched/assimilator.C @@ -27,16 +27,13 @@ #include "parse.h" #include "util.h" #include "config.h" +#include "sched_util.h" #include "assimilate_handler.h" #define ASSIMILATOR_LOCKFILE "assimilator.out" CONFIG config; -void write_log(char* p) { - fprintf(stderr, "%s: %s", timestamp(), p); -} - // assimilate all WUs that need it // return nonzero if did anything // @@ -48,6 +45,8 @@ bool do_pass(APP& app) { char buf[MAX_BLOB_SIZE]; unsigned int i; + check_stop_trigger(); + wu.appid = app.id; wu.assimilate_state = ASSIMILATE_READY; while (!db_workunit_enum_app_assimilate_state(wu)) { @@ -118,6 +117,7 @@ int main(int argc, char** argv) { int i; char buf[256]; + check_stop_trigger(); for (i=1; i\n" @@ -545,15 +521,14 @@ int tables_file() { int main(int argc, char** argv) { CONFIG config; int retval, i; - bool do_update_teams = false; char dir[256]; + check_stop_trigger(); + strcpy(dir, ""); strcpy(zip_cmd, ""); for (i=1; i and ) // Also patch the download URL (redundant) @@ -85,12 +81,6 @@ void replace_file_name( } } -void check_trigger() { - FILE* f = fopen(TRIGGER_FILENAME, "r"); - if (!f) return; - exit(0); -} - void make_work() { CONFIG config; char * p; @@ -200,7 +190,7 @@ void make_work() { sprintf(buf, "added result: %s_%s\n", wu.name, suffix); write_log(buf); nresults_left--; - check_trigger(); + check_stop_trigger(); } } @@ -208,7 +198,7 @@ int main(int argc, char** argv) { bool asynch = false; int i; - unlink(TRIGGER_FILENAME); + check_stop_trigger(); for (i=1; i + +#include "parse.h" +#include "util.h" +#include "main.h" +#include "sched_util.h" +#include "server_types.h" + +void write_log(char* p) { + fprintf(stderr, "%s: %s", timestamp(), p); +} + +void check_stop_trigger() { + FILE* f = fopen(STOP_TRIGGER_FILENAME, "r"); + if (!f) return; + exit(0); +} + +// fill in the nusers, total_credit and expavg_credit fields +// of the team table. +// This may take a while; don't do it often +// +int update_teams() { + TEAM team; + int retval; + + while (!db_team_enum(team)) { + retval = get_team_credit(team); + if (retval) return retval; + + // update the team record + retval = db_team_update(team); + if (retval) return retval; + } + return 0; +} + +int get_team_credit(TEAM &t) { + int nusers; + double expavg_credit, total_credit; + int retval; + + // count the number of users on a team + retval = db_user_count_team(t, nusers); + if (retval) return retval; + + // get the summed credit values for a team + retval = db_user_sum_team_expavg_credit(t, expavg_credit); + if (retval) return retval; + retval = db_user_sum_team_total_credit(t, total_credit); + if (retval) return retval; + + t.nusers = nusers; + t.total_credit = total_credit; + t.expavg_credit = expavg_credit; + + return 0; +} + +// update an exponential average of credit per second. +// +void update_average(double credit_assigned_time, double credit, double& avg, double& avg_time) { + time_t now = time(0); + + // decrease existing average according to how long it's been + // since it was computed + // + if (avg_time) { + double deltat = now - avg_time; + avg *= exp(-deltat*ALPHA); + } + if (credit_assigned_time) { + double deltat = now - credit_assigned_time; + // Add (credit)/(number of days to return result) to credit, which + // is the average number of cobblestones per day + avg += credit/(deltat/86400); + } + avg_time = now; +} + diff --git a/sched/sched_util.h b/sched/sched_util.h new file mode 100644 index 0000000000..a9333d7728 --- /dev/null +++ b/sched/sched_util.h @@ -0,0 +1,49 @@ +// The contents of this file are subject to the Mozilla Public License +// Version 1.0 (the "License"); you may not use this file except in +// compliance with the License. You may obtain a copy of the License at +// http://www.mozilla.org/MPL/ +// +// Software distributed under the License is distributed on an "AS IS" +// basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the +// License for the specific language governing rights and limitations +// under the License. +// +// The Original Code is the Berkeley Open Infrastructure for Network Computing. +// +// The Initial Developer of the Original Code is the SETI@home project. +// Portions created by the SETI@home project are Copyright (C) 2002 +// University of California at Berkeley. All Rights Reserved. +// +// Contributor(s): +// + +#ifndef _SCHED_UTIL_ +#define _SCHED_UTIL_ + +#include + +// "average credit" uses an exponential decay so that recent +// activity is weighted more heavily. +// H is the "half-life" period: the average goes down by 1/2 +// if idle for this period. +// Specifically, the weighting function W(t) is +// W(t) = exp(t/(H*log(2))*H*log(2). +// The average credit is the sum of X*W(t(X)) +// over units of credit X that were granted t(X) time ago. + +#define LOG2 M_LN2 + // log(2) +#define SECONDS_IN_DAY (3600*24) +#define AVG_HALF_LIFE (SECONDS_IN_DAY*7) +#define ALPHA (LOG2/AVG_HALF_LIFE) + +#define STOP_TRIGGER_FILENAME "stop_server" + +extern void write_log(char*); +extern void check_stop_trigger(); +extern int get_team_credit(TEAM&); +extern int update_teams(); +extern void update_average(double, double, double&, double&); + +#endif + diff --git a/sched/update_stats.C b/sched/update_stats.C new file mode 100644 index 0000000000..a6efcf474c --- /dev/null +++ b/sched/update_stats.C @@ -0,0 +1,123 @@ +// The contents of this file are subject to the Mozilla Public License +// Version 1.0 (the "License"); you may not use this file except in +// compliance with the License. You may obtain a copy of the License at +// http://www.mozilla.org/MPL/ +// +// Software distributed under the License is distributed on an "AS IS" +// basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the +// License for the specific language governing rights and limitations +// under the License. +// +// The Original Code is the Berkeley Open Infrastructure for Network Computing. +// +// The Initial Developer of the Original Code is the SETI@home project. +// Portions created by the SETI@home project are Copyright (C) 2002 +// University of California at Berkeley. All Rights Reserved. +// +// Contributor(s): +// + +// update_stats: update exponential average credit for users and hosts, +// and calculate total users/credit for teams +// +// usage: update_stats [-update_teams] [-update_users] [-update_hosts] + +#include +#include +#include +#include + +#include "db.h" +#include "util.h" +#include "config.h" +#include "sched_util.h" + +#define LOCKFILE "update_stats.out" + +int update_users() { + USER user; + int retval; + + while (!db_user_enum_id(user)) { + update_average(0, 0, user.expavg_credit, user.expavg_time); + retval = db_user_update(user); + if (retval) return retval; + } + + return 0; +} + +int update_hosts() { + HOST host; + int retval; + + while (!db_host_enum_id(host)) { + update_average(0, 0, host.expavg_credit, host.expavg_time); + retval = db_host_update(host); + if (retval) return retval; + } + + return 0; +} + +int main(int argc, char** argv) { + CONFIG config; + int retval, i; + bool do_update_teams = false; + bool do_update_users = false; + bool do_update_hosts = false; + + check_stop_trigger(); + + for (i=1; i