- Implementation of automatic credit leveling for cpu based projects that

wish to use it. - The script calculate_credit_multiplier (expected to be run daily as a config.xml task) looks at the ratio of granted credit to CPU time for recent results for each app. Multiplier is calculated to cause median hosts granted credit per cpu second to equal to equal that expected from its benchmarks. This is 30-day exponentially averaged with the previous value of the multplier and stored in the table credit_multplier. - When a result is received the server adjusts claimed credit by the value the multiplier had when the result was sent. svn path=/trunk/boinc/; revision=15661
2008-07-22 23:36:55 +00:00 · 2008-07-22 23:36:55 +00:00 · a5a6f693cd
parent a7eb1597a7
commit a5a6f693cd
8 changed files with 178 additions and 13 deletions
--- a/24
+++ b/24
@ -5922,3 +5922,27 @@ David  21 July 2008

    sched/
        sched_config.h
+
+Eric  22 July 2008
+    - Implementation of automatic credit leveling for cpu based projects that
+      wish to use it.
+    - The script calculate_credit_multiplier (expected to be run daily as
+      a config.xml task) looks at the ratio of granted credit to CPU time 
+      for recent results for each app.  Multiplier is calculated to cause 
+      median hosts granted credit per cpu second to equal to equal that 
+      expected from its benchmarks.  This is 30-day exponentially averaged 
+      with the previous value of the multplier and stored in the table 
+      credit_multplier.
+    - When a result is received the server adjusts claimed credit by the
+      value the multiplier had when the result was sent.
+
+    tools/
+        calculate_credit_multiplier
+    db/
+        boinc_db.[Ch]
+    db/
+        schema.sql
+    sched/
+        sched_util.[Ch]
+	sched_result.C
+
--- a/db/boinc_db.C
+++ b/db/boinc_db.C
@ -68,6 +68,7 @@ void ASSIGNMENT::clear() {memset(this, 0, sizeof(*this));}
 void TRANSITIONER_ITEM::clear() {memset(this, 0, sizeof(*this));}
 void VALIDATOR_ITEM::clear() {memset(this, 0, sizeof(*this));}
 void SCHED_RESULT_ITEM::clear() {memset(this, 0, sizeof(*this));}
+void CREDIT_MULTIPLIER::clear() {memset(this, 0, sizeof(*this));}

 DB_PLATFORM::DB_PLATFORM(DB_CONN* dc) :
    DB_BASE("platform", dc?dc:&boinc_db){}
@ -93,6 +94,8 @@ DB_MSG_TO_HOST::DB_MSG_TO_HOST(DB_CONN* dc) :
    DB_BASE("msg_to_host", dc?dc:&boinc_db){}
 DB_ASSIGNMENT::DB_ASSIGNMENT(DB_CONN* dc) :
    DB_BASE("assignment", dc?dc:&boinc_db){}
+DB_CREDIT_MULTIPLIER::DB_CREDIT_MULTIPLIER(DB_CONN* dc) :
+    DB_BASE("credit_multiplier", dc?dc:&boinc_db){}
 DB_TRANSITIONER_ITEM_SET::DB_TRANSITIONER_ITEM_SET(DB_CONN* dc) :
    DB_BASE_SPECIAL(dc?dc:&boinc_db){}
 DB_VALIDATOR_ITEM_SET::DB_VALIDATOR_ITEM_SET(DB_CONN* dc) :
@ -118,6 +121,7 @@ int DB_RESULT::get_id() {return id;}
 int DB_MSG_FROM_HOST::get_id() {return id;}
 int DB_MSG_TO_HOST::get_id() {return id;}
 int DB_ASSIGNMENT::get_id() {return id;}
+int DB_CREDIT_MULTIPLIER::get_id() {return id;}

 void DB_PLATFORM::db_print(char* buf){
    sprintf(buf,
@ -913,6 +917,56 @@ void DB_ASSIGNMENT::db_parse(MYSQL_ROW& r) {
    resultid = atoi(r[i++]);
 }

+void DB_CREDIT_MULTIPLIER::db_print(char* buf) {
+    sprintf(buf,
+        "appid=%d, "
+        "time=%d, "
+        "multiplier=%f ",
+        appid,
+        time,
+        multiplier
+    );
+}
+
+void DB_CREDIT_MULTIPLIER::db_parse(MYSQL_ROW& r) {
+    int i=0;
+    clear();
+    id = atoi(r[i++]);
+    appid = atoi(r[i++]);
+    time = atoi(r[i++]);
+    multiplier = atof(r[i++]);
+}
+
+void DB_CREDIT_MULTIPLIER::get_nearest(int Appid, int Time) {
+    char query[MAX_QUERY_LEN];
+    MYSQL_ROW row;
+    MYSQL_RES *rp;
+    // set default values.
+    clear();
+    multiplier=1;
+    time=::time(NULL);
+    appid=Appid;
+
+    snprintf(query,MAX_QUERY_LEN,
+	"select * from credit_multiplier where appid=%d and"
+	  "abs(time-%d)=("
+	    "select min(abs(time-%d)) from credit_multiplier where appid=%d"
+	  ") limit 1",
+	Appid,Time,Time,Appid
+    );
+    if (db->do_query(query) != 0) return;
+    rp = mysql_store_result(db->mysql);
+    if (!rp) return;
+
+    row = mysql_fetch_row(rp);
+    if (!row) {
+      mysql_free_result(rp);
+    } else {
+      db_parse(row);
+    }
+    return;
+}
+
 void TRANSITIONER_ITEM::parse(MYSQL_ROW& r) {
    int i=0;
    clear();
@ -1483,6 +1537,7 @@ void SCHED_RESULT_ITEM::parse(MYSQL_ROW& r) {
    id = atoi(r[i++]);
    strcpy2(name, r[i++]);
    workunitid = atoi(r[i++]);
+    appid = atoi(r[i++]);
    server_state = atoi(r[i++]);
    hostid = atoi(r[i++]);
    userid = atoi(r[i++]);
@ -1516,6 +1571,7 @@ int DB_SCHED_RESULT_ITEM_SET::enumerate() {
        "   id, "
        "   name, "
        "   workunitid, "
+	"   appid, "
        "   server_state, "
        "   hostid, "
        "   userid, "
--- a/db/boinc_db.h
+++ b/db/boinc_db.h
@ -565,6 +565,23 @@ struct TRANSITIONER_ITEM {
    void parse(MYSQL_ROW&);
 };

+struct CREDIT_MULTIPLIER {
+    int id;
+    int appid;
+    int time;
+    double multiplier;
+
+    void clear();
+};
+    
+struct DB_CREDIT_MULTIPLIER : public DB_BASE, public CREDIT_MULTIPLIER {
+    DB_CREDIT_MULTIPLIER(DB_CONN* p=0);
+    int get_id();
+    void db_print(char *);
+    void db_parse(MYSQL_ROW &row);
+    void get_nearest(int appid, int time);
+};
+
 struct VALIDATOR_ITEM {
    WORKUNIT wu;
    RESULT res;
@ -573,6 +590,8 @@ struct VALIDATOR_ITEM {
    void parse(MYSQL_ROW&);
 };

+    
+
 class DB_PLATFORM : public DB_BASE, public PLATFORM {
 public:
    DB_PLATFORM(DB_CONN* p=0);
@ -716,6 +735,7 @@ public:
    int update_workunit(WORKUNIT&);
 };

+
 // used by the feeder and scheduler for outgoing work
 //
 struct WORK_ITEM {
@ -773,6 +793,7 @@ struct SCHED_RESULT_ITEM {
    int id;
    char name[256];
    int workunitid;
+    int appid;
    int server_state;
    int client_state;
    int validate_state;
--- a/db/schema.sql
+++ b/db/schema.sql
@ -587,6 +587,7 @@ create table notify (
 -- to automatically adjust granted credit.
 create table credit_multiplier (
    id                  serial          primary key,
+    appid		integer	        not null,
    time                integer         not null,
    multiplier          double          not null default 0
 ) engine=MyISAM;
--- a/sched/sched_result.C
+++ b/sched/sched_result.C
@ -250,6 +250,9 @@ int handle_results(SCHEDULER_REQUEST& sreq, SCHEDULER_REPLY& reply) {
        } else {
            srip->claimed_credit = srip->cpu_time * reply.host.claimed_credit_per_cpu_sec;
        }
+	// Regardless of the method of claiming credit, multiply by the
+	// application's credit multiplier at the time of result creation.
+	srip->claimed_credit*=credit_multiplier(srip->appid,srip->sent_time);
        if (config.debug_handle_results) {
            log_messages.printf(MSG_DEBUG,
                "cpu time %f credit/sec %f, claimed credit %f\n", srip->cpu_time, reply.host.claimed_credit_per_cpu_sec, srip->claimed_credit
--- a/sched/sched_util.C
+++ b/sched/sched_util.C
@ -247,6 +247,12 @@ double fpops_to_credit(double fpops, double intops) {
    return std::max(fpc, intc);
 }

+double credit_multiplier(int appid, time_t create_time) {
+    DB_CREDIT_MULTIPLIER mult;
+    mult.get_nearest(appid,create_time);
+    return mult.multiplier;
+}
+
 int count_results(char* query, int& n) {
    DB_RESULT result;
    int retval = result.count(n, query);
--- a/sched/sched_util.h
+++ b/sched/sched_util.h
@ -41,6 +41,7 @@ extern bool check_stop_sched();
 extern void install_stop_signal_handler();
 extern int try_fopen(const char* path, FILE*& f, const char* mode);
 extern void get_log_path(char*, const char*);
+extern double credit_multiplier(int, time_t);

 // convert filename to path in a hierarchical directory system
 //
--- a/tools/calculate_credit_multiplier
+++ b/tools/calculate_credit_multiplier
@ -1,12 +1,23 @@
 #! /bin/sh
 dbhost=`grep db_host ../config.xml | tr '[\<\>]' '[  ]' | head -1 | awk '{print $2}'`
+replica_dbhost=`grep replica_db_host ../config.xml | tr '[\<\>]' '[  ]' | head -1 | awk '{print $2}'`
 dbuser=`grep db_user ../config.xml | tr '[\<\>]' '[  ]' | head -1 | awk '{print $2}'`
 dbname=`grep db_name ../config.xml | tr '[\<\>]' '[  ]' | head -1 | awk '{print $2}'`
+if [ -z "${replica_dbhost}" ] ; then
+  replica_dbhost=${dbhost}
+fi
+
 MYSQL="mysql -D $dbname -h $dbhost -u $dbuser -N -B"
+MYSQL_R="mysql -D $dbname -h $replica_dbhost -u $dbuser -N -B"

 DECAY_TIME=30*86400
+RUN_INCREMENT=86400
+MAX_NRESULTS=10000
+FILE_DELETE_READY=1
+FILE_DELETE_DONE=2
+FILE_DELETE_ERROR=3

-function median_query() {
+function median_host_query() {
  $MYSQL --execute="
     create temporary table medians (id int auto_increment primary key)
       select $1 from host 
@ -23,21 +34,63 @@ function median_query() {
     drop table ids;"
 }

-median_credit_per_cpu_sec=`median_query credit_per_cpu_sec`
-median_p_fpops=`median_query p_fpops`
-median_p_iops=`median_query p_iops`
-ratio=`$MYSQL --execute="select ($median_p_fpops+$median_p_iops)/($median_credit_per_cpu_sec)*5.787037037037e-13"`
+function get_recent_credited_results() {
+  $MYSQL_R --execute="
+    create temporary table recent_results 
+      select 
+        granted_credit,
+	cpu_time,
+        hostid,
+	p_fpops,
+	p_iops
+      from host,result where 
+        file_delete_state in ($FILE_DELETE_READY,$FILE_DELETE_DONE) and
+        granted_credit>0 and
+        received_time>unix_timestamp(now())-$DECAY_TIME and
+	unix_timestamp(result.mod_time)>unix_timestamp(now())-$RUN_INCREMENT and
+	host.rpc_time>unix_timestamp(now())-$DECAY_TIME and
+	appid=$1 and
+	result.hostid=host.id
+      limit $MAX_NRESULTS;
+    create temporary table merged_results 
+        select sum(granted_credit)/sum(cpu_time) as granted_rate,
+	hostid,
+	(p_fpops+p_iops)*5.787037037037e-13 as bench_rate 
+    from recent_results 
+    group by hostid;
+    create temporary table medians (id int auto_increment primary key)
+      select (bench_rate+1e-10)/(granted_rate+1e-10) as mult 
+      from merged_results 
+      order by mult; 
+    create temporary table ids
+      select round(avg(id)-0.5) as id from medians;
+    insert into ids select round(avg(id)+0.5) from medians;
+    select avg(mult) from medians where id in (
+      select id from ids
+    );
+    drop table ids;
+    drop table medians;
+    drop table merged_results;
+    drop table recent_results;"
+}

-nrows=`$MYSQL --execute="select count(id) from credit_multiplier"`
-if [ $nrows = 0 ] ; then
-  $MYSQL --execute="insert into credit_multiplier values (0,unix_timestamp(now())-86400,1.0)"
-fi

-last_value=`$MYSQL --execute="select multiplier from credit_multiplier where (unix_timestamp(now())-time)=(select min(unix_timestamp(now())-time) from credit_multiplier)"`
-last_time=`$MYSQL --execute="select unix_timestamp(now())-time from credit_multiplier where (unix_timestamp(now())-time)=(select min(unix_timestamp(now())-time) from credit_multiplier)"`
+appids=`$MYSQL --execute="select id from app where deprecated=0 and beta=0"`

-value=`$MYSQL --execute="select ($last_value*(($DECAY_TIME)-$last_time)+$ratio*$last_time)/($DECAY_TIME)"`
+for appid in $appids ; do
+  ratio=`get_recent_credited_results $appid`

-$MYSQL --execute="insert into credit_multiplier values (0,unix_timestamp(now()),$value);"
+  nrows=`$MYSQL --execute="select count(id) from credit_multiplier where appid=$appid"`
+  if [ $nrows = 0 ] ; then
+    $MYSQL --execute="insert into credit_multiplier values (0,$appid,unix_timestamp(now())-86400,1.0)"
+  fi
+
+    last_value=`$MYSQL --execute="select multiplier from credit_multiplier where appid=$appid and (unix_timestamp(now())-time)=(select min(unix_timestamp(now())-time) from credit_multiplier where appid=$appid)"`
+    last_time=`$MYSQL --execute="select unix_timestamp(now())-time from credit_multiplier where appid=$appid and (unix_timestamp(now())-time)=(select min(unix_timestamp(now())-time) from credit_multiplier where appid=$appid)"`
+
+    value=`$MYSQL --execute="select ($last_value*(($DECAY_TIME)-$last_time)+$ratio*$last_time)/($DECAY_TIME)"`
+
+    $MYSQL --execute="insert into credit_multiplier values (0,$appid,unix_timestamp(now()),$value);"
+done

 exit 0