David 24 Apr 2007

- Server: add new database field host.error_rate.
        Stores a dynamic estimate of the fraction of the hosts's
        results that fail validation (updated by the validator).

    client/
        sim.C
        cs_prefs.C
    db/
        boinc_db.C,h
    html/ops/
        db_update.php
    sched/
        handle_request.C
        sched_msgs.h
        validator.C

svn path=/trunk/boinc/; revision=12466

checkin_notes

@@ -3954,3 +3954,20 @@ Rom    24 Apr 2007
     
     win_build/
         *.vcproj
+
+David  24 Apr 2007
+    - Server: add new database field host.error_rate.
+        Stores a dynamic estimate of the fraction of the hosts's
+        results that fail validation (updated by the validator).
+
+    client/
+        sim.C
+        cs_prefs.C
+    db/
+        boinc_db.C,h
+    html/ops/
+        db_update.php
+    sched/
+        handle_request.C
+        sched_msgs.h
+        validator.C

client/cs_prefs.C

@@ -407,7 +407,7 @@ void CLIENT_STATE::read_global_prefs() {
 		(host_info.m_nbytes*global_prefs.ram_max_used_idle_frac)/MEGA
     );
     msg_printf(NULL, MSG_INFO,
-		"Preferences: limit disk usage to %.2fGB",
+		"Preferences limit disk usage to %.2fGB",
         allowed_disk_usage()/GIGA
     );
     // max_cpus, bandwidth limits may have changed

client/sim.C

@@ -352,13 +352,14 @@ bool CLIENT_STATE::simulate_rpc(PROJECT* _p) {
         wup->rsc_fpops_est = ap->fpops_est;
         results.push_back(rp);
         double ops = ap->fpops.sample();
+        if (ops < 0) ops = 0;
         rp->final_cpu_time = ops/net_fpops;
         rp->report_deadline = now + ap->latency_bound;
         sprintf(buf, "got job %s: CPU time %.2f, deadline %s<br>",
             rp->name, rp->final_cpu_time, time_to_string(rp->report_deadline)
         );
         html_msg += buf;
-        p->work_request -= ap->fpops_est/net_fpops;
+        p->work_request -= p->duration_correction_factor*ap->fpops_est/net_fpops;
     }
     p->work_request = 0;
     request_schedule_cpus("simulate_rpc");
@@ -920,7 +921,7 @@ void parse_error(char* file, int retval) {
 }
 
 void help(char* prog) {
-    fprintf(stderr, "usage: %s [--duration X] [--delta X]\n", prog);
+    fprintf(stderr, "usage: %s [--duration X] [--delta X] [--dirs ...]\n", prog);
     exit(1);
 }
 

db/boinc_db.C

@@ -358,7 +358,8 @@ void DB_HOST::db_print(char* buf){
         "credit_per_cpu_sec=%.15e, "
         "venue='%s', nresults_today=%d, "
         "avg_turnaround=%f, "
-        "host_cpid='%s', external_ip_addr='%s', max_results_day=%d ",
+        "host_cpid='%s', external_ip_addr='%s', max_results_day=%d, "
+        "error_rate=%f ",
         create_time, userid,
         rpc_seqno, rpc_time,
         total_credit, expavg_credit, expavg_time,
@@ -376,7 +377,8 @@ void DB_HOST::db_print(char* buf){
         credit_per_cpu_sec,
         venue, nresults_today,
         avg_turnaround,
-        host_cpid, external_ip_addr, max_results_day
+        host_cpid, external_ip_addr, max_results_day,
+        error_rate
     );
     UNESCAPE(domain_name);
     UNESCAPE(serialnum);
@@ -434,10 +436,12 @@ void DB_HOST::db_parse(MYSQL_ROW &r) {
     strcpy2(host_cpid, r[i++]);
     strcpy2(external_ip_addr, r[i++]);
     max_results_day = atoi(r[i++]);
+    error_rate = atof(r[i++]);
 }
 
-// update fields that differ from the argument HOST.
+// Update fields that differ from the argument HOST.
-// called from scheduler (handle_request.C)
+// Called from scheduler (handle_request.C),
+// so only include fields modified by the scheduler.
 //
 int DB_HOST::update_diff(HOST& h) {
     char buf[LARGE_BLOB_SIZE], updates[LARGE_BLOB_SIZE], query[LARGE_BLOB_SIZE];

db/boinc_db.h

@@ -280,6 +280,8 @@ struct HOST {
     char external_ip_addr[256]; // IP address seen by scheduler
     int max_results_day;    // maximum # of results to send per day per CPU
         // this is dynamically adjusted to limit work sent to bad hosts
+    double error_rate;      // dynamic estimate of fraction of results
+                            // that fail validation
 
     // the following not stored in DB
     //

db/schema.sql

@@ -6,8 +6,9 @@
     html/
         inc/
             host.inc (host)
-        ops/
             db_ops.inc
+        ops/
+            db_update.php
         user/
             create_account_action.php (user)
             team_create_action.php (team)
@@ -167,6 +168,7 @@ create table host (
     host_cpid           varchar(254),
     external_ip_addr    varchar(254),
     max_results_day     integer     not null,
+    error_rate          double      not null default 0,
 
     primary key (id)
 ) type=InnoDB;

doc/boinc_news.php

@@ -1,6 +1,17 @@
 <?
 
 $project_news = array(
+array("April 23, 2007",
+    "Krystof Dolezal's
+    <a href=http://www.kd-web.info/clanky.php>Flash-based BOINC tutorials</a>
+    are now available in English and Slovak,
+    as well as Czech."
+),
+array("April 23, 2007",
+    "BOINCstats and BAM! are now available in the Czech language
+    (<a href=http://cz.boincstats.com>http://cz.boincstats.com</a>).
+    Thanks to Zelvuska for the translation."
+),
 array("April 17, 2007",
     "<a href=http://desktopgrid.hu/>Desktopgrid.hu</a>
     has been created by the Computer and Automation Research Institute

doc/index.php

@@ -68,7 +68,7 @@ function show_participate() {
         ".sprintf(tr(HOME_P3), "<a href=help.php>", "</a>")."
         <center>
         <a href=download.php><b>".tr(HOME_DOWNLOAD)."</b></a>
-        | <a href=participate.php><b><nobr>".tr(HOME_MORE_INFO)."</nobr></b></a> 
+        | <a href=trac/wiki/RunningBoinc><b><nobr>".tr(HOME_MORE_INFO)."</nobr></b></a> 
         | <a href=links.php><b><nobr>".tr(HOME_WEB_SITES)."</nobr></b></a>
         | <a href=addons.php><b>".tr(HOME_ADD_ONS)."</b></a>
         | <a href=poll.php><b><nobr>".tr(HOME_SURVEY)."</nobr></b></a>
@@ -109,10 +109,10 @@ function show_other() {
         <tr><td bgcolor=$light_blue><font size=4>Other info</font></td></tr>
         <tr><td>
             <ul>
-            <li> <a href=intro.php>Overview</a>
+            <li> <a href=trac/wiki/BoincIntro/>Overview</a>
             <li> <a href=trac/>Software development</a>
             <li> <a href=translation.php>Translation</a> of web and GUI text
-            <li> <a href=contact.php>Personnel and contributors</a>
+            <li> <a href=trac/wiki/ProjectPeople>Personnel and contributors</a>
             <li> BOINC <a href=email_lists.php>email lists</a>
             <li> BOINC <a href=dev/>message boards</a>
             <li> <a href=papers.php>Papers and talks</a> about BOINC
@@ -190,9 +190,4 @@ echo "
 ";
 
 page_tail(true, true);
-echo "
-    <script language=\"JavaScript\" type=\"text/javascript\" src=\"wz_tooltip.js\"></script>
-    </body>
-    </html>
-";
 ?>

doc/links.php

@@ -118,7 +118,7 @@ Show if the servers of various projects are up or down.
 <h3>Miscellaneous</h3>
 ";
 $misc_sites = array(
-    array("http://www.kd-web.info/clanky.php", "Flash-based BOINC tutorial", "(in Czech)"),
+    array("http://www.kd-web.info/clanky.php", "Flash-based BOINC tutorial", "(in Czech, English, and Slovak)"),
     array("http://www.myboinc.com/", "BOINC Users of the Day"),
     array("http://groups.myspace.com/BOINConMYSPACE", "BOINC on MySpace"),
     array("http://www.boincuk.com/repository.php", "bunc", "(excellent newsletter produced by BOINC UK)"),
@@ -196,14 +196,14 @@ language("English", array(
 ));
 language("Estonian", array(
     site("http://boinc.tmac.pri.ee", "boinc.tmac.pri.ee"),
-    site("http://setimehed.net/", "setimehed.net"),
+    //site("http://setimehed.net/", "setimehed.net"),
 ));
 language("Finnish", array(
     site( "http://www.universe-examiners.org/",
         "Universe Examiners"),
-    site(
+    //site("http://news.universe-examiners.org/asennus/boinc.html",
-    "http://news.universe-examiners.org/asennus/boinc.html",
+    //    "BOINC instructions in Finnish"
-        "BOINC instructions in Finnish")
+    //),
 ));
 language("French", array(
     site("http://boincfrance.org", "BOINCFRANCE.ORG"),
@@ -247,7 +247,7 @@ language("Korean", array(
 ));
 
 language("Polish", array(
-    site("http://www.boinc-polska.org/", "BOINC-Polska.org"),
+    //site("http://www.boinc-polska.org/", "BOINC-Polska.org"),
     site("http://www.boinc.org.pl/", "Team boinc.pl"),
     site("http://www.boinc.prv.pl", "BOINC@Kolobrzeg"),
     site("http://www.boincatpoland.org", "BOINC@Poland"),

doc/logo.php

@@ -62,7 +62,7 @@ $logos = array(
         <img src=logos/markus/Logo2/MarkusBoinc.jpg>
         <br>... and <a href=logos/markus>various others</a>"
     ),
-    array("<a href=http://www.boincproject.org/>Invisible Design</a>",
+    array("Invisible Design",
         "<img src=logos/logo.boinc.240x80-01.jpg>
         ... and <a href=logos/>many variants</a> in different sizes and colors,
         and for specific countries."

doc/project_cookbook.php

@@ -1,6 +1,6 @@
 <?php
 require_once("docutil.php");
-page_head("Project creation cookbook");
+page_head("Project creation cookbook [Deprecated - wiki]");
 echo "
 <h2>Make skeletal project</h2>
 <ul>

doc/translation.php

@@ -97,7 +97,7 @@ the character set (e.g. iso-8859-1) used in the translation.
 Here are links to the translation files for
 <ul>
 <li> <a href=http://boinc.berkeley.edu/trac/browser/trunk/boinc/languages/translations/>this web site</a>
-<li> <a href=http://boinc.berkeley.edu/trac/browser/trunk/boinc/html/languages/translations/>the BOINC-supplied part of project web sites.
+<li> <a href=http://boinc.berkeley.edu/trac/browser/trunk/boinc/html/languages/translations/>the BOINC-supplied part of project web sites</a>.
 <li> <a href=http://boinc.berkeley.edu/trac/browser/trunk/boinc/locale/client/>the BOINC Manager</a>.
 </ul>
 

doc/validate_simple.php

@@ -1,7 +1,7 @@
 <?php
 
 require_once("docutil.php");
-page_head("Simple validator framework");
+page_head("Simple validator framework [deprecated: SimpleValidation]");
 echo "
 To create a validator using the simple framework,
 you must supply four functions:

html/ops/db_update.php

@@ -360,6 +360,11 @@ function update_4_07_2007() {
     );
 }
 
+function update_4_24_2007() {
+    do_query('alter table host add error_rate double not null default 0');
+    
+}
+
 // modify the following to call the function you want.
 // Make sure you do all needed functions, in order.
 // (Look at your DB structure using "explain" queries to see

sched/handle_request.C

@@ -330,6 +330,7 @@ make_new_host:
         host.userid = reply.user.id;
         host.rpc_seqno = 0;
         host.expavg_time = time(0);
+        host.error_rate = 0.1;
         strcpy(host.venue, reply.user.venue);
         host.fix_nans();
         retval = host.insert();

sched/sched_msgs.h

@@ -23,10 +23,10 @@
 #include "msg_log.h"
 
 class SCHED_MSG_LOG : public MSG_LOG {
-    int debug_level;
     const char* v_format_kind(int kind) const;
     bool v_message_wanted(int kind) const;
 public:
+    int debug_level;
     enum Kind {
         MSG_CRITICAL=1,
         MSG_NORMAL=2,

sched/validator.C

@@ -79,11 +79,20 @@ double max_granted_credit = 0;
 double max_claimed_credit = 0;
 bool grant_claimed_credit = false;
 
+void update_error_rate(DB_HOST& host, bool valid) {
+    if (host.error_rate > 1) host.error_rate = 1;
+    if (host.error_rate <= 0) host.error_rate = 0.1;
+    //
+    host.error_rate *= 0.95;
+    if (!valid) {
+        host.error_rate += 0.05;
+    }
+}
 
-// here when a result has been validated and its granted_credit as been set.
+// Here when a result has been validated and its granted_credit as been set.
-// grant credit to host, user and team
+// Grant credit to host, user and team, and update host error rate.
 //
-int grant_credit(RESULT& result) {
+int is_valid(RESULT& result) {
     DB_USER user;
     DB_HOST host;
     DB_TEAM team;
@@ -147,10 +156,12 @@ int grant_credit(RESULT& result) {
         );
     }
 
+    double old_error_rate = host.error_rate;
+    update_error_rate(host, true);
     sprintf(
         buf,
-        "total_credit=total_credit+%f, expavg_credit=%f, expavg_time=%f, avg_turnaround=%f, credit_per_cpu_sec=%f",
+        "total_credit=total_credit+%f, expavg_credit=%f, expavg_time=%f, avg_turnaround=%f, credit_per_cpu_sec=%f, error_rate=%f",
-        result.granted_credit,  host.expavg_credit, host.expavg_time, host.avg_turnaround, host.credit_per_cpu_sec
+        result.granted_credit,  host.expavg_credit, host.expavg_time, host.avg_turnaround, host.credit_per_cpu_sec, host.error_rate
     );
     retval = host.update_field(buf);
     if (retval) {
@@ -160,6 +171,10 @@ int grant_credit(RESULT& result) {
             result.id, result.hostid, retval
         );
     }
+    log_messages.printf(SCHED_MSG_LOG::MSG_DEBUG,
+        "[HOST#%d] error rate %f->%f\n",
+        host.id, old_error_rate, host.error_rate
+    );
 
     if (user.teamid) {
         retval = team.lookup_id(user.teamid);
@@ -189,6 +204,39 @@ int grant_credit(RESULT& result) {
     return 0;
 }
 
+int is_invalid(RESULT& result) {
+    char buf[256];
+    int retval;
+    DB_HOST host;
+
+    retval = host.lookup_id(result.hostid);
+    if (retval) {
+        log_messages.printf(
+            SCHED_MSG_LOG::MSG_CRITICAL,
+            "[RESULT#%d] lookup of host %d failed %d\n",
+            result.id, result.hostid, retval
+        );
+        return retval;
+    }
+    double old_error_rate = host.error_rate;
+    update_error_rate(host, false);
+    sprintf(buf, "error_rate=%f", host.error_rate);
+    retval = host.update_field(buf);
+    if (retval) {
+        log_messages.printf(
+            SCHED_MSG_LOG::MSG_CRITICAL,
+            "[RESULT#%d] update of host %d failed %d\n",
+            result.id, result.hostid, retval
+        );
+        return retval;
+    }
+    log_messages.printf(SCHED_MSG_LOG::MSG_DEBUG,
+        "[HOST#%d] invalid result; error rate %f->%f\n",
+        host.id, old_error_rate, host.error_rate
+    );
+    return 0;
+}
+
 // Return zero iff we resolved the WU
 //
 int handle_wu(
@@ -277,7 +325,7 @@ int handle_wu(
                     "[RESULT#%d %s] pair_check() matched: setting result to valid; credit %f\n",
                     result.id, result.name, result.granted_credit
                 );
-                retval = grant_credit(result);
+                retval = is_valid(result);
                 if (retval) {
                     log_messages.printf(
                         SCHED_MSG_LOG::MSG_NORMAL,
@@ -293,6 +341,7 @@ int handle_wu(
                     "[RESULT#%d %s] pair_check() didn't match: setting result to invalid\n",
                     result.id, result.name
                 );
+                is_invalid(result);
             }
             if (update_result) {
                 log_messages.printf(
@@ -385,11 +434,11 @@ int handle_wu(
                     if (max_granted_credit && result.granted_credit > max_granted_credit) {
                         result.granted_credit = max_granted_credit;
                     }
-                    retval = grant_credit(result);
+                    retval = is_valid(result);
                     if (retval) {
                         log_messages.printf(
                             SCHED_MSG_LOG::MSG_DEBUG,
-                            "[RESULT#%d %s] grant_credit() failed: %d\n",
+                            "[RESULT#%d %s] is_valid() failed: %d\n",
                             result.id, result.name, retval
                         );
                     }
@@ -400,6 +449,7 @@ int handle_wu(
                     );
                     break;
                 case VALIDATE_STATE_INVALID:
+                    is_invalid(result);
                     update_result = true;
                     break;
                 case VALIDATE_STATE_INIT:
@@ -426,8 +476,7 @@ int handle_wu(
                 // the transitioner - doing so creates a race condition
                 //
                 transition_time = NEVER;
-                log_messages.printf(
+                log_messages.printf(SCHED_MSG_LOG::MSG_DEBUG,
-                    SCHED_MSG_LOG::MSG_DEBUG,
                     "[WU#%d %s] Found a canonical result: id=%d\n",
                     wu.id, wu.name, canonicalid
                 );
@@ -643,7 +692,9 @@ int main(int argc, char** argv) {
         exit(1);
     }
 
-    log_messages.printf(SCHED_MSG_LOG::MSG_NORMAL, "Starting validator\n");
+    log_messages.printf(SCHED_MSG_LOG::MSG_NORMAL,
+        "Starting validator, debug level %d\n", log_messages.debug_level
+    );
     if (wu_id_modulus) {
         log_messages.printf(SCHED_MSG_LOG::MSG_NORMAL,
             "Modulus %d, remainder %d\n", wu_id_modulus, wu_id_remainder