- client: change the way upload/download throughput

estimates are maintained.
	The old way took concurrency into account but ignored compression;
	the new way does the converse.
- client: maintain new stat: recent average transfer rates.
	This track average up/down throughputs
	(over wall clock time, not just while xfers are active)
	with a half-life of 1 day.
	Will eventually add a new pref: don't fetch new work if
	either of these is above a threshold

svn path=/trunk/boinc/; revision=13407

checkin_notes

@@ -7956,3 +7956,20 @@ David  16 Aug 2007
 	lib/
 		util.C
 
+David  17 Aug 2007
+	- client: change the way upload/download throughput
+		estimates are maintained.
+		The old way took concurrency into account but ignored compression;
+		the new way does the converse.
+	- client: maintain new stat: recent average transfer rates.
+		This track average up/down throughputs
+		(over wall clock time, not just while xfers are active)
+		with a half-life of 1 day.
+		Will eventually add a new pref: don't fetch new work if
+		either of these is above a threshold
+
+	client/
+		client_state.C
+		file_names.C
+		http_curl.C,h
+		net_stats.C,h

client/client_state.C

@@ -572,7 +572,7 @@ bool CLIENT_STATE::poll_slow_events() {
     POLL_ACTION(update_results         , update_results         );
     POLL_ACTION(gui_http               , gui_http.poll          );
     if (!network_suspended) {
-        net_stats.poll(*file_xfers, *http_ops);
+        net_status.poll();
         POLL_ACTION(acct_mgr               , acct_mgr_info.poll     );
         POLL_ACTION(file_xfers             , file_xfers->poll       );
         POLL_ACTION(pers_file_xfers        , pers_file_xfers->poll  );

client/file_names.C

@@ -291,12 +291,14 @@ bool is_image_file(const char* filename) {
     return false;
 }
 
-int set_to_project_group(const char* path) {
 #ifdef SANDBOX
+int set_to_project_group(const char* path) {
     if (g_use_sandbox) {
         if (boinc_exec(SETPROJECTGRP_FILE_NAME, (char*)path))
             return ERR_CHOWN;
     }
+#else
+int set_to_project_group(const char*) {
 #endif
     return 0;
 }

client/http_curl.C

@@ -835,7 +835,6 @@ void HTTP_OP::reset() {
     fileOut = NULL;
     connect_error = 0;
     bytes_xferred = 0;
-    xfer_speed = 0;
     bSentHeader = false;
     close_socket();
 }
@@ -938,16 +937,44 @@ void HTTP_OP_SET::got_select(FDSET_GROUP&, double timeout) {
         // update byte counts and transfer speed
         //
         if (hop->want_download) {
-            bytes_down += hop->bytes_xferred;
+            // SIZE_DOWNLOAD is the byte count "on the wire"
-            curlErr = curl_easy_getinfo(hop->curlEasy, 
+            // (possible with compression)
-                CURLINFO_SPEED_DOWNLOAD, &hop->xfer_speed
+            // TOTAL_TIME is the elapsed time of the download
+            // STARTTRANSFER_TIME is portion of elapsed time involved
+            // with setup (connection establishment etc.)
+            // SPEED_DOWNLOAD is bytes/sec based on uncompressed size
+            // (we don't use it)
+            //
+            double size_download, total_time, starttransfer_time;
+			curlErr = curl_easy_getinfo(hop->curlEasy, 
+                CURLINFO_SIZE_DOWNLOAD, &size_download
             );
+			curlErr = curl_easy_getinfo(hop->curlEasy, 
+                CURLINFO_TOTAL_TIME, &total_time
+            );
+			curlErr = curl_easy_getinfo(hop->curlEasy, 
+                CURLINFO_STARTTRANSFER_TIME, &starttransfer_time
+            );
+            double dt = total_time - starttransfer_time;
+            if (dt > 0) {
+                gstate.net_stats.down.update(size_download, dt);
+            }
         }
         if (hop->want_upload) {
-            bytes_up += hop->bytes_xferred;  
+            double size_upload, total_time, starttransfer_time;
-            curlErr = curl_easy_getinfo(hop->curlEasy, 
+			curlErr = curl_easy_getinfo(hop->curlEasy, 
-                CURLINFO_SPEED_UPLOAD, &hop->xfer_speed
+                CURLINFO_SIZE_UPLOAD, &size_upload
             );
+			curlErr = curl_easy_getinfo(hop->curlEasy, 
+                CURLINFO_TOTAL_TIME, &total_time
+            );
+			curlErr = curl_easy_getinfo(hop->curlEasy, 
+                CURLINFO_STARTTRANSFER_TIME, &starttransfer_time
+            );
+            double dt = total_time - starttransfer_time;
+            if (dt > 0) {
+                gstate.net_stats.up.update(size_upload, dt);
+            }
         }
 
         // if proxy/socks server uses authentication and its not set yet,

client/http_curl.h

@@ -94,10 +94,12 @@ public:
         // the above two MUST be long (not int)
         // otherwise breaks on 64-bit machines
     double start_time;
-    double xfer_speed;
+    double bytes_xferred;
-    double bytes_xferred;   // bytes transferred overall
+        // uncompressed bytes transferred in this session
 	double start_bytes_xferred;
-
+    double xfer_speed;
+        // tranfer rate based on elapsed time and bytes_xferred
+        // (hence doesn't reflect compression; used only for GUI)
 	int http_op_state;      // values above
     int http_op_type;       // HTTP_OP_* (see above)
     int http_op_retval;

client/net_stats.C

@@ -19,18 +19,15 @@
 
 // NET_STATS estimates average network throughput,
 // i.e. the average total throughput in both the up and down directions.
-// Here's how it works: NET_STATS::poll() is called every second or so.
+//
-// If there are any file transfers active,
+// NET_STATUS keeps track of whether we have a physical connection,
-// it increments elapsed time and byte counts,
+// and whether we need one
-// and maintains an exponential average of throughput.
 
 #include "cpp.h"
 
 #ifdef _WIN32
 #include "boinc_win.h"
-#endif
+#else
-
-#ifndef _WIN32
 #include "config.h"
 #include <cstring>
 #include <cmath>
@@ -40,6 +37,7 @@
 #include "time.h"
 #include "str_util.h"
 #include "error_numbers.h"
+#include "util.h"
 
 #include "client_msgs.h"
 #include "client_state.h"
@@ -47,105 +45,75 @@
 
 #include "net_stats.h"
 
-#define EXP_DECAY_RATE (1./3600)
+#define NET_RATE_HALF_LIFE  (7*86400)
 
 NET_STATUS net_status;
 
 NET_STATS::NET_STATS() {
-    last_time = 0;
     memset(&up, 0, sizeof(up));
     memset(&down, 0, sizeof(down));
 }
 
-void NET_INFO::update(double dt, double nb, bool active) {
+// called after file xfer to update rates
-    //msg_printf(NULL, MSG_INFO, "dt %f nb %f active %d", dt, nb, active);
-    if (active) {
-        delta_t += dt;
-        delta_nbytes += nb-last_bytes;
-    }
-    last_bytes = nb;
-}
-
-double NET_INFO::throughput() {
-    double x, tp, new_tp=0;
-    if (starting_throughput > 0) {
-        if (delta_t > 0) {
-            x = exp(-delta_t*EXP_DECAY_RATE);
-            tp = delta_nbytes/delta_t;
-            new_tp = x*starting_throughput + (1-x)*tp;
-        } else {
-            new_tp = starting_throughput;
-        }
-    } else if (delta_t > 0) {
-        new_tp = delta_nbytes/delta_t;
-    } else {
-    }
-#if 0
-    msg_printf(NULL, MSG_INFO, "start %f delta_t %f delta_nb %f new_tp %f",
-        starting_throughput, delta_t, delta_nbytes, new_tp
-    );
-#endif
-    starting_throughput = new_tp;
-    delta_nbytes = delta_t = 0;
-    return new_tp;
-}
-
-void NET_STATS::poll(FILE_XFER_SET& fxs, HTTP_OP_SET& hops) {
-    double dt;
-    bool upload_active, download_active;
-
-    if (last_time == 0) {
-        dt = 0;
-    } else {
-        dt = gstate.now - last_time;
-    }
-    last_time = gstate.now;
-
-    fxs.check_active(upload_active, download_active);
-    up.update(dt, hops.bytes_up, upload_active);
-    down.update(dt, hops.bytes_down, download_active);
-
-	if (net_status.need_to_contact_reference_site && gstate.gui_http.state==GUI_HTTP_STATE_IDLE) {
-		net_status.contact_reference_site();
-	}
-}
-
-// Write XML based network statistics
 //
+void NET_INFO::update(double nbytes, double dt) {
+    if (nbytes == 0 || dt==0) return;
+    double bytes_sec = nbytes/dt;
+    if (max_rate == 0) {
+        max_rate = bytes_sec;   // first time
+    } else {
+        // somewhat arbitrary weighting formula
+        //
+        double w = log(nbytes)/500;
+        if (w>1) w = 1;
+        max_rate = w*bytes_sec + (1-w)*max_rate;
+    }
+    double start_time = gstate.now - dt;
+    update_average(
+        start_time,
+        nbytes,
+        NET_RATE_HALF_LIFE,
+        avg_rate,
+        avg_time
+    );
+}
+
 int NET_STATS::write(MIOFILE& out) {
     out.printf(
         "<net_stats>\n"
-        "    <bwup>%g</bwup>\n"
+        "    <bwup>%f</bwup>\n"
-        "    <bwdown>%g</bwdown>\n"
+        "    <avg_up>%f</avg_up>\n"
+        "    <avg_time_up>%f</avg_time_up>\n"
+        "    <bwdown>%f</bwdown>\n"
+        "    <avg_down>%f</avg_down>\n"
+        "    <avg_time_down>%f</avg_time_down>\n"
         "</net_stats>\n",
-        up.throughput(),
+        up.max_rate,
-        down.throughput()
+        up.avg_rate,
+        up.avg_time,
+        down.max_rate,
+        down.avg_rate,
+        down.avg_time
     );
     return 0;
 }
 
-// Read XML based network statistics
-//
 int NET_STATS::parse(MIOFILE& in) {
     char buf[256];
-    double bwup, bwdown;
 
     memset(this, 0, sizeof(NET_STATS));
     while (in.fgets(buf, 256)) {
         if (match_tag(buf, "</net_stats>")) return 0;
-        else if (parse_double(buf, "<bwup>", bwup)) {
+        if (parse_double(buf, "<bwup>", up.max_rate)) continue;
-            up.starting_throughput = bwup;
+        if (parse_double(buf, "<avg_up>", up.avg_rate)) continue;
-            continue;
+        if (parse_double(buf, "<avg_time_up>", up.avg_time)) continue;
-        }
+        if (parse_double(buf, "<bwdown>", down.max_rate)) continue;
-        else if (parse_double(buf, "<bwdown>", bwdown)) {
+        if (parse_double(buf, "<avg_down>", down.avg_rate)) continue;
-            down.starting_throughput = bwdown;
+        if (parse_double(buf, "<avg_time_down>", down.avg_time)) continue;
-            continue;
+        if (log_flags.unparsed_xml) {
-        } else {
+            msg_printf(NULL, MSG_INFO,
-            if (log_flags.unparsed_xml) {
+                "[unparsed_xml] Unrecognized network statistics line: %s", buf
-                msg_printf(NULL, MSG_INFO,
+            );
-                    "[unparsed_xml] Unrecognized network statistics line: %s", buf
-                );
-            }
         }
     }
     return ERR_XML_PARSE;
@@ -284,4 +252,10 @@ void LOOKUP_WEBSITE_OP::handle_reply(int http_op_retval) {
     }
 }
 
+void NET_STATUS::poll() {
+	if (net_status.need_to_contact_reference_site && gstate.gui_http.state==GUI_HTTP_STATE_IDLE) {
+		net_status.contact_reference_site();
+	}
+}
+
 const char *BOINC_RCSID_733b4006f5 = "$Id$";

client/net_stats.h

@@ -36,24 +36,25 @@ class HTTP_OP_SET;
 // there's one of these each for upload and download
 //
 struct NET_INFO {
-    double delta_t;         // elapsed time of file transfer activity
+    double max_rate;
-                            // in this session of client
+        // estimate of max transfer rate; computed as an average of
-    double delta_nbytes;    // bytes transferred in this session
+        // the rates of recent file transfers, weighted by file size.
-    double last_bytes;
+        // This ignores concurrency of transfers.
-    double starting_throughput; // throughput at start of session
+    double avg_rate;        // recent average transfer rate
+    double avg_time;        // when avg_rate was last updated
+    void update(double nbytes, double dt);
+        // updates the above vars
 
-    void update(double dt, double nb, bool active);
-    double throughput();
 };
 
 class NET_STATS {
 public:
-    double last_time;
+    //double last_time;
     NET_INFO up;
     NET_INFO down;
 
     NET_STATS();
-    void poll(FILE_XFER_SET&, HTTP_OP_SET&);
+    //void poll(FILE_XFER_SET&, HTTP_OP_SET&);
 
     int write(MIOFILE&);
     int parse(MIOFILE&);
@@ -92,6 +93,7 @@ public:
         show_ref_message = false;
 		last_comm_time = 0;
     }
+    void poll();
 };
 
 // This is used to access a reference website (like yahoo or google)