From 449e3fa25707ab18166707df8a95d39cb8fa7e16 Mon Sep 17 00:00:00 2001 From: David Anderson Date: Fri, 17 Aug 2007 18:05:39 +0000 Subject: [PATCH] - client: change the way upload/download throughput estimates are maintained. The old way took concurrency into account but ignored compression; the new way does the converse. - client: maintain new stat: recent average transfer rates. This track average up/down throughputs (over wall clock time, not just while xfers are active) with a half-life of 1 day. Will eventually add a new pref: don't fetch new work if either of these is above a threshold svn path=/trunk/boinc/; revision=13407 --- checkin_notes | 17 +++++ client/client_state.C | 2 +- client/file_names.C | 4 +- client/http_curl.C | 41 ++++++++++--- client/http_curl.h | 8 ++- client/net_stats.C | 140 +++++++++++++++++------------------------- client/net_stats.h | 20 +++--- 7 files changed, 128 insertions(+), 104 deletions(-) diff --git a/checkin_notes b/checkin_notes index 06942818f7..1f2ab9fa71 100755 --- a/checkin_notes +++ b/checkin_notes @@ -7956,3 +7956,20 @@ David 16 Aug 2007 lib/ util.C +David 17 Aug 2007 + - client: change the way upload/download throughput + estimates are maintained. + The old way took concurrency into account but ignored compression; + the new way does the converse. + - client: maintain new stat: recent average transfer rates. + This track average up/down throughputs + (over wall clock time, not just while xfers are active) + with a half-life of 1 day. + Will eventually add a new pref: don't fetch new work if + either of these is above a threshold + + client/ + client_state.C + file_names.C + http_curl.C,h + net_stats.C,h diff --git a/client/client_state.C b/client/client_state.C index 49cc1776b1..e3c8dd029a 100644 --- a/client/client_state.C +++ b/client/client_state.C @@ -572,7 +572,7 @@ bool CLIENT_STATE::poll_slow_events() { POLL_ACTION(update_results , update_results ); POLL_ACTION(gui_http , gui_http.poll ); if (!network_suspended) { - net_stats.poll(*file_xfers, *http_ops); + net_status.poll(); POLL_ACTION(acct_mgr , acct_mgr_info.poll ); POLL_ACTION(file_xfers , file_xfers->poll ); POLL_ACTION(pers_file_xfers , pers_file_xfers->poll ); diff --git a/client/file_names.C b/client/file_names.C index df6fad4bbc..bd8a5f7d83 100644 --- a/client/file_names.C +++ b/client/file_names.C @@ -291,12 +291,14 @@ bool is_image_file(const char* filename) { return false; } -int set_to_project_group(const char* path) { #ifdef SANDBOX +int set_to_project_group(const char* path) { if (g_use_sandbox) { if (boinc_exec(SETPROJECTGRP_FILE_NAME, (char*)path)) return ERR_CHOWN; } +#else +int set_to_project_group(const char*) { #endif return 0; } diff --git a/client/http_curl.C b/client/http_curl.C index 3247d1ef96..0e3665094e 100644 --- a/client/http_curl.C +++ b/client/http_curl.C @@ -835,7 +835,6 @@ void HTTP_OP::reset() { fileOut = NULL; connect_error = 0; bytes_xferred = 0; - xfer_speed = 0; bSentHeader = false; close_socket(); } @@ -938,16 +937,44 @@ void HTTP_OP_SET::got_select(FDSET_GROUP&, double timeout) { // update byte counts and transfer speed // if (hop->want_download) { - bytes_down += hop->bytes_xferred; - curlErr = curl_easy_getinfo(hop->curlEasy, - CURLINFO_SPEED_DOWNLOAD, &hop->xfer_speed + // SIZE_DOWNLOAD is the byte count "on the wire" + // (possible with compression) + // TOTAL_TIME is the elapsed time of the download + // STARTTRANSFER_TIME is portion of elapsed time involved + // with setup (connection establishment etc.) + // SPEED_DOWNLOAD is bytes/sec based on uncompressed size + // (we don't use it) + // + double size_download, total_time, starttransfer_time; + curlErr = curl_easy_getinfo(hop->curlEasy, + CURLINFO_SIZE_DOWNLOAD, &size_download ); + curlErr = curl_easy_getinfo(hop->curlEasy, + CURLINFO_TOTAL_TIME, &total_time + ); + curlErr = curl_easy_getinfo(hop->curlEasy, + CURLINFO_STARTTRANSFER_TIME, &starttransfer_time + ); + double dt = total_time - starttransfer_time; + if (dt > 0) { + gstate.net_stats.down.update(size_download, dt); + } } if (hop->want_upload) { - bytes_up += hop->bytes_xferred; - curlErr = curl_easy_getinfo(hop->curlEasy, - CURLINFO_SPEED_UPLOAD, &hop->xfer_speed + double size_upload, total_time, starttransfer_time; + curlErr = curl_easy_getinfo(hop->curlEasy, + CURLINFO_SIZE_UPLOAD, &size_upload ); + curlErr = curl_easy_getinfo(hop->curlEasy, + CURLINFO_TOTAL_TIME, &total_time + ); + curlErr = curl_easy_getinfo(hop->curlEasy, + CURLINFO_STARTTRANSFER_TIME, &starttransfer_time + ); + double dt = total_time - starttransfer_time; + if (dt > 0) { + gstate.net_stats.up.update(size_upload, dt); + } } // if proxy/socks server uses authentication and its not set yet, diff --git a/client/http_curl.h b/client/http_curl.h index 6f51cb2eaa..f0e058aa46 100644 --- a/client/http_curl.h +++ b/client/http_curl.h @@ -94,10 +94,12 @@ public: // the above two MUST be long (not int) // otherwise breaks on 64-bit machines double start_time; - double xfer_speed; - double bytes_xferred; // bytes transferred overall + double bytes_xferred; + // uncompressed bytes transferred in this session double start_bytes_xferred; - + double xfer_speed; + // tranfer rate based on elapsed time and bytes_xferred + // (hence doesn't reflect compression; used only for GUI) int http_op_state; // values above int http_op_type; // HTTP_OP_* (see above) int http_op_retval; diff --git a/client/net_stats.C b/client/net_stats.C index af3b811ec4..521cafb51b 100644 --- a/client/net_stats.C +++ b/client/net_stats.C @@ -19,18 +19,15 @@ // NET_STATS estimates average network throughput, // i.e. the average total throughput in both the up and down directions. -// Here's how it works: NET_STATS::poll() is called every second or so. -// If there are any file transfers active, -// it increments elapsed time and byte counts, -// and maintains an exponential average of throughput. +// +// NET_STATUS keeps track of whether we have a physical connection, +// and whether we need one #include "cpp.h" #ifdef _WIN32 #include "boinc_win.h" -#endif - -#ifndef _WIN32 +#else #include "config.h" #include #include @@ -40,6 +37,7 @@ #include "time.h" #include "str_util.h" #include "error_numbers.h" +#include "util.h" #include "client_msgs.h" #include "client_state.h" @@ -47,105 +45,75 @@ #include "net_stats.h" -#define EXP_DECAY_RATE (1./3600) +#define NET_RATE_HALF_LIFE (7*86400) NET_STATUS net_status; NET_STATS::NET_STATS() { - last_time = 0; memset(&up, 0, sizeof(up)); memset(&down, 0, sizeof(down)); } -void NET_INFO::update(double dt, double nb, bool active) { - //msg_printf(NULL, MSG_INFO, "dt %f nb %f active %d", dt, nb, active); - if (active) { - delta_t += dt; - delta_nbytes += nb-last_bytes; - } - last_bytes = nb; -} - -double NET_INFO::throughput() { - double x, tp, new_tp=0; - if (starting_throughput > 0) { - if (delta_t > 0) { - x = exp(-delta_t*EXP_DECAY_RATE); - tp = delta_nbytes/delta_t; - new_tp = x*starting_throughput + (1-x)*tp; - } else { - new_tp = starting_throughput; - } - } else if (delta_t > 0) { - new_tp = delta_nbytes/delta_t; - } else { - } -#if 0 - msg_printf(NULL, MSG_INFO, "start %f delta_t %f delta_nb %f new_tp %f", - starting_throughput, delta_t, delta_nbytes, new_tp - ); -#endif - starting_throughput = new_tp; - delta_nbytes = delta_t = 0; - return new_tp; -} - -void NET_STATS::poll(FILE_XFER_SET& fxs, HTTP_OP_SET& hops) { - double dt; - bool upload_active, download_active; - - if (last_time == 0) { - dt = 0; - } else { - dt = gstate.now - last_time; - } - last_time = gstate.now; - - fxs.check_active(upload_active, download_active); - up.update(dt, hops.bytes_up, upload_active); - down.update(dt, hops.bytes_down, download_active); - - if (net_status.need_to_contact_reference_site && gstate.gui_http.state==GUI_HTTP_STATE_IDLE) { - net_status.contact_reference_site(); - } -} - -// Write XML based network statistics +// called after file xfer to update rates // +void NET_INFO::update(double nbytes, double dt) { + if (nbytes == 0 || dt==0) return; + double bytes_sec = nbytes/dt; + if (max_rate == 0) { + max_rate = bytes_sec; // first time + } else { + // somewhat arbitrary weighting formula + // + double w = log(nbytes)/500; + if (w>1) w = 1; + max_rate = w*bytes_sec + (1-w)*max_rate; + } + double start_time = gstate.now - dt; + update_average( + start_time, + nbytes, + NET_RATE_HALF_LIFE, + avg_rate, + avg_time + ); +} + int NET_STATS::write(MIOFILE& out) { out.printf( "\n" - " %g\n" - " %g\n" + " %f\n" + " %f\n" + " %f\n" + " %f\n" + " %f\n" + " %f\n" "\n", - up.throughput(), - down.throughput() + up.max_rate, + up.avg_rate, + up.avg_time, + down.max_rate, + down.avg_rate, + down.avg_time ); return 0; } -// Read XML based network statistics -// int NET_STATS::parse(MIOFILE& in) { char buf[256]; - double bwup, bwdown; memset(this, 0, sizeof(NET_STATS)); while (in.fgets(buf, 256)) { if (match_tag(buf, "")) return 0; - else if (parse_double(buf, "", bwup)) { - up.starting_throughput = bwup; - continue; - } - else if (parse_double(buf, "", bwdown)) { - down.starting_throughput = bwdown; - continue; - } else { - if (log_flags.unparsed_xml) { - msg_printf(NULL, MSG_INFO, - "[unparsed_xml] Unrecognized network statistics line: %s", buf - ); - } + if (parse_double(buf, "", up.max_rate)) continue; + if (parse_double(buf, "", up.avg_rate)) continue; + if (parse_double(buf, "", up.avg_time)) continue; + if (parse_double(buf, "", down.max_rate)) continue; + if (parse_double(buf, "", down.avg_rate)) continue; + if (parse_double(buf, "", down.avg_time)) continue; + if (log_flags.unparsed_xml) { + msg_printf(NULL, MSG_INFO, + "[unparsed_xml] Unrecognized network statistics line: %s", buf + ); } } return ERR_XML_PARSE; @@ -284,4 +252,10 @@ void LOOKUP_WEBSITE_OP::handle_reply(int http_op_retval) { } } +void NET_STATUS::poll() { + if (net_status.need_to_contact_reference_site && gstate.gui_http.state==GUI_HTTP_STATE_IDLE) { + net_status.contact_reference_site(); + } +} + const char *BOINC_RCSID_733b4006f5 = "$Id$"; diff --git a/client/net_stats.h b/client/net_stats.h index cf7fd6e05a..fbe4885839 100644 --- a/client/net_stats.h +++ b/client/net_stats.h @@ -36,24 +36,25 @@ class HTTP_OP_SET; // there's one of these each for upload and download // struct NET_INFO { - double delta_t; // elapsed time of file transfer activity - // in this session of client - double delta_nbytes; // bytes transferred in this session - double last_bytes; - double starting_throughput; // throughput at start of session + double max_rate; + // estimate of max transfer rate; computed as an average of + // the rates of recent file transfers, weighted by file size. + // This ignores concurrency of transfers. + double avg_rate; // recent average transfer rate + double avg_time; // when avg_rate was last updated + void update(double nbytes, double dt); + // updates the above vars - void update(double dt, double nb, bool active); - double throughput(); }; class NET_STATS { public: - double last_time; + //double last_time; NET_INFO up; NET_INFO down; NET_STATS(); - void poll(FILE_XFER_SET&, HTTP_OP_SET&); + //void poll(FILE_XFER_SET&, HTTP_OP_SET&); int write(MIOFILE&); int parse(MIOFILE&); @@ -92,6 +93,7 @@ public: show_ref_message = false; last_comm_time = 0; } + void poll(); }; // This is used to access a reference website (like yahoo or google)