diff --git a/checkin_notes b/checkin_notes index 9c71405d2e..c30d1010ab 100644 --- a/checkin_notes +++ b/checkin_notes @@ -8546,3 +8546,16 @@ Charlie 19 Nov 2011 clientgui/ sg_ProjectPanel.cpp + +David 20 Nov 2011 + - client: on network_available() GUI RPC, + clear project-level upload and download backoffs, + as well as RPC and individual xfer backoffs. + This was an oversight. + + client/ + client_types.h + net_stats.cpp + ssim/ + ssim.cpp + makefile diff --git a/client/client_types.h b/client/client_types.h index 7433a75feb..f27f320934 100644 --- a/client/client_types.h +++ b/client/client_types.h @@ -198,6 +198,13 @@ struct FILE_XFER_BACKOFF { file_xfer_failures = 0; next_xfer_time = 0; } + + // clear backoff but maintain failure count; + // called when network becomes available + // + void clear_temporary() { + next_xfer_time = 0; + } }; // statistics at a specific day diff --git a/client/net_stats.cpp b/client/net_stats.cpp index e5ccebe00d..6498450474 100644 --- a/client/net_stats.cpp +++ b/client/net_stats.cpp @@ -172,6 +172,8 @@ void NET_STATUS::network_available() { for (i=0; imin_rpc_time = 0; + p->upload_backoff.clear_temporary(); + p->download_backoff.clear_temporary(); } // tell active tasks that network is available (for Folding@home) diff --git a/ssim/makefile b/ssim/makefile index adb8579b1c..ee96057f84 100644 --- a/ssim/makefile +++ b/ssim/makefile @@ -1,2 +1,2 @@ ssim: ssim.cpp des.h - g++ -g -o ssim ssim.cpp + g++ -g -Wall -o ssim ssim.cpp diff --git a/ssim/ssim.cpp b/ssim/ssim.cpp index adf4a1fb9b..64d175264a 100644 --- a/ssim/ssim.cpp +++ b/ssim/ssim.cpp @@ -44,17 +44,17 @@ // The bottom-level data units ("chunks") are stored on hosts, // with R-fold replication -#define ENCODING_N 10 -#define ENCODING_K 5 -#define ENCODING_M 15 +#define ENCODING_N 4 +#define ENCODING_K 2 +#define ENCODING_M 6 #define ENCODING_LEVELS 1 -#define REPLICATION_LEVEL 2 +#define REPLICATION_LEVEL 1 // When we need to reconstruct an encoded unit on the server, // we try to upload N_UPLOAD subunits, // where N <= N_UPLOAD <= M -#define N_UPLOAD 12 +#define N_UPLOAD 5 // Terminology: // @@ -62,7 +62,10 @@ // An encoded data unit is "present_on_server" if at least N // of its subunits are present_on_server (recursive definition). -// A chunk is "recoverable" if it is present on at least 1 host. +// A data unit is "recoverable" if it can be reconstruct on the server, +// based on current state. +// A chunk is "recoverable" if it is assigned at least 1 host. +// (if it is downloading, it's still present on the server) // An encoded data unit is "recoverable" if at least N // of its subunits are recoverable. @@ -110,28 +113,32 @@ struct CHUNK; struct META_CHUNK; struct DFILE; struct HOST; +set hosts; struct CHUNK_ON_HOST : public EVENT { HOST* host; CHUNK* chunk; + char name[256]; bool present_on_host; bool transfer_in_progress; // upload if present_on_host, else download virtual void handle(); }; +static int next_host_id=0; struct HOST : public EVENT { + int id; set chunks; virtual void handle(); + HOST() { + t = sim.now + ran_exp(HOST_LIFE_MEAN); + id = next_host_id++; + hosts.insert(this); + } }; -set hosts; - struct HOST_ARRIVAL : public EVENT { virtual void handle() { - HOST* h = new HOST; - h->t = t + ran_exp(HOST_LIFE_MEAN); - hosts.insert(h); - sim.insert(h); + sim.insert(new HOST); t += ran_exp(86400./HOSTS_PER_DAY); sim.insert(this); } @@ -139,22 +146,28 @@ struct HOST_ARRIVAL : public EVENT { struct REPORT_STATS : public EVENT { virtual void handle() { - printf("%f: %d hosts\n", t, hosts.size()); + printf("%f: %lu hosts\n", t, hosts.size()); t += 86400; sim.insert(this); } }; +void die(const char* msg) { + printf("%.0f: %s\n", sim.now, msg); + exit(1); +} + // base class for chunks and meta-chunks // struct DATA_UNIT { - virtual bool recoverable(){}; + virtual bool recoverable(){die("recoverable undef"); return false;}; // can be reconstructed w/o reconstructing parent, // assuming that current downloads succeed - virtual void start_upload(){}; - virtual void assign(){}; + virtual void start_upload(){die("start_upload undef"); }; + virtual void assign(){die("assign undef"); }; bool present_on_server; bool is_uploading; + char name[64]; }; struct CHUNK : DATA_UNIT { @@ -162,14 +175,40 @@ struct CHUNK : DATA_UNIT { META_CHUNK* parent; double size; - CHUNK(META_CHUNK* mc, double s) { - parent = mc; - size = s; - } + CHUNK(META_CHUNK* mc, double s, int index); virtual void assign(); void host_failed(CHUNK_ON_HOST* p); void upload_complete(); + virtual bool recoverable() { + return (!hosts.empty()); + } + virtual void start_upload() { + // if there's another replica, start upload of 1st instance + // NOTE: all instances are inherently present_on_host + // + CHUNK_ON_HOST *c = *(hosts.begin()); + c->transfer_in_progress = true; + c->t = sim.now + size/UPLOAD_BYTES_SEC; + printf("%.0f: starting upload of %s\n", sim.now, c->name); + sim.insert(c); + } + void download_complete() { + // see if we can remove chunk from server + // + int n=0; + for (unsigned int i=0; ipresent_on_host) { + n++; + } + } + if (n >= REPLICATION_LEVEL) { + printf("%.0f: %s replicated, removing from server\n", sim.now, name); + present_on_server = false; + } + + } }; struct META_CHUNK : DATA_UNIT { @@ -179,25 +218,9 @@ struct META_CHUNK : DATA_UNIT { DFILE* dfile; bool uploading; - META_CHUNK(DFILE* d, META_CHUNK* par, double size, int encoding_level) { - dfile = d; - parent = par; - if (encoding_level) { - for (int j=0; j= ENCODING_N) { - for (int i=0; i= ENCODING_N) { + uploading = true; + for (unsigned int i=0; irecoverable()) { c->start_upload(); } } + } else { + printf("%.0f: only %d recoverable children\n", sim.now, n); } } @@ -227,23 +256,46 @@ struct META_CHUNK : DATA_UNIT { } } + // this is called only if we're uploading + // void child_upload_complete() { - } - - void upload_complete() { + int n = 0; + for (unsigned int i=0; ipresent_on_server) { + n++; + } + } + if (n >= ENCODING_N) { + present_on_server = true; + } + assign(); + if (parent && parent->uploading) { + parent->child_upload_complete(); + } } }; +static int next_file_id=0; + struct DFILE : EVENT { META_CHUNK* meta_chunk; double size; + int id; set unused_hosts; // hosts that don't have any packets of this file + DFILE(double s) { + id = next_file_id++; + unused_hosts = hosts; + size = s; + } + // the creation of a file // virtual void handle() { - meta_chunk = new META_CHUNK(this, NULL, size, ENCODING_LEVELS); + printf("creating file %d\n", id); + meta_chunk = new META_CHUNK(this, NULL, size, ENCODING_LEVELS, id); meta_chunk->assign(); } }; @@ -257,17 +309,21 @@ void CHUNK_ON_HOST::handle() { if (present_on_host) { // it was an upload chunk->upload_complete(); // create new replicas if needed + printf("%.f: upload of %s completed\n", sim.now, name); } else { present_on_host = true; + printf("%.f: download of %s completed\n", sim.now, name); + chunk->download_complete(); } } // the host has departed // -void HOST:: handle() { +void HOST::handle() { set::iterator i = hosts.find(this); hosts.erase(i); + printf("%.0f: host %d failed\n", sim.now, id); set::iterator p; for (p = chunks.begin(); p != chunks.end(); p++) { CHUNK_ON_HOST* c = *p; @@ -276,27 +332,30 @@ void HOST:: handle() { } } +CHUNK::CHUNK(META_CHUNK* mc, double s, int index) { + parent = mc; + present_on_server = true; + size = s; + sprintf(name, "%s.%d", parent->name, index); +} + void CHUNK::host_failed(CHUNK_ON_HOST* p) { set::iterator i = hosts.find(p); hosts.erase(i); + printf("%.0f: handling loss of %s\n", sim.now, p->name); if (present_on_server) { // if data is on server, make a new replica // assign(); } else if (!hosts.empty()) { - // if there's another replica, start upload of 1st instance - // NOTE: all instances are inherently present_on_host - // - CHUNK_ON_HOST *c = *(hosts.begin()); - c->transfer_in_progress = true; - c->t = sim.now + size/UPLOAD_BYTES_SEC; - sim.insert(c); + start_upload(); } else { parent->child_unrecoverable(); } } void CHUNK::upload_complete() { + present_on_server = true; assign(); if (parent->uploading) { parent->child_upload_complete(); @@ -305,26 +364,67 @@ void CHUNK::upload_complete() { void CHUNK::assign() { while (hosts.size() < REPLICATION_LEVEL) { + if (parent->dfile->unused_hosts.size() == 0) { + die("no more hosts!\n"); + } set::iterator i = parent->dfile->unused_hosts.begin(); HOST* h = *i; parent->dfile->unused_hosts.erase(i); CHUNK_ON_HOST *c = new CHUNK_ON_HOST(); + sprintf(c->name, "chunk %s on host %d", name, h->id); + printf("%.0f: assigning chunk %s to host %d\n", sim.now, name, h->id); c->host = h; c->chunk = this; c->t = sim.now + size/DOWNLOAD_BYTES_SEC; + hosts.insert(c); + h->chunks.insert(c); sim.insert(c); } } +META_CHUNK::META_CHUNK( + DFILE* d, META_CHUNK* par, double size, int encoding_level, int index +) { + dfile = d; + parent = par; + if (parent) { + sprintf(name, "%s.%d", parent->name, index); + } else { + sprintf(name, "%d", index); + } + if (encoding_level) { + for (int j=0; j dfiles; int main() { +#if 0 HOST_ARRIVAL *h = new HOST_ARRIVAL; h->t = 0; sim.insert(h); REPORT_STATS* r = new REPORT_STATS; r->t = 0; sim.insert(r); +#endif + + for (int i=0; i<100; i++) { + sim.insert(new HOST); + } + sim.insert(new DFILE(1e12)); sim.simulate(200*86400); }