From e5aa5f813a575ebf59e26979750fc3fcc1fd4e23 Mon Sep 17 00:00:00 2001 From: David Anderson Date: Wed, 18 Jan 2012 02:24:18 +0000 Subject: [PATCH] - storage simulator work - lib: recent checkin broke strip_whitespace(string&). svn path=/trunk/boinc/; revision=25089 --- checkin_notes | 11 +++ lib/str_util.cpp | 10 +-- ssim/plot | 6 ++ ssim/ssim.cpp | 230 +++++++++++++++++++++++++++++++++++------------ ssim/ssim.php | 93 +++++++++++++++++++ version.h | 2 +- 6 files changed, 289 insertions(+), 63 deletions(-) create mode 100755 ssim/plot create mode 100644 ssim/ssim.php diff --git a/checkin_notes b/checkin_notes index 214438a577..979cca8094 100644 --- a/checkin_notes +++ b/checkin_notes @@ -655,3 +655,14 @@ Rom 17 Jan 2012 samples/vboxwrapper vbox.cpp + +David 17 Jan 2012 + - storage simulator work + - lib: recent checkin broke strip_whitespace(string&). + + ssim/ + ssim.cpp + plot + ssim.php + lib/ + str_util.cpp diff --git a/lib/str_util.cpp b/lib/str_util.cpp index ec81feab3f..ed442d2ef8 100644 --- a/lib/str_util.cpp +++ b/lib/str_util.cpp @@ -312,7 +312,7 @@ int parse_command_line(char* p, char** argv) { // remove whitespace from start and end of a string // void strip_whitespace(char *str) { - char *s=str; + char *s = str; while (*s) { if (!isascii(*s)) break; if (!isspace(*s)) break; @@ -320,8 +320,8 @@ void strip_whitespace(char *str) { } if (s != str) strcpy_overlap(str, s); - int n=strlen(str); - while(n>0) { + int n = strlen(str); + while (n>0) { n--; if (!isascii(str[n])) break; if (!isspace(str[n])) break; @@ -339,9 +339,9 @@ void strip_whitespace(string& str) { int n = (int) str.length(); while (n>0) { + if (!isascii(str[n-1])) break; + if (!isspace(str[n-1])) break; n--; - if (!isascii(str[n])) break; - if (!isspace(str[n])) break; } str.erase(n, str.length()-n); } diff --git a/ssim/plot b/ssim/plot new file mode 100755 index 0000000000..324647d6b2 --- /dev/null +++ b/ssim/plot @@ -0,0 +1,6 @@ +#!/bin/sh + +gnuplot < disk.gp > disk.png +gnuplot < upload.gp > upload.png +gnuplot < download.gp > download.png +gnuplot < fault_tol.gp > fault_tol.png diff --git a/ssim/ssim.cpp b/ssim/ssim.cpp index d20b15b485..596bf1aee3 100644 --- a/ssim/ssim.cpp +++ b/ssim/ssim.cpp @@ -19,6 +19,30 @@ // // Simulates the storage of files on a dynamic set of hosts. +// usage: ssim +// [--policy filename] +// [--host_life_mean x] +// [--connect_interval x] +// [--mean_xfer_rate x] +// +// outputs: +// stdout: log info +// summary.txt: format +// fault tolerance min +// disk_usage mean +// upload_mean +// download_mean + +#include +#include +#include +#include +#include + +#include "des.h" + +using std::set; + // We simulate policies based on coding and replication. // // Coding means that data is divided into M = N+K units, @@ -45,31 +69,59 @@ struct PARAMS { // - the population is unbounded // - host lifetime is exponentially distributed // - the time needed to transfer n bytes of data to/from a host is - // U1*connect_interval + (U2+.5)*n/mean_transfer_rate; + // U1*connect_interval + (U2+.5)*n/mean_xfer_rate; // where U1 and U2 are uniform random vars // (U1 is per-transfer, U2 is per-host) // double host_life_mean; double connect_interval; - double mean_transfer_rate; + double mean_xfer_rate; + int replication; int coding_levels; CODING codings[10]; - int replication; PARAMS() { // default parameters // host_life_mean = 100.*86400; connect_interval = 86400.; - mean_transfer_rate = .2e6; + mean_xfer_rate = .2e6; coding_levels = 1; - codings[0].n = 4; - codings[0].k = 2; - codings[0].m = 6; - codings[0].n_upload = 5; + codings[0].n = 10; + codings[0].k = 6; + codings[0].m = 16; + codings[0].n_upload = 12; replication = 2; } + int parse_policy(const char* filename) { + int n; + FILE* f = fopen(filename, "r"); + if (!f) { + fprintf(stderr, "No policy file %s\n", filename); + exit(1); + } + n = fscanf(f, "%d", &replication); + if (n != 1) { + fprintf(stderr, "parse error in %s\n", filename); + exit(1); + } + n = fscanf(f, "%d", &coding_levels); + if (n != 1) { + fprintf(stderr, "parse error in %s\n", filename); + exit(1); + } + for (int i=0; i -#include -#include -#include -#include - -#include "des.h" - -using std::set; - #define EVENT_DEBUG #define SAMPLE_DEBUG //#define RECOVERY_DEBUG @@ -154,7 +196,7 @@ struct HOST : public EVENT { HOST() { t = sim.now + ran_exp(p.host_life_mean); id = next_host_id++; - transfer_rate = p.mean_transfer_rate*(drand() + .5); + transfer_rate = p.mean_xfer_rate*(drand() + .5); hosts.insert(this); } }; @@ -193,6 +235,7 @@ struct CHUNK_ON_HOST : public EVENT { inline bool download_in_progress() { return (transfer_in_progress && !present_on_host); } + void remove(); }; #define PRESENT 0 @@ -252,31 +295,53 @@ struct META_CHUNK : DATA_UNIT { // keeps track of a time-varying property of a file // (server disk usage, up/download rate, fault tolerance level) // + +typedef enum {DISK, NETWORK, FAULT_TOLERANCE} STATS_KIND; + struct STATS_ITEM { + STATS_KIND kind; double value; double integral; - double max_val; - double max_val_time; - double min_val; - double min_val_time; + double extreme_val; + double extreme_val_time; double prev_t; double start_time; bool first; char name[256]; + FILE* f; - STATS_ITEM() { + void init(const char* n, const char* filename, STATS_KIND k) { + f = fopen(filename, "w"); + strcpy(name, n); + kind = k; value = 0; integral = 0; - max_val = 0; - max_val_time = 0; - min_val = INT_MAX; - min_val_time = 0; + switch (kind) { + case DISK: + case NETWORK: + extreme_val = 0; + break; + case FAULT_TOLERANCE: + extreme_val = INT_MAX; + break; + } + extreme_val_time = 0; first = true; } void sample(double v, bool collecting_stats) { #ifdef SAMPLE_DEBUG - printf("%s: %s: %f -> %f\n", now_str(), name, value, v); + switch (kind) { + case DISK: + printf("%s: %s: %fGB -> %fGB\n", now_str(), name, value/1e9, v/1e9); + break; + case NETWORK: + printf("%s: %s: %fMbps -> %fMbps\n", now_str(), name, value/1e6, v/1e6); + break; + case FAULT_TOLERANCE: + printf("%s: %s: %.0f -> %.0f\n", now_str(), name, value, v); + break; + } #endif double old_val = value; value = v; @@ -289,30 +354,50 @@ struct STATS_ITEM { double dt = sim.now - prev_t; prev_t = sim.now; integral += dt*old_val; - if (v > max_val) { - max_val = v; - max_val_time = sim.now; - } - if (v < min_val) { - min_val = v; - min_val_time = sim.now; + switch (kind) { + case DISK: + case NETWORK: + if (v > extreme_val) { + extreme_val = v; + extreme_val_time = sim.now; + } + break; + case FAULT_TOLERANCE: + if (v < extreme_val) { + extreme_val = v; + extreme_val_time = sim.now; + } + break; } + + fprintf(f, "%f %f\n", sim.now, old_val); + fprintf(f, "%f %f\n", sim.now, v); } void sample_inc(double inc, bool collecting_stats) { sample(value+inc, collecting_stats); } - void print(bool show_min) { + void print() { sample_inc(0, true); double dt = sim.now - start_time; - printf(" mean: %f\n", integral/dt); - if (show_min) { - printf(" min: %f\n", min_val); - } else { - printf(" max: %f\n", max_val); + switch (kind) { + case DISK: + printf(" mean: %fGB. Max: %fGB at %s\n", + (integral/dt)/1e9, extreme_val/1e9, time_str(extreme_val_time) + ); + break; + case NETWORK: + printf(" mean: %fMbps. Max: %fMbps at %s\n", + (integral/dt)/1e6, extreme_val/1e6, time_str(extreme_val_time) + ); + break; + case FAULT_TOLERANCE: + printf(" mean: %.2f. Min: %.0f at %s\n", + integral/dt, extreme_val, time_str(extreme_val_time) + ); + break; } - printf(" time of max: %s\n", time_str(max_val_time)); } }; @@ -342,10 +427,10 @@ struct DFILE : EVENT { unused_hosts = hosts; #endif size = s; - strcpy(disk_usage.name, "Disk usage"); - strcpy(upload_rate.name, "Upload rate"); - strcpy(download_rate.name, "Download rate"); - strcpy(fault_tolerance.name, "Fault tolerance"); + disk_usage.init("Disk usage", "disk.dat", DISK); + upload_rate.init("Upload rate", "upload.dat", NETWORK); + download_rate.init("Download rate", "download.dat", NETWORK); + fault_tolerance.init("Fault tolerance", "fault_tol.dat", FAULT_TOLERANCE); } // the creation of a file @@ -374,13 +459,13 @@ struct DFILE : EVENT { void print_stats() { printf("Statistics for file %d\n", id); printf(" Server disk usage:\n"); - disk_usage.print(false); + disk_usage.print(); printf(" Upload rate:\n"); - upload_rate.print(false); + upload_rate.print(); printf(" Download rate:\n"); - download_rate.print(false); + download_rate.print(); printf(" Fault tolerance level:\n"); - fault_tolerance.print(true); + fault_tolerance.print(); } }; @@ -457,6 +542,25 @@ void CHUNK_ON_HOST::handle() { } } +void CHUNK_ON_HOST::remove() { + if (transfer_in_progress) { + sim.remove(this); + if (!transfer_wait) { + if (present_on_host) { + chunk->parent->dfile->upload_rate.sample_inc( + -host->transfer_rate, + chunk->parent->dfile->collecting_stats() + ); + } else { + chunk->parent->dfile->download_rate.sample_inc( + -host->transfer_rate, + chunk->parent->dfile->collecting_stats() + ); + } + } + } +} + // the host has failed // void HOST::handle() { @@ -470,9 +574,7 @@ void HOST::handle() { for (p = chunks.begin(); p != chunks.end(); p++) { CHUNK_ON_HOST* c = *p; c->chunk->host_failed(c); - if (c->transfer_in_progress) { - sim.remove(c); - } + c->remove(); delete c; } } @@ -843,7 +945,21 @@ void CHUNK::recovery_action() { set dfiles; -int main() { +int main(int argc, char** argv) { + for (int i=1; it = 0; @@ -855,10 +971,10 @@ int main() { sim.insert(new HOST); } #endif - DFILE* dfile = new DFILE(1e2); + DFILE* dfile = new DFILE(1e12); sim.insert(dfile); - sim.simulate(200*86400); + sim.simulate(1000*86400); printf("%s: simulation finished\n", now_str()); dfile->print_stats(); diff --git a/ssim/ssim.php b/ssim/ssim.php new file mode 100644 index 0000000000..ef5217b16d --- /dev/null +++ b/ssim/ssim.php @@ -0,0 +1,93 @@ +name = $filename; + $x->policy = array(); + $f = fopen($filename, "r"); + if (!$f) die("no file $filename\n"); + while (($buf = fgets($f)) !== false) { + $w = explode(" ", $buf); + switch ($w[0]) { + case "policy": + $x->policy[] = trim($w[1]); + break; + case "host_life_mean": + $x->host_life_mean = array(); + for ($i=1; $ihost_life_mean[] = (double)($w[$i]); + } + break; + case "connect_interval": + $x->connect_interval = (double)($w[1]); + break; + case "mean_xfer_rate": + $x->mean_xfer_rate = (double)($w[1]); + break; + } + } + fclose($f); + return $x; +} + +function make_graph($input, $title, $index) { + $gp_filename = $input->name."_$prefix.gp"; + $f = fopen($gp_filename, "w"); + fprintf($f, "set terminal png small size 1024, 768 +set title \"$title\" +set yrange[0:] +plot "); + $n = sizeof($input->policy); + $i = 0; + foreach ($input->policy as $p) { + $fname = $input->name."_$p.dat"; + fprintf($f, "plot $fname using 1:$index title \"$p\" with lines"); + if ($i < $n-1) { + fprintf($f, ", \\"); + } + $i++; + } + fclose($f); + $png_filename = $input->name."_$prefix.png"; + $cmd = "gnuplot < $gp_filename > $png_filename"; + system($cmd); +} + +if ($argc != 2) { + die("usage: ssim.php infile\n"); +} +$input = parse_input_file($argv[1]); +foreach ($input->policy as $p) { + $datafile = fopen($input->name."_$p.dat", "w"); + if (!file_exists($p)) { + die("no policy file '$p'\n"); + } + foreach ($input->host_life_mean as $hlm) { + $cmd = "ssim --policy_file $p --host_life_mean $hlm --connect_interval $input->connect_interval --mean_xfer_rate $input->mean_xfer_rate"; + //system($cmd); + echo "$cmd\n"; + list($du, $ul, $dl, $ft) = parse_output_file("summary.txt"); + printf($datafile, "$hlm $du $ul $dl $ft\n"); + } +} + +exit(); + +make_graph($input, "du", "Disk usage", 2); +make_graph($input, "ub", "Upload bandwidth", 3); +make_graph($input, "db", "Download bandwidth", 4); +make_graph($input, "ft", "Fault tolerance", 5); + +?> diff --git a/version.h b/version.h index 08acc420ea..cc00aeda38 100644 --- a/version.h +++ b/version.h @@ -16,7 +16,7 @@ #define BOINC_VERSION_STRING "7.0.10" /* Package is a pre-release (Alpha/Beta) package */ -/* #define BOINC_PRERELEASE 1 */ +#define BOINC_PRERELEASE 1 #if (defined(_WIN32) || defined(__APPLE__)) /* Name of package */