From d0413886fa2c88bd1a99a05ab92511ab27827c1d Mon Sep 17 00:00:00 2001 From: David Anderson Date: Fri, 24 Feb 2012 21:13:00 +0000 Subject: [PATCH] - storage stuff svn path=/trunk/boinc/; revision=25340 --- checkin_notes | 9 ++++ vda/vda.cpp | 4 +- vda/vda_lib.h | 9 ++-- vda/vda_lib2.cpp | 6 ++- vda/vdad.cpp | 123 +++++++++++++++++++++++++---------------------- 5 files changed, 84 insertions(+), 67 deletions(-) diff --git a/checkin_notes b/checkin_notes index b792159039..dcd796f200 100644 --- a/checkin_notes +++ b/checkin_notes @@ -2156,3 +2156,12 @@ David 24 Feb 2012 client/ work_fetch.cpp + +David 24 Feb 2012 + - storage stuff + + vda/ + vdad.cpp + vda.cpp + vda_lib2.cpp + vda_lib.h diff --git a/vda/vda.cpp b/vda/vda.cpp index 66d99f9498..341c6653d6 100644 --- a/vda/vda.cpp +++ b/vda/vda.cpp @@ -52,7 +52,7 @@ int handle_add(const char* path) { *p = 0; strcpy(filename, p+1); - // make sure there's a policy file in the dir + // make sure there's a valid policy file in the dir // sprintf(buf, "%s/boinc_meta.txt", dir); retval = policy.parse(buf); @@ -61,7 +61,7 @@ int handle_add(const char* path) { return -1; } - // add a DB record and schedule it for updating + // add a DB record and mark it for update // vf.create_time = dtime(); strcpy(vf.dir, dir); diff --git a/vda/vda_lib.h b/vda/vda_lib.h index 796416b951..d1893323a8 100644 --- a/vda/vda_lib.h +++ b/vda/vda_lib.h @@ -147,12 +147,9 @@ struct META_CHUNK : DATA_UNIT { ); // used by vdad - META_CHUNK(VDA_FILE_AUX* d, META_CHUNK* p) { - dfile = d; - parent = p; - } - int init(const char* dir, const char* fname, POLICY&, int level); - int get_state(const char* dir, const char* fname, POLICY&, int level); + META_CHUNK(VDA_FILE_AUX* d, META_CHUNK* p, int index); + int init(const char* dir, POLICY&, int level); + int get_state(const char* dir, POLICY&, int level); virtual int recovery_plan(); virtual int recovery_action(double); diff --git a/vda/vda_lib2.cpp b/vda/vda_lib2.cpp index b0027ef7bd..67230c2df9 100644 --- a/vda/vda_lib2.cpp +++ b/vda/vda_lib2.cpp @@ -27,7 +27,11 @@ CHUNK::CHUNK(META_CHUNK* mc, double s, int index) { parent = mc; present_on_server = true; size = s; - sprintf(name, "%s.%d", parent->name, index); + if (strlen(parent->name)) { + sprintf(name, "%s.%d", parent->name, index); + } else { + sprintf(name, "%d", index); + } } // assign this chunk to a host diff --git a/vda/vdad.cpp b/vda/vdad.cpp index 245c1e0809..4506050855 100644 --- a/vda/vdad.cpp +++ b/vda/vdad.cpp @@ -44,6 +44,8 @@ using std::vector; // Coding/fname_k01.ext // Coding/fname_m01.ext // +// Assume there's no extension +// void encoder_filename( const char* base, const char* ext, CODING& c, int i, char* buf ) { @@ -60,23 +62,26 @@ void encoder_filename( j = i+1; ch = 'k'; } - sprintf(buf, "%s_%c%0*d%s", base, ch, ndigits, j, ext); + sprintf(buf, "%s_%c%0*d.%s", base, ch, ndigits, j, ext); } +#define DATA_FILENAME "data" +#define DATA_EXT "ext" + // encode a meta-chunk. -// precondition: "dir" contains a file "fname". +// precondition: "dir" contains a file "data". // postcondition: dir contains // a subdir Coding with encoded chunks -// subdirs fname_0 ... fname_m, +// subdirs data.0 .. data.m // each containing a same-named symbolic link to the corresponding chunk // // The size of these chunks is returned in "size" // -int encode(const char* dir, const char* fname, CODING& c, double& size) { +int encode(const char* dir, CODING& c, double& size) { char cmd[1024]; sprintf(cmd, - "cd %s; /mydisks/b/users/boincadm/vda_test/encoder %s %d %d cauchy_good 32 1024 500000", - dir, fname, c.n, c.k + "cd %s; /mydisks/b/users/boincadm/vda_test/encoder %s.%s %d %d cauchy_good 32 1024 500000", + dir, DATA_FILENAME, DATA_EXT, c.n, c.k ); printf("%s\n", cmd); int s = system(cmd); @@ -85,28 +90,21 @@ int encode(const char* dir, const char* fname, CODING& c, double& size) { if (status != 32) return -1; // encoder returns 32 for some reason } - char base[256], ext[256]; - strcpy(base, fname); - char* p = strchr(base, '.'); - if (p) { - strcpy(ext, p); - *p = 0; - } else { - strcpy(ext, ""); - } + // make symlinks + // for (int i=0; iname)) { + sprintf(name, "%s.%d", parent->name, index); + } else { + sprintf(name, "%d", index); + } + } else { + strcpy(name, ""); + } +} + +// initialize a meta-chunk: +// encode it, then recursively initialize its meta-chunk children // -int META_CHUNK::init(const char* dir, const char* fname, POLICY& p, int level) { +int META_CHUNK::init(const char* dir, POLICY& p, int level) { double size; + CODING& c = p.codings[level]; - int retval = encode(dir, fname, c, size); + int retval = encode(dir, c, size); if (retval) return retval; p.chunk_sizes[level] = size; + if (level+1 < p.coding_levels) { for (int i=0; iinit(child_dir, child_fname, p, level+1); + char child_dir[1024]; + sprintf(child_dir, "%s/%s.%d", dir, DATA_FILENAME, i); + META_CHUNK* mc = new META_CHUNK(dfile, parent, i); + retval = mc->init(child_dir, p, level+1); if (retval) return retval; children.push_back(mc); } @@ -152,9 +165,15 @@ int META_CHUNK::init(const char* dir, const char* fname, POLICY& p, int level) { // leaving only the bottom-level chunks // int VDA_FILE_AUX::init() { - char buf[1024]; - meta_chunk = new META_CHUNK(this, NULL); - int retval = meta_chunk->init(dir, name, policy, 0); + char buf[1024], buf2[1024]; + sprintf(buf, "%s/%s.%s", dir, DATA_FILENAME, DATA_EXT); + sprintf(buf2, "%s/%s", dir, name); + int retval = symlink(buf2, buf); + if (retval) { + return ERR_SYMLINK; + } + meta_chunk = new META_CHUNK(this, NULL, 0); + retval = meta_chunk->init(dir, policy, 0); if (retval) return retval; sprintf(buf, "%s/chunk_sizes.txt", dir); FILE* f = fopen(buf, "w"); @@ -165,19 +184,16 @@ int VDA_FILE_AUX::init() { return 0; } -int META_CHUNK::get_state( - const char* dir, const char* fname, POLICY& p, int level -) { +int META_CHUNK::get_state(const char* dir, POLICY& p, int level) { int retval; CODING& c = p.codings[level]; if (level+1 < p.coding_levels) { for (int i=0; iget_state(child_dir, child_fname, p, level+1); + char child_dir[1024]; + sprintf(child_dir, "%s/%s.%d", dir, DATA_FILENAME, i); + META_CHUNK* mc = new META_CHUNK(dfile, this, i); + retval = mc->get_state(child_dir, p, level+1); if (retval) return retval; children.push_back(mc); } @@ -191,25 +207,13 @@ int META_CHUNK::get_state( } int get_chunk_numbers(VDA_CHUNK_HOST& vch, vector& chunk_numbers) { - char* p, *q; - p = vch.name; - - // find the last __ in filename - // + char* p = vch.name; while (1) { - q = strstr(p, "__"); - if (!q) { - if (p == vch.name) return ERR_NOT_FOUND; - } else { - break; - } - p = q; - } - p += 2; - while (p) { + p = strchr(p, '.'); + if (!p) break; + p++; int i = atoi(p); chunk_numbers.push_back(i); - p = strchr(p, '_'); } return 0; } @@ -230,8 +234,8 @@ int VDA_FILE_AUX::get_state() { if (n != 1) return -1; } fclose(f); - meta_chunk = new META_CHUNK(this, NULL); - int retval = meta_chunk->get_state(dir, name, policy, 0); + meta_chunk = new META_CHUNK(this, NULL, 0); + int retval = meta_chunk->get_state(dir, policy, 0); if (retval) return retval; DB_VDA_CHUNK_HOST vch; sprintf(buf, "where vda_file_id=%d", id); @@ -248,7 +252,10 @@ int VDA_FILE_AUX::get_state() { return retval; } if ((int)(chunk_numbers.size()) != policy.coding_levels) { - log_messages.printf(MSG_CRITICAL, "too many get_chunk_numbers\n"); + log_messages.printf(MSG_CRITICAL, + "wrong get_chunk_numbers: got %d, expected %d\n", + (int)(chunk_numbers.size()), policy.coding_levels + ); return -1; } META_CHUNK* mc = meta_chunk; @@ -271,7 +278,7 @@ int handle_file(VDA_FILE_AUX& vf, DB_VDA_FILE& dvf) { int retval; char buf[1024]; - log_messages.printf(MSG_NORMAL, "processing file%d\n", vf.id); + log_messages.printf(MSG_NORMAL, "processing file %s\n", vf.name); // read the policy file //