Rooba
/
boinc
mirror of https://github.com/BOINC/boinc.git
1
0
Fork 0
Code Issues 1 Packages Projects Releases Wiki Activity

- storage: add some code 

svn path=/trunk/boinc/; revision=25400
This commit is contained in:
David Anderson 2012-03-11 01:51:07 +00:00
parent 8f8caec6e7
commit 67d6ab7cf2
7 changed files with 197 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
checkin_notes
View File

@ -2526,3 +2526,11 @@ Rom 9 Mar 2012
samples/vboxwrapper/
vbox.cpp
vboxwrapper.cpp
David 10 Mar 2012
- storage: add some code
vda/
vda_lib.cpp,h
ssim.cpp
vdad.cpp

12
vda/ssim.cpp
View File

@ -450,6 +450,18 @@ void CHUNK::download_complete() {
sfp->recover();
}
int META_CHUNK::encode() {
return 0;
}
int META_CHUNK::decode() {
return 0;
}
int DATA_UNIT::delete_file() {
return 0;
}
const char* status_str(int status) {
switch (status) {
case PRESENT: return "present";

6
vda/state.txt
View File

@ -2,7 +2,7 @@
the processing of a dfile by the vdad
DATA_UNIT
int status (PRESENT, RECOVERABLE, UNRECOVERABLE)
int status (PRESENT, RECOVERABLE, UNRECOVERABLE)
bool keep_present
unit is currently PRESENT
keep enough descendant chunks on server so that
@ -12,10 +12,10 @@ DATA_UNIT
start enough uploads to make it PRESENT
META_CHUNK
bool need_reconstruct
bool need_reconstruct
decode this unit during this pass.
This is set only if some child is UNREC
bool needed_by_parent
bool needed_by_parent
this unit is needed to reconstruct parent.
(don't delete it after reconstructing)

1
vda/storage.txt
View File

@ -151,6 +151,7 @@ dir/
data_m040.vda
0/
data.vda (symlink to ../Coding/data_k001.vda)
these are retained even when the file is deleted
if this is a meta-chunk:
Coding/

112
vda/vda_lib.cpp
View File

@ -15,6 +15,8 @@
// You should have received a copy of the GNU Lesser General Public License
// along with BOINC. If not, see <http://www.gnu.org/licenses/>.
// Code that's shared by the simulator and vdad
#include <stdio.h>
#include <string.h>
@ -51,12 +53,14 @@ META_CHUNK::META_CHUNK(
j
));
}
bottom_level = false;
} else {
for (int j=0; j<coding.m; j++) {
children.push_back(
new CHUNK(this, size/coding.n, j)
);
}
bottom_level = true;
}
}
@ -318,15 +322,16 @@ int CHUNK::recovery_action(double now) {
return 0;
}
#if 0
void META_CHUNK::decide_reconstruct() {
unsigned int i;
if (some_child_is_unrecoverable()) {
if (status == PRESENT) {
need_reconstruct = true;
} else if (status == RECOVERABLE) {
need_present = true;
for (i=0; i<children.size(); i++) {
DATA_UNIT* c = children[i];
DATA_UNIT& c = *(children[i]);
if (c.in_recovery_set) {
if (c.status == PRESENT) {
c.keep_present = true;
@ -340,10 +345,10 @@ void META_CHUNK::decide_reconstruct() {
if (needed_by_parent) {
need_reconstruct = true;
}
if (need_reconstruct and !bottom_level()) {
if (need_reconstruct and !bottom_level) {
int n = 0;
for (i=0; i<children.size(); i++) {
DATA_UNIT* c = children[i];
META_CHUNK& c = *(META_CHUNK*)children[i];
if (c.status == PRESENT) {
c.needed_by_parent = true;
n++;
@ -356,7 +361,7 @@ void META_CHUNK::decide_reconstruct() {
if (keep_present) {
int n = 0;
for (i=0; i<children.size(); i++) {
DATA_UNIT* c = children[i];
DATA_UNIT& c = *(children[i]);
if (c.status == PRESENT) {
c.keep_present = true;
n++;
@ -366,12 +371,103 @@ void META_CHUNK::decide_reconstruct() {
}
}
}
if (!bottom_level()) {
if (!bottom_level) {
for (i=0; i<children.size(); i++) {
META_CHUNK* c = (META_CHUNK*)children[i];
META_CHUNK& c = *(META_CHUNK*)children[i];
c.decide_reconstruct();
}
}
}
#endif
int META_CHUNK::reconstruct_and_cleanup() {
unsigned int i;
int retval;
if (!bottom_level) {
for (i=0; i<children.size(); i++) {
META_CHUNK& c = *(META_CHUNK*)children[i];
retval = c.reconstruct_and_cleanup();
if (retval) return retval;
}
}
if (need_reconstruct) {
retval = decode();
if (retval) return retval;
retval = expand();
if (retval) return retval;
if (!needed_by_parent) {
delete_file();
}
}
if (bottom_level) {
int npresent = coding.m;
for (i=0; i<children.size(); i++) {
CHUNK& c = *(CHUNK*)children[i];
if (c.status != UNRECOVERABLE && !c.keep_present) {
if (!keep_present || npresent > coding.n) {
c.delete_file();
npresent--;
c.new_present_on_server = false;
}
}
}
}
return 0;
}
int META_CHUNK::expand() {
unsigned int i;
int retval;
if (bottom_level) {
bool do_encode = false;
for (i=0; i<children.size(); i++) {
CHUNK& c = *(CHUNK*)children[i];
if (c.status != PRESENT && c.need_more_replicas()) {
do_encode = true;
break;
}
}
if (do_encode) {
retval = encode();
if (retval) return retval;
}
for (i=0; i<children.size(); i++) {
CHUNK& c = *(CHUNK*)children[i];
if (c.need_more_replicas() || c.download_in_progress()) {
c.new_present_on_server = true;
} else {
c.new_present_on_server = false;
c.delete_file();
}
}
} else {
bool do_encode = false;
for (i=0; i<children.size(); i++) {
META_CHUNK& c = *(META_CHUNK*)children[i];
if (c.status == UNRECOVERABLE) {
do_encode = true;
break;
}
}
if (do_encode) {
retval = encode();
if (retval) return retval;
for (i=0; i<children.size(); i++) {
META_CHUNK& c = *(META_CHUNK*)children[i];
if (c.status != UNRECOVERABLE) {
c.delete_file();
}
}
for (i=0; i<children.size(); i++) {
META_CHUNK& c = *(META_CHUNK*)children[i];
if (c.status == UNRECOVERABLE) {
retval = c.expand();
if (retval) return retval;
c.delete_file();
}
}
}
}
return 0;
}

26
vda/vda_lib.h
View File

@ -86,6 +86,11 @@ struct DATA_UNIT {
int min_failures;
// min # of host failures that would make this unrecoverable
char name[64];
char dir[1024];
bool keep_present;
bool need_present;
int delete_file();
};
struct META_CHUNK : DATA_UNIT {
@ -96,6 +101,10 @@ struct META_CHUNK : DATA_UNIT {
VDA_FILE_AUX* dfile;
bool uploading;
CODING coding;
bool bottom_level;
bool need_reconstruct;
bool needed_by_parent;
double child_size;
// used by ssim
META_CHUNK(
@ -110,6 +119,19 @@ struct META_CHUNK : DATA_UNIT {
virtual int recovery_plan();
virtual int recovery_action(double);
void decide_reconstruct();
int reconstruct_and_cleanup();
int expand();
bool some_child_is_unrecoverable() {
for (unsigned int i=0; i<children.size(); i++) {
DATA_UNIT& d = *(children[i]);
if (d.status == UNRECOVERABLE) return true;
}
return false;
}
int decode();
int encode();
};
struct CHUNK : DATA_UNIT {
@ -117,6 +139,7 @@ struct CHUNK : DATA_UNIT {
META_CHUNK* parent;
double size;
bool present_on_server;
bool new_present_on_server;
CHUNK(META_CHUNK* mc, double s, int index);
@ -128,4 +151,7 @@ struct CHUNK : DATA_UNIT {
int assign();
virtual int recovery_plan();
virtual int recovery_action(double);
bool need_more_replicas() {
return ((int)hosts.size() < parent->dfile->policy.replication);
}
};

60
vda/vdad.cpp
View File

@ -80,25 +80,25 @@ void encoder_filename(
//
// The size of these chunks is returned in "size"
//
int encode(const char* dir, CODING& c, double& size) {
int META_CHUNK::encode() {
char cmd[1024];
sprintf(cmd,
"cd %s; /mydisks/b/users/boincadm/vda_test/encoder %s %d %d cauchy_good 32 1024 500000",
dir, DATA_FILENAME, c.n, c.k
dir, DATA_FILENAME, coding.n, coding.k
);
printf("%s\n", cmd);
int s = system(cmd);
if (WIFEXITED(s)) {
int status = WEXITSTATUS(s);
if (status != 32) return -1; // encoder returns 32 for some reason
int st = WEXITSTATUS(s);
if (st != 32) return -1; // encoder returns 32 for some reason
}
// make symlinks
//
for (int i=0; i<c.m; i++) {
for (int i=0; i<coding.m; i++) {
char enc_filename[1024], target_path[1024];
char dir_name[1024], link_name[1024];
encoder_filename("data", "vda", c, i, enc_filename);
encoder_filename("data", "vda", coding, i, enc_filename);
sprintf(target_path, "%s/Coding/%s", dir, enc_filename);
sprintf(dir_name, "%s/%d", dir, i);
int retval = mkdir(dir_name, 0777);
@ -115,12 +115,35 @@ int encode(const char* dir, CODING& c, double& size) {
return retval;
}
if (i == 0) {
file_size(target_path, size);
file_size(target_path, child_size);
}
}
return 0;
}
int META_CHUNK::decode() {
char cmd[1024];
sprintf(cmd,
"cd %s; /mydisks/b/users/boincadm/vda_test/decoder %s",
dir, DATA_FILENAME
);
printf("%s\n", cmd);
int s = system(cmd);
if (WIFEXITED(s)) {
int st = WEXITSTATUS(s);
if (st != 32) return -1; // encoder returns 32 for some reason
}
return 0;
}
int DATA_UNIT::delete_file() {
char path[1024], buf[1024];
sprintf(path, "%s/data.vda", dir);
size_t n = readlink(path, buf, 1024);
buf[n] = 0;
return unlink(buf);
}
CHUNK::CHUNK(META_CHUNK* mc, double s, int index) {
parent = mc;
present_on_server = true;
@ -130,6 +153,7 @@ CHUNK::CHUNK(META_CHUNK* mc, double s, int index) {
} else {
sprintf(name, "%d", index);
}
sprintf(dir, "%s/%d", mc->dir, index);
}
// assign this chunk to a host
@ -302,17 +326,18 @@ META_CHUNK::META_CHUNK(VDA_FILE_AUX* d, META_CHUNK* p, int index) {
// initialize a meta-chunk:
// encode it, then recursively initialize its meta-chunk children
//
int META_CHUNK::init(const char* dir, POLICY& p, int level) {
int META_CHUNK::init(const char* _dir, POLICY& p, int level) {
double size;
char child_dir[1024];
CODING& c = p.codings[level];
int retval = encode(dir, c, size);
strcpy(dir, _dir);
coding = p.codings[level];
int retval = encode();
if (retval) return retval;
p.chunk_sizes[level] = size;
p.chunk_sizes[level] = child_size;
if (level+1 < p.coding_levels) {
for (int i=0; i<c.m; i++) {
for (int i=0; i<coding.m; i++) {
sprintf(child_dir, "%s/%d", dir, i);
META_CHUNK* mc = new META_CHUNK(dfile, parent, i);
retval = mc->init(child_dir, p, level+1);
@ -320,7 +345,7 @@ int META_CHUNK::init(const char* dir, POLICY& p, int level) {
children.push_back(mc);
}
} else {
for (int i=0; i<c.m; i++) {
for (int i=0; i<coding.m; i++) {
CHUNK* cp = new CHUNK(this, p.chunk_sizes[level], i);
children.push_back(cp);
@ -375,12 +400,13 @@ int VDA_FILE_AUX::init() {
return 0;
}
int META_CHUNK::get_state(const char* dir, POLICY& p, int level) {
int META_CHUNK::get_state(const char* _dir, POLICY& p, int level) {
int retval;
CODING& c = p.codings[level];
strcpy(dir, _dir);
coding = p.codings[level];
if (level+1 < p.coding_levels) {
for (int i=0; i<c.m; i++) {
for (int i=0; i<coding.m; i++) {
char child_dir[1024];
sprintf(child_dir, "%s/%s.%d", dir, DATA_FILENAME, i);
META_CHUNK* mc = new META_CHUNK(dfile, this, i);
@ -389,7 +415,7 @@ int META_CHUNK::get_state(const char* dir, POLICY& p, int level) {
children.push_back(mc);
}
} else {
for (int i=0; i<c.m; i++) {
for (int i=0; i<coding.m; i++) {
CHUNK* ch = new CHUNK(this, p.chunk_sizes[level], i);
children.push_back(ch);
}