- storage: add some code

svn path=/trunk/boinc/; revision=25400

checkin_notes

@@ -2526,3 +2526,11 @@ Rom    9 Mar 2012
     samples/vboxwrapper/
         vbox.cpp
         vboxwrapper.cpp
+
+David  10 Mar 2012
+    - storage: add some code
+
+    vda/
+        vda_lib.cpp,h
+        ssim.cpp
+        vdad.cpp

vda/ssim.cpp

@@ -450,6 +450,18 @@ void CHUNK::download_complete() {
     sfp->recover();
 }
 
+int META_CHUNK::encode() {
+    return 0;
+}
+
+int META_CHUNK::decode() {
+    return 0;
+}
+
+int DATA_UNIT::delete_file() {
+    return 0;
+}
+
 const char* status_str(int status) {
     switch (status) {
     case PRESENT: return "present";

vda/state.txt

@@ -2,7 +2,7 @@
     the processing of a dfile by the vdad
 
 DATA_UNIT
-	int status (PRESENT, RECOVERABLE, UNRECOVERABLE)
+    int status (PRESENT, RECOVERABLE, UNRECOVERABLE)
     bool keep_present
         unit is currently PRESENT
         keep enough descendant chunks on server so that
@@ -12,10 +12,10 @@ DATA_UNIT
         start enough uploads to make it PRESENT
 
     META_CHUNK
-		bool need_reconstruct
+        bool need_reconstruct
             decode this unit during this pass.
             This is set only if some child is UNREC
-		bool needed_by_parent
+        bool needed_by_parent
             this unit is needed to reconstruct parent.
             (don't delete it after reconstructing)
 

vda/storage.txt

@@ -151,6 +151,7 @@ dir/
         data_m040.vda
     0/
         data.vda (symlink to ../Coding/data_k001.vda)
+            these are retained even when the file is deleted
 
         if this is a meta-chunk:
         Coding/

vda/vda_lib.cpp

@@ -15,6 +15,8 @@
 // You should have received a copy of the GNU Lesser General Public License
 // along with BOINC.  If not, see <http://www.gnu.org/licenses/>.
 
+// Code that's shared by the simulator and vdad
+
 #include <stdio.h>
 #include <string.h>
 
@@ -51,12 +53,14 @@ META_CHUNK::META_CHUNK(
                 j
             ));
         }
+        bottom_level = false;
     } else {
         for (int j=0; j<coding.m; j++) {
             children.push_back(
                 new CHUNK(this, size/coding.n, j)
             );
         }
+        bottom_level = true;
     }
 }
 
@@ -318,15 +322,16 @@ int CHUNK::recovery_action(double now) {
     return 0;
 }
 
-#if 0
 void META_CHUNK::decide_reconstruct() {
+    unsigned int i;
+
     if (some_child_is_unrecoverable()) {
         if (status == PRESENT) {
             need_reconstruct = true;
         } else if (status == RECOVERABLE) {
             need_present = true;
             for (i=0; i<children.size(); i++) {
-                DATA_UNIT* c = children[i];
+                DATA_UNIT& c = *(children[i]);
                 if (c.in_recovery_set) {
                     if (c.status == PRESENT) {
                         c.keep_present = true;
@@ -340,10 +345,10 @@ void META_CHUNK::decide_reconstruct() {
     if (needed_by_parent) {
         need_reconstruct = true;
     }
-    if (need_reconstruct and !bottom_level()) {
+    if (need_reconstruct and !bottom_level) {
         int n = 0;
         for (i=0; i<children.size(); i++) {
-            DATA_UNIT* c = children[i];
+            META_CHUNK& c = *(META_CHUNK*)children[i];
             if (c.status == PRESENT) {
                 c.needed_by_parent = true;
                 n++;
@@ -356,7 +361,7 @@ void META_CHUNK::decide_reconstruct() {
     if (keep_present) {
         int n = 0;
         for (i=0; i<children.size(); i++) {
-            DATA_UNIT* c = children[i];
+            DATA_UNIT& c = *(children[i]);
             if (c.status == PRESENT) {
                 c.keep_present = true;
                 n++;
@@ -366,12 +371,103 @@ void META_CHUNK::decide_reconstruct() {
             }
         }
     }
-    if (!bottom_level()) {
+    if (!bottom_level) {
         for (i=0; i<children.size(); i++) {
-            META_CHUNK* c = (META_CHUNK*)children[i];
+            META_CHUNK& c = *(META_CHUNK*)children[i];
             c.decide_reconstruct();
         }
     }
 }
 
-#endif
+int META_CHUNK::reconstruct_and_cleanup() {
+    unsigned int i;
+    int retval;
+
+    if (!bottom_level) {
+        for (i=0; i<children.size(); i++) {
+            META_CHUNK& c = *(META_CHUNK*)children[i];
+            retval = c.reconstruct_and_cleanup();
+            if (retval) return retval;
+        }
+    }
+    if (need_reconstruct) {
+        retval = decode();
+        if (retval) return retval;
+        retval = expand();
+        if (retval) return retval;
+        if (!needed_by_parent) {
+            delete_file();
+        }
+    }
+    if (bottom_level) {
+        int npresent = coding.m;
+        for (i=0; i<children.size(); i++) {
+            CHUNK& c = *(CHUNK*)children[i];
+            if (c.status != UNRECOVERABLE && !c.keep_present) {
+                if (!keep_present || npresent > coding.n) {
+                    c.delete_file();
+                    npresent--;
+                    c.new_present_on_server = false;
+                }
+            }
+        }
+    }
+    return 0;
+}
+
+int META_CHUNK::expand() {
+    unsigned int i;
+    int retval;
+
+    if (bottom_level) {
+        bool do_encode = false;
+        for (i=0; i<children.size(); i++) {
+            CHUNK& c = *(CHUNK*)children[i];
+            if (c.status != PRESENT && c.need_more_replicas()) {
+                do_encode = true;
+                break;
+            }
+        }
+        if (do_encode) {
+            retval = encode();
+            if (retval) return retval;
+        }
+        for (i=0; i<children.size(); i++) {
+            CHUNK& c = *(CHUNK*)children[i];
+            if (c.need_more_replicas() || c.download_in_progress()) {
+                c.new_present_on_server = true;
+            } else {
+                c.new_present_on_server = false;
+                c.delete_file();
+            }
+        }
+    } else {
+        bool do_encode = false;
+        for (i=0; i<children.size(); i++) {
+            META_CHUNK& c = *(META_CHUNK*)children[i];
+            if (c.status == UNRECOVERABLE) {
+                do_encode = true;
+                break;
+            }
+        }
+        if (do_encode) {
+            retval = encode();
+            if (retval) return retval;
+            for (i=0; i<children.size(); i++) {
+                META_CHUNK& c = *(META_CHUNK*)children[i];
+                if (c.status != UNRECOVERABLE) {
+                    c.delete_file();
+                }
+            }
+            for (i=0; i<children.size(); i++) {
+                META_CHUNK& c = *(META_CHUNK*)children[i];
+                if (c.status == UNRECOVERABLE) {
+                    retval = c.expand();
+                    if (retval) return retval;
+                    c.delete_file();
+                }
+            }
+        }
+    }
+    return 0;
+}

vda/vda_lib.h

@@ -86,6 +86,11 @@ struct DATA_UNIT {
     int min_failures;
         // min # of host failures that would make this unrecoverable
     char name[64];
+    char dir[1024];
+    bool keep_present;
+    bool need_present;
+
+    int delete_file();
 };
 
 struct META_CHUNK : DATA_UNIT {
@@ -96,6 +101,10 @@ struct META_CHUNK : DATA_UNIT {
     VDA_FILE_AUX* dfile;
     bool uploading;
     CODING coding;
+    bool bottom_level;
+    bool need_reconstruct;
+    bool needed_by_parent;
+    double child_size;
 
     // used by ssim
     META_CHUNK(
@@ -110,6 +119,19 @@ struct META_CHUNK : DATA_UNIT {
 
     virtual int recovery_plan();
     virtual int recovery_action(double);
+
+    void decide_reconstruct();
+    int reconstruct_and_cleanup();
+    int expand();
+    bool some_child_is_unrecoverable() {
+        for (unsigned int i=0; i<children.size(); i++) {
+            DATA_UNIT& d = *(children[i]);
+            if (d.status == UNRECOVERABLE) return true;
+        }
+        return false;
+    }
+    int decode();
+    int encode();
 };
 
 struct CHUNK : DATA_UNIT {
@@ -117,6 +139,7 @@ struct CHUNK : DATA_UNIT {
     META_CHUNK* parent;
     double size;
     bool present_on_server;
+    bool new_present_on_server;
 
     CHUNK(META_CHUNK* mc, double s, int index);
 
@@ -128,4 +151,7 @@ struct CHUNK : DATA_UNIT {
     int assign();
     virtual int recovery_plan();
     virtual int recovery_action(double);
+    bool need_more_replicas() {
+        return ((int)hosts.size() < parent->dfile->policy.replication);
+    }
 };

vda/vdad.cpp

@@ -80,25 +80,25 @@ void encoder_filename(
 //
 // The size of these chunks is returned in "size"
 //
-int encode(const char* dir, CODING& c, double& size) {
+int META_CHUNK::encode() {
     char cmd[1024];
     sprintf(cmd,
         "cd %s; /mydisks/b/users/boincadm/vda_test/encoder %s %d %d cauchy_good 32 1024 500000",
-        dir, DATA_FILENAME, c.n, c.k
+        dir, DATA_FILENAME, coding.n, coding.k
     );
     printf("%s\n", cmd);
     int s = system(cmd);
     if (WIFEXITED(s)) {
-        int status = WEXITSTATUS(s);
+        int st = WEXITSTATUS(s);
-        if (status != 32) return -1;    // encoder returns 32 for some reason
+        if (st != 32) return -1;    // encoder returns 32 for some reason
     }
 
     // make symlinks
     //
-    for (int i=0; i<c.m; i++) {
+    for (int i=0; i<coding.m; i++) {
         char enc_filename[1024], target_path[1024];
         char dir_name[1024], link_name[1024];
-        encoder_filename("data", "vda", c, i, enc_filename);
+        encoder_filename("data", "vda", coding, i, enc_filename);
         sprintf(target_path, "%s/Coding/%s", dir, enc_filename);
         sprintf(dir_name, "%s/%d", dir, i);
         int retval = mkdir(dir_name, 0777);
@@ -115,12 +115,35 @@ int encode(const char* dir, CODING& c, double& size) {
             return retval;
         }
         if (i == 0) {
-            file_size(target_path, size);
+            file_size(target_path, child_size);
         }
     }
     return 0;
 }
 
+int META_CHUNK::decode() {
+    char cmd[1024];
+    sprintf(cmd,
+        "cd %s; /mydisks/b/users/boincadm/vda_test/decoder %s",
+        dir, DATA_FILENAME
+    );
+    printf("%s\n", cmd);
+    int s = system(cmd);
+    if (WIFEXITED(s)) {
+        int st = WEXITSTATUS(s);
+        if (st != 32) return -1;    // encoder returns 32 for some reason
+    }
+    return 0;
+}
+
+int DATA_UNIT::delete_file() {
+    char path[1024], buf[1024];
+    sprintf(path, "%s/data.vda", dir);
+    size_t n = readlink(path, buf, 1024);
+    buf[n] = 0;
+    return unlink(buf);
+}
+
 CHUNK::CHUNK(META_CHUNK* mc, double s, int index) {
     parent = mc;
     present_on_server = true;
@@ -130,6 +153,7 @@ CHUNK::CHUNK(META_CHUNK* mc, double s, int index) {
     } else {
         sprintf(name, "%d", index);
     }
+    sprintf(dir, "%s/%d", mc->dir, index);
 }
 
 // assign this chunk to a host
@@ -302,17 +326,18 @@ META_CHUNK::META_CHUNK(VDA_FILE_AUX* d, META_CHUNK* p, int index) {
 // initialize a meta-chunk:
 // encode it, then recursively initialize its meta-chunk children
 //
-int META_CHUNK::init(const char* dir, POLICY& p, int level) {
+int META_CHUNK::init(const char* _dir, POLICY& p, int level) {
     double size;
     char child_dir[1024];
 
-    CODING& c = p.codings[level];
+    strcpy(dir, _dir);
-    int retval = encode(dir, c, size);
+    coding = p.codings[level];
+    int retval = encode();
     if (retval) return retval;
-    p.chunk_sizes[level] = size;
+    p.chunk_sizes[level] = child_size;
 
     if (level+1 < p.coding_levels) {
-        for (int i=0; i<c.m; i++) {
+        for (int i=0; i<coding.m; i++) {
             sprintf(child_dir, "%s/%d", dir, i);
             META_CHUNK* mc = new META_CHUNK(dfile, parent, i);
             retval = mc->init(child_dir, p, level+1);
@@ -320,7 +345,7 @@ int META_CHUNK::init(const char* dir, POLICY& p, int level) {
             children.push_back(mc);
         }
     } else {
-        for (int i=0; i<c.m; i++) {
+        for (int i=0; i<coding.m; i++) {
             CHUNK* cp = new CHUNK(this, p.chunk_sizes[level], i);
             children.push_back(cp);
 
@@ -375,12 +400,13 @@ int VDA_FILE_AUX::init() {
     return 0;
 }
 
-int META_CHUNK::get_state(const char* dir, POLICY& p, int level) {
+int META_CHUNK::get_state(const char* _dir, POLICY& p, int level) {
     int retval;
 
-    CODING& c = p.codings[level];
+    strcpy(dir, _dir);
+    coding = p.codings[level];
     if (level+1 < p.coding_levels) {
-        for (int i=0; i<c.m; i++) {
+        for (int i=0; i<coding.m; i++) {
             char child_dir[1024];
             sprintf(child_dir, "%s/%s.%d", dir, DATA_FILENAME, i);
             META_CHUNK* mc = new META_CHUNK(dfile, this, i);
@@ -389,7 +415,7 @@ int META_CHUNK::get_state(const char* dir, POLICY& p, int level) {
             children.push_back(mc);
         }
     } else {
-        for (int i=0; i<c.m; i++) {
+        for (int i=0; i<coding.m; i++) {
             CHUNK* ch = new CHUNK(this, p.chunk_sizes[level], i);
             children.push_back(ch);
         }