- storage stuff (work in progress)

- small code shuffle


svn path=/trunk/boinc/; revision=25274

checkin_notes

@@ -1757,14 +1757,14 @@ David  15 Feb 2012
     client/
         hostinfo_network.cpp
 
-David  15 Feb 2012
+David  16 Feb 2012
     - client: fetch work from backup (zero resource share) projects
         only if a device instance is idle
 
     client/
         work_fetch.cpp
 
-David  15 Feb 2012
+David  16 Feb 2012
     - web: change wording of buffer-size prefs
 
     html/inc/
@@ -1774,8 +1774,24 @@ David  15 Feb 2012
     vda/
         sched_vda.cpp
 
-Charlie 15 Feb 2012
+Charlie 16 Feb 2012
     - client: Remove some temporary debugging code.
 
     client/
         coproc_detect.cpp
+
+David  16 Feb 2012
+    - storage stuff (work in progress)
+    - small code shuffle
+
+    db/
+        boinc_db.h
+    sched/
+        delete_file.cpp
+    tools/
+        backend_lib.cpp,h
+    lib/
+        str_util.h
+    vda/
+        sched_vda.cpp
+        makefile

db/boinc_db.h

@@ -1150,6 +1150,11 @@ struct VDA_CHUNK_HOST {
     bool transfer_in_progress;
     bool transfer_wait;
     double transition_time;
+
+    // the following not in DB
+    //
+    bool found;
+
     void clear();
 };
 

lib/str_util.h

@@ -54,6 +54,10 @@ inline bool starts_with(std::string const& s, std::string const& prefix) {
     return s.substr(0, prefix.size()) == prefix;
 }
 
+inline bool starts_with(const char* s, const char* prefix) {
+    return (strncmp(s, prefix, strlen(prefix)) == 0);
+}
+
 inline void downcase_string(std::string& w) {
     for (std::string::iterator p = w.begin(); p != w.end(); ++p) {
         *p = (char)tolower((int)*p);

sched/delete_file.cpp

@@ -33,6 +33,7 @@
 #include <string>
 #include <iostream>
 
+#include "backend_lib.h"
 #include "boinc_db.h"
 #include "str_util.h"
 #include "svn_version.h"
@@ -51,23 +52,6 @@ void usage(char* name) {
     );
 }
 
-int delete_host_file(int host_id, const char* file_name) {
-    DB_MSG_TO_HOST mth;
-    int retval;
-    mth.clear();
-    mth.create_time = time(0);
-    mth.hostid = host_id;
-    mth.handled = false;
-    sprintf(mth.xml, "<delete_file_info>%s</delete_file_info>\n", file_name);
-    sprintf(mth.variety, "delete_file");
-    retval = mth.insert();
-    if (retval) {
-        fprintf(stderr, "msg_to_host.insert(): %s\n", boincerror(retval));
-        return retval;
-    }
-    return 0;
-}
-
 int main(int argc, char** argv) {
     int i, retval;
     char file_name[256];

tools/backend_lib.cpp

@@ -464,6 +464,23 @@ int put_file(
     return 0;
 }
 
+int delete_host_file(int host_id, const char* file_name) {
+    DB_MSG_TO_HOST mth;
+    int retval;
+    mth.clear();
+    mth.create_time = time(0);
+    mth.hostid = host_id;
+    mth.handled = false;
+    sprintf(mth.xml, "<delete_file_info>%s</delete_file_info>\n", file_name);
+    sprintf(mth.variety, "delete_file");
+    retval = mth.insert();
+    if (retval) {
+        fprintf(stderr, "msg_to_host.insert(): %s\n", boincerror(retval));
+        return retval;
+    }
+    return 0;
+}
+
 int cancel_jobs(int min_id, int max_id) {
     DB_WORKUNIT wu;
     DB_RESULT result;

tools/backend_lib.h

@@ -73,6 +73,10 @@ extern int create_work(
 
 extern int stage_file(const char*, bool);
 
+// the following 3 functions are for programs other than the schedule
+// to do file operations.
+// They work by creating MSG_TO_HOST records in the DB
+//
 extern int put_file(
     int host_id, const char* file_name, vector<const char*> urls,
     const char* md5, double nbytes, double report_deadline
@@ -86,6 +90,8 @@ extern int get_file(
     R_RSA_PRIVATE_KEY& key
 );
 
+extern int delete_host_file(int host_id, const char* file_name);
+
 // cancel jobs from min_id to max_id inclusive
 //
 extern int cancel_jobs(int min_id, int max_id);

vda/makefile

@@ -1,6 +1,6 @@
-all: ssim vda vdad
+all: ssim vda vdad sched_vda.o
 
-CCFLAGS = -I /usr/include/mysql -I ../db -I ../lib -I ../db
+CCFLAGS = -I /usr/include/mysql -I ../db -I ../lib -I ../db -I ../sched
 
 LIBS = ../sched/libsched.a ../lib/libboinc.a
 
@@ -16,3 +16,5 @@ vdad: vdad.cpp vda_lib.o vda_lib2.o
 	g++ -g $(CCFLAGS) -Wall -o vdad vdad.cpp vda_lib.o vda_lib2.o $(LIBS) $(MYSQL_LIBS)
 vda: vda.cpp vda_lib.o vda_lib2.o
 	g++ -g $(CCFLAGS) -Wall -o vda vda.cpp vda_lib.o vda_lib2.o $(LIBS) $(MYSQL_LIBS)
+sched_vda.o: sched_vda.cpp vda_lib.h
+	g++ -c $(CCFLAGS) sched_vda.cpp

vda/sched_vda.cpp

@@ -22,14 +22,30 @@
 #include <string>
 
 #include "sched_types.h"
+#include "sched_msgs.h"
 
 using std::map;
 using std::string;
+using std::pair;
 
+static int mark_for_update(int vda_file_id) {
+    DB_VDA_FILE f;
+    f.id = vda_file_id;
+    return f.update_field("need_update=1");
+}
+
+// relevant fields of SCHEDULER_REQUEST
+// file_infos: list of sticky files
+// file_xfer_results: list of completed file xfers
+//
 void handle_vda() {
+    int retval;
+    unsigned int i;
     map<string, DB_VDA_CHUNK_HOST> chunks;
         // chunks that are supposed to be on this host
 
+    // get a list of the vda_chunk_host records for this host
+    //
     DB_VDA_CHUNK_HOST ch;
     char buf[256];
     sprintf(buf, "host_id=%d", g_reply->host.id);
@@ -44,7 +60,61 @@ void handle_vda() {
             );
             return;
         }
-        chunks.insert(ch.name, ch);
+        chunks.insert(pair<string, DB_VDA_CHUNK_HOST>(string(ch.name), ch));
     }
 
+    // process completed uploads:
+    // - verify md5 of upload
+    // - move it from upload dir to vda_file dir
+    // - mark vda_file for update
+    // - clear transfer_in_progress flag in vda_chunk_host
+    //
+    for (i=0; i<g_request->file_xfer_results.size(); i++) {
+        RESULT& r = g_request->file_xfer_results[i];
+        if (!starts_with(r.name, "upload_")) continue;
+        char* file_name = r.name + strlen("upload_");
+        if (!strstr(file_name, "vdafile")) continue;
+    }
+
+    // process list of present files;
+    // - create a vda_chunk_host record if needed
+    // - set present_on_host flag in vda_chunk_host
+    // - mark our in-memory vda_chunk_host record as "found"
+    // - mark vda_file for update
+    //
+    for (i=0; i<g_request->file_infos.size(); i++) {
+        FILE_INFO& fi = g_request->file_infos[i];
+    }
+
+    // for each vda_chunk_host not in file list:
+    // - delete from DB
+    // - mark vda_file for update
+    //
+    map<string, DB_VDA_CHUNK_HOST>::iterator it;
+    it = chunks.begin();
+    while (it != chunks.end()) {
+        DB_VDA_CHUNK_HOST& ch = (*it).second;
+        if (!ch.found) {
+            ch.delete_from_db();
+            chunks.erase(it);
+            mark_for_update(ch.vda_file_id);
+        } else {
+            it++;
+        }
+    }
+
+    // if process is using more than its share of disk space,
+    // remove some chunks and mark vda_files for update
+    //
+    if (g_request->host.d_project_share) {
+        double x = g_request->host.d_boinc_used_project;
+        it = chunks.begin();
+        while (x > g_request->host.d_project_share && it != chunks.end()) {
+            DB_VDA_CHUNK_HOST& ch = (*it).second;
+            FILE_INFO fi;
+            strcpy(fi.name, ch.name);
+            g_reply->file_deletes.push_back(fi);
+            it++;
+        }
+    }
 }