diff --git a/checkin_notes b/checkin_notes index 47e3d30c28..d851e18784 100644 --- a/checkin_notes +++ b/checkin_notes @@ -1757,14 +1757,14 @@ David 15 Feb 2012 client/ hostinfo_network.cpp -David 15 Feb 2012 +David 16 Feb 2012 - client: fetch work from backup (zero resource share) projects only if a device instance is idle client/ work_fetch.cpp -David 15 Feb 2012 +David 16 Feb 2012 - web: change wording of buffer-size prefs html/inc/ @@ -1774,8 +1774,24 @@ David 15 Feb 2012 vda/ sched_vda.cpp -Charlie 15 Feb 2012 +Charlie 16 Feb 2012 - client: Remove some temporary debugging code. client/ coproc_detect.cpp + +David 16 Feb 2012 + - storage stuff (work in progress) + - small code shuffle + + db/ + boinc_db.h + sched/ + delete_file.cpp + tools/ + backend_lib.cpp,h + lib/ + str_util.h + vda/ + sched_vda.cpp + makefile diff --git a/db/boinc_db.h b/db/boinc_db.h index 4e0f0808f5..3b8e740340 100644 --- a/db/boinc_db.h +++ b/db/boinc_db.h @@ -1150,6 +1150,11 @@ struct VDA_CHUNK_HOST { bool transfer_in_progress; bool transfer_wait; double transition_time; + + // the following not in DB + // + bool found; + void clear(); }; diff --git a/lib/str_util.h b/lib/str_util.h index ba1e2aca2d..cbafceb2de 100644 --- a/lib/str_util.h +++ b/lib/str_util.h @@ -54,6 +54,10 @@ inline bool starts_with(std::string const& s, std::string const& prefix) { return s.substr(0, prefix.size()) == prefix; } +inline bool starts_with(const char* s, const char* prefix) { + return (strncmp(s, prefix, strlen(prefix)) == 0); +} + inline void downcase_string(std::string& w) { for (std::string::iterator p = w.begin(); p != w.end(); ++p) { *p = (char)tolower((int)*p); diff --git a/sched/delete_file.cpp b/sched/delete_file.cpp index 6d5c1755e2..106c897211 100644 --- a/sched/delete_file.cpp +++ b/sched/delete_file.cpp @@ -33,6 +33,7 @@ #include #include +#include "backend_lib.h" #include "boinc_db.h" #include "str_util.h" #include "svn_version.h" @@ -51,23 +52,6 @@ void usage(char* name) { ); } -int delete_host_file(int host_id, const char* file_name) { - DB_MSG_TO_HOST mth; - int retval; - mth.clear(); - mth.create_time = time(0); - mth.hostid = host_id; - mth.handled = false; - sprintf(mth.xml, "%s\n", file_name); - sprintf(mth.variety, "delete_file"); - retval = mth.insert(); - if (retval) { - fprintf(stderr, "msg_to_host.insert(): %s\n", boincerror(retval)); - return retval; - } - return 0; -} - int main(int argc, char** argv) { int i, retval; char file_name[256]; diff --git a/tools/backend_lib.cpp b/tools/backend_lib.cpp index ad5dbfc7b7..ff9bbdaa76 100644 --- a/tools/backend_lib.cpp +++ b/tools/backend_lib.cpp @@ -464,6 +464,23 @@ int put_file( return 0; } +int delete_host_file(int host_id, const char* file_name) { + DB_MSG_TO_HOST mth; + int retval; + mth.clear(); + mth.create_time = time(0); + mth.hostid = host_id; + mth.handled = false; + sprintf(mth.xml, "%s\n", file_name); + sprintf(mth.variety, "delete_file"); + retval = mth.insert(); + if (retval) { + fprintf(stderr, "msg_to_host.insert(): %s\n", boincerror(retval)); + return retval; + } + return 0; +} + int cancel_jobs(int min_id, int max_id) { DB_WORKUNIT wu; DB_RESULT result; diff --git a/tools/backend_lib.h b/tools/backend_lib.h index 1d6f315ee7..3f9868262b 100644 --- a/tools/backend_lib.h +++ b/tools/backend_lib.h @@ -73,6 +73,10 @@ extern int create_work( extern int stage_file(const char*, bool); +// the following 3 functions are for programs other than the schedule +// to do file operations. +// They work by creating MSG_TO_HOST records in the DB +// extern int put_file( int host_id, const char* file_name, vector urls, const char* md5, double nbytes, double report_deadline @@ -86,6 +90,8 @@ extern int get_file( R_RSA_PRIVATE_KEY& key ); +extern int delete_host_file(int host_id, const char* file_name); + // cancel jobs from min_id to max_id inclusive // extern int cancel_jobs(int min_id, int max_id); diff --git a/vda/makefile b/vda/makefile index 6ae2e61637..2bb784cca5 100644 --- a/vda/makefile +++ b/vda/makefile @@ -1,6 +1,6 @@ -all: ssim vda vdad +all: ssim vda vdad sched_vda.o -CCFLAGS = -I /usr/include/mysql -I ../db -I ../lib -I ../db +CCFLAGS = -I /usr/include/mysql -I ../db -I ../lib -I ../db -I ../sched LIBS = ../sched/libsched.a ../lib/libboinc.a @@ -16,3 +16,5 @@ vdad: vdad.cpp vda_lib.o vda_lib2.o g++ -g $(CCFLAGS) -Wall -o vdad vdad.cpp vda_lib.o vda_lib2.o $(LIBS) $(MYSQL_LIBS) vda: vda.cpp vda_lib.o vda_lib2.o g++ -g $(CCFLAGS) -Wall -o vda vda.cpp vda_lib.o vda_lib2.o $(LIBS) $(MYSQL_LIBS) +sched_vda.o: sched_vda.cpp vda_lib.h + g++ -c $(CCFLAGS) sched_vda.cpp diff --git a/vda/sched_vda.cpp b/vda/sched_vda.cpp index cb800896a0..bd8c69d89c 100644 --- a/vda/sched_vda.cpp +++ b/vda/sched_vda.cpp @@ -22,14 +22,30 @@ #include #include "sched_types.h" +#include "sched_msgs.h" using std::map; using std::string; +using std::pair; +static int mark_for_update(int vda_file_id) { + DB_VDA_FILE f; + f.id = vda_file_id; + return f.update_field("need_update=1"); +} + +// relevant fields of SCHEDULER_REQUEST +// file_infos: list of sticky files +// file_xfer_results: list of completed file xfers +// void handle_vda() { + int retval; + unsigned int i; map chunks; // chunks that are supposed to be on this host + // get a list of the vda_chunk_host records for this host + // DB_VDA_CHUNK_HOST ch; char buf[256]; sprintf(buf, "host_id=%d", g_reply->host.id); @@ -44,7 +60,61 @@ void handle_vda() { ); return; } - chunks.insert(ch.name, ch); + chunks.insert(pair(string(ch.name), ch)); } + // process completed uploads: + // - verify md5 of upload + // - move it from upload dir to vda_file dir + // - mark vda_file for update + // - clear transfer_in_progress flag in vda_chunk_host + // + for (i=0; ifile_xfer_results.size(); i++) { + RESULT& r = g_request->file_xfer_results[i]; + if (!starts_with(r.name, "upload_")) continue; + char* file_name = r.name + strlen("upload_"); + if (!strstr(file_name, "vdafile")) continue; + } + + // process list of present files; + // - create a vda_chunk_host record if needed + // - set present_on_host flag in vda_chunk_host + // - mark our in-memory vda_chunk_host record as "found" + // - mark vda_file for update + // + for (i=0; ifile_infos.size(); i++) { + FILE_INFO& fi = g_request->file_infos[i]; + } + + // for each vda_chunk_host not in file list: + // - delete from DB + // - mark vda_file for update + // + map::iterator it; + it = chunks.begin(); + while (it != chunks.end()) { + DB_VDA_CHUNK_HOST& ch = (*it).second; + if (!ch.found) { + ch.delete_from_db(); + chunks.erase(it); + mark_for_update(ch.vda_file_id); + } else { + it++; + } + } + + // if process is using more than its share of disk space, + // remove some chunks and mark vda_files for update + // + if (g_request->host.d_project_share) { + double x = g_request->host.d_boinc_used_project; + it = chunks.begin(); + while (x > g_request->host.d_project_share && it != chunks.end()) { + DB_VDA_CHUNK_HOST& ch = (*it).second; + FILE_INFO fi; + strcpy(fi.name, ch.name); + g_reply->file_deletes.push_back(fi); + it++; + } + } }