From 127e905e0d5307cbd4cbd3328a6d7fa1c6e55281 Mon Sep 17 00:00:00 2001 From: David Anderson Date: Wed, 29 Feb 2012 07:22:59 +0000 Subject: [PATCH] - storage stuff. Getting there. svn path=/trunk/boinc/; revision=25355 --- checkin_notes | 13 +++++++++ db/boinc_db.cpp | 2 +- sched/handle_request.cpp | 2 +- vda/sched_vda.cpp | 16 +++++------ vda/vda.cpp | 34 +++++++++++++++++++++++ vda/vda_policy.h | 3 ++ vda/vdad.cpp | 60 ++++++++++++++++++++++++++++------------ 7 files changed, 103 insertions(+), 27 deletions(-) diff --git a/checkin_notes b/checkin_notes index b25d422f34..436068bd3b 100644 --- a/checkin_notes +++ b/checkin_notes @@ -2336,3 +2336,16 @@ David 28 Feb 2012 sched_types.cpp,h vda/ sched_vda.cpp + +David 28 Feb 2012 + - storage stuff. Getting there. + + db/ + boinc_db.cpp + sched/ + handle_request.cpp + vda/ + vdad.cpp + vda_policy.h + sched_vda.cpp + vda.cpp diff --git a/db/boinc_db.cpp b/db/boinc_db.cpp index 1e3b93e78c..ce62c05335 100644 --- a/db/boinc_db.cpp +++ b/db/boinc_db.cpp @@ -2367,7 +2367,7 @@ void DB_VDA_CHUNK_HOST::db_print(char* buf) { "vda_file_id=%d, " "host_id=%d, " "name='%s', " - "size=%f " + "size=%f, " "present_on_host=%d, " "transfer_in_progress=%d, " "transfer_wait=%d, " diff --git a/sched/handle_request.cpp b/sched/handle_request.cpp index 1daecb2fe3..d4e13f377a 100644 --- a/sched/handle_request.cpp +++ b/sched/handle_request.cpp @@ -1141,7 +1141,7 @@ void process_request(char* code_sign_key) { ) { g_reply->insert_message("No work available", "low"); g_reply->set_delay(DELAY_NO_WORK_SKIP); - if (!config.msg_to_host) { + if (!config.msg_to_host && !config.enable_vda) { log_messages.printf(MSG_NORMAL, "No work - skipping DB access\n"); return; } diff --git a/vda/sched_vda.cpp b/vda/sched_vda.cpp index 9e387f6a40..6d66827f3e 100644 --- a/vda/sched_vda.cpp +++ b/vda/sched_vda.cpp @@ -158,9 +158,9 @@ static int process_present_file(FILE_INFO& fi, CHUNK_LIST& chunks) { // static int process_missing_chunks(CHUNK_LIST& chunks) { CHUNK_LIST::iterator it; - it = chunks.begin(); - while (it != chunks.end()) { + for (it = chunks.begin(); it != chunks.end(); it++) { DB_VDA_CHUNK_HOST& ch = it->second; + if (!ch.present_on_host && ch.transfer_in_progress) continue; if (!ch.found) { if (config.debug_vda) { log_messages.printf(MSG_NORMAL, @@ -168,10 +168,8 @@ static int process_missing_chunks(CHUNK_LIST& chunks) { ); } ch.delete_from_db(); - chunks.erase(it); + ch.transfer_in_progress = false; mark_for_update(ch.vda_file_id); - } else { - it++; } } return 0; @@ -193,6 +191,7 @@ static int enforce_quota(CHUNK_LIST& chunks) { CHUNK_LIST::iterator it = chunks.begin(); while (x > g_request->host.d_project_share && it != chunks.end()) { DB_VDA_CHUNK_HOST& ch = it->second; + if (!ch.found) continue; FILE_INFO fi; strcpy(fi.name, ch.name); if (config.debug_vda) { @@ -210,7 +209,7 @@ static int enforce_quota(CHUNK_LIST& chunks) { // issue upload and download commands // static int issue_transfer_commands(CHUNK_LIST& chunks) { - char xml_buf[1024], file_name[1024]; + char xml_buf[8192], file_name[1024]; int retval; char url[1024]; vector urls; @@ -256,7 +255,8 @@ static int issue_transfer_commands(CHUNK_LIST& chunks) { get_chunk_url(vf, ch.name, url); urls.push_back(url); get_chunk_dir(vf, ch.name, chunk_dir); - get_chunk_md5(chunk_dir, md5); + retval = get_chunk_md5(chunk_dir, md5); + if (retval) return retval; retval = put_file_xml( file_name, urls, @@ -294,7 +294,7 @@ void handle_vda() { // DB_VDA_CHUNK_HOST ch; char buf[256]; - sprintf(buf, "host_id=%d", g_reply->host.id); + sprintf(buf, "where host_id=%d", g_reply->host.id); while (1) { retval = ch.enumerate(buf); if (retval == ERR_DB_NOT_FOUND) break; diff --git a/vda/vda.cpp b/vda/vda.cpp index 341c6653d6..8a00aa862f 100644 --- a/vda/vda.cpp +++ b/vda/vda.cpp @@ -25,6 +25,7 @@ #include "boinc_db.h" #include "filesys.h" #include "sched_config.h" +#include "sched_util.h" #include "util.h" #include "vda_lib.h" @@ -77,6 +78,29 @@ int handle_add(const char* path) { return 0; } +int handle_remove(const char* name) { + DB_VDA_FILE vf; + char buf[1024]; + sprintf(buf, "where name='%s'", name); + int retval = vf.lookup(buf); + if (retval) return retval; + + // delete DB records + // + DB_VDA_CHUNK_HOST ch; + sprintf(buf, "vda_file_id=%d", vf.id); + ch.delete_from_db_multi(buf); + vf.delete_from_db(); + + dir_hier_path(name, config.download_dir, config.uldl_dir_fanout, buf); + unlink(buf); + retval = chdir(vf.dir); + if (retval) perror("chdir"); + retval = system("/bin/rm -r [0-9]* Coding data.vda"); + if (retval) perror("system"); + return 0; +} + int main(int argc, char** argv) { int retval = config.parse_file(); if (retval) { @@ -101,6 +125,16 @@ int main(int argc, char** argv) { } exit(retval); } + if (!strcmp(argv[i], "remove")) { + if (argc != 3) usage(); + retval = handle_remove(argv[++i]); + if (retval) { + fprintf(stderr, "error %d\n", retval); + } else { + printf("file removed successfully\n"); + } + exit(retval); + } usage(); } usage(); diff --git a/vda/vda_policy.h b/vda/vda_policy.h index 6f028e93bb..506899dbf1 100644 --- a/vda/vda_policy.h +++ b/vda/vda_policy.h @@ -36,6 +36,9 @@ struct POLICY { int coding_levels; CODING codings[10]; double chunk_sizes[10]; + double chunk_size() { + return chunk_sizes[coding_levels-1]; + } char description[256]; // derived from the above diff --git a/vda/vdad.cpp b/vda/vdad.cpp index a8621e06d1..241b6c8b87 100644 --- a/vda/vdad.cpp +++ b/vda/vdad.cpp @@ -33,6 +33,8 @@ using std::set; #include "boinc_db.h" #include "sched_config.h" +#include "sched_util.h" +#include "md5_file.h" #include "error_numbers.h" #include "util.h" @@ -67,23 +69,22 @@ void encoder_filename( sprintf(buf, "%s_%c%0*d.%s", base, ch, ndigits, j, ext); } -#define DATA_FILENAME "data" -#define DATA_EXT "ext" +#define DATA_FILENAME "data.vda" // encode a meta-chunk. -// precondition: "dir" contains a file "data". +// precondition: "dir" contains a file "data.vda". // postcondition: dir contains // a subdir Coding with encoded chunks -// subdirs data.0 .. data.m -// each containing a same-named symbolic link to the corresponding chunk +// subdirs 0/ .. m/ +// each containing a symbolic link "data.vda" to the corresponding chunk // // The size of these chunks is returned in "size" // int encode(const char* dir, CODING& c, double& size) { char cmd[1024]; sprintf(cmd, - "cd %s; /mydisks/b/users/boincadm/vda_test/encoder %s.%s %d %d cauchy_good 32 1024 500000", - dir, DATA_FILENAME, DATA_EXT, c.n, c.k + "cd %s; /mydisks/b/users/boincadm/vda_test/encoder %s %d %d cauchy_good 32 1024 500000", + dir, DATA_FILENAME, c.n, c.k ); printf("%s\n", cmd); int s = system(cmd); @@ -95,21 +96,22 @@ int encode(const char* dir, CODING& c, double& size) { // make symlinks // for (int i=0; idfile->id; - strcpy(ch.name, name); ch.host_id = host_id; + strcpy(ch.name, name); + ch.size = parent->dfile->policy.chunk_size(); ch.present_on_host = 0; ch.transfer_in_progress = true; ch.transfer_wait = true; @@ -301,6 +304,7 @@ META_CHUNK::META_CHUNK(VDA_FILE_AUX* d, META_CHUNK* p, int index) { // int META_CHUNK::init(const char* dir, POLICY& p, int level) { double size; + char child_dir[1024]; CODING& c = p.codings[level]; int retval = encode(dir, c, size); @@ -309,8 +313,7 @@ int META_CHUNK::init(const char* dir, POLICY& p, int level) { if (level+1 < p.coding_levels) { for (int i=0; iinit(child_dir, p, level+1); if (retval) return retval; @@ -320,6 +323,17 @@ int META_CHUNK::init(const char* dir, POLICY& p, int level) { for (int i=0; iinit(dir, policy, 0); if (retval) return retval; @@ -346,6 +362,16 @@ int VDA_FILE_AUX::init() { fprintf(f, "%.0f\n", policy.chunk_sizes[i]); } fclose(f); + + // create symlink from download dir + // + dir_hier_path(name, config.download_dir, config.uldl_dir_fanout, buf); + retval = symlink(buf2, buf); + if (retval) { + log_messages.printf(MSG_CRITICAL, "symlink %s %s failed\n", buf2, buf); + return ERR_SYMLINK; + } + return 0; }