Rooba
/
boinc
mirror of https://github.com/BOINC/boinc.git
1
0
Fork 0
Code Issues 1 Packages Projects Releases Wiki Activity

- storage stuff. Getting there. 

svn path=/trunk/boinc/; revision=25355
This commit is contained in:
David Anderson 2012-02-29 07:22:59 +00:00
parent 516e5ad798
commit 127e905e0d
7 changed files with 103 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
checkin_notes
View File

@ -2336,3 +2336,16 @@ David 28 Feb 2012
sched_types.cpp,h
vda/
sched_vda.cpp
David 28 Feb 2012
- storage stuff. Getting there.
db/
boinc_db.cpp
sched/
handle_request.cpp
vda/
vdad.cpp
vda_policy.h
sched_vda.cpp
vda.cpp

2
db/boinc_db.cpp
View File

@ -2367,7 +2367,7 @@ void DB_VDA_CHUNK_HOST::db_print(char* buf) {
"vda_file_id=%d, "
"host_id=%d, "
"name='%s', "
"size=%f "
"size=%f, "
"present_on_host=%d, "
"transfer_in_progress=%d, "
"transfer_wait=%d, "

2
sched/handle_request.cpp
View File

@ -1141,7 +1141,7 @@ void process_request(char* code_sign_key) {
) {
g_reply->insert_message("No work available", "low");
g_reply->set_delay(DELAY_NO_WORK_SKIP);
if (!config.msg_to_host) {
if (!config.msg_to_host && !config.enable_vda) {
log_messages.printf(MSG_NORMAL, "No work - skipping DB access\n");
return;
}

16
vda/sched_vda.cpp
View File

@ -158,9 +158,9 @@ static int process_present_file(FILE_INFO& fi, CHUNK_LIST& chunks) {
//
static int process_missing_chunks(CHUNK_LIST& chunks) {
CHUNK_LIST::iterator it;
it = chunks.begin();
while (it != chunks.end()) {
for (it = chunks.begin(); it != chunks.end(); it++) {
DB_VDA_CHUNK_HOST& ch = it->second;
if (!ch.present_on_host && ch.transfer_in_progress) continue;
if (!ch.found) {
if (config.debug_vda) {
log_messages.printf(MSG_NORMAL,
@ -168,10 +168,8 @@ static int process_missing_chunks(CHUNK_LIST& chunks) {
);
}
ch.delete_from_db();
chunks.erase(it);
ch.transfer_in_progress = false;
mark_for_update(ch.vda_file_id);
} else {
it++;
}
}
return 0;
@ -193,6 +191,7 @@ static int enforce_quota(CHUNK_LIST& chunks) {
CHUNK_LIST::iterator it = chunks.begin();
while (x > g_request->host.d_project_share && it != chunks.end()) {
DB_VDA_CHUNK_HOST& ch = it->second;
if (!ch.found) continue;
FILE_INFO fi;
strcpy(fi.name, ch.name);
if (config.debug_vda) {
@ -210,7 +209,7 @@ static int enforce_quota(CHUNK_LIST& chunks) {
// issue upload and download commands
//
static int issue_transfer_commands(CHUNK_LIST& chunks) {
char xml_buf[1024], file_name[1024];
char xml_buf[8192], file_name[1024];
int retval;
char url[1024];
vector<const char*> urls;
@ -256,7 +255,8 @@ static int issue_transfer_commands(CHUNK_LIST& chunks) {
get_chunk_url(vf, ch.name, url);
urls.push_back(url);
get_chunk_dir(vf, ch.name, chunk_dir);
get_chunk_md5(chunk_dir, md5);
retval = get_chunk_md5(chunk_dir, md5);
if (retval) return retval;
retval = put_file_xml(
file_name,
urls,
@ -294,7 +294,7 @@ void handle_vda() {
//
DB_VDA_CHUNK_HOST ch;
char buf[256];
sprintf(buf, "host_id=%d", g_reply->host.id);
sprintf(buf, "where host_id=%d", g_reply->host.id);
while (1) {
retval = ch.enumerate(buf);
if (retval == ERR_DB_NOT_FOUND) break;

34
vda/vda.cpp
View File

@ -25,6 +25,7 @@
#include "boinc_db.h"
#include "filesys.h"
#include "sched_config.h"
#include "sched_util.h"
#include "util.h"
#include "vda_lib.h"
@ -77,6 +78,29 @@ int handle_add(const char* path) {
return 0;
}
int handle_remove(const char* name) {
DB_VDA_FILE vf;
char buf[1024];
sprintf(buf, "where name='%s'", name);
int retval = vf.lookup(buf);
if (retval) return retval;
// delete DB records
//
DB_VDA_CHUNK_HOST ch;
sprintf(buf, "vda_file_id=%d", vf.id);
ch.delete_from_db_multi(buf);
vf.delete_from_db();
dir_hier_path(name, config.download_dir, config.uldl_dir_fanout, buf);
unlink(buf);
retval = chdir(vf.dir);
if (retval) perror("chdir");
retval = system("/bin/rm -r [0-9]* Coding data.vda");
if (retval) perror("system");
return 0;
}
int main(int argc, char** argv) {
int retval = config.parse_file();
if (retval) {
@ -101,6 +125,16 @@ int main(int argc, char** argv) {
}
exit(retval);
}
if (!strcmp(argv[i], "remove")) {
if (argc != 3) usage();
retval = handle_remove(argv[++i]);
if (retval) {
fprintf(stderr, "error %d\n", retval);
} else {
printf("file removed successfully\n");
}
exit(retval);
}
usage();
}
usage();

3
vda/vda_policy.h
View File

@ -36,6 +36,9 @@ struct POLICY {
int coding_levels;
CODING codings[10];
double chunk_sizes[10];
double chunk_size() {
return chunk_sizes[coding_levels-1];
}
char description[256]; // derived from the above

60
vda/vdad.cpp
View File

@ -33,6 +33,8 @@ using std::set;
#include "boinc_db.h"
#include "sched_config.h"
#include "sched_util.h"
#include "md5_file.h"
#include "error_numbers.h"
#include "util.h"
@ -67,23 +69,22 @@ void encoder_filename(
sprintf(buf, "%s_%c%0*d.%s", base, ch, ndigits, j, ext);
}
#define DATA_FILENAME "data"
#define DATA_EXT "ext"
#define DATA_FILENAME "data.vda"
// encode a meta-chunk.
// precondition: "dir" contains a file "data".
// precondition: "dir" contains a file "data.vda".
// postcondition: dir contains
// a subdir Coding with encoded chunks
// subdirs data.0 .. data.m
// each containing a same-named symbolic link to the corresponding chunk
// subdirs 0/ .. m/
// each containing a symbolic link "data.vda" to the corresponding chunk
//
// The size of these chunks is returned in "size"
//
int encode(const char* dir, CODING& c, double& size) {
char cmd[1024];
sprintf(cmd,
"cd %s; /mydisks/b/users/boincadm/vda_test/encoder %s.%s %d %d cauchy_good 32 1024 500000",
dir, DATA_FILENAME, DATA_EXT, c.n, c.k
"cd %s; /mydisks/b/users/boincadm/vda_test/encoder %s %d %d cauchy_good 32 1024 500000",
dir, DATA_FILENAME, c.n, c.k
);
printf("%s\n", cmd);
int s = system(cmd);
@ -95,21 +96,22 @@ int encode(const char* dir, CODING& c, double& size) {
// make symlinks
//
for (int i=0; i<c.m; i++) {
char enc_filename[1024], target_path[1024], chunk_name[1024];
char enc_filename[1024], target_path[1024];
char dir_name[1024], link_name[1024];
encoder_filename(DATA_FILENAME, DATA_EXT, c, i, enc_filename);
encoder_filename("data", "vda", c, i, enc_filename);
sprintf(target_path, "%s/Coding/%s", dir, enc_filename);
sprintf(chunk_name, "%s.%d", DATA_FILENAME, i);
sprintf(dir_name, "%s/%s", dir, chunk_name);
sprintf(dir_name, "%s/%d", dir, i);
int retval = mkdir(dir_name, 0777);
if (retval) {
perror("mkdir");
return retval;
}
sprintf(link_name, "%s/%s.%s", dir_name, DATA_FILENAME, DATA_EXT);
sprintf(link_name, "%s/%s", dir_name, DATA_FILENAME);
retval = symlink(target_path, link_name);
if (retval) {
perror("symlink");
log_messages.printf(MSG_CRITICAL,
"encode(): symlink %s %s failed\n", target_path, link_name
);
return retval;
}
if (i == 0) {
@ -140,8 +142,9 @@ int CHUNK::assign() {
DB_VDA_CHUNK_HOST ch;
ch.create_time = dtime();
ch.vda_file_id = parent->dfile->id;
strcpy(ch.name, name);
ch.host_id = host_id;
strcpy(ch.name, name);
ch.size = parent->dfile->policy.chunk_size();
ch.present_on_host = 0;
ch.transfer_in_progress = true;
ch.transfer_wait = true;
@ -301,6 +304,7 @@ META_CHUNK::META_CHUNK(VDA_FILE_AUX* d, META_CHUNK* p, int index) {
//
int META_CHUNK::init(const char* dir, POLICY& p, int level) {
double size;
char child_dir[1024];
CODING& c = p.codings[level];
int retval = encode(dir, c, size);
@ -309,8 +313,7 @@ int META_CHUNK::init(const char* dir, POLICY& p, int level) {
if (level+1 < p.coding_levels) {
for (int i=0; i<c.m; i++) {
char child_dir[1024];
sprintf(child_dir, "%s/%s.%d", dir, DATA_FILENAME, i);
sprintf(child_dir, "%s/%d", dir, i);
META_CHUNK* mc = new META_CHUNK(dfile, parent, i);
retval = mc->init(child_dir, p, level+1);
if (retval) return retval;
@ -320,6 +323,17 @@ int META_CHUNK::init(const char* dir, POLICY& p, int level) {
for (int i=0; i<c.m; i++) {
CHUNK* cp = new CHUNK(this, p.chunk_sizes[level], i);
children.push_back(cp);
// write the chunk's MD5 to a file
//
char file_path[1024], md5_file_path[1024];
sprintf(file_path, "%s/%d/%s", dir, i, DATA_FILENAME);
sprintf(md5_file_path, "%s/%d/md5.txt", dir, i);
char md5[64];
md5_file(file_path, md5, size);
FILE* f = fopen(md5_file_path, "w");
fprintf(f, "%s\n", md5);
fclose(f);
}
}
return 0;
@ -331,12 +345,14 @@ int META_CHUNK::init(const char* dir, POLICY& p, int level) {
//
int VDA_FILE_AUX::init() {
char buf[1024], buf2[1024];
sprintf(buf, "%s/%s.%s", dir, DATA_FILENAME, DATA_EXT);
sprintf(buf, "%s/%s", dir, DATA_FILENAME);
sprintf(buf2, "%s/%s", dir, name);
int retval = symlink(buf2, buf);
if (retval) {
log_messages.printf(MSG_CRITICAL, "symlink %s %s failed\n", buf2, buf);
return ERR_SYMLINK;
}
meta_chunk = new META_CHUNK(this, NULL, 0);
retval = meta_chunk->init(dir, policy, 0);
if (retval) return retval;
@ -346,6 +362,16 @@ int VDA_FILE_AUX::init() {
fprintf(f, "%.0f\n", policy.chunk_sizes[i]);
}
fclose(f);
// create symlink from download dir
//
dir_hier_path(name, config.download_dir, config.uldl_dir_fanout, buf);
retval = symlink(buf2, buf);
if (retval) {
log_messages.printf(MSG_CRITICAL, "symlink %s %s failed\n", buf2, buf);
return ERR_SYMLINK;
}
return 0;
}