mirror of https://github.com/BOINC/boinc.git
- server: factor process_input_template() into smaller pieces
svn path=/trunk/boinc/; revision=24108
This commit is contained in:
David Anderson
parent
c529c3c50d
commit
f75ffd88f2
|
|
@ -5464,3 +5464,8 @@ David 1 Sept 2011
|
|||
api/
|
||||
reduce_main.cpp
|
||||
|
||||
David 1 Sept 2011
|
||||
- server: factor process_input_template() into smaller pieces
|
||||
|
||||
tools/
|
||||
process_input_template.cpp
|
||||
|
|
|
|||
|
|
@ -132,6 +132,238 @@ static void write_md5_info(
|
|||
return;
|
||||
}
|
||||
|
||||
static int process_file_info(
|
||||
XML_PARSER& xp, string& out, int ninfiles, const char** infiles,
|
||||
SCHED_CONFIG& config_loc
|
||||
) {
|
||||
vector<string> urls;
|
||||
bool generated_locally = false;
|
||||
int retval, file_number = -1;
|
||||
double nbytes, nbytesdef = -1;
|
||||
string md5str, urlstr, tmpstr;
|
||||
char buf[BLOB_SIZE], path[256], top_download_path[256], md5[33], url[256];
|
||||
|
||||
out += "<file_info>\n";
|
||||
while (!xp.get_tag()) {
|
||||
if (xp.parse_int("number", file_number)) {
|
||||
continue;
|
||||
} else if (xp.parse_bool("generated_locally", generated_locally)) {
|
||||
continue;
|
||||
} else if (xp.parse_string("url", urlstr)) {
|
||||
urls.push_back(urlstr);
|
||||
continue;
|
||||
} else if (xp.parse_string("md5_cksum", md5str)) {
|
||||
continue;
|
||||
} else if (xp.parse_double("nbytes", nbytesdef)) {
|
||||
continue;
|
||||
} else if (xp.match_tag("/file_info")) {
|
||||
if (nbytesdef != -1 || md5str != "" || urlstr != "") {
|
||||
if (nbytesdef == -1 || md5str == "" || urlstr == "") {
|
||||
fprintf(stderr, "All file properties must be defined "
|
||||
"if at least one is defined (url, md5_cksum, nbytes)!\n"
|
||||
);
|
||||
return ERR_XML_PARSE;
|
||||
}
|
||||
}
|
||||
if (file_number < 0) {
|
||||
fprintf(stderr, "No file number found\n");
|
||||
return ERR_XML_PARSE;
|
||||
}
|
||||
if (file_number >= ninfiles) {
|
||||
fprintf(stderr,
|
||||
"Too few input files given; need at least %d\n",
|
||||
file_number+1
|
||||
);
|
||||
return ERR_XML_PARSE;
|
||||
}
|
||||
if (generated_locally) {
|
||||
sprintf(buf,
|
||||
" <name>%s</name>\n"
|
||||
" <generated_locally/>\n"
|
||||
"</file_info>\n",
|
||||
infiles[file_number]
|
||||
);
|
||||
} else if (nbytesdef == -1) {
|
||||
// here if nybtes was not supplied; stage the file
|
||||
//
|
||||
dir_hier_path(
|
||||
infiles[file_number], config_loc.download_dir,
|
||||
config_loc.uldl_dir_fanout, path, true
|
||||
);
|
||||
|
||||
// if file isn't found in hierarchy,
|
||||
// look for it at top level and copy
|
||||
//
|
||||
if (!boinc_file_exists(path)) {
|
||||
sprintf(top_download_path,
|
||||
"%s/%s",config_loc.download_dir,
|
||||
infiles[file_number]
|
||||
);
|
||||
boinc_copy(top_download_path, path);
|
||||
}
|
||||
|
||||
if (!config_loc.cache_md5_info || !got_md5_info(path, md5, &nbytes)) {
|
||||
retval = md5_file(path, md5, nbytes);
|
||||
if (retval) {
|
||||
fprintf(stderr, "process_input_template: md5_file %d\n", retval);
|
||||
return retval;
|
||||
} else if (config_loc.cache_md5_info) {
|
||||
write_md5_info(path, md5, nbytes);
|
||||
}
|
||||
}
|
||||
|
||||
dir_hier_url(
|
||||
infiles[file_number], config_loc.download_url,
|
||||
config_loc.uldl_dir_fanout, url
|
||||
);
|
||||
sprintf(buf,
|
||||
" <name>%s</name>\n"
|
||||
" <url>%s</url>\n"
|
||||
" <md5_cksum>%s</md5_cksum>\n"
|
||||
" <nbytes>%.0f</nbytes>\n"
|
||||
"</file_info>\n",
|
||||
infiles[file_number],
|
||||
url,
|
||||
md5,
|
||||
nbytes
|
||||
);
|
||||
} else {
|
||||
// here if nbytes etc. was supplied,
|
||||
// i.e the file is already staged, possibly remotely
|
||||
//
|
||||
urlstr = "";
|
||||
for (unsigned int i=0; i<urls.size(); i++) {
|
||||
urlstr += " <url>" + urls.at(i) + "</url>\n";
|
||||
}
|
||||
sprintf(buf,
|
||||
" <name>%s</name>\n"
|
||||
"%s"
|
||||
" <md5_cksum>%s</md5_cksum>\n"
|
||||
" <nbytes>%.0f</nbytes>\n"
|
||||
"</file_info>\n",
|
||||
infiles[file_number],
|
||||
urlstr.c_str(),
|
||||
md5str.c_str(),
|
||||
nbytesdef
|
||||
);
|
||||
}
|
||||
out += buf;
|
||||
break;
|
||||
} else {
|
||||
retval = xp.copy_element(tmpstr);
|
||||
if (retval) return retval;
|
||||
out += tmpstr;
|
||||
out += "\n";
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int process_workunit(
|
||||
XML_PARSER& xp, WORKUNIT& wu, string& out,
|
||||
const char** infiles,
|
||||
const char* command_line,
|
||||
const char* additional_xml
|
||||
) {
|
||||
char buf[256], open_name[256];
|
||||
int file_number;
|
||||
string tmpstr, cmdline;
|
||||
int retval;
|
||||
|
||||
out += "<workunit>\n";
|
||||
if (command_line) {
|
||||
//fprintf(stderr, "appending command line: %s\n", command_line);
|
||||
out += "<command_line>\n";
|
||||
out += command_line;
|
||||
out += "\n</command_line>\n";
|
||||
}
|
||||
while (!xp.get_tag()) {
|
||||
if (xp.match_tag("/workunit")) {
|
||||
if (additional_xml && strlen(additional_xml)) {
|
||||
out += additional_xml;
|
||||
out += "\n";
|
||||
}
|
||||
out += "</workunit>";
|
||||
break;
|
||||
} else if (xp.match_tag("file_ref")) {
|
||||
out += "<file_ref>\n";
|
||||
bool found_file_number = false, found_open_name = false;
|
||||
while (!xp.get_tag()) {
|
||||
if (xp.parse_int("file_number", file_number)) {
|
||||
sprintf(buf, " <file_name>%s</file_name>\n",
|
||||
infiles[file_number]
|
||||
);
|
||||
out += buf;
|
||||
found_file_number = true;
|
||||
continue;
|
||||
} else if (xp.parse_str("open_name", open_name, sizeof(open_name))) {
|
||||
sprintf(buf, " <open_name>%s</open_name>\n", open_name);
|
||||
out += buf;
|
||||
found_open_name = true;
|
||||
continue;
|
||||
} else if (xp.match_tag("/file_ref")) {
|
||||
if (!found_file_number) {
|
||||
fprintf(stderr, "No file number found\n");
|
||||
return ERR_XML_PARSE;
|
||||
}
|
||||
if (!found_open_name) {
|
||||
fprintf(stderr, "No open name found\n");
|
||||
return ERR_XML_PARSE;
|
||||
}
|
||||
out += "</file_ref>\n";
|
||||
break;
|
||||
} else if (xp.parse_string("file_name", tmpstr)) {
|
||||
fprintf(stderr, "<file_name> ignored in <file_ref> element.\n");
|
||||
continue;
|
||||
} else {
|
||||
retval = xp.copy_element(tmpstr);
|
||||
if (retval) return retval;
|
||||
out += tmpstr;
|
||||
out += "\n";
|
||||
}
|
||||
}
|
||||
} else if (xp.parse_string("command_line", cmdline)) {
|
||||
if (command_line) {
|
||||
fprintf(stderr, "Can't specify command line twice");
|
||||
return ERR_XML_PARSE;
|
||||
}
|
||||
out += "<command_line>\n";
|
||||
out += cmdline;
|
||||
out += "\n</command_line>\n";
|
||||
} else if (xp.parse_double("rsc_fpops_est", wu.rsc_fpops_est)) {
|
||||
continue;
|
||||
} else if (xp.parse_double("rsc_fpops_bound", wu.rsc_fpops_bound)) {
|
||||
continue;
|
||||
} else if (xp.parse_double("rsc_memory_bound", wu.rsc_memory_bound)) {
|
||||
continue;
|
||||
} else if (xp.parse_double("rsc_bandwidth_bound", wu.rsc_bandwidth_bound)) {
|
||||
continue;
|
||||
} else if (xp.parse_double("rsc_disk_bound", wu.rsc_disk_bound)) {
|
||||
continue;
|
||||
} else if (xp.parse_int("batch", wu.batch)) {
|
||||
continue;
|
||||
} else if (xp.parse_int("delay_bound", wu.delay_bound)){
|
||||
continue;
|
||||
} else if (xp.parse_int("min_quorum", wu.min_quorum)) {
|
||||
continue;
|
||||
} else if (xp.parse_int("target_nresults", wu.target_nresults)) {
|
||||
continue;
|
||||
} else if (xp.parse_int("max_error_results", wu.max_error_results)) {
|
||||
continue;
|
||||
} else if (xp.parse_int("max_total_results", wu.max_total_results)) {
|
||||
continue;
|
||||
} else if (xp.parse_int("max_success_results", wu.max_success_results)) {
|
||||
continue;
|
||||
} else {
|
||||
retval = xp.copy_element(tmpstr);
|
||||
if (retval) return retval;
|
||||
out += tmpstr;
|
||||
out += "\n";
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
// fill in the workunit's XML document (wu.xml_doc)
|
||||
// by scanning the input template, macro-substituting the input files,
|
||||
// and putting in the command line element and additional XML
|
||||
|
|
@ -145,12 +377,9 @@ int process_input_template(
|
|||
const char* command_line,
|
||||
const char* additional_xml
|
||||
) {
|
||||
char buf[BLOB_SIZE], md5[33], path[256], url[256], top_download_path[256];
|
||||
string out, cmdline, md5str, urlstr, tmpstr;
|
||||
int retval, file_number;
|
||||
double nbytes, nbytesdef;
|
||||
char open_name[256];
|
||||
bool found=false;
|
||||
string out;
|
||||
int retval;
|
||||
bool found_workunit=false;
|
||||
int nfiles_parsed = 0;
|
||||
|
||||
out = "";
|
||||
|
|
@ -162,229 +391,27 @@ int process_input_template(
|
|||
if (xp.match_tag("input_template")) continue;
|
||||
if (xp.match_tag("/input_template")) continue;
|
||||
if (xp.match_tag("file_info")) {
|
||||
vector<string> urls;
|
||||
bool generated_locally = false;
|
||||
file_number = nbytesdef = -1;
|
||||
md5str = urlstr = "";
|
||||
out += "<file_info>\n";
|
||||
while (!xp.get_tag()) {
|
||||
if (xp.parse_int("number", file_number)) {
|
||||
continue;
|
||||
} else if (xp.parse_bool("generated_locally", generated_locally)) {
|
||||
continue;
|
||||
} else if (xp.parse_string("url", urlstr)) {
|
||||
urls.push_back(urlstr);
|
||||
continue;
|
||||
} else if (xp.parse_string("md5_cksum", md5str)) {
|
||||
continue;
|
||||
} else if (xp.parse_double("nbytes", nbytesdef)) {
|
||||
continue;
|
||||
} else if (xp.match_tag("/file_info")) {
|
||||
if (nbytesdef != -1 || md5str != "" || urlstr != "") {
|
||||
if (nbytesdef == -1 || md5str == "" || urlstr == "") {
|
||||
fprintf(stderr, "All file properties must be defined "
|
||||
"if at least one is defined (url, md5_cksum, nbytes)!\n"
|
||||
);
|
||||
return ERR_XML_PARSE;
|
||||
}
|
||||
}
|
||||
if (file_number < 0) {
|
||||
fprintf(stderr, "No file number found\n");
|
||||
return ERR_XML_PARSE;
|
||||
}
|
||||
if (file_number >= ninfiles) {
|
||||
fprintf(stderr,
|
||||
"Too few input files given; need at least %d\n",
|
||||
file_number+1
|
||||
);
|
||||
return ERR_XML_PARSE;
|
||||
}
|
||||
nfiles_parsed++;
|
||||
if (generated_locally) {
|
||||
sprintf(buf,
|
||||
" <name>%s</name>\n"
|
||||
" <generated_locally/>\n"
|
||||
"</file_info>\n",
|
||||
infiles[file_number]
|
||||
);
|
||||
} else if (nbytesdef == -1) {
|
||||
// here if nybtes was not supplied; stage the file
|
||||
//
|
||||
dir_hier_path(
|
||||
infiles[file_number], config_loc.download_dir,
|
||||
config_loc.uldl_dir_fanout, path, true
|
||||
);
|
||||
|
||||
// if file isn't found in hierarchy,
|
||||
// look for it at top level and copy
|
||||
//
|
||||
if (!boinc_file_exists(path)) {
|
||||
sprintf(top_download_path,
|
||||
"%s/%s",config_loc.download_dir,
|
||||
infiles[file_number]
|
||||
);
|
||||
boinc_copy(top_download_path, path);
|
||||
}
|
||||
|
||||
if (!config_loc.cache_md5_info || !got_md5_info(path, md5, &nbytes)) {
|
||||
retval = md5_file(path, md5, nbytes);
|
||||
if (retval) {
|
||||
fprintf(stderr, "process_input_template: md5_file %d\n", retval);
|
||||
return retval;
|
||||
} else if (config_loc.cache_md5_info) {
|
||||
write_md5_info(path, md5, nbytes);
|
||||
}
|
||||
}
|
||||
|
||||
dir_hier_url(
|
||||
infiles[file_number], config_loc.download_url,
|
||||
config_loc.uldl_dir_fanout, url
|
||||
);
|
||||
sprintf(buf,
|
||||
" <name>%s</name>\n"
|
||||
" <url>%s</url>\n"
|
||||
" <md5_cksum>%s</md5_cksum>\n"
|
||||
" <nbytes>%.0f</nbytes>\n"
|
||||
"</file_info>\n",
|
||||
infiles[file_number],
|
||||
url,
|
||||
md5,
|
||||
nbytes
|
||||
);
|
||||
} else {
|
||||
// here if nbytes etc. was supplied,
|
||||
// i.e the file is already staged, possibly remotely
|
||||
//
|
||||
urlstr = "";
|
||||
for (unsigned int i=0; i<urls.size(); i++) {
|
||||
urlstr += " <url>" + urls.at(i) + "</url>\n";
|
||||
}
|
||||
sprintf(buf,
|
||||
" <name>%s</name>\n"
|
||||
"%s"
|
||||
" <md5_cksum>%s</md5_cksum>\n"
|
||||
" <nbytes>%.0f</nbytes>\n"
|
||||
"</file_info>\n",
|
||||
infiles[file_number],
|
||||
urlstr.c_str(),
|
||||
md5str.c_str(),
|
||||
nbytesdef
|
||||
);
|
||||
}
|
||||
out += buf;
|
||||
break;
|
||||
} else {
|
||||
retval = xp.copy_element(tmpstr);
|
||||
if (retval) return retval;
|
||||
out += tmpstr;
|
||||
out += "\n";
|
||||
}
|
||||
}
|
||||
retval = process_file_info(xp, out, ninfiles, infiles, config_loc);
|
||||
if (retval) return retval;
|
||||
nfiles_parsed++;
|
||||
} else if (xp.match_tag("workunit")) {
|
||||
found = true;
|
||||
out += "<workunit>\n";
|
||||
if (command_line) {
|
||||
//fprintf(stderr, "appending command line: %s\n", command_line);
|
||||
out += "<command_line>\n";
|
||||
out += command_line;
|
||||
out += "\n</command_line>\n";
|
||||
}
|
||||
while (!xp.get_tag()) {
|
||||
if (xp.match_tag("/workunit")) {
|
||||
if (additional_xml && strlen(additional_xml)) {
|
||||
out += additional_xml;
|
||||
out += "\n";
|
||||
}
|
||||
out += "</workunit>";
|
||||
break;
|
||||
} else if (xp.match_tag("file_ref")) {
|
||||
out += "<file_ref>\n";
|
||||
bool found_file_number = false, found_open_name = false;
|
||||
while (!xp.get_tag()) {
|
||||
if (xp.parse_int("file_number", file_number)) {
|
||||
sprintf(buf, " <file_name>%s</file_name>\n",
|
||||
infiles[file_number]
|
||||
);
|
||||
out += buf;
|
||||
found_file_number = true;
|
||||
continue;
|
||||
} else if (xp.parse_str("open_name", open_name, sizeof(open_name))) {
|
||||
sprintf(buf, " <open_name>%s</open_name>\n", open_name);
|
||||
out += buf;
|
||||
found_open_name = true;
|
||||
continue;
|
||||
} else if (xp.match_tag("/file_ref")) {
|
||||
if (!found_file_number) {
|
||||
fprintf(stderr, "No file number found\n");
|
||||
return ERR_XML_PARSE;
|
||||
}
|
||||
if (!found_open_name) {
|
||||
fprintf(stderr, "No open name found\n");
|
||||
return ERR_XML_PARSE;
|
||||
}
|
||||
out += "</file_ref>\n";
|
||||
break;
|
||||
} else if (xp.parse_string("file_name", tmpstr)) {
|
||||
fprintf(stderr, "<file_name> ignored in <file_ref> element.\n");
|
||||
continue;
|
||||
} else {
|
||||
retval = xp.copy_element(tmpstr);
|
||||
if (retval) return retval;
|
||||
out += tmpstr;
|
||||
out += "\n";
|
||||
}
|
||||
}
|
||||
} else if (xp.parse_string("command_line", cmdline)) {
|
||||
if (command_line) {
|
||||
fprintf(stderr, "Can't specify command line twice");
|
||||
return ERR_XML_PARSE;
|
||||
}
|
||||
out += "<command_line>\n";
|
||||
out += cmdline;
|
||||
out += "\n</command_line>\n";
|
||||
} else if (xp.parse_double("rsc_fpops_est", wu.rsc_fpops_est)) {
|
||||
continue;
|
||||
} else if (xp.parse_double("rsc_fpops_bound", wu.rsc_fpops_bound)) {
|
||||
continue;
|
||||
} else if (xp.parse_double("rsc_memory_bound", wu.rsc_memory_bound)) {
|
||||
continue;
|
||||
} else if (xp.parse_double("rsc_bandwidth_bound", wu.rsc_bandwidth_bound)) {
|
||||
continue;
|
||||
} else if (xp.parse_double("rsc_disk_bound", wu.rsc_disk_bound)) {
|
||||
continue;
|
||||
} else if (xp.parse_int("batch", wu.batch)) {
|
||||
continue;
|
||||
} else if (xp.parse_int("delay_bound", wu.delay_bound)){
|
||||
continue;
|
||||
} else if (xp.parse_int("min_quorum", wu.min_quorum)) {
|
||||
continue;
|
||||
} else if (xp.parse_int("target_nresults", wu.target_nresults)) {
|
||||
continue;
|
||||
} else if (xp.parse_int("max_error_results", wu.max_error_results)) {
|
||||
continue;
|
||||
} else if (xp.parse_int("max_total_results", wu.max_total_results)) {
|
||||
continue;
|
||||
} else if (xp.parse_int("max_success_results", wu.max_success_results)) {
|
||||
continue;
|
||||
} else {
|
||||
retval = xp.copy_element(tmpstr);
|
||||
if (retval) return retval;
|
||||
out += tmpstr;
|
||||
out += "\n";
|
||||
}
|
||||
}
|
||||
found_workunit = true;
|
||||
retval = process_workunit(
|
||||
xp, wu, out, infiles, command_line, additional_xml
|
||||
);
|
||||
if (retval) return retval;
|
||||
}
|
||||
}
|
||||
if (!found) {
|
||||
if (!found_workunit) {
|
||||
fprintf(stderr, "process_input_template: bad WU template - no <workunit>\n");
|
||||
return -1;
|
||||
return ERR_XML_PARSE;
|
||||
}
|
||||
if (nfiles_parsed != ninfiles) {
|
||||
fprintf(stderr,
|
||||
"process_input_template: %d input files listed, but template has %d\n",
|
||||
ninfiles, nfiles_parsed
|
||||
);
|
||||
return -1;
|
||||
return ERR_XML_PARSE;
|
||||
}
|
||||
if (out.size() > sizeof(wu.xml_doc)-1) {
|
||||
fprintf(stderr,
|
||||
|
|
|
|||
Loading…
Reference in New Issue