diff --git a/checkin_notes b/checkin_notes index 6c0b26724a..45f1da6086 100644 --- a/checkin_notes +++ b/checkin_notes @@ -5464,3 +5464,8 @@ David 1 Sept 2011 api/ reduce_main.cpp +David 1 Sept 2011 + - server: factor process_input_template() into smaller pieces + + tools/ + process_input_template.cpp diff --git a/tools/process_input_template.cpp b/tools/process_input_template.cpp index 064baabfd5..c88541ced8 100644 --- a/tools/process_input_template.cpp +++ b/tools/process_input_template.cpp @@ -132,6 +132,238 @@ static void write_md5_info( return; } +static int process_file_info( + XML_PARSER& xp, string& out, int ninfiles, const char** infiles, + SCHED_CONFIG& config_loc +) { + vector urls; + bool generated_locally = false; + int retval, file_number = -1; + double nbytes, nbytesdef = -1; + string md5str, urlstr, tmpstr; + char buf[BLOB_SIZE], path[256], top_download_path[256], md5[33], url[256]; + + out += "\n"; + while (!xp.get_tag()) { + if (xp.parse_int("number", file_number)) { + continue; + } else if (xp.parse_bool("generated_locally", generated_locally)) { + continue; + } else if (xp.parse_string("url", urlstr)) { + urls.push_back(urlstr); + continue; + } else if (xp.parse_string("md5_cksum", md5str)) { + continue; + } else if (xp.parse_double("nbytes", nbytesdef)) { + continue; + } else if (xp.match_tag("/file_info")) { + if (nbytesdef != -1 || md5str != "" || urlstr != "") { + if (nbytesdef == -1 || md5str == "" || urlstr == "") { + fprintf(stderr, "All file properties must be defined " + "if at least one is defined (url, md5_cksum, nbytes)!\n" + ); + return ERR_XML_PARSE; + } + } + if (file_number < 0) { + fprintf(stderr, "No file number found\n"); + return ERR_XML_PARSE; + } + if (file_number >= ninfiles) { + fprintf(stderr, + "Too few input files given; need at least %d\n", + file_number+1 + ); + return ERR_XML_PARSE; + } + if (generated_locally) { + sprintf(buf, + " %s\n" + " \n" + "\n", + infiles[file_number] + ); + } else if (nbytesdef == -1) { + // here if nybtes was not supplied; stage the file + // + dir_hier_path( + infiles[file_number], config_loc.download_dir, + config_loc.uldl_dir_fanout, path, true + ); + + // if file isn't found in hierarchy, + // look for it at top level and copy + // + if (!boinc_file_exists(path)) { + sprintf(top_download_path, + "%s/%s",config_loc.download_dir, + infiles[file_number] + ); + boinc_copy(top_download_path, path); + } + + if (!config_loc.cache_md5_info || !got_md5_info(path, md5, &nbytes)) { + retval = md5_file(path, md5, nbytes); + if (retval) { + fprintf(stderr, "process_input_template: md5_file %d\n", retval); + return retval; + } else if (config_loc.cache_md5_info) { + write_md5_info(path, md5, nbytes); + } + } + + dir_hier_url( + infiles[file_number], config_loc.download_url, + config_loc.uldl_dir_fanout, url + ); + sprintf(buf, + " %s\n" + " %s\n" + " %s\n" + " %.0f\n" + "\n", + infiles[file_number], + url, + md5, + nbytes + ); + } else { + // here if nbytes etc. was supplied, + // i.e the file is already staged, possibly remotely + // + urlstr = ""; + for (unsigned int i=0; i\n"; + } + sprintf(buf, + " %s\n" + "%s" + " %s\n" + " %.0f\n" + "\n", + infiles[file_number], + urlstr.c_str(), + md5str.c_str(), + nbytesdef + ); + } + out += buf; + break; + } else { + retval = xp.copy_element(tmpstr); + if (retval) return retval; + out += tmpstr; + out += "\n"; + } + } + return 0; +} + +static int process_workunit( + XML_PARSER& xp, WORKUNIT& wu, string& out, + const char** infiles, + const char* command_line, + const char* additional_xml +) { + char buf[256], open_name[256]; + int file_number; + string tmpstr, cmdline; + int retval; + + out += "\n"; + if (command_line) { + //fprintf(stderr, "appending command line: %s\n", command_line); + out += "\n"; + out += command_line; + out += "\n\n"; + } + while (!xp.get_tag()) { + if (xp.match_tag("/workunit")) { + if (additional_xml && strlen(additional_xml)) { + out += additional_xml; + out += "\n"; + } + out += ""; + break; + } else if (xp.match_tag("file_ref")) { + out += "\n"; + bool found_file_number = false, found_open_name = false; + while (!xp.get_tag()) { + if (xp.parse_int("file_number", file_number)) { + sprintf(buf, " %s\n", + infiles[file_number] + ); + out += buf; + found_file_number = true; + continue; + } else if (xp.parse_str("open_name", open_name, sizeof(open_name))) { + sprintf(buf, " %s\n", open_name); + out += buf; + found_open_name = true; + continue; + } else if (xp.match_tag("/file_ref")) { + if (!found_file_number) { + fprintf(stderr, "No file number found\n"); + return ERR_XML_PARSE; + } + if (!found_open_name) { + fprintf(stderr, "No open name found\n"); + return ERR_XML_PARSE; + } + out += "\n"; + break; + } else if (xp.parse_string("file_name", tmpstr)) { + fprintf(stderr, " ignored in element.\n"); + continue; + } else { + retval = xp.copy_element(tmpstr); + if (retval) return retval; + out += tmpstr; + out += "\n"; + } + } + } else if (xp.parse_string("command_line", cmdline)) { + if (command_line) { + fprintf(stderr, "Can't specify command line twice"); + return ERR_XML_PARSE; + } + out += "\n"; + out += cmdline; + out += "\n\n"; + } else if (xp.parse_double("rsc_fpops_est", wu.rsc_fpops_est)) { + continue; + } else if (xp.parse_double("rsc_fpops_bound", wu.rsc_fpops_bound)) { + continue; + } else if (xp.parse_double("rsc_memory_bound", wu.rsc_memory_bound)) { + continue; + } else if (xp.parse_double("rsc_bandwidth_bound", wu.rsc_bandwidth_bound)) { + continue; + } else if (xp.parse_double("rsc_disk_bound", wu.rsc_disk_bound)) { + continue; + } else if (xp.parse_int("batch", wu.batch)) { + continue; + } else if (xp.parse_int("delay_bound", wu.delay_bound)){ + continue; + } else if (xp.parse_int("min_quorum", wu.min_quorum)) { + continue; + } else if (xp.parse_int("target_nresults", wu.target_nresults)) { + continue; + } else if (xp.parse_int("max_error_results", wu.max_error_results)) { + continue; + } else if (xp.parse_int("max_total_results", wu.max_total_results)) { + continue; + } else if (xp.parse_int("max_success_results", wu.max_success_results)) { + continue; + } else { + retval = xp.copy_element(tmpstr); + if (retval) return retval; + out += tmpstr; + out += "\n"; + } + } + return 0; +} + // fill in the workunit's XML document (wu.xml_doc) // by scanning the input template, macro-substituting the input files, // and putting in the command line element and additional XML @@ -145,12 +377,9 @@ int process_input_template( const char* command_line, const char* additional_xml ) { - char buf[BLOB_SIZE], md5[33], path[256], url[256], top_download_path[256]; - string out, cmdline, md5str, urlstr, tmpstr; - int retval, file_number; - double nbytes, nbytesdef; - char open_name[256]; - bool found=false; + string out; + int retval; + bool found_workunit=false; int nfiles_parsed = 0; out = ""; @@ -162,229 +391,27 @@ int process_input_template( if (xp.match_tag("input_template")) continue; if (xp.match_tag("/input_template")) continue; if (xp.match_tag("file_info")) { - vector urls; - bool generated_locally = false; - file_number = nbytesdef = -1; - md5str = urlstr = ""; - out += "\n"; - while (!xp.get_tag()) { - if (xp.parse_int("number", file_number)) { - continue; - } else if (xp.parse_bool("generated_locally", generated_locally)) { - continue; - } else if (xp.parse_string("url", urlstr)) { - urls.push_back(urlstr); - continue; - } else if (xp.parse_string("md5_cksum", md5str)) { - continue; - } else if (xp.parse_double("nbytes", nbytesdef)) { - continue; - } else if (xp.match_tag("/file_info")) { - if (nbytesdef != -1 || md5str != "" || urlstr != "") { - if (nbytesdef == -1 || md5str == "" || urlstr == "") { - fprintf(stderr, "All file properties must be defined " - "if at least one is defined (url, md5_cksum, nbytes)!\n" - ); - return ERR_XML_PARSE; - } - } - if (file_number < 0) { - fprintf(stderr, "No file number found\n"); - return ERR_XML_PARSE; - } - if (file_number >= ninfiles) { - fprintf(stderr, - "Too few input files given; need at least %d\n", - file_number+1 - ); - return ERR_XML_PARSE; - } - nfiles_parsed++; - if (generated_locally) { - sprintf(buf, - " %s\n" - " \n" - "\n", - infiles[file_number] - ); - } else if (nbytesdef == -1) { - // here if nybtes was not supplied; stage the file - // - dir_hier_path( - infiles[file_number], config_loc.download_dir, - config_loc.uldl_dir_fanout, path, true - ); - - // if file isn't found in hierarchy, - // look for it at top level and copy - // - if (!boinc_file_exists(path)) { - sprintf(top_download_path, - "%s/%s",config_loc.download_dir, - infiles[file_number] - ); - boinc_copy(top_download_path, path); - } - - if (!config_loc.cache_md5_info || !got_md5_info(path, md5, &nbytes)) { - retval = md5_file(path, md5, nbytes); - if (retval) { - fprintf(stderr, "process_input_template: md5_file %d\n", retval); - return retval; - } else if (config_loc.cache_md5_info) { - write_md5_info(path, md5, nbytes); - } - } - - dir_hier_url( - infiles[file_number], config_loc.download_url, - config_loc.uldl_dir_fanout, url - ); - sprintf(buf, - " %s\n" - " %s\n" - " %s\n" - " %.0f\n" - "\n", - infiles[file_number], - url, - md5, - nbytes - ); - } else { - // here if nbytes etc. was supplied, - // i.e the file is already staged, possibly remotely - // - urlstr = ""; - for (unsigned int i=0; i\n"; - } - sprintf(buf, - " %s\n" - "%s" - " %s\n" - " %.0f\n" - "\n", - infiles[file_number], - urlstr.c_str(), - md5str.c_str(), - nbytesdef - ); - } - out += buf; - break; - } else { - retval = xp.copy_element(tmpstr); - if (retval) return retval; - out += tmpstr; - out += "\n"; - } - } + retval = process_file_info(xp, out, ninfiles, infiles, config_loc); + if (retval) return retval; + nfiles_parsed++; } else if (xp.match_tag("workunit")) { - found = true; - out += "\n"; - if (command_line) { - //fprintf(stderr, "appending command line: %s\n", command_line); - out += "\n"; - out += command_line; - out += "\n\n"; - } - while (!xp.get_tag()) { - if (xp.match_tag("/workunit")) { - if (additional_xml && strlen(additional_xml)) { - out += additional_xml; - out += "\n"; - } - out += ""; - break; - } else if (xp.match_tag("file_ref")) { - out += "\n"; - bool found_file_number = false, found_open_name = false; - while (!xp.get_tag()) { - if (xp.parse_int("file_number", file_number)) { - sprintf(buf, " %s\n", - infiles[file_number] - ); - out += buf; - found_file_number = true; - continue; - } else if (xp.parse_str("open_name", open_name, sizeof(open_name))) { - sprintf(buf, " %s\n", open_name); - out += buf; - found_open_name = true; - continue; - } else if (xp.match_tag("/file_ref")) { - if (!found_file_number) { - fprintf(stderr, "No file number found\n"); - return ERR_XML_PARSE; - } - if (!found_open_name) { - fprintf(stderr, "No open name found\n"); - return ERR_XML_PARSE; - } - out += "\n"; - break; - } else if (xp.parse_string("file_name", tmpstr)) { - fprintf(stderr, " ignored in element.\n"); - continue; - } else { - retval = xp.copy_element(tmpstr); - if (retval) return retval; - out += tmpstr; - out += "\n"; - } - } - } else if (xp.parse_string("command_line", cmdline)) { - if (command_line) { - fprintf(stderr, "Can't specify command line twice"); - return ERR_XML_PARSE; - } - out += "\n"; - out += cmdline; - out += "\n\n"; - } else if (xp.parse_double("rsc_fpops_est", wu.rsc_fpops_est)) { - continue; - } else if (xp.parse_double("rsc_fpops_bound", wu.rsc_fpops_bound)) { - continue; - } else if (xp.parse_double("rsc_memory_bound", wu.rsc_memory_bound)) { - continue; - } else if (xp.parse_double("rsc_bandwidth_bound", wu.rsc_bandwidth_bound)) { - continue; - } else if (xp.parse_double("rsc_disk_bound", wu.rsc_disk_bound)) { - continue; - } else if (xp.parse_int("batch", wu.batch)) { - continue; - } else if (xp.parse_int("delay_bound", wu.delay_bound)){ - continue; - } else if (xp.parse_int("min_quorum", wu.min_quorum)) { - continue; - } else if (xp.parse_int("target_nresults", wu.target_nresults)) { - continue; - } else if (xp.parse_int("max_error_results", wu.max_error_results)) { - continue; - } else if (xp.parse_int("max_total_results", wu.max_total_results)) { - continue; - } else if (xp.parse_int("max_success_results", wu.max_success_results)) { - continue; - } else { - retval = xp.copy_element(tmpstr); - if (retval) return retval; - out += tmpstr; - out += "\n"; - } - } + found_workunit = true; + retval = process_workunit( + xp, wu, out, infiles, command_line, additional_xml + ); + if (retval) return retval; } } - if (!found) { + if (!found_workunit) { fprintf(stderr, "process_input_template: bad WU template - no \n"); - return -1; + return ERR_XML_PARSE; } if (nfiles_parsed != ninfiles) { fprintf(stderr, "process_input_template: %d input files listed, but template has %d\n", ninfiles, nfiles_parsed ); - return -1; + return ERR_XML_PARSE; } if (out.size() > sizeof(wu.xml_doc)-1) { fprintf(stderr,