Rooba
/
boinc
mirror of https://github.com/BOINC/boinc.git
1
0
Fork 0
Code Issues 1 Packages Projects Releases Wiki Activity

Daemons: enhance validator framework 

The validator handler can now pass unknown arguments to the project specific handler.
Projects that have there own validator need to implement the validate_handler_init() function and handle project specific arguments there. They also need to supply a validate_handler_usage() function that printf()'s a description of the custom options. For examples see sample_substr_validator.cpp or script_validator.cpp
The validator test harness was also adopted to use this new functions.

This brings the implementation of the validator framework on the same level as the assimilator framework where similar changes where made in 0038d275c and dd004404a.
This commit is contained in:
Christian Beer 2016-08-16 11:06:14 +02:00
parent 3b3c575694
commit 2c36e7246d
10 changed files with 163 additions and 92 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -153,6 +153,7 @@ sched/trickle_credit
sched/trickle_deadline
sched/trickle_echo
sched/update_stats
sched/validator_test
sched/wu_check
tools/cancel_jobs
tools/create_work

7
sched/makefile_validator_test
View File

@ -5,7 +5,8 @@
VALIDATOR_SRC = sample_bitwise_validator.cpp
# where libmysqlclient is
MYSQL_LIB_DIR = /usr/lib64/mysql
MYSQL_LIB := $(shell mysql_config --libs)
MYSQL_INC := $(shell mysql_config --include)
validator_test: validator_test.cpp $(VALIDATOR_SRC)
g++ -g -I.. -I../lib -I../db -o validator_test validator_test.cpp $(VALIDATOR_SRC) validate_util.o -L . -lsched -L ../lib -lboinc -L $(MYSQL_LIB_DIR) -lmysqlclient
validator_test: validator_test.cpp $(VALIDATOR_SRC) makefile_validator_test
g++ -g -I.. -I../lib -I../db $(MYSQL_INC) -o validator_test validator_test.cpp $(VALIDATOR_SRC) validate_util.o -L . -lsched -L ../lib -lboinc $(MYSQL_LIB)

33
sched/sample_bitwise_validator.cpp
View File

@ -38,7 +38,6 @@
using std::string;
using std::vector;
bool first = true;
bool is_gzip = false;
// if true, files are gzipped; skip header when comparing
@ -47,6 +46,25 @@ struct FILE_CKSUM_LIST {
~FILE_CKSUM_LIST(){}
};
int validate_handler_init(int argc, char** argv) {
// handle project specific arguments here
for (int i=1; i<argc; i++) {
if (is_arg(argv[i], "is_gzip")) {
is_gzip = true;
}
}
return 0;
}
void validate_handler_usage() {
// describe the project specific arguments here
fprintf(stderr,
" Custom options:\n"
" [--is_gzip] files are gzipped; skip header when comparing\n"
);
}
bool files_match(FILE_CKSUM_LIST& f1, FILE_CKSUM_LIST& f2) {
if (f1.files.size() != f2.files.size()) return false;
for (unsigned int i=0; i<f1.files.size(); i++) {
@ -55,14 +73,6 @@ bool files_match(FILE_CKSUM_LIST& f1, FILE_CKSUM_LIST& f2) {
return true;
}
void parse_cmdline() {
for (int i=1; i<g_argc; i++) {
if (!strcmp(g_argv[i], "--is_gzip")) {
is_gzip = true;
}
}
}
int init_result(RESULT& result, void*& data) {
int retval;
FILE_CKSUM_LIST* fcl = new FILE_CKSUM_LIST;
@ -70,11 +80,6 @@ int init_result(RESULT& result, void*& data) {
char md5_buf[MD5_LEN];
double nbytes;
if (first) {
parse_cmdline();
first = false;
}
retval = get_output_file_infos(result, files);
if (retval) {
log_messages.printf(MSG_CRITICAL,

31
sched/sample_substr_validator.cpp
View File

@ -23,37 +23,44 @@
// (default: accept it if the string is present)
#include "sched_msgs.h"
#include "sched_util_basic.h"
#include "validate_util2.h"
#include "validator.h"
bool first = true;
char* stderr_string;
bool reject_if_present = false;
void parse_cmdline() {
int validate_handler_init(int argc, char** argv) {
// handle project specific arguments here
bool found = false;
for (int i=1; i<g_argc; i++) {
if (!strcmp(g_argv[i], "--stderr_string")) {
stderr_string = g_argv[++i];
for (int i=1; i<argc; i++) {
if (is_arg(argv[i], "stderr_string")) {
stderr_string = argv[++i];
found = true;
}
if (!strcmp(g_argv[i], "--reject_if_present")) {
} else if (is_arg(argv[i], "reject_if_present")) {
reject_if_present = true;
}
}
if (!found) {
log_messages.printf(MSG_CRITICAL,
"--stderr_string missing from command line\n"
);
exit(1);
return 1;
}
return 0;
}
void validate_handler_usage() {
// describe the project specific arguments here
fprintf(stderr,
" Custom options:\n"
" --stderr_string X accept task if X is present in stderr_out\n"
" [--reject_if_present] reject (invalidate) the task if X is present\n"
);
}
int init_result(RESULT& r, void*&) {
if (first) {
parse_cmdline();
first = false;
}
if (strstr(r.stderr_out, stderr_string)) {
return reject_if_present?-1:0;
} else {

13
sched/sample_trivial_validator.cpp
View File

@ -19,6 +19,19 @@
#include "validate_util2.h"
int validate_handler_init(int, char**) {
return 0;
}
void validate_handler_usage() {
// describe the project specific arguments here
//fprintf(stderr,
// " Custom options:\n"
// " [--project_option X] a project specific option\n"
//);
}
int init_result(RESULT&, void*&) {
return 0;
}

45
sched/script_validator.cpp
View File

@ -56,42 +56,53 @@
using std::string;
using std::vector;
bool first = true;
vector<string> init_script, compare_script;
// first element is script path, other elements are args
void parse_cmdline() {
for (int i=1; i<g_argc; i++) {
if (!strcmp(g_argv[i], "--init_script")) {
init_script = split(g_argv[++i], ' ');
int validate_handler_init(int argc, char** argv) {
// handle project specific arguments here
for (int i=1; i<argc; i++) {
if (is_arg(argv[i], "init_script")) {
init_script = split(argv[++i], ' ');
if (init_script.size() == 1) {
init_script.push_back(string("files"));
}
} else if (!strcmp(g_argv[i], "--compare_script")) {
compare_script = split(g_argv[++i], ' ');
} else if (is_arg(argv[i], "compare_script")) {
compare_script = split(argv[++i], ' ');
if (compare_script.size() == 1) {
compare_script.push_back("files");
compare_script.push_back("files2");
}
}
}
if (!init_script.size() && !compare_script.size()) {
log_messages.printf(MSG_CRITICAL,
"script names missing from command line\n"
);
exit(1);
return 1;
}
return 0;
}
void validate_handler_usage() {
// describe the project specific arguments here
fprintf(stderr,
" A validator that runs scripts to check and compare results, \n"
" so that you can do your validation in Python, PHP, Perl, bash, etc.\n"
" Custom options:\n"
" --init_script \"scriptname arg1 ... argn\" checks the validity of a task,\n"
" e.g. that the output files have the proper format. Needs to exit with zero if the files are valid.\n"
" --compare_script \"scriptname arg1 ... argn\" compares two tasks. \n"
" Needs to return zero if the output files are equivalent.\n"
" See script_validator.cpp for more usage information.\n"
);
}
int init_result(RESULT& result, void*&) {
unsigned int i, j;
char buf[256];
if (first) {
parse_cmdline();
first = false;
}
if (!init_script.size()) return 0;
vector<string> paths;
int retval;
retval = get_output_file_paths(result, paths);
@ -127,14 +138,6 @@ int compare_results(RESULT& r1, void*, RESULT const& r2, void*, bool& match) {
unsigned int i, j;
char buf[256];
if (first) {
parse_cmdline();
first = false;
}
if (!compare_script.size()) {
match = true;
return 0;
}
vector<string> paths1, paths2;
int retval;
retval = get_output_file_paths(r1, paths1);

3
sched/validate_util.h
View File

@ -53,5 +53,8 @@ extern bool standalone;
// used by validator_test.
extern void remove_random_from_filename(const char* in, char* out);
extern int validate_handler_init(int argc, char** argv);
extern void validate_handler_usage();
#endif

77
sched/validator.cpp
View File

@ -772,6 +772,40 @@ int main_loop() {
return 0;
}
void usage(char* name) {
fprintf(stderr,
"This program is a 'validator'; it handles completed tasks.\n"
"Normally it is run as a daemon from config.xml.\n"
"See: https://boinc.berkeley.edu/trac/wiki/BackendPrograms\n\n"
);
fprintf(stderr, "usage: %s [options]\n"
" Options:\n"
" --app name Process tasks for the given application\n"
" [--one_pass_N_WU N] Validate at most N WUs, then exit\n"
" [--one_pass] Make one pass through WU table, then exit\n"
" [--dry_run] Don't update db, just write logs (for debugging)\n"
" [--mod n i] Process only WUs with (id mod n) == i\n"
" [--max_wu_id n] Process only WUs with id <= n\n"
" [--min_wu_id n] Process only WUs with id >= n\n"
" [--max_granted_credit X] Grant no more than this amount of credit to a result\n"
" [--grant_claimed_credit] Grant the claimed credit, regardless of what other results for this workunit claimed\n"
" [--update_credited_job] Add record to credited_job table after granting credit\n"
" [--credit_from_wu] Credit is specified in WU XML\n"
" [--credit_from_runtime X] Grant credit based on runtime (max X seconds)and estimated FLOPS\n"
" [--no_credit] Don't grant credit\n"
" [--sleep_interval n] Set sleep-interval to n\n"
" [--wu_id n] Process WU with given ID\n"
" [-d level|--debug_level n] Set log verbosity level\n"
" [-h|--help] Print this usage information and exit\n"
" [-v|--version] Print version information and exit\n"
"\n",
name
);
validate_handler_usage();
}
// For use by project-supplied routines check_set() and check_pair()
//
int debug_level=0;
@ -779,30 +813,9 @@ int debug_level=0;
int main(int argc, char** argv) {
int i, retval;
const char *usage =
"\nUsage: %s --app <app-name> [OPTIONS]\n"
"Start validator for application <app-name>\n\n"
"Optional arguments:\n"
" --one_pass_N_WU N Validate at most N WUs, then exit\n"
" --one_pass Make one pass through WU table, then exit\n"
" --dry_run Don't update db, just write logs (for debugging)\n"
" --mod n i Process only WUs with (id mod n) == i\n"
" --max_wu_id n Process only WUs with id <= n\n"
" --min_wu_id n Process only WUs with id >= n\n"
" --max_granted_credit X Grant no more than this amount of credit to a result\n"
" --update_credited_job Add record to credited_job table after granting credit\n"
" --credit_from_wu Credit is specified in WU XML\n"
" --credit_from_runtime X Grant credit based on runtime (max X seconds)and estimated FLOPS\n"
" --no_credit Don't grant credit\n"
" --sleep_interval n Set sleep-interval to n\n"
" --wu_id n Process WU with given ID\n"
" -d n, --debug_level n Set log verbosity level, 1-4\n"
" -h | --help Show this\n"
" -v | --version Show version information\n";
if (argc > 1) {
if (is_arg(argv[1], "h") || is_arg(argv[1], "help")) {
printf (usage, argv[0] );
usage(argv[0]);
exit(0);
} else if (is_arg(argv[1], "v") || is_arg(argv[1], "version")) {
printf("%s\n", SVN_VERSION);
@ -812,6 +825,7 @@ int main(int argc, char** argv) {
check_stop_daemons();
int j=1;
for (i=1; i<argc; i++) {
if (is_arg(argv[i], "one_pass_N_WU")) {
one_pass_N_WU = atoi(argv[++i]);
@ -850,17 +864,16 @@ int main(int argc, char** argv) {
wu_id = atoi(argv[++i]);
one_pass = true;
} else {
//log_messages.printf(MSG_CRITICAL, "unrecognized arg: %s\n", argv[i]);
// unknown arg - pass to handler
argv[j++] = argv[i];
}
}
g_argc = argc;
g_argv = argv;
if (app_name[0] == 0) {
log_messages.printf(MSG_CRITICAL,
"must use '--app' to specify an application\n"
);
printf (usage, argv[0] );
usage(argv[0]);
exit(1);
}
@ -882,10 +895,6 @@ int main(int argc, char** argv) {
exit(1);
}
log_messages.printf(MSG_NORMAL,
"Starting validator, debug level %d\n", log_messages.debug_level
);
if (credit_from_runtime) {
log_messages.printf(MSG_NORMAL,
"using credit from runtime, max runtime: %f\n", max_runtime
@ -908,6 +917,14 @@ int main(int argc, char** argv) {
);
}
argv[j] = 0;
retval = validate_handler_init(j, argv);
if (retval) exit(1);
log_messages.printf(MSG_NORMAL,
"Starting validator, debug level %d\n", log_messages.debug_level
);
install_stop_signal_handler();
main_loop();

3
sched/validator.h
View File

@ -24,6 +24,3 @@ extern WORKUNIT* g_wup;
// A pointer to the WU currently being processed;
// you can access this in your init_result() etc. functions
// (which are passed RESULT but not WORKUNIT)
extern int g_argc;
extern char** g_argv;

42
sched/validator_test.cpp
View File

@ -28,17 +28,25 @@
#include <stdio.h>
#include "svn_version.h"
#include "sched_util_basic.h"
#include "validate_util.h"
#include "validate_util2.h"
int g_argc;
char **g_argv;
void usage(char* prog) {
fprintf(stderr,
"usage: %s file1 file2\n", prog
"This program is a test validator; \n"
"You can test your custom handler with this\n"
);
exit(1);
fprintf(stderr, "usage: %s [options] file1 file2\n"
" Options:\n"
" [-h|--help] Print this usage information and exit\n"
" [-v|--version] Print version information and exit\n"
" file1 Path to file to be compared (must be second to last)\n"
" file2 Path to file to be compared (must be last)\n"
"\n",
prog);
validate_handler_usage();
}
int get_output_file_info(RESULT r, OUTPUT_FILE_INFO& fi) {
@ -47,18 +55,34 @@ int get_output_file_info(RESULT r, OUTPUT_FILE_INFO& fi) {
}
int main(int argc, char** argv) {
if (argc != 3) {
int retval;
if (argc > 1) {
if (is_arg(argv[1], "h") || is_arg(argv[1], "help")) {
usage(argv[0]);
exit(0);
} else if (is_arg(argv[1], "v") || is_arg(argv[1], "version")) {
printf("%s\n", SVN_VERSION);
exit(0);
}
}
if (argc < 3) {
usage(argv[0]);
exit(0);
}
retval = validate_handler_init(argc, argv);
if (retval) exit(1);
void* data1, *data2;
RESULT r1, r2;
bool match;
standalone = true;
sprintf(r1.xml_doc_in, "<file_ref><file_name>%s</file_name></file_ref>", argv[1]);
sprintf(r2.xml_doc_in, "<file_ref><file_name>%s</file_name></file_ref>", argv[2]);
int retval;
sprintf(r1.xml_doc_in, "<file_ref><file_name>%s</file_name></file_ref>", argv[argc-2]);
sprintf(r2.xml_doc_in, "<file_ref><file_name>%s</file_name></file_ref>", argv[argc-1]);
retval = init_result(r1, data1);
if (retval) {
fprintf(stderr, "init_result(r1) returned %d\n", retval);