diff --git a/db/boinc_db.cpp b/db/boinc_db.cpp index 77f9c9951f..44c884d782 100644 --- a/db/boinc_db.cpp +++ b/db/boinc_db.cpp @@ -901,7 +901,8 @@ void DB_WORKUNIT::db_print(char* buf){ "fileset_id=%lu, " "app_version_id=%ld, " "transitioner_flags=%d, " - "size_class=%d ", + "size_class=%d, " + "keywords='%s' ", create_time, appid, name, xml_doc, batch, rsc_fpops_est, rsc_fpops_bound, rsc_memory_bound, rsc_disk_bound, @@ -921,7 +922,8 @@ void DB_WORKUNIT::db_print(char* buf){ fileset_id, app_version_id, transitioner_flags, - size_class + size_class, + keywords ); } @@ -944,7 +946,8 @@ void DB_WORKUNIT::db_print_values(char* buf) { "%lu, " "%ld, " "%d, " - "%d)", + "%d, " + "'%s')", create_time, appid, name, xml_doc, batch, rsc_fpops_est, rsc_fpops_bound, @@ -965,7 +968,8 @@ void DB_WORKUNIT::db_print_values(char* buf) { fileset_id, app_version_id, transitioner_flags, - size_class + size_class, + keywords ); } @@ -1005,6 +1009,7 @@ void DB_WORKUNIT::db_parse(MYSQL_ROW &r) { app_version_id = atol(r[i++]); transitioner_flags = atoi(r[i++]); size_class = atoi(r[i++]); + strcpy2(keywords, r[i++]); } void DB_CREDITED_JOB::db_print(char* buf){ @@ -2020,6 +2025,7 @@ void WORK_ITEM::parse(MYSQL_ROW& r) { wu.app_version_id = atol(r[i++]); wu.transitioner_flags = atoi(r[i++]); wu.size_class = atoi(r[i++]); + strcpy2(wu.keywords, r[i++]); } int DB_WORK_ITEM::enumerate( diff --git a/db/boinc_db.h b/db/boinc_db.h index 3d016981d1..86ace2a764 100644 --- a/db/boinc_db.h +++ b/db/boinc_db.h @@ -18,7 +18,14 @@ #ifndef _BOINC_DB_ #define _BOINC_DB_ -// Structures corresponding to database records. +// Structures passed to and from DB queries. +// +// Mostly these correspond to DB tables, and inherit structs +// defined in boinc_db_types.h +// But some of them - TRANSITIONER_ITEM, STATE_COUNTS, SCHED_RESULT_ITEM, etc. - +// combine the info from multiple tables (from joins) +// or have subsets of table data. +// // Some of these types have counterparts in client/types.h, // but don't be deceived - client and server have different variants. diff --git a/db/boinc_db_types.h b/db/boinc_db_types.h index 03795fb585..c129121ec9 100644 --- a/db/boinc_db_types.h +++ b/db/boinc_db_types.h @@ -480,9 +480,9 @@ struct WORKUNIT { int size_class; // -1 means none; encode this here so that transitioner // doesn't have to look up app + char keywords[256]; + // keywords, as space-separated integers - // the following not used in the DB - char app_name[256]; void clear(); WORKUNIT(){clear();} }; diff --git a/db/schema.sql b/db/schema.sql index 7ca2760ed8..404a633137 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -256,6 +256,7 @@ create table workunit ( app_version_id integer not null, transitioner_flags tinyint not null, size_class smallint not null default -1, + keywords varchar(254) not null, primary key (id) ) engine=InnoDB; diff --git a/html/ops/db_update.php b/html/ops/db_update.php index 8a2c960aa8..e0414d7c30 100644 --- a/html/ops/db_update.php +++ b/html/ops/db_update.php @@ -1049,6 +1049,12 @@ function update_6_13_2017() { "); } +function update_7_21_2017() { + do_query("alter table workunit + add column keywords varchar(254) not null + "); +} + // Updates are done automatically if you use "upgrade". // // If you need to do updates manually, @@ -1098,6 +1104,7 @@ $db_updates = array ( array(27015, "update_2_17_2017"), array(27016, "update_3_17_2017"), array(27017, "update_6_13_2017"), + array(27018, "update_7_21_2017"), ); ?> diff --git a/lib/keyword.cpp b/lib/keyword.cpp index b08fd8265c..3abe47cbb8 100644 --- a/lib/keyword.cpp +++ b/lib/keyword.cpp @@ -18,7 +18,6 @@ // utility functions for keywords #include -#include #include "parse.h" #include "keyword.h" @@ -54,15 +53,13 @@ void USER_KEYWORDS::write(FILE* f) { fprintf(f, "\n"); } -double keyword_score(USER_KEYWORDS& user_keywords, JOB_KEYWORDS& job_keywords) { - double score = 0; - for (unsigned int i=0; i #include "parse.h" @@ -24,12 +27,12 @@ struct USER_KEYWORDS { std::vector yes; std::vector no; int parse(XML_PARSER&); - void clear() { + inline void clear() { yes.clear(); no.clear(); } void write(FILE*); - bool empty() { + inline bool empty() { return yes.empty() && no.empty(); } }; @@ -38,6 +41,12 @@ struct JOB_KEYWORDS { std::vector ids; void parse_str(char*); // parse space-separated list + inline bool empty() { + return ids.empty(); + } + inline void clear() { + ids.clear(); + } }; -extern double keyword_score(USER_KEYWORDS&, JOB_KEYWORDS&); +#endif diff --git a/lib/str_util.h b/lib/str_util.h index 840b2870b1..91751e3965 100644 --- a/lib/str_util.h +++ b/lib/str_util.h @@ -42,6 +42,10 @@ extern char* precision_time_to_string(double); extern void secs_to_hmsf(double, char*); extern std::string timediff_format(double); +inline bool empty(char* p) { + return p[0] == 0; +} + inline bool ends_with(std::string const& s, std::string const& suffix) { return s.size()>=suffix.size() && diff --git a/sched/Makefile.am b/sched/Makefile.am index ccb66c2c4a..8de97c9f39 100644 --- a/sched/Makefile.am +++ b/sched/Makefile.am @@ -167,6 +167,7 @@ cgi_sources = \ sched_customize.cpp \ sched_files.cpp \ sched_hr.cpp \ + sched_keyword.cpp \ sched_limit.cpp \ sched_locality.cpp \ sched_main.cpp \ diff --git a/sched/sched_check.cpp b/sched/sched_check.cpp index a34a6e398e..b1ae195b8d 100644 --- a/sched/sched_check.cpp +++ b/sched/sched_check.cpp @@ -375,9 +375,10 @@ int wu_is_infeasible_fast( // Do checks that require DB access for whether we can send this job, // and return: -// 0 if OK to send -// 1 if can't send to this host -// 2 if can't send to ANY host +// CHECK_OK if OK to send +// CHECK_NO_HOST if can't send to this host +// CHECK_NO_ANY if can't send to ANY host +// e.g. WU error mask is nonzero // int slow_check( WU_RESULT& wu_result, // the job cache entry. @@ -402,7 +403,7 @@ int slow_check( log_messages.printf(MSG_CRITICAL, "send_work: can't get result count (%s)\n", boincerror(retval) ); - return 1; + return CHECK_NO_HOST; } else { if (n>0) { if (config.debug_send_job) { @@ -411,7 +412,7 @@ int slow_check( g_reply->user.id, n, wu.id ); } - return 1; + return CHECK_NO_HOST; } } } else if (config.one_result_per_host_per_wu) { @@ -426,7 +427,7 @@ int slow_check( log_messages.printf(MSG_CRITICAL, "send_work: can't get result count (%s)\n", boincerror(retval) ); - return 1; + return CHECK_NO_HOST; } else { if (n>0) { if (config.debug_send_job) { @@ -435,7 +436,7 @@ int slow_check( g_reply->host.id, n, wu.id ); } - return 1; + return CHECK_NO_HOST; } } } @@ -454,13 +455,13 @@ int slow_check( log_messages.printf(MSG_CRITICAL, "can't get fields for [WU#%lu]: %s\n", db_wu.id, boincerror(retval) ); - return 1; + return CHECK_NO_HOST; } // check wu.error_mask // if (vals[2] != 0) { - return 2; + return CHECK_NO_ANY; } if (app_hr_type(*app)) { @@ -477,7 +478,7 @@ int slow_check( // are processed first. // wu_result.infeasible_count++; - return 1; + return CHECK_NO_HOST; } } if (app->homogeneous_app_version) { @@ -491,11 +492,11 @@ int slow_check( ); } wu_result.infeasible_count++; - return 1; + return CHECK_NO_HOST; } } } - return 0; + return CHECK_OK; } // Check for pathological conditions that mean diff --git a/sched/sched_check.h b/sched/sched_check.h index 59fca0f165..bea673c9a2 100644 --- a/sched/sched_check.h +++ b/sched/sched_check.h @@ -22,6 +22,8 @@ #include "sched_shmem.h" #include "sched_types.h" +// return values from wu_is_infeasible_fast() +// #define INFEASIBLE_MEM 1 #define INFEASIBLE_DISK 2 #define INFEASIBLE_CPU 3 @@ -42,7 +44,15 @@ extern int wu_is_infeasible_fast( int res_server_state, int res_priority, double res_report_deadline, APP&, BEST_APP_VERSION& ); + +// return values from slow_check() +// +#define CHECK_OK 0 +#define CHECK_NO_HOST 1 +#define CHECK_NO_ANY 2 + extern int slow_check(WU_RESULT&, APP*, BEST_APP_VERSION*); + extern bool result_still_sendable(DB_RESULT& result, WORKUNIT& wu); extern bool app_not_selected(int appid); diff --git a/sched/sched_config.cpp b/sched/sched_config.cpp index 6a7bb4f69d..7b7c595fbd 100644 --- a/sched/sched_config.cpp +++ b/sched/sched_config.cpp @@ -298,6 +298,7 @@ int SCHED_CONFIG::parse(FILE* f) { if (xp.parse_double("version_select_random_factor", version_select_random_factor)) continue; if (xp.parse_double("maintenance_delay", maintenance_delay)) continue; if (xp.parse_bool("credit_by_app", credit_by_app)) continue; + if (xp.parse_bool("keyword_sched", keyword_sched)) continue; //////////// SCHEDULER LOG FLAGS ///////// @@ -308,6 +309,7 @@ int SCHED_CONFIG::parse(FILE* f) { if (xp.parse_bool("debug_edf_sim_workload", debug_edf_sim_workload)) continue; if (xp.parse_bool("debug_fcgi", debug_fcgi)) continue; if (xp.parse_bool("debug_handle_results", debug_handle_results)) continue; + if (xp.parse_bool("debug_keyword", debug_keyword)) continue; if (xp.parse_bool("debug_locality", debug_locality)) continue; if (xp.parse_bool("debug_locality_lite", debug_locality_lite)) continue; if (xp.parse_bool("debug_prefs", debug_prefs)) continue; diff --git a/sched/sched_config.h b/sched/sched_config.h index 49385e6abf..75757db866 100644 --- a/sched/sched_config.h +++ b/sched/sched_config.h @@ -180,6 +180,8 @@ struct SCHED_CONFIG { // to calculate projected_flops when choosing version. bool credit_by_app; // store per-app credit info in credit_user and credit_team + bool keyword_sched; + // score jobs based on keywords // time intervals double maintenance_delay; @@ -194,6 +196,7 @@ struct SCHED_CONFIG { bool debug_fcgi; bool debug_client_files; // stuff related to sticky files on client bool debug_handle_results; + bool debug_keyword; bool debug_locality; // locality scheduling bool debug_locality_lite; // locality scheduling Lite bool debug_prefs; diff --git a/sched/sched_keyword.cpp b/sched/sched_keyword.cpp index 9bb4a4ffe1..6ef935dc84 100644 --- a/sched/sched_keyword.cpp +++ b/sched/sched_keyword.cpp @@ -16,23 +16,32 @@ // along with BOINC. If not, see . // code related to keyword-based job scoring +// +// A job's keywords are stored in workunit.keywords as a char string. +// We don't want to parse that every time we score the job, +// so we maintain a list of JOB_KEYWORDs paralleling the job array. #include +#include #include "sched_main.h" #include "keyword.h" JOB_KEYWORDS *job_keywords_array; +// compute the score increment for the given job and user keywords +// (or -1 if the keywords are incompatible) +// double keyword_score_aux( - USER_KEYWORDS& user_keywords, JOB_KEYWORDS& job_keywords + USER_KEYWORDS& uks, JOB_KEYWORDS& jks ) { double score = 0; - for (unsigned int i=0; imax_wu_results]; } diff --git a/sched/sched_keyword.h b/sched/sched_keyword.h index ed61fb14d1..5814e575b1 100644 --- a/sched/sched_keyword.h +++ b/sched/sched_keyword.h @@ -18,7 +18,11 @@ #ifndef BOINC_SCHED_KEYWORD_H #define BOINC_SCHED_KEYWORD_H +// see sched_keywork.cpp + extern double keyword_score(int); extern void keyword_sched_remove_job(int index); +extern void keyword_sched_init(); + #endif diff --git a/sched/sched_main.cpp b/sched/sched_main.cpp index c5584df507..98efcd09fc 100644 --- a/sched/sched_main.cpp +++ b/sched/sched_main.cpp @@ -61,6 +61,7 @@ #include "handle_request.h" #include "sched_config.h" #include "sched_files.h" +#include "sched_keyword.h" #include "sched_msgs.h" #include "sched_types.h" #include "sched_util.h" @@ -543,6 +544,9 @@ int main(int argc, char** argv) { send_message("Server error: can't attach shared memory", config.maintenance_delay); goto done; } + if (config.keyword_sched) { + keyword_sched_init(); + } if (strlen(config.debug_req_reply_dir)) { struct stat statbuf; diff --git a/sched/sched_score.cpp b/sched/sched_score.cpp index dea57c0987..91c14697fa 100644 --- a/sched/sched_score.cpp +++ b/sched/sched_score.cpp @@ -31,6 +31,7 @@ #include "sched_check.h" #include "sched_config.h" #include "sched_hr.h" +#include "sched_keyword.h" #include "sched_main.h" #include "sched_msgs.h" #include "sched_send.h" @@ -54,7 +55,8 @@ static int get_size_class(APP& app, double es) { // Also do some initial screening, // and return false if can't send the job to host // -bool JOB::get_score(WU_RESULT& wu_result) { +bool JOB::get_score(int array_index) { + WU_RESULT& wu_result = ssp->wu_results[array_index]; score = 0; if (!app->beta && wu_result.need_reliable) { @@ -130,6 +132,14 @@ bool JOB::get_score(WU_RESULT& wu_result) { score += wu_result.res_priority; + if (config.keyword_sched) { + double x = keyword_score(array_index); + if (x < 0) { + return false; + } + score += x; + } + if (config.debug_send_job) { log_messages.printf(MSG_NORMAL, "[send_job]: score %f for result %lu\n", score, wu_result.resultid @@ -215,7 +225,7 @@ void send_work_score_type(int rt) { job.index = i; job.result_id = wu_result.resultid; - if (!job.get_score(wu_result)) { + if (!job.get_score(i)) { if (config.debug_send_job) { log_messages.printf(MSG_NORMAL, "[send_job] [RESULT#%lu] get_score() returned false\n", @@ -298,11 +308,14 @@ void send_work_score_type(int rt) { sema_locked = false; switch (slow_check(wu_result, job.app, job.bavp)) { - case 1: + case CHECK_NO_HOST: wu_result.state = WR_STATE_PRESENT; break; - case 2: + case CHECK_NO_ANY: wu_result.state = WR_STATE_EMPTY; + if (config.keyword_sched) { + keyword_sched_remove_job(job.index); + } break; default: // slow_check() refreshes fields of wu_result.workunit; @@ -315,6 +328,7 @@ void send_work_score_type(int rt) { // (since otherwise feeder might overwrite it) // wu_result.state = WR_STATE_EMPTY; + keyword_sched_remove_job(job.index); // reread result from DB, make sure it's still unsent // TODO: from here to end of add_result_to_reply() diff --git a/sched/sched_score.h b/sched/sched_score.h index 4a05857429..693a55ac5d 100644 --- a/sched/sched_score.h +++ b/sched/sched_score.h @@ -24,7 +24,7 @@ struct JOB { APP* app; BEST_APP_VERSION* bavp; - bool get_score(WU_RESULT&); + bool get_score(int); }; extern void send_work_score(); diff --git a/sched/sched_types.cpp b/sched/sched_types.cpp index 0dd654100f..12b26b0899 100644 --- a/sched/sched_types.cpp +++ b/sched/sched_types.cpp @@ -1373,6 +1373,7 @@ int HOST::parse(XML_PARSER& xp) { p_ncpus = 1; double dtemp; string stemp; + int x; while (!xp.get_tag()) { if (xp.match_tag("/host_info")) return 0; if (xp.parse_int("timezone", timezone)) continue; @@ -1404,6 +1405,10 @@ int HOST::parse(XML_PARSER& xp) { continue; } + // unused fields + // + if (xp.parse_int("n_usable_coprocs", x)) continue; + // parse deprecated fields to avoid error messages // if (xp.parse_double("p_calculated", dtemp)) continue; diff --git a/tools/create_work.cpp b/tools/create_work.cpp index 028e13bf62..c443b3c984 100644 --- a/tools/create_work.cpp +++ b/tools/create_work.cpp @@ -62,6 +62,7 @@ void usage() { " [ -d n ]\n" " [ --delay_bound x ]\n" " [ --hr_class n ]\n" + " [ --keywords 'n1 n2 ...' ]\n" " [ --max_error_results n ]\n" " [ --max_success_results n ]\n" " [ --max_total_results n ]\n" @@ -336,6 +337,8 @@ int main(int argc, char** argv) { verbose = true; } else if (arg(argv, i, "continue_on_error")) { continue_on_error = true; + } else if (arg(argv, i, "keywords")) { + strcpy(jd.wu.keywords, argv[++i]); } else { if (!strncmp("-", argv[i], 1)) { fprintf(stderr, "create_work: bad argument '%s'\n", argv[i]);