diff --git a/checkin_notes b/checkin_notes
index 2d3ff40a38..a38c05776b 100755
--- a/checkin_notes
+++ b/checkin_notes
@@ -15483,11 +15483,6 @@ Rom July 19 2004
Karl 2004-07-21
- fix bug in "couldn't find app" messages
-
- sched/
- message_handler.C
- validator.C
-
- allow '.msi' as extension for installation package
tools/
@@ -15502,3 +15497,42 @@ Karl 2004-07-21
old_news.php
project.sample/
project_news.inc
+
+ sched/
+ message_handler.C
+ validator.C
+
+David 21 July 2004
+ - Database optimization in the transitioner:
+ add several results in a single SQL insert statement
+ (using values (...),(...),(...))
+ instead of separate inserts.
+ This is enabled by a BATCH_INSERT compile flag in transitioner.C
+ - Add a USE_TRANSACTIONS flag to transitioner.
+ Leave it off for now.
+ - For some reason we weren't processing the UPLOAD_URL symbol
+ in result templates.
+ Restore this, using the info in the config file.
+ - transitioner: if anything fails (result creation, WU update), quit.
+ - result.random must be initialize by caller on insert
+ (not too relevant since we don't use it anymore)
+ - start/commit_transaction are now members of DB_CONN
+ rather than DB_BASE_SPECIAL
+ - scheduler: in scan_work_array, do all "cheap" checks
+ (i.e. those that don't require DB access)
+ before releasing the semaphore.
+ Thus if all the results are infeasible,
+ we'll acquire/release the semaphore once, not 100 times.
+ - re-enable team name search
+
+ db/
+ boinc_db.C,h
+ db_base.C,h
+ html/user/
+ team.php
+ team_lookup.php
+ sched/
+ sched_send.C
+ transitioner.C
+ tools/
+ backend_lib.C,h
diff --git a/db/boinc_db.C b/db/boinc_db.C
index 836a32e1c5..36718625da 100644
--- a/db/boinc_db.C
+++ b/db/boinc_db.C
@@ -492,6 +492,34 @@ void DB_RESULT::db_print(char* buf){
UNESCAPE(stderr_out);
}
+void DB_RESULT::db_print_values(char* buf){
+ ESCAPE(xml_doc_out);
+ ESCAPE(stderr_out);
+ sprintf(
+ buf,
+ "(0, %d, %d, "
+ "%d, %d, %d, "
+ "%d, %d, "
+ "%d, %d, %d, "
+ "'%s', %.15e, "
+ "'%s', '%s', '%s', "
+ "%d, %d, %d, "
+ "%.15e, %.15e, %f, %d, "
+ "%d, %d, %d, %d)",
+ create_time, workunitid,
+ server_state, outcome, client_state,
+ hostid, userid,
+ report_deadline, sent_time, received_time,
+ name, cpu_time,
+ xml_doc_in, xml_doc_out, stderr_out,
+ batch, file_delete_state, validate_state,
+ claimed_credit, granted_credit, opaque, random,
+ app_version_num, appid, exit_status, teamid
+ );
+ UNESCAPE(xml_doc_out);
+ UNESCAPE(stderr_out);
+}
+
void DB_RESULT::db_parse(MYSQL_ROW &r) {
int i=0;
clear();
@@ -524,11 +552,6 @@ void DB_RESULT::db_parse(MYSQL_ROW &r) {
teamid = atoi(r[i++]);
}
-int DB_RESULT::insert() {
- random = lrand48();
- return DB_BASE::insert();
-}
-
void DB_MSG_FROM_HOST::db_print(char* buf) {
ESCAPE(xml);
sprintf(buf,
diff --git a/db/boinc_db.h b/db/boinc_db.h
index 48dfcea4c2..c654c5d0ce 100755
--- a/db/boinc_db.h
+++ b/db/boinc_db.h
@@ -533,9 +533,9 @@ public:
class DB_RESULT : public DB_BASE, public RESULT {
public:
DB_RESULT();
- int insert();
int get_id();
void db_print(char*);
+ void db_print_values(char*);
void db_parse(MYSQL_ROW &row);
void operator=(RESULT& r) {RESULT::operator=(r);}
};
diff --git a/db/db_base.C b/db/db_base.C
index b138f72fcb..c0162b3035 100644
--- a/db/db_base.C
+++ b/db/db_base.C
@@ -60,6 +60,14 @@ const char* DB_CONN::error_string() {
return mysql?mysql_error(mysql):"Not connected";
}
+int DB_CONN::start_transaction() {
+ return do_query("START TRANSACTION");
+}
+
+int DB_CONN::commit_transaction() {
+ return do_query("COMMIT");
+}
+
DB_BASE::DB_BASE(DB_CONN& p, char *tn) : db(&p), table_name(tn) {
is_high_priority = false;
}
@@ -76,6 +84,12 @@ int DB_BASE::insert() {
return db->do_query(query);
}
+int DB_BASE::insert_batch(const char* values) {
+ char query[MAX_QUERY_LEN];
+ sprintf(query, "insert into %s values %s", table_name, values);
+ return db->do_query(query);
+}
+
// update an entire record
//
int DB_BASE::update() {
@@ -253,17 +267,9 @@ int DB_BASE::sum(double& x, char* field, char* clause) {
DB_BASE_SPECIAL::DB_BASE_SPECIAL(DB_CONN& p) : db(&p) {
}
-int DB_BASE_SPECIAL::start_transaction() {
- return db->do_query("START TRANSACTION");
-}
-
-int DB_BASE_SPECIAL::commit_transaction() {
- return db->do_query("COMMIT");
-}
-
// convert a string into a form that allows it to be used
// in SQL queries delimited by single quotes:
-// replace ' with \', \ with \\
+// replace ' with \', '\' with '\\'
//
void escape_string(char* field, int len) {
char buf[MAX_QUERY_LEN];
diff --git a/db/db_base.h b/db/db_base.h
index 1066cae6c7..5a0d2dc9b8 100644
--- a/db/db_base.h
+++ b/db/db_base.h
@@ -68,6 +68,8 @@ public:
const char* error_string();
MYSQL* mysql;
+ int start_transaction();
+ int commit_transaction();
};
// Base for derived classes that can access the DB
@@ -77,6 +79,7 @@ class DB_BASE {
public:
DB_BASE(DB_CONN&, char *table_name);
int insert();
+ int insert_batch(const char*);
int update();
int update_field(char*);
int lookup_id(int id);
@@ -106,9 +109,6 @@ public:
DB_CONN* db;
CURSOR cursor;
-
- int start_transaction();
- int commit_transaction();
};
void escape_string(char* field, int len);
diff --git a/html/user/team.php b/html/user/team.php
index 719fb4ec34..59584ede42 100644
--- a/html/user/team.php
+++ b/html/user/team.php
@@ -23,7 +23,7 @@ echo "".PROJECT." participants may form teams.
diff --git a/html/user/team_lookup.php b/html/user/team_lookup.php
index 30da1398dc..2977b18cee 100644
--- a/html/user/team_lookup.php
+++ b/html/user/team_lookup.php
@@ -12,14 +12,7 @@
$length = count($words);
$name_lc = strtolower($team_name);
- echo "Team lookup disabled";
- page_tail();
- exit();
-
- $query = sprintf(
- "select * from team where name_lc like '%s'",
- "%$name_lc%"
- );
+ $query = "select * from team where name like '$name_lc%'";
$result_list = mysql_query($query);
page_head("Search Results");
if ($result_list) {
diff --git a/sched/sched_send.C b/sched/sched_send.C
index de08a27785..98c743325d 100644
--- a/sched/sched_send.C
+++ b/sched/sched_send.C
@@ -490,47 +490,44 @@ static void scan_work_array(
WU_RESULT& wu_result = ss.wu_results[i];
- // the following should be a critical section
- //
+ // do fast checks on this wu_result;
+ // i.e. ones that don't require DB access
+ // if any check fails, continue
+
switch (wu_result.state) {
case WR_STATE_EMPTY:
case WR_STATE_CHECKED_OUT:
continue;
}
- wu_result.state = WR_STATE_CHECKED_OUT;
- unlock_sema();
- // from here on in this loop, don't continue;
- // you can only goto dont_send (so that we reacquire semaphore)
+ if (wreq.infeasible_only && (wu_result.infeasible_count==0)) {
+ continue;
+ }
if (wu_result.workunit.rsc_disk_bound > wreq.disk_available) {
wreq.insufficient_disk = true;
wu_result.infeasible_count++;
- goto dont_send;
- }
-
- if (wreq.infeasible_only && (wu_result.infeasible_count==0)) {
- goto dont_send;
+ continue;
}
// don't send if we're already sending a result for same WU
//
if (already_in_reply(wu_result, reply)) {
- goto dont_send;
+ continue;
}
// don't send if host can't handle it
//
wu = wu_result.workunit;
- if (!wu_is_feasible(wu, reply.host, wreq,
- sreq.resource_share_fraction)
- ) {
+ if (!wu_is_feasible(
+ wu, reply.host, wreq, sreq.resource_share_fraction
+ )) {
log_messages.printf(
SCHED_MSG_LOG::DEBUG, "[HOST#%d] [WU#%d %s] WU is infeasible\n",
reply.host.id, wu.id, wu.name
);
wu_result.infeasible_count++;
- goto dont_send;
+ continue;
}
// Find the app and app_version for the client's platform.
@@ -540,14 +537,14 @@ static void scan_work_array(
app = ss.lookup_app(wu.appid);
found = sreq.has_version(*app);
if (!found) {
- goto dont_send;
+ continue;
}
avp = NULL;
} else {
found = find_app_version(wreq, wu, platform, ss, app, avp);
if (!found) {
wu_result.infeasible_count++;
- goto dont_send;
+ continue;
}
// see if the core client is too old.
@@ -555,12 +552,18 @@ static void scan_work_array(
// isn't the result's fault
//
if (!app_core_compatible(wreq, *avp)) {
- goto dont_send;
+ continue;
}
}
+ // end of fast checks - mark wu_result as checked out and release sema.
+ // from here on in this loop, don't continue on failure;
+ // instead, goto dont_send (so that we reacquire semaphore)
+
+ wu_result.state = WR_STATE_CHECKED_OUT;
+ unlock_sema();
+
// Don't send if we've already sent a result of this WU to this user.
- // NOTE: do this check last since it involves a DB access
//
if (config.one_result_per_user_per_wu) {
sprintf(buf,
diff --git a/sched/transitioner.C b/sched/transitioner.C
index ad0a421e30..a8d87c2dd4 100644
--- a/sched/transitioner.C
+++ b/sched/transitioner.C
@@ -47,6 +47,9 @@ using namespace std;
#define SELECT_LIMIT 100
+#define BATCH_INSERT 1
+//#define USE_TRANSACTIONS 1
+
int startup_time;
SCHED_CONFIG config;
R_RSA_PRIVATE_KEY key;
@@ -59,7 +62,7 @@ int result_suffix(char* name) {
return 0;
}
-void handle_wu(
+int handle_wu(
DB_TRANSITIONER_ITEM_SET& transitioner,
std::vector& items
) {
@@ -226,8 +229,11 @@ void handle_wu(
}
} else if (items[0].assimilate_state == ASSIMILATE_INIT) {
// If no error, generate new results if needed.
- // NOTE!! `n' must be a SIGNED integer!
+ // NOTE: n must be signed
+ //
int n = items[0].target_nresults - nunsent - ninprogress - nsuccess;
+ string values;
+ char value_buf[MAX_QUERY_LEN];
if (n > 0) {
log_messages.printf(
SCHED_MSG_LOG::NORMAL,
@@ -238,9 +244,10 @@ void handle_wu(
sprintf(suffix, "%d", max_result_suffix+i+1);
char rtfpath[256];
sprintf(rtfpath, "../%s", items[0].result_template_file);
+#ifdef BATCH_INSERT
retval = create_result(
items[0].id, items[0].appid, items[0].name,
- rtfpath, suffix, key, config.upload_url
+ rtfpath, suffix, key, config.upload_url, value_buf
);
if (retval) {
log_messages.printf(
@@ -248,9 +255,41 @@ void handle_wu(
"[WU#%d %s] create_result() %d\n",
items[0].id, items[0].name, retval
);
- break;
+ return retval;
}
+ if (i==0) {
+ values = value_buf;
+ } else {
+ values += ",";
+ values += value_buf;
+ }
+#else
+ retval = create_result(
+ items[0].id, items[0].appid, items[0].name,
+ rtfpath, suffix, key, config.upload_url, 0
+ );
+ if (retval) {
+ log_messages.printf(
+ SCHED_MSG_LOG::CRITICAL,
+ "[WU#%d %s] create_result() %d\n",
+ items[0].id, items[0].name, retval
+ );
+ return retval;
+ }
+#endif
}
+#ifdef BATCH_INSERT
+ DB_RESULT r;
+ retval = r.insert_batch(values.c_str());
+ if (retval) {
+ log_messages.printf(
+ SCHED_MSG_LOG::CRITICAL,
+ "[WU#%d %s] insert_batch() %d\n",
+ items[0].id, items[0].name, retval
+ );
+ return retval;
+ }
+#endif
}
}
@@ -359,7 +398,9 @@ void handle_wu(
SCHED_MSG_LOG::CRITICAL,
"[WU#%d %s] workunit.update() == %d\n", items[0].id, items[0].name, retval
);
+ return retval;
}
+ return 0;
}
bool do_pass() {
@@ -375,12 +416,13 @@ bool do_pass() {
while (!transitioner.enumerate((int)time(0), mod_n, mod_i, SELECT_LIMIT, items)) {
did_something = true;
+#ifdef USE_TRANSACTIONS
log_messages.printf(
SCHED_MSG_LOG::DEBUG,
"[WU#%d %s] Starting Transaction...\n",
items[0].id, items[0].name
- );
- retval = transitioner.start_transaction();
+ );
+ retval = boinc_db.start_transaction();
if (retval) {
log_messages.printf(
SCHED_MSG_LOG::CRITICAL,
@@ -388,15 +430,25 @@ bool do_pass() {
items[0].id, items[0].name, retval
);
}
+#endif
- handle_wu(transitioner, items);
+ retval = handle_wu(transitioner, items);
+ if (retval) {
+ log_messages.printf(
+ SCHED_MSG_LOG::CRITICAL,
+ "[WU#%d %s] handle_wu: %d; quitting\n",
+ items[0].id, items[0].name, retval
+ );
+ exit(1);
+ }
+#ifdef USE_TRANSACTIONS
log_messages.printf(
SCHED_MSG_LOG::DEBUG,
"[WU#%d %s] Committing Transaction...\n",
items[0].id, items[0].name
);
- retval = transitioner.commit_transaction();
+ retval = boinc_db.commit_transaction();
if (retval) {
log_messages.printf(
SCHED_MSG_LOG::CRITICAL,
@@ -410,6 +462,7 @@ bool do_pass() {
items[0].id, items[0].name
);
}
+#endif
check_stop_daemons();
}
diff --git a/tools/backend_lib.C b/tools/backend_lib.C
index 2c14175b5f..8ee153c9b3 100644
--- a/tools/backend_lib.C
+++ b/tools/backend_lib.C
@@ -22,6 +22,7 @@
#include
#include
#include
+#include
#include "boinc_db.h"
#include "crypt.h"
@@ -36,6 +37,12 @@
#include "fcgi_stdio.h"
#endif
+static struct random_init {
+ random_init() {
+ srand48(getpid() + time(0));
+ }
+} random_init;
+
int read_file(FILE* f, char* buf, int len) {
int n = fread(buf, 1, len, f);
buf[n] = 0;
@@ -212,7 +219,9 @@ int create_result(
char* result_template_filename,
char* result_name_suffix,
R_RSA_PRIVATE_KEY& key,
- char* upload_url
+ char* upload_url,
+ char* query_string
+ // if nonzero, write value list here; else do insert
) {
DB_RESULT result;
char base_outfile_name[256];
@@ -226,8 +235,10 @@ int create_result(
retval = read_filename(result_template_filename, result_template, sizeof(result_template));
if (retval) {
- fprintf(stderr, "Failed to read result template file '%s': %d\n",
- result_template_filename, retval);
+ fprintf(stderr,
+ "Failed to read result template file '%s': %d\n",
+ result_template_filename, retval
+ );
return retval;
}
@@ -246,12 +257,16 @@ int create_result(
}
safe_strncpy(result.xml_doc_in, result_template, sizeof(result.xml_doc_in));
- // NOTE: result::insert() sets random
+ result.random = lrand48();
- retval = result.insert();
- if (retval) {
- fprintf(stderr, "result.insert(): %d\n", retval);
- return retval;
+ if (query_string) {
+ result.db_print_values(query_string);
+ } else {
+ retval = result.insert();
+ if (retval) {
+ fprintf(stderr, "result.insert(): %d\n", retval);
+ return retval;
+ }
}
return 0;
}
diff --git a/tools/backend_lib.h b/tools/backend_lib.h
index 648f9401dd..a28eda88ac 100644
--- a/tools/backend_lib.h
+++ b/tools/backend_lib.h
@@ -45,7 +45,8 @@ extern int create_result(
char* result_template_filename,
char* suffix,
R_RSA_PRIVATE_KEY& key,
- char* upload_url
+ char* upload_url,
+ char* query_string=0
);
extern int create_work(