From 0038d275cd55ce414f92656db93e3b9d993458a0 Mon Sep 17 00:00:00 2001
From: Christian Beer <christian.beer@aei.mpg.de>
Date: Thu, 8 Oct 2015 12:14:45 +0200
Subject: [PATCH 1/4] Daemons: enhance assimilator framework The assimilator
 handler can now pass unknown arguments to the project specific handler.
 Projects that have there own assimilators need to implement the
 assimilate_handler_init() function and handle project specific arguments
 there. For examples see sample_assimilator.cpp or script_assimilator.cpp I
 also added some checks for missing arguments and clarified some error
 messages.

---
 sched/assimilate_handler.h         |  7 +--
 sched/assimilator.cpp              | 89 ++++++++++++++++++++----------
 sched/sample_assimilator.cpp       |  8 ++-
 sched/sample_dummy_assimilator.cpp |  7 ++-
 sched/script_assimilator.cpp       | 21 +++----
 sched/single_job_assimilator.cpp   |  7 ++-
 6 files changed, 89 insertions(+), 50 deletions(-)

diff --git a/sched/assimilate_handler.h b/sched/assimilate_handler.h
index ab602369d3..56a15cde12 100644
--- a/sched/assimilate_handler.h
+++ b/sched/assimilate_handler.h
@@ -1,6 +1,6 @@
 // This file is part of BOINC.
 // http://boinc.berkeley.edu
-// Copyright (C) 2008 University of California
+// Copyright (C) 2015 University of California
 //
 // BOINC is free software; you can redistribute it and/or modify it
 // under the terms of the GNU Lesser General Public License
@@ -34,7 +34,4 @@ extern int assimilate_handler(
     RESULT&                 // the canonical instance
 );
 
-extern int g_argc;
-extern char** g_argv;
-extern char* results_prefix;
-extern char* transcripts_prefix;
+extern int assimilate_handler_init(int argc, char** argv);
diff --git a/sched/assimilator.cpp b/sched/assimilator.cpp
index 7f0783cee9..b98db705da 100644
--- a/sched/assimilator.cpp
+++ b/sched/assimilator.cpp
@@ -1,6 +1,6 @@
 // This file is part of BOINC.
 // http://boinc.berkeley.edu
-// Copyright (C) 2014 University of California
+// Copyright (C) 2015 University of California
 //
 // BOINC is free software; you can redistribute it and/or modify it
 // under the terms of the GNU Lesser General Public License
@@ -50,12 +50,8 @@ bool noinsert = false;
 int wu_id_modulus=0, wu_id_remainder=0;
 int sleep_interval = SLEEP_INTERVAL;
 int one_pass_N_WU=0;
-int g_argc;
-char** g_argv;
-char* results_prefix = NULL;
-char* transcripts_prefix = NULL;
 
-void usage(char** argv) {
+void usage(char* name) {
     fprintf(stderr,
         "This program is an 'assimilator'; it handles completed jobs.\n"
         "Normally it is run as a daemon from config.xml.\n"
@@ -74,9 +70,8 @@ void usage(char** argv) {
         "    [--noinsert]          Don't insert records in app-specific DB\n"
         "    [-h | --help]                 Show this\n"
         "    [-v | --version]      Show version information\n",
-        argv[0]
+        name
     );
-    exit(0);
 }
 
 // assimilate all WUs that need it
@@ -100,7 +95,7 @@ bool do_pass(APP& app) {
     }
 
     sprintf(buf,
-        "where appid=%lu and assimilate_state=%d %s limit %d",
+        "where appid=%ld and assimilate_state=%d %s limit %d",
         app.id, ASSIMILATE_READY, mod_clause,
         one_pass_N_WU ? one_pass_N_WU : 1000
     );
@@ -127,7 +122,7 @@ bool do_pass(APP& app) {
             "[%s] assimilating WU %lu; state=%d\n", wu.name, wu.id, wu.assimilate_state
         );
 
-        sprintf(buf, "where workunitid=%lu", wu.id);
+        sprintf(buf, "where workunitid=%ld", wu.id);
         canonical_result.clear();
         bool found = false;
         while (1) {
@@ -206,6 +201,13 @@ bool do_pass(APP& app) {
     return did_something;
 }
 
+void missing_argument(char* name, char* arg) {
+    log_messages.printf(MSG_CRITICAL,
+        "%s requires an argument\n\n", arg
+    );
+    usage(name);
+}
+
 int main(int argc, char** argv) {
     int retval;
     bool one_pass = false;
@@ -215,22 +217,38 @@ int main(int argc, char** argv) {
 
     strcpy(app.name, "");
     check_stop_daemons();
-    g_argc = argc;
-    g_argv = argv;
+
+    int j=1;
     for (i=1; i<argc; i++) {
         if (is_arg(argv[i], "one_pass_N_WU")) {
-            one_pass_N_WU = atoi(argv[++i]);
+            if (!argv[++i]) {
+                missing_argument(argv[0], argv[--i]);
+                exit(1);
+            }
+            one_pass_N_WU = atoi(argv[i]);
             one_pass = true;
         } else if (is_arg(argv[i], "sleep_interval")) {
-            sleep_interval = atoi(argv[++i]);
+            if (!argv[++i]) {
+                missing_argument(argv[0], argv[--i]);
+                exit(1);
+            }
+            sleep_interval = atoi(argv[i]);
         } else if (is_arg(argv[i], "one_pass")) {
             one_pass = true;
         } else if (is_arg(argv[i], "d") || is_arg(argv[i], "debug_level")) {
-            int dl = atoi(argv[++i]);
+            if (!argv[++i]) {
+                missing_argument(argv[0], argv[--i]);
+                exit(1);
+            }
+            int dl = atoi(argv[i]);
             log_messages.set_debug_level(dl);
             if (dl ==4) g_print_queries = true;
         } else if (is_arg(argv[i], "app")) {
-            safe_strcpy(app.name, argv[++i]);
+            if (!argv[++i]) {
+                missing_argument(argv[0], argv[--i]);
+                exit(1);
+            }
+            safe_strcpy(app.name, argv[i]);
         } else if (is_arg(argv[i], "dont_update_db")) {
             // This option is for testing your assimilator.  When set,
             // it ensures that the assimilator does not actually modify
@@ -244,25 +262,31 @@ int main(int argc, char** argv) {
             // (as opposed to the boinc) DB.
             noinsert = true;
         } else if (is_arg(argv[i], "mod")) {
-            wu_id_modulus   = atoi(argv[++i]);
-            wu_id_remainder = atoi(argv[++i]);
+            if (!argv[++i]) {
+                missing_argument(argv[0], argv[--i]);
+                exit(1);
+            }
+            wu_id_modulus   = atoi(argv[i]);
+            if (!argv[++i]) {
+                missing_argument(argv[0], argv[--i]);
+                exit(1);
+            }
+            wu_id_remainder = atoi(argv[i]);
         } else if (is_arg(argv[i], "help") || is_arg(argv[i], "h")) {
-            usage(argv);
+            usage(argv[0]);
+            exit(0);
         } else if (is_arg(argv[i], "v") || is_arg(argv[i], "version")) {
             printf("%s\n", SVN_VERSION);
             exit(0);
-        } else if (is_arg(argv[i], "results_prefix")) {
-            results_prefix=argv[++i];
-        } else if (is_arg(argv[i], "transcripts_prefix")) {
-            transcripts_prefix=argv[++i];
         } else {
-            // project-specific part might parse extra args
-            //log_messages.printf(MSG_CRITICAL, "Unrecognized arg: %s\n", argv[i]);
+            // unknown arg - pass to handler
+            argv[j++] = argv[i];
         }
     }
 
     if (!strlen(app.name)) {
-        usage(argv);
+        usage(argv[0]);
+        exit(1);
     }
 
     if (wu_id_modulus) {
@@ -280,19 +304,24 @@ int main(int argc, char** argv) {
         exit(1);
     }
 
-    log_messages.printf(MSG_NORMAL, "Starting\n");
-
     retval = boinc_db.open(config.db_name, config.db_host, config.db_user, config.db_passwd);
     if (retval) {
-        log_messages.printf(MSG_CRITICAL, "Can't open DB\n");
+        log_messages.printf(MSG_CRITICAL, "boinc_db.open failed: %s\n", boincerror(retval));
         exit(1);
     }
     sprintf(buf, "where name='%s'", app.name);
     retval = app.lookup(buf);
     if (retval) {
-        log_messages.printf(MSG_CRITICAL, "Can't find app\n");
+        log_messages.printf(MSG_CRITICAL, "Can't find app: %s\n", app.name);
         exit(1);
     }
+
+    argv[j] = 0;
+    retval = assimilate_handler_init(j, argv);
+    if (retval) exit(1);
+
+    log_messages.printf(MSG_NORMAL, "Starting assimilator handler\n");
+
     install_stop_signal_handler();
     do {
         if (!do_pass(app)) {
diff --git a/sched/sample_assimilator.cpp b/sched/sample_assimilator.cpp
index f31165543b..67d4240d32 100644
--- a/sched/sample_assimilator.cpp
+++ b/sched/sample_assimilator.cpp
@@ -1,6 +1,6 @@
 // This file is part of BOINC.
 // http://boinc.berkeley.edu
-// Copyright (C) 2008 University of California
+// Copyright (C) 2015 University of California
 //
 // BOINC is free software; you can redistribute it and/or modify it
 // under the terms of the GNU Lesser General Public License
@@ -29,6 +29,7 @@
 #include "sched_msgs.h"
 #include "validate_util.h"
 #include "sched_config.h"
+#include "assimilate_handler.h"
 
 using std::vector;
 using std::string;
@@ -44,6 +45,11 @@ int write_error(char* p) {
     return 0;
 }
 
+int assimilate_handler_init(int argc, char** argv) {
+    // handle project specific arguments here
+    return 0;
+}
+
 int assimilate_handler(
     WORKUNIT& wu, vector<RESULT>& /*results*/, RESULT& canonical_result
 ) {
diff --git a/sched/sample_dummy_assimilator.cpp b/sched/sample_dummy_assimilator.cpp
index 258bd2f642..72491f3134 100644
--- a/sched/sample_dummy_assimilator.cpp
+++ b/sched/sample_dummy_assimilator.cpp
@@ -1,6 +1,6 @@
 // This file is part of BOINC.
 // http://boinc.berkeley.edu
-// Copyright (C) 2008 University of California
+// Copyright (C) 2015 University of California
 //
 // BOINC is free software; you can redistribute it and/or modify it
 // under the terms of the GNU Lesser General Public License
@@ -31,6 +31,11 @@
 using std::vector;
 using std::string;
 
+int assimilate_handler_init(int argc, char** argv) {
+    // handle project specific arguments here
+    return 0;
+}
+
 int assimilate_handler(
     WORKUNIT& wu, vector<RESULT>& /*results*/, RESULT& canonical_result
 ) {
diff --git a/sched/script_assimilator.cpp b/sched/script_assimilator.cpp
index eb02e62143..91f55ac620 100644
--- a/sched/script_assimilator.cpp
+++ b/sched/script_assimilator.cpp
@@ -1,6 +1,6 @@
 // This file is part of BOINC.
 // http://boinc.berkeley.edu
-// Copyright (C) 2014 University of California
+// Copyright (C) 2015 University of California
 //
 // BOINC is free software; you can redistribute it and/or modify it
 // under the terms of the GNU Lesser General Public License
@@ -47,6 +47,7 @@
 #include "boinc_db.h"
 #include "error_numbers.h"
 #include "sched_msgs.h"
+#include "sched_util.h"
 #include "validate_util.h"
 #include "validator.h"
 #include "sched_config.h"
@@ -54,13 +55,13 @@
 using std::vector;
 using std::string;
 
-bool first = true;
 vector<string> script;
 
-void parse_cmdline() {
-    for (int i=1; i<g_argc; i++) {
-        if (!strcmp(g_argv[i], "--script")) {
-            script = split(g_argv[++i], ' ');
+int assimilate_handler_init(int argc, char** argv) {
+    // handle project specific arguments here
+    for (int i=1; i<argc; i++) {
+        if (is_arg(argv[i], "script")) {
+            script = split(argv[++i], ' ');
             if (script.size() == 1) {
                 script.push_back("wu_id");
                 script.push_back("files");
@@ -71,8 +72,9 @@ void parse_cmdline() {
         log_messages.printf(MSG_CRITICAL,
             "script name missing from command line\n"
         );
-        exit(1);
+        return 1;
     }
+    return 0;
 }
 
 int assimilate_handler(
@@ -82,11 +84,6 @@ int assimilate_handler(
     char cmd[4096], buf[256];
     unsigned int i, j;
 
-    if (first) {
-        parse_cmdline();
-        first = false;
-    }
-
     if (wu.canonical_resultid) {
         sprintf(cmd, "../bin/%s", script[0].c_str());
         vector<string> paths;
diff --git a/sched/single_job_assimilator.cpp b/sched/single_job_assimilator.cpp
index 55caa05392..198799bbe9 100644
--- a/sched/single_job_assimilator.cpp
+++ b/sched/single_job_assimilator.cpp
@@ -1,6 +1,6 @@
 // This file is part of BOINC.
 // http://boinc.berkeley.edu
-// Copyright (C) 2008 University of California
+// Copyright (C) 2015 University of California
 //
 // BOINC is free software; you can redistribute it and/or modify it
 // under the terms of the GNU Lesser General Public License
@@ -39,6 +39,11 @@
 using std::vector;
 using std::string;
 
+int assimilate_handler_init(int argc, char** argv) {
+    // handle project specific arguments here
+    return 0;
+}
+
 int assimilate_handler(
     WORKUNIT& wu, vector<RESULT>& /*results*/, RESULT& canonical_result
 ) {

From dd004404a1458119a98bc5b07ad68b0a8141544c Mon Sep 17 00:00:00 2001
From: Christian Beer <christian.beer@aei.mpg.de>
Date: Tue, 13 Oct 2015 16:39:04 +0200
Subject: [PATCH 2/4] Daemons: add usage function to project specific handler

Projects that have there own assimilator need to implement the assimilate_handler_usage() function and should printf() a description of those custom options. For examples see sample_assimilator.cpp or script_assimilator.cpp
---
 sched/assimilate_handler.h         | 1 +
 sched/assimilator.cpp              | 1 +
 sched/sample_assimilator.cpp       | 8 ++++++++
 sched/sample_dummy_assimilator.cpp | 8 ++++++++
 sched/script_assimilator.cpp       | 9 +++++++++
 sched/single_job_assimilator.cpp   | 8 ++++++++
 6 files changed, 35 insertions(+)

diff --git a/sched/assimilate_handler.h b/sched/assimilate_handler.h
index 56a15cde12..7286c86145 100644
--- a/sched/assimilate_handler.h
+++ b/sched/assimilate_handler.h
@@ -35,3 +35,4 @@ extern int assimilate_handler(
 );
 
 extern int assimilate_handler_init(int argc, char** argv);
+extern void assimilate_handler_usage();
diff --git a/sched/assimilator.cpp b/sched/assimilator.cpp
index b98db705da..5822be8ee8 100644
--- a/sched/assimilator.cpp
+++ b/sched/assimilator.cpp
@@ -72,6 +72,7 @@ void usage(char* name) {
         "    [-v | --version]      Show version information\n",
         name
     );
+    assimilate_handler_usage();
 }
 
 // assimilate all WUs that need it
diff --git a/sched/sample_assimilator.cpp b/sched/sample_assimilator.cpp
index 67d4240d32..f3ac3652a9 100644
--- a/sched/sample_assimilator.cpp
+++ b/sched/sample_assimilator.cpp
@@ -50,6 +50,14 @@ int assimilate_handler_init(int argc, char** argv) {
     return 0;
 }
 
+void assimilate_handler_usage() {
+    // describe the project specific arguments here
+    //fprintf(stderr,
+    //    "    custom options:\n"
+    //    "    [--project_option X]  a project specific option\n"
+    //);
+}
+
 int assimilate_handler(
     WORKUNIT& wu, vector<RESULT>& /*results*/, RESULT& canonical_result
 ) {
diff --git a/sched/sample_dummy_assimilator.cpp b/sched/sample_dummy_assimilator.cpp
index 72491f3134..02014ead21 100644
--- a/sched/sample_dummy_assimilator.cpp
+++ b/sched/sample_dummy_assimilator.cpp
@@ -36,6 +36,14 @@ int assimilate_handler_init(int argc, char** argv) {
     return 0;
 }
 
+void assimilate_handler_usage() {
+    // describe the project specific arguments here
+    //fprintf(stderr,
+    //    "    custom options:\n"
+    //    "    [--project_option X]  a project specific option\n"
+    //);
+}
+
 int assimilate_handler(
     WORKUNIT& wu, vector<RESULT>& /*results*/, RESULT& canonical_result
 ) {
diff --git a/sched/script_assimilator.cpp b/sched/script_assimilator.cpp
index 91f55ac620..bd7083196c 100644
--- a/sched/script_assimilator.cpp
+++ b/sched/script_assimilator.cpp
@@ -77,6 +77,15 @@ int assimilate_handler_init(int argc, char** argv) {
     return 0;
 }
 
+void assimilate_handler_usage() {
+    // describe the project specific arguments here
+    fprintf(stderr,
+        "    custom options:\n"
+        "    --script \"X\"  call script to assimilate job\n"
+        "                    see comment in script_assimilator.cpp for details\n"
+    );
+}
+
 int assimilate_handler(
     WORKUNIT& wu, vector<RESULT>& /*results*/, RESULT& canonical_result
 ) {
diff --git a/sched/single_job_assimilator.cpp b/sched/single_job_assimilator.cpp
index 198799bbe9..7959159672 100644
--- a/sched/single_job_assimilator.cpp
+++ b/sched/single_job_assimilator.cpp
@@ -44,6 +44,14 @@ int assimilate_handler_init(int argc, char** argv) {
     return 0;
 }
 
+void assimilate_handler_usage() {
+    // describe the project specific arguments here
+    //fprintf(stderr,
+    //    "    custom options:\n"
+    //    "    [--project_option X]  a project specific option\n"
+    //);
+}
+
 int assimilate_handler(
     WORKUNIT& wu, vector<RESULT>& /*results*/, RESULT& canonical_result
 ) {

From d160c7fab5a1e410330facbde11de67d2f8d05a5 Mon Sep 17 00:00:00 2001
From: Christian Beer <christian.beer@aei.mpg.de>
Date: Thu, 29 Oct 2015 09:16:11 +0100
Subject: [PATCH 3/4] Assimilator: cosmetic changes to usage output

---
 sched/assimilator.cpp              | 3 ++-
 sched/sample_assimilator.cpp       | 2 +-
 sched/sample_dummy_assimilator.cpp | 2 +-
 sched/script_assimilator.cpp       | 2 +-
 sched/single_job_assimilator.cpp   | 2 +-
 5 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/sched/assimilator.cpp b/sched/assimilator.cpp
index 5822be8ee8..ca3d7e46b3 100644
--- a/sched/assimilator.cpp
+++ b/sched/assimilator.cpp
@@ -69,7 +69,8 @@ void usage(char* name) {
         "    [--dont_update_db]    Don't update DB (for testing)\n"
         "    [--noinsert]          Don't insert records in app-specific DB\n"
         "    [-h | --help]                 Show this\n"
-        "    [-v | --version]      Show version information\n",
+        "    [-v | --version]      Show version information\n"
+        "\n",
         name
     );
     assimilate_handler_usage();
diff --git a/sched/sample_assimilator.cpp b/sched/sample_assimilator.cpp
index f3ac3652a9..c2562ef165 100644
--- a/sched/sample_assimilator.cpp
+++ b/sched/sample_assimilator.cpp
@@ -53,7 +53,7 @@ int assimilate_handler_init(int argc, char** argv) {
 void assimilate_handler_usage() {
     // describe the project specific arguments here
     //fprintf(stderr,
-    //    "    custom options:\n"
+    //    "    Custom options:\n"
     //    "    [--project_option X]  a project specific option\n"
     //);
 }
diff --git a/sched/sample_dummy_assimilator.cpp b/sched/sample_dummy_assimilator.cpp
index 02014ead21..c2e9a5ea49 100644
--- a/sched/sample_dummy_assimilator.cpp
+++ b/sched/sample_dummy_assimilator.cpp
@@ -39,7 +39,7 @@ int assimilate_handler_init(int argc, char** argv) {
 void assimilate_handler_usage() {
     // describe the project specific arguments here
     //fprintf(stderr,
-    //    "    custom options:\n"
+    //    "    Custom options:\n"
     //    "    [--project_option X]  a project specific option\n"
     //);
 }
diff --git a/sched/script_assimilator.cpp b/sched/script_assimilator.cpp
index bd7083196c..f41783f884 100644
--- a/sched/script_assimilator.cpp
+++ b/sched/script_assimilator.cpp
@@ -80,7 +80,7 @@ int assimilate_handler_init(int argc, char** argv) {
 void assimilate_handler_usage() {
     // describe the project specific arguments here
     fprintf(stderr,
-        "    custom options:\n"
+        "    Custom options:\n"
         "    --script \"X\"  call script to assimilate job\n"
         "                    see comment in script_assimilator.cpp for details\n"
     );
diff --git a/sched/single_job_assimilator.cpp b/sched/single_job_assimilator.cpp
index 7959159672..f530547532 100644
--- a/sched/single_job_assimilator.cpp
+++ b/sched/single_job_assimilator.cpp
@@ -47,7 +47,7 @@ int assimilate_handler_init(int argc, char** argv) {
 void assimilate_handler_usage() {
     // describe the project specific arguments here
     //fprintf(stderr,
-    //    "    custom options:\n"
+    //    "    Custom options:\n"
     //    "    [--project_option X]  a project specific option\n"
     //);
 }

From d536505f7e1a405dd9ceb0e74d66345644833dd7 Mon Sep 17 00:00:00 2001
From: Christian Beer <christian.beer@aei.mpg.de>
Date: Fri, 30 Oct 2015 14:20:21 +0100
Subject: [PATCH 4/4] Assimilator: remove --noinsert option

please implement this in the project specific part if needed
---
 sched/assimilator.cpp | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

diff --git a/sched/assimilator.cpp b/sched/assimilator.cpp
index ca3d7e46b3..5e30c7b5ae 100644
--- a/sched/assimilator.cpp
+++ b/sched/assimilator.cpp
@@ -46,7 +46,6 @@ using std::vector;
 #define SLEEP_INTERVAL 10
 
 bool update_db = true;
-bool noinsert = false;
 int wu_id_modulus=0, wu_id_remainder=0;
 int sleep_interval = SLEEP_INTERVAL;
 int one_pass_N_WU=0;
@@ -66,8 +65,7 @@ void usage(char* name) {
         "    [--one_pass]          Do one DB enumeration, then exit\n"
         "    [--one_pass_N_WU N]   Process at most N jobs\n"
         "    [-d | --debug_level N]       Set verbosity level (1 to 4)\n"
-        "    [--dont_update_db]    Don't update DB (for testing)\n"
-        "    [--noinsert]          Don't insert records in app-specific DB\n"
+        "    [--dont_update_db]    Don't update BOINC DB (for testing)\n"
         "    [-h | --help]                 Show this\n"
         "    [-v | --version]      Show version information\n"
         "\n",
@@ -258,11 +256,6 @@ int main(int argc, char** argv) {
             // your assimilator over and over again without affecting
             // your project.
             update_db = false;
-        } else if (is_arg(argv[i], "noinsert")) {
-            // This option is also for testing and is used to
-            // prevent the inserting of results into the *backend*
-            // (as opposed to the boinc) DB.
-            noinsert = true;
         } else if (is_arg(argv[i], "mod")) {
             if (!argv[++i]) {
                 missing_argument(argv[0], argv[--i]);