diff --git a/checkin_notes b/checkin_notes index 1344803791..7a049f15be 100644 --- a/checkin_notes +++ b/checkin_notes @@ -3621,4 +3621,31 @@ Eric K May 6 2008 api/ x_opengl.C +David May 6 2008 + - scheduler: add preliminary support for "job size matching" + (attempt to send big jobs to fast hosts, small jobs to slow hosts). + - have "census" compute mean/stdev of host speeds, + write it to a file perf_info.txt + - have feeder compute mean/stdev of sizes of jobs in shmem + - have feeder read perf_info.txt into shmem + - scheduler: add some debugging messages for app version selection + - Add LGPL license to a few files + - upgrade/setup scripts: copy census to bin/ + py/Boinc/ + setup_project.py + sched/ + assimilate_handler.C + census.C + feeder.C + hr_info.C,h + sched_array.h + sched_config.C,h + sched_plan.C + sched_resend.h + sched_shmem.h + sched_timezone.h + server_types.C + single_job_assimilator.C + validate_util2.h + validator.h diff --git a/doc/addon_data.php b/doc/addon_data.php index 291471f172..8f6150c560 100644 --- a/doc/addon_data.php +++ b/doc/addon_data.php @@ -466,14 +466,14 @@ array('boinc_server_status.zip', ); $web = array( -array('http://download.conmunix.net/pub/boinc_lcs/boinc_lcs_3.0_beta.tar.gz', +array('http://download.conmunix.net/pub/boinc_lcs/boinc_lcs_3.0.tar.gz', 'Boinc LCS', - '3.0 beta', + '3.0 final', 'Shows the current state and other information from an unlimited number of BOINC clients connected to the Internet.', 'http://www.conmunix.net/boinc-lcs', 'Apache, PHP', - 'Boinc LCS is a free PHP based script, that allows you to monitor the current state (and other information) from each of your connected BOINC clients. It runs on a simple webserver and since version 2.1 it is also platform independent. Boinc LCS is released under the GNU/GPL license. You can modify and redistribute or just using it!', - '1193023938' + 'Boinc LCS is a free PHP based script, that allows you to monitor the current state (and other information) from each of your connected BOINC clients. It runs on a simple webserver and is platform independent. Boinc LCS is released under the GNU/GPL license. You can modify and redistribute or just use it!', + '1210049520' ), array('b-sig.zip', 'Forum signature showing jobs in progress', diff --git a/doc/boinc_news.php b/doc/boinc_news.php index ffb6fcff93..e34d1a249f 100644 --- a/doc/boinc_news.php +++ b/doc/boinc_news.php @@ -1,6 +1,9 @@ Boinc LCS (a free PHP script for monitoring remote BOINC clients) has been released." +), array("Apr 20, 2008", "The BOINC user documentation has been moved from Trac diff --git a/py/Boinc/setup_project.py b/py/Boinc/setup_project.py index 8c6757573b..1a35dfa6cd 100644 --- a/py/Boinc/setup_project.py +++ b/py/Boinc/setup_project.py @@ -367,7 +367,7 @@ def install_boinc_files(dest_dir, web_only): 'file_deleter', 'sample_dummy_assimilator', 'sample_assimilator', 'sample_work_generator', 'single_job_assimilator', - 'update_stats', 'db_dump', 'db_purge', 'show_shmem' ]) + 'update_stats', 'db_dump', 'db_purge', 'show_shmem', 'census' ]) map(lambda (s): install(srcdir('tools',s), dir('bin',s)), [ 'create_work', 'xadd', 'dbcheck_files_exist', 'run_in_ops', 'update_versions', 'parse_config', 'grep_logs', 'db_query', diff --git a/sched/assimilate_handler.h b/sched/assimilate_handler.h index e68b63363b..400b3e5e61 100644 --- a/sched/assimilate_handler.h +++ b/sched/assimilate_handler.h @@ -1,3 +1,22 @@ +// Berkeley Open Infrastructure for Network Computing +// http://boinc.berkeley.edu +// Copyright (C) 2008 University of California +// +// This is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; +// either version 2.1 of the License, or (at your option) any later version. +// +// This software is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +// See the GNU Lesser General Public License for more details. +// +// To view the GNU Lesser General Public License visit +// http://www.gnu.org/copyleft/lesser.html +// or write to the Free Software Foundation, Inc., +// 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + #include #include "boinc_db.h" #define DEFER_ASSIMILATION 123321 diff --git a/sched/census.C b/sched/census.C index 4f744c4af7..f6534b1d8e 100644 --- a/sched/census.C +++ b/sched/census.C @@ -70,4 +70,5 @@ int main(int argc, char** argv) { hri.init(); hri.scan_db(); hri.write_file(); + hri.perf_info.write_file(); } diff --git a/sched/feeder.C b/sched/feeder.C index 53b2b7bdaf..5c0c9da570 100644 --- a/sched/feeder.C +++ b/sched/feeder.C @@ -98,6 +98,7 @@ #include #include #include +#include #include #include #include @@ -352,8 +353,33 @@ void weighted_interleave(double* weights, int n, int k, int* v, int* count) { free(x); } +// update the job size statistics fields of array entries +// +static void update_stats() { + int i, n=0; + double sum=0, sum_sqr=0; + + for (i=0; imax_wu_results; i++) { + WU_RESULT& wu_result = ssp->wu_results[i]; + if (wu_result.state != WR_STATE_PRESENT) continue; + n++; + double e = wu_result.workunit.rsc_fpops_est; + sum += e; + sum_sqr += e*e; + } + double mean = sum/n; + double stdev = sqrt((sum_sqr - sum*mean)/n); + for (i=0; imax_wu_results; i++) { + WU_RESULT& wu_result = ssp->wu_results[i]; + if (wu_result.state != WR_STATE_PRESENT) continue; + double e = wu_result.workunit.rsc_fpops_est; + double diff = e - mean; + wu_result.fpops_size = diff/stdev; + } +} + // Make one pass through the work array, filling in empty slots. -// Return true if should call this again immediately. +// Return true if we filled in any. // static bool scan_work_array(vector &work_items) { int i; @@ -471,6 +497,10 @@ void feeder_loop() { "No action; sleeping %.2f sec\n", sleep_interval ); boinc_sleep(sleep_interval); + } else { + if (config.job_size_matching) { + update_stats(); + } } fflush(stdout); @@ -704,6 +734,15 @@ int main(int argc, char** argv) { } hr_init(); + if (config.job_size_matching) { + retval = ssp->perf_info.read_file(); + if (retval) { + log_messages.printf(MSG_CRITICAL, + "can't read perf_info file; run census\n" + ); + exit(1); + } + } signal(SIGUSR1, show_state); diff --git a/sched/hr_info.C b/sched/hr_info.C index 1dd16a6540..cf8255a383 100644 --- a/sched/hr_info.C +++ b/sched/hr_info.C @@ -1,7 +1,25 @@ +// Berkeley Open Infrastructure for Network Computing +// http://boinc.berkeley.edu +// Copyright (C) 2008 University of California +// +// This is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; +// either version 2.1 of the License, or (at your option) any later version. +// +// This software is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +// See the GNU Lesser General Public License for more details. +// +// To view the GNU Lesser General Public License visit +// http://www.gnu.org/copyleft/lesser.html +// or write to the Free Software Foundation, Inc., +// 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + #include -#ifdef HAVE_MALLOC_H #include -#endif +#include #include "error_numbers.h" #include "sched_msgs.h" @@ -66,11 +84,17 @@ void HR_INFO::init() { void HR_INFO::scan_db() { DB_HOST host; int retval; - int i; + int i, n=0; + double sum=0, sum_sqr=0; while (1) { retval = host.enumerate("where expavg_credit>1"); if (retval) break; + if (host.p_fpops > 1e7 && host.p_fpops < 1e13) { + n++; + sum += host.p_fpops; + sum_sqr += host.p_fpops*host.p_fpops; + } //printf("host %d: %s | %s | %s\n", host.id, host.os_name, host.p_vendor, host.p_model); for (i=1; i ban_os; vector ban_cpu; + bool job_size_matching; // log flags // bool debug_version_select; diff --git a/sched/sched_plan.C b/sched/sched_plan.C index 046e1b28b1..982773ca6c 100644 --- a/sched/sched_plan.C +++ b/sched/sched_plan.C @@ -27,6 +27,7 @@ // (you need to prevent that from being overwritten too) // In either case, put your version under source-code control, e.g. SVN +#include "sched_config.h" #include "sched_msgs.h" #include "sched_plan.h" @@ -80,6 +81,11 @@ bool app_plan(SCHEDULER_REQUEST& sreq, char* plan_class, HOST_USAGE& hu) { return true; } } + if (config.debug_version_select) { + log_messages.printf(MSG_DEBUG, + "Host lacks CUDA coprocessor for plan class %s\n", plan_class + ); + } return false; } log_messages.printf(MSG_CRITICAL, diff --git a/sched/sched_resend.h b/sched/sched_resend.h index 8392f9159a..aea3311f3b 100644 --- a/sched/sched_resend.h +++ b/sched/sched_resend.h @@ -1,2 +1,20 @@ -extern bool resend_lost_work(SCHEDULER_REQUEST&, SCHEDULER_REPLY&); +// Berkeley Open Infrastructure for Network Computing +// http://boinc.berkeley.edu +// Copyright (C) 2008 University of California +// +// This is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; +// either version 2.1 of the License, or (at your option) any later version. +// +// This software is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +// See the GNU Lesser General Public License for more details. +// +// To view the GNU Lesser General Public License visit +// http://www.gnu.org/copyleft/lesser.html +// or write to the Free Software Foundation, Inc., +// 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +extern bool resend_lost_work(SCHEDULER_REQUEST&, SCHEDULER_REPLY&); diff --git a/sched/sched_shmem.h b/sched/sched_shmem.h index 21a0a9353b..3981ee41c4 100644 --- a/sched/sched_shmem.h +++ b/sched/sched_shmem.h @@ -27,6 +27,7 @@ #define _SCHED_SHMEM_H_ #include "boinc_db.h" +#include "hr_info.h" // the following must be at least as large as DB tables // (counting only non-deprecated entries for the current major version) @@ -60,6 +61,7 @@ struct WU_RESULT { int resultid; int time_added_to_shared_memory; int result_priority; + double fpops_size; // measured in stdevs }; // this struct is followed in memory by an array of WU_RESULTS @@ -68,7 +70,7 @@ struct SCHED_SHMEM { bool ready; // feeder sets to true when init done // the following fields let the scheduler make sure // that the shared mem has the right format - int ss_size; // size of this struct, including array + int ss_size; // size of this struct, including WU_RESULT array int platform_size; // sizeof(PLATFORM) int app_size; // sizeof(APP) int app_version_size; // sizeof(APP_VERSION) @@ -84,6 +86,7 @@ struct SCHED_SHMEM { int max_app_versions; int max_assignments; int max_wu_results; + PERF_INFO perf_info; PLATFORM platforms[MAX_PLATFORMS]; APP apps[MAX_APPS]; APP_VERSION app_versions[MAX_APP_VERSIONS]; diff --git a/sched/sched_timezone.h b/sched/sched_timezone.h index e6df13944b..ebc23b8414 100644 --- a/sched/sched_timezone.h +++ b/sched/sched_timezone.h @@ -1,3 +1,22 @@ +// Berkeley Open Infrastructure for Network Computing +// http://boinc.berkeley.edu +// Copyright (C) 2008 University of California +// +// This is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; +// either version 2.1 of the License, or (at your option) any later version. +// +// This software is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +// See the GNU Lesser General Public License for more details. +// +// To view the GNU Lesser General Public License visit +// http://www.gnu.org/copyleft/lesser.html +// or write to the Free Software Foundation, Inc., +// 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + extern void process_av_timezone( SCHEDULER_REPLY& reply, APP_VERSION* avp, APP_VERSION& av2 ); diff --git a/sched/server_types.C b/sched/server_types.C index ffa6176240..9d776fe20f 100644 --- a/sched/server_types.C +++ b/sched/server_types.C @@ -16,6 +16,7 @@ // http://www.gnu.org/copyleft/lesser.html // or write to the Free Software Foundation, Inc., // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + #include "config.h" #include #include diff --git a/sched/single_job_assimilator.C b/sched/single_job_assimilator.C index 85354233bd..4efb04a06f 100644 --- a/sched/single_job_assimilator.C +++ b/sched/single_job_assimilator.C @@ -16,7 +16,7 @@ // http://www.gnu.org/copyleft/lesser.html // or write to the Free Software Foundation, Inc., // 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -// + // assimilator for single jobs. // - if success, move the output file(s) to job directory // - delete job description file diff --git a/sched/validate_util2.h b/sched/validate_util2.h index d27ef02dcf..b4191988de 100644 --- a/sched/validate_util2.h +++ b/sched/validate_util2.h @@ -1,3 +1,22 @@ +// Berkeley Open Infrastructure for Network Computing +// http://boinc.berkeley.edu +// Copyright (C) 2008 University of California +// +// This is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; +// either version 2.1 of the License, or (at your option) any later version. +// +// This software is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +// See the GNU Lesser General Public License for more details. +// +// To view the GNU Lesser General Public License visit +// http://www.gnu.org/copyleft/lesser.html +// or write to the Free Software Foundation, Inc., +// 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + #ifndef _VALIDATE_UTIL2_ #define _VALIDATE_UTIL2_ diff --git a/sched/validator.h b/sched/validator.h index be72dafe70..ffdd1ec344 100644 --- a/sched/validator.h +++ b/sched/validator.h @@ -1,3 +1,22 @@ +// Berkeley Open Infrastructure for Network Computing +// http://boinc.berkeley.edu +// Copyright (C) 2008 University of California +// +// This is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; +// either version 2.1 of the License, or (at your option) any later version. +// +// This software is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +// See the GNU Lesser General Public License for more details. +// +// To view the GNU Lesser General Public License visit +// http://www.gnu.org/copyleft/lesser.html +// or write to the Free Software Foundation, Inc., +// 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + #include "sched_config.h" extern SCHED_CONFIG config;