// This file is part of BOINC. // http://boinc.berkeley.edu // Copyright (C) 2008 University of California // // BOINC is free software; you can redistribute it and/or modify it // under the terms of the GNU Lesser General Public License // as published by the Free Software Foundation, // either version 3 of the License, or (at your option) any later version. // // BOINC is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. // See the GNU Lesser General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with BOINC. If not, see . // // This file contains functions that can be customized to // implement project-specific scheduling policies. // The functions are: // // wu_is_infeasible_custom() // Decide whether host can run a job using a particular app version. // In addition it can: // - set the app version's resource usage and/or FLOPS rate estimate // (by assigning to bav.host_usage) // - modify command-line args // (by assigning to bav.host_usage.cmdline) // - set the job's FLOPS count // (by assigning to wu.rsc_fpops_est) // // app_plan() // Decide whether host can use an app version, // and if so what resources it will use // // app_plan_uses_gpu(): // Which plan classes use GPUs // // JOB::get_score(): // Determine the value of sending a particular job to host; // (used only by "matchmaker" scheduling) // // WARNING: if you modify this file, you must prevent it from // being overwritten the next time you update BOINC source code. // You can either: // 1) write-protect this file, or // 2) put this in a differently-named file and change the Makefile.am // (and write-protect that) // In either case, put your version under source-code control, e.g. SVN #include using std::string; #include "str_util.h" #include "util.h" #include "sched_config.h" #include "sched_main.h" #include "sched_msgs.h" #include "sched_send.h" #include "sched_score.h" #include "sched_shmem.h" #include "sched_version.h" #include "sched_customize.h" #include "plan_class_spec.h" GPU_REQUIREMENTS gpu_requirements[NPROC_TYPES]; bool wu_is_infeasible_custom(WORKUNIT& wu, APP& app, BEST_APP_VERSION& bav) { #if 0 // example: if WU name contains "_v1", don't use GPU apps. // Note: this is slightly suboptimal. // If the host is able to accept both GPU and CPU jobs, // we'll skip this job rather than send it for the CPU. // Fixing this would require a big architectural change. // if (strstr(wu.name, "_v1") && bav.host_usage.proc_type != PROC_TYPE_CPU) { return true; } #endif #if 0 // example: for NVIDIA GPU app, // wu.batch is the minimum number of GPU processors. // Don't send if #procs is less than this. // if (!strcmp(app.name, "foobar") && bav.host_usage.proc_type == PROC_TYPE_NVIDIA_GPU) { int n = g_request->coprocs.nvidia.prop.multiProcessorCount; if (n < wu.batch) { return true; } } #endif #if 0 // example: if GPU app and WU name contains ".vlar", don't send // if (bav.host_usage.proc_type != PROC_TYPE_CPU) { if (strstr(wu.name, ".vlar")) { return true; } } #endif return false; } // the following is for an app that can use anywhere from 1 to 64 threads // static inline bool app_plan_mt(SCHEDULER_REQUEST&, HOST_USAGE& hu) { double ncpus = g_wreq->effective_ncpus; // number of usable CPUs, taking user prefs into account if (ncpus < 2) return false; int nthreads = (int)ncpus; if (nthreads > 64) nthreads = 64; hu.avg_ncpus = nthreads; hu.max_ncpus = nthreads; sprintf(hu.cmdline, "--nthreads %d", nthreads); hu.projected_flops = capped_host_fpops()*hu.avg_ncpus*.99; // the .99 ensures that on uniprocessors a sequential app // will be used in preferences to this hu.peak_flops = capped_host_fpops()*hu.avg_ncpus; if (config.debug_version_select) { log_messages.printf(MSG_NORMAL, "[version] Multi-thread app projected %.2fGS\n", hu.projected_flops/1e9 ); } return true; } static bool ati_check(COPROC_ATI& c, HOST_USAGE& hu, int min_driver_version, bool need_amd_libs, double min_ram, double ndevs, // # of GPUs used; can be fractional double cpu_frac, // fraction of FLOPS performed by CPU double flops_scale ) { if (c.version_num) { gpu_requirements[PROC_TYPE_AMD_GPU].update(min_driver_version, min_ram); } if (need_amd_libs) { if (!c.amdrt_detected) { if (config.debug_version_select) { log_messages.printf(MSG_NORMAL, "[version] AMD run time libraries not found\n" ); } return false; } } else { if (!c.atirt_detected) { if (config.debug_version_select) { log_messages.printf(MSG_NORMAL, "[version] ATI run time libraries not found\n" ); } return false; } } if (c.version_num < min_driver_version) { if (config.debug_version_select) { int app_major=min_driver_version/10000000; int app_minor=(min_driver_version%10000000)/10000; int app_rev=(min_driver_version%10000); int dev_major=c.version_num/10000000; int dev_minor=(c.version_num%10000000)/10000; int dev_rev=(c.version_num%10000); log_messages.printf(MSG_NORMAL, "[version] Bad display driver revision %d.%d.%d<%d.%d.%d.\n", dev_major,dev_minor,dev_rev,app_major,app_minor,app_rev ); } return false; } if (c.available_ram < min_ram) { if (config.debug_version_select) { log_messages.printf(MSG_NORMAL, "[version] Insufficient GPU RAM %f>%f.\n", min_ram, c.available_ram ); } return false; } hu.gpu_ram = min_ram; hu.proc_type = PROC_TYPE_AMD_GPU; hu.gpu_usage = ndevs; coproc_perf( capped_host_fpops(), flops_scale * hu.gpu_usage*c.peak_flops, cpu_frac, hu.projected_flops, hu.avg_ncpus ); hu.peak_flops = hu.gpu_usage*c.peak_flops + hu.avg_ncpus*capped_host_fpops(); hu.max_ncpus = hu.avg_ncpus; return true; } #define ATI_MIN_RAM 250*MEGA static inline bool app_plan_ati( SCHEDULER_REQUEST& sreq, char* plan_class, HOST_USAGE& hu ) { COPROC_ATI& c = sreq.coprocs.ati; if (!c.count) { if (config.debug_version_select) { log_messages.printf(MSG_NORMAL,"[version] Host has no ATI GPUs\n"); } return false; } if (!strcmp(plan_class, "ati")) { if (!ati_check(c, hu, ati_version_int(1, 0, 0), true, ATI_MIN_RAM, 1, .01, .20 )) { return false; } } if (!strcmp(plan_class, "ati13amd")) { if (!ati_check(c, hu, ati_version_int(1, 3, 0), true, ATI_MIN_RAM, 1, .01, .21 )) { return false; } } if (!strcmp(plan_class, "ati13ati")) { if (!ati_check(c, hu, ati_version_int(1, 3, 186), false, ATI_MIN_RAM, 1, .01, .22 )) { return false; } } if (!strcmp(plan_class, "ati14")) { if (!ati_check(c, hu, ati_version_int(1, 4, 0), false, ATI_MIN_RAM, 1, .01, .23 )) { return false; } } if (config.debug_version_select) { log_messages.printf(MSG_NORMAL, "[version] %s ATI app projected %.2fG peak %.2fG %.3f CPUs\n", plan_class, hu.projected_flops/1e9, hu.peak_flops/1e9, hu.avg_ncpus ); } return true; } #define CUDA_MIN_DRIVER_VERSION 17700 #define CUDA23_MIN_CUDA_VERSION 2030 #define CUDA23_MIN_DRIVER_VERSION 19038 #define CUDA3_MIN_CUDA_VERSION 3000 #define CUDA3_MIN_DRIVER_VERSION 19500 #define CUDA_OPENCL_MIN_DRIVER_VERSION 19713 #define CUDA_OPENCL_101_MIN_DRIVER_VERSION 28013 static bool cuda_check(COPROC_NVIDIA& c, HOST_USAGE& hu, int min_cc, int max_cc, int min_cuda_version, int min_driver_version, double min_ram, double ndevs, // # of GPUs used; can be fractional double cpu_frac, // fraction of FLOPS performed by CPU double flops_scale ) { int cc = c.prop.major*100 + c.prop.minor; if (min_cc && (cc < min_cc)) { log_messages.printf(MSG_NORMAL, "[version] App requires compute capability > %d.%d (has %d.%d).\n", min_cc/100, min_cc%100, c.prop.major, c.prop.minor ); return false; } if (max_cc && cc >= max_cc) { log_messages.printf(MSG_NORMAL, "[version] App requires compute capability <= %d.%d (has %d.%d).\n", max_cc/100, max_cc%100, c.prop.major, c.prop.minor ); return false; } if (c.display_driver_version) { gpu_requirements[PROC_TYPE_NVIDIA_GPU].update(min_driver_version, min_ram); } // Old BOINC clients report display driver version; // newer ones report CUDA RT version. // Some Linux doesn't return either. // if (!c.cuda_version && !c.display_driver_version) { if (config.debug_version_select) { log_messages.printf(MSG_NORMAL, "[version] Client did not provide cuda or driver version.\n" ); } return false; } if (c.cuda_version) { if (min_cuda_version && (c.cuda_version < min_cuda_version)) { if (config.debug_version_select) { double app_version=(double)(min_cuda_version/1000)+(double)(min_cuda_version%100)/100.0; double client_version=(double)(c.cuda_version/1000)+(double)(c.cuda_version%100)/100.0; log_messages.printf(MSG_NORMAL, "[version] Bad CUDA version %f>%f.\n", app_version, client_version ); } return false; } } if (c.display_driver_version) { if (min_driver_version && (c.display_driver_version < min_driver_version)) { if (config.debug_version_select) { double app_version=(double)(min_driver_version)/100.0; double client_version=(double)(c.display_driver_version)/100.0; log_messages.printf(MSG_NORMAL, "[version] Bad display driver revision %f>%f.\n", app_version, client_version ); } return false; } } if (c.available_ram < min_ram) { if (config.debug_version_select) { log_messages.printf(MSG_NORMAL, "[version] Insufficient GPU RAM %f>%f.\n", min_ram, c.available_ram ); } return false; } hu.gpu_ram = min_ram; hu.proc_type = PROC_TYPE_NVIDIA_GPU; hu.gpu_usage = ndevs; coproc_perf( capped_host_fpops(), flops_scale * hu.gpu_usage*c.peak_flops, cpu_frac, hu.projected_flops, hu.avg_ncpus ); hu.peak_flops = hu.gpu_usage*c.peak_flops + hu.avg_ncpus*capped_host_fpops(); hu.max_ncpus = hu.avg_ncpus; return true; } // the following is for an app that uses an NVIDIA GPU // static inline bool app_plan_cuda( SCHEDULER_REQUEST& sreq, char* plan_class, HOST_USAGE& hu ) { COPROC_NVIDIA& c = sreq.coprocs.nvidia; if (!c.count) { return false; } // Macs require 6.10.28 // if (strstr(sreq.host.os_name, "Darwin") && (sreq.core_client_version < 61028)) { return false; } // for CUDA 2.3, we need to check the CUDA RT version. // Old BOINC clients report display driver version; // newer ones report CUDA RT version // if (!strcmp(plan_class, "cuda_fermi")) { if (!cuda_check(c, hu, 200, 0, CUDA3_MIN_CUDA_VERSION, CUDA3_MIN_DRIVER_VERSION, 384*MEGA, 1, .01, .22 )) { return false; } } else if (!strcmp(plan_class, "cuda23")) { if (!cuda_check(c, hu, 100, 200, // change to zero if app is compiled to byte code CUDA23_MIN_CUDA_VERSION, CUDA23_MIN_DRIVER_VERSION, 384*MEGA, 1, .01, .21 )) { return false; } } else if (!strcmp(plan_class, "cuda")) { if (!cuda_check(c, hu, 100, 200, // change to zero if app is compiled to byte code 0, CUDA_MIN_DRIVER_VERSION, 254*MEGA, 1, .01, .20 )) { return false; } } else { log_messages.printf(MSG_CRITICAL, "UNKNOWN PLAN CLASS %s\n", plan_class ); return false; } if (config.debug_version_select) { log_messages.printf(MSG_NORMAL, "[version] %s app projected %.2fG peak %.2fG %.3f CPUs\n", plan_class, hu.projected_flops/1e9, hu.peak_flops/1e9, hu.avg_ncpus ); } return true; } // The following is for a non-CPU-intensive application. // Say that we'll use 1% of a CPU. // This will cause the client (6.7+) to run it at non-idle priority // static inline bool app_plan_nci(SCHEDULER_REQUEST&, HOST_USAGE& hu) { hu.avg_ncpus = .01; hu.max_ncpus = .01; hu.projected_flops = capped_host_fpops()*1.01; // The *1.01 is needed to ensure that we'll send this app // version rather than a non-plan-class one hu.peak_flops = capped_host_fpops()*.01; return true; } // the following is for an app version that requires a processor with SSE3, // and will run 10% faster than the non-SSE3 version // static inline bool app_plan_sse3( SCHEDULER_REQUEST& sreq, HOST_USAGE& hu ) { downcase_string(sreq.host.p_features); if (!strstr(sreq.host.p_features, "sse3")) { // Pre-6.x clients report CPU features in p_model // if (!strstr(sreq.host.p_model, "sse3")) { //add_no_work_message("Your CPU lacks SSE3"); return false; } } hu.avg_ncpus = 1; hu.max_ncpus = 1; hu.projected_flops = 1.1*capped_host_fpops(); hu.peak_flops = capped_host_fpops(); return true; } static inline bool opencl_check( COPROC& cp, HOST_USAGE& hu, int min_opencl_device_version, double min_global_mem_size, double ndevs, double cpu_frac, double flops_scale ) { if (cp.opencl_prop.opencl_device_version_int < min_opencl_device_version) { return false; } if (cp.opencl_prop.global_mem_size < min_global_mem_size) { return false; } hu.gpu_ram = min_global_mem_size; if (!strcmp(cp.type, "NVIDIA")) { hu.proc_type = PROC_TYPE_NVIDIA_GPU; hu.gpu_usage = ndevs; } else if (!strcmp(cp.type, "ATI")) { hu.proc_type = PROC_TYPE_AMD_GPU; hu.gpu_usage = ndevs; } coproc_perf( capped_host_fpops(), flops_scale * ndevs * cp.peak_flops, cpu_frac, hu.projected_flops, hu.avg_ncpus ); hu.peak_flops = ndevs*cp.peak_flops + hu.avg_ncpus*capped_host_fpops(); hu.max_ncpus = hu.avg_ncpus; return true; } static inline bool app_plan_opencl( SCHEDULER_REQUEST& sreq, const char* plan_class, HOST_USAGE& hu ) { if (strstr(plan_class, "nvidia")) { COPROC_NVIDIA& c = sreq.coprocs.nvidia; if (!c.count) return false; if (!c.have_opencl) return false; if (!strcmp(plan_class, "opencl_nvidia_101")) { return opencl_check( c, hu, 101, 256*MEGA, 1, .1, .2 ); } else { log_messages.printf(MSG_CRITICAL, "Unknown plan class: %s\n", plan_class ); return false; } } else if (strstr(plan_class, "ati")) { COPROC_ATI& c = sreq.coprocs.ati; if (!c.count) return false; if (!c.have_opencl) return false; if (!strcmp(plan_class, "opencl_ati_101")) { return opencl_check( c, hu, 101, 256*MEGA, 1, .1, .2 ); } else { log_messages.printf(MSG_CRITICAL, "Unknown plan class: %s\n", plan_class ); return false; } // maybe add a clause for multicore CPU } else { log_messages.printf(MSG_CRITICAL, "Unknown plan class: %s\n", plan_class ); return false; } } // handles vbox_[32|64][_mt] // "mt" is tailored to the needs of CERN: // use 1 or 2 CPUs static inline bool app_plan_vbox( SCHEDULER_REQUEST& sreq, char* plan_class, HOST_USAGE& hu ) { bool can_use_multicore = true; // host must run 7.0+ client // if (sreq.core_client_major_version < 7) { add_no_work_message("BOINC client 7.0+ required for Virtualbox jobs"); return false; } // host must have VirtualBox 3.2 or later // if (strlen(sreq.host.virtualbox_version) == 0) { add_no_work_message("VirtualBox is not installed"); return false; } int n, maj, min, rel; n = sscanf(sreq.host.virtualbox_version, "%d.%d.%d", &maj, &min, &rel); if ((n != 3) || (maj < 3) || (maj == 3 and min < 2)) { add_no_work_message("VirtualBox version 3.2 or later is required"); return false; } // host must have VM acceleration in order to run multi-core jobs // if (strstr(plan_class, "mt")) { if ((!strstr(sreq.host.p_features, "vmx") && !strstr(sreq.host.p_features, "svm")) || sreq.host.p_vm_extensions_disabled ) { can_use_multicore = false; } } // only send the version for host's primary platform. // A Win64 host can't run a 32-bit VM app: // it will look in the 32-bit half of the registry and fail // PLATFORM* p = g_request->platforms.list[0]; if (is_64b_platform(p->name)) { if (!strstr(plan_class, "64")) return false; } else { if (strstr(plan_class, "64")) return false; } double flops_scale = 1; hu.avg_ncpus = 1; hu.max_ncpus = 1; if (strstr(plan_class, "mt")) { if (can_use_multicore) { // Use number of usable CPUs, taking user prefs into account double ncpus = g_wreq->effective_ncpus; // CernVM on average uses between 25%-50% of a second core // Total on a dual-core machine is between 65%-75% if (ncpus > 1.5) ncpus = 1.5; hu.avg_ncpus = ncpus; hu.max_ncpus = 2.0; sprintf(hu.cmdline, "--nthreads %f", ncpus); } // use the non-mt version rather than the mt version with 1 CPU // flops_scale = .99; } hu.projected_flops = flops_scale * capped_host_fpops()*hu.avg_ncpus; hu.peak_flops = capped_host_fpops()*hu.max_ncpus; if (config.debug_version_select) { log_messages.printf(MSG_NORMAL, "[version] %s app projected %.2fG\n", plan_class, hu.projected_flops/1e9 ); } return true; } PLAN_CLASS_SPECS plan_class_specs; // app planning function. // See http://boinc.berkeley.edu/trac/wiki/AppPlan // bool app_plan(SCHEDULER_REQUEST& sreq, char* plan_class, HOST_USAGE& hu) { char buf[256]; static bool check_plan_class_spec = true; static bool have_plan_class_spec = false; static bool bad_plan_class_spec = false; if (config.debug_version_select) { log_messages.printf(MSG_NORMAL, "[version] Checking plan class '%s'\n", plan_class ); } if (check_plan_class_spec) { check_plan_class_spec = false; strcpy(buf, config.project_dir); strcat(buf, "/plan_class_spec.xml"); int retval = plan_class_specs.parse_file(buf); if (retval == ERR_FOPEN) { if (config.debug_version_select) { log_messages.printf(MSG_NORMAL, "[version] Couldn't open plan class spec file '%s'\n", buf ); } have_plan_class_spec = false; } else if (retval) { log_messages.printf(MSG_CRITICAL, "Error parsing plan class spec file '%s'\n", buf ); bad_plan_class_spec = true; } else { if (config.debug_version_select) { log_messages.printf(MSG_NORMAL, "[version] reading plan classes from file '%s'\n", buf ); } have_plan_class_spec = true; } } if (bad_plan_class_spec) { return false; } if (have_plan_class_spec) { return plan_class_specs.check(sreq, plan_class, hu); } if (!strcmp(plan_class, "mt")) { return app_plan_mt(sreq, hu); } else if (strstr(plan_class, "opencl")) { return app_plan_opencl(sreq, plan_class, hu); } else if (strstr(plan_class, "ati")) { return app_plan_ati(sreq, plan_class, hu); } else if (strstr(plan_class, "cuda")) { return app_plan_cuda(sreq, plan_class, hu); } else if (!strcmp(plan_class, "nci")) { return app_plan_nci(sreq, hu); } else if (!strcmp(plan_class, "sse3")) { return app_plan_sse3(sreq, hu); } else if (strstr(plan_class, "vbox")) { return app_plan_vbox(sreq, plan_class, hu); } log_messages.printf(MSG_CRITICAL, "Unknown plan class: %s\n", plan_class ); return false; } // compute a "score" for sending this job to this host. // Return false if the WU is infeasible. // Otherwise set est_time and disk_usage. // bool JOB::get_score() { WORKUNIT wu; int retval; WU_RESULT& wu_result = ssp->wu_results[index]; wu = wu_result.workunit; app = ssp->lookup_app(wu.appid); if (app->non_cpu_intensive) return false; score = 0; // Find the best app version to use. // bavp = get_app_version(wu, true, false); if (!bavp) return false; retval = wu_is_infeasible_fast( wu, wu_result.res_server_state, wu_result.res_priority, wu_result.res_report_deadline, *app, *bavp ); if (retval) { if (config.debug_send) { log_messages.printf(MSG_NORMAL, "[send] [HOST#%d] [WU#%d %s] WU is infeasible: %s\n", g_reply->host.id, wu.id, wu.name, infeasible_string(retval) ); } return false; } score = 1; #if 0 // example: for CUDA app, wu.batch is the minimum number of processors. // add min/actual to score // (this favors sending jobs that need lots of procs to GPUs that have them) // IF YOU USE THIS, USE THE PART IN wu_is_infeasible_custom() ALSO // if (!strcmp(app->name, "foobar") && bavp->host_usage.ncudas) { int n = g_request->coproc_cuda->prop.multiProcessorCount; score += ((double)wu.batch)/n; } #endif // check if user has selected apps, // and send beta work to beta users // if (app->beta && !config.distinct_beta_apps) { if (g_wreq->allow_beta_work) { score += 1; } else { return false; } } else { if (app_not_selected(wu)) { if (!g_wreq->allow_non_preferred_apps) { return false; } else { // Allow work to be sent, but it will not get a bump in its score } } else { score += 1; } } // if job needs to get done fast, send to fast/reliable host // if (bavp->reliable && (wu_result.need_reliable)) { score += 1; } // if job already committed to an HR class, // try to send to host in that class // if (wu_result.infeasible_count) { score += 1; } // Favor jobs that will run fast // score += bavp->host_usage.projected_flops/1e9; // match large jobs to fast hosts // if (config.job_size_matching) { double host_stdev = (capped_host_fpops() - ssp->perf_info.host_fpops_mean)/ ssp->perf_info.host_fpops_stddev; double diff = host_stdev - wu_result.fpops_size; score -= diff*diff; } // TODO: If user has selected some apps but will accept jobs from others, // try to send them jobs from the selected apps // est_time = estimate_duration(wu, *bavp); disk_usage = wu.rsc_disk_bound; return true; } void handle_file_xfer_results() { for (unsigned int i=0; ifile_xfer_results.size(); i++) { RESULT& r = g_request->file_xfer_results[i]; log_messages.printf(MSG_NORMAL, "completed file xfer %s\n", r.name ); g_reply->result_acks.push_back(string(r.name)); } }