2008-08-06 18:36:30 +00:00
|
|
|
// This file is part of BOINC.
|
2005-01-20 23:22:22 +00:00
|
|
|
// http://boinc.berkeley.edu
|
2019-01-12 21:43:48 +00:00
|
|
|
// Copyright (C) 2019 University of California
|
2004-08-05 22:02:33 +00:00
|
|
|
//
|
2008-08-06 18:36:30 +00:00
|
|
|
// BOINC is free software; you can redistribute it and/or modify it
|
|
|
|
// under the terms of the GNU Lesser General Public License
|
|
|
|
// as published by the Free Software Foundation,
|
|
|
|
// either version 3 of the License, or (at your option) any later version.
|
2004-08-05 22:02:33 +00:00
|
|
|
//
|
2008-08-06 18:36:30 +00:00
|
|
|
// BOINC is distributed in the hope that it will be useful,
|
2005-01-20 23:22:22 +00:00
|
|
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
|
|
|
|
// See the GNU Lesser General Public License for more details.
|
2004-08-05 22:02:33 +00:00
|
|
|
//
|
2008-08-06 18:36:30 +00:00
|
|
|
// You should have received a copy of the GNU Lesser General Public License
|
|
|
|
// along with BOINC. If not, see <http://www.gnu.org/licenses/>.
|
2004-08-05 22:02:33 +00:00
|
|
|
|
2011-07-19 20:52:41 +00:00
|
|
|
// upload a file from a host
|
|
|
|
//
|
- server: change the following from per-host to per-(host, app version):
- daily quota mechanism
- reliable mechanism (accelerated retries)
- "trusted" mechanism (adaptive replication)
- scheduler: enforce host scale probation only for apps with
host_scale_check set.
- validator: do scale probation on invalid results
(need this in addition to error and timeout cases)
- feeder: update app version scales every 10 min, not 10 sec
- back-end apps: support --foo as well as -foo for options
Notes:
- If you have, say, cuda, cuda23 and cuda_fermi plan classes,
a host will have separate quotas for each one.
That means it could error out on 100 jobs for cuda_fermi,
and when its quota goes to zero,
error out on 100 jobs for cuda23, etc.
This is intentional; there may be cases where one version
works but not the others.
- host.error_rate and host.max_results_day are deprecated
TODO:
- the values in the app table for limits on jobs in progress etc.
should override rather than config.xml.
Implementation notes:
scheduler:
process_request():
read all host_app_versions for host at start;
Compute "reliable" and "trusted" for each one.
write modified records at end
get_app_version():
add "reliable_only" arg; if set, use only reliable versions
skip over-quota versions
Multi-pass scheduling: if have at least one reliable version,
do a pass for jobs that need reliable,
and use only reliable versions.
Then clear best_app_versions cache.
Score-based scheduling: for need-reliable jobs,
it will pick the fastest version,
then give a score bonus if that version happens to be reliable.
When get back a successful result from client:
increase daily quota
When get back an error result from client:
impose scale probation
decrease daily quota if not aborted
Validator:
when handling a WU, create a vector of HOST_APP_VERSION
parallel to vector of RESULT.
Pass it to assign_credit_set().
Make copies of originals so we can update only modified ones
update HOST_APP_VERSION error rates
Transitioner:
decrease quota on timeout
svn path=/trunk/boinc/; revision=21181
2010-04-15 03:13:56 +00:00
|
|
|
// get_file [options]
|
|
|
|
// --host_id N ID of host to upload from
|
2011-07-19 20:52:41 +00:00
|
|
|
// --file_name name file name
|
|
|
|
// [ --url x ] URL of upload server (can specify several)
|
2011-07-20 22:27:01 +00:00
|
|
|
// [ --max_latency x ] max latency, seconds (default 1 week)
|
|
|
|
// [ --max_nbytes x ] max file size (default 1 GB)
|
2006-11-03 21:42:49 +00:00
|
|
|
//
|
|
|
|
// Run from the project root dir.
|
2004-08-05 22:02:33 +00:00
|
|
|
|
2005-11-21 18:34:44 +00:00
|
|
|
#include "config.h"
|
2004-08-05 22:02:33 +00:00
|
|
|
|
|
|
|
#if HAVE_UNISTD_H
|
|
|
|
#include <unistd.h>
|
2005-11-21 18:34:44 +00:00
|
|
|
#endif
|
2012-05-09 16:11:50 +00:00
|
|
|
#include <sys/param.h>
|
2004-08-05 22:02:33 +00:00
|
|
|
#include <stdlib.h>
|
2008-02-27 23:26:38 +00:00
|
|
|
#include <string>
|
2004-08-05 22:02:33 +00:00
|
|
|
#include <time.h>
|
2007-05-11 16:30:13 +00:00
|
|
|
|
2011-04-24 02:00:27 +00:00
|
|
|
#include "backend_lib.h"
|
2013-08-23 00:01:45 +00:00
|
|
|
#include "filesys.h"
|
2013-06-07 00:31:46 +00:00
|
|
|
#include "str_replace.h"
|
2007-05-11 16:30:13 +00:00
|
|
|
|
2004-08-05 22:02:33 +00:00
|
|
|
#include "sched_config.h"
|
|
|
|
#include "sched_util.h"
|
2009-09-17 17:56:59 +00:00
|
|
|
#include "svn_version.h"
|
2004-08-05 22:02:33 +00:00
|
|
|
|
2011-04-23 19:02:17 +00:00
|
|
|
void usage() {
|
2011-07-19 20:52:41 +00:00
|
|
|
fprintf(stderr, "Gets a file from a host.\n"
|
2011-04-23 19:02:17 +00:00
|
|
|
"Usage: get_file [options]\n\n"
|
2011-07-19 20:52:41 +00:00
|
|
|
" --host_id id host DB ID\n"
|
|
|
|
" --file_name name Name of file\n"
|
|
|
|
" [-- url X] URL of upload server\n"
|
2011-04-23 19:02:17 +00:00
|
|
|
" [ -v | --version ] Show version\n"
|
|
|
|
" [ -h | --help ] Show help\n"
|
2009-09-17 17:56:59 +00:00
|
|
|
);
|
|
|
|
}
|
2004-08-05 22:02:33 +00:00
|
|
|
|
|
|
|
int main(int argc, char** argv) {
|
|
|
|
int i, retval;
|
|
|
|
char file_name[256];
|
|
|
|
int host_id;
|
2011-07-19 20:52:41 +00:00
|
|
|
vector<const char*> urls;
|
2011-07-20 22:27:01 +00:00
|
|
|
double max_latency = 7*86400;
|
|
|
|
double max_nbytes = 1e9;
|
2004-08-05 22:02:33 +00:00
|
|
|
|
|
|
|
strcpy(file_name, "");
|
|
|
|
host_id = 0;
|
|
|
|
|
|
|
|
check_stop_daemons();
|
|
|
|
|
- server: change the following from per-host to per-(host, app version):
- daily quota mechanism
- reliable mechanism (accelerated retries)
- "trusted" mechanism (adaptive replication)
- scheduler: enforce host scale probation only for apps with
host_scale_check set.
- validator: do scale probation on invalid results
(need this in addition to error and timeout cases)
- feeder: update app version scales every 10 min, not 10 sec
- back-end apps: support --foo as well as -foo for options
Notes:
- If you have, say, cuda, cuda23 and cuda_fermi plan classes,
a host will have separate quotas for each one.
That means it could error out on 100 jobs for cuda_fermi,
and when its quota goes to zero,
error out on 100 jobs for cuda23, etc.
This is intentional; there may be cases where one version
works but not the others.
- host.error_rate and host.max_results_day are deprecated
TODO:
- the values in the app table for limits on jobs in progress etc.
should override rather than config.xml.
Implementation notes:
scheduler:
process_request():
read all host_app_versions for host at start;
Compute "reliable" and "trusted" for each one.
write modified records at end
get_app_version():
add "reliable_only" arg; if set, use only reliable versions
skip over-quota versions
Multi-pass scheduling: if have at least one reliable version,
do a pass for jobs that need reliable,
and use only reliable versions.
Then clear best_app_versions cache.
Score-based scheduling: for need-reliable jobs,
it will pick the fastest version,
then give a score bonus if that version happens to be reliable.
When get back a successful result from client:
increase daily quota
When get back an error result from client:
impose scale probation
decrease daily quota if not aborted
Validator:
when handling a WU, create a vector of HOST_APP_VERSION
parallel to vector of RESULT.
Pass it to assign_credit_set().
Make copies of originals so we can update only modified ones
update HOST_APP_VERSION error rates
Transitioner:
decrease quota on timeout
svn path=/trunk/boinc/; revision=21181
2010-04-15 03:13:56 +00:00
|
|
|
for (i=1; i<argc; i++) {
|
|
|
|
if (is_arg(argv[i], "host_id")) {
|
|
|
|
if (!argv[++i]) {
|
2011-04-23 19:02:17 +00:00
|
|
|
usage();
|
2009-09-17 17:56:59 +00:00
|
|
|
exit(1);
|
|
|
|
}
|
|
|
|
host_id = atoi(argv[i]);
|
- server: change the following from per-host to per-(host, app version):
- daily quota mechanism
- reliable mechanism (accelerated retries)
- "trusted" mechanism (adaptive replication)
- scheduler: enforce host scale probation only for apps with
host_scale_check set.
- validator: do scale probation on invalid results
(need this in addition to error and timeout cases)
- feeder: update app version scales every 10 min, not 10 sec
- back-end apps: support --foo as well as -foo for options
Notes:
- If you have, say, cuda, cuda23 and cuda_fermi plan classes,
a host will have separate quotas for each one.
That means it could error out on 100 jobs for cuda_fermi,
and when its quota goes to zero,
error out on 100 jobs for cuda23, etc.
This is intentional; there may be cases where one version
works but not the others.
- host.error_rate and host.max_results_day are deprecated
TODO:
- the values in the app table for limits on jobs in progress etc.
should override rather than config.xml.
Implementation notes:
scheduler:
process_request():
read all host_app_versions for host at start;
Compute "reliable" and "trusted" for each one.
write modified records at end
get_app_version():
add "reliable_only" arg; if set, use only reliable versions
skip over-quota versions
Multi-pass scheduling: if have at least one reliable version,
do a pass for jobs that need reliable,
and use only reliable versions.
Then clear best_app_versions cache.
Score-based scheduling: for need-reliable jobs,
it will pick the fastest version,
then give a score bonus if that version happens to be reliable.
When get back a successful result from client:
increase daily quota
When get back an error result from client:
impose scale probation
decrease daily quota if not aborted
Validator:
when handling a WU, create a vector of HOST_APP_VERSION
parallel to vector of RESULT.
Pass it to assign_credit_set().
Make copies of originals so we can update only modified ones
update HOST_APP_VERSION error rates
Transitioner:
decrease quota on timeout
svn path=/trunk/boinc/; revision=21181
2010-04-15 03:13:56 +00:00
|
|
|
} else if (is_arg(argv[i], "file_name")) {
|
|
|
|
if (!argv[++i]) {
|
2011-04-23 19:02:17 +00:00
|
|
|
usage();
|
2004-08-05 22:02:33 +00:00
|
|
|
exit(1);
|
|
|
|
}
|
2013-06-04 05:42:53 +00:00
|
|
|
safe_strcpy(file_name, argv[i]);
|
- server: change the following from per-host to per-(host, app version):
- daily quota mechanism
- reliable mechanism (accelerated retries)
- "trusted" mechanism (adaptive replication)
- scheduler: enforce host scale probation only for apps with
host_scale_check set.
- validator: do scale probation on invalid results
(need this in addition to error and timeout cases)
- feeder: update app version scales every 10 min, not 10 sec
- back-end apps: support --foo as well as -foo for options
Notes:
- If you have, say, cuda, cuda23 and cuda_fermi plan classes,
a host will have separate quotas for each one.
That means it could error out on 100 jobs for cuda_fermi,
and when its quota goes to zero,
error out on 100 jobs for cuda23, etc.
This is intentional; there may be cases where one version
works but not the others.
- host.error_rate and host.max_results_day are deprecated
TODO:
- the values in the app table for limits on jobs in progress etc.
should override rather than config.xml.
Implementation notes:
scheduler:
process_request():
read all host_app_versions for host at start;
Compute "reliable" and "trusted" for each one.
write modified records at end
get_app_version():
add "reliable_only" arg; if set, use only reliable versions
skip over-quota versions
Multi-pass scheduling: if have at least one reliable version,
do a pass for jobs that need reliable,
and use only reliable versions.
Then clear best_app_versions cache.
Score-based scheduling: for need-reliable jobs,
it will pick the fastest version,
then give a score bonus if that version happens to be reliable.
When get back a successful result from client:
increase daily quota
When get back an error result from client:
impose scale probation
decrease daily quota if not aborted
Validator:
when handling a WU, create a vector of HOST_APP_VERSION
parallel to vector of RESULT.
Pass it to assign_credit_set().
Make copies of originals so we can update only modified ones
update HOST_APP_VERSION error rates
Transitioner:
decrease quota on timeout
svn path=/trunk/boinc/; revision=21181
2010-04-15 03:13:56 +00:00
|
|
|
} else if (is_arg(argv[i], "h") || is_arg(argv[i], "help")) {
|
2011-04-23 19:02:17 +00:00
|
|
|
usage();
|
2009-09-17 17:56:59 +00:00
|
|
|
exit(0);
|
- server: change the following from per-host to per-(host, app version):
- daily quota mechanism
- reliable mechanism (accelerated retries)
- "trusted" mechanism (adaptive replication)
- scheduler: enforce host scale probation only for apps with
host_scale_check set.
- validator: do scale probation on invalid results
(need this in addition to error and timeout cases)
- feeder: update app version scales every 10 min, not 10 sec
- back-end apps: support --foo as well as -foo for options
Notes:
- If you have, say, cuda, cuda23 and cuda_fermi plan classes,
a host will have separate quotas for each one.
That means it could error out on 100 jobs for cuda_fermi,
and when its quota goes to zero,
error out on 100 jobs for cuda23, etc.
This is intentional; there may be cases where one version
works but not the others.
- host.error_rate and host.max_results_day are deprecated
TODO:
- the values in the app table for limits on jobs in progress etc.
should override rather than config.xml.
Implementation notes:
scheduler:
process_request():
read all host_app_versions for host at start;
Compute "reliable" and "trusted" for each one.
write modified records at end
get_app_version():
add "reliable_only" arg; if set, use only reliable versions
skip over-quota versions
Multi-pass scheduling: if have at least one reliable version,
do a pass for jobs that need reliable,
and use only reliable versions.
Then clear best_app_versions cache.
Score-based scheduling: for need-reliable jobs,
it will pick the fastest version,
then give a score bonus if that version happens to be reliable.
When get back a successful result from client:
increase daily quota
When get back an error result from client:
impose scale probation
decrease daily quota if not aborted
Validator:
when handling a WU, create a vector of HOST_APP_VERSION
parallel to vector of RESULT.
Pass it to assign_credit_set().
Make copies of originals so we can update only modified ones
update HOST_APP_VERSION error rates
Transitioner:
decrease quota on timeout
svn path=/trunk/boinc/; revision=21181
2010-04-15 03:13:56 +00:00
|
|
|
} else if (is_arg(argv[i], "v") || is_arg(argv[i], "version")) {
|
2009-09-17 17:56:59 +00:00
|
|
|
printf("%s\n", SVN_VERSION);
|
|
|
|
exit(0);
|
2011-07-19 20:52:41 +00:00
|
|
|
} else if (is_arg(argv[i], "url")) {
|
|
|
|
urls.push_back(argv[++i]);
|
2011-07-20 22:27:01 +00:00
|
|
|
} else if (is_arg(argv[i], "max_latency")) {
|
|
|
|
max_latency = atof(argv[++i]);
|
|
|
|
} else if (is_arg(argv[i], "max_nbytes")) {
|
|
|
|
max_nbytes = atof(argv[++i]);
|
2009-09-17 17:56:59 +00:00
|
|
|
} else {
|
2011-04-23 19:02:17 +00:00
|
|
|
usage();
|
2009-09-17 17:56:59 +00:00
|
|
|
exit(1);
|
2004-08-05 22:02:33 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!strlen(file_name) || host_id == 0) {
|
2011-04-23 19:02:17 +00:00
|
|
|
usage();
|
2004-08-05 22:02:33 +00:00
|
|
|
exit(1);
|
|
|
|
}
|
|
|
|
|
2009-05-07 13:54:51 +00:00
|
|
|
retval = config.parse_file();
|
2004-08-05 22:02:33 +00:00
|
|
|
if (retval) {
|
2009-05-07 13:54:51 +00:00
|
|
|
fprintf(stderr, "Can't parse config.xml: %s\n", boincerror(retval));
|
2004-08-05 22:02:33 +00:00
|
|
|
exit(1);
|
|
|
|
}
|
|
|
|
|
2006-11-03 21:42:49 +00:00
|
|
|
retval = boinc_db.open(
|
|
|
|
config.db_name, config.db_host, config.db_user, config.db_passwd
|
|
|
|
);
|
2004-08-05 22:02:33 +00:00
|
|
|
if (retval) {
|
2020-05-06 20:01:21 +00:00
|
|
|
fprintf(stderr, "boinc_db.open failed: %s\n",
|
|
|
|
boinc_db.error_string()
|
|
|
|
);
|
2004-08-05 22:02:33 +00:00
|
|
|
exit(1);
|
|
|
|
}
|
|
|
|
|
2011-07-19 20:52:41 +00:00
|
|
|
if (urls.size() == 0) {
|
|
|
|
urls.push_back(config.upload_url);
|
|
|
|
}
|
|
|
|
|
2011-07-20 22:27:01 +00:00
|
|
|
R_RSA_PRIVATE_KEY key;
|
|
|
|
bool generate_upload_certificate = !config.dont_generate_upload_certificates;
|
|
|
|
if (generate_upload_certificate) {
|
2012-05-09 16:11:50 +00:00
|
|
|
char keypath[MAXPATHLEN];
|
2011-07-20 22:27:01 +00:00
|
|
|
sprintf(keypath, "%s/upload_private", config.key_dir);
|
|
|
|
retval = read_key_file(keypath, key);
|
|
|
|
if (retval) {
|
|
|
|
fprintf(stderr, "can't read key\n");
|
|
|
|
exit(1);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2012-02-24 22:55:11 +00:00
|
|
|
retval = create_get_file_msg(
|
2011-07-20 22:27:01 +00:00
|
|
|
host_id, file_name, urls, max_nbytes,
|
|
|
|
dtime() + max_latency,
|
|
|
|
generate_upload_certificate, key
|
|
|
|
);
|
|
|
|
|
2011-07-19 20:52:41 +00:00
|
|
|
if (retval) {
|
|
|
|
fprintf(stderr, "get_file() failed: %s\n", boincerror(retval));
|
|
|
|
}
|
2004-08-05 22:02:33 +00:00
|
|
|
boinc_db.close();
|
|
|
|
}
|