// This file is part of BOINC.
// http://boinc.berkeley.edu
// Copyright (C) 2008 University of California
//
// BOINC is free software; you can redistribute it and/or modify it
// under the terms of the GNU Lesser General Public License
// as published by the Free Software Foundation,
// either version 3 of the License, or (at your option) any later version.
//
// BOINC is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
// See the GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with BOINC. If not, see .
#include "cpp.h"
#ifdef _WIN32
#include "boinc_win.h"
#else
#include "config.h"
#include
#include
#endif
#include "error_numbers.h"
#include "md5_file.h"
#include "parse.h"
#include "util.h"
#include "str_util.h"
#include "filesys.h"
#include "log_flags.h"
#include "file_names.h"
#include "client_state.h"
#include "client_types.h"
#include "client_msgs.h"
using std::vector;
PERS_FILE_XFER::PERS_FILE_XFER() {
nretry = 0;
first_request_time = gstate.now;
next_request_time = first_request_time;
time_so_far = 0;
last_bytes_xferred = 0;
pers_xfer_done = false;
fip = NULL;
fxp = NULL;
}
PERS_FILE_XFER::~PERS_FILE_XFER() {
if (fip) {
fip->pers_file_xfer = NULL;
}
}
int PERS_FILE_XFER::init(FILE_INFO* f, bool is_file_upload) {
fxp = NULL;
fip = f;
is_upload = is_file_upload;
pers_xfer_done = false;
const char* p = f->get_init_url();
if (!p) {
msg_printf(NULL, MSG_INTERNAL_ERROR, "No URL for file transfer of %s", f->name);
return ERR_NULL;
}
return 0;
}
int PERS_FILE_XFER::start_xfer() {
if (is_upload) {
if (gstate.exit_before_upload) {
exit(0);
}
return fxp->init_upload(*fip);
} else {
return fxp->init_download(*fip);
}
}
// Possibly create and start a file transfer
//
int PERS_FILE_XFER::create_xfer() {
FILE_XFER *file_xfer;
int retval;
// Decide whether to start a new file transfer
//
if (!gstate.start_new_file_xfer(*this)) {
return ERR_IDLE_PERIOD;
}
// if download, see if file already exists and is valid
//
if (!is_upload) {
char pathname[256];
get_pathname(fip, pathname, sizeof(pathname));
if (!fip->verify_file(true, false)) {
retval = fip->set_permissions();
fip->status = FILE_PRESENT;
pers_xfer_done = true;
if (log_flags.file_xfer) {
msg_printf(
fip->project, MSG_INFO,
"File %s exists already, skipping download", fip->name
);
}
return 0;
} else {
// Mark file as not present but don't delete it.
// It might partly downloaded.
//
fip->status = FILE_NOT_PRESENT;
}
}
file_xfer = new FILE_XFER;
fxp = file_xfer;
retval = start_xfer();
if (!retval) retval = gstate.file_xfers->insert(file_xfer);
if (retval) {
if (log_flags.http_debug) {
msg_printf(
fip->project, MSG_INFO, "[file_xfer] Couldn't start %s of %s",
(is_upload ? "upload" : "download"), fip->name
);
msg_printf(
fip->project, MSG_INFO, "[file_xfer] URL %s: %s",
fip->get_current_url(), boincerror(retval)
);
}
fxp->file_xfer_retval = retval;
transient_failure(retval);
delete fxp;
fxp = NULL;
return retval;
}
if (log_flags.file_xfer) {
msg_printf(
fip->project, MSG_INFO, "Started %s of %s",
(is_upload ? "upload" : "download"), fip->name
);
}
if (log_flags.file_xfer_debug) {
msg_printf(fip->project, MSG_INFO,
"[file_xfer] URL: %s\n",
fip->get_current_url()
);
}
return 0;
}
// Poll the status of this persistent file transfer.
// If it's time to start it, then attempt to start it.
// If it has finished or failed, then deal with it appropriately
//
bool PERS_FILE_XFER::poll() {
int retval;
if (pers_xfer_done) {
return false;
}
if (!fxp) {
// No file xfer is active.
// Either initial or resume after failure.
// See if it's time to try again.
//
if (gstate.now < next_request_time) {
return false;
}
FILE_XFER_BACKOFF& fxb = fip->project->file_xfer_backoff(is_upload);
if (!fxb.ok_to_transfer()) {
#if 0
if (log_flags.file_xfer_debug) {
msg_printf(fip->project, MSG_INFO,
"[file_xfer] delaying %s of %s: project-wide backoff %f sec",
is_upload?"upload":"download", fip->name,
fxb.next_xfer_time - gstate.now
);
}
#endif
return false;
}
last_time = gstate.now;
retval = create_xfer();
return (retval == 0);
}
// copy bytes_xferred for use in GUI
//
last_bytes_xferred = fxp->bytes_xferred;
if (fxp->is_upload) {
last_bytes_xferred += fxp->file_offset;
}
// don't count suspended periods in total time
//
double diff = gstate.now - last_time;
if (diff <= 2) {
time_so_far += diff;
}
last_time = gstate.now;
if (fxp->file_xfer_done) {
if (log_flags.file_xfer_debug) {
msg_printf(fip->project, MSG_INFO,
"[file_xfer] file transfer status %d",
fxp->file_xfer_retval
);
}
switch (fxp->file_xfer_retval) {
case 0:
fip->project->file_xfer_backoff(is_upload).file_xfer_succeeded();
if (log_flags.file_xfer) {
msg_printf(
fip->project, MSG_INFO, "Finished %s of %s",
is_upload?"upload":"download", fip->name
);
}
if (log_flags.file_xfer_debug) {
if (fxp->xfer_speed < 0) {
msg_printf(fip->project, MSG_INFO, "[file_xfer] No data transferred");
} else {
msg_printf(
fip->project, MSG_INFO, "[file_xfer] Throughput %d bytes/sec",
(int)fxp->xfer_speed
);
}
}
pers_xfer_done = true;
break;
case ERR_UPLOAD_PERMANENT:
permanent_failure(fxp->file_xfer_retval);
break;
case ERR_NOT_FOUND:
case ERR_FILE_NOT_FOUND:
case HTTP_STATUS_NOT_FOUND: // won't happen - converted in http_curl.C
if (is_upload) {
// if we get a "not found" on an upload,
// the project must not have a file_upload_handler.
// Treat this as a transient error.
//
msg_printf(fip->project, MSG_INFO,
"Project file upload handler is missing"
);
transient_failure(fxp->file_xfer_retval);
} else {
permanent_failure(fxp->file_xfer_retval);
}
break;
default:
if (log_flags.file_xfer) {
msg_printf(
fip->project, MSG_INFO, "Temporarily failed %s of %s: %s",
is_upload?"upload":"download", fip->name,
boincerror(fxp->file_xfer_retval)
);
}
transient_failure(fxp->file_xfer_retval);
}
// If we transferred any bytes, or there are >1 URLs,
// set upload_offset back to -1
// so that we'll query file size on next retry.
// Otherwise leave it as is, avoiding unnecessary size query.
//
if (last_bytes_xferred || (fip->urls.size() > 1)) {
fip->upload_offset = -1;
}
// fxp could have already been freed and zeroed above
// so check before trying to remove
//
if (fxp) {
gstate.file_xfers->remove(fxp);
delete fxp;
fxp = NULL;
}
return true;
}
return false;
}
void PERS_FILE_XFER::permanent_failure(int retval) {
gstate.file_xfers->remove(fxp);
delete fxp;
fxp = NULL;
fip->status = retval;
pers_xfer_done = true;
if (log_flags.file_xfer) {
msg_printf(
fip->project, MSG_INFO, "Giving up on %s of %s: %s",
is_upload?"upload":"download", fip->name, boincerror(retval)
);
}
fip->error_msg = boincerror(retval);
}
// Handle a transient failure
//
void PERS_FILE_XFER::transient_failure(int retval) {
// If it was a bad range request, delete the file and start over
//
if (retval == HTTP_STATUS_RANGE_REQUEST_ERROR) {
fip->delete_file();
return;
}
// If too much time has elapsed, give up
//
if ((gstate.now - first_request_time) > gstate.file_xfer_giveup_period) {
permanent_failure(ERR_TIMEOUT);
}
// Cycle to the next URL to try.
// If we reach the URL that we started at, then back off.
// Otherwise immediately try the next URL
//
if (fip->get_next_url()) {
start_xfer();
} else {
do_backoff();
}
}
// per-file backoff policy: sets next_request_time
//
void PERS_FILE_XFER::do_backoff() {
double backoff = 0;
// don't count it as a server failure if network is down
//
if (!net_status.need_physical_connection) {
nretry++;
}
// keep track of transient failures per project (not currently used)
//
PROJECT* p = fip->project;
p->file_xfer_backoff(is_upload).file_xfer_failed(p);
// Do an exponential backoff of e^nretry seconds,
// keeping within the bounds of pers_retry_delay_min and
// pers_retry_delay_max
//
backoff = calculate_exponential_backoff(
nretry, gstate.pers_retry_delay_min, gstate.pers_retry_delay_max
);
next_request_time = gstate.now + backoff;
msg_printf(fip->project, MSG_INFO,
"Backing off %s on %s of %s",
timediff_format(backoff).c_str(),
is_upload?"upload":"download",
fip->name
);
}
void PERS_FILE_XFER::abort() {
if (fxp) {
gstate.file_xfers->remove(fxp);
delete fxp;
fxp = NULL;
}
fip->status = ERR_ABORTED_VIA_GUI;
fip->error_msg = "user requested transfer abort";
pers_xfer_done = true;
}
// Parse XML information about a persistent file transfer
//
int PERS_FILE_XFER::parse(MIOFILE& fin) {
char buf[256];
while (fin.fgets(buf, 256)) {
if (match_tag(buf, "")) return 0;
else if (parse_int(buf, "", nretry)) continue;
else if (parse_double(buf, "", first_request_time)) {
continue;
}
else if (parse_double(buf, "", next_request_time)) {
continue;
}
else if (parse_double(buf, "", time_so_far)) continue;
else if (parse_double(buf, "", last_bytes_xferred)) continue;
else {
if (log_flags.unparsed_xml) {
msg_printf(NULL, MSG_INFO,
"[unparsed_xml] Unparsed line in file transfer info: %s", buf
);
}
}
}
return ERR_XML_PARSE;
}
// Write XML information about a persistent file transfer
//
int PERS_FILE_XFER::write(MIOFILE& fout) {
fout.printf(
" \n"
" %d\n"
" %f\n"
" %f\n"
" %f\n"
" %f\n"
" \n",
nretry, first_request_time, next_request_time, time_so_far, last_bytes_xferred
);
if (fxp) {
fout.printf(
" \n"
" %f\n"
" %f\n"
" %f\n"
" %s\n"
" \n",
fxp->bytes_xferred,
fxp->file_offset,
fxp->xfer_speed,
fxp->m_url
);
}
return 0;
}
// suspend file transfers by killing them.
// They'll restart automatically later.
//
void PERS_FILE_XFER::suspend() {
if (fxp) {
last_bytes_xferred = fxp->bytes_xferred; // save bytes transferred
if (fxp->is_upload) {
last_bytes_xferred += fxp->file_offset;
}
gstate.file_xfers->remove(fxp); // this removes from http_op_set too
delete fxp;
fxp = 0;
}
fip->upload_offset = -1;
}
PERS_FILE_XFER_SET::PERS_FILE_XFER_SET(FILE_XFER_SET* p) {
file_xfers = p;
}
// Run through the set, starting any transfers that need to be
// started and deleting any that have finished
//
bool PERS_FILE_XFER_SET::poll() {
unsigned int i;
bool action = false;
static double last_time=0;
if (gstate.now - last_time < PERS_FILE_XFER_POLL_PERIOD) return false;
last_time = gstate.now;
// try to finish ones we've already started
//
for (i=0; ilast_bytes_xferred) continue;
action |= pers_file_xfers[i]->poll();
}
for (i=0; ilast_bytes_xferred) continue;
action |= pers_file_xfers[i]->poll();
}
if (action) gstate.set_client_state_dirty("pers_file_xfer_set poll");
return action;
}
// Insert a PERS_FILE_XFER object into the set.
// We will decide which ones to start when we hit the polling loop
//
int PERS_FILE_XFER_SET::insert(PERS_FILE_XFER* pfx) {
pers_file_xfers.push_back(pfx);
return 0;
}
// Remove a PERS_FILE_XFER object from the set.
// What should the action here be?
//
int PERS_FILE_XFER_SET::remove(PERS_FILE_XFER* pfx) {
vector::iterator iter;
iter = pers_file_xfers.begin();
while (iter != pers_file_xfers.end()) {
if (*iter == pfx) {
iter = pers_file_xfers.erase(iter);
return 0;
}
iter++;
}
msg_printf(
pfx->fip->project, MSG_INTERNAL_ERROR,
"Persistent file transfer object not found"
);
return ERR_NOT_FOUND;
}
// suspend all PERS_FILE_XFERs
//
void PERS_FILE_XFER_SET::suspend() {
unsigned int i;
for (i=0; isuspend();
}
}
// add a random delay 0..x to all transfers
// (used when emerging from bandwidth quota suspension)
//
void PERS_FILE_XFER_SET::add_random_delay(double x) {
unsigned int i;
double y = gstate.now + x*drand();
for (i=0; i pers_file_xfers[i]->next_request_time) {
pers_file_xfers[i]->next_request_time = y;
}
}
}