boinc/client/pers_file_xfer.C

// Berkeley Open Infrastructure for Network Computing
// http://boinc.berkeley.edu
// Copyright (C) 2005 University of California
//
// This is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation;
// either version 2.1 of the License, or (at your option) any later version.
//
// This software is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
// See the GNU Lesser General Public License for more details.
//
// To view the GNU Lesser General Public License visit
// http://www.gnu.org/copyleft/lesser.html
// or write to the Free Software Foundation, Inc.,
// 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

#include "cpp.h"

#ifdef _WIN32
#include "boinc_win.h"
#endif

#ifndef _WIN32
#include "config.h"
#include <cmath>
#include <cstdlib>
#endif

#include "error_numbers.h"
#include "md5_file.h"
#include "parse.h"
#include "util.h"
#include "filesys.h"

#include "log_flags.h"
#include "file_names.h"
#include "client_state.h"
#include "client_types.h"
#include "client_msgs.h"

using std::vector;

PERS_FILE_XFER::PERS_FILE_XFER() {
    nretry = 0;
    first_request_time = gstate.now;
    next_request_time = first_request_time;
    time_so_far = 0;
    last_bytes_xferred = 0;
    pers_xfer_done = false;
    fip = NULL;
    fxp = NULL;
}

PERS_FILE_XFER::~PERS_FILE_XFER() {
    if (fip) {
        fip->pers_file_xfer = NULL;
    }
}

int PERS_FILE_XFER::init(FILE_INFO* f, bool is_file_upload) {
    fxp = NULL;
    fip = f;
    is_upload = is_file_upload;
    pers_xfer_done = false;
    const char* p = f->get_init_url(is_file_upload);
    if (!p) {
        msg_printf(NULL, MSG_INTERNAL_ERROR, "No URL for file transfer of %s", f->name);
        return ERR_NULL;
    }
    return 0;
}

// Try to start the file transfer associated with this persistent file transfer.
//
int PERS_FILE_XFER::start_xfer() {
    FILE_XFER *file_xfer;
    int retval;

    // Decide whether to start a new file transfer
    //
    if (!gstate.start_new_file_xfer(*this)) {
        return ERR_IDLE_PERIOD;
    }

    // Does the file exist already? this could happen for example if we are
    // downloading an application which exists from a previous installation
    //
    if (!is_upload) {
        char pathname[256];
        get_pathname(fip, pathname);

        // see if file already exists and is valid
        //
        if (!fip->verify_file(true, false)) {
            retval = fip->set_permissions();
            fip->status = FILE_PRESENT;
            pers_xfer_done = true;

            if (log_flags.file_xfer) {
                msg_printf(
                    fip->project, MSG_INFO,
                    "File %s exists already, skipping download", fip->name
                );
            }

            return 0;
        } else {
            fip->status = FILE_NOT_PRESENT;
        }
    }

    // Create a new FILE_XFER object and initialize a
    // download or upload for the persistent file transfer
    //
    file_xfer = new FILE_XFER;
    file_xfer->set_proxy(&gstate.proxy_info);
    if (is_upload) {
        if (gstate.exit_before_upload) {
            exit(0);
        }
        retval = file_xfer->init_upload(*fip);
    } else {
        retval = file_xfer->init_download(*fip);
    }
    fxp = file_xfer;
    if (!retval) retval = gstate.file_xfers->insert(file_xfer);
    if (retval) {
        if (log_flags.http_debug) {
            msg_printf(
                fip->project, MSG_INFO, "[file_xfer_debug] Couldn't start %s of %s",
                (is_upload ? "upload" : "download"), fip->name
            );
            msg_printf(
                fip->project, MSG_INFO, "[file_xfer_debug] URL %s: %s",
                fip->get_current_url(is_upload), boincerror(retval)
            );
        }

        fxp->file_xfer_retval = retval;
        handle_xfer_failure();
        delete fxp;
        fxp = NULL;
        return retval;
    }
    if (log_flags.file_xfer) {
        msg_printf(
            fip->project, MSG_INFO, "[file_xfer] Started %s of file %s",
            (is_upload ? "upload" : "download"), fip->name
        );
    }
    if (log_flags.file_xfer_debug) {
        msg_printf(0, MSG_INFO,
            "[file_xfer_debug] PERS_FILE_XFER::start_xfer(): URL: %s\n",
            fip->get_current_url(is_upload)
        );
    }
    return 0;
}

// Poll the status of this persistent file transfer.
// If it's time to start it, then attempt to start it.
// If it has finished or failed, then deal with it appropriately
//
bool PERS_FILE_XFER::poll() {
    int retval;

    if (pers_xfer_done) {
        return false;
    }
    if (!fxp) {
        // No file xfer is active.
        // Either initial or resume after failure.
        // See if it's time to try again.
        //
        if (gstate.now < next_request_time) {
            return false;
        }
#if 0
        if (gstate.now < fip->project->next_file_xfer_time(is_upload)) {
            return false;
        }
#endif
        last_time = gstate.now;
        fip->upload_offset = -1;
        retval = start_xfer();
        return (retval == 0);
    }

    // copy bytes_xferred for use in GUI
    last_bytes_xferred = fxp->bytes_xferred;
    if (fxp->is_upload) {
        last_bytes_xferred += fxp->file_offset;
    }

    // don't count suspended periods in total time
    //
    double diff = gstate.now - last_time;
    if (diff <= 2) {
        time_so_far += diff;
    }
    last_time = gstate.now;

    if (fxp->file_xfer_done) {
        if (log_flags.file_xfer_debug) {
            msg_printf(0, MSG_INFO,
                "[file_xfer_debug] PERS_FILE_XFER::poll(): file transfer status %d",
                fxp->file_xfer_retval
            );
        }
        if (fxp->file_xfer_retval == 0) {
            // The transfer finished with no errors.
            fip->project->file_xfer_succeeded(is_upload);
            if (log_flags.file_xfer) {
                msg_printf(
                    fip->project, MSG_INFO, "[file_xfer] Finished %s of file %s",
                    is_upload?"upload":"download", fip->name
                );
                if (fxp->xfer_speed < 0) {
                    msg_printf(fip->project, MSG_INFO, "[file_xfer] No data transferred");
                } else {
                    msg_printf(
                        fip->project, MSG_INFO, "[file_xfer] Throughput %d bytes/sec",
                        (int)fxp->xfer_speed
                    );
                }
            }
            pers_xfer_done = true;
        } else if (fxp->file_xfer_retval == ERR_UPLOAD_PERMANENT) {
            if (log_flags.file_xfer) {
                msg_printf(
                    fip->project, MSG_INFO, "[file_xfer] Permanently failed %s of %s",
                    is_upload?"upload":"download", fip->name
                );
            }
            try_next_url("server rejected file");
        } else if (fxp->file_xfer_retval == ERR_NOT_FOUND) {
            if (log_flags.file_xfer) {
                msg_printf(
                    fip->project, MSG_INFO, "[file_xfer] Permanently failed %s of %s",
                    is_upload?"upload":"download", fip->name
                );
            }
            try_next_url("File not found on client");
        } else {
            if (log_flags.file_xfer) {
                msg_printf(
                    fip->project, MSG_INFO, "[file_xfer] Temporarily failed %s of %s: %s",
                    is_upload?"upload":"download", fip->name,
                    boincerror(fxp->file_xfer_retval)
                );
            }
            handle_xfer_failure();
        }

        // fxp could have already been freed and zeroed by try_next_url
        // so check before trying to remove
        //
        if (fxp) {
            gstate.file_xfers->remove(fxp);
            delete fxp;
            fxp = NULL;
        }
        return true;
    }
    return false;
}

void PERS_FILE_XFER::xfer_failed(const char* why) {
    gstate.file_xfers->remove(fxp);
    delete fxp;
    fxp = NULL;
    if (is_upload) {
        fip->status = ERR_GIVEUP_UPLOAD;
    } else {
        fip->status = ERR_GIVEUP_DOWNLOAD;
    }
    pers_xfer_done = true;
    if (log_flags.file_xfer) {
        msg_printf(
            fip->project, MSG_INFO, "Giving up on %s of %s: %s",
            is_upload?"upload":"download", fip->name, why
        );
    }
    fip->error_msg = why;
}

// A file transfer (to a particular server)
// has had a failure
// TODO ?? transient ? permanent? terminology??
//
// Takes a reason why a transfer has failed.
//
// Checks to see if there are no more valid URLs listed in the file_info.
//
// If no more urls are present, the file is then given up on, the reason is
// listed in the error_msg field, and the appropriate status code is given.
//
// If there are more URLs to try, the file_xfer is restarted with these new
// urls until a good transfer is made or it completely gives up.
//
void PERS_FILE_XFER::try_next_url(const char* why) {
    if (fip->get_next_url(fip->upload_when_present) == NULL) {
        xfer_failed(why);
        fip->delete_file();
    } else {
        if (is_upload) {
            if (gstate.exit_before_upload) {
                exit(0);
            }
            fxp->init_upload(*fip);
        } else {
            fxp->init_download(*fip);
        }
    }
}

// Handle a transfer failure
//
void PERS_FILE_XFER::handle_xfer_failure() {

    // If it was a bad range request, delete the file and start over
    //
    if (fxp->file_xfer_retval == HTTP_STATUS_RANGE_REQUEST_ERROR) {
        fip->delete_file();
        return;
    }

    if (fxp->file_xfer_retval == HTTP_STATUS_NOT_FOUND) {
        if (fxp->is_upload) {
            // If it is uploading and receives a HTTP_STATUS_NOT_FOUND then
            // the file upload handler could not be found.
            // This is hopefully transient.
            //
            retry_or_backoff();
            return;
        } else {
            try_next_url("file was not found on server");
            return;
        }
    }

    // See if it's time to give up on the persistent file xfer
    //
    if ((gstate.now - first_request_time) > gstate.file_xfer_giveup_period) {
        try_next_url("too much elapsed time");
    } else {
        retry_or_backoff();
    }
}

// Cycle to the next URL, or if we've hit all URLs in this cycle,
// backoff and try again later
//
void PERS_FILE_XFER::retry_or_backoff() {
    // Cycle to the next URL to try
    // If we reach the URL that we started at, then we have tried all
    // servers without success
    //

    if (fip->get_next_url(is_upload) == NULL) {
        fip->project->file_xfer_failed(is_upload);
        do_backoff();
    }
}

// per-file backoff policy: sets next_request_time
//
void PERS_FILE_XFER::do_backoff() {
    double backoff = 0;

    // don't count it as a server failure if network is down
    //
    if (!net_status.need_physical_connection) {
        nretry++;
    }

    // Do an exponential backoff of e^nretry seconds,
    // keeping within the bounds of pers_retry_delay_min and
    // pers_retry_delay_max
    //
    backoff = calculate_exponential_backoff(
        nretry, gstate.pers_retry_delay_min, gstate.pers_retry_delay_max
    );
    next_request_time = gstate.now + backoff;
    msg_printf(fip->project, MSG_INFO,
        "Backing off %s on %s of file %s",
        timediff_format(backoff).c_str(),
        is_upload?"upload":"download",
        fip->name
    );
}

void PERS_FILE_XFER::abort() {
    if (fxp) {
        gstate.file_xfers->remove(fxp);
        delete fxp;
        fxp = NULL;
    }
    fip->status = is_upload?ERR_GIVEUP_UPLOAD:ERR_GIVEUP_DOWNLOAD;
    fip->error_msg = "user requested transfer abort";
    pers_xfer_done = true;
}

// Parse XML information about a persistent file transfer
//
int PERS_FILE_XFER::parse(MIOFILE& fin) {
    char buf[256];

    while (fin.fgets(buf, 256)) {
        if (match_tag(buf, "</persistent_file_xfer>")) return 0;
        else if (parse_int(buf, "<num_retries>", nretry)) continue;
        else if (parse_double(buf, "<first_request_time>", first_request_time)) {
            validate_time(first_request_time);
            continue;
        }
        else if (parse_double(buf, "<next_request_time>", next_request_time)) {
            validate_time(next_request_time);
            continue;
        }
        else if (parse_double(buf, "<time_so_far>", time_so_far)) continue;
        else if (parse_double(buf, "<last_bytes_xferred>", last_bytes_xferred)) continue;
        else {
            if (log_flags.unparsed_xml) {
                msg_printf(NULL, MSG_INFO,
                    "[unparsed_xml] Unparsed line in file transfer info: %s", buf
                );
            }
        }
    }
    return ERR_XML_PARSE;
}

// Write XML information about a persistent file transfer
//
int PERS_FILE_XFER::write(MIOFILE& fout) {
    fout.printf(
        "    <persistent_file_xfer>\n"
        "        <num_retries>%d</num_retries>\n"
        "        <first_request_time>%f</first_request_time>\n"
        "        <next_request_time>%f</next_request_time>\n"
        "        <time_so_far>%f</time_so_far>\n"
        "        <last_bytes_xferred>%f</last_bytes_xferred>\n"
        "    </persistent_file_xfer>\n",
        nretry, first_request_time, next_request_time, time_so_far, last_bytes_xferred
    );
    if (fxp) {
        fout.printf(
            "    <file_xfer>\n"
            "        <bytes_xferred>%f</bytes_xferred>\n"
            "        <file_offset>%f</file_offset>\n"
            "        <xfer_speed>%f</xfer_speed>\n"
            "        <url>%s</url>\n"
            "    </file_xfer>\n",
            fxp->bytes_xferred,
            fxp->file_offset,
            fxp->xfer_speed,
            fxp->m_url
        );
    }
    return 0;
}

// suspend file transfers by killing them.
// They'll restart automatically later.
//
void PERS_FILE_XFER::suspend() {
    if (fxp) {
        last_bytes_xferred = fxp->bytes_xferred;  // save bytes transferred
        if (fxp->is_upload) {
            last_bytes_xferred += fxp->file_offset;
        }
        gstate.file_xfers->remove(fxp);     // this removes from http_op_set too
        delete fxp;
        fxp = 0;
    }
}

PERS_FILE_XFER_SET::PERS_FILE_XFER_SET(FILE_XFER_SET* p) {
    file_xfers = p;
}

// Run through the set, starting any transfers that need to be
// started and deleting any that have finished
//
bool PERS_FILE_XFER_SET::poll() {
    unsigned int i;
    bool action = false;
    static double last_time=0;

    if (gstate.now - last_time < 1.0) return false;
    last_time = gstate.now;

    for (i=0; i<pers_file_xfers.size(); i++) {
        action |= pers_file_xfers[i]->poll();
    }

    if (action) gstate.set_client_state_dirty("pers_file_xfer_set poll");

    return action;
}

// Insert a PERS_FILE_XFER object into the set.
// We will decide which ones to start when we hit the polling loop
//
int PERS_FILE_XFER_SET::insert(PERS_FILE_XFER* pfx) {
    pers_file_xfers.push_back(pfx);
    return 0;
}

// Remove a PERS_FILE_XFER object from the set.
// What should the action here be?
//
int PERS_FILE_XFER_SET::remove(PERS_FILE_XFER* pfx) {
    vector<PERS_FILE_XFER*>::iterator iter;

    iter = pers_file_xfers.begin();
    while (iter != pers_file_xfers.end()) {
        if (*iter == pfx) {
            iter = pers_file_xfers.erase(iter);
            return 0;
        }
        iter++;
    }
    msg_printf(
        pfx->fip->project, MSG_INTERNAL_ERROR,
        "Persistent file transfer object not found"
    );
    return ERR_NOT_FOUND;
}

// suspend all PERS_FILE_XFERs
//
void PERS_FILE_XFER_SET::suspend() {
    unsigned int i;

    for (i=0; i<pers_file_xfers.size(); i++) {
        pers_file_xfers[i]->suspend();
    }
}

const char *BOINC_RCSID_76edfcfb49 = "$Id$";