From 96c0ded3a8475c41bfbdd7f8a76a17aa978bcb64 Mon Sep 17 00:00:00 2001 From: Eric Heien Date: Wed, 21 Aug 2002 19:12:42 +0000 Subject: [PATCH] Persistent file xfer, minor Mac fixes svn path=/trunk/boinc/; revision=356 --- client/client_state.C | 42 +-------- client/client_state.h | 6 +- client/client_types.C | 28 ++++-- client/client_types.h | 2 +- client/cs_files.C | 19 +++- client/file_xfer.C | 39 +++++++- client/file_xfer.h | 2 + client/http.h | 3 + client/mac/mac_main.h | 2 +- client/main.C | 21 ++--- client/pers_file_xfer.C | 201 +++++++++++++++++++--------------------- client/pers_file_xfer.h | 3 +- lib/error_numbers.h | 2 + 13 files changed, 196 insertions(+), 174 deletions(-) diff --git a/client/client_state.C b/client/client_state.C index 6c56fe9ff7..528fef2a5d 100644 --- a/client/client_state.C +++ b/client/client_state.C @@ -201,23 +201,20 @@ bool CLIENT_STATE::do_something() { check_suspend_activities(); if (!activities_suspended) { + // Call these functions in bottom to top order in + // respect to the FSMs hierarchy net_xfers->poll(999999, nbytes); if (nbytes) action = true; - // If pers_xfers returns true, we've made a change to a - // persistent transfer which must be recorded in the - // client_state.xml file - if (pers_xfers->poll()) { - action = client_state_dirty = true; - } action |= http_ops->poll(); action |= file_xfers->poll(); action |= active_tasks.poll(); action |= active_tasks.poll_time(); action |= scheduler_rpc_poll(); - action |= garbage_collect(); action |= start_apps(); + action |= pers_xfers->poll(); action |= handle_running_apps(); - action |= start_file_xfers(); + action |= handle_pers_file_xfers(); + action |= garbage_collect(); action |= update_results(); write_state_file_if_needed(); } @@ -649,11 +646,9 @@ void CLIENT_STATE::print_counts() { // bool CLIENT_STATE::garbage_collect() { unsigned int i; - PERS_FILE_XFER* pfxp; FILE_INFO* fip; RESULT* rp; WORKUNIT* wup; - vector::iterator pers_iter; vector::iterator result_iter; vector::iterator wu_iter; vector::iterator fi_iter; @@ -668,33 +663,6 @@ bool CLIENT_STATE::garbage_collect() { fip = file_infos[i]; fip->ref_cnt = 0; } - - // delete PERS_FILE_XFERs that have finished and their - // associated FILE_INFO and FILE_XFER objects - // - pers_iter = pers_xfers->pers_file_xfers.begin(); - while (pers_iter != pers_xfers->pers_file_xfers.end()) { - pfxp = *pers_iter; - if (pfxp->xfer_done) { - // Set the status of the related file info to - // ERR_GIVEUP. The failure will be reported to the - // server and related file infos, results, and workunits - // will be deleted if necessary - if (pfxp->pers_xfer_retval == ERR_GIVEUP) { - pfxp->fip->status = ERR_GIVEUP; - } - - pers_iter = pers_xfers->pers_file_xfers.erase(pers_iter); - pfxp->fip->pers_file_xfer = NULL; - delete pfxp; - - // Update the client_state file - client_state_dirty = true; - action = true; - } else { - pers_iter++; - } - } // delete RESULTs that have been finished and reported; // reference-count files referred to by other results diff --git a/client/client_state.h b/client/client_state.h index f3a4342a4c..99238739f1 100644 --- a/client/client_state.h +++ b/client/client_state.h @@ -50,7 +50,8 @@ public: double allowed_disk_usage(); void update_net_stats(bool is_upload, double nbytes, double nsecs); int insert_file_xfer( FILE_XFER *fxp ); - int giveup_after; + unsigned int giveup_after; + bool client_state_dirty; vector projects; vector apps; @@ -72,7 +73,6 @@ private: int version; char* platform_name; unsigned int nslots; - bool client_state_dirty; bool exit_when_idle; bool run_time_test; bool activities_suspended; @@ -97,7 +97,7 @@ private: int app_finished(ACTIVE_TASK&); bool start_apps(); bool handle_running_apps(); - bool start_file_xfers(); + bool handle_pers_file_xfers(); void print_counts(); bool garbage_collect(); bool update_results(); diff --git a/client/client_types.C b/client/client_types.C index bb909dda14..8d54bd998f 100644 --- a/client/client_types.C +++ b/client/client_types.C @@ -19,6 +19,9 @@ #include "windows_cpp.h" +#include +#include + #include "error_numbers.h" #include "file_names.h" #include "filesys.h" @@ -211,16 +214,24 @@ FILE_INFO::FILE_INFO() { FILE_INFO::~FILE_INFO() { } -int FILE_INFO::parse_server_response(char *buf) { - int status = -1; - - parse_double(buf, "", upload_offset); - parse_int(buf, "", status); - // TODO: decide what to do with error string - //if (!parse_str(buf, "", upload_offset) ) return -1; +// Set the appropriate permissions depending on whether +// it's an executable or normal file +// TODO: implement Windows equivalent +int FILE_INFO::set_permissions() { + int retval; + char pathname[256]; - return status; + get_pathname(this, pathname); +#ifndef _WIN32 + if (executable) { + retval = chmod(pathname, S_IEXEC|S_IREAD|S_IWRITE); + } else { + retval = chmod(pathname, S_IREAD|S_IWRITE); + } +#endif + return retval; } + // If from server, make an exact copy of everything // except the start/end tags and the element. // @@ -347,6 +358,7 @@ int FILE_INFO::delete_file() { char path[256]; get_pathname(this, path); + status = FILE_NOT_PRESENT; return file_delete(path); } diff --git a/client/client_types.h b/client/client_types.h index ad7eedfc71..8766746e37 100644 --- a/client/client_types.h +++ b/client/client_types.h @@ -123,7 +123,7 @@ public: FILE_INFO(); ~FILE_INFO(); - int parse_server_response(char*); + int set_permissions(); int parse(FILE*, bool from_server); int write(FILE*, bool to_server); int delete_file(); // attempt to delete the underlying file diff --git a/client/cs_files.C b/client/cs_files.C index 5bd566233b..f07564ae59 100644 --- a/client/cs_files.C +++ b/client/cs_files.C @@ -71,10 +71,10 @@ int verify_downloaded_file(char* pathname, FILE_INFO& file_info) { return 0; } -// scan all FILE_INFOs. -// start downloads and uploads as needed. +// scan all FILE_INFOs and PERS_FILE_XFERs. +// start and finish downloads and uploads as needed. // -bool CLIENT_STATE::start_file_xfers() { +bool CLIENT_STATE::handle_pers_file_xfers() { unsigned int i; FILE_INFO* fip; PERS_FILE_XFER *pfx; @@ -103,5 +103,18 @@ bool CLIENT_STATE::start_file_xfers() { action = true; } } + + for (i=0; ipers_file_xfers.size(); i++) { + pfx = pers_xfers->pers_file_xfers[i]; + // If the transfer finished, remove the PERS_FILE_XFER object + // from the set and delete it + if (pfx->xfer_done) { + pfx->fip->pers_file_xfer = NULL; + pers_xfers->remove(pfx); + delete pfx; + action = true; + } + } + return action; } diff --git a/client/file_xfer.C b/client/file_xfer.C index fc12a6fc30..72ed7de06c 100644 --- a/client/file_xfer.C +++ b/client/file_xfer.C @@ -24,6 +24,7 @@ #include "filesys.h" #include "log_flags.h" #include "file_xfer.h" +#include "parse.h" #include "error_numbers.h" FILE_XFER::FILE_XFER() { @@ -34,6 +35,7 @@ FILE_XFER::FILE_XFER() { fip = NULL; strcpy(pathname,""); strcpy(header,""); + file_size_query = false; //state = ? } @@ -84,6 +86,7 @@ int FILE_XFER::init_upload(FILE_INFO& file_info) { "%s\n", file_info.name ); + file_size_query = true; return HTTP_OP::init_post2(fip->get_url(), header, NULL, 0); } else { sprintf(header, @@ -101,10 +104,24 @@ int FILE_XFER::init_upload(FILE_INFO& file_info) { file_info.nbytes, file_info.upload_offset ); + file_size_query = false; return HTTP_OP::init_post2(fip->get_url(), header, pathname, fip->upload_offset); } } +// Parse the server response in req1 +// +int FILE_XFER::parse_server_response(double &offset) { + int status = -1; + + parse_double(req1, "", offset); + parse_int(req1, "", status); + // TODO: decide what to do with error string + //if (!parse_str(req1, "", upload_offset) ) return -1; + + return status; +} + // Returns the total time that the file xfer has taken // double FILE_XFER::elapsed_time() { @@ -159,6 +176,7 @@ bool FILE_XFER_SET::poll() { unsigned int i; FILE_XFER* fxp; bool action = false; + int retval; for (i=0; ihttp_op_retval); } - if (fxp->http_op_retval == 200) { - fxp->file_xfer_retval = 0; + if (fxp->http_op_retval == HTTP_OK) { + // If this was a file size query, restart the transfer + // using the remote file size information + if (fxp->file_size_query) { + // Parse the server's response. + retval = fxp->parse_server_response(fxp->fip->upload_offset); + + if (retval) { + fxp->fip->upload_offset = -1; + remove(fxp); + i--; + fxp->file_xfer_retval = retval; + } else { + // Restart the upload, using the newly obtained upload_offset + retval = fxp->init_upload(*fxp->fip); + } + } } else { // Remove the transfer from the set. The actual object - // will be removed later by it's associated PERS_FILE_XFER + // will be removed later by it's associated PERS_FILE_XFER? remove(fxp); fxp->file_xfer_retval = fxp->http_op_retval; i--; diff --git a/client/file_xfer.h b/client/file_xfer.h index debcb742ea..9bf53cb583 100644 --- a/client/file_xfer.h +++ b/client/file_xfer.h @@ -40,12 +40,14 @@ public: char pathname[256]; char header[4096]; int state; + bool file_size_query; FILE_XFER(); ~FILE_XFER(); //int init_download(char* url, char* outfile); //int init_upload(char* url, char* infile); + int parse_server_response(double &offset); int init_download(FILE_INFO&); int init_upload(FILE_INFO&); bool file_xfer_done; diff --git a/client/http.h b/client/http.h index 8e85703438..bc2e4ecd11 100644 --- a/client/http.h +++ b/client/http.h @@ -93,6 +93,9 @@ public: #define HTTP_STATE_REPLY_BODY 6 #define HTTP_STATE_DONE 7 +#define HTTP_OK 200 +#define HTTP_RANGE_REQUEST_ERROR 416 + extern int read_http_reply_header(int socket, HTTP_REPLY_HEADER&); #endif diff --git a/client/mac/mac_main.h b/client/mac/mac_main.h index 9acfaf5b2a..3ccb42860f 100644 --- a/client/mac/mac_main.h +++ b/client/mac/mac_main.h @@ -23,7 +23,7 @@ extern "C" { #endif -void InitToolbox(void); +void InitMainWindow(void); void DisplayBOINCStatusWindow (int left, int top, int width, int height); pascal OSStatus MainAppEventHandler(EventHandlerCallRef appHandler, EventRef theEvent, void* appData); pascal void BOINCPollLoopProcessor(EventLoopTimerRef inTimer, void* timeData); diff --git a/client/main.C b/client/main.C index e846add131..f6f9b20e7c 100644 --- a/client/main.C +++ b/client/main.C @@ -71,22 +71,19 @@ int initialize_prefs() { #ifdef __APPLE_CC__ #include "mac_main.h" -mac_main() { +int main() { signal(SIGPIPE, SIG_IGN); read_log_flags(); + + if (gstate.init()) return -1; + + // mac_setup won't return until the main application loop has quit if (!mac_setup ()) return -1; - retval = gstate.init(); - if (retval) exit(retval); - while (1) { - if (!gstate.do_something()) { - } - if (gstate.time_to_exit() || user_requested_exit) { - break; - } - } + // Afterwards, we clean up and exit mac_cleanup (); } -#endif + +#else int main(int argc, char** argv) { int retval; @@ -112,3 +109,5 @@ int main(int argc, char** argv) { gstate.exit(); return 0; } + +#endif diff --git a/client/pers_file_xfer.C b/client/pers_file_xfer.C index cb9f970578..b27e1362e1 100644 --- a/client/pers_file_xfer.C +++ b/client/pers_file_xfer.C @@ -22,9 +22,6 @@ #include #include -#include -#include - #include "client_state.h" #include "client_types.h" #include "error_numbers.h" @@ -51,7 +48,6 @@ int PERS_FILE_XFER::init(FILE_INFO* the_file, bool is_file_upload) { first_request_time = time(NULL); next_request_time = first_request_time; is_upload = is_file_upload; - pers_xfer_retval = -1; xfer_done = false; return 0; @@ -66,34 +62,35 @@ bool PERS_FILE_XFER::start_xfer() { // Decide whether to start a new file transfer // - if (gstate.start_new_file_xfer()) { - // Create a new FILE_XFER object and initialize a - // download or upload for the persistent file transfer - // - file_xfer = new FILE_XFER; - if (is_upload) { - retval = file_xfer->init_upload(*fip); - } else { - retval = file_xfer->init_download(*fip); - } - if (!retval) { - retval = gstate.insert_file_xfer(file_xfer); - } - if (retval) { - fprintf( - stderr, "couldn't start %s for %s: error %d\n", - (is_upload ? "upload" : "download"), fip->get_url(), retval + if (!gstate.start_new_file_xfer()) { + return false; + } + + // Create a new FILE_XFER object and initialize a + // download or upload for the persistent file transfer + // + file_xfer = new FILE_XFER; + if (is_upload) { + retval = file_xfer->init_upload(*fip); + } else { + retval = file_xfer->init_download(*fip); + } + if (retval) { + fprintf( + stderr, "couldn't start %s for %s: error %d\n", + (is_upload ? "upload" : "download"), fip->get_url(), retval + ); + } else { + retval = gstate.insert_file_xfer(file_xfer); + if (retval) return false; + fxp = file_xfer; + if (log_flags.file_xfer) { + printf( + "started %s of %s\n", + (is_upload ? "upload" : "download"), fip->get_url() ); - } else { - fxp = file_xfer; - if (log_flags.file_xfer) { - printf( - "started %s of %s\n", - (is_upload ? "upload" : "download"), fip->get_url() - ); - } - return true; } + return true; } return false; } @@ -103,11 +100,10 @@ bool PERS_FILE_XFER::start_xfer() { // If it has finished or failed, then deal with it appropriately // bool PERS_FILE_XFER::poll(unsigned int now) { - double exp_backoff; int retval; char pathname[256]; - - if (!fxp) { + + if (!fxp && !xfer_done) { // No file xfer is active. // We must be waiting after a failure. // See if it's time to try again. @@ -120,42 +116,23 @@ bool PERS_FILE_XFER::poll(unsigned int now) { } if (fxp->file_xfer_done) { + if (log_flags.file_xfer) { + printf( "file transfer done for %s; retval %d\n", + fip->get_url(), fxp->file_xfer_retval ); + } if (fxp->file_xfer_retval == 0) { - // The transfer finished with no errors. We will clean up the - // PERS_FILE_XFER object in garbage_collect() later - // - if (log_flags.file_xfer) { - printf( - "file transfer done for %s; retval %d\n", - fip->get_url(), fxp->file_xfer_retval - ); - } + // The transfer finished with no errors. if (fip->generated_locally) { - // If this was a file size query, redo the transfer with - // the information - if (fip->upload_offset<0) { - // Parse the server's response - // TODO: handle error response - fip->parse_server_response(fxp->req1); + // If the file was generated locally (for upload), update stats + // and delete the local copy of the file if needed + gstate.update_net_stats(true, fip->nbytes, fxp->elapsed_time()); - // Reset the file transfer so we start the actual transfer - fxp = NULL; - } else { - // Parse the server's response - // TODO: handle error response - fip->parse_server_response(fxp->req1); - - // If the file was generated locally (for upload), update stats - // and delete the local copy of the file if needed - // - gstate.update_net_stats(true, fip->nbytes, fxp->elapsed_time()); - - // file has been uploaded - delete if not sticky - if (!fip->sticky) { - fip->delete_file(); - } - fip->uploaded = true; + // file has been uploaded - delete if not sticky + if (!fip->sticky) { + fip->delete_file(); } + fip->uploaded = true; + xfer_done = true; } else { // Otherwise we downloaded the file. Update stats, verify // the file with RSA or MD5, and change permissions @@ -171,53 +148,18 @@ bool PERS_FILE_XFER::poll(unsigned int now) { } // Set the appropriate permissions depending on whether // it's an executable or normal file - if (fip->executable) { -#ifndef _WIN32 - retval = chmod(pathname, S_IEXEC|S_IREAD|S_IWRITE); -#endif - } else { - get_pathname(fip, pathname); -#ifndef _WIN32 - retval = chmod(pathname, S_IREAD|S_IWRITE); -#endif - } + retval = fip->set_permissions(); fip->status = FILE_PRESENT; } + xfer_done = true; } - xfer_done = true; - pers_xfer_retval = 0; } else { // file xfer failed. - // If it was a bad range request, delete the file and start over - if (fxp->file_xfer_retval == 416) { - fip->delete_file(); - // TODO: remove fxp from file_xfer_set here and at other - // necessary places - fxp = NULL; - } else { - // Cycle to the next URL to try - fip->current_url = (fip->current_url + 1)%fip->urls.size(); - - // If we reach the URL that we started at, then we have tried all - // servers without success - if (fip->current_url == fip->start_url) { - nretry++; - exp_backoff = exp(((double)rand()/(double)RAND_MAX)*nretry); - // Do an exponential backoff of e^nretry seconds, keeping - // within the bounds of PERS_RETRY_DELAY_MIN and - // PERS_RETRY_DELAY_MAX - next_request_time = now+(int)max(PERS_RETRY_DELAY_MIN,min(PERS_RETRY_DELAY_MAX,exp_backoff)); - } - } - - // See if it's time to give up on the persistent file xfer - // - if ((now - first_request_time) > gstate.giveup_after) { - xfer_done = true; - pers_xfer_retval = ERR_GIVEUP; - } + handle_xfer_failure(now); } - // Disassociate the finished file transfer + // remove fxp from file_xfer_set here and deallocate it + gstate.file_xfers->remove(fxp); + delete fxp; fxp = NULL; return true; @@ -226,6 +168,48 @@ bool PERS_FILE_XFER::poll(unsigned int now) { return false; } +// Handle a transfer failure +// +void PERS_FILE_XFER::handle_xfer_failure(unsigned int cur_time) { + // If it was a bad range request, delete the file and start over + if (fxp->file_xfer_retval == HTTP_RANGE_REQUEST_ERROR) { + fip->delete_file(); + } + + retry_and_backoff(cur_time); + + // See if it's time to give up on the persistent file xfer + // + if ((cur_time - first_request_time) > gstate.giveup_after) { + // Set the associated files status to a ERR_GIVEUP failure + fip->status = ERR_GIVEUP; + xfer_done = true; + } +} + +// Cycle to the next URL, or if we've hit all URLs in this cycle, +// backoff and try again later +// +int PERS_FILE_XFER::retry_and_backoff(unsigned int cur_time) { + double exp_backoff; + + // Cycle to the next URL to try + fip->current_url = (fip->current_url + 1)%fip->urls.size(); + + // If we reach the URL that we started at, then we have tried all + // servers without success + if (fip->current_url == fip->start_url) { + nretry++; + exp_backoff = exp(((double)rand()/(double)RAND_MAX)*nretry); + // Do an exponential backoff of e^nretry seconds, keeping + // within the bounds of PERS_RETRY_DELAY_MIN and + // PERS_RETRY_DELAY_MAX + next_request_time = cur_time+(int)max(PERS_RETRY_DELAY_MIN,min(PERS_RETRY_DELAY_MAX,exp_backoff)); + } + + return 0; +} + // Parse XML information about a single persistent file transfer // int PERS_FILE_XFER::parse(FILE* fin) { @@ -259,6 +243,9 @@ PERS_FILE_XFER_SET::PERS_FILE_XFER_SET(FILE_XFER_SET* p) { file_xfers = p; } +// Run through the set, starting any transfers that need to be +// started and deleting any that have finished +// bool PERS_FILE_XFER_SET::poll() { unsigned int i; bool action = false; @@ -268,6 +255,8 @@ bool PERS_FILE_XFER_SET::poll() { action |= pers_file_xfers[i]->poll(now); } + if (action) gstate.client_state_dirty = true; + return action; } diff --git a/client/pers_file_xfer.h b/client/pers_file_xfer.h index d1963dae78..8525e2241d 100644 --- a/client/pers_file_xfer.h +++ b/client/pers_file_xfer.h @@ -44,13 +44,14 @@ class PERS_FILE_XFER { bool is_upload; public: - int pers_xfer_retval; bool xfer_done; FILE_XFER* fxp; // nonzero if file xfer in progress FILE_INFO* fip; int init(FILE_INFO*, bool is_file_upload); bool poll(unsigned int now); + void handle_xfer_failure(unsigned int cur_time); + int retry_and_backoff(unsigned int cur_time); int write(FILE* fout); int parse(FILE* fin); bool start_xfer(); diff --git a/lib/error_numbers.h b/lib/error_numbers.h index 2ea2bb36d0..952458bf26 100755 --- a/lib/error_numbers.h +++ b/lib/error_numbers.h @@ -29,7 +29,9 @@ #define ERR_RENAME -109 #define ERR_UNLINK -110 #define ERR_OPENDIR -111 +// Unexpected XML tag or XML format #define ERR_XML_PARSE -112 +// Couldn't resolve hostname #define ERR_GETHOSTBYNAME -113 // too much time has elapsed without progress on file xfer #define ERR_GIVEUP -114