Rooba
/
boinc
mirror of https://github.com/BOINC/boinc.git
1
0
Fork 0
Code Issues 1 Packages Projects Releases Wiki Activity

Persistent file xfer, minor Mac fixes 

svn path=/trunk/boinc/; revision=356
This commit is contained in:
Eric Heien 2002-08-21 19:12:42 +00:00
parent c9fea02a4e
commit 96c0ded3a8
13 changed files with 196 additions and 174 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
client/client_state.C
View File

@ -201,23 +201,20 @@ bool CLIENT_STATE::do_something() {
check_suspend_activities();
if (!activities_suspended) {
// Call these functions in bottom to top order in
// respect to the FSMs hierarchy
net_xfers->poll(999999, nbytes);
if (nbytes) action = true;
// If pers_xfers returns true, we've made a change to a
// persistent transfer which must be recorded in the
// client_state.xml file
if (pers_xfers->poll()) {
action = client_state_dirty = true;
}
action |= http_ops->poll();
action |= file_xfers->poll();
action |= active_tasks.poll();
action |= active_tasks.poll_time();
action |= scheduler_rpc_poll();
action |= garbage_collect();
action |= start_apps();
action |= pers_xfers->poll();
action |= handle_running_apps();
action |= start_file_xfers();
action |= handle_pers_file_xfers();
action |= garbage_collect();
action |= update_results();
write_state_file_if_needed();
}
@ -649,11 +646,9 @@ void CLIENT_STATE::print_counts() {
//
bool CLIENT_STATE::garbage_collect() {
unsigned int i;
PERS_FILE_XFER* pfxp;
FILE_INFO* fip;
RESULT* rp;
WORKUNIT* wup;
vector<PERS_FILE_XFER*>::iterator pers_iter;
vector<RESULT*>::iterator result_iter;
vector<WORKUNIT*>::iterator wu_iter;
vector<FILE_INFO*>::iterator fi_iter;
@ -668,33 +663,6 @@ bool CLIENT_STATE::garbage_collect() {
fip = file_infos[i];
fip->ref_cnt = 0;
}
// delete PERS_FILE_XFERs that have finished and their
// associated FILE_INFO and FILE_XFER objects
//
pers_iter = pers_xfers->pers_file_xfers.begin();
while (pers_iter != pers_xfers->pers_file_xfers.end()) {
pfxp = *pers_iter;
if (pfxp->xfer_done) {
// Set the status of the related file info to
// ERR_GIVEUP. The failure will be reported to the
// server and related file infos, results, and workunits
// will be deleted if necessary
if (pfxp->pers_xfer_retval == ERR_GIVEUP) {
pfxp->fip->status = ERR_GIVEUP;
}
pers_iter = pers_xfers->pers_file_xfers.erase(pers_iter);
pfxp->fip->pers_file_xfer = NULL;
delete pfxp;
// Update the client_state file
client_state_dirty = true;
action = true;
} else {
pers_iter++;
}
}
// delete RESULTs that have been finished and reported;
// reference-count files referred to by other results

6
client/client_state.h
View File

@ -50,7 +50,8 @@ public:
double allowed_disk_usage();
void update_net_stats(bool is_upload, double nbytes, double nsecs);
int insert_file_xfer( FILE_XFER *fxp );
int giveup_after;
unsigned int giveup_after;
bool client_state_dirty;
vector<PROJECT*> projects;
vector<APP*> apps;
@ -72,7 +73,6 @@ private:
int version;
char* platform_name;
unsigned int nslots;
bool client_state_dirty;
bool exit_when_idle;
bool run_time_test;
bool activities_suspended;
@ -97,7 +97,7 @@ private:
int app_finished(ACTIVE_TASK&);
bool start_apps();
bool handle_running_apps();
bool start_file_xfers();
bool handle_pers_file_xfers();
void print_counts();
bool garbage_collect();
bool update_results();

28
client/client_types.C
View File

@ -19,6 +19,9 @@
#include "windows_cpp.h"
#include <sys/stat.h>
#include <sys/types.h>
#include "error_numbers.h"
#include "file_names.h"
#include "filesys.h"
@ -211,16 +214,24 @@ FILE_INFO::FILE_INFO() {
FILE_INFO::~FILE_INFO() {
}
int FILE_INFO::parse_server_response(char *buf) {
int status = -1;
parse_double(buf, "<nbytes>", upload_offset);
parse_int(buf, "<status>", status);
// TODO: decide what to do with error string
//if (!parse_str(buf, "<error>", upload_offset) ) return -1;
// Set the appropriate permissions depending on whether
// it's an executable or normal file
// TODO: implement Windows equivalent
int FILE_INFO::set_permissions() {
int retval;
char pathname[256];
return status;
get_pathname(this, pathname);
#ifndef _WIN32
if (executable) {
retval = chmod(pathname, S_IEXEC|S_IREAD|S_IWRITE);
} else {
retval = chmod(pathname, S_IREAD|S_IWRITE);
}
#endif
return retval;
}
// If from server, make an exact copy of everything
// except the start/end tags and the <xml_signature> element.
//
@ -347,6 +358,7 @@ int FILE_INFO::delete_file() {
char path[256];
get_pathname(this, path);
status = FILE_NOT_PRESENT;
return file_delete(path);
}

2
client/client_types.h
View File

@ -123,7 +123,7 @@ public:
FILE_INFO();
~FILE_INFO();
int parse_server_response(char*);
int set_permissions();
int parse(FILE*, bool from_server);
int write(FILE*, bool to_server);
int delete_file(); // attempt to delete the underlying file

19
client/cs_files.C
View File

@ -71,10 +71,10 @@ int verify_downloaded_file(char* pathname, FILE_INFO& file_info) {
return 0;
}
// scan all FILE_INFOs.
// start downloads and uploads as needed.
// scan all FILE_INFOs and PERS_FILE_XFERs.
// start and finish downloads and uploads as needed.
//
bool CLIENT_STATE::start_file_xfers() {
bool CLIENT_STATE::handle_pers_file_xfers() {
unsigned int i;
FILE_INFO* fip;
PERS_FILE_XFER *pfx;
@ -103,5 +103,18 @@ bool CLIENT_STATE::start_file_xfers() {
action = true;
}
}
for (i=0; i<pers_xfers->pers_file_xfers.size(); i++) {
pfx = pers_xfers->pers_file_xfers[i];
// If the transfer finished, remove the PERS_FILE_XFER object
// from the set and delete it
if (pfx->xfer_done) {
pfx->fip->pers_file_xfer = NULL;
pers_xfers->remove(pfx);
delete pfx;
action = true;
}
}
return action;
}

39
client/file_xfer.C
View File

@ -24,6 +24,7 @@
#include "filesys.h"
#include "log_flags.h"
#include "file_xfer.h"
#include "parse.h"
#include "error_numbers.h"
FILE_XFER::FILE_XFER() {
@ -34,6 +35,7 @@ FILE_XFER::FILE_XFER() {
fip = NULL;
strcpy(pathname,"");
strcpy(header,"");
file_size_query = false;
//state = ?
}
@ -84,6 +86,7 @@ int FILE_XFER::init_upload(FILE_INFO& file_info) {
"<file_size_req>%s</file_size_req>\n",
file_info.name
);
file_size_query = true;
return HTTP_OP::init_post2(fip->get_url(), header, NULL, 0);
} else {
sprintf(header,
@ -101,10 +104,24 @@ int FILE_XFER::init_upload(FILE_INFO& file_info) {
file_info.nbytes,
file_info.upload_offset
);
file_size_query = false;
return HTTP_OP::init_post2(fip->get_url(), header, pathname, fip->upload_offset);
}
}
// Parse the server response in req1
//
int FILE_XFER::parse_server_response(double &offset) {
int status = -1;
parse_double(req1, "<nbytes>", offset);
parse_int(req1, "<status>", status);
// TODO: decide what to do with error string
//if (!parse_str(req1, "<error>", upload_offset) ) return -1;
return status;
}
// Returns the total time that the file xfer has taken
//
double FILE_XFER::elapsed_time() {
@ -159,6 +176,7 @@ bool FILE_XFER_SET::poll() {
unsigned int i;
FILE_XFER* fxp;
bool action = false;
int retval;
for (i=0; i<file_xfers.size(); i++) {
fxp = file_xfers[i];
@ -169,11 +187,26 @@ bool FILE_XFER_SET::poll() {
if (log_flags.file_xfer_debug) {
printf("http retval: %d\n", fxp->http_op_retval);
}
if (fxp->http_op_retval == 200) {
fxp->file_xfer_retval = 0;
if (fxp->http_op_retval == HTTP_OK) {
// If this was a file size query, restart the transfer
// using the remote file size information
if (fxp->file_size_query) {
// Parse the server's response.
retval = fxp->parse_server_response(fxp->fip->upload_offset);
if (retval) {
fxp->fip->upload_offset = -1;
remove(fxp);
i--;
fxp->file_xfer_retval = retval;
} else {
// Restart the upload, using the newly obtained upload_offset
retval = fxp->init_upload(*fxp->fip);
}
}
} else {
// Remove the transfer from the set. The actual object
// will be removed later by it's associated PERS_FILE_XFER
// will be removed later by it's associated PERS_FILE_XFER?
remove(fxp);
fxp->file_xfer_retval = fxp->http_op_retval;
i--;

2
client/file_xfer.h
View File

@ -40,12 +40,14 @@ public:
char pathname[256];
char header[4096];
int state;
bool file_size_query;
FILE_XFER();
~FILE_XFER();
//int init_download(char* url, char* outfile);
//int init_upload(char* url, char* infile);
int parse_server_response(double &offset);
int init_download(FILE_INFO&);
int init_upload(FILE_INFO&);
bool file_xfer_done;

3
client/http.h
View File

@ -93,6 +93,9 @@ public:
#define HTTP_STATE_REPLY_BODY 6
#define HTTP_STATE_DONE 7
#define HTTP_OK 200
#define HTTP_RANGE_REQUEST_ERROR 416
extern int read_http_reply_header(int socket, HTTP_REPLY_HEADER&);
#endif

2
client/mac/mac_main.h
View File

@ -23,7 +23,7 @@
extern "C" {
#endif
void InitToolbox(void);
void InitMainWindow(void);
void DisplayBOINCStatusWindow (int left, int top, int width, int height);
pascal OSStatus MainAppEventHandler(EventHandlerCallRef appHandler, EventRef theEvent, void* appData);
pascal void BOINCPollLoopProcessor(EventLoopTimerRef inTimer, void* timeData);

21
client/main.C
View File

@ -71,22 +71,19 @@ int initialize_prefs() {
#ifdef __APPLE_CC__
#include "mac_main.h"
mac_main() {
int main() {
signal(SIGPIPE, SIG_IGN);
read_log_flags();
if (gstate.init()) return -1;
// mac_setup won't return until the main application loop has quit
if (!mac_setup ()) return -1;
retval = gstate.init();
if (retval) exit(retval);
while (1) {
if (!gstate.do_something()) {
}
if (gstate.time_to_exit() || user_requested_exit) {
break;
}
}
// Afterwards, we clean up and exit
mac_cleanup ();
}
#endif
#else
int main(int argc, char** argv) {
int retval;
@ -112,3 +109,5 @@ int main(int argc, char** argv) {
gstate.exit();
return 0;
}
#endif

201
client/pers_file_xfer.C
View File

@ -22,9 +22,6 @@
#include <math.h>
#include <stdlib.h>
#include <sys/stat.h>
#include <sys/types.h>
#include "client_state.h"
#include "client_types.h"
#include "error_numbers.h"
@ -51,7 +48,6 @@ int PERS_FILE_XFER::init(FILE_INFO* the_file, bool is_file_upload) {
first_request_time = time(NULL);
next_request_time = first_request_time;
is_upload = is_file_upload;
pers_xfer_retval = -1;
xfer_done = false;
return 0;
@ -66,34 +62,35 @@ bool PERS_FILE_XFER::start_xfer() {
// Decide whether to start a new file transfer
//
if (gstate.start_new_file_xfer()) {
// Create a new FILE_XFER object and initialize a
// download or upload for the persistent file transfer
//
file_xfer = new FILE_XFER;
if (is_upload) {
retval = file_xfer->init_upload(*fip);
} else {
retval = file_xfer->init_download(*fip);
}
if (!retval) {
retval = gstate.insert_file_xfer(file_xfer);
}
if (retval) {
fprintf(
stderr, "couldn't start %s for %s: error %d\n",
(is_upload ? "upload" : "download"), fip->get_url(), retval
if (!gstate.start_new_file_xfer()) {
return false;
}
// Create a new FILE_XFER object and initialize a
// download or upload for the persistent file transfer
//
file_xfer = new FILE_XFER;
if (is_upload) {
retval = file_xfer->init_upload(*fip);
} else {
retval = file_xfer->init_download(*fip);
}
if (retval) {
fprintf(
stderr, "couldn't start %s for %s: error %d\n",
(is_upload ? "upload" : "download"), fip->get_url(), retval
);
} else {
retval = gstate.insert_file_xfer(file_xfer);
if (retval) return false;
fxp = file_xfer;
if (log_flags.file_xfer) {
printf(
"started %s of %s\n",
(is_upload ? "upload" : "download"), fip->get_url()
);
} else {
fxp = file_xfer;
if (log_flags.file_xfer) {
printf(
"started %s of %s\n",
(is_upload ? "upload" : "download"), fip->get_url()
);
}
return true;
}
return true;
}
return false;
}
@ -103,11 +100,10 @@ bool PERS_FILE_XFER::start_xfer() {
// If it has finished or failed, then deal with it appropriately
//
bool PERS_FILE_XFER::poll(unsigned int now) {
double exp_backoff;
int retval;
char pathname[256];
if (!fxp) {
if (!fxp && !xfer_done) {
// No file xfer is active.
// We must be waiting after a failure.
// See if it's time to try again.
@ -120,42 +116,23 @@ bool PERS_FILE_XFER::poll(unsigned int now) {
}
if (fxp->file_xfer_done) {
if (log_flags.file_xfer) {
printf( "file transfer done for %s; retval %d\n",
fip->get_url(), fxp->file_xfer_retval );
}
if (fxp->file_xfer_retval == 0) {
// The transfer finished with no errors. We will clean up the
// PERS_FILE_XFER object in garbage_collect() later
//
if (log_flags.file_xfer) {
printf(
"file transfer done for %s; retval %d\n",
fip->get_url(), fxp->file_xfer_retval
);
}
// The transfer finished with no errors.
if (fip->generated_locally) {
// If this was a file size query, redo the transfer with
// the information
if (fip->upload_offset<0) {
// Parse the server's response
// TODO: handle error response
fip->parse_server_response(fxp->req1);
// If the file was generated locally (for upload), update stats
// and delete the local copy of the file if needed
gstate.update_net_stats(true, fip->nbytes, fxp->elapsed_time());
// Reset the file transfer so we start the actual transfer
fxp = NULL;
} else {
// Parse the server's response
// TODO: handle error response
fip->parse_server_response(fxp->req1);
// If the file was generated locally (for upload), update stats
// and delete the local copy of the file if needed
//
gstate.update_net_stats(true, fip->nbytes, fxp->elapsed_time());
// file has been uploaded - delete if not sticky
if (!fip->sticky) {
fip->delete_file();
}
fip->uploaded = true;
// file has been uploaded - delete if not sticky
if (!fip->sticky) {
fip->delete_file();
}
fip->uploaded = true;
xfer_done = true;
} else {
// Otherwise we downloaded the file. Update stats, verify
// the file with RSA or MD5, and change permissions
@ -171,53 +148,18 @@ bool PERS_FILE_XFER::poll(unsigned int now) {
}
// Set the appropriate permissions depending on whether
// it's an executable or normal file
if (fip->executable) {
#ifndef _WIN32
retval = chmod(pathname, S_IEXEC|S_IREAD|S_IWRITE);
#endif
} else {
get_pathname(fip, pathname);
#ifndef _WIN32
retval = chmod(pathname, S_IREAD|S_IWRITE);
#endif
}
retval = fip->set_permissions();
fip->status = FILE_PRESENT;
}
xfer_done = true;
}
xfer_done = true;
pers_xfer_retval = 0;
} else {
// file xfer failed.
// If it was a bad range request, delete the file and start over
if (fxp->file_xfer_retval == 416) {
fip->delete_file();
// TODO: remove fxp from file_xfer_set here and at other
// necessary places
fxp = NULL;
} else {
// Cycle to the next URL to try
fip->current_url = (fip->current_url + 1)%fip->urls.size();
// If we reach the URL that we started at, then we have tried all
// servers without success
if (fip->current_url == fip->start_url) {
nretry++;
exp_backoff = exp(((double)rand()/(double)RAND_MAX)*nretry);
// Do an exponential backoff of e^nretry seconds, keeping
// within the bounds of PERS_RETRY_DELAY_MIN and
// PERS_RETRY_DELAY_MAX
next_request_time = now+(int)max(PERS_RETRY_DELAY_MIN,min(PERS_RETRY_DELAY_MAX,exp_backoff));
}
}
// See if it's time to give up on the persistent file xfer
//
if ((now - first_request_time) > gstate.giveup_after) {
xfer_done = true;
pers_xfer_retval = ERR_GIVEUP;
}
handle_xfer_failure(now);
}
// Disassociate the finished file transfer
// remove fxp from file_xfer_set here and deallocate it
gstate.file_xfers->remove(fxp);
delete fxp;
fxp = NULL;
return true;
@ -226,6 +168,48 @@ bool PERS_FILE_XFER::poll(unsigned int now) {
return false;
}
// Handle a transfer failure
//
void PERS_FILE_XFER::handle_xfer_failure(unsigned int cur_time) {
// If it was a bad range request, delete the file and start over
if (fxp->file_xfer_retval == HTTP_RANGE_REQUEST_ERROR) {
fip->delete_file();
}
retry_and_backoff(cur_time);
// See if it's time to give up on the persistent file xfer
//
if ((cur_time - first_request_time) > gstate.giveup_after) {
// Set the associated files status to a ERR_GIVEUP failure
fip->status = ERR_GIVEUP;
xfer_done = true;
}
}
// Cycle to the next URL, or if we've hit all URLs in this cycle,
// backoff and try again later
//
int PERS_FILE_XFER::retry_and_backoff(unsigned int cur_time) {
double exp_backoff;
// Cycle to the next URL to try
fip->current_url = (fip->current_url + 1)%fip->urls.size();
// If we reach the URL that we started at, then we have tried all
// servers without success
if (fip->current_url == fip->start_url) {
nretry++;
exp_backoff = exp(((double)rand()/(double)RAND_MAX)*nretry);
// Do an exponential backoff of e^nretry seconds, keeping
// within the bounds of PERS_RETRY_DELAY_MIN and
// PERS_RETRY_DELAY_MAX
next_request_time = cur_time+(int)max(PERS_RETRY_DELAY_MIN,min(PERS_RETRY_DELAY_MAX,exp_backoff));
}
return 0;
}
// Parse XML information about a single persistent file transfer
//
int PERS_FILE_XFER::parse(FILE* fin) {
@ -259,6 +243,9 @@ PERS_FILE_XFER_SET::PERS_FILE_XFER_SET(FILE_XFER_SET* p) {
file_xfers = p;
}
// Run through the set, starting any transfers that need to be
// started and deleting any that have finished
//
bool PERS_FILE_XFER_SET::poll() {
unsigned int i;
bool action = false;
@ -268,6 +255,8 @@ bool PERS_FILE_XFER_SET::poll() {
action |= pers_file_xfers[i]->poll(now);
}
if (action) gstate.client_state_dirty = true;
return action;
}

3
client/pers_file_xfer.h
View File

@ -44,13 +44,14 @@ class PERS_FILE_XFER {
bool is_upload;
public:
int pers_xfer_retval;
bool xfer_done;
FILE_XFER* fxp; // nonzero if file xfer in progress
FILE_INFO* fip;
int init(FILE_INFO*, bool is_file_upload);
bool poll(unsigned int now);
void handle_xfer_failure(unsigned int cur_time);
int retry_and_backoff(unsigned int cur_time);
int write(FILE* fout);
int parse(FILE* fin);
bool start_xfer();

2
lib/error_numbers.h
View File

@ -29,7 +29,9 @@
#define ERR_RENAME -109
#define ERR_UNLINK -110
#define ERR_OPENDIR -111
// Unexpected XML tag or XML format
#define ERR_XML_PARSE -112
// Couldn't resolve hostname
#define ERR_GETHOSTBYNAME -113
// too much time has elapsed without progress on file xfer
#define ERR_GIVEUP -114