diff --git a/TODO b/TODO index ce5c644e65..47d4e52c19 100644 --- a/TODO +++ b/TODO @@ -1,5 +1,14 @@ HIGH-PRIORITY (must be done to support SETI@home) +- Code-signing + research tools for code-signing + +- Upload authentication + Each result contains a "certificate", signed with project key, giving + - list of: file name, max size + - min, max times to xfer + modify put program to decrypt certificate, enforce name/size/time limits + - Network retry policies can't download file: when to give up? how to retry? exponential backoff @@ -142,6 +151,10 @@ LOW-PRIORITY - test - implement file upload/download requests + This can be done in the current WU/result paradigm; + WU has a special "null" application + use sticky input files to download; + use upload-when-done output files to upload - preferences finish PHP web interface diff --git a/api/api.C b/api/api.C index 424641c906..76e0564162 100644 --- a/api/api.C +++ b/api/api.C @@ -22,6 +22,7 @@ #include #include +#include "../sched/parse.h" #include "api.h" int MFILE::open(char* path, char* mode) { @@ -81,3 +82,77 @@ int MFILE::flush() { len = 0; return fflush(f); } + +void write_core_file(FILE* f, APP_IN& ai) { + fprintf(f, + "%d\n" + "%d\n" + "%f\n" + "%f\n" + "%f\n", + ai.graphics.xsize, + ai.graphics.ysize, + ai.graphics.refresh_period, + ai.checkpoint_period, + ai.poll_period + ); +} + +void parse_core_file(FILE* f, APP_IN& ai) { + char buf[256]; + + while (fgets(buf, 256, f)) { + if (match_tag(buf, "")) { + strcpy(ai.app_preferences, ""); + while (fgets(buf, 256, f)) { + if (match_tag(buf, "")) break; + strcat(ai.app_preferences, buf); + } + continue; + } + else if (parse_int(buf, "", ai.graphics.xsize)) continue; + else if (parse_int(buf, "", ai.graphics.ysize)) continue; + else if (parse_double(buf, "", ai.graphics.refresh_period)) continue; + else if (parse_double(buf, "", ai.checkpoint_period)) continue; + else if (parse_double(buf, "", ai.poll_period)) continue; + else fprintf(stderr, "read_core_file: unrecognized %s", buf); + } +} + +void write_app_file(FILE* f, APP_OUT& ao) { + fprintf(f, + "%f\n" + "%f\n", + ao.percent_done, + ao.cpu_time_at_checkpoint + ); + if (ao.checkpointed) { + fprintf(f, "\n"); + } +} + +void parse_app_file(FILE* f, APP_OUT& ao) { +} + +void boinc_init(APP_IN& ai) { + FILE* f; + + memset(&ai, 0, sizeof(ai)); + f = fopen(CORE_TO_APP_FILE, "r"); + if (f) { + parse_core_file(f, ai); + unlink(CORE_TO_APP_FILE); + } +} + +double boinc_time() { + return double_time(); +} + +void boinc_poll(APP_IN& ai, APP_OUT& ao) { + FILE* f; + + f = fopen("_app_temp", "w"); + write_app_file(f, ao); + rename("_app_temp", APP_TO_CORE_FILE); +} diff --git a/api/api.h b/api/api.h index 5d86c9457b..d5273f8d1c 100644 --- a/api/api.h +++ b/api/api.h @@ -17,17 +17,17 @@ // Contributor(s): // +#include + +#ifndef BOINC_API +#define BOINC_API + // MFILE supports a primitive form of checkpointing. // Write all your output (and restart file) to MFILEs. // The output is buffered in memory. // Then close all the MFILEs; // all the buffers will be flushed to disk, almost atomically. -#include - -#ifndef BOINC_API -#define BOINC_API - class MFILE { char* buf; int len; @@ -42,4 +42,44 @@ public: int flush(); }; +// An application that wants to be well-behaved should do the following: +// +// - call boinc_init() at startup +// - call boinc_time() periodically. +// This is cheap - it gets the time of day. +// - checkpoint as often as requested by core +// - boinc_poll(): +// Call this as often as requested by core + +struct APP_IN_GRAPHICS { + int xsize; + int ysize; + double refresh_period; + char shmem_seg_name[32]; +}; + +struct APP_OUT_GRAPHICS { +}; + +struct APP_IN { + char app_preferences[4096]; + APP_IN_GRAPHICS graphics; + double checkpoint_period; // recommended checkpoint period + double poll_period; // recommended poll period +}; + +struct APP_OUT { + double percent_done; + double cpu_time_at_checkpoint; + bool checkpointed; // true iff checkpointed since last call +}; + +void boinc_init(APP_IN&); +double boinc_time(); +void boinc_poll(APP_IN&, APP_OUT&); +double boinc_cpu_time(); + +#define CORE_TO_APP_FILE "core_to_app.xml" +#define APP_TO_CORE_FILE "app_to_core.xml" + #endif diff --git a/checkin_notes b/checkin_notes index 6475859d20..3828ae4e41 100755 --- a/checkin_notes +++ b/checkin_notes @@ -434,3 +434,48 @@ Michael Gary June 08, 2002 server_types.C (added) server_types.h (added) show_shmem.C (added) + +David A June 9 2002 + - added support for multiple URLs in a FILE_INFO + (e.g. multiple servers from which the file can be downloaded) + - started work on "persistent file transfer": a layer on top of + FILE_XFER that manages restarting from failed connections, + and that implements a give-up policy + - added offset arguments to GET and PUT HTTP operations. + NOTE: this will work fine for downloading files (GET) + but we'll have to use something else for upload, + since the standard PUT handler doesn't do offsets, + and we need security functionality in any case. + - added preliminary version of application API for + communicating with core client. + - use tags instead of and + (makes things simpler) + + TODO + notes + api/ + api.C,h + client/ + configure.in + client_state.C + client_types.C,h + cs_files.C + error_numbers.h + file_xfer.h + http.C,h + pers_file_xfer.C,h + test_http.C + html_ops/ + db.php + html_user/ + db.inc + test/ + 1sec_result + account2.xml + *_result + *_wu + init.inc + laptop_prefs.xml + test_*.php + tools/ + add.C diff --git a/client/client_state.C b/client/client_state.C index 64cfaee11a..e4e1a00928 100644 --- a/client/client_state.C +++ b/client/client_state.C @@ -224,6 +224,9 @@ int CLIENT_STATE::write_state_file() { FILE* f = fopen(STATE_FILE_TEMP, "w"); int retval; + if (log_flags.state_debug) { + printf("Writing state file\n"); + } if (!f) { fprintf(stderr, "can't open temp state file: %s\n", STATE_FILE_TEMP); return ERR_FOPEN; @@ -264,6 +267,9 @@ int CLIENT_STATE::write_state_file() { fprintf(f, "\n"); fclose(f); retval = rename(STATE_FILE_TEMP, STATE_FILE_NAME); + if (log_flags.state_debug) { + printf("Done writing state file\n"); + } if (retval) return ERR_RENAME; return 0; } @@ -497,7 +503,9 @@ bool CLIENT_STATE::garbage_collect() { while (wu_iter != workunits.end()) { wup = *wu_iter; if (wup->ref_cnt == 0) { - if (log_flags.state_debug) printf("deleting workunit %s\n", wup->name); + if (log_flags.state_debug) { + printf("deleting workunit %s\n", wup->name); + } delete wup; wu_iter = workunits.erase(wu_iter); action = true; diff --git a/client/client_types.C b/client/client_types.C index dd7996791b..ed3257719e 100644 --- a/client/client_types.C +++ b/client/client_types.C @@ -113,11 +113,18 @@ int APP::write(FILE* out) { return 0; } +FILE_INFO::FILE_INFO() { +} + +FILE_INFO::~FILE_INFO() { +} + int FILE_INFO::parse(FILE* in) { char buf[256]; + STRING256 url; strcpy(name, ""); - strcpy(url, ""); + //strcpy(url, ""); strcpy(md5_cksum, ""); nbytes = 0; generated_locally = false; @@ -128,10 +135,14 @@ int FILE_INFO::parse(FILE* in) { sticky = false; project = NULL; file_xfer = NULL; + urls.clear(); while (fgets(buf, 256, in)) { if (match_tag(buf, "")) return 0; else if (parse_str(buf, "", name)) continue; - else if (parse_str(buf, "", url)) continue; + else if (parse_str(buf, "", url.text)) { + urls.push_back(url); + continue; + } else if (parse_str(buf, "", md5_cksum)) continue; else if (parse_double(buf, "", nbytes)) continue; else if (match_tag(buf, "")) generated_locally = true; @@ -146,13 +157,13 @@ int FILE_INFO::parse(FILE* in) { } int FILE_INFO::write(FILE* out, bool to_server) { + unsigned int i; fprintf(out, "\n" " %s\n" - " %s\n" " %s\n" " %f\n", - name, url, md5_cksum, nbytes + name, md5_cksum, nbytes ); if (!to_server) { if (generated_locally) fprintf(out, " \n"); @@ -162,6 +173,9 @@ int FILE_INFO::write(FILE* out, bool to_server) { if (upload_when_present) fprintf(out, " \n"); if (sticky) fprintf(out, " \n"); } + for (i=0; i%s\n", urls[i].text); + } fprintf(out, "\n"); return 0; } @@ -180,17 +194,14 @@ int APP_VERSION::parse(FILE* in) { FILE_REF file_ref; strcpy(app_name, ""); - //strcpy(file_name, ""); version_num = 0; app = NULL; project = NULL; - //file_info = NULL; while (fgets(buf, 256, in)) { if (match_tag(buf, "")) return 0; else if (parse_str(buf, "", app_name)) continue; - //else if (parse_str(buf, "", file_name)) continue; else if (match_tag(buf, "")) { - file_ref.parse(in, ""); + file_ref.parse(in); app_files.push_back(file_ref); continue; } @@ -205,14 +216,12 @@ int APP_VERSION::write(FILE* out) { fprintf(out, "\n" " %s\n" - //" %s\n" " %d\n", app_name, - //file_name, version_num ); for (i=0; i\n" @@ -220,7 +229,7 @@ int APP_VERSION::write(FILE* out) { return 0; } -int FILE_REF::parse(FILE* in, char* end_tag) { +int FILE_REF::parse(FILE* in) { char buf[256]; strcpy(file_name, ""); @@ -228,7 +237,7 @@ int FILE_REF::parse(FILE* in, char* end_tag) { fd = -1; main_program = false; while (fgets(buf, 256, in)) { - if (match_tag(buf, end_tag)) return 0; + if (match_tag(buf, "")) return 0; else if (parse_str(buf, "", file_name)) continue; else if (parse_str(buf, "", open_name)) continue; else if (parse_int(buf, "", fd)) continue; @@ -238,12 +247,12 @@ int FILE_REF::parse(FILE* in, char* end_tag) { return 1; } -int FILE_REF::write(FILE* out, char* tag) { - fprintf(out, " <%s>\n", tag); +int FILE_REF::write(FILE* out) { if (strlen(open_name)) { fprintf(out, " %s\n", open_name); } fprintf(out, + " \n" " %s\n", file_name ); @@ -253,7 +262,7 @@ int FILE_REF::write(FILE* out, char* tag) { if (main_program) { fprintf(out, " \n"); } - fprintf(out, " \n", tag); + fprintf(out, " \n"); return 0; } @@ -272,10 +281,11 @@ int WORKUNIT::parse(FILE* in) { if (match_tag(buf, "")) return 0; else if (parse_str(buf, "", name)) continue; else if (parse_str(buf, "", app_name)) continue; + else if (parse_int(buf, "", version_num)) continue; else if (parse_str(buf, "", command_line)) continue; else if (parse_str(buf, "", env_vars)) continue; - else if (match_tag(buf, "")) { - file_ref.parse(in, ""); + else if (match_tag(buf, "")) { + file_ref.parse(in); input_files.push_back(file_ref); continue; } @@ -296,7 +306,7 @@ int WORKUNIT::write(FILE* out) { name, app_name, version_num, command_line, env_vars ); for (i=0; i\n"); return 0; @@ -321,8 +331,8 @@ int RESULT::parse(FILE* in, char* end_tag) { if (match_tag(buf, end_tag)) return 0; else if (parse_str(buf, "", name)) continue; else if (parse_str(buf, "", wu_name)) continue; - else if (match_tag(buf, "")) { - file_ref.parse(in, ""); + else if (match_tag(buf, "")) { + file_ref.parse(in); output_files.push_back(file_ref); continue; } @@ -372,7 +382,7 @@ int RESULT::write(FILE* out, bool to_server) { wu_name ); for (i=0; i urls; + FILE_INFO(); + ~FILE_INFO(); int parse(FILE*); int write(FILE*, bool to_server); int delete_file(); // attempt to delete the underlying file @@ -99,17 +107,15 @@ struct FILE_REF { bool main_program; FILE_INFO* file_info; - int parse(FILE*, char*); - int write(FILE*, char*); + int parse(FILE*); + int write(FILE*); }; struct APP_VERSION { char app_name[256]; - //char file_name[256]; int version_num; APP* app; PROJECT* project; - //FILE_INFO* file_info; vector app_files; int parse(FILE*); diff --git a/client/configure.in b/client/configure.in index 51e7b31151..cf391e6d85 100644 --- a/client/configure.in +++ b/client/configure.in @@ -37,7 +37,7 @@ AC_CHECK_LIB(stdc++, cerr) dnl Checks for header files. AC_HEADER_DIRENT AC_HEADER_STDC -AC_CHECK_HEADERS(fcntl.h sys/time.h unistd.h sys/select.h sys/statvfs.h sys/swap.h) +AC_CHECK_HEADERS(fcntl.h sys/time.h unistd.h sys/select.h sys/statvfs.h sys/swap.h sys/systeminfo.h) dnl Checks for typedefs, structures, and compiler characteristics. AC_C_CONST diff --git a/client/cs_files.C b/client/cs_files.C index 5a043dc54a..54dbf833cc 100644 --- a/client/cs_files.C +++ b/client/cs_files.C @@ -55,17 +55,20 @@ bool CLIENT_STATE::start_file_xfers() { if (!fip->generated_locally && !fip->file_present && !fxp) { fxp = new FILE_XFER; get_pathname(fip, pathname); - fxp->init_download( fip->url, pathname); + fxp->init_download(fip->urls[0].text, pathname); retval = file_xfers->insert(fxp); if (retval) { fprintf(stderr, "couldn't start download for %s: error %d\n", - fip->url, retval + fip->urls[0].text, retval ); } else { fip->file_xfer = fxp; if (log_flags.file_xfer) { - printf("started download of %s to %s\n", fip->url, pathname); + printf( + "started download of %s to %s\n", + fip->urls[0].text, pathname + ); } } action = true; @@ -75,17 +78,20 @@ bool CLIENT_STATE::start_file_xfers() { ) { fxp = new FILE_XFER; get_pathname(fip, pathname); - fxp->init_upload( fip->url, pathname); + fxp->init_upload( fip->urls[0].text, pathname); retval = file_xfers->insert(fxp); if (retval) { fprintf(stderr, "couldn't start upload for %s: error %d\n", - fip->url, retval + fip->urls[0].text, retval ); } else { fip->file_xfer = fxp; if (log_flags.file_xfer) { - printf("started upload of %s to %s\n", pathname, fip->url); + printf( + "started upload of %s to %s\n", + pathname, fip->urls[0].text + ); } } action = true; @@ -95,7 +101,7 @@ bool CLIENT_STATE::start_file_xfers() { if (log_flags.file_xfer) { printf( "file transfer done for %s; retval %d\n", - fip->url, fxp->file_xfer_retval + fip->urls[0].text, fxp->file_xfer_retval ); } file_xfers->remove(fxp); diff --git a/client/error_numbers.h b/client/error_numbers.h index 9dcc1f5380..9e6a4c4c8e 100644 --- a/client/error_numbers.h +++ b/client/error_numbers.h @@ -31,3 +31,5 @@ #define ERR_OPENDIR -111 #define ERR_XML_PARSE -112 #define ERR_GETHOSTBYNAME -113 +#define ERR_GIVEUP -114 + // too much time has elapsed without progress on file xfer diff --git a/client/file_xfer.h b/client/file_xfer.h index a2b678c4de..e6d9e4fbbc 100644 --- a/client/file_xfer.h +++ b/client/file_xfer.h @@ -20,7 +20,8 @@ #ifndef _FILE_XFER_ #define _FILE_XFER_ -// FILE_XFER objects encapsulate the transfer of file +// FILE_XFER objects encapsulate the transfer of a file +// to/from a particular server. // TODO: use the HTTP Range header fields to do partial xfers #include "client_types.h" diff --git a/client/hostinfo_unix.C b/client/hostinfo_unix.C index bca12fa84e..799ea13efa 100644 --- a/client/hostinfo_unix.C +++ b/client/hostinfo_unix.C @@ -32,7 +32,9 @@ #ifdef _WIN32 #include #else +#if HAVE_SYS_SYSTEMINFO_H #include +#endif #include #include #include diff --git a/client/http.C b/client/http.C index 9a3d8c828d..e1b067e6e5 100644 --- a/client/http.C +++ b/client/http.C @@ -54,9 +54,34 @@ void parse_url(char* url, char* host, char* file) { // Note: HTTP 1.1 keeps connection open. // We use 1.0 so we don't have to count bytes. // -void http_get_request_header(char* buf, char* host, char* file) { + +void http_get_request_header(char* buf, char* host, char* file, int offset) { + if (offset) { + sprintf(buf, + "GET /%s;byte-range %d- HTTP/1.0\015\012" + "User-Agent: BOINC client\015\012" + "Host: %s:80\015\012" + "Accept: */*\015\012" + "\015\012", + file, offset, + host + ); + } else { + sprintf(buf, + "GET /%s HTTP/1.0\015\012" + "User-Agent: BOINC client\015\012" + "Host: %s:80\015\012" + "Accept: */*\015\012" + "\015\012", + file, + host + ); + } +} + +void http_head_request_header(char* buf, char* host, char* file) { sprintf(buf, - "GET /%s HTTP/1.0\015\012" + "HEAD /%s HTTP/1.0\015\012" "User-Agent: BOINC client\015\012" "Host: %s:80\015\012" "Accept: */*\015\012" @@ -78,17 +103,34 @@ void http_post_request_header(char* buf, char* host, char* file, int size) { ); } -void http_put_request_header(char* buf, char* host, char* file, int size) { - sprintf(buf, - "PUT /%s HTTP/1.0\015\012" - "Pragma: no-cache\015\012" - "Cache-Control: no-cache\015\012" - "Host: %s:80\015\012" - "Content-Type: application/octet-stream\015\012" - "Content-Length: %d\015\012" - "\015\012", - file, host, size - ); +void http_put_request_header( + char* buf, char* host, char* file, int size, int offset +) { + if (offset) { + sprintf(buf, + "PUT /%s;byte-range %d- HTTP/1.0\015\012" + "Pragma: no-cache\015\012" + "Cache-Control: no-cache\015\012" + "Host: %s:80\015\012" + "Content-Type: application/octet-stream\015\012" + "Content-Length: %d\015\012" + "\015\012", + file, offset, + host, size + ); + } else { + sprintf(buf, + "PUT /%s HTTP/1.0\015\012" + "Pragma: no-cache\015\012" + "Cache-Control: no-cache\015\012" + "Host: %s:80\015\012" + "Content-Type: application/octet-stream\015\012" + "Content-Length: %d\015\012" + "\015\012", + file, + host, size + ); + } } int read_http_reply_header(int socket, HTTP_REPLY_HEADER& header) { @@ -127,7 +169,16 @@ HTTP_OP::HTTP_OP() { HTTP_OP::~HTTP_OP() { } -int HTTP_OP::init_get(char* url, char* out) { +int HTTP_OP::init_head(char* url) { + parse_url(url, hostname, filename); + NET_XFER::init(hostname, 80, HTTP_BLOCKSIZE); + http_op_type = HTTP_OP_HEAD; + http_op_state = HTTP_STATE_CONNECTING; + return 0; +} + +int HTTP_OP::init_get(char* url, char* out, int off) { + offset = off; parse_url(url, hostname, filename); NET_XFER::init(hostname, 80, HTTP_BLOCKSIZE); strcpy(outfile, out); @@ -150,9 +201,10 @@ int HTTP_OP::init_post(char* url, char* in, char* out) { return 0; } -int HTTP_OP::init_put(char* url, char* in) { +int HTTP_OP::init_put(char* url, char* in, int off) { int retval; + offset = off; parse_url(url, hostname, filename); NET_XFER::init(hostname, 80, HTTP_BLOCKSIZE); strcpy(infile, in); @@ -206,11 +258,14 @@ bool HTTP_OP_SET::poll() { ); break; case HTTP_OP_GET: - http_get_request_header(hdr, htp->hostname, htp->filename); + http_get_request_header(hdr, htp->hostname, htp->filename, htp->offset); + break; + case HTTP_OP_HEAD: + http_head_request_header(hdr, htp->hostname, htp->filename); break; case HTTP_OP_PUT: http_put_request_header( - hdr, htp->hostname, htp->filename, htp->content_length + hdr, htp->hostname, htp->filename, htp->content_length, htp->offset ); break; } @@ -238,6 +293,7 @@ bool HTTP_OP_SET::poll() { htp->do_file_io = true; break; case HTTP_OP_GET: + case HTTP_OP_HEAD: htp->http_op_state = HTTP_STATE_REPLY_HEADER; htp->want_upload = false; htp->want_download = true; @@ -269,6 +325,10 @@ bool HTTP_OP_SET::poll() { break; } switch (htp->http_op_type) { + case HTTP_OP_HEAD: + htp->http_op_state = HTTP_STATE_DONE; + htp->http_op_retval = 0; + break; case HTTP_OP_GET: case HTTP_OP_POST: htp->http_op_state = HTTP_STATE_REPLY_BODY; diff --git a/client/http.h b/client/http.h index 655c5f5b5b..b19b7a4bef 100644 --- a/client/http.h +++ b/client/http.h @@ -30,6 +30,7 @@ struct HTTP_REPLY_HEADER { #define HTTP_OP_GET 1 #define HTTP_OP_POST 2 #define HTTP_OP_PUT 3 +#define HTTP_OP_HEAD 4 // represents an HTTP request in progress // @@ -43,14 +44,16 @@ public: char infile[256]; char outfile[256]; int content_length; + int offset; HTTP_REPLY_HEADER hrh; int http_op_state; // values below int http_op_type; int http_op_retval; - int init_get(char* url, char* outfile); + int init_head(char* url); + int init_get(char* url, char* outfile, int offset=0); int init_post(char* url, char* infile, char* outfile); - int init_put(char* url, char* infile); + int init_put(char* url, char* infile, int offset=0); bool http_op_done(); }; diff --git a/client/net_xfer.C b/client/net_xfer.C index f860bbbca1..2e411a91b4 100644 --- a/client/net_xfer.C +++ b/client/net_xfer.C @@ -217,7 +217,7 @@ int NET_XFER_SET::do_select(int max_bytes, int& bytes_transferred) { #ifdef _WIN32 getsockopt(fd, SOL_SOCKET, SO_ERROR, (char *)&n, (int *)&intsize); #else - getsockopt(fd, SOL_SOCKET, SO_ERROR, &n, (int *)&intsize); + getsockopt(fd, SOL_SOCKET, SO_ERROR, &n, (unsigned int *)&intsize); #endif if (n) { if (log_flags.net_xfer_debug) { diff --git a/client/pers_file_xfer.C b/client/pers_file_xfer.C new file mode 100644 index 0000000000..ffc252dfd7 --- /dev/null +++ b/client/pers_file_xfer.C @@ -0,0 +1,73 @@ +// The contents of this file are subject to the Mozilla Public License +// Version 1.0 (the "License"); you may not use this file except in +// compliance with the License. You may obtain a copy of the License at +// http://www.mozilla.org/MPL/ +// +// Software distributed under the License is distributed on an "AS IS" +// basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the +// License for the specific language governing rights and limitations +// under the License. +// +// The Original Code is the Berkeley Open Network Computing. +// +// The Initial Developer of the Original Code is the SETI@home project. +// Portions created by the SETI@home project are Copyright (C) 2002 +// University of California at Berkeley. All Rights Reserved. +// +// Contributor(s): +// + +// PERS_FILE_XFER represents a persistent file transfer. +// A set of URL is given. + +// For download, the object attempts to download the file +// from any of the URLs. +// If one fails or is not available, try another, +// using an exponential backoff policy to avoid flooding servers. + +// For upload, try to upload the file to the first URL; +// if that fails try the others. + +int PERS_FILE_XFER::init(FILE_INFO&, bool is_upload) { +} + +// +void PERS_FILE_XFER::try() { +} + +void PERS_FILE_XFER::poll(unsigned int now) { + if (fxp) { + if (fxp->file_xfer_done) { + if (fxp->file_xfer_retval == 0) { + } else { + // file xfer failed. + // See if it's time to give up on the persistent file xfer + // + diff = now - fip->last_xfer_time; + if (diff > PERS_GIVEUP) { + pers_xfer_done = true; + pers_file_xfer_retval = ERR_GIVEUP; + } + } + } + } else { + // No file xfer is active. + // We must be waiting after a failure. + // See if it's time to try again. + // + if (now > retry_time) { + try(); + } + } +} + +int PERS_FILE_XFER_SET::poll() { + unsigned int ; + PERS_FILE_XFER* pfxp; + bool action = false; + int now = time(0); + + for (i=0; ipoll(now); + } +} diff --git a/client/pers_file_xfer.h b/client/pers_file_xfer.h new file mode 100644 index 0000000000..fdad8d1735 --- /dev/null +++ b/client/pers_file_xfer.h @@ -0,0 +1,55 @@ +// The contents of this file are subject to the Mozilla Public License +// Version 1.0 (the "License"); you may not use this file except in +// compliance with the License. You may obtain a copy of the License at +// http://www.mozilla.org/MPL/ +// +// Software distributed under the License is distributed on an "AS IS" +// basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the +// License for the specific language governing rights and limitations +// under the License. +// +// The Original Code is the Berkeley Open Network Computing. +// +// The Initial Developer of the Original Code is the SETI@home project. +// Portions created by the SETI@home project are Copyright (C) 2002 +// University of California at Berkeley. All Rights Reserved. +// +// Contributor(s): +// + +// PERS_FILE_XFER represents a persistent file transfer. +// A set of URL is given in the FILE_INFO. + +// For download, the object attempts to download the file +// from any of the URLs. +// If one fails or is not available, try another, +// using an exponential backoff policy to avoid flooding servers. + +// For upload, try to upload the file to the first URL; +// if that fails try the others. + +#define PERS_RETRY_DELAY_MIN 60 +#define PERS_RETRY_DELAY_MAX (256*60) +#define PERS_GIVEUP (3600*24*7) + // give up on xfer if this time elapses since last byte xferred + +class PERS_FILE_XFER { + int url_index; // which URL to try next + int nretry; // # of retries so far + FILE_INFO* fip; + bool is_upload; + FILE_XFER* fxp; // nonzero if file xfer in progress + int retry_time; // don't retry until this time + + int init(FILE_INFO&, bool is_upload); +}; + +class PERS_FILE_XFER_SET { + vectorpers_file_xfers; + FILE_XFER_SET* file_xfers; +public: + PERS_FILE_XFER_SET(FILE_XFER_SET*); + int insert(PERS_FILE_XFER*); + int remove(PERS_FILE_XFER*); + int poll(); +}; diff --git a/client/test_http.C b/client/test_http.C index c244be288a..2a74e02832 100644 --- a/client/test_http.C +++ b/client/test_http.C @@ -21,22 +21,43 @@ #include #include "http.h" +#include "log_flags.h" #include "net_xfer.h" #include "util.h" int main() { NET_XFER_SET nxs; HTTP_OP_SET hos(&nxs); - HTTP_OP *op1, *op2; - int n; + HTTP_OP *op1=0, *op2=0, *op3=0; + int retval, n; + log_flags.http_debug = true; + log_flags.net_xfer_debug = true; + +#if 0 op1 = new HTTP_OP; - op2 = new HTTP_OP; - op1->init_get("http://localhost.localdomain/my_index.html", "test_out1"); - op2->init_post("http://localhost.localdomain/test-cgi/cgi", "test_in1", "test_out2"); - + retval = op1->init_get("http://localhost.localdomain/my_index.html", "test_out1"); + if (retval) { + printf("init_post: %d\n", retval); + exit(1); + } hos.insert(op1); + + op2 = new HTTP_OP; + retval = op2->init_post("http://localhost.localdomain/test-cgi/cgi", "test_in1", "test_out2"); + if (retval) { + printf("init_post: %d\n", retval); + exit(1); + } hos.insert(op2); +#endif + op3 = new HTTP_OP; + retval = op3->init_head("http://localhost.localdomain/my_index2.html"); + if (retval) { + printf("init_post: %d\n", retval); + exit(1); + } + hos.insert(op3); while (1) { nxs.poll(100000, n); @@ -51,8 +72,13 @@ int main() { hos.remove(op2); op2 = 0; } - if (!op1 && !op2) break; - boinc_sleep(1); + if (op3 && op3->http_op_done()) { + printf("op3 done; status %d\n", op3->hrh.status); + hos.remove(op3); + op3 = 0; + } + if (!op1 && !op2 && !op3) break; + sleep(1); } printf("all done\n"); } diff --git a/html/ops/db.php b/html/ops/db.php index 89658864cf..7dca125574 100644 --- a/html/ops/db.php +++ b/html/ops/db.php @@ -35,11 +35,6 @@ while ($res = mysql_fetch_object($result)) { show_result($res); } - echo "
Prefs"; - $result = mysql_query("select * from prefs"); - while ($prefs = mysql_fetch_object($result)) { - show_prefs($prefs); - } echo "
Users"; $result = mysql_query("select * from user"); while ($user = mysql_fetch_object($result)) { diff --git a/html/user/db.inc b/html/user/db.inc index ea28f5b102..bc6696e6fc 100644 --- a/html/user/db.inc +++ b/html/user/db.inc @@ -143,16 +143,10 @@ function show_user($user) { row("email_addr", $user->email_addr); row("country", $user->country); row("postal_code", $user->postal_code); - echo "\n"; -} - -function show_prefs($prefs) { - start_table(); - row("ID", $prefs->id); - row("created", time_str($prefs->create_time)); - row("modified", time_str($prefs->modified_time)); - row("userid", $prefs->userid); - row("XML doc", "
".htmlspecialchars($prefs->xml_doc)."
"); + row("total credit", $user->total_credit); + row("recent average credit", $user->expavg_credit); + row("preferences", "
".htmlspecialchars($user->prefs)."
"); + row("prefs mod time", time_str($user->prefs_mod_time)); echo "\n"; } diff --git a/notes b/notes index 31bb27e688..47e64b9448 100644 --- a/notes +++ b/notes @@ -1,14 +1,5 @@ Abstractions -"Project": each is described by a URL. -Each has its own database and control server. - -"Application": a particular program. -A project may have several applications. - -"Account": each user has a separate account with each project. -Each account has a unique email address and a server-assigned authenticator. - -------------------- Client files two files: @@ -58,41 +49,6 @@ write events to log file: logging flag is part of preferences -------------------- -division between database and XML -proposal: move as much info as possible out of the DB into XML files. -examples: -- workunits and results - WUs and results are described by XML files listing - their inputs, outputs, etc. - The DB entry for a WU contains only info relevant to scheduling: - memory/disk/communication requirement -- user info - A configuration is an XML file, opaque to the scheduling server - --------------------- -WUs and results -- WUs and results are desribed by XML files that describe their - input and output files. -- each client computation is represented by a "result" DB record, - which is created BEFORE the client requests it. - The application server system must keep the DB supplied - with result records, or clients will starve. - - NOTE: this is necessary to control where output files go. - Could also have a scheme where each application has a - "template" result file. - This instructs the client to create its own output file names. - When the client returns the result, - the server creates the result record and plugs in file names. --------------------- -File info - input files - "sticky": don't delete after result done - URL (if not already on client) - output files - "sticky": don't delete after result done - URL (optional; send here after result done) --------------------- file xfer commands implemented as WU/result pairs whose app is "file_xfer". Can have just one input file, one output. @@ -124,122 +80,6 @@ WU attributes in DB, sched server not all input files available -------------------- -Workunit affinity - -This mechanism allows a sequence of WUs to get executed on the same host, -but allows the sequence to migrate (or be duplicated) if needed. - -result attributes: -previous_resultid - This result is a "successor" to the previous one. - If all the sticky input and output files of the previous WU are present, - this WU can be executed efficiently. -has_successor - This result has a successor. - -How it works: -The project generates a sequence of WUs, -each with one or more results. -It chains the results together into sequences. - -When a client completes a result with successor, -it retains the result record. - -NOTE: one goal of this design is to avoid the scheduler -having to know about individual files --------------------- -Scheduler request - -The client sends all its results with successors - -scheduler algorithm: - if there any results with predecessors - for which the client has all sticky files, - send them in preference to any other results - - - --------------------- -database tables - -application -platform -app_version -core_version -account -file -workunit - applicationid - file1 name1 - file2 name2 - nresults -result - workunitid - accountid - fileid - boolean verified -host --------------------- -State maintained on client -Config file (XML) - - - 123123 - // last time user added project or changed CPU shares - 1000 - 10 - // if estimated work falls below this, try to get more - 10 - // don't get more work if estimate is above this - 20 - // zero means don't work while user active - - - http://wjwjwj - 123123 - blah.blah - blah.blah - 1 - 100 - 5.44 - // this is zeroed out each time shares updated - 123123123 - // stored on client only; not sent to server in general - foo@bar - - - - sfkjf - akdjsfd - 123123 - - - - skdjf - - foo - blah - // name by which app refers to file - - - - 12938 - - - ... - - - --------------------- -Security notes: --------------------- -Client directory structure -top-level dir - project dir (one per project) - CPU dir (one per CPU) - contains symbolic links to application file, - all input and output files --------------------- Client logic ["network xfer" object encapsulates a set of file xfers in progress] ["processor" object: one for each CPU] @@ -331,10 +171,67 @@ Disk usage Projects For each project: - master URL + user name + project's master URL email address authenticator resource % show email address on web site? accept emails from project? project-specific prefs +------------------------------ +retry policies: +general issues: + when and where to retry? + when to declare overall failure? + what to do if overall failure? + what needs to be saved in state file? + +file xfer + download + round-robin through URLs with random exponential backoff + after connection failure or HTTP error. + 2X from 1 minute up to 256 minutes + Overall failure after 1 week since last successful xfer + flag result as "file download failed", + abort other file xfers, + delete other files. + write log entry + State file: + record time of last successful xfer + upload + same as for download? + Use HTTP features to find file size on server + +scheduler RPC + order projects according to hosts's "debt" to them. + Attempt to contact them in this order. + For each project, try all URLs in sequence with no delay. + If still need more work after a given RPC, + keep going to next project. + If still not enough work after a given "round", + do exponential backoff + 2X from 1 minute up to 256 minutes + If reach 256 minutes, show error message to user and write to log + nothing saved in state file +------------------ +Core/App connection + two unidirectional message streams. + files "core_to_app.xml" and "app_to_core.xml". + + core->app: + initially: + requested frequency of app->core messages + app preferences + name of graphics shared-mem segment + recommended graphics parameters + frame rate + size + recommended checkpoint period + whether to do graphics + thereafter: + recommended graphics params + app->core + percent done + I just checkpointed + CPU time so far diff --git a/test/1sec_result b/test/1sec_result index 5d126d6ed2..40ad05f381 100644 --- a/test/1sec_result +++ b/test/1sec_result @@ -7,8 +7,8 @@ - + out - +
diff --git a/test/account2.xml b/test/account2.xml index 8ed9428aaf..e2a973e407 100644 --- a/test/account2.xml +++ b/test/account2.xml @@ -1,15 +1,14 @@ - localhost - http://localhost/cgi-bin/boinc-cgi/cgi + http://localhost/boinc-cgi/cgi david@localhost 3f7b90793a0175ad0bda68684e8bd136 2 zoot - http://zoot/cgi-bin/boinc-cgi/cgi + http://zoot/boinc-cgi/cgi david@localhost 3f7b90793a0175ad0bda68684e8bd136 10 diff --git a/test/concat_result b/test/concat_result index 5d126d6ed2..40ad05f381 100644 --- a/test/concat_result +++ b/test/concat_result @@ -7,8 +7,8 @@ - + out - +
diff --git a/test/concat_wu b/test/concat_wu index 03791f4782..937158aa1d 100644 --- a/test/concat_wu +++ b/test/concat_wu @@ -11,13 +11,13 @@ concat - + in1 - - +
+ in2 - + in1 in2 out diff --git a/test/init.inc b/test/init.inc index 3111565ff3..9a03b6569c 100644 --- a/test/init.inc +++ b/test/init.inc @@ -97,7 +97,7 @@ function add_user($prefs_file) { $cmd = "../tools/add user -email_addr $BOINC_EMAIL -user_name David -web_password foobar -authenticator 3f7b90793a0175ad0bda68684e8bd136 "; if ($prefs_file) { - $cmd = $cmd." -prefs_file=$prefs_file"; + $cmd = $cmd." -prefs_file $prefs_file"; } PassThru($cmd); } diff --git a/test/laptop_prefs.xml b/test/laptop_prefs.xml index a4f1feb132..07ab1144fd 100644 --- a/test/laptop_prefs.xml +++ b/test/laptop_prefs.xml @@ -1,3 +1,17 @@ - -10 -1 + + + 10 + 1 + 0.4 + 50 + 0.4 + + http://localhost.localdomain + david@localdomain + 123892398 + 10 + + Tahiti Sunset + + \n" + diff --git a/test/test_1sec.php b/test/test_1sec.php index bd3b891a69..3f47a3a404 100644 --- a/test/test_1sec.php +++ b/test/test_1sec.php @@ -1,9 +1,12 @@ #! /usr/local/bin/php diff --git a/test/test_concat.php b/test/test_concat.php index 0b4a5aa6f7..9441993d76 100644 --- a/test/test_concat.php +++ b/test/test_concat.php @@ -16,7 +16,10 @@ add_user(null); add_app("concat"); create_work("-appname concat -wu_name concat_wu -wu_template concat_wu -result_template concat_result -nresults 2 input input"); + start_feeder(); run_client(); + stop_feeder(); + check_results_done(); compare_file("concat_wu_0_0", "concat_correct_output"); compare_file("concat_wu_1_0", "concat_correct_output"); ?> diff --git a/test/test_dynamic.php b/test/test_dynamic.php index 34db9cb274..b8275740ad 100755 --- a/test/test_dynamic.php +++ b/test/test_dynamic.php @@ -5,6 +5,7 @@ include_once("init.inc"); + check_env_vars(); clear_db(); clear_data_dirs(); init_client_dirs("account1.xml"); diff --git a/test/test_prefs.php b/test/test_prefs.php index 0afe5e8157..4c156a9245 100644 --- a/test/test_prefs.php +++ b/test/test_prefs.php @@ -5,6 +5,7 @@ include_once("init.inc"); + check_env_vars(); clear_db(); clear_data_dirs(); init_client_dirs("account1.xml"); diff --git a/test/test_projects.php b/test/test_projects.php index a8bc97bb71..69e5200303 100755 --- a/test/test_projects.php +++ b/test/test_projects.php @@ -5,6 +5,7 @@ include_once("init.inc"); + check_env_vars(); clear_db(); clear_data_dirs(); init_client_dirs("account2.xml"); @@ -14,5 +15,7 @@ add_user(null); add_app("upper_case"); create_work("-appname upper_case -wu_name uc_wu -wu_template uc_wu -result_template uc_result -nresults 2 small_input"); - //run_client(); + start_feeder(); + run_client(); + stop_feeder(); ?> diff --git a/test/test_stderr.php b/test/test_stderr.php index 827112c209..8010656f87 100644 --- a/test/test_stderr.php +++ b/test/test_stderr.php @@ -5,6 +5,7 @@ include_once("init.inc"); + check_env_vars(); clear_db(); clear_data_dirs(); init_client_dirs("account1.xml"); @@ -14,7 +15,9 @@ add_user(null); add_app("upper_case"); create_work("-appname upper_case -wu_name uc_wu -wu_template uc_wu -result_template uc_result -nresults 2 input input"); + start_feeder(); run_client(); + stop_feeder(); check_results_done(); compare_file("uc_wu_0_0", "uc_correct_output"); diff --git a/test/test_uc.php b/test/test_uc.php index df7a31cd1e..4829c3239c 100644 --- a/test/test_uc.php +++ b/test/test_uc.php @@ -15,11 +15,8 @@ add_user(null); add_app("upper_case"); create_work("-appname upper_case -wu_name uc_wu -wu_template uc_wu -result_template uc_result -nresults 2 input input"); - echo "starting feeder\n"; start_feeder(); - echo "started feeder\n"; run_client(); - echo "ran client\n"; stop_feeder(); check_results_done(); diff --git a/test/test_uc_slow.php b/test/test_uc_slow.php index b008d97a0a..6a652d59f8 100644 --- a/test/test_uc_slow.php +++ b/test/test_uc_slow.php @@ -1,10 +1,12 @@ #! /usr/local/bin/php - + 1 - + diff --git a/test/uc_wu b/test/uc_wu index 5dad18a7e4..533aa3b7c2 100644 --- a/test/uc_wu +++ b/test/uc_wu @@ -6,8 +6,8 @@ upper_case - + 0 - + diff --git a/test/ucs_result b/test/ucs_result index 5d126d6ed2..40ad05f381 100644 --- a/test/ucs_result +++ b/test/ucs_result @@ -7,8 +7,8 @@ - + out - + diff --git a/test/ucs_wu b/test/ucs_wu index c32080e006..6544525c75 100644 --- a/test/ucs_wu +++ b/test/ucs_wu @@ -6,8 +6,8 @@ uc_slow - + in - + diff --git a/tools/add.C b/tools/add.C index 23851085f9..80a9750591 100644 --- a/tools/add.C +++ b/tools/add.C @@ -86,7 +86,7 @@ void add_app_version() { strcpy(app.name, app_name); retval = db_app_lookup_name(app); if (retval) { - fprintf(stderr, "can't find app %s\n", app_name); + fprintf(stderr, "add_app_version(): can't find app %s\n", app_name); db_print_error("db_app_lookup_name"); return; } @@ -94,7 +94,7 @@ void add_app_version() { strcpy(platform.name, platform_name); retval = db_platform_lookup_name(platform); if (retval) { - fprintf(stderr, "can't find platform %s\n", platform_name); + fprintf(stderr, "add_app_version(): can't find platform %s\n", platform_name); db_print_error("db_platform_lookup_name"); return; }