Rooba
/
boinc
mirror of https://github.com/BOINC/boinc.git
1
0
Fork 0
Code Issues 1 Packages Projects Releases Wiki Activity

multiple URLs per file 

svn path=/trunk/boinc/; revision=108
This commit is contained in:
David Anderson 2002-06-10 06:14:18 +00:00
parent 142a50b8ff
commit 7d98c6f445
40 changed files with 617 additions and 280 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
TODO
View File

@ -1,5 +1,14 @@
HIGH-PRIORITY (must be done to support SETI@home)
- Code-signing
research tools for code-signing
- Upload authentication
Each result contains a "certificate", signed with project key, giving
- list of: file name, max size
- min, max times to xfer
modify put program to decrypt certificate, enforce name/size/time limits
- Network retry policies
can't download file: when to give up? how to retry?
exponential backoff
@ -142,6 +151,10 @@ LOW-PRIORITY
- test
- implement file upload/download requests
This can be done in the current WU/result paradigm;
WU has a special "null" application
use sticky input files to download;
use upload-when-done output files to upload
- preferences
finish PHP web interface

75
api/api.C
View File

@ -22,6 +22,7 @@
#include <stdio.h>
#include <string.h>
#include "../sched/parse.h"
#include "api.h"
int MFILE::open(char* path, char* mode) {
@ -81,3 +82,77 @@ int MFILE::flush() {
len = 0;
return fflush(f);
}
void write_core_file(FILE* f, APP_IN& ai) {
fprintf(f,
"<graphics_xsize>%d</graphics_xsize>\n"
"<graphics_ysize>%d</graphics_ysize>\n"
"<graphics_refresh_period>%f</graphics_refresh_period>\n"
"<checkpoint_period>%f</checkpoint_period>\n"
"<poll_period>%f</poll_period>\n",
ai.graphics.xsize,
ai.graphics.ysize,
ai.graphics.refresh_period,
ai.checkpoint_period,
ai.poll_period
);
}
void parse_core_file(FILE* f, APP_IN& ai) {
char buf[256];
while (fgets(buf, 256, f)) {
if (match_tag(buf, "<app_specific_prefs>")) {
strcpy(ai.app_preferences, "");
while (fgets(buf, 256, f)) {
if (match_tag(buf, "</app_specific_prefs>")) break;
strcat(ai.app_preferences, buf);
}
continue;
}
else if (parse_int(buf, "<graphics_xsize>", ai.graphics.xsize)) continue;
else if (parse_int(buf, "<graphics_ysize>", ai.graphics.ysize)) continue;
else if (parse_double(buf, "<graphics_refresh_period>", ai.graphics.refresh_period)) continue;
else if (parse_double(buf, "<checkpoint_period>", ai.checkpoint_period)) continue;
else if (parse_double(buf, "<poll_period>", ai.poll_period)) continue;
else fprintf(stderr, "read_core_file: unrecognized %s", buf);
}
}
void write_app_file(FILE* f, APP_OUT& ao) {
fprintf(f,
"<percent_done>%f</percent_done>\n"
"<cpu_time_at_checkpoint>%f</cpu_time_at_checkpoint>\n",
ao.percent_done,
ao.cpu_time_at_checkpoint
);
if (ao.checkpointed) {
fprintf(f, "<checkpointed/>\n");
}
}
void parse_app_file(FILE* f, APP_OUT& ao) {
}
void boinc_init(APP_IN& ai) {
FILE* f;
memset(&ai, 0, sizeof(ai));
f = fopen(CORE_TO_APP_FILE, "r");
if (f) {
parse_core_file(f, ai);
unlink(CORE_TO_APP_FILE);
}
}
double boinc_time() {
return double_time();
}
void boinc_poll(APP_IN& ai, APP_OUT& ao) {
FILE* f;
f = fopen("_app_temp", "w");
write_app_file(f, ao);
rename("_app_temp", APP_TO_CORE_FILE);
}

50
api/api.h
View File

@ -17,17 +17,17 @@
// Contributor(s):
//
#include <stdio.h>
#ifndef BOINC_API
#define BOINC_API
// MFILE supports a primitive form of checkpointing.
// Write all your output (and restart file) to MFILEs.
// The output is buffered in memory.
// Then close all the MFILEs;
// all the buffers will be flushed to disk, almost atomically.
#include <stdio.h>
#ifndef BOINC_API
#define BOINC_API
class MFILE {
char* buf;
int len;
@ -42,4 +42,44 @@ public:
int flush();
};
// An application that wants to be well-behaved should do the following:
//
// - call boinc_init() at startup
// - call boinc_time() periodically.
// This is cheap - it gets the time of day.
// - checkpoint as often as requested by core
// - boinc_poll():
// Call this as often as requested by core
struct APP_IN_GRAPHICS {
int xsize;
int ysize;
double refresh_period;
char shmem_seg_name[32];
};
struct APP_OUT_GRAPHICS {
};
struct APP_IN {
char app_preferences[4096];
APP_IN_GRAPHICS graphics;
double checkpoint_period; // recommended checkpoint period
double poll_period; // recommended poll period
};
struct APP_OUT {
double percent_done;
double cpu_time_at_checkpoint;
bool checkpointed; // true iff checkpointed since last call
};
void boinc_init(APP_IN&);
double boinc_time();
void boinc_poll(APP_IN&, APP_OUT&);
double boinc_cpu_time();
#define CORE_TO_APP_FILE "core_to_app.xml"
#define APP_TO_CORE_FILE "app_to_core.xml"
#endif

45
checkin_notes
View File

@ -434,3 +434,48 @@ Michael Gary June 08, 2002
server_types.C (added)
server_types.h (added)
show_shmem.C (added)
David A June 9 2002
- added support for multiple URLs in a FILE_INFO
(e.g. multiple servers from which the file can be downloaded)
- started work on "persistent file transfer": a layer on top of
FILE_XFER that manages restarting from failed connections,
and that implements a give-up policy
- added offset arguments to GET and PUT HTTP operations.
NOTE: this will work fine for downloading files (GET)
but we'll have to use something else for upload,
since the standard PUT handler doesn't do offsets,
and we need security functionality in any case.
- added preliminary version of application API for
communicating with core client.
- use <file_ref> tags instead of <input_file> and <output_file>
(makes things simpler)
TODO
notes
api/
api.C,h
client/
configure.in
client_state.C
client_types.C,h
cs_files.C
error_numbers.h
file_xfer.h
http.C,h
pers_file_xfer.C,h
test_http.C
html_ops/
db.php
html_user/
db.inc
test/
1sec_result
account2.xml
*_result
*_wu
init.inc
laptop_prefs.xml
test_*.php
tools/
add.C

10
client/client_state.C
View File

@ -224,6 +224,9 @@ int CLIENT_STATE::write_state_file() {
FILE* f = fopen(STATE_FILE_TEMP, "w");
int retval;
if (log_flags.state_debug) {
printf("Writing state file\n");
}
if (!f) {
fprintf(stderr, "can't open temp state file: %s\n", STATE_FILE_TEMP);
return ERR_FOPEN;
@ -264,6 +267,9 @@ int CLIENT_STATE::write_state_file() {
fprintf(f, "</client_state>\n");
fclose(f);
retval = rename(STATE_FILE_TEMP, STATE_FILE_NAME);
if (log_flags.state_debug) {
printf("Done writing state file\n");
}
if (retval) return ERR_RENAME;
return 0;
}
@ -497,7 +503,9 @@ bool CLIENT_STATE::garbage_collect() {
while (wu_iter != workunits.end()) {
wup = *wu_iter;
if (wup->ref_cnt == 0) {
if (log_flags.state_debug) printf("deleting workunit %s\n", wup->name);
if (log_flags.state_debug) {
printf("deleting workunit %s\n", wup->name);
}
delete wup;
wu_iter = workunits.erase(wu_iter);
action = true;

54
client/client_types.C
View File

@ -113,11 +113,18 @@ int APP::write(FILE* out) {
return 0;
}
FILE_INFO::FILE_INFO() {
}
FILE_INFO::~FILE_INFO() {
}
int FILE_INFO::parse(FILE* in) {
char buf[256];
STRING256 url;
strcpy(name, "");
strcpy(url, "");
//strcpy(url, "");
strcpy(md5_cksum, "");
nbytes = 0;
generated_locally = false;
@ -128,10 +135,14 @@ int FILE_INFO::parse(FILE* in) {
sticky = false;
project = NULL;
file_xfer = NULL;
urls.clear();
while (fgets(buf, 256, in)) {
if (match_tag(buf, "</file_info>")) return 0;
else if (parse_str(buf, "<name>", name)) continue;
else if (parse_str(buf, "<url>", url)) continue;
else if (parse_str(buf, "<url>", url.text)) {
urls.push_back(url);
continue;
}
else if (parse_str(buf, "<md5_cksum>", md5_cksum)) continue;
else if (parse_double(buf, "<nbytes>", nbytes)) continue;
else if (match_tag(buf, "<generated_locally/>")) generated_locally = true;
@ -146,13 +157,13 @@ int FILE_INFO::parse(FILE* in) {
}
int FILE_INFO::write(FILE* out, bool to_server) {
unsigned int i;
fprintf(out,
"<file_info>\n"
" <name>%s</name>\n"
" <url>%s</url>\n"
" <md5_cksum>%s</md5_cksum>\n"
" <nbytes>%f</nbytes>\n",
name, url, md5_cksum, nbytes
name, md5_cksum, nbytes
);
if (!to_server) {
if (generated_locally) fprintf(out, " <generated_locally/>\n");
@ -162,6 +173,9 @@ int FILE_INFO::write(FILE* out, bool to_server) {
if (upload_when_present) fprintf(out, " <upload_when_present/>\n");
if (sticky) fprintf(out, " <sticky/>\n");
}
for (i=0; i<urls.size(); i++) {
fprintf(out, "<url>%s</url>\n", urls[i].text);
}
fprintf(out, "</file_info>\n");
return 0;
}
@ -180,17 +194,14 @@ int APP_VERSION::parse(FILE* in) {
FILE_REF file_ref;
strcpy(app_name, "");
//strcpy(file_name, "");
version_num = 0;
app = NULL;
project = NULL;
//file_info = NULL;
while (fgets(buf, 256, in)) {
if (match_tag(buf, "</app_version>")) return 0;
else if (parse_str(buf, "<app_name>", app_name)) continue;
//else if (parse_str(buf, "<file_name>", file_name)) continue;
else if (match_tag(buf, "<file_ref>")) {
file_ref.parse(in, "</file_ref>");
file_ref.parse(in);
app_files.push_back(file_ref);
continue;
}
@ -205,14 +216,12 @@ int APP_VERSION::write(FILE* out) {
fprintf(out,
"<app_version>\n"
" <app_name>%s</app_name>\n"
//" <file_name>%s</file_name>\n"
" <version_num>%d</version_num>\n",
app_name,
//file_name,
version_num
);
for (i=0; i<app_files.size(); i++) {
app_files[i].write(out, "file_assoc");
app_files[i].write(out);
}
fprintf(out,
"</app_version>\n"
@ -220,7 +229,7 @@ int APP_VERSION::write(FILE* out) {
return 0;
}
int FILE_REF::parse(FILE* in, char* end_tag) {
int FILE_REF::parse(FILE* in) {
char buf[256];
strcpy(file_name, "");
@ -228,7 +237,7 @@ int FILE_REF::parse(FILE* in, char* end_tag) {
fd = -1;
main_program = false;
while (fgets(buf, 256, in)) {
if (match_tag(buf, end_tag)) return 0;
if (match_tag(buf, "</file_ref>")) return 0;
else if (parse_str(buf, "<file_name>", file_name)) continue;
else if (parse_str(buf, "<open_name>", open_name)) continue;
else if (parse_int(buf, "<fd>", fd)) continue;
@ -238,12 +247,12 @@ int FILE_REF::parse(FILE* in, char* end_tag) {
return 1;
}
int FILE_REF::write(FILE* out, char* tag) {
fprintf(out, " <%s>\n", tag);
int FILE_REF::write(FILE* out) {
if (strlen(open_name)) {
fprintf(out, " <open_name>%s</open_name>\n", open_name);
}
fprintf(out,
" <file_ref>\n"
" <file_name>%s</file_name>\n",
file_name
);
@ -253,7 +262,7 @@ int FILE_REF::write(FILE* out, char* tag) {
if (main_program) {
fprintf(out, " <main_program/>\n");
}
fprintf(out, " </%s>\n", tag);
fprintf(out, " </file_ref>\n");
return 0;
}
@ -272,10 +281,11 @@ int WORKUNIT::parse(FILE* in) {
if (match_tag(buf, "</workunit>")) return 0;
else if (parse_str(buf, "<name>", name)) continue;
else if (parse_str(buf, "<app_name>", app_name)) continue;
else if (parse_int(buf, "<version_num>", version_num)) continue;
else if (parse_str(buf, "<command_line>", command_line)) continue;
else if (parse_str(buf, "<env_vars>", env_vars)) continue;
else if (match_tag(buf, "<input_file>")) {
file_ref.parse(in, "</input_file>");
else if (match_tag(buf, "<file_ref>")) {
file_ref.parse(in);
input_files.push_back(file_ref);
continue;
}
@ -296,7 +306,7 @@ int WORKUNIT::write(FILE* out) {
name, app_name, version_num, command_line, env_vars
);
for (i=0; i<input_files.size(); i++) {
input_files[i].write(out, "input_file");
input_files[i].write(out);
}
fprintf(out, "</workunit>\n");
return 0;
@ -321,8 +331,8 @@ int RESULT::parse(FILE* in, char* end_tag) {
if (match_tag(buf, end_tag)) return 0;
else if (parse_str(buf, "<name>", name)) continue;
else if (parse_str(buf, "<wu_name>", wu_name)) continue;
else if (match_tag(buf, "<output_file>")) {
file_ref.parse(in, "</output_file>");
else if (match_tag(buf, "<file_ref>")) {
file_ref.parse(in);
output_files.push_back(file_ref);
continue;
}
@ -372,7 +382,7 @@ int RESULT::write(FILE* out, bool to_server) {
wu_name
);
for (i=0; i<output_files.size(); i++) {
output_files[i].write(out, "output_file");
output_files[i].write(out);
}
} else {
for (i=0; i<output_files.size(); i++) {

18
client/client_types.h
View File

@ -36,6 +36,10 @@
class FILE_XFER;
struct STRING256 {
char text[256];
};
class PROJECT {
public:
char name[256]; // descriptive. not unique
@ -66,9 +70,10 @@ struct APP {
int write(FILE*);
};
struct FILE_INFO {
class FILE_INFO {
public:
char name[256];
char url[256]; // TODO: allow multiple URLs
//char url[256]; // TODO: allow multiple URLs
char md5_cksum[33];
double nbytes;
bool generated_locally; // file is produced by app
@ -80,7 +85,10 @@ struct FILE_INFO {
FILE_XFER* file_xfer; // nonzero if in the process of being up/downloaded
PROJECT* project;
int ref_cnt;
vector<STRING256> urls;
FILE_INFO();
~FILE_INFO();
int parse(FILE*);
int write(FILE*, bool to_server);
int delete_file(); // attempt to delete the underlying file
@ -99,17 +107,15 @@ struct FILE_REF {
bool main_program;
FILE_INFO* file_info;
int parse(FILE*, char*);
int write(FILE*, char*);
int parse(FILE*);
int write(FILE*);
};
struct APP_VERSION {
char app_name[256];
//char file_name[256];
int version_num;
APP* app;
PROJECT* project;
//FILE_INFO* file_info;
vector<FILE_REF> app_files;
int parse(FILE*);

2
client/configure.in
View File

@ -37,7 +37,7 @@ AC_CHECK_LIB(stdc++, cerr)
dnl Checks for header files.
AC_HEADER_DIRENT
AC_HEADER_STDC
AC_CHECK_HEADERS(fcntl.h sys/time.h unistd.h sys/select.h sys/statvfs.h sys/swap.h)
AC_CHECK_HEADERS(fcntl.h sys/time.h unistd.h sys/select.h sys/statvfs.h sys/swap.h sys/systeminfo.h)
dnl Checks for typedefs, structures, and compiler characteristics.
AC_C_CONST

20
client/cs_files.C
View File

@ -55,17 +55,20 @@ bool CLIENT_STATE::start_file_xfers() {
if (!fip->generated_locally && !fip->file_present && !fxp) {
fxp = new FILE_XFER;
get_pathname(fip, pathname);
fxp->init_download( fip->url, pathname);
fxp->init_download(fip->urls[0].text, pathname);
retval = file_xfers->insert(fxp);
if (retval) {
fprintf(stderr,
"couldn't start download for %s: error %d\n",
fip->url, retval
fip->urls[0].text, retval
);
} else {
fip->file_xfer = fxp;
if (log_flags.file_xfer) {
printf("started download of %s to %s\n", fip->url, pathname);
printf(
"started download of %s to %s\n",
fip->urls[0].text, pathname
);
}
}
action = true;
@ -75,17 +78,20 @@ bool CLIENT_STATE::start_file_xfers() {
) {
fxp = new FILE_XFER;
get_pathname(fip, pathname);
fxp->init_upload( fip->url, pathname);
fxp->init_upload( fip->urls[0].text, pathname);
retval = file_xfers->insert(fxp);
if (retval) {
fprintf(stderr,
"couldn't start upload for %s: error %d\n",
fip->url, retval
fip->urls[0].text, retval
);
} else {
fip->file_xfer = fxp;
if (log_flags.file_xfer) {
printf("started upload of %s to %s\n", pathname, fip->url);
printf(
"started upload of %s to %s\n",
pathname, fip->urls[0].text
);
}
}
action = true;
@ -95,7 +101,7 @@ bool CLIENT_STATE::start_file_xfers() {
if (log_flags.file_xfer) {
printf(
"file transfer done for %s; retval %d\n",
fip->url, fxp->file_xfer_retval
fip->urls[0].text, fxp->file_xfer_retval
);
}
file_xfers->remove(fxp);

2
client/error_numbers.h
View File

@ -31,3 +31,5 @@
#define ERR_OPENDIR -111
#define ERR_XML_PARSE -112
#define ERR_GETHOSTBYNAME -113
#define ERR_GIVEUP -114
// too much time has elapsed without progress on file xfer

3
client/file_xfer.h
View File

@ -20,7 +20,8 @@
#ifndef _FILE_XFER_
#define _FILE_XFER_
// FILE_XFER objects encapsulate the transfer of file
// FILE_XFER objects encapsulate the transfer of a file
// to/from a particular server.
// TODO: use the HTTP Range header fields to do partial xfers
#include "client_types.h"

2
client/hostinfo_unix.C
View File

@ -32,7 +32,9 @@
#ifdef _WIN32
#include <windows.h>
#else
#if HAVE_SYS_SYSTEMINFO_H
#include <sys/systeminfo.h>
#endif
#include <sys/utsname.h>
#include <unistd.h>
#include <netdb.h>

94
client/http.C
View File

@ -54,9 +54,34 @@ void parse_url(char* url, char* host, char* file) {
// Note: HTTP 1.1 keeps connection open.
// We use 1.0 so we don't have to count bytes.
//
void http_get_request_header(char* buf, char* host, char* file) {
void http_get_request_header(char* buf, char* host, char* file, int offset) {
if (offset) {
sprintf(buf,
"GET /%s;byte-range %d- HTTP/1.0\015\012"
"User-Agent: BOINC client\015\012"
"Host: %s:80\015\012"
"Accept: */*\015\012"
"\015\012",
file, offset,
host
);
} else {
sprintf(buf,
"GET /%s HTTP/1.0\015\012"
"User-Agent: BOINC client\015\012"
"Host: %s:80\015\012"
"Accept: */*\015\012"
"\015\012",
file,
host
);
}
}
void http_head_request_header(char* buf, char* host, char* file) {
sprintf(buf,
"GET /%s HTTP/1.0\015\012"
"HEAD /%s HTTP/1.0\015\012"
"User-Agent: BOINC client\015\012"
"Host: %s:80\015\012"
"Accept: */*\015\012"
@ -78,17 +103,34 @@ void http_post_request_header(char* buf, char* host, char* file, int size) {
);
}
void http_put_request_header(char* buf, char* host, char* file, int size) {
sprintf(buf,
"PUT /%s HTTP/1.0\015\012"
"Pragma: no-cache\015\012"
"Cache-Control: no-cache\015\012"
"Host: %s:80\015\012"
"Content-Type: application/octet-stream\015\012"
"Content-Length: %d\015\012"
"\015\012",
file, host, size
);
void http_put_request_header(
char* buf, char* host, char* file, int size, int offset
) {
if (offset) {
sprintf(buf,
"PUT /%s;byte-range %d- HTTP/1.0\015\012"
"Pragma: no-cache\015\012"
"Cache-Control: no-cache\015\012"
"Host: %s:80\015\012"
"Content-Type: application/octet-stream\015\012"
"Content-Length: %d\015\012"
"\015\012",
file, offset,
host, size
);
} else {
sprintf(buf,
"PUT /%s HTTP/1.0\015\012"
"Pragma: no-cache\015\012"
"Cache-Control: no-cache\015\012"
"Host: %s:80\015\012"
"Content-Type: application/octet-stream\015\012"
"Content-Length: %d\015\012"
"\015\012",
file,
host, size
);
}
}
int read_http_reply_header(int socket, HTTP_REPLY_HEADER& header) {
@ -127,7 +169,16 @@ HTTP_OP::HTTP_OP() {
HTTP_OP::~HTTP_OP() {
}
int HTTP_OP::init_get(char* url, char* out) {
int HTTP_OP::init_head(char* url) {
parse_url(url, hostname, filename);
NET_XFER::init(hostname, 80, HTTP_BLOCKSIZE);
http_op_type = HTTP_OP_HEAD;
http_op_state = HTTP_STATE_CONNECTING;
return 0;
}
int HTTP_OP::init_get(char* url, char* out, int off) {
offset = off;
parse_url(url, hostname, filename);
NET_XFER::init(hostname, 80, HTTP_BLOCKSIZE);
strcpy(outfile, out);
@ -150,9 +201,10 @@ int HTTP_OP::init_post(char* url, char* in, char* out) {
return 0;
}
int HTTP_OP::init_put(char* url, char* in) {
int HTTP_OP::init_put(char* url, char* in, int off) {
int retval;
offset = off;
parse_url(url, hostname, filename);
NET_XFER::init(hostname, 80, HTTP_BLOCKSIZE);
strcpy(infile, in);
@ -206,11 +258,14 @@ bool HTTP_OP_SET::poll() {
);
break;
case HTTP_OP_GET:
http_get_request_header(hdr, htp->hostname, htp->filename);
http_get_request_header(hdr, htp->hostname, htp->filename, htp->offset);
break;
case HTTP_OP_HEAD:
http_head_request_header(hdr, htp->hostname, htp->filename);
break;
case HTTP_OP_PUT:
http_put_request_header(
hdr, htp->hostname, htp->filename, htp->content_length
hdr, htp->hostname, htp->filename, htp->content_length, htp->offset
);
break;
}
@ -238,6 +293,7 @@ bool HTTP_OP_SET::poll() {
htp->do_file_io = true;
break;
case HTTP_OP_GET:
case HTTP_OP_HEAD:
htp->http_op_state = HTTP_STATE_REPLY_HEADER;
htp->want_upload = false;
htp->want_download = true;
@ -269,6 +325,10 @@ bool HTTP_OP_SET::poll() {
break;
}
switch (htp->http_op_type) {
case HTTP_OP_HEAD:
htp->http_op_state = HTTP_STATE_DONE;
htp->http_op_retval = 0;
break;
case HTTP_OP_GET:
case HTTP_OP_POST:
htp->http_op_state = HTTP_STATE_REPLY_BODY;

7
client/http.h
View File

@ -30,6 +30,7 @@ struct HTTP_REPLY_HEADER {
#define HTTP_OP_GET 1
#define HTTP_OP_POST 2
#define HTTP_OP_PUT 3
#define HTTP_OP_HEAD 4
// represents an HTTP request in progress
//
@ -43,14 +44,16 @@ public:
char infile[256];
char outfile[256];
int content_length;
int offset;
HTTP_REPLY_HEADER hrh;
int http_op_state; // values below
int http_op_type;
int http_op_retval;
int init_get(char* url, char* outfile);
int init_head(char* url);
int init_get(char* url, char* outfile, int offset=0);
int init_post(char* url, char* infile, char* outfile);
int init_put(char* url, char* infile);
int init_put(char* url, char* infile, int offset=0);
bool http_op_done();
};

2
client/net_xfer.C
View File

@ -217,7 +217,7 @@ int NET_XFER_SET::do_select(int max_bytes, int& bytes_transferred) {
#ifdef _WIN32
getsockopt(fd, SOL_SOCKET, SO_ERROR, (char *)&n, (int *)&intsize);
#else
getsockopt(fd, SOL_SOCKET, SO_ERROR, &n, (int *)&intsize);
getsockopt(fd, SOL_SOCKET, SO_ERROR, &n, (unsigned int *)&intsize);
#endif
if (n) {
if (log_flags.net_xfer_debug) {

73
client/pers_file_xfer.C Normal file
View File

@ -0,0 +1,73 @@
// The contents of this file are subject to the Mozilla Public License
// Version 1.0 (the "License"); you may not use this file except in
// compliance with the License. You may obtain a copy of the License at
// http://www.mozilla.org/MPL/
//
// Software distributed under the License is distributed on an "AS IS"
// basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the
// License for the specific language governing rights and limitations
// under the License.
//
// The Original Code is the Berkeley Open Network Computing.
//
// The Initial Developer of the Original Code is the SETI@home project.
// Portions created by the SETI@home project are Copyright (C) 2002
// University of California at Berkeley. All Rights Reserved.
//
// Contributor(s):
//
// PERS_FILE_XFER represents a persistent file transfer.
// A set of URL is given.
// For download, the object attempts to download the file
// from any of the URLs.
// If one fails or is not available, try another,
// using an exponential backoff policy to avoid flooding servers.
// For upload, try to upload the file to the first URL;
// if that fails try the others.
int PERS_FILE_XFER::init(FILE_INFO&, bool is_upload) {
}
//
void PERS_FILE_XFER::try() {
}
void PERS_FILE_XFER::poll(unsigned int now) {
if (fxp) {
if (fxp->file_xfer_done) {
if (fxp->file_xfer_retval == 0) {
} else {
// file xfer failed.
// See if it's time to give up on the persistent file xfer
//
diff = now - fip->last_xfer_time;
if (diff > PERS_GIVEUP) {
pers_xfer_done = true;
pers_file_xfer_retval = ERR_GIVEUP;
}
}
}
} else {
// No file xfer is active.
// We must be waiting after a failure.
// See if it's time to try again.
//
if (now > retry_time) {
try();
}
}
}
int PERS_FILE_XFER_SET::poll() {
unsigned int ;
PERS_FILE_XFER* pfxp;
bool action = false;
int now = time(0);
for (i=0; i<pers_file_xfers.size(); i++) {
pers_file_xfers[i]->poll(now);
}
}

55
client/pers_file_xfer.h Normal file
View File

@ -0,0 +1,55 @@
// The contents of this file are subject to the Mozilla Public License
// Version 1.0 (the "License"); you may not use this file except in
// compliance with the License. You may obtain a copy of the License at
// http://www.mozilla.org/MPL/
//
// Software distributed under the License is distributed on an "AS IS"
// basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the
// License for the specific language governing rights and limitations
// under the License.
//
// The Original Code is the Berkeley Open Network Computing.
//
// The Initial Developer of the Original Code is the SETI@home project.
// Portions created by the SETI@home project are Copyright (C) 2002
// University of California at Berkeley. All Rights Reserved.
//
// Contributor(s):
//
// PERS_FILE_XFER represents a persistent file transfer.
// A set of URL is given in the FILE_INFO.
// For download, the object attempts to download the file
// from any of the URLs.
// If one fails or is not available, try another,
// using an exponential backoff policy to avoid flooding servers.
// For upload, try to upload the file to the first URL;
// if that fails try the others.
#define PERS_RETRY_DELAY_MIN 60
#define PERS_RETRY_DELAY_MAX (256*60)
#define PERS_GIVEUP (3600*24*7)
// give up on xfer if this time elapses since last byte xferred
class PERS_FILE_XFER {
int url_index; // which URL to try next
int nretry; // # of retries so far
FILE_INFO* fip;
bool is_upload;
FILE_XFER* fxp; // nonzero if file xfer in progress
int retry_time; // don't retry until this time
int init(FILE_INFO&, bool is_upload);
};
class PERS_FILE_XFER_SET {
vector<PERS_FILE_XFER>pers_file_xfers;
FILE_XFER_SET* file_xfers;
public:
PERS_FILE_XFER_SET(FILE_XFER_SET*);
int insert(PERS_FILE_XFER*);
int remove(PERS_FILE_XFER*);
int poll();
};

42
client/test_http.C
View File

@ -21,22 +21,43 @@
#include <unistd.h>
#include "http.h"
#include "log_flags.h"
#include "net_xfer.h"
#include "util.h"
int main() {
NET_XFER_SET nxs;
HTTP_OP_SET hos(&nxs);
HTTP_OP *op1, *op2;
int n;
HTTP_OP *op1=0, *op2=0, *op3=0;
int retval, n;
log_flags.http_debug = true;
log_flags.net_xfer_debug = true;
#if 0
op1 = new HTTP_OP;
op2 = new HTTP_OP;
op1->init_get("http://localhost.localdomain/my_index.html", "test_out1");
op2->init_post("http://localhost.localdomain/test-cgi/cgi", "test_in1", "test_out2");
retval = op1->init_get("http://localhost.localdomain/my_index.html", "test_out1");
if (retval) {
printf("init_post: %d\n", retval);
exit(1);
}
hos.insert(op1);
op2 = new HTTP_OP;
retval = op2->init_post("http://localhost.localdomain/test-cgi/cgi", "test_in1", "test_out2");
if (retval) {
printf("init_post: %d\n", retval);
exit(1);
}
hos.insert(op2);
#endif
op3 = new HTTP_OP;
retval = op3->init_head("http://localhost.localdomain/my_index2.html");
if (retval) {
printf("init_post: %d\n", retval);
exit(1);
}
hos.insert(op3);
while (1) {
nxs.poll(100000, n);
@ -51,8 +72,13 @@ int main() {
hos.remove(op2);
op2 = 0;
}
if (!op1 && !op2) break;
boinc_sleep(1);
if (op3 && op3->http_op_done()) {
printf("op3 done; status %d\n", op3->hrh.status);
hos.remove(op3);
op3 = 0;
}
if (!op1 && !op2 && !op3) break;
sleep(1);
}
printf("all done\n");
}

5
html/ops/db.php
View File

@ -35,11 +35,6 @@
while ($res = mysql_fetch_object($result)) {
show_result($res);
}
echo "<hr>Prefs";
$result = mysql_query("select * from prefs");
while ($prefs = mysql_fetch_object($result)) {
show_prefs($prefs);
}
echo "<hr>Users";
$result = mysql_query("select * from user");
while ($user = mysql_fetch_object($result)) {

14
html/user/db.inc
View File

@ -143,16 +143,10 @@ function show_user($user) {
row("email_addr", $user->email_addr);
row("country", $user->country);
row("postal_code", $user->postal_code);
echo "</table>\n";
}
function show_prefs($prefs) {
start_table();
row("ID", $prefs->id);
row("created", time_str($prefs->create_time));
row("modified", time_str($prefs->modified_time));
row("userid", $prefs->userid);
row("XML doc", "<pre>".htmlspecialchars($prefs->xml_doc)."</pre>");
row("total credit", $user->total_credit);
row("recent average credit", $user->expavg_credit);
row("preferences", "<pre>".htmlspecialchars($user->prefs)."</pre>");
row("prefs mod time", time_str($user->prefs_mod_time));
echo "</table>\n";
}

219
notes
View File

@ -1,14 +1,5 @@
Abstractions
"Project": each is described by a URL.
Each has its own database and control server.
"Application": a particular program.
A project may have several applications.
"Account": each user has a separate account with each project.
Each account has a unique email address and a server-assigned authenticator.
--------------------
Client files
two files:
@ -58,41 +49,6 @@ write events to log file:
logging flag is part of preferences
--------------------
division between database and XML
proposal: move as much info as possible out of the DB into XML files.
examples:
- workunits and results
WUs and results are described by XML files listing
their inputs, outputs, etc.
The DB entry for a WU contains only info relevant to scheduling:
memory/disk/communication requirement
- user info
A configuration is an XML file, opaque to the scheduling server
--------------------
WUs and results
- WUs and results are desribed by XML files that describe their
input and output files.
- each client computation is represented by a "result" DB record,
which is created BEFORE the client requests it.
The application server system must keep the DB supplied
with result records, or clients will starve.
NOTE: this is necessary to control where output files go.
Could also have a scheme where each application has a
"template" result file.
This instructs the client to create its own output file names.
When the client returns the result,
the server creates the result record and plugs in file names.
--------------------
File info
input files
"sticky": don't delete after result done
URL (if not already on client)
output files
"sticky": don't delete after result done
URL (optional; send here after result done)
--------------------
file xfer commands
implemented as WU/result pairs whose app is "file_xfer".
Can have just one input file, one output.
@ -124,122 +80,6 @@ WU attributes in DB, sched server
not all input files available
--------------------
Workunit affinity
This mechanism allows a sequence of WUs to get executed on the same host,
but allows the sequence to migrate (or be duplicated) if needed.
result attributes:
previous_resultid
This result is a "successor" to the previous one.
If all the sticky input and output files of the previous WU are present,
this WU can be executed efficiently.
has_successor
This result has a successor.
How it works:
The project generates a sequence of WUs,
each with one or more results.
It chains the results together into sequences.
When a client completes a result with successor,
it retains the result record.
NOTE: one goal of this design is to avoid the scheduler
having to know about individual files
--------------------
Scheduler request
The client sends all its results with successors
scheduler algorithm:
if there any results with predecessors
for which the client has all sticky files,
send them in preference to any other results
--------------------
database tables
application
platform
app_version
core_version
account
file
workunit
applicationid
file1 name1
file2 name2
nresults
result
workunitid
accountid
fileid
boolean verified
host
--------------------
State maintained on client
Config file (XML)
<config>
<update-time>123123</update-time>
// last time user added project or changed CPU shares
<max-disk-mb>1000</max-disk-mb>
<min-work-hrs>10</min-work-hrs>
// if estimated work falls below this, try to get more
<max-work-hrs>10</max-work-hrs>
// don't get more work if estimate is above this
<max_ram_while_user_active>20</max_ram_while_user_active>
// zero means don't work while user active
<projects>
<project>
<url>http://wjwjwj</url>
<dont-contact-until>123123</dont_contact_until>
<control-server>blah.blah</control-server>
</control-server>blah.blah</control-server>
<cpu-share>1</cpu-share>
<max-disk-mb>100</max-disk-mb>
<cpu-total>5.44</cpu-total>
// this is zeroed out each time shares updated
<password>123123123</password>
// stored on client only; not sent to server in general
<email-address>foo@bar</email-address>
</home-project>
<file>
<md5>sfkjf</md5>
<url>akdjsfd</url>
<size>123123</size>
<complete/>
</file>
<workunit>
<name>skdjf</name>
<file>
<name>foo</name>
<appname>blah</appname>
// name by which app refers to file
</file>
</workunit>
<result>
<workunit-name>12938</workunit-name>
<result>
</project>
...
</projects>
</config>
--------------------
Security notes:
--------------------
Client directory structure
top-level dir
project dir (one per project)
CPU dir (one per CPU)
contains symbolic links to application file,
all input and output files
--------------------
Client logic
["network xfer" object encapsulates a set of file xfers in progress]
["processor" object: one for each CPU]
@ -331,10 +171,67 @@ Disk usage
Projects
For each project:
master URL
user name
project's master URL
email address
authenticator
resource %
show email address on web site?
accept emails from project?
project-specific prefs
------------------------------
retry policies:
general issues:
when and where to retry?
when to declare overall failure?
what to do if overall failure?
what needs to be saved in state file?
file xfer
download
round-robin through URLs with random exponential backoff
after connection failure or HTTP error.
2X from 1 minute up to 256 minutes
Overall failure after 1 week since last successful xfer
flag result as "file download failed",
abort other file xfers,
delete other files.
write log entry
State file:
record time of last successful xfer
upload
same as for download?
Use HTTP features to find file size on server
scheduler RPC
order projects according to hosts's "debt" to them.
Attempt to contact them in this order.
For each project, try all URLs in sequence with no delay.
If still need more work after a given RPC,
keep going to next project.
If still not enough work after a given "round",
do exponential backoff
2X from 1 minute up to 256 minutes
If reach 256 minutes, show error message to user and write to log
nothing saved in state file
------------------
Core/App connection
two unidirectional message streams.
files "core_to_app.xml" and "app_to_core.xml".
core->app:
initially:
requested frequency of app->core messages
app preferences
name of graphics shared-mem segment
recommended graphics parameters
frame rate
size
recommended checkpoint period
whether to do graphics
thereafter:
recommended graphics params
app->core
percent done
I just checkpointed
CPU time so far

4
test/1sec_result
View File

@ -7,8 +7,8 @@
<result>
<name><RESULT_NAME/></name>
<wu_name><WU_NAME/></wu_name>
<output_file>
<file_ref>
<file_name><OUTFILE_0/></file_name>
<open_name>out</open_name>
</output_file>
</file_ref>
</result>

5
test/account2.xml
View File

@ -1,15 +1,14 @@
<accounts>
<project>
<home_project/>
<domain>localhost</domain>
<scheduler_url>http://localhost/cgi-bin/boinc-cgi/cgi</scheduler_url>
<scheduler_url>http://localhost/boinc-cgi/cgi</scheduler_url>
<email_addr>david@localhost</email_addr>
<authenticator>3f7b90793a0175ad0bda68684e8bd136</authenticator>
<resource_share>2</resource_share>
</project>
<project>
<domain>zoot</domain>
<scheduler_url>http://zoot/cgi-bin/boinc-cgi/cgi</scheduler_url>
<scheduler_url>http://zoot/boinc-cgi/cgi</scheduler_url>
<email_addr>david@localhost</email_addr>
<authenticator>3f7b90793a0175ad0bda68684e8bd136</authenticator>
<resource_share>10</resource_share>

4
test/concat_result
View File

@ -7,8 +7,8 @@
<result>
<name><RESULT_NAME/></name>
<wu_name><WU_NAME/></wu_name>
<output_file>
<file_ref>
<file_name><OUTFILE_0/></file_name>
<open_name>out</open_name>
</output_file>
</file_ref>
</result>

8
test/concat_wu
View File

@ -11,13 +11,13 @@
<workunit>
<name><WU_NAME/></name>
<app_name>concat</app_name>
<input_file>
<file_ref>
<file_name><INFILE_0/></file_name>
<open_name>in1</open_name>
</input_file>
<input_file>
</file_ref>
<file_ref>
<file_name><INFILE_1/></file_name>
<open_name>in2</open_name>
</input_file>
</file_ref>
<command_line>in1 in2 out</command_line>
</workunit>

2
test/init.inc
View File

@ -97,7 +97,7 @@ function add_user($prefs_file) {
$cmd = "../tools/add user -email_addr $BOINC_EMAIL -user_name David -web_password foobar -authenticator 3f7b90793a0175ad0bda68684e8bd136 ";
if ($prefs_file) {
$cmd = $cmd." -prefs_file=$prefs_file";
$cmd = $cmd." -prefs_file $prefs_file";
}
PassThru($cmd);
}

20
test/laptop_prefs.xml
View File

@ -1,3 +1,17 @@
<stop_on_batteries/>
<hi_water_secs>10</hi_water_secs>
<lo_water_secs>1</lo_water_secs>
<preferences>
<dont_run_on_batteries/>
<high_water_days>10</high_water_days>
<low_water_days>1</low_water_days>
<disk_max_used_gb>0.4</disk_max_used_gb>
<disk_max_used_pct>50</disk_max_used_pct>
<disk_min_free_gb>0.4</disk_min_free_gb>
<project>
<master_url>http://localhost.localdomain</master_url>
<email_addr>david@localdomain</email_addr>
<authenticator>123892398</authenticator>
<resource_share>10</resource_share>
<project_specific>
<color-scheme>Tahiti Sunset</color-scheme>
</project_specific>
</project>\n"
</preferences>

9
test/test_1sec.php
View File

@ -1,9 +1,12 @@
#! /usr/local/bin/php
<?php
// test the 1sec application
// This tests whether CPU time is divided correctly between projects
// TODO: make this test what it's supposed to test
include_once("init.inc");
check_env_vars();
clear_db();
clear_data_dirs();
init_client_dirs("account2.xml");
@ -12,7 +15,7 @@
add_user(null);
add_app("1sec");
create_work("-appname 1sec -wu_name 1sec_wu -wu_template 1sec_wu -result_template 1sec_result -nresults 10");
//run_client();
//compare_file("concat_wu_0_0", "1sec_correct_output");
//compare_file("concat_wu_1_0", "1sec_correct_output");
start_feeder();
run_client();
stop_feeder();
?>

3
test/test_concat.php
View File

@ -16,7 +16,10 @@
add_user(null);
add_app("concat");
create_work("-appname concat -wu_name concat_wu -wu_template concat_wu -result_template concat_result -nresults 2 input input");
start_feeder();
run_client();
stop_feeder();
check_results_done();
compare_file("concat_wu_0_0", "concat_correct_output");
compare_file("concat_wu_1_0", "concat_correct_output");
?>

1
test/test_dynamic.php
View File

@ -5,6 +5,7 @@
include_once("init.inc");
check_env_vars();
clear_db();
clear_data_dirs();
init_client_dirs("account1.xml");

1
test/test_prefs.php
View File

@ -5,6 +5,7 @@
include_once("init.inc");
check_env_vars();
clear_db();
clear_data_dirs();
init_client_dirs("account1.xml");

5
test/test_projects.php
View File

@ -5,6 +5,7 @@
include_once("init.inc");
check_env_vars();
clear_db();
clear_data_dirs();
init_client_dirs("account2.xml");
@ -14,5 +15,7 @@
add_user(null);
add_app("upper_case");
create_work("-appname upper_case -wu_name uc_wu -wu_template uc_wu -result_template uc_result -nresults 2 small_input");
//run_client();
start_feeder();
run_client();
stop_feeder();
?>

3
test/test_stderr.php
View File

@ -5,6 +5,7 @@
include_once("init.inc");
check_env_vars();
clear_db();
clear_data_dirs();
init_client_dirs("account1.xml");
@ -14,7 +15,9 @@
add_user(null);
add_app("upper_case");
create_work("-appname upper_case -wu_name uc_wu -wu_template uc_wu -result_template uc_result -nresults 2 input input");
start_feeder();
run_client();
stop_feeder();
check_results_done();
compare_file("uc_wu_0_0", "uc_correct_output");

3
test/test_uc.php
View File

@ -15,11 +15,8 @@
add_user(null);
add_app("upper_case");
create_work("-appname upper_case -wu_name uc_wu -wu_template uc_wu -result_template uc_result -nresults 2 input input");
echo "starting feeder\n";
start_feeder();
echo "started feeder\n";
run_client();
echo "ran client\n";
stop_feeder();
check_results_done();

4
test/test_uc_slow.php
View File

@ -1,10 +1,12 @@
#! /usr/local/bin/php
<?php
// test the uc_slow application
// test the client checkpoint/restart mechanism,
// using the uc_slow application
//
include_once("init.inc");
check_env_vars();
clear_db();
clear_data_dirs();
init_client_dirs("account1.xml");

4
test/uc_result
View File

@ -7,8 +7,8 @@
<result>
<name><RESULT_NAME/></name>
<wu_name><WU_NAME/></wu_name>
<output_file>
<file_ref>
<file_name><OUTFILE_0/></file_name>
<fd>1</fd>
</output_file>
</file_ref>
</result>

4
test/uc_wu
View File

@ -6,8 +6,8 @@
<workunit>
<name><WU_NAME/></name>
<app_name>upper_case</app_name>
<input_file>
<file_ref>
<file_name><INFILE_0/></file_name>
<fd>0</fd>
</input_file>
</file_ref>
</workunit>

4
test/ucs_result
View File

@ -7,8 +7,8 @@
<result>
<name><RESULT_NAME/></name>
<wu_name><WU_NAME/></wu_name>
<output_file>
<file_ref>
<file_name><OUTFILE_0/></file_name>
<open_name>out</open_name>
</output_file>
</file_ref>
</result>

4
test/ucs_wu
View File

@ -6,8 +6,8 @@
<workunit>
<name><WU_NAME/></name>
<app_name>uc_slow</app_name>
<input_file>
<file_ref>
<file_name><INFILE_0/></file_name>
<open_name>in</open_name>
</input_file>
</file_ref>
</workunit>

4
tools/add.C
View File

@ -86,7 +86,7 @@ void add_app_version() {
strcpy(app.name, app_name);
retval = db_app_lookup_name(app);
if (retval) {
fprintf(stderr, "can't find app %s\n", app_name);
fprintf(stderr, "add_app_version(): can't find app %s\n", app_name);
db_print_error("db_app_lookup_name");
return;
}
@ -94,7 +94,7 @@ void add_app_version() {
strcpy(platform.name, platform_name);
retval = db_platform_lookup_name(platform);
if (retval) {
fprintf(stderr, "can't find platform %s\n", platform_name);
fprintf(stderr, "add_app_version(): can't find platform %s\n", platform_name);
db_print_error("db_platform_lookup_name");
return;
}