shared memory communication

svn path=/trunk/boinc/; revision=1063
This commit is contained in:
Eric Heien 2003-03-17 19:24:38 +00:00
parent 2d3ae2c71f
commit d362a835ee
12 changed files with 230 additions and 111 deletions

View File

@ -66,6 +66,7 @@ extern BOOL win_loop_done;
#endif
#include "parse.h"
#include "shmem.h"
#include "util.h"
#include "error_numbers.h"
#include "graphics_api.h"
@ -97,6 +98,7 @@ static bool write_frac_done = false;
static bool this_process_active;
static bool time_to_quit = false;
bool using_opengl = false;
static APP_CLIENT_SHM *app_client_shm;
// read the INIT_DATA and FD_INIT files
//
@ -165,6 +167,7 @@ int boinc_init() {
boinc_install_signal_handlers();
set_timer(timer_period);
setup_shared_mem();
return 0;
}
@ -263,7 +266,7 @@ void boinc_quit(int sig) {
int boinc_finish(int status) {
last_checkpoint_cpu_time = boinc_cpu_time();
write_fraction_done_file(fraction_done,last_checkpoint_cpu_time,last_checkpoint_cpu_time);
update_app_progress(fraction_done, last_checkpoint_cpu_time, last_checkpoint_cpu_time);
#ifdef _WIN32
// Stop the timer
timeKillEvent(timer_id);
@ -277,6 +280,7 @@ int boinc_finish(int status) {
}
#endif
#endif
cleanup_shared_mem();
exit(status);
return 0;
}
@ -317,13 +321,13 @@ bool boinc_time_to_checkpoint() {
switch (eventState) {
case WAIT_OBJECT_0:
case WAIT_ABANDONED:
time_to_quit = true;
break;
time_to_quit = true;
break;
}
#endif
if (write_frac_done) {
write_fraction_done_file(fraction_done, boinc_cpu_time(), last_checkpoint_cpu_time);
update_app_progress(fraction_done, boinc_cpu_time(), last_checkpoint_cpu_time);
time_until_fraction_done_update = aid.fraction_done_update_period;
write_frac_done = false;
}
@ -339,7 +343,7 @@ bool boinc_time_to_checkpoint() {
int boinc_checkpoint_completed() {
last_checkpoint_cpu_time = boinc_cpu_time();
write_fraction_done_file(fraction_done,last_checkpoint_cpu_time,last_checkpoint_cpu_time);
update_app_progress(fraction_done, last_checkpoint_cpu_time, last_checkpoint_cpu_time);
ready_to_checkpoint = false;
time_until_checkpoint = aid.checkpoint_period;
// If it's time to quit, call boinc_finish which will
@ -403,7 +407,7 @@ double boinc_cpu_time() {
totTime = tKernel.QuadPart + tUser.QuadPart;
// Runtimes in 100-nanosecond units
cpu_secs += totTime / 10000000.0;
cpu_secs += totTime / 1.e7;
// Convert to seconds and return
return cpu_secs;
@ -503,6 +507,45 @@ int set_timer(double period) {
return retval;
}
void setup_shared_mem(void) {
#ifdef API_IGNORE_CLIENT
fprintf( stderr, "Ignoring client, so not attaching to shared memory.\n" );
return;
#endif
#ifdef HAVE_SYS_SHM_H
#ifdef HAVE_SYS_IPC_H
if (attach_shmem(aid.shm_key, (void**)&app_client_shm)) {
app_client_shm = NULL;
}
#endif
#endif
}
void cleanup_shared_mem(void) {
#ifdef HAVE_SYS_SHM_H
#ifdef HAVE_SYS_IPC_H
if (app_client_shm != NULL)
detach_shmem(app_client_shm);
#endif
#endif
}
int update_app_progress(double frac_done, double cpu_t, double cp_cpu_t) {
if (app_client_shm == NULL || (app_client_shm->access != APP_ACCESS_OK))
return -1;
sprintf( app_client_shm->message_buf,
"<fraction_done>%f</fraction_done>\n"
"<current_cpu_time>%f</current_cpu_time>\n"
"<checkpoint_cpu_time>%f</checkpoint_cpu_time>\n",
frac_done, cpu_t, cp_cpu_t );
app_client_shm->access = CLIENT_ACCESS_OK;
return 0;
}
int write_init_data_file(FILE* f, APP_INIT_DATA& ai) {
if (strlen(ai.app_preferences)) {
fprintf(f, "<app_preferences>\n%s</app_preferences>\n", ai.app_preferences);
@ -522,6 +565,7 @@ int write_init_data_file(FILE* f, APP_INIT_DATA& ai) {
"<user_expavg_credit>%f</user_expavg_credit>\n"
"<host_total_credit>%f</host_total_credit>\n"
"<host_expavg_credit>%f</host_expavg_credit>\n"
"<shm_key>%d</shm_key>\n"
"<checkpoint_period>%f</checkpoint_period>\n"
"<fraction_done_update_period>%f</fraction_done_update_period>\n",
ai.wu_cpu_time,
@ -529,6 +573,7 @@ int write_init_data_file(FILE* f, APP_INIT_DATA& ai) {
ai.user_expavg_credit,
ai.host_total_credit,
ai.host_expavg_credit,
ai.shm_key,
ai.checkpoint_period,
ai.fraction_done_update_period
);
@ -555,6 +600,7 @@ int parse_init_data_file(FILE* f, APP_INIT_DATA& ai) {
else if (parse_double(buf, "<host_total_credit>", ai.host_total_credit)) continue;
else if (parse_double(buf, "<host_expavg_credit>", ai.host_expavg_credit)) continue;
else if (parse_double(buf, "<wu_cpu_time>", ai.wu_cpu_time)) continue;
else if (parse_int(buf, "<shm_key>", ai.shm_key)) continue;
else if (parse_double(buf, "<checkpoint_period>", ai.checkpoint_period)) continue;
else if (parse_double(buf, "<fraction_done_update_period>", ai.fraction_done_update_period)) continue;
else fprintf(stderr, "parse_init_data_file: unrecognized %s", buf);
@ -562,40 +608,6 @@ int parse_init_data_file(FILE* f, APP_INIT_DATA& ai) {
return 0;
}
int write_fraction_done_file(double pct, double cpu, double checkpoint_cpu) {
FILE* f = fopen(FRACTION_DONE_TEMP_FILE, "w");
if (!f) return -1;
fprintf(f,
"<fraction_done>%f</fraction_done>\n"
"<cpu_time>%f</cpu_time>\n"
"<checkpoint_cpu_time>%f</checkpoint_cpu_time>\n",
pct,
cpu,
checkpoint_cpu
);
fclose(f);
#ifdef _WIN32
unlink(FRACTION_DONE_FILE);
#endif
rename(FRACTION_DONE_TEMP_FILE, FRACTION_DONE_FILE);
return 0;
}
int parse_fraction_done_file(FILE* f, double& pct, double& cpu, double& checkpoint_cpu) {
char buf[256];
while (fgets(buf, 256, f)) {
if (parse_double(buf, "<fraction_done>", pct)) continue;
else if (parse_double(buf, "<cpu_time>", cpu)) continue;
else if (parse_double(buf, "<checkpoint_cpu_time>", checkpoint_cpu)) continue;
else fprintf(stderr, "parse_fraction_done_file: unrecognized %s", buf);
}
return 0;
}
// TODO: this should handle arbitrarily many fd/filename pairs.
// Also, give the tags better names
int write_fd_init_file(FILE* f, char *file_name, int fdesc, int input_file ) {

View File

@ -28,7 +28,7 @@
#ifndef _BOINC_API
#define _BOINC_API
#define DEFAULT_FRACTION_DONE_UPDATE_PERIOD 10
#define DEFAULT_FRACTION_DONE_UPDATE_PERIOD 1
#define DEFAULT_CHECKPOINT_PERIOD 300
// MFILE supports a primitive form of checkpointing.
@ -62,9 +62,18 @@ struct APP_INIT_DATA {
double host_total_credit;
double host_expavg_credit;
double checkpoint_period; // recommended checkpoint period
int shm_key;
double fraction_done_update_period;
};
#define APP_ACCESS_OK 0
#define CLIENT_ACCESS_OK 1
struct APP_CLIENT_SHM {
int access;
char message_buf[4096];
};
extern int boinc_init();
extern int boinc_get_init_data(APP_INIT_DATA&);
extern int boinc_finish(int);
@ -79,20 +88,19 @@ extern int boinc_install_signal_handlers();
/////////// API ENDS HERE - IMPLEMENTATION STUFF FOLLOWS
int update_app_progress(double, double, double);
int write_init_data_file(FILE* f, APP_INIT_DATA&);
int parse_init_data_file(FILE* f, APP_INIT_DATA&);
int write_fd_init_file(FILE*, char*, int, int);
int parse_fd_init_file(FILE*);
int write_fraction_done_file(double, double, double);
int parse_fraction_done_file(FILE*, double&, double&, double&);
#define INIT_DATA_FILE "init_data.xml"
#define GRAPHICS_DATA_FILE "graphics.xml"
#define FD_INIT_FILE "fd_init.xml"
#define FRACTION_DONE_FILE "fraction_done.xml"
#define FRACTION_DONE_TEMP_FILE "fraction_done.tmp"
#define STDERR_FILE "stderr.txt"
int set_timer(double period);
void setup_shared_mem();
void cleanup_shared_mem();
#endif

View File

@ -16,7 +16,7 @@ CC = @CC@ $(CFLAGS) -I @top_srcdir@/api -I@top_srcdir@/lib
APIOBJS = ../api/boinc_api.o ../api/graphics_api.o
X11APIOBJS = ../api/boinc_api.x11.o ../api/graphics_api.x11.o ../api/x_opengl.x11.o
LIBS = ../api/mfile.o ../lib/parse.o ../lib/filesys.o ../lib/util.o
LIBS = ../api/mfile.o ../lib/parse.o ../lib/filesys.o ../lib/shmem.o ../lib/util.o
CLIBS = @LIBS@

View File

@ -3819,3 +3819,20 @@ David Mar 15 2003
sched/
feeder.C
Eric March 17, 2003
- Changed app->client communication to use shared memory rather
than files. The client sets up a shared memory segment when
starting the app. The app attaches to it and writes XML tags
into a 4K text buffer every second.
configure.in
api/
boinc_api.C,h
apps/
Makefile.in
client/
app.C,h
cs_apps.C
lib/
shmem.C,h

View File

@ -64,6 +64,7 @@
#include "file_names.h"
#include "log_flags.h"
#include "parse.h"
#include "shmem.h"
#include "util.h"
#include "app.h"
@ -86,6 +87,7 @@ ACTIVE_TASK::ACTIVE_TASK() {
result = NULL;
wup = NULL;
app_version = NULL;
app_client_shm = NULL;
pid = 0;
slot = 0;
state = PROCESS_UNINITIALIZED;
@ -151,6 +153,7 @@ int ACTIVE_TASK::start(bool first_time) {
aid.host_expavg_credit = wup->project->host_expavg_credit;
aid.checkpoint_period = DEFAULT_CHECKPOINT_PERIOD;
aid.fraction_done_update_period = DEFAULT_FRACTION_DONE_UPDATE_PERIOD;
aid.shm_key = 0;
aid.wu_cpu_time = checkpoint_cpu_time;
sprintf(init_data_path, "%s%s%s", slot_dir, PATH_SEPARATOR, INIT_DATA_FILE);
@ -163,6 +166,9 @@ int ACTIVE_TASK::start(bool first_time) {
show_message(wup->project, buf, MSG_ERROR);
return ERR_FOPEN;
}
#ifdef HAVE_SYS_IPC_H
aid.shm_key = ftok( init_data_path, slot );
#endif
retval = write_init_data_file(f, aid);
if (retval) return retval;
@ -321,9 +327,17 @@ int ACTIVE_TASK::start(bool first_time) {
thread_handle = process_info.hThread;
#else
char* argv[100];
// Setup shared memory to communicate between processes
// The app must attach to this shmem segment by calling boinc_init()
//
shm_key = aid.shm_key;
// Destroy any shared memory still hanging around from previous runs
//destroy_shmem(shm_key);
pid = fork();
if (pid == 0) {
// from here on we're running in a new process.
// If an error happens, exit nonzero so that the core client
// knows there was a problem.
@ -349,6 +363,13 @@ int ACTIVE_TASK::start(bool first_time) {
perror("execv");
exit(1);
}
// Create shared memory after forking, to prevent problems with the
// child inheriting bad information about the segment
if (!create_shmem(shm_key, sizeof(APP_CLIENT_SHM), (void**)&app_client_shm)) {
app_client_shm->access = APP_ACCESS_OK;
}
if (log_flags.task_debug) printf("forked process: pid %d\n", pid);
#endif
state = PROCESS_RUNNING;
@ -415,28 +436,13 @@ bool ACTIVE_TASK_SET::poll() {
#ifdef _WIN32
unsigned long exit_code;
FILETIME creation_time, exit_time, kernel_time, user_time;
ULARGE_INTEGER tKernel, tUser;
LONGLONG totTime;
bool found = false;
for (int i=0; i<active_tasks.size(); i++) {
atp = active_tasks[i];
if (GetExitCodeProcess(atp->pid_handle, &exit_code)) {
//
// Get the elapsed CPU time
if (GetProcessTimes(
atp->pid_handle, &creation_time, &exit_time,
&kernel_time, &user_time
)) {
tKernel.LowPart = kernel_time.dwLowDateTime;
tKernel.HighPart = kernel_time.dwHighDateTime;
tUser.LowPart = user_time.dwLowDateTime;
tUser.HighPart = user_time.dwHighDateTime;
// Runtimes in 100-nanosecond units
totTime = tKernel.QuadPart + tUser.QuadPart;
}
// Get the elapsed CPU time, checkpoint CPU time
atp->check_app_status();
atp->result->final_cpu_time = atp->checkpoint_cpu_time;
if (exit_code != STILL_ACTIVE) {
found = true;
@ -469,11 +475,10 @@ bool ACTIVE_TASK_SET::poll() {
}
if (found) return true;
#else
struct rusage rs;
int pid;
int stat;
pid = wait3(&stat, WNOHANG, &rs);
pid = wait3(&stat, WNOHANG, 0);
if (pid > 0) {
if (log_flags.task_debug) printf("process %d is done\n", pid);
atp = lookup_pid(pid);
@ -481,11 +486,11 @@ bool ACTIVE_TASK_SET::poll() {
fprintf(stderr, "ACTIVE_TASK_SET::poll(): pid %d not found\n", pid);
return true;
}
double x = rs.ru_utime.tv_sec + rs.ru_utime.tv_usec/1.e6;
atp->result->final_cpu_time = atp->starting_cpu_time + x;
atp->check_app_status();
atp->result->final_cpu_time = atp->checkpoint_cpu_time;
if (atp->state == PROCESS_ABORT_PENDING) {
atp->state = PROCESS_ABORTED;
atp->result->active_task_state = PROCESS_ABORTED;
atp->result->active_task_state = PROCESS_ABORTED;
gstate.report_result_error(
*(atp->result), 0, "process was aborted\n"
);
@ -525,6 +530,7 @@ bool ACTIVE_TASK_SET::poll() {
}
}
destroy_shmem(atp->shm_key);
atp->read_stderr_file();
clean_out_dir(atp->slot_dir);
@ -573,8 +579,8 @@ bool ACTIVE_TASK::read_stderr_file() {
//
int ACTIVE_TASK_SET::wait_for_exit(double wait_time) {
bool all_exited;
unsigned int i,n;
ACTIVE_TASK *atp;
unsigned int i,n;
ACTIVE_TASK *atp;
for (i=0; i<10; i++) {
boinc_sleep(wait_time/10.0);
@ -588,9 +594,7 @@ int ACTIVE_TASK_SET::wait_for_exit(double wait_time) {
}
}
if (all_exited) {
return 0;
}
if (all_exited) return 0;
}
return -1;
@ -745,31 +749,71 @@ int ACTIVE_TASK_SET::restart_tasks() {
return 0;
}
// See if the app has generated a new fraction-done file.
int ACTIVE_TASK::get_cpu_time() {
#ifdef _WIN32
FILETIME creation_time, exit_time, kernel_time, user_time;
ULARGE_INTEGER tKernel, tUser;
LONGLONG totTime;
// Get the elapsed CPU time
if (GetProcessTimes( pid_handle, &creation_time, &exit_time, &kernel_time, &user_time )) {
tKernel.LowPart = kernel_time.dwLowDateTime;
tKernel.HighPart = kernel_time.dwHighDateTime;
tUser.LowPart = user_time.dwLowDateTime;
tUser.HighPart = user_time.dwHighDateTime;
// Runtimes in 100 nanosecond units
totTime = tKernel.QuadPart + tUser.QuadPart;
current_cpu_time = checkpoint_cpu_time = starting_cpu_time + totTime/1.e7;
return 0;
}
#else
struct rusage rs;
pid_t ret_pid;
int stat;
ret_pid = wait4(pid, &stat, WNOHANG, &rs);
if (ret_pid > 0) {
double x = rs.ru_utime.tv_sec + rs.ru_utime.tv_usec/1.e6;
current_cpu_time = checkpoint_cpu_time = starting_cpu_time + x;
return 0;
}
#endif
return -1;
}
// See if the app has generated a new fraction-done buffer.
// If so read it and return true.
//
bool ACTIVE_TASK::check_app_status_files() {
FILE* f;
char path[256];
bool found = false;
int retval;
sprintf(path, "%s%s%s", slot_dir, PATH_SEPARATOR, FRACTION_DONE_FILE);
f = fopen(path, "r");
if (f) {
found = true;
retval = parse_fraction_done_file(f, fraction_done, current_cpu_time, checkpoint_cpu_time);
fclose(f);
if (retval) return false;
retval = file_delete(path);
if (retval) {
fprintf(stderr,
"ACTIVE_TASK.check_app_status_files: could not delete %s: %d\n",
path, retval
);
bool ACTIVE_TASK::check_app_status() {
if (app_client_shm == NULL) {
fraction_done = 0;
get_cpu_time();
} else {
if (app_client_shm->access == CLIENT_ACCESS_OK) {
fraction_done = current_cpu_time = checkpoint_cpu_time = 0.0;
parse_double(app_client_shm->message_buf, "<fraction_done>", fraction_done);
parse_double(app_client_shm->message_buf, "<current_cpu_time>", current_cpu_time);
parse_double(app_client_shm->message_buf, "<checkpoint_cpu_time>", checkpoint_cpu_time);
app_client_shm->access = APP_ACCESS_OK;
return false;
}
}
return found;
return false;
}
// Check status of all active tasks
//
void ACTIVE_TASK_SET::check_apps() {
unsigned int i;
ACTIVE_TASK *atp;
for (i=0; i<active_tasks.size(); i++) {
atp = active_tasks[i];
atp->check_app_status();
}
}
// Returns the estimated time to completion (in seconds) of this task,
@ -811,7 +855,7 @@ bool ACTIVE_TASK_SET::poll_time() {
for (i=0; i<active_tasks.size(); i++) {
atp = active_tasks[i];
updated |= atp->check_app_status_files();
updated |= atp->check_app_status();
}
return updated;

View File

@ -28,6 +28,7 @@
#include <stdio.h>
#include <vector>
#include "client_types.h"
#include "boinc_api.h"
class CLIENT_STATE;
typedef int PROCESS_ID;
@ -55,11 +56,14 @@ class ACTIVE_TASK {
public:
#ifdef _WIN32
HANDLE pid_handle, thread_handle, quitRequestEvent;
#else
key_t shm_key;
#endif
RESULT* result;
WORKUNIT* wup;
APP_VERSION* app_version;
PROCESS_ID pid;
APP_CLIENT_SHM *app_client_shm;
int slot; // which slot (determines directory)
int state;
int exit_status;
@ -98,7 +102,8 @@ public:
int suspend();
int unsuspend();
bool check_app_status_files();
int get_cpu_time();
bool check_app_status();
double est_time_to_completion();
bool read_stderr_file();
@ -120,6 +125,7 @@ public:
int restart_tasks();
void request_tasks_exit();
void kill_tasks();
void check_apps();
int get_free_slot(int total_slots);
int write(FILE*);

View File

@ -59,15 +59,17 @@ int CLIENT_STATE::cleanup_and_exit() {
}
int CLIENT_STATE::exit_tasks() {
// TODO: unsuspend active tasks so they have a chance to checkpoint
// Send a request to the tasks to exit
active_tasks.request_tasks_exit();
// Wait a second for them to exit normally
active_tasks.wait_for_exit(1);
// And then just kill them
active_tasks.kill_tasks();
// Wait a second for them to exit normally, if they don't then kill them
if (active_tasks.wait_for_exit(1))
active_tasks.kill_tasks();
// Check their final CPU time
active_tasks.check_apps();
return 0;
}

View File

@ -29,7 +29,7 @@ dnl Checks for header files.
AC_HEADER_DIRENT
AC_HEADER_STDC
AC_HEADER_SYS_WAIT
AC_CHECK_HEADERS(fcntl.h malloc.h strings.h sys/time.h unistd.h sys/systeminfo.h sys/resource.h sys/types.h dirent.h sys/utsname.h netdb.h netinet/in.h arpa/inet.h signal.h sys/wait.h sys/file.h)
AC_CHECK_HEADERS(fcntl.h malloc.h strings.h sys/time.h unistd.h sys/systeminfo.h sys/resource.h sys/types.h dirent.h sys/utsname.h netdb.h netinet/in.h arpa/inet.h signal.h sys/wait.h sys/file.h sys/ipc.h sys/shm.h)
AC_CHECK_HEADERS(mysql/include/mysql_com.h mysql/mysql_com.h)
dnl Checks for typedefs, structures, and compiler characteristics.

View File

@ -1,4 +1,12 @@
#ifdef HAVE_SYS_TYPES_H
#include <sys/types.h>
#endif
#ifdef HAVE_SYS_IPC_H
#include <sys/ipc.h>
#endif
#ifdef HAVE_SYS_MSG_H
#include <sys/msg.h>
#endif
extern int create_message_queue(key_t);
extern int receive_message(key_t,void*,size_t,bool);

View File

@ -27,12 +27,14 @@
#include "shmem.h"
int create_shmem(key_t key, int size, void** pp){
int create_shmem(key_t key, int size, void** pp) {
int id;
char buf[256];
assert(pp!=NULL);
id = shmget(key, size, IPC_CREAT|0777);
if (id < 0) {
perror("create_shmem: shmget");
sprintf(buf, "create_shmem: shmget: key: %x size: %d", (unsigned int)key, size);
perror(buf);
return -1;
}
return attach_shmem(key, pp);
@ -64,17 +66,19 @@ int destroy_shmem(key_t key){
int attach_shmem(key_t key, void** pp){
void* p;
char buf[256];
int id;
assert(pp!=NULL);
//fprintf(stderr, "%x\n", key);
id = shmget(key, 0, 0);
if (id < 0) {
perror("attach_shmem: shmget");
sprintf(buf, "attach_shmem: shmget: key: %x mem_addr: %d", (unsigned int)key, (int)pp);
perror(buf);
return -1;
}
p = shmat(id, 0, 0);
if ((int)p == -1) {
perror("attach_shmem: shmat");
sprintf(buf, "attach_shmem: shmat: key: %x mem_addr: %d", (unsigned int)key, (int)pp);
perror(buf);
return -1;
}
*pp = p;
@ -88,3 +92,20 @@ int detach_shmem(void* p) {
if (retval) perror("detach_shmem: shmdt");
return retval;
}
int shmem_info(key_t key) {
int id;
struct shmid_ds buf;
char buf2[256];
id = shmget(key, 0, 0);
if (id < 0) {
sprintf(buf2, "shmem_info: shmget: key: %x", (unsigned int)key);
perror(buf2);
return -1;
}
shmctl(id, IPC_STAT, &buf);
fprintf( stderr, "id: %d, size: %d, nattach: %d\n", id, buf.shm_segsz, buf.shm_nattch );
return 0;
}

View File

@ -19,3 +19,5 @@ extern int attach_shmem(key_t, void**);
// detach from a shared-mem segment
//
extern int detach_shmem(void*);
extern int shmem_info(key_t key);

5
todo
View File

@ -4,7 +4,7 @@ BUGS (arranged from high to low priority)
- window closes and does not reopen when workunit finishes
and new workunit starts
- CPU time updates infrequently (every 10 seconds),
should there be a user control for this?
add user control for this (HD write frequency)
- Client treats URL "maggie/ap/" different than URL "maggie/ap",
though this isn't really a bug it might be good to fix anyway
- global battery/user active prefs are always true in the client
@ -17,8 +17,6 @@ HIGH-PRIORITY (should do for beta test)
- Implement Screensaver "blank screen" functionality
multiple preference sets
implement server watchdogs
est_time_to_completion doesn't work for non-running tasks
@ -37,6 +35,7 @@ THINGS TO TEST (preferably with test scripts)
- WU failure: too many errors
- WU failure: too many good results
- credit is granted even if result arrives very late
- multiple preference sets
-----------------------
MEDIUM-PRIORITY (should do before public release)
-----------------------