diff --git a/dcapi/boinc/dc_boinc.h b/dcapi/boinc/dc_boinc.h index 90c49acfbb..0a9d68f0e1 100644 --- a/dcapi/boinc/dc_boinc.h +++ b/dcapi/boinc/dc_boinc.h @@ -88,6 +88,10 @@ struct _DC_Workunit GList *input_files; int num_inputs; + /* Remote input file definitions. Elements are of type DC_RemoteFile */ + GList *remote_input_files; + int num_remote_inputs; + /* Output file definitions. Elements are of type char * */ GList *output_files; int num_outputs; diff --git a/dcapi/boinc/wu.C b/dcapi/boinc/wu.C index a4b2dd8e11..36620055b7 100644 --- a/dcapi/boinc/wu.C +++ b/dcapi/boinc/wu.C @@ -25,6 +25,7 @@ #include #include #include +#include #include "dc_boinc.h" @@ -417,6 +418,7 @@ static void wudesc_text(GMarkupParseContext *ctx, const char *text, static int write_wudesc(const DC_Workunit *wu) { DC_PhysicalFile *file; + DC_RemoteFile *rfile; GList *l; FILE *f; int i; @@ -449,6 +451,13 @@ static int write_wudesc(const DC_Workunit *wu) file->mode, file->label); } + for (l = wu->remote_input_files; l; l = l->next) + { + rfile = (DC_RemoteFile *)l->data; + fprintf(f, "\t%s\n", + rfile->label); + } + for (l = wu->output_files; l; l = l->next) fprintf(f, "\t%s\n", (char *)l->data); @@ -592,6 +601,15 @@ void DC_destroyWU(DC_Workunit *wu) _DC_destroyPhysicalFile(file); } + while (wu->remote_input_files) + { + DC_RemoteFile *file = (DC_RemoteFile *)wu->remote_input_files->data; + + wu->remote_input_files = g_list_delete_link(wu->remote_input_files, + wu->remote_input_files); + _DC_destroyRemoteFile(file); + } + while (wu->output_files) { g_free(wu->output_files->data); @@ -655,7 +673,6 @@ static int check_logical_name(DC_Workunit *wu, const char *logicalFileName) for (l = wu->input_files; l; l = l->next) { DC_PhysicalFile *file = (DC_PhysicalFile *)l->data; - if (!strcmp(file->label, logicalFileName)) { DC_log(LOG_ERR, "File %s is already registered as an " @@ -663,6 +680,16 @@ static int check_logical_name(DC_Workunit *wu, const char *logicalFileName) return DC_ERR_BADPARAM; } } + for (l = wu->remote_input_files; l; l = l->next) + { + DC_RemoteFile *file = (DC_RemoteFile *)l->data; + if (!strcmp(file->label, logicalFileName)) + { + DC_log(LOG_ERR, "File %s is already registered as a " + "remote input file", logicalFileName); + return DC_ERR_BADPARAM; + } + } for (l = wu->output_files; l; l = l->next) { if (!strcmp((char *)l->data, logicalFileName)) @@ -772,6 +799,36 @@ int DC_addWUInputAdvanced(DC_Workunit *wu, const char *logicalFileName, const ch return 0; } +int DC_addWURemoteInput(DC_Workunit *wu, const char *logicalFileName, const char *URL, + const char *md5, const int size) +{ + int ret; + DC_RemoteFile *file; + + /* Sanity checks */ + if (!wu || !logicalFileName || !URL || !md5 || !size) + { + DC_log(LOG_ERR, "%s: Missing arguments", __func__); + return DC_ERR_BADPARAM; + } + ret = check_logical_name(wu, logicalFileName); + if (ret) + return ret; + + file = _DC_createRemoteFile(logicalFileName, URL, md5, size); + if (!file) + return DC_ERR_INTERNAL; + + wu->remote_input_files = g_list_append(wu->remote_input_files, file); + wu->num_remote_inputs++; + + if (wu->serialized) + write_wudesc(wu); + + return 0; + +} + int DC_addWUOutput(DC_Workunit *wu, const char *logicalFileName) { int ret; @@ -928,6 +985,16 @@ static void append_wu_file_info(GString *tmpl, int idx) g_string_append(tmpl, "\n"); } +static void append_wu_remote_file_info(GString *tmpl, int idx, DC_RemoteFile *file) +{ + g_string_append(tmpl, "\n"); + g_string_append_printf(tmpl, "\t%d\n", idx); + g_string_append_printf(tmpl, "\t%s\n", file->url); + g_string_append_printf(tmpl, "\t%s\n", file->remotefilehash); + g_string_append_printf(tmpl, "\t%d\n", file->remotefilesize); + g_string_append(tmpl, "\n"); +} + static void append_wu_file_ref(GString *tmpl, int idx, const char *label) { g_string_append(tmpl, "\t\n"); @@ -940,7 +1007,7 @@ static char *generate_wu_template(DC_Workunit *wu) { struct wu_params params; GString *tmpl, *cmd; - int i, num_inputs; + int i, num_inputs, num_remote_inputs; GList *l; char *p; @@ -948,23 +1015,36 @@ static char *generate_wu_template(DC_Workunit *wu) /* Generate the file info block */ num_inputs = wu->num_inputs; + num_remote_inputs = wu->num_remote_inputs; if (wu->ckpt_name) num_inputs++; tmpl = g_string_new(""); for (i = 0; i < num_inputs; i++) append_wu_file_info(tmpl, i); + for (l = wu->remote_input_files; l && i < num_inputs + num_remote_inputs; + l = l->next, i++) + { + DC_RemoteFile *file = (DC_RemoteFile *)l->data; + append_wu_remote_file_info(tmpl, i, file); + } /* Checkpoint file, if exists */ if (wu->ckpt_name) append_wu_file_info(tmpl, i++); /* Generate the workunit description */ g_string_append(tmpl, "\n"); - for (i = 0, l = wu->input_files; l && i < wu->num_inputs; + for (i = 0, l = wu->input_files; l && i < num_inputs; l = l->next, i++) { DC_PhysicalFile *file = (DC_PhysicalFile *)l->data; + append_wu_file_ref(tmpl, i, file->label); + } + for (l = wu->remote_input_files; l && i < num_inputs + num_remote_inputs; + l = l->next, i++) + { + DC_PhysicalFile *file = (DC_PhysicalFile *)l->data; append_wu_file_ref(tmpl, i, file->label); } @@ -1200,6 +1280,7 @@ int DC_submitWU(DC_Workunit *wu) dbwu.priority = wu->priority; wu_template = generate_wu_template(wu); + printf("WU template is: %s", wu_template); result_template_file = generate_result_template(wu); if (!result_template_file) { @@ -1219,7 +1300,7 @@ int DC_submitWU(DC_Workunit *wu) free(cfgval); /* Create the input file name array as required by create_work() */ - ninputs = wu->num_inputs; + ninputs = wu->num_inputs + wu->num_remote_inputs; if (wu->ckpt_name) ninputs++; @@ -1230,6 +1311,13 @@ int DC_submitWU(DC_Workunit *wu) DC_PhysicalFile *file = (DC_PhysicalFile *)l->data; infiles[i] = get_input_download_name(wu, file->label, file->physicalfilename); } + for (l = wu->remote_input_files; l && i < wu->num_remote_inputs+wu->num_inputs; + l = l->next, i++) + { + DC_RemoteFile *file = (DC_RemoteFile *)l->data; + //infiles[i] = get_input_download_name(wu, file->label, NULL); + infiles[i] = g_strdup_printf("%s_%s", file->label, md5_string(file->url)); + } if (wu->ckpt_name) infiles[i++] = get_input_download_name(wu, wu->ckpt_name, NULL); /* Terminator so we can use g_strfreev() later */ diff --git a/dcapi/common/util.c b/dcapi/common/util.c index 5384ee49e2..1a3f96c304 100644 --- a/dcapi/common/util.c +++ b/dcapi/common/util.c @@ -166,6 +166,29 @@ DC_PhysicalFile *_DC_createPhysicalFile(const char *label, return file; } +DC_RemoteFile *_DC_createRemoteFile(const char *label, + const char *url, const char *md5, const int size) +{ + DC_RemoteFile *file; + + file = malloc(sizeof(*file)); + if (!file) + return NULL; + + file->label = strdup(label); + file->url = strdup(url); + file->remotefilehash = strdup(md5); + file->remotefilesize = size; + + if (!file->label || !file->url || !file->remotefilehash) + { + _DC_destroyRemoteFile(file); + return NULL; + } + + return file; +} + void _DC_destroyPhysicalFile(DC_PhysicalFile *file) { if (!file) @@ -178,5 +201,15 @@ void _DC_destroyPhysicalFile(DC_PhysicalFile *file) if (file->physicalfilehash) free(file->physicalfilehash); free(file); - +} + +void _DC_destroyRemoteFile(DC_RemoteFile *file) +{ + if (!file) + return; + + free(file->label); + free(file->url); + free(file->remotefilehash); + free(file); } diff --git a/dcapi/condor/condor_master.c b/dcapi/condor/condor_master.c index abaf66b12a..365afa2abf 100644 --- a/dcapi/condor/condor_master.c +++ b/dcapi/condor/condor_master.c @@ -438,6 +438,13 @@ int DC_addWUInputAdvanced(DC_Workunit *wu, const char *logicalFileName, const ch return DC_ERR_INTERNAL; } +int DC_addWURemoteInput(DC_Workunit *wu, const char *logicalFileName, const char *URL, + const char *md5, const int size) +{ + DC_log(LOG_ERR,"Function \"%s\" is not implemented in this backend.",__func__); + return DC_ERR_INTERNAL; +} + /* Sets an input file for the work unit. */ int DC_addWUInput(DC_Workunit *wu, diff --git a/dcapi/doc/dc-api-sections.txt b/dcapi/doc/dc-api-sections.txt index 1ae17e8770..b02f08b695 100644 --- a/dcapi/doc/dc-api-sections.txt +++ b/dcapi/doc/dc-api-sections.txt @@ -76,6 +76,7 @@ DC_destroyMasterEvent DC_getWUNumber DC_createWU DC_addWUInput +DC_addWURemoteInput DC_addWUOutput DC_setWUPriority DC_serializeWU diff --git a/dcapi/doc/tmpl/dc.sgml b/dcapi/doc/tmpl/dc.sgml index 6bf4087a2a..6d8a8b17c5 100644 --- a/dcapi/doc/tmpl/dc.sgml +++ b/dcapi/doc/tmpl/dc.sgml @@ -272,6 +272,23 @@ units in state %DC_WU_READY (i.e., before the work unit is submitted). @Returns: 0 if successful or a #DC_ErrorCode. + + +Adds a remote input file to a work unit. This function may be called only for +work units in state %DC_WU_READY (i.e., before the work unit is submitted). + + +@wu: the work unit to add the input file to. +@logicalFileName: the logical file name the client will use to refer to this + input file. The client should call the DC_resolveFileName() function + with the %DC_FILE_IN file type and this name to get the real name of + the file on the local system. +@URL: remote URL of the input file. +@md5: the MD5 hash of the remote input file. +@size: the size of the remote input file in bytes. +@Returns: 0 if successful or a #DC_ErrorCode. + + Adds an output file specification to a work unit. Files registered using this diff --git a/dcapi/doc/tmpl/dc_common.sgml b/dcapi/doc/tmpl/dc_common.sgml index 91fe2a24c6..dce61b82e1 100644 --- a/dcapi/doc/tmpl/dc_common.sgml +++ b/dcapi/doc/tmpl/dc_common.sgml @@ -96,6 +96,18 @@ Describes a physical file. @label: logical name of the file as used by the application. @path: the real path name of the file. @mode: usage mode of the file. +@physicalfilename: +@physicalfilehash: + + + +Describes a remote file. + + +@label: logical name of the file as used by the application. +@url: URL of the remote file. +@remotefilehash: MD5 hash of the remote file. +@remotefilesize: size of the remote file in bytes. diff --git a/dcapi/include/dc.h b/dcapi/include/dc.h index 41d6963fb6..bf84367499 100644 --- a/dcapi/include/dc.h +++ b/dcapi/include/dc.h @@ -149,6 +149,10 @@ DC_Workunit *DC_createWU(const char *clientName, const char *arguments[], int DC_addWUInput(DC_Workunit *wu, const char *logicalFileName, const char *URL, DC_FileMode fileMode); +/* Sets a remote input file for the work unit. */ +int DC_addWURemoteInput(DC_Workunit *wu, const char *logicalFileName, const char *URL, + const char *md5, const int size); + /* Sets an input file for the work unit. */ int DC_addWUInputAdvanced(DC_Workunit *wu, const char *logicalFileName, const char *URL, DC_FileMode fileMode, const char *physicalFileName, const char *physicalFileHashString); diff --git a/dcapi/include/dc_common.h b/dcapi/include/dc_common.h index f9ebbe8de2..a91d5b6bfc 100644 --- a/dcapi/include/dc_common.h +++ b/dcapi/include/dc_common.h @@ -90,6 +90,16 @@ struct _DC_PhysicalFile char *physicalfilehash; }; +/* Descriptor of a remote file */ +typedef struct _DC_RemoteFile DC_RemoteFile; +struct _DC_RemoteFile +{ + char *label; + char *url; + char *remotefilehash; + int remotefilesize; +}; + /******************************************************************** * Function prototypes diff --git a/dcapi/include/dc_internal.h b/dcapi/include/dc_internal.h index 3c987f1057..f9bbfb8d2b 100644 --- a/dcapi/include/dc_internal.h +++ b/dcapi/include/dc_internal.h @@ -59,10 +59,18 @@ int _DC_parseBoolean(const char *value) __attribute__((visibility("hidden"))); DC_PhysicalFile *_DC_createPhysicalFile(const char *label, const char *path) __attribute__((visibility("hidden"))); +/* Allocates a remote file descriptor */ +DC_RemoteFile *_DC_createRemoteFile(const char *label, const char *url, const char *md5, const int size) + __attribute__((visibility("hidden"))); + /* De-allocates a physical file descriptor */ void _DC_destroyPhysicalFile(DC_PhysicalFile *file) __attribute__((visibility("hidden"))); +/* De-allocates a remote file descriptor */ +void _DC_destroyRemoteFile(DC_RemoteFile *file) + __attribute__((visibility("hidden"))); + /* Initializa a config file for a client */ int _DC_initClientConfig(const char *clientName, FILE *f) __attribute__((visibility("hidden"))); diff --git a/dcapi/local/local_master.c b/dcapi/local/local_master.c index b720267001..5b6b3a6bf1 100644 --- a/dcapi/local/local_master.c +++ b/dcapi/local/local_master.c @@ -508,7 +508,14 @@ int DC_addWUInputAdvanced(DC_Workunit *wu, const char *logicalFileName, const ch DC_log(LOG_ERR,"Function \"%s\" is not implemented in this backend.",__func__); return DC_ERR_INTERNAL; } - + +int DC_addWURemoteInput(DC_Workunit *wu, const char *logicalFileName, const char *URL, + const char *md5, const int size) +{ + DC_log(LOG_ERR,"Function \"%s\" is not implemented in this backend.",__func__); + return DC_ERR_INTERNAL; +} + int DC_addWUInput(DC_Workunit *wu, const char *logicalFileName, const char *URL, DC_FileMode fileMode) {