Rooba
/
boinc
mirror of https://github.com/BOINC/boinc.git
1
0
Fork 0
Code Issues 1 Packages Projects Releases Wiki Activity

Added support for "remote" files at least in BOINC. 

git-svn-id: svn+ssh://cvs.lpds.sztaki.hu/var/lib/svn/szdg/dcapi/trunk@2385 a7169a2c-3604-0410-bc95-c702d8d87f7a
This commit is contained in:
zfarkas 2010-09-13 13:14:18 +00:00 committed by Adam Visegradi
parent d47ae1c89e
commit 1397ae1a2b
11 changed files with 197 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
dcapi/boinc/dc_boinc.h
View File

@ -88,6 +88,10 @@ struct _DC_Workunit
GList *input_files;
int num_inputs;
/* Remote input file definitions. Elements are of type DC_RemoteFile */
GList *remote_input_files;
int num_remote_inputs;
/* Output file definitions. Elements are of type char * */
GList *output_files;
int num_outputs;

96
dcapi/boinc/wu.C
View File

@ -25,6 +25,7 @@
#include <sched_util.h>
#include <sched_config.h>
#include <backend_lib.h>
#include <md5_file.h>
#include "dc_boinc.h"
@ -417,6 +418,7 @@ static void wudesc_text(GMarkupParseContext *ctx, const char *text,
static int write_wudesc(const DC_Workunit *wu)
{
DC_PhysicalFile *file;
DC_RemoteFile *rfile;
GList *l;
FILE *f;
int i;
@ -449,6 +451,13 @@ static int write_wudesc(const DC_Workunit *wu)
file->mode, file->label);
}
for (l = wu->remote_input_files; l; l = l->next)
{
rfile = (DC_RemoteFile *)l->data;
fprintf(f, "\t<remote_input_label>%s</remote_input_label>\n",
rfile->label);
}
for (l = wu->output_files; l; l = l->next)
fprintf(f, "\t<output_label>%s</output_label>\n", (char *)l->data);
@ -592,6 +601,15 @@ void DC_destroyWU(DC_Workunit *wu)
_DC_destroyPhysicalFile(file);
}
while (wu->remote_input_files)
{
DC_RemoteFile *file = (DC_RemoteFile *)wu->remote_input_files->data;
wu->remote_input_files = g_list_delete_link(wu->remote_input_files,
wu->remote_input_files);
_DC_destroyRemoteFile(file);
}
while (wu->output_files)
{
g_free(wu->output_files->data);
@ -655,7 +673,6 @@ static int check_logical_name(DC_Workunit *wu, const char *logicalFileName)
for (l = wu->input_files; l; l = l->next)
{
DC_PhysicalFile *file = (DC_PhysicalFile *)l->data;
if (!strcmp(file->label, logicalFileName))
{
DC_log(LOG_ERR, "File %s is already registered as an "
@ -663,6 +680,16 @@ static int check_logical_name(DC_Workunit *wu, const char *logicalFileName)
return DC_ERR_BADPARAM;
}
}
for (l = wu->remote_input_files; l; l = l->next)
{
DC_RemoteFile *file = (DC_RemoteFile *)l->data;
if (!strcmp(file->label, logicalFileName))
{
DC_log(LOG_ERR, "File %s is already registered as a "
"remote input file", logicalFileName);
return DC_ERR_BADPARAM;
}
}
for (l = wu->output_files; l; l = l->next)
{
if (!strcmp((char *)l->data, logicalFileName))
@ -772,6 +799,36 @@ int DC_addWUInputAdvanced(DC_Workunit *wu, const char *logicalFileName, const ch
return 0;
}
int DC_addWURemoteInput(DC_Workunit *wu, const char *logicalFileName, const char *URL,
const char *md5, const int size)
{
int ret;
DC_RemoteFile *file;
/* Sanity checks */
if (!wu || !logicalFileName || !URL || !md5 || !size)
{
DC_log(LOG_ERR, "%s: Missing arguments", __func__);
return DC_ERR_BADPARAM;
}
ret = check_logical_name(wu, logicalFileName);
if (ret)
return ret;
file = _DC_createRemoteFile(logicalFileName, URL, md5, size);
if (!file)
return DC_ERR_INTERNAL;
wu->remote_input_files = g_list_append(wu->remote_input_files, file);
wu->num_remote_inputs++;
if (wu->serialized)
write_wudesc(wu);
return 0;
}
int DC_addWUOutput(DC_Workunit *wu, const char *logicalFileName)
{
int ret;
@ -928,6 +985,16 @@ static void append_wu_file_info(GString *tmpl, int idx)
g_string_append(tmpl, "</file_info>\n");
}
static void append_wu_remote_file_info(GString *tmpl, int idx, DC_RemoteFile *file)
{
g_string_append(tmpl, "<file_info>\n");
g_string_append_printf(tmpl, "\t<number>%d</number>\n", idx);
g_string_append_printf(tmpl, "\t<url>%s</url>\n", file->url);
g_string_append_printf(tmpl, "\t<md5_cksum>%s</md5_cksum>\n", file->remotefilehash);
g_string_append_printf(tmpl, "\t<nbytes>%d</nbytes>\n", file->remotefilesize);
g_string_append(tmpl, "</file_info>\n");
}
static void append_wu_file_ref(GString *tmpl, int idx, const char *label)
{
g_string_append(tmpl, "\t<file_ref>\n");
@ -940,7 +1007,7 @@ static char *generate_wu_template(DC_Workunit *wu)
{
struct wu_params params;
GString *tmpl, *cmd;
int i, num_inputs;
int i, num_inputs, num_remote_inputs;
GList *l;
char *p;
@ -948,23 +1015,36 @@ static char *generate_wu_template(DC_Workunit *wu)
/* Generate the file info block */
num_inputs = wu->num_inputs;
num_remote_inputs = wu->num_remote_inputs;
if (wu->ckpt_name)
num_inputs++;
tmpl = g_string_new("");
for (i = 0; i < num_inputs; i++)
append_wu_file_info(tmpl, i);
for (l = wu->remote_input_files; l && i < num_inputs + num_remote_inputs;
l = l->next, i++)
{
DC_RemoteFile *file = (DC_RemoteFile *)l->data;
append_wu_remote_file_info(tmpl, i, file);
}
/* Checkpoint file, if exists */
if (wu->ckpt_name)
append_wu_file_info(tmpl, i++);
/* Generate the workunit description */
g_string_append(tmpl, "<workunit>\n");
for (i = 0, l = wu->input_files; l && i < wu->num_inputs;
for (i = 0, l = wu->input_files; l && i < num_inputs;
l = l->next, i++)
{
DC_PhysicalFile *file = (DC_PhysicalFile *)l->data;
append_wu_file_ref(tmpl, i, file->label);
}
for (l = wu->remote_input_files; l && i < num_inputs + num_remote_inputs;
l = l->next, i++)
{
DC_PhysicalFile *file = (DC_PhysicalFile *)l->data;
append_wu_file_ref(tmpl, i, file->label);
}
@ -1200,6 +1280,7 @@ int DC_submitWU(DC_Workunit *wu)
dbwu.priority = wu->priority;
wu_template = generate_wu_template(wu);
printf("WU template is: %s", wu_template);
result_template_file = generate_result_template(wu);
if (!result_template_file)
{
@ -1219,7 +1300,7 @@ int DC_submitWU(DC_Workunit *wu)
free(cfgval);
/* Create the input file name array as required by create_work() */
ninputs = wu->num_inputs;
ninputs = wu->num_inputs + wu->num_remote_inputs;
if (wu->ckpt_name)
ninputs++;
@ -1230,6 +1311,13 @@ int DC_submitWU(DC_Workunit *wu)
DC_PhysicalFile *file = (DC_PhysicalFile *)l->data;
infiles[i] = get_input_download_name(wu, file->label, file->physicalfilename);
}
for (l = wu->remote_input_files; l && i < wu->num_remote_inputs+wu->num_inputs;
l = l->next, i++)
{
DC_RemoteFile *file = (DC_RemoteFile *)l->data;
//infiles[i] = get_input_download_name(wu, file->label, NULL);
infiles[i] = g_strdup_printf("%s_%s", file->label, md5_string(file->url));
}
if (wu->ckpt_name)
infiles[i++] = get_input_download_name(wu, wu->ckpt_name, NULL);
/* Terminator so we can use g_strfreev() later */

35
dcapi/common/util.c
View File

@ -166,6 +166,29 @@ DC_PhysicalFile *_DC_createPhysicalFile(const char *label,
return file;
}
DC_RemoteFile *_DC_createRemoteFile(const char *label,
const char *url, const char *md5, const int size)
{
DC_RemoteFile *file;
file = malloc(sizeof(*file));
if (!file)
return NULL;
file->label = strdup(label);
file->url = strdup(url);
file->remotefilehash = strdup(md5);
file->remotefilesize = size;
if (!file->label || !file->url || !file->remotefilehash)
{
_DC_destroyRemoteFile(file);
return NULL;
}
return file;
}
void _DC_destroyPhysicalFile(DC_PhysicalFile *file)
{
if (!file)
@ -178,5 +201,15 @@ void _DC_destroyPhysicalFile(DC_PhysicalFile *file)
if (file->physicalfilehash)
free(file->physicalfilehash);
free(file);
}
void _DC_destroyRemoteFile(DC_RemoteFile *file)
{
if (!file)
return;
free(file->label);
free(file->url);
free(file->remotefilehash);
free(file);
}

7
dcapi/condor/condor_master.c
View File

@ -438,6 +438,13 @@ int DC_addWUInputAdvanced(DC_Workunit *wu, const char *logicalFileName, const ch
return DC_ERR_INTERNAL;
}
int DC_addWURemoteInput(DC_Workunit *wu, const char *logicalFileName, const char *URL,
const char *md5, const int size)
{
DC_log(LOG_ERR,"Function \"%s\" is not implemented in this backend.",__func__);
return DC_ERR_INTERNAL;
}
/* Sets an input file for the work unit. */
int
DC_addWUInput(DC_Workunit *wu,

1
dcapi/doc/dc-api-sections.txt
View File

@ -76,6 +76,7 @@ DC_destroyMasterEvent
DC_getWUNumber
DC_createWU
DC_addWUInput
DC_addWURemoteInput
DC_addWUOutput
DC_setWUPriority
DC_serializeWU

17
dcapi/doc/tmpl/dc.sgml
View File

@ -272,6 +272,23 @@ units in state %DC_WU_READY (i.e., before the work unit is submitted).
@Returns: 0 if successful or a #DC_ErrorCode.
<!-- ##### FUNCTION DC_addWURemoteInput ##### -->
<para>
Adds a remote input file to a work unit. This function may be called only for
work units in state %DC_WU_READY (i.e., before the work unit is submitted).
</para>
@wu: the work unit to add the input file to.
@logicalFileName: the logical file name the client will use to refer to this
input file. The client should call the DC_resolveFileName() function
with the %DC_FILE_IN file type and this name to get the real name of
the file on the local system.
@URL: remote URL of the input file.
@md5: the MD5 hash of the remote input file.
@size: the size of the remote input file in bytes.
@Returns: 0 if successful or a #DC_ErrorCode.
<!-- ##### FUNCTION DC_addWUOutput ##### -->
<para>
Adds an output file specification to a work unit. Files registered using this

12
dcapi/doc/tmpl/dc_common.sgml
View File

@ -96,6 +96,18 @@ Describes a physical file.
@label: logical name of the file as used by the application.
@path: the real path name of the file.
@mode: usage mode of the file.
@physicalfilename:
@physicalfilehash:
<!-- ##### STRUCT DC_RemoteFile ##### -->
<para>
Describes a remote file.
</para>
@label: logical name of the file as used by the application.
@url: URL of the remote file.
@remotefilehash: MD5 hash of the remote file.
@remotefilesize: size of the remote file in bytes.
<!-- ##### FUNCTION DC_getCfgStr ##### -->
<para>

4
dcapi/include/dc.h
View File

@ -149,6 +149,10 @@ DC_Workunit *DC_createWU(const char *clientName, const char *arguments[],
int DC_addWUInput(DC_Workunit *wu, const char *logicalFileName, const char *URL,
DC_FileMode fileMode);
/* Sets a remote input file for the work unit. */
int DC_addWURemoteInput(DC_Workunit *wu, const char *logicalFileName, const char *URL,
const char *md5, const int size);
/* Sets an input file for the work unit. */
int DC_addWUInputAdvanced(DC_Workunit *wu, const char *logicalFileName, const char *URL,
DC_FileMode fileMode, const char *physicalFileName, const char *physicalFileHashString);

10
dcapi/include/dc_common.h
View File

@ -90,6 +90,16 @@ struct _DC_PhysicalFile
char *physicalfilehash;
};
/* Descriptor of a remote file */
typedef struct _DC_RemoteFile DC_RemoteFile;
struct _DC_RemoteFile
{
char *label;
char *url;
char *remotefilehash;
int remotefilesize;
};
/********************************************************************
* Function prototypes

8
dcapi/include/dc_internal.h
View File

@ -59,10 +59,18 @@ int _DC_parseBoolean(const char *value) __attribute__((visibility("hidden")));
DC_PhysicalFile *_DC_createPhysicalFile(const char *label, const char *path)
__attribute__((visibility("hidden")));
/* Allocates a remote file descriptor */
DC_RemoteFile *_DC_createRemoteFile(const char *label, const char *url, const char *md5, const int size)
__attribute__((visibility("hidden")));
/* De-allocates a physical file descriptor */
void _DC_destroyPhysicalFile(DC_PhysicalFile *file)
__attribute__((visibility("hidden")));
/* De-allocates a remote file descriptor */
void _DC_destroyRemoteFile(DC_RemoteFile *file)
__attribute__((visibility("hidden")));
/* Initializa a config file for a client */
int _DC_initClientConfig(const char *clientName, FILE *f)
__attribute__((visibility("hidden")));

9
dcapi/local/local_master.c
View File

@ -508,7 +508,14 @@ int DC_addWUInputAdvanced(DC_Workunit *wu, const char *logicalFileName, const ch
DC_log(LOG_ERR,"Function \"%s\" is not implemented in this backend.",__func__);
return DC_ERR_INTERNAL;
}
int DC_addWURemoteInput(DC_Workunit *wu, const char *logicalFileName, const char *URL,
const char *md5, const int size)
{
DC_log(LOG_ERR,"Function \"%s\" is not implemented in this backend.",__func__);
return DC_ERR_INTERNAL;
}
int DC_addWUInput(DC_Workunit *wu, const char *logicalFileName, const char *URL,
DC_FileMode fileMode)
{