Add input file URL substitution for http and attic files (TODO: docs).

git-svn-id: svn+ssh://cvs.lpds.sztaki.hu/var/lib/svn/szdg/dcapi/trunk@2505 a7169a2c-3604-0410-bc95-c702d8d87f7a

dcapi/boinc/dc_boinc.h

@@ -44,6 +44,10 @@ extern "C" {
 #define CFG_ENABLESUSPEND	"EnableSuspend"
 /* Client uses native BOINC API instead of DC-API */
 #define CFG_NATIVECLIENT	"NativeClient"
+/* Remote file rewrite regexp match string */
+#define CFG_REGEXPMATCH		"InputURLRewriteRegExpMatch"
+/* Remote file rewrite regexp replace string */
+#define CFG_REGEXPREPLACE	"InputURLRewriteRegExpReplace"
 
 /* File types in the working directory */
 typedef enum

dcapi/boinc/wu.C

@@ -286,6 +286,46 @@ static DC_Workunit *alloc_wu(void)
 	return wu;
 }
 
+/**
+ * Perform regular expression replacement for a given URL.
+ * @param url the URL to perform the replace on
+ * @return NULL-terminated string array of replacement result(s)
+ */
+gchar **replace_regex(const char *url)
+{
+	if (!url)
+		return NULL;
+
+	char *regex = DC_getCfgStr(CFG_REGEXPMATCH);
+	if (!regex)
+		return NULL;
+
+	char *replace = DC_getCfgStr(CFG_REGEXPREPLACE);
+	if (!replace)
+	{
+		g_free(regex);
+		return NULL;
+	}
+
+	GRegex *reg = g_regex_new(regex, (GRegexCompileFlags)0, (GRegexMatchFlags)0, NULL);
+	if (!reg)
+	{
+		g_free(regex);
+		g_free(replace);
+		return NULL;
+	}
+
+	gchar *res = g_regex_replace(reg, url, -1, 0, replace, (GRegexMatchFlags)0, NULL);
+	if (!res)
+	{
+		g_free(regex);
+		g_free(replace);
+		return NULL;
+	}
+
+	return g_strsplit(res, "\n", 0);
+}
+
 /********************************************************************
  * The WU description parser
  */
@@ -719,7 +759,7 @@ int DC_addWUInput(DC_Workunit *wu, const char *logicalFileName, const char *URL,
 	int ret;
 	va_list ap;
 	char *workpath;
-	DC_PhysicalFile *file;
+	DC_PhysicalFile *file = NULL;
 	DC_RemoteFile *rfile;
 
 	/* Sanity checks */
@@ -771,19 +811,40 @@ int DC_addWUInput(DC_Workunit *wu, const char *logicalFileName, const char *URL,
 		size_t size = va_arg(ap, size_t);
 		va_end(ap);
 
+		// If regular expression for the Attic file gives valid substitutions,
+		// use substitued URLs and keep the file remote
+		gchar **repl = replace_regex(URL);
+		if (repl)
+		{
+			g_strfreev(repl);
+			_DC_destroyPhysicalFile(file);
+
+			rfile = _DC_createRemoteFile(logicalFileName, URL, md5, size);
+			if (!rfile)
+				return DC_ERR_INTERNAL;
+
+			wu->remote_input_files = g_list_append(wu->remote_input_files, rfile);
+			wu->num_remote_inputs++;
+
+			if (wu->serialized)
+			write_wudesc(wu);
+
+			return 0;
+		}
+
 		char *physicalFileName = g_strrstr(URL,"/");
-		if ( !physicalFileName )
+		if (!physicalFileName)
-                {
+		{
-                        DC_log(LOG_ERR, "URL of attic file has wrong format! URL: '%s'", URL);
+			DC_log(LOG_ERR, "URL of attic file has wrong format! URL: '%s'", URL);
-                        return DC_ERR_BADPARAM;
+			return DC_ERR_BADPARAM;
-                }
+		}
 		physicalFileName++;
 		physicalFileName = g_strdup(physicalFileName);
-		
+
-		char *physicalFileHashString = g_strdup_printf("%s %i\n",md5,(int)size);
+		char *physicalFileHashString = g_strdup_printf("%s %i\n", md5, (int)size);
 
 		DC_log(LOG_DEBUG, "Attic file details: logic: %s, url:%s, md5:%s, size:%i, file:%s, filecont:%s",
-			logicalFileName,URL,md5,(int)size,physicalFileName,physicalFileHashString);
+			logicalFileName, URL, md5, (int)size, physicalFileName, physicalFileHashString);
 
 		file->physicalfilename = strdup(physicalFileName);
 		g_free(physicalFileName);
@@ -905,14 +966,14 @@ static int install_input_files(DC_Workunit *wu)
 		/* By creating the hard link ourselves we prevent BOINC from
 		 * copying the file later */
 		ret = link(file->path, dest);
-		if (ret) 
+		if (ret)
 		{
 			if (file->physicalfilename)
 			{
 				if (errno == EEXIST)
 				{
 					DC_log(LOG_NOTICE, "File %s already exists under Boinc at %s. Skipping...",file->physicalfilename,dest);
-				} 
+				}
 				else if (errno == ENOENT)
 				{
 					FILE *f;
@@ -921,22 +982,22 @@ static int install_input_files(DC_Workunit *wu)
 					{
 						DC_log(LOG_ERR, "Cannot create file %s: %s",dest,strerror(errno));
 						g_free(dest);
-	                                        return DC_ERR_INTERNAL;
+						return DC_ERR_INTERNAL;
 					}
 					fprintf(f, "This file has been generated by DC-API.\nOriginal location of this file:%s", dest);
 					fclose(f);
-                                        DC_log(LOG_DEBUG, "File %s has been created by DC-API under Boinc.",dest);
+					DC_log(LOG_DEBUG, "File %s has been created by DC-API under Boinc.",dest);
 				}
 				else
 				{
-	                                DC_log(LOG_ERR, "---Failed to install file %s to the "
+					DC_log(LOG_ERR, "---Failed to install file %s to the "
-                                        	"Boinc download directory at %s: %s",
+						"Boinc download directory at %s: %s",
-	                                        file->path, dest, strerror(errno));
+						file->path, dest, strerror(errno));
-	                                g_free(dest);
+					g_free(dest);
-        	                        return DC_ERR_INTERNAL;
+					return DC_ERR_INTERNAL;
 				}
 			}
-			else 
+			else
 			{
 				DC_log(LOG_ERR, "Failed to install file %s to the "
 					"Boinc download directory at %s: %s",
@@ -950,7 +1011,7 @@ static int install_input_files(DC_Workunit *wu)
 			const char *hashFileExt=".md5";
 			GString *hashFile;
 			FILE *f;
-			
+
 			hashFile = g_string_new(dest);
 			g_string_append(hashFile, hashFileExt);
 
@@ -975,7 +1036,7 @@ static int install_input_files(DC_Workunit *wu)
 				fclose(f);
 				DC_log(LOG_DEBUG, "Hash file \"%s\" has been created with content \"%s\".", hashFile->str, file->physicalfilehash);
 			}
-			
+
 			g_string_free(hashFile, TRUE);
 		}
 		g_free(dest);
@@ -1047,7 +1108,18 @@ static void append_wu_remote_file_info(GString *tmpl, int idx, DC_RemoteFile *fi
 {
 	g_string_append(tmpl, "<file_info>\n");
 	g_string_append_printf(tmpl, "\t<number>%d</number>\n", idx);
-	g_string_append_printf(tmpl, "\t<url>%s</url>\n", file->url);
+	gchar **alts = replace_regex(file->url);
+	if (!alts)
+	{
+		g_string_append_printf(tmpl, "\t<url>%s</url>\n", file->url);
+	}
+	else
+	{
+		int i;
+		for (i = 0; alts[i]; i++)
+			g_string_append_printf(tmpl, "\t<url>%s</url>\n", alts[i]);
+		g_strfreev(alts);
+	}
 	g_string_append_printf(tmpl, "\t<md5_cksum>%s</md5_cksum>\n", file->remotefilehash);
 	g_string_append_printf(tmpl, "\t<nbytes>%lu</nbytes>\n", file->remotefilesize);
 	g_string_append(tmpl, "</file_info>\n");