From 156930d204664d7e180da4b7fd3774d9b29a79dc Mon Sep 17 00:00:00 2001
From: David Anderson <davea@ssl.berkeley.edu>
Date: Fri, 6 Aug 2004 11:42:41 +0000
Subject: [PATCH] *** empty log message ***

svn path=/trunk/boinc/; revision=4003
---
 checkin_notes               | 31 ++++++++++++++++++++++++++
 html/inc/profile.inc        | 14 ++++++------
 html/user/stats.php         |  2 ++
 lib/util.C                  | 22 +++++++++++++++++--
 lib/util.h                  | 12 ++++++++++-
 sched/file_deleter.C        | 43 +++++++++++++++++++++++++++----------
 sched/file_upload_handler.C |  7 ++++--
 sched/make_work.C           | 10 ++++++---
 sched/sched_config.C        |  1 +
 sched/sched_config.h        |  3 +++
 sched/validate_util.C       | 10 ++++-----
 sched/wu_check.C            |  3 ++-
 tools/Makefile.am           |  1 +
 tools/Makefile.in           | 33 +++++++++++++++++++++++++---
 tools/backend_lib.C         | 33 ++++++++++++++++------------
 tools/backend_lib.h         | 20 ++---------------
 tools/create_work.C         |  9 ++------
 tools/dir_hier_move.C       | 40 +++++++++++++++++++++++++++++++++-
 tools/dir_hier_path.C       |  2 +-
 19 files changed, 220 insertions(+), 76 deletions(-)

diff --git a/checkin_notes b/checkin_notes
index b498990938..ad90bff354 100755
--- a/checkin_notes
+++ b/checkin_notes
@@ -15953,3 +15953,34 @@ Daniel 2004-08-05
         boinc_cli.dsp,vcproj
         boinc_gui.dsp,vcproj
 
+David  6 Aug 2004
+    - Changed server software to use the hierarchical directory mechanism.
+        This means using dir_hier_path() to generate the pathname
+        of an input or output file,
+        rather than just concatenating directory and filename.
+        Also (in process_wu_template()) use dir_hier_url()
+        to generate the appropriate URL
+    - Add a scheme for transitioning from flat to hierarchical download dir.
+        Can have a <download_dir_alt> element in config.xml,
+        giving the path of an old, flat dir.
+        file_deleter will look here if it can't find file in main download dir.
+    - Pass around SCHED_CONFIG in a few interfaces
+        (e.g. create_work()) in order to access uldl_dir_fanout
+
+    html/inc/
+        profile.inc
+    lib/
+        util.C,h
+    sched/
+        file_deleter.C
+        file_upload_handler.C
+        make_work.C
+        sched_config.C,h
+        validate_util.C
+        wu_check.C
+    tools/
+        Makefile.am
+        backend_lib.C,h
+        create_work.C
+        dir_hier_move.C
+        dir_hier_path.C
diff --git a/html/inc/profile.inc b/html/inc/profile.inc
index e73e11c51d..6aad98ecfa 100644
--- a/html/inc/profile.inc
+++ b/html/inc/profile.inc
@@ -425,11 +425,11 @@ function show_profile($userid, $verify_mode=false) {
 
     if (!$verify_mode) {
         $logged_in_user = get_logged_in_user(false);  // (false) since anyone can look at profiles.;
-	if (!$logged_in_user || ($user->id != $logged_in_user->id)) {
-	  $caching = true;
-	  $cache_args = "userid=$userid";
-	  start_cache(USER_PROFILE_TTL,$cache_args);
-	}
+        if (!$logged_in_user || ($user->id != $logged_in_user->id)) {
+            $caching = true;
+            $cache_args = "userid=$userid";
+            start_cache(USER_PROFILE_TTL,$cache_args);
+        }
     }
 
 
@@ -455,10 +455,10 @@ function show_profile($userid, $verify_mode=false) {
     show_profile_summary($user, $profile, $can_edit, $verify_mode);
     echo "<br><br>";
     show_profile_heading1();
-    echo ":", $profile->response1;
+    echo $profile->response1;
     echo "<br><br>";
     show_profile_heading2();
-    echo ":", $profile->response2;
+    echo $profile->response2;
     if (!$verify_mode) {
         page_tail();
     }
diff --git a/html/user/stats.php b/html/user/stats.php
index a4f1283ba1..baa5c4e173 100644
--- a/html/user/stats.php
+++ b/html/user/stats.php
@@ -20,6 +20,8 @@ This data can be summarized and represented as Web pages.
 Examples:
 <ul>
 <li>
+<a href=http://www.setisynergy.com/stats/index.php>BOINC Statistics for the WORLD!</a>
+<li>
 <a href=http://www.boinc.dk/index.php?page=statistics>http://www.boinc.dk</a>,
 developed by <a href=mailto:stats@boinc.dk>Janus Kristensen</a>.
 <li>
diff --git a/lib/util.C b/lib/util.C
index 594a8875c1..2eb52a4938 100755
--- a/lib/util.C
+++ b/lib/util.C
@@ -750,7 +750,8 @@ void update_average(
 }
 
 int dir_hier_path(
-    char* filename, char* root, int fanout, char* result, bool create
+    const char* filename, const char* root, int fanout, char* result,
+    bool create
 ) {
     int sum=0;
     char dir[256];
@@ -761,7 +762,7 @@ int dir_hier_path(
         return 0;
     }
 
-    char* p = filename;
+    char* p = (char*)filename;
     while (*p) sum += *p++;
     sum %= fanout;
     sprintf(dir, "%s/%x", root, sum);
@@ -774,3 +775,20 @@ int dir_hier_path(
     sprintf(result, "%s/%s", dir, filename);
     return 0;
 }
+
+int dir_hier_url(
+    const char* filename, const char* root, int fanout, char* result
+) {
+    int sum=0;
+
+    if (fanout==0) {
+        sprintf(result, "%s/%s", root, filename);
+        return 0;
+    }
+
+    char* p = (char*)filename;
+    while (*p) sum += *p++;
+    sum %= fanout;
+    sprintf(result, "%s/%x/%s", root, sum, filename);
+    return 0;
+}
diff --git a/lib/util.h b/lib/util.h
index 519d964511..1628e83839 100755
--- a/lib/util.h
+++ b/lib/util.h
@@ -114,7 +114,17 @@ int boinc_thread_cpu_time(HANDLE thread_handle, double& cpu, double& ws);
 
 extern void update_average(double, double, double, double&, double&);
 
+// convert filename to path in a hierarchical directory system
+//
 extern int dir_hier_path(
-    char* filename, char* root, int fanout, char* result, bool create=false
+    const char* filename, const char* root, int fanout, char* result,
+    bool create=false
 );
+
+// convert filename to URL in a hierarchical directory system
+//
+extern int dir_hier_url(
+    const char* filename, const char* root, int fanout, char* result
+);
+
 #endif
diff --git a/sched/file_deleter.C b/sched/file_deleter.C
index 75aab34d96..ae72746825 100644
--- a/sched/file_deleter.C
+++ b/sched/file_deleter.C
@@ -41,7 +41,7 @@ int wu_delete_files(WORKUNIT& wu) {
     char* p;
     char filename[256], pathname[256], buf[LARGE_BLOB_SIZE];
     bool no_delete=false;
-    int count_deleted = 0;
+    int count_deleted = 0, retval;
 
     safe_strcpy(buf, wu.xml_doc);
 
@@ -56,10 +56,21 @@ int wu_delete_files(WORKUNIT& wu) {
             no_delete = true;
         } else if (match_tag(p, "</file_info>")) {
             if (!no_delete) {
-                sprintf(pathname, "%s/%s", config.download_dir, filename);
-                log_messages.printf(SCHED_MSG_LOG::NORMAL, "[%s] deleting download/%s\n", wu.name, filename);
-                unlink(pathname);
-                ++count_deleted;
+                retval = dir_hier_path(
+                    filename, config.download_dir, config.uldl_dir_fanout,
+                    pathname
+                );
+                if (retval) {
+                    log_messages.printf(SCHED_MSG_LOG::CRITICAL, "[%s] dir_hier_path: %d\n", wu.name, retval);
+                } else {
+                    log_messages.printf(SCHED_MSG_LOG::NORMAL, "[%s] deleting download/%s\n", wu.name, filename);
+                    retval = unlink(pathname);
+                    if (retval && strlen(config.download_dir_alt)) {
+                        sprintf(pathname, "%s/%s", config.download_dir_alt, filename);
+                        unlink(pathname);
+                    }
+                    ++count_deleted;
+                }
             }
         }
         p = strtok(0, "\n");
@@ -85,13 +96,23 @@ int result_delete_files(RESULT& result) {
             no_delete = true;
         } else if (match_tag(p, "</file_info>")) {
             if (!no_delete) {
-                sprintf(pathname, "%s/%s", config.upload_dir, filename);
-                retval = unlink(pathname);
-                ++count_deleted;
-                log_messages.printf(SCHED_MSG_LOG::NORMAL,
-                    "[%s] unlinked %s; retval %d\n",
-                    result.name, filename, retval
+                retval = dir_hier_path(
+                    filename, config.upload_dir, config.uldl_dir_fanout,
+                    pathname
                 );
+                if (retval) {
+                    log_messages.printf(SCHED_MSG_LOG::CRITICAL,
+                        "[%s] dir_hier_path: %d\n",
+                        result.name, retval
+                    );
+                } else {
+                    retval = unlink(pathname);
+                    ++count_deleted;
+                    log_messages.printf(SCHED_MSG_LOG::NORMAL,
+                        "[%s] unlinked %s; retval %d\n",
+                        result.name, filename, retval
+                    );
+                }
             }
         }
         p = strtok(0, "\n");
diff --git a/sched/file_upload_handler.C b/sched/file_upload_handler.C
index 83189a838b..39a14c304f 100644
--- a/sched/file_upload_handler.C
+++ b/sched/file_upload_handler.C
@@ -267,7 +267,10 @@ int handle_file_upload(FILE* in, R_RSA_PUBLIC_KEY& key) {
                 );
             }
 
-            sprintf(path, "%s/%s", config.upload_dir, file_info.name);
+            retval = dir_hier_path(
+                file_info.name, config.upload_dir, config.uldl_dir_fanout,
+                path, true
+            );
             log_messages.printf(
                 SCHED_MSG_LOG::NORMAL,
                 "Starting upload of %s from %s [offset=%.0f, nbytes=%.0f]\n",
@@ -312,7 +315,7 @@ int handle_get_file_size(char* file_name) {
     // TODO: check to ensure path doesn't point somewhere bad
     // Use 64-bit variant
     //
-    sprintf(path, "%s/%s", config.upload_dir, file_name );
+    dir_hier_path(file_name, config.upload_dir, config.uldl_dir_fanout, path);
     retval = stat( path, &sbuf );
     if (retval && errno != ENOENT) {
         log_messages.printf(SCHED_MSG_LOG::DEBUG, "handle_get_file_size(): [%s] returning error\n", file_name);
diff --git a/sched/make_work.C b/sched/make_work.C
index 807b611f32..2d60d06c00 100644
--- a/sched/make_work.C
+++ b/sched/make_work.C
@@ -174,9 +174,13 @@ void make_work() {
                 sprintf(
                     new_file_name, "%s__%d_%d", file_name, start_time, seqno++
                 );
-                sprintf(pathname, "%s/%s", config.download_dir, file_name);
-                sprintf(
-                    new_pathname, "%s/%s",config.download_dir, new_file_name
+                dir_hier_path(
+                    file_name, config.download_dir, config.uldl_dir_fanout,
+                    pathname
+                );
+                dir_hier_path(
+                    new_file_name, config.download_dir, config.uldl_dir_fanout,
+                    new_pathname, true
                 );
                 sprintf(command,"ln %s %s", pathname, new_pathname);
                 log_messages.printf(SCHED_MSG_LOG::DEBUG, "executing command: %s\n", command);
diff --git a/sched/sched_config.C b/sched/sched_config.C
index fd2129b7bd..8ba7967703 100644
--- a/sched/sched_config.C
+++ b/sched/sched_config.C
@@ -49,6 +49,7 @@ int SCHED_CONFIG::parse(char* buf) {
     parse_str(buf, "<key_dir>", key_dir, sizeof(key_dir));
     parse_str(buf, "<download_url>", download_url, sizeof(download_url));
     parse_str(buf, "<download_dir>", download_dir, sizeof(download_dir));
+    parse_str(buf, "<download_dir_alt>", download_dir_alt, sizeof(download_dir_alt));
     parse_str(buf, "<upload_url>", upload_url, sizeof(upload_url));
     parse_str(buf, "<upload_dir>", upload_dir, sizeof(upload_dir));
     if (match_tag(buf, "<one_result_per_user_per_wu/>")) {
diff --git a/sched/sched_config.h b/sched/sched_config.h
index 93f78d04db..a80327c401 100644
--- a/sched/sched_config.h
+++ b/sched/sched_config.h
@@ -33,6 +33,9 @@ public:
     char key_dir[256];
     char download_url[256];
     char download_dir[256];
+    char download_dir_alt[256];
+        // old download dir, assumed to be flat
+        // (file deleter looks here if not in main dir)
     char upload_url[256];
     char upload_dir[256];
     bool one_result_per_user_per_wu;
diff --git a/sched/validate_util.C b/sched/validate_util.C
index 834c08d4ef..a0cd5f172f 100644
--- a/sched/validate_util.C
+++ b/sched/validate_util.C
@@ -26,6 +26,7 @@
 
 #include "error_numbers.h"
 #include "parse.h"
+#include "util.h"
 
 #include "sched_util.h"
 #include "sched_config.h"
@@ -39,15 +40,14 @@ extern SCHED_CONFIG config;
 
 // get the name of a result's (first) output file
 //
-int get_output_file_path(RESULT const& result, string& path) {
-    char buf[256];
+int get_output_file_path(RESULT const& result, string& path_str) {
+    char buf[256], path[256];
     bool flag;
 
     flag = parse_str(result.xml_doc_in, "<name>", buf, sizeof(buf));
     if (!flag) return ERR_XML_PARSE;
-    path = config.upload_dir;
-    path += '/';
-    path += buf;
+    dir_hier_path(buf, config.upload_dir, config.uldl_dir_fanout, path);
+    path_str = path;
     return 0;
 }
 
diff --git a/sched/wu_check.C b/sched/wu_check.C
index 1836b1fc4e..3e43a78749 100644
--- a/sched/wu_check.C
+++ b/sched/wu_check.C
@@ -24,6 +24,7 @@
 #include <cstdio>
 
 #include "parse.h"
+#include "util.h"
 #include "error_numbers.h"
 #include "boinc_db.h"
 #include "sched_config.h"
@@ -42,7 +43,7 @@ int get_file_path(WORKUNIT& wu, char* path) {
     bool flag;
     flag = parse_str(wu.xml_doc, "<name>", buf, sizeof(buf));
     if (!flag) return ERR_XML_PARSE;
-    sprintf(path, "%s/%s", config.download_dir, buf);
+    dir_hier_path(buf, config.download_dir, config.uldl_dir_fanout, path);
     return 0;
 }
 
diff --git a/tools/Makefile.am b/tools/Makefile.am
index 5e74e54eaf..62e2b4f1ac 100644
--- a/tools/Makefile.am
+++ b/tools/Makefile.am
@@ -44,4 +44,5 @@ dir_hier_move_SOURCES = \
     dir_hier_move.C \
     ../sched/sched_config.C     \
     ../lib/parse.C \
+    ../lib/filesys.C \
     ../lib/util.C
diff --git a/tools/Makefile.in b/tools/Makefile.in
index cd3be47216..7f6e66cd47 100644
--- a/tools/Makefile.in
+++ b/tools/Makefile.in
@@ -220,6 +220,7 @@ dir_hier_move_SOURCES = \
     dir_hier_move.C \
     ../sched/sched_config.C     \
     ../lib/parse.C \
+    ../lib/filesys.C \
     ../lib/util.C
 
 subdir = tools
@@ -237,7 +238,8 @@ am_create_work_OBJECTS = create_work.$(OBJEXT) backend_lib.$(OBJEXT) \
 create_work_OBJECTS = $(am_create_work_OBJECTS)
 create_work_LDFLAGS =
 am_dir_hier_move_OBJECTS = dir_hier_move.$(OBJEXT) \
-	sched_config.$(OBJEXT) parse.$(OBJEXT) util.$(OBJEXT)
+	sched_config.$(OBJEXT) parse.$(OBJEXT) filesys.$(OBJEXT) \
+	util.$(OBJEXT)
 dir_hier_move_OBJECTS = $(am_dir_hier_move_OBJECTS)
 dir_hier_move_LDADD = $(LDADD)
 dir_hier_move_DEPENDENCIES =
@@ -260,8 +262,9 @@ am__depfiles_maybe = depfiles
 @AMDEP_TRUE@	./$(DEPDIR)/boinc_db.Po ./$(DEPDIR)/create_work.Po \
 @AMDEP_TRUE@	./$(DEPDIR)/crypt.Po ./$(DEPDIR)/db_base.Po \
 @AMDEP_TRUE@	./$(DEPDIR)/dir_hier_move.Po \
-@AMDEP_TRUE@	./$(DEPDIR)/dir_hier_path.Po ./$(DEPDIR)/md5.Po \
-@AMDEP_TRUE@	./$(DEPDIR)/md5_file.Po ./$(DEPDIR)/parse.Po \
+@AMDEP_TRUE@	./$(DEPDIR)/dir_hier_path.Po ./$(DEPDIR)/filesys.Po \
+@AMDEP_TRUE@	./$(DEPDIR)/md5.Po ./$(DEPDIR)/md5_file.Po \
+@AMDEP_TRUE@	./$(DEPDIR)/parse.Po \
 @AMDEP_TRUE@	./$(DEPDIR)/process_result_template.Po \
 @AMDEP_TRUE@	./$(DEPDIR)/sched_config.Po \
 @AMDEP_TRUE@	./$(DEPDIR)/sign_executable.Po ./$(DEPDIR)/util.Po
@@ -325,6 +328,7 @@ sched_config.$(OBJEXT): ../sched/sched_config.C
 create_work$(EXEEXT): $(create_work_OBJECTS) $(create_work_DEPENDENCIES) 
 	@rm -f create_work$(EXEEXT)
 	$(CXXLINK) $(create_work_LDFLAGS) $(create_work_OBJECTS) $(create_work_LDADD) $(LIBS)
+filesys.$(OBJEXT): ../lib/filesys.C
 dir_hier_move$(EXEEXT): $(dir_hier_move_OBJECTS) $(dir_hier_move_DEPENDENCIES) 
 	@rm -f dir_hier_move$(EXEEXT)
 	$(CXXLINK) $(dir_hier_move_LDFLAGS) $(dir_hier_move_OBJECTS) $(dir_hier_move_LDADD) $(LIBS)
@@ -348,6 +352,7 @@ distclean-compile:
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/db_base.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dir_hier_move.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dir_hier_path.Po@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/filesys.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/md5.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/md5_file.Po@am__quote@
 @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/parse.Po@am__quote@
@@ -535,6 +540,28 @@ sched_config.obj: ../sched/sched_config.C
 @AMDEP_TRUE@@am__fastdepCXX_FALSE@	$(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
 @am__fastdepCXX_FALSE@	$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o sched_config.obj `if test -f '../sched/sched_config.C'; then $(CYGPATH_W) '../sched/sched_config.C'; else $(CYGPATH_W) '$(srcdir)/../sched/sched_config.C'`
 
+filesys.o: ../lib/filesys.C
+@am__fastdepCXX_TRUE@	if $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT filesys.o -MD -MP -MF "$(DEPDIR)/filesys.Tpo" \
+@am__fastdepCXX_TRUE@	  -c -o filesys.o `test -f '../lib/filesys.C' || echo '$(srcdir)/'`../lib/filesys.C; \
+@am__fastdepCXX_TRUE@	then mv "$(DEPDIR)/filesys.Tpo" "$(DEPDIR)/filesys.Po"; \
+@am__fastdepCXX_TRUE@	else rm -f "$(DEPDIR)/filesys.Tpo"; exit 1; \
+@am__fastdepCXX_TRUE@	fi
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@	source='../lib/filesys.C' object='filesys.o' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@	depfile='$(DEPDIR)/filesys.Po' tmpdepfile='$(DEPDIR)/filesys.TPo' @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@	$(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCXX_FALSE@	$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o filesys.o `test -f '../lib/filesys.C' || echo '$(srcdir)/'`../lib/filesys.C
+
+filesys.obj: ../lib/filesys.C
+@am__fastdepCXX_TRUE@	if $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT filesys.obj -MD -MP -MF "$(DEPDIR)/filesys.Tpo" \
+@am__fastdepCXX_TRUE@	  -c -o filesys.obj `if test -f '../lib/filesys.C'; then $(CYGPATH_W) '../lib/filesys.C'; else $(CYGPATH_W) '$(srcdir)/../lib/filesys.C'`; \
+@am__fastdepCXX_TRUE@	then mv "$(DEPDIR)/filesys.Tpo" "$(DEPDIR)/filesys.Po"; \
+@am__fastdepCXX_TRUE@	else rm -f "$(DEPDIR)/filesys.Tpo"; exit 1; \
+@am__fastdepCXX_TRUE@	fi
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@	source='../lib/filesys.C' object='filesys.obj' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@	depfile='$(DEPDIR)/filesys.Po' tmpdepfile='$(DEPDIR)/filesys.TPo' @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@	$(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCXX_FALSE@	$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o filesys.obj `if test -f '../lib/filesys.C'; then $(CYGPATH_W) '../lib/filesys.C'; else $(CYGPATH_W) '$(srcdir)/../lib/filesys.C'`
+
 .c.o:
 @am__fastdepCC_TRUE@	if $(COMPILE) -MT $@ -MD -MP -MF "$(DEPDIR)/$*.Tpo" \
 @am__fastdepCC_TRUE@	  -c -o $@ `test -f '$<' || echo '$(srcdir)/'`$<; \
diff --git a/tools/backend_lib.C b/tools/backend_lib.C
index 0e8444e5c7..8a51ce6b06 100644
--- a/tools/backend_lib.C
+++ b/tools/backend_lib.C
@@ -64,14 +64,12 @@ int read_filename(const char* path, char* buf, int len) {
 static int process_wu_template(
     WORKUNIT& wu,
     char* tmplate,
-    const char* dirpath,
     const char** infiles,
     int n,
-    const char* upload_url,
-    const char* download_url
+    SCHED_CONFIG& config
 ) {
     char* p;
-    char buf[LARGE_BLOB_SIZE], md5[33], path[256];
+    char buf[LARGE_BLOB_SIZE], md5[33], path[256], url[256];
     char out[LARGE_BLOB_SIZE];
     int retval, file_number;
     double nbytes;
@@ -93,20 +91,27 @@ static int process_wu_template(
                         fprintf(stderr, "No file number found\n");
                         return ERR_XML_PARSE;
                     }
-                    sprintf(path, "%s/%s", dirpath, infiles[file_number]);
+                    dir_hier_path(
+                        infiles[file_number], config.download_dir,
+                        config.uldl_dir_fanout, path
+                    );
                     retval = md5_file(path, md5, nbytes);
                     if (retval) {
                         fprintf(stderr, "process_wu_template: md5_file %d\n", retval);
                         return retval;
                     }
+                    dir_hier_url(
+                        infiles[file_number], config.download_url,
+                        config.uldl_dir_fanout, url
+                    );
                     sprintf(buf,
                         "    <name>%s</name>\n"
-                        "    <url>%s/%s</url>\n"
+                        "    <url>%s</url>\n"
                         "    <md5_cksum>%s</md5_cksum>\n"
                         "    <nbytes>%.0f</nbytes>\n"
                         "</file_info>\n",
                         infiles[file_number],
-                        download_url, infiles[file_number],
+                        url,
                         md5,
                         nbytes
                     );
@@ -277,13 +282,15 @@ int create_result(
 
 // make sure a WU's input files are actually there
 //
-int check_files(char** infiles, int ninfiles, char* download_dir) {
+int check_files(char** infiles, int ninfiles, SCHED_CONFIG& config) {
     int i;
     char path[256];
     FILE* f;
 
     for (i=0; i<ninfiles; i++) {
-        sprintf(path, "%s/%s", download_dir, infiles[i]);
+        dir_hier_path(
+            infiles[i], config.download_dir, config.uldl_dir_fanout, path
+        );
         f = fopen(path, "r");
         if (f) {
             fclose(f);
@@ -299,18 +306,17 @@ int create_work(
     const char* _wu_template,
     const char* result_template_filename,
     const char* result_template_filepath,
-    const char* infile_dir,
     const char** infiles,
     int ninfiles,
     R_RSA_PRIVATE_KEY& key,
-    const char* upload_url, const char* download_url
+    SCHED_CONFIG& config
 ) {
     int retval;
     char _result_template[LARGE_BLOB_SIZE];
     char wu_template[LARGE_BLOB_SIZE];
 
 #if 0
-    retval = check_files(infiles, ninfiles, download_dir);
+    retval = check_files(infiles, ninfiles, config);
     if (retval) {
         fprintf(stderr, "Missing input file: %s\n", infiles[0]);
         return -1;
@@ -320,8 +326,7 @@ int create_work(
     strcpy(wu_template, _wu_template);
     wu.create_time = time(0);
     retval = process_wu_template(
-        wu, wu_template, infile_dir, infiles, ninfiles,
-        upload_url, download_url
+        wu, wu_template, infiles, ninfiles, config
     );
     if (retval) {
         fprintf(stderr, "process_wu_template: %d\n", retval);
diff --git a/tools/backend_lib.h b/tools/backend_lib.h
index a28eda88ac..da87e4ce94 100644
--- a/tools/backend_lib.h
+++ b/tools/backend_lib.h
@@ -21,6 +21,7 @@
 #define H_BACKEND_LIB
 
 #include "crypt.h"
+#include "sched_config.h"
 #include "boinc_db.h"
 
 extern int add_signatures(char*, R_RSA_PRIVATE_KEY&);
@@ -54,27 +55,10 @@ extern int create_work(
     const char* wu_template,
     const char* result_template_filename,
     const char* result_template_filepath,
-    const char* infile_dir,
     const char** infiles,
     int ninfiles,
     R_RSA_PRIVATE_KEY&,
-    const char* upload_url,
-    const char* download_url
+    SCHED_CONFIG&
 );
 
-#if 0
-extern int create_sequence_group(
-    DB_WORKUNIT& wu,
-    char* wu_template,
-    char* result_template,
-    char* infile_dir,
-    char** infiles,
-    int ninfiles,
-    R_RSA_PRIVATE_KEY&,
-    char* upload_url,
-    char* download_url,
-    int nsteps
-);
-#endif
-
 #endif
diff --git a/tools/create_work.C b/tools/create_work.C
index a42f919a51..a03ed26398 100644
--- a/tools/create_work.C
+++ b/tools/create_work.C
@@ -64,7 +64,6 @@ int main(int argc, char** argv) {
     int i, ninfiles;
     R_RSA_PRIVATE_KEY key;
     char download_dir[256], db_name[256], db_passwd[256],db_user[256],db_host[256];
-    char upload_url[256], download_url[256];
     char buf[256];
     SCHED_CONFIG config;
 
@@ -150,9 +149,7 @@ int main(int argc, char** argv) {
         strcpy(db_passwd, config.db_passwd);
         strcpy(db_user, config.db_user);
         strcpy(db_host, config.db_host);
-        strcpy(download_url, config.download_url);
         strcpy(download_dir, config.download_dir);
-        strcpy(upload_url, config.upload_url);
         sprintf(keyfile, "%s/upload_private", config.key_dir);
     }
 
@@ -188,13 +185,11 @@ int main(int argc, char** argv) {
         wu,
         wu_template,
         result_template_file,
-	result_template_path,
-        download_dir,
+        result_template_path,
         const_cast<const char **>(infiles),
         ninfiles,
         key,
-        upload_url,
-        download_url
+        config
     );
     if (retval) {
         fprintf(stderr, "create_work: %d\n", retval);
diff --git a/tools/dir_hier_move.C b/tools/dir_hier_move.C
index 69c27338e7..8ec7a92523 100644
--- a/tools/dir_hier_move.C
+++ b/tools/dir_hier_move.C
@@ -1,2 +1,40 @@
-main() {
+// dir_hier_move src_dir dst_dir fanout
+//
+// move files from src_dir (flat) into dst_dir (hierarchical)
+// with the given fanout
+
+#include <stdio.h>
+#include <string>
+#include <stdlib.h>
+
+#include "filesys.h"
+#include "util.h"
+
+int main(int argc, char** argv) {
+    char* src_dir, *dst_dir;
+    int fanout=0;
+    std::string filename;
+    char dst_path[256], src_path[256];
+    int retval;
+    
+    if (argc != 4) exit(1);
+    src_dir = argv[1];
+    dst_dir = argv[2];
+    fanout = atoi(argv[3]);
+    if (!fanout) exit(1);
+
+    DirScanner scanner(src_dir);
+    while (scanner.scan(filename)) {
+        retval = dir_hier_path(filename.c_str(), dst_dir, fanout, dst_path, true);
+        if (retval) {
+            fprintf(stderr, "dir_hier_path: %d\n", retval);
+            exit(1);
+        }
+        sprintf(src_path, "%s/%s", src_dir, filename.c_str());
+        retval = rename(src_path, dst_path);
+        if (retval) {
+            fprintf(stderr, "rename: %d\n", retval);
+            exit(1);
+        }
+    }
 }
diff --git a/tools/dir_hier_path.C b/tools/dir_hier_path.C
index eee78c9ad0..bbfacf85e9 100644
--- a/tools/dir_hier_path.C
+++ b/tools/dir_hier_path.C
@@ -8,7 +8,7 @@ int main(int argc, char** argv) {
     char path[256];
     int retval;
 
-    retval = config.parse_file();
+    retval = config.parse_file("..");
     if (retval) exit(1);
 
     dir_hier_path(argv[1], "", config.uldl_dir_fanout, path);