Client: measure disk usage in terms of allocated disk space, not file size.

On filesystems that use compression, the disk space allocated to a file
is generally less than the (logical) size of the file.

In this case, the client can fail to get work
because it thinks there's insufficient disk space.

Fix: change things so that the client measures disk usage
(per-project and total) in terms of allocated space rather than file size.

But still use logical file sizes in two places:
- in checking that downloaded files are the right size
- in calculating a job's disk usage to see if it exceeds
the project-specified limit.

New functions: file_size_alloc() and dir_size_alloc().
These are like file_size() and dir_size() except they return
allocated rather than logical size.

client/app.h

@@ -78,6 +78,7 @@ struct ACTIVE_TASK {
     double peak_working_set_size;
     double peak_swap_size;
     double peak_disk_usage;
+        // based on real (not allocated/compressed) file sizes
 
     // START OF ITEMS ALSO SAVED IN CLIENT STATE FILE
 
@@ -204,7 +205,9 @@ struct ACTIVE_TASK {
     void cleanup_task();
 
     int current_disk_usage(double&);
-        // disk used by output files and temp files of this task
+        // total sizes of output files and temp files of this task
+        // This is compared with project-specified limits
+        // to decide whether to abort job; no other use.
     int get_free_slot(RESULT*);
     int start(bool test=false);         // start a process
 

client/cs_prefs.cpp

@@ -90,14 +90,14 @@ int CLIENT_STATE::get_disk_usages() {
     for (i=0; i<projects.size(); i++) {
         p = projects[i];
         p->disk_usage = 0;
-        retval = dir_size(p->project_dir(), size);
+        retval = dir_size_alloc(p->project_dir(), size);
         if (!retval) p->disk_usage = size;
     }
 
     for (i=0; i<active_tasks.active_tasks.size(); i++) {
         ACTIVE_TASK* atp = active_tasks.active_tasks[i];
         get_slot_dir(atp->slot, buf, sizeof(buf));
-        retval = dir_size(buf, size);
+        retval = dir_size_alloc(buf, size);
         if (retval) continue;
         atp->wup->project->disk_usage += size;
     }
@@ -105,7 +105,7 @@ int CLIENT_STATE::get_disk_usages() {
         p = projects[i];
         total_disk_usage += p->disk_usage;
     }
-    retval = dir_size(".", size, false);
+    retval = dir_size_alloc(".", size, false);
     if (!retval) {
         client_disk_usage = size;
         total_disk_usage += size;

client/gui_rpc_server_ops.cpp

@@ -168,8 +168,8 @@ static void handle_get_disk_usage(GUI_RPC_CONN& grc) {
         );
     }
 
-    dir_size(".", boinc_non_project, false);
-    dir_size("locale", size, false);
+    dir_size_alloc(".", boinc_non_project, false);
+    dir_size_alloc("locale", size, false);
     boinc_non_project += size;
 #ifdef __APPLE__
     if (gstate.launched_by_manager) {
@@ -179,9 +179,13 @@ static void handle_get_disk_usage(GUI_RPC_CONN& grc) {
         OSStatus err = noErr;
         
         retval = proc_pidpath(getppid(), path, sizeof(path));
-        if (retval <= 0) err = fnfErr;
-        if (! err) dir_size(path, manager_size, true);
-        if (! err) boinc_non_project += manager_size;
+        if (retval <= 0) {
+            err = fnfErr;
+        }
+        if (!err) {
+            dir_size_alloc(path, manager_size, true);
+            boinc_non_project += manager_size;
+        }
     }
 #endif
     boinc_total = boinc_non_project;

lib/filesys.cpp

@@ -370,6 +370,32 @@ int file_size(const char* path, double& size) {
 #endif
 }
 
+// get file allocation size, i.e. how much disk space does it use.
+// This can be less than the file size on compressed filesystems,
+// or if the file has holes.
+// It can also be slightly more.
+//
+int file_size_alloc(const char* path, double& size) {
+#if defined(_WIN32) && !defined(__CYGWIN32__) && !defined(__MINGW32__)
+    HANDLE h = CreateFileA(path, 0, FILE_SHARE_READ|FILE_SHARE_WRITE|FILE_SHARE_DELETE, 0, OPEN_EXISTING, 0, 0);
+    if (h == INVALID_HANDLE_VALUE) return ERR_STAT;
+    LARGE_INTEGER lisize;
+    if (GetCompressedFileSizeEx(h, &lisize)) {
+        size = (double) lisize.QuadPart;
+        CloseHandle(h);
+        return 0;
+    }
+    return ERR_STAT;
+#else
+    int retval;
+    struct stat sbuf;
+    retval = stat(path, &sbuf);
+    if (retval) return ERR_NOT_FOUND;
+    size = ((double)sbuf.st_blocks)*512.;
+    return 0;
+#endif
+}
+
 int boinc_truncate(const char* path, double size) {
     int retval;
 #if defined(_WIN32) && !defined(__CYGWIN32__)
@@ -487,6 +513,76 @@ int dir_size(const char* dirpath, double& size, bool recurse) {
     return 0;
 }
 
+// return total allocated size of files in directory and optionally its subdirectories
+// Win: use special version because stat() is slow, can be avoided
+// Unix: follow symbolic links
+//
+int dir_size_alloc(const char* dirpath, double& size, bool recurse) {
+#ifdef WIN32
+    char buf[_MAX_PATH];
+    char path2[_MAX_PATH];
+    double dsize = 0.0;
+    WIN32_FIND_DATAA findData;
+
+    size = 0.0;
+    snprintf(path2, sizeof(path2), "%s/*", dirpath);
+    path2[sizeof(path2)-1] = 0;
+
+    HANDLE hFind = ::FindFirstFileA(path2, &findData);
+    if (INVALID_HANDLE_VALUE == hFind) return ERR_OPENDIR;
+    do {
+        snprintf(buf, sizeof(buf), "%.*s/%.*s", DIR_LEN, dirpath, FILE_LEN, findData.cFileName);
+        buf[sizeof(buf)-1] = 0;
+        if (findData.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) {
+            if (!recurse) continue;
+            if (!strcmp(findData.cFileName, ".")) continue;
+            if (!strcmp(findData.cFileName, "..")) continue;
+
+            dsize = 0.0;
+            dir_size_alloc(buf, dsize, true);
+            size += dsize;
+        } else {
+            double s;
+            if (file_size_alloc(buf, s) == 0) {
+                size += s;
+            }
+        }
+    } while (FindNextFileA(hFind, &findData));
+
+    ::FindClose(hFind);
+#else
+    char filename[MAXPATHLEN], subdir[MAXPATHLEN];
+    int retval=0;
+    DIRREF dirp;
+    double x;
+
+    size = 0.0;
+    dirp = dir_open(dirpath);
+    if (!dirp) return ERR_OPENDIR;
+    while (1) {
+        retval = dir_scan(filename, dirp, sizeof(filename));
+        if (retval) break;
+
+        snprintf(subdir, sizeof(subdir), "%.*s/%.*s", DIR_LEN, dirpath, FILE_LEN, filename);
+        subdir[sizeof(subdir)-1] = 0;
+
+        if (is_dir(subdir)) {
+            if (recurse) {
+                retval = dir_size_alloc(subdir, x);
+                if (retval) continue;
+                size += x;
+            }
+        } else if (is_file(subdir)) {
+            retval = file_size_alloc(subdir, x);
+            if (retval) continue;
+            size += x;
+        }
+    }
+    dir_close(dirp);
+#endif
+    return 0;
+}
+
 FILE* boinc_fopen(const char* path, const char* mode) {
     // if opening for read, and file isn't there,
     // leave now (avoid 5-second delay!!)

lib/filesys.h

@@ -95,8 +95,10 @@ extern "C" {
 #ifdef __cplusplus
 
 extern int file_size(const char*, double&);
-extern int clean_out_dir(const char*);
+extern int file_size_alloc(const char*, double&);
 extern int dir_size(const char* dirpath, double&, bool recurse=true);
+extern int dir_size_alloc(const char* dirpath, double&, bool recurse=true);
+extern int clean_out_dir(const char*);
 extern int get_filesystem_info(double& total, double& free, char* path=const_cast<char *>("."));
 extern bool is_path_absolute(const std::string path);