Merge pull request #3707 from BOINC/dpa_disk_alloc

Client: measure disk usage in terms of allocated disk space, not file size

client/app.h

@@ -78,6 +78,7 @@ struct ACTIVE_TASK {
     double peak_working_set_size;
     double peak_swap_size;
     double peak_disk_usage;
+        // based on real (not allocated/compressed) file sizes
 
     // START OF ITEMS ALSO SAVED IN CLIENT STATE FILE
 
@@ -204,7 +205,9 @@ struct ACTIVE_TASK {
     void cleanup_task();
 
     int current_disk_usage(double&);
-        // disk used by output files and temp files of this task
+        // total sizes of output files and temp files of this task
+        // This is compared with project-specified limits
+        // to decide whether to abort job; no other use.
     int get_free_slot(RESULT*);
     int start(bool test=false);         // start a process
 

client/cs_prefs.cpp

@@ -90,14 +90,14 @@ int CLIENT_STATE::get_disk_usages() {
     for (i=0; i<projects.size(); i++) {
         p = projects[i];
         p->disk_usage = 0;
-        retval = dir_size(p->project_dir(), size);
+        retval = dir_size_alloc(p->project_dir(), size);
         if (!retval) p->disk_usage = size;
     }
 
     for (i=0; i<active_tasks.active_tasks.size(); i++) {
         ACTIVE_TASK* atp = active_tasks.active_tasks[i];
         get_slot_dir(atp->slot, buf, sizeof(buf));
-        retval = dir_size(buf, size);
+        retval = dir_size_alloc(buf, size);
         if (retval) continue;
         atp->wup->project->disk_usage += size;
     }
@@ -105,7 +105,7 @@ int CLIENT_STATE::get_disk_usages() {
         p = projects[i];
         total_disk_usage += p->disk_usage;
     }
-    retval = dir_size(".", size, false);
+    retval = dir_size_alloc(".", size, false);
     if (!retval) {
         client_disk_usage = size;
         total_disk_usage += size;
@@ -119,7 +119,7 @@ int CLIENT_STATE::get_disk_usages() {
 // - each project has a "disk_resource_share" (DRS)
 //   This is the resource share plus .1*(max resource share).
 //   This ensures that backup projects get some disk.
-// - each project as a "desired_disk_usage (DDU)", 
+// - each project has a "desired_disk_usage (DDU)", 
 //   which is either its current usage
 //   or an amount sent from the scheduler.
 // - each project has a "quota": (available space)*(drs/total_drs).

client/gui_rpc_server_ops.cpp

@@ -168,8 +168,8 @@ static void handle_get_disk_usage(GUI_RPC_CONN& grc) {
         );
     }
 
-    dir_size(".", boinc_non_project, false);
-    dir_size("locale", size, false);
+    dir_size_alloc(".", boinc_non_project, false);
+    dir_size_alloc("locale", size, false);
     boinc_non_project += size;
 #ifdef __APPLE__
     if (gstate.launched_by_manager) {
@@ -179,9 +179,13 @@ static void handle_get_disk_usage(GUI_RPC_CONN& grc) {
         OSStatus err = noErr;
         
         retval = proc_pidpath(getppid(), path, sizeof(path));
-        if (retval <= 0) err = fnfErr;
-        if (! err) dir_size(path, manager_size, true);
-        if (! err) boinc_non_project += manager_size;
+        if (retval <= 0) {
+            err = fnfErr;
+        }
+        if (!err) {
+            dir_size_alloc(path, manager_size, true);
+            boinc_non_project += manager_size;
+        }
     }
 #endif
     boinc_total = boinc_non_project;

client/project.cpp

@@ -417,6 +417,8 @@ int PROJECT::write_state(MIOFILE& out, bool gui_rpc) {
         "    <rec>%f</rec>\n"
         "    <rec_time>%f</rec_time>\n"
         "    <resource_share>%f</resource_share>\n"
+        "    <disk_usage>%f</disk_usage>\n"
+        "    <disk_share>%f</disk_share>\n"
         "    <desired_disk_usage>%f</desired_disk_usage>\n"
         "    <duration_correction_factor>%f</duration_correction_factor>\n"
         "    <sched_rpc_pending>%d</sched_rpc_pending>\n"
@@ -454,6 +456,7 @@ int PROJECT::write_state(MIOFILE& out, bool gui_rpc) {
         pwf.rec,
         pwf.rec_time,
         resource_share,
+        disk_usage, disk_share,
         desired_disk_usage,
         duration_correction_factor,
         sched_rpc_pending,
@@ -557,10 +560,8 @@ int PROJECT::write_state(MIOFILE& out, bool gui_rpc) {
             "    <cpu_ec>%f</cpu_ec>\n"
             "    <cpu_time>%f</cpu_time>\n"
             "    <gpu_ec>%f</gpu_ec>\n"
-            "    <gpu_time>%f</gpu_time>\n"
-            "    <disk_usage>%f</disk_usage>\n"
-            "    <disk_share>%f</disk_share>\n",
-            cpu_ec, cpu_time, gpu_ec, gpu_time, disk_usage, disk_share
+            "    <gpu_time>%f</gpu_time>\n",
+            cpu_ec, cpu_time, gpu_ec, gpu_time
         );
     }
     out.printf(

lib/filesys.cpp

@@ -370,6 +370,35 @@ int file_size(const char* path, double& size) {
 #endif
 }
 
+// get file allocation size, i.e. how much disk space does it use.
+// This can be less than the file size on compressed filesystems,
+// or if the file has holes.
+// It can also be slightly more.
+//
+int file_size_alloc(const char* path, double& size) {
+#if defined(_WIN32) && !defined(__CYGWIN32__) && !defined(__MINGW32__)
+    DWORD hi;
+    DWORD lo = GetCompressedFileSizeA(path, &hi);
+    if (lo != INVALID_FILE_SIZE) {
+        ULONGLONG x = (((ULONGLONG)hi) << 32) + lo;
+        size = (double) x;
+        return 0;
+    }
+    return ERR_STAT;
+#else
+    int retval;
+    struct stat sbuf;
+    retval = stat(path, &sbuf);
+    if (retval) return ERR_NOT_FOUND;
+#ifdef _WIN32 // cygwin, mingw
+    size = (double)sbuf.st_size;
+#else
+    size = ((double)sbuf.st_blocks)*512.;
+#endif
+    return 0;
+#endif
+}
+
 int boinc_truncate(const char* path, double size) {
     int retval;
 #if defined(_WIN32) && !defined(__CYGWIN32__)
@@ -487,6 +516,76 @@ int dir_size(const char* dirpath, double& size, bool recurse) {
     return 0;
 }
 
+// return total allocated size of files in directory and optionally its subdirectories
+// Win: use special version because stat() is slow, can be avoided
+// Unix: follow symbolic links
+//
+int dir_size_alloc(const char* dirpath, double& size, bool recurse) {
+#ifdef WIN32
+    char buf[_MAX_PATH];
+    char path2[_MAX_PATH];
+    double dsize = 0.0;
+    WIN32_FIND_DATAA findData;
+
+    size = 0.0;
+    snprintf(path2, sizeof(path2), "%s/*", dirpath);
+    path2[sizeof(path2)-1] = 0;
+
+    HANDLE hFind = ::FindFirstFileA(path2, &findData);
+    if (INVALID_HANDLE_VALUE == hFind) return ERR_OPENDIR;
+    do {
+        snprintf(buf, sizeof(buf), "%.*s/%.*s", DIR_LEN, dirpath, FILE_LEN, findData.cFileName);
+        buf[sizeof(buf)-1] = 0;
+        if (findData.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) {
+            if (!recurse) continue;
+            if (!strcmp(findData.cFileName, ".")) continue;
+            if (!strcmp(findData.cFileName, "..")) continue;
+
+            dsize = 0.0;
+            dir_size_alloc(buf, dsize, true);
+            size += dsize;
+        } else {
+            double s;
+            if (file_size_alloc(buf, s) == 0) {
+                size += s;
+            }
+        }
+    } while (FindNextFileA(hFind, &findData));
+
+    ::FindClose(hFind);
+#else
+    char filename[MAXPATHLEN], subdir[MAXPATHLEN];
+    int retval=0;
+    DIRREF dirp;
+    double x;
+
+    size = 0.0;
+    dirp = dir_open(dirpath);
+    if (!dirp) return ERR_OPENDIR;
+    while (1) {
+        retval = dir_scan(filename, dirp, sizeof(filename));
+        if (retval) break;
+
+        snprintf(subdir, sizeof(subdir), "%.*s/%.*s", DIR_LEN, dirpath, FILE_LEN, filename);
+        subdir[sizeof(subdir)-1] = 0;
+
+        if (is_dir(subdir)) {
+            if (recurse) {
+                retval = dir_size_alloc(subdir, x);
+                if (retval) continue;
+                size += x;
+            }
+        } else if (is_file(subdir)) {
+            retval = file_size_alloc(subdir, x);
+            if (retval) continue;
+            size += x;
+        }
+    }
+    dir_close(dirp);
+#endif
+    return 0;
+}
+
 FILE* boinc_fopen(const char* path, const char* mode) {
     // if opening for read, and file isn't there,
     // leave now (avoid 5-second delay!!)

lib/filesys.h

@@ -95,8 +95,10 @@ extern "C" {
 #ifdef __cplusplus
 
 extern int file_size(const char*, double&);
-extern int clean_out_dir(const char*);
+extern int file_size_alloc(const char*, double&);
 extern int dir_size(const char* dirpath, double&, bool recurse=true);
+extern int dir_size_alloc(const char* dirpath, double&, bool recurse=true);
+extern int clean_out_dir(const char*);
 extern int get_filesystem_info(double& total, double& free, char* path=const_cast<char *>("."));
 extern bool is_path_absolute(const std::string path);
 

lib/gui_rpc_client_print.cpp

@@ -99,7 +99,7 @@ void PROJECT::print() {
     printf("   ended: %s\n", ended?"yes":"no");
     printf("   suspended via GUI: %s\n", suspended_via_gui?"yes":"no");
     printf("   don't request more work: %s\n", dont_request_more_work?"yes":"no");
-    printf("   disk usage: %f\n", disk_usage);
+    printf("   disk usage: %.2fMB\n", disk_usage/MEGA);
     time_t foo = (time_t)last_rpc_time;
     printf("   last RPC: %s\n", ctime(&foo));
     printf("   project files downloaded: %f\n", project_files_downloaded_time);
@@ -428,8 +428,8 @@ void PROJECTS::print_urls() {
 void DISK_USAGE::print() {
     unsigned int i;
     printf("======== Disk usage ========\n");
-    printf("total: %f\n", d_total);
-    printf("free: %f\n", d_free);
+    printf("total: %.2fMB\n", d_total/MEGA);
+    printf("free: %.2fMB\n", d_free/MEGA);
     for (i=0; i<projects.size(); i++) {
         printf("%d) -----------\n", i+1);
         projects[i]->print_disk_usage();