From e88e3d9937c1d19eca93548b46f91dc5562a81b8 Mon Sep 17 00:00:00 2001 From: Brian Boshes Date: Thu, 5 Aug 2004 21:11:06 +0000 Subject: [PATCH] *** empty log message *** svn path=/trunk/boinc/; revision=3992 --- client/cs_apps.C | 2 +- client/cs_data.C | 521 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 522 insertions(+), 1 deletion(-) create mode 100644 client/cs_data.C diff --git a/client/cs_apps.C b/client/cs_apps.C index ba97e7d93a..6ca208536b 100644 --- a/client/cs_apps.C +++ b/client/cs_apps.C @@ -548,7 +548,7 @@ void CLIENT_STATE::handle_file_xfer_apps() { RESULT* rp = *i; if(rp->wup->avp->app_files.size() == 0 && rp->state == RESULT_FILES_DOWNLOADED) { rp->state = RESULT_FILES_UPLOADING; - rp->reset_result_files(); + rp->reset_files(); } } } diff --git a/client/cs_data.C b/client/cs_data.C new file mode 100644 index 0000000000..c610e44cec --- /dev/null +++ b/client/cs_data.C @@ -0,0 +1,521 @@ +// The contents of this file are subject to the BOINC Public License +// Version 1.0 (the "License"); you may not use this file except in +// compliance with the License. You may obtain a copy of the License at +// http://boinc.berkeley.edu/license_1.0.txt +// +// Software distributed under the License is distributed on an "AS IS" +// basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the +// License for the specific language governing rights and limitations +// under the License. +// +// The Original Code is the Berkeley Open Infrastructure for Network Computing. +// +// The Initial Developer of the Original Code is the SETI@home project. +// Portions created by the SETI@home project are Copyright (C) 2002 +// University of California at Berkeley. All Rights Reserved. +// +// Contributor(s): +// + +// Includes the methods required for managing saved data on the client +// utilized by the client to managed its deletion policy and communicate with +// the user and the server what the status of the data storage is +// + +#include "cpp.h" + +#ifdef _WIN32 +#include "boinc_win.h" +#endif + +#ifndef _WIN32 +#include +#include +#endif + +#include "filesys.h" +#include "client_types.h" +#include "client_state.h" + +using std::vector; + +extern void data_overflow_notify(PROJECT*); + +// Polling function called in do_somthing() +// +// Makes sure the disk bounds have not been drastically violated +// Returns true if something has been violated and tries to correct +// the problem +// TODO: notify the user of corrections made? +// +bool CLIENT_STATE::data_manager_poll() { + double tdu, adu; + + total_disk_usage(tdu); + allowed_disk_usage(adu); + + // first delete files from offenders only + if(tdu > adu) { + if(size_overflow) { + return false; + } + return fix_data_overflow(tdu, adu); + } + + if(size_overflow) { + calc_all_proj_size(); + size_overflow = false; + } + + return false; +} + +bool CLIENT_STATE::fix_data_overflow(double tdu, double adu) { + double space_needed, deleted_space; + unsigned int i; + int priority; + bool deleted; + PROJECT* p; + // First, get accurate sizes as of right now + calc_all_proj_size(); + + // first delete files from offenders only + + space_needed = tdu - adu; + get_more_disk_space(NULL, space_needed); + total_disk_usage(tdu); + + i = 0; + priority = P_LOW; + deleted = false; + + while(tdu > adu) { + // there is an offender that doesn't want to give up its files + // try deleting files from other projects first + if(i >= projects.size()) { + if(!deleted){ + if(priority >= P_HIGH) { + break; + } else { + priority++; + } + } + deleted = false; + i = 0; + } + p = projects[i]; + deleted_space = select_delete(p, 1, priority); + if(deleted_space != 0) deleted = true; + tdu -= deleted_space; + i++; + } + + i = 0; + deleted = false; + + while(tdu > adu) { + // still not enough space, projects give up non-active WUs + if(i >= projects.size()) { + if(!deleted) { + size_overflow = true; + data_overflow_notify(NULL); + return true; + } + i = 0; + } + p = projects[i]; + deleted_space = delete_results(p, 1); + if(deleted_space != 0) deleted = true; + tdu -= deleted_space; + i++; + } + return false; +} + +// calculates the size of all the projects +// use after any deletions to ensure accurate number for size +// +int CLIENT_STATE::calc_all_proj_size() { + for (unsigned int i=0; i < projects.size(); ++i) { + calc_proj_size(projects[i]); + } + return 0; +} + +// recalculates the total size of the project directory +// and any associated active task directories +// +int CLIENT_STATE::calc_proj_size(PROJECT* p) { + p->size = 0; + project_disk_usage(p, p->size); + compute_share_disk_size(p); + return 0; +} + + +// Any space that is unallocated, returns number of bytes +// +int CLIENT_STATE::anything_free(double& size) { + double total_size = 0; + double disk_available; + + for (unsigned int i=0; i < projects.size(); ++i) { + total_size += projects[i]->size; + } + allowed_project_disk_usage(disk_available); + size = disk_available - total_size; + if(size > 0) { + return 0; + } else { + size = 0; + return 1; + } +} + +// Tries to get more disk space for a project. Takes some percentage +// of the disk space that is not utilzed by a project and awards the +// space to the project to be used. Returns true if this was successful, +// false if it was not. It will be uncessful if all the projects are at their +// limits and the currect project is trying to exceed theirs. +// +// This function will try its best to allow a project to grow in size +// It will delete files from all projects that have a larger gap than +// the current project. +// +bool CLIENT_STATE::get_more_disk_space(PROJECT *p, double space_needed) { + PROJECT * other_p = NULL; + double total_space = 0; + double free_space; + double offend_size; + int priority = 0; + + // check to see if there is enough extra space floating around + // such will be the case after calling this after a deletion + if(anything_free(free_space)) { + total_space += free_space; + if (total_space > space_needed) { + return true; + } + } + + // If the function has not exited with true, this means that all available + // space is being used by some combination of projects + // Check if one of the projects is over its resource share and give the space + // to the requesting project + + reset_checks(); + priority = P_LOW; + + while(total_space < space_needed) { + other_p = greatest_offender(); + if(other_p == NULL || (other_p == p)) { + if(priority > P_HIGH) { + return false; + } else { + priority++; + reset_checks(); + continue; + } + } + other_p->checked = true; + + offend_size = offender(other_p); + if(space_needed - total_space > offend_size) { + total_space += select_delete(other_p, offend_size, priority); + } else { + total_space += select_delete(other_p, (space_needed - total_space), priority); + } + } + return true; +} + +// try and delete this many bytes of data from the greatest offender +// return the amount of disk space that was actually freed by the delete +// +double CLIENT_STATE::select_delete(PROJECT* p, double space_to_delete, int priority) { + FILE_INFO* next_file = NULL; + double deleted_space = 0; + double total_space = 0; + + garbage_collect(); + + // if not, then it must start selecting and deleting files + // until enough space is freed up + while(total_space < space_to_delete) { + deleted_space = delete_next_file(p, priority); + if(deleted_space == 0) break; + total_space += deleted_space; + } + // delete all files that you can after reflagging stickys + garbage_collect(); + // calculate the size of the project after the deletion + calc_proj_size(p); + return total_space; +} + +// Tries to delete everything but files associated with the active task +// If this function exits with 0 and there is still not enough space, +// the active task needs to be suspended and not restarted until there is +// enough space to run that project. + +double CLIENT_STATE::delete_results(PROJECT *p, double space_to_delete) { + double deleted_space = 0; + double oldsize = p->size; + + while(deleted_space < space_to_delete) { + if(delete_inactive_results(p)) { + break; + } + calc_proj_size(p); + deleted_space += oldsize - p->size; + } + return deleted_space; +} + +// Returns the next file based on the deletion policy of the project. +// Returns true if a file that can be deleted was found, +// false otherwise +// +double CLIENT_STATE::delete_next_file(PROJECT* p, int priority) { + FILE_INFO* retval = NULL; + double space_freed = 0; + + if(p->deletion_policy_expire) { + space_freed = delete_expired(p); + } + if(space_freed == 0) { + retval = get_priority_or_lru(p, priority); + if(retval != NULL) { + retval->sticky = false; + space_freed = retval->nbytes; + } + } + return space_freed; +} + +// Returns the file_info from the project that has the lowest priority +// +FILE_INFO* CLIENT_STATE::get_priority_or_lru(PROJECT* p, int priority) { + FILE_INFO* fip; + unsigned int i; + double lowest_p = 0; + FILE_INFO* lowest = NULL; + + for(i = 0; i < file_infos.size(); i++) { + fip = file_infos[i]; + + // files that have no wu's or results and are permenant + if(fip->ref_cnt==0 && fip->project == p + && fip->sticky && fip->priority <= priority) { + if(lowest == NULL) { + lowest = fip; + lowest_p = fip->priority; + } else if(fip->priority < lowest_p) { + lowest = file_infos[i]; + lowest_p = lowest->priority; + } else if(fip->priority == lowest_p) { + if(p->deletion_policy_expire && fip->exp_date < lowest->exp_date) { + lowest = file_infos[i]; + } else if(fip->time_last_used < lowest->time_last_used) { + lowest = file_infos[i]; + } + } + } + } + return lowest; +} + +// Deletes all expired file_infos +// Returns the amount of bytes freed +// +double CLIENT_STATE::delete_expired(PROJECT* p) { + FILE_INFO* fip; + double time_now = time(0); + double space_expired = 0; + unsigned int i; + + for(i = 0; i < file_infos.size(); i++) { + fip = file_infos[i]; + // files that have no wu's or results and are permenant + if(fip->ref_cnt==0 && fip->project == p && fip->sticky) { + if(fip->exp_date > time_now) { + fip->sticky = false; + space_expired += fip->nbytes; + } + } + } + return space_expired; +} + +// Delete any files that associated with inactive results +// by marking their results to acknowledged +// +int CLIENT_STATE::delete_inactive_results(PROJECT *p) { + bool deleted = false; + RESULT* result; + unsigned int i; + + for(i = 0; i < results.size(); i++) { + result = results[i]; + if(!result->is_active || result->state < RESULT_COMPUTE_DONE) { + result->got_server_ack = true; + unstick_result_files(result); + deleted = true; + } + } + if(deleted) { + garbage_collect(); + return 0; + } else { + return 1; + } +} + +// should be called after forcebly deleting any result +// ensures any files that were supposed to by permanent are +// deleted as well, as we are already low on disk space +// +int CLIENT_STATE::unstick_result_files(RESULT *rp) { + WORKUNIT* wup; + int retval = 1; + unsigned int i; + for (i=0; ioutput_files.size(); i++) { + retval = 0; + rp->output_files[i].file_info->sticky = false; + } + wup = rp->wup; + for (i=0; iinput_files.size(); i++) { + retval = 0; + wup->input_files[i].file_info->sticky = false; + } + return retval; +} + +// returns the number of bytes the greatest offender is over his usual resource share +// the argument is returned with the number of bytes and the offending +// project is returned +// +PROJECT* CLIENT_STATE::greatest_offender() { + PROJECT* g_offender = NULL; + PROJECT* current_suspect; + double max_offense = 0; + for (unsigned int i=0; i < projects.size(); ++i) { + if(!projects[i]->checked){ + current_suspect = projects[i]; + if(offender(current_suspect) > max_offense) { + g_offender = current_suspect; + max_offense = offender(current_suspect); + } + } + } + return g_offender; +} +// returns the number of bytes the project is offending by +// will be negative if it is not an offender +// +double CLIENT_STATE::offender(PROJECT* p) { + if(p->share_size == 0) { + calc_all_proj_size(); + } + return (p->size - p->share_size); +} + +// Computes the percentage of the actual resource share that +// has been awarded to this project when compared with the totals +// from all other projects +// +double CLIENT_STATE::compute_resource_share(PROJECT *p) { + double total_resource_share = 0; + + for (unsigned int i=0; i < projects.size(); ++i) { + total_resource_share += projects[i]->resource_share; + } + return p->resource_share/total_resource_share; +} + +// Computes the size of the allowed disk share in number of bytes. +// This number may be smaller than the actual disk usage of the project +// since projects are allowed to grow outside of their disk bounds if there +// is space not utilzed by other projects +// +int CLIENT_STATE::compute_share_disk_size(PROJECT *p) { + double disk_available; + allowed_project_disk_usage(disk_available); + p->share_size = disk_available * compute_resource_share(p); + return 0; +} + +// resets the checked flag for all projects in gstate to false; +// +int CLIENT_STATE::reset_checks() { + unsigned int i; + for (i=0; i < projects.size(); ++i) { + projects[i]->checked = false; + } + return 0; +} + +int CLIENT_STATE::total_potentially_offender(PROJECT* p, double& tps) { + PROJECT* other_p; + unsigned int i; + + garbage_collect(); + anything_free(tps); + + for(i = 0; i 0) { + return 0; + } else { + tps = 0; + return 1; + } +} + +int CLIENT_STATE::total_potentially_self(PROJECT* p, double& tps) { + FILE_INFO* fip; + unsigned int i; + total_potentially_offender(p, tps); + for(i = 0; i < file_infos.size(); i++) { + fip = file_infos[i]; + if(fip->ref_cnt == 0 && fip->project == p && fip->sticky) { + if(!p->deletion_policy_expire) { + tps += fip->nbytes; + } else if(p->deletion_policy_expire && (fip->exp_date > time(0))) { + tps += fip->nbytes; + } + } + } + if(tps > 0) { + return 0; + } else { + tps = 0; + return 1; + } +} + +double CLIENT_STATE::proj_potentially_free(PROJECT* p) { + double offend_share = offender(p); + double tps = 0; + FILE_INFO* fip; + unsigned int i; + if(offend_share <= 0) { + return 0; + } + for(i = 0; i < file_infos.size(); i++) { + if(tps > offend_share) break; + fip = file_infos[i]; + if(fip->ref_cnt==0 && fip->project == p && fip->sticky) { + if(!p->deletion_policy_expire) { + tps += fip->nbytes; + } else if(p->deletion_policy_expire && (fip->exp_date > time(0))) { + tps += fip->nbytes; + } + } + } + return tps; +} \ No newline at end of file