From 4749dca86ef8bf394d8d9d45d8d0dbbf8185ac5d Mon Sep 17 00:00:00 2001 From: Karl Chen Date: Fri, 17 Oct 2003 09:04:13 +0000 Subject: [PATCH] *** empty log message *** svn path=/trunk/boinc/; revision=2496 --- checkin_notes | 16 ++ py/Boinc/database.py | 480 ++-------------------------------- py/Boinc/db_base.py | 533 ++++++++++++++++++++++++++++++++++++++ py/Boinc/setup_project.py | 6 +- tools/make_project | 3 +- 5 files changed, 584 insertions(+), 454 deletions(-) create mode 100644 py/Boinc/db_base.py diff --git a/checkin_notes b/checkin_notes index 2eaaa45989..4c36e697f7 100755 --- a/checkin_notes +++ b/checkin_notes @@ -6854,3 +6854,19 @@ Oliver 16 Oct 2003 graphics_api.C gutil.C,h windows_opengl.C + +Karl 2003-10-16 + - updated python database API with new workunit.userid field + + - refactored database.py into db_base.py and merged in changes from + CourseSurvey system + - improved object caching and implemented lazy lookup of relational + objects; database-heavy python programs should now be much faster + + - added 1 to + config.XML in make_project (but not testbase) + + py/Boinc/ + db_base.py (new) + database.py + diff --git a/py/Boinc/database.py b/py/Boinc/database.py index 6f0bfe62d4..d060d5a58f 100644 --- a/py/Boinc/database.py +++ b/py/Boinc/database.py @@ -1,7 +1,5 @@ ## $Id$ -## this module is from quarl's HKN CourseSurvey database.py - ''' Defines database backend library and database table and object relationships. @@ -30,404 +28,13 @@ for user in database.Users.find(): ''' -from __future__ import generators import boinc_path_config from Boinc import configxml from Boinc.util import * -import sys, os, weakref -import MySQLdb, MySQLdb.cursors +from Boinc.db_base import * ID = '$Id$' -boincdb = None - -class DatabaseInconsistency(Exception): - def __init__(self, descript=None, search_table=None, search_kwargs=None): - self.descript = descript - self.search_table = search_table - self.search_kwargs = search_kwargs - self.search_tree = [] - def __str__(self): - return ("""** DATABASE INCONSISTENCY ** - %s - search_table = %s - search_kwargs = %s - search_tree = [ -%s - ] """ %( - self.descript, - self.search_table, - self.search_kwargs, - '\n'.join( - map(lambda o:" %s#%s %s"%(o._table.table,o.__dict__.get('id'),o), self.search_tree)) - )) - -class Debug: - def __init__(self): - self.html = False - def printline(self,s): - if self.html: - print ""%s - else: - print >>sys.stderr, "##", s - -debug = Debug() -debug.mysql = not not os.environ.get('BOINC_DEBUG_DB') - -def _commit_object(tablename, paramdict, id=None): - """Takes a tablename, boincdb object, a parameter dict, and an - optional id. Puts together the appropriate SQL command to commit - the object to the database. Executes it. Returns the object's - id.""" - assert(boincdb) - cursor = boincdb.cursor() - equalcommands = [] - for key in paramdict.keys(): - value = paramdict[key] - if value == None: - minicommand = '' - continue - else: - minicommand = "%s='%s'"%(key,boincdb.escape_string(str(value))) - equalcommands.append(minicommand) - if id == None: - command = 'INSERT INTO %s SET %s' % \ - (tablename, ', '.join(equalcommands)) - #print command, '--', paramdict - if debug.mysql: - debug.printline("query: "+command) - cursor.execute(command) - id = cursor.insert_id() #porters note: works w/MySQLdb only - else: - command = "UPDATE %s SET %s WHERE id=%d" % \ - (tablename, ', '.join(equalcommands), id) - if debug.mysql: - debug.printline("query: "+command) - cursor.execute(command) - cursor.close() - boincdb.commit() - return id -def _remove_object(command, id=None): - """Takes a command string, boincdb object, and optional id. If an - id is given, it assembles the SQL command and deletes the object - from the database. Does nothing if no id is given.""" - assert(boincdb) - if id == None: - pass - else: - cursor = boincdb.cursor() - command = 'DELETE FROM ' + command + \ - ' WHERE id=%d' % id - if debug.mysql: - debug.printline("query: "+command) - cursor.execute(command) - cursor.close() - boincdb.commit() -def _select_object(table, searchdict, extra_args="", extra_params=[], select_what=None): - assert(boincdb) - parameters = extra_params[:] - join = None - if '_join' in searchdict: - join = searchdict['_join'] - del searchdict['_join'] - if '_extra_params' in searchdict: - parameters += searchdict['_extra_params'] - del searchdict['_extra_params'] - command = 'SELECT %s from %s'%((select_what or "%s.*"%table) ,table) - if join: - command += "," + join - for (key,value) in searchdict.items(): - # note: if value == 0, we want to look for it. - if value != None and value != '': - escaped_value = boincdb.escape_string(str(value)) - if key == 'text': - parameters.append("instr(%s,'%s')"%(key,escaped_value)) - else: - parameters.append("%s='%s'"%(key,escaped_value)) - if parameters: - command += ' WHERE ' + ' AND '.join(parameters) - if extra_args: - command += ' ' + extra_args.strip() - cursor = boincdb.cursor() - if debug.mysql: - debug.printline("query: "+command) - cursor.execute(command) - return cursor - -def _select_object_fetchall(*args, **kwargs): - cursor = apply(_select_object, args, kwargs) - results = cursor.fetchall() - cursor.close() - return results - -def _select_object_iterate(*args, **kwargs): - cursor = apply(_select_object, args, kwargs) - while True: - result = cursor.fetchone() - if not result: return - yield result - -def _select_count_objects(*args, **kwargs): - kwargs['select_what'] = 'count(*)' - cursor = apply(_select_object, args, kwargs) - result = cursor.fetchone().values()[0] - cursor.close() - return result - -OBJECT_CACHE_SIZE = 1024 - -class DatabaseTable: - def __init__(self, table, columns, extra_columns=[], - select_args = None, sort_results = False): - self.table = table - self.lcolumns = columns - self.columns = list2dict(columns) - self.extra_columns = list2dict(extra_columns) - # self.object_class = object_class - self.select_args = select_args - self.sort_results = sort_results - ## self.objects is a mapping from id->object which weakly references - ## all current objects from this table. this guarantees that if a - ## find() returns a row for which we already created an object in - ## memory, we return the same one. - self.objects = weakref.WeakValueDictionary() - ## self.object_cache is a list of the N most recently retrieved - ## objects. its values aren't really used; the list is used to ensure - ## the strong reference count for self.objects[object] is nonzero to - ## ensure is lifetime. - ## - ## This means if you look up database.Apps[1]: the first lookup does a - ## MySQL SELECT; afterwards database.Apps[1] is free (only requires a - ## lookup in database.Apps.objects). To prevent this object cache - ## from growing without bound, database.Apps.object is a weak-valued - ## dictionary. This means that if no one refers to the object, it is - ## deleted from the dictionary. database.Apps.object_cache maintains - ## a list of the OBJECT_CACHE_SIZE most recent lookups, which forces - ## their strong reference count. - self.object_cache = [] - self.defdict = {} - for key in self.lcolumns: - if key == 'id': - self.defdict[key] = None - elif key.endswith('id'): - self.defdict[key[:-2]] = None - elif key.endswith('ids'): - self.defdict[key[:-3]] = None - else: - self.defdict[key] = None - - def _cache(self, object): - """Maintain up to OBJECT_CACHE_SIZE objects in the object_cache list. - - The object's existence in this cache ensures its strong reference - count is nonzero, so that it doesn't get implicitly dropped from - self.objects.""" - if len(self.object_cache) >= OBJECT_CACHE_SIZE: - self.object_cache = self.object_cache[OBJECT_CACHE_SIZE/2:] - self.object_cache.append(object) - - def count(self, **kwargs): - """Return the number of database objects matching keywords. - - Arguments are the same format as find().""" - if kwargs.keys() == ['id']: - # looking up by ID only, look in cache first: - id = kwargs['id'] - if not id: - return 0 - if id in self.objects: - return 1 - kwargs = self.dict2database_fields(kwargs) - return _select_count_objects(self.table, kwargs, - extra_args=self.select_args) - - def find(self, **kwargs): - """Return a list of database objects matching keywords. - - Allowed keywords are specified by self.columns. - - Objects are cached by ID so repeated lookups are quick. - """ - if kwargs.keys() == ['id']: - # looking up by ID only, look in cache first: - id = kwargs['id'] - if not id: - return [None] - try: - return [self.objects[id]] - except KeyError: - pass - limbo_object = self.object_class(id=None) # prevent possible id recursion - limbo_object.in_limbo = 1 - self.objects[id] = limbo_object - self._cache(limbo_object) - kwargs = self.dict2database_fields(kwargs) - results = _select_object_fetchall(self.table, kwargs, - extra_args=self.select_args) - objects = map(self._create_object_from_sql_result, results) - if self.sort_results: - objects.sort() - return objects - def iterate(self, **kwargs): - """Same as find(), but using generators. - - No sorting.""" - if kwargs.keys() == ['id']: - # looking up by ID only, look in cache first: - id = kwargs['id'] - if not id: - return - try: - yield self.objects[id] - return - except KeyError: - pass - limbo_object = self.object_class(id=None) # prevent possible id recursion - limbo_object.in_limbo = 1 - self.objects[id] = limbo_object - self._cache(limbo_object) - kwargs = self.dict2database_fields(kwargs) - for result in _select_object_iterate(self.table, kwargs, - extra_args=self.select_args): - yield self._create_object_from_sql_result(result) - return - - def _create_object_from_sql_result(self, result): - id = result['id'] - try: - # object already exists in cache? - object = self.objects[id] - if 'in_limbo' in object.__dict__: - # earlier we set the object cache so that we don't recurse; - # update it now with real values and delete the 'in limbo' - # flag - del object.__dict__['in_limbo'] - object.do_init(result) - except KeyError: - # create the object - looking up instructors, etc - object = apply(self.object_class, [], result) - if object.id: - self.objects[object.id] = object - self._cache(object) - return object - def find1(self, **kwargs): - '''Return a single result. Raises a DatabaseInconsistency if not - exactly 1 result returned.''' - objects = apply(self.find, [], kwargs) - if len(objects) != 1: - raise DatabaseInconsistency( - descript="find1: expected 1 result but found %d"%len(objects), - search_table = self.table, - search_kwargs = kwargs) - return objects[0] - def __getitem__(self, id): - '''Lookup (possibly cached) object by id. Returns None if id==None.''' - return id and self.find1(id=id) - def objdict2database_fields(self, indict): - dict = {} - for key in self.columns: - if key.endswith('id'): - obj = indict[key[:-2]] - dict[key] = obj and obj.id or 0 - else: - dict[key] = indict[key] - return dict - def _valid_query_keys(self): - return self.columns.keys()+self.extra_columns.keys()+['_join','_extra_params'] - def dict2database_fields(self, indict): - indict = indict.copy() - dict = {} - if 'id' in indict: - dict['id'] = indict['id'] - del indict['id'] - for key in self._valid_query_keys(): - if key.endswith('id'): - xkey = key[:-2] - if xkey in indict: - obj = indict[xkey] - dict[key] = obj and obj.id - del indict[xkey] - else: - if key in indict: - dict[key] = indict[key] - del indict[key] - if len(indict): - raise ValueError('Invalid key(s): %s'%indict) - return dict - -class DatabaseObject: - id_lookups = {} # set near end of file - def database_fields_to_self(self, dict): - columns = self._table.columns - self.__dict__.update(self._table.defdict) # set defaults to None - # set this first so that if we get a DatabaseInconsistency we can see - # the id - self.id = dict.get('id') - for (key, value) in dict.items(): - if key == 'id': - # self.id = value - continue - if key or key+'id' in columns: - if key.endswith('id'): - xkey = key[:-2] - self.__dict__[xkey] = self.id_lookups[xkey]._table[value] - else: - self.__dict__[key] = value - else: - # print '### columns=%s'%columns - raise ValueError("database '%s' object doesn't take argument '%s'"%( - self._table.table, key)) - - def do_init(self, kwargs): - try: - self.database_fields_to_self(kwargs) - except DatabaseInconsistency, e: - e.search_tree.append(self) - raise - # if no id then object starts dirty - self._set_dirty(not self.id) - - def __init__(self, **kwargs): - self.do_init(kwargs) - - def __eq__(self, other): - return other!=None and self.id == other.id - def __ne__(self, other): - return not (self == other) - def __hash__(self): - return self.id or 0 - - def _commit_params(self, paramdict): - """Commits the object to the boincdb database.""" - self.id = _commit_object(self._table.table, paramdict, self.id) - - def commit(self, force=False): - if force or self._dirty: - self._commit_params(self._table.objdict2database_fields(self.__dict__)) - self._set_dirty(False) - - def remove(self): - """Removes the object from the boincdb database.""" - _remove_object(self._table.table, self.id) - self.id = None - - def dset(self, key, value): - if self.__dict__[key] != value: - self.__dict__[key] = value - self._set_dirty() - def __setattr__(self, key, value): - if key in self._table.columns or key+'id' in self._table.columns: - self.dset(key, value) - else: - self.__dict__[key] = value - def _set_dirty(self, value=True): - self.__dict__['_dirty'] = value - - # TODO: move this to db_med - def URL(self): - """Relative form of absURL()""" - return make_url_relative(self.absURL()) - class Project(DatabaseObject): _table = DatabaseTable( table = 'project', @@ -621,28 +228,18 @@ class Workseq(DatabaseObject): 'wuid_last_sent', 'workseqid_master' ]) -def _connectp(dbname, user, passwd, host='localhost'): - """Takes a database name, a username, and password. Connects to - SQL server and makes a new Boincdb.""" - global boincdb - if boincdb: - raise 'Already connected' - boincdb = MySQLdb.connect(db=dbname,host=host,user=user,passwd=passwd, - cursorclass=MySQLdb.cursors.DictCursor) - def connect(config = None, nodb = False): """Connect if not already connected, using config values.""" - global boincdb - if boincdb: + if get_dbconnection(): return 0 config = config or configxml.default_config().config if nodb: db = '' else: db = config.db_name - _connectp(db, - config.__dict__.get('db_user',''), - config.__dict__.get('db_passwd', '')) + do_connect(db=db, + user=config.__dict__.get('db_user',''), + passwd=config.__dict__.get('db_passwd', '')) return 1 def _execute_sql_script(cursor, filename): @@ -653,10 +250,9 @@ def _execute_sql_script(cursor, filename): def create_database(config = None, drop_first = False): ''' creates a new database. ''' - global boincdb config = config or configxml.default_config().config connect(config, nodb=True) - cursor = boincdb.cursor() + cursor = get_dbconnection().cursor() if drop_first: cursor.execute("drop database if exists %s"%config.db_name) cursor.execute("create database %s"%config.db_name) @@ -669,48 +265,28 @@ def create_database(config = None, drop_first = False): # alias connect_default_config = connect -def close(): - """Closes the connection to the sql boinc and deletes the Boincdb object.""" - boincdb.close() +database_classes_ = [ Project, + Platform, + CoreVersion, + App, + AppVersion, + User, + Team, + Host, + Workunit, + Result, + Workseq ] -database_classes = [ Project, - Platform, - CoreVersion, - App, - AppVersion, - User, - Team, - Host, - Workunit, - Result, - Workseq ] - -for Class in database_classes: - # these couldn't be defined earlier because the classes weren't defined yet. - Class._table.object_class = Class - DatabaseObject.id_lookups[Class._table.table] = Class - -DatabaseObject.id_lookups['canonical_result'] = Result - -Projects = Project._table -Platforms = Platform._table +Projects = Project._table +Platforms = Platform._table CoreVersions = CoreVersion._table -Apps = App._table -AppVersions = AppVersion._table -Users = User._table -Teams = Team._table -Hosts = Host._table -Workunits = Workunit._table -Results = Result._table -Workseqs = Workseq._table +Apps = App._table +AppVersions = AppVersion._table +Users = User._table +Teams = Team._table +Hosts = Host._table +Workunits = Workunit._table +Results = Result._table +Workseqs = Workseq._table -database_tables = map(lambda c: c._table, database_classes) - -# def check_database_consistency(): -# '''Raises DatabaseInconsistency on error. - -# Loads the entire database into memory so will take a while. -# ''' -# for table in database_tables: -# print 'Checking', table.table -# print ' checked', len(table.find()) +init_table_classes(database_classes_,{'canonical_result': Result}) diff --git a/py/Boinc/db_base.py b/py/Boinc/db_base.py new file mode 100644 index 0000000000..2515b2c7ae --- /dev/null +++ b/py/Boinc/db_base.py @@ -0,0 +1,533 @@ +## $Id$ + +# quarl 2003-10-16 initial version based on conglomeration of +# coursesurvey/database.py and boinc/database.py + +# quarl 2003-10-16 implemented lazy lookups + +# DB_BASE - an awesome view of an SQL database in Python. All relational +# objects are lazily cached. I.e. if table WORKUNIT has field RESULTID, wu = +# database.Workunits.find1(name='Wu1') will look up Wu1; accessing wu.result +# will do a database.Results.find1(id=wu.resultid) the first time + +from __future__ import generators + +import MySQLdb, MySQLdb.cursors +import sys, os, weakref + +ID = '$Id$' + +dbconnection = None + +def list2dict(list): + dict = {} + for k in list: dict[k] = None + return dict + +class DatabaseInconsistency(Exception): + def __init__(self, descript=None, search_table=None, search_kwargs=None): + self.descript = descript + self.search_table = search_table + self.search_kwargs = search_kwargs + self.search_tree = [] + def __str__(self): + return ("""** DATABASE INCONSISTENCY ** + %s + search_table = %s + search_kwargs = %s + search_tree = [ +%s + ] """ %( + self.descript, + self.search_table, + self.search_kwargs, + '\n'.join( + map(lambda o:" %s#%s %s"%(o._table.table,o.__dict__.get('id'),o), self.search_tree)) + )) + +class Debug: + def __init__(self): + self.html = False + def printline(self,s): + if self.html: + print ""%s + else: + print >>sys.stderr, "##", s + +debug = Debug() +debug.mysql = not not os.environ.get('DEBUG_DB') + +def _commit_object(tablename, paramdict, id=None): + """Takes a tablename, a parameter dict, and an optional id. Puts together + the appropriate SQL command to commit the object to the database. + Executes it. Returns the object's id.""" + assert(dbconnection) + cursor = dbconnection.cursor() + equalcommands = [] + for key in paramdict.keys(): + value = paramdict[key] + if value == None: + continue + elif isinstance(value, int): + equalcommands.append('%s=%d' %(key,value)) + else: + equalcommands.append("%s='%s'"%(key,dbconnection.escape_string(str(value)))) + if id == None: + command = 'INSERT INTO %s SET %s' % \ + (tablename, ', '.join(equalcommands)) + if debug.mysql: + debug.printline("query: "+command) + cursor.execute(command) + id = cursor.insert_id() #porters note: works w/MySQLdb only + else: + command = "UPDATE %s SET %s WHERE id=%d" % \ + (tablename, ', '.join(equalcommands), id) + if debug.mysql: + debug.printline("query: "+command) + cursor.execute(command) + cursor.close() + dbconnection.commit() + return id +def _remove_object(command, id=None): + """Takes a command string, dbconnection object, and optional id. If an + id is given, it assembles the SQL command and deletes the object + from the database. Does nothing if no id is given.""" + assert(dbconnection) + if id == None: + pass + else: + cursor = dbconnection.cursor() + command = 'DELETE FROM ' + command + \ + ' WHERE id=%d' % id + if debug.mysql: + debug.printline("query: "+command) + cursor.execute(command) + cursor.close() + dbconnection.commit() +def _select_object(table, searchdict, extra_args="", extra_params=[], select_what=None): + assert(dbconnection) + parameters = extra_params[:] + join = None + if '_join' in searchdict: + join = searchdict['_join'] + del searchdict['_join'] + if '_extra_params' in searchdict: + parameters += searchdict['_extra_params'] + del searchdict['_extra_params'] + command = 'SELECT %s from %s'%((select_what or "%s.*"%table) ,table) + if join: + command += "," + join + for (key,value) in searchdict.items(): + # note: if value == 0, we want to look for it. + if value != None and value != '': + escaped_value = dbconnection.escape_string(str(value)) + if key == 'text': + parameters.append("instr(%s,'%s')"%(key,escaped_value)) + else: + parameters.append("%s='%s'"%(key,escaped_value)) + if parameters: + command += ' WHERE ' + ' AND '.join(parameters) + if extra_args: + command += ' ' + extra_args.strip() + cursor = dbconnection.cursor() + if debug.mysql: + debug.printline("query: "+command) + cursor.execute(command) + return cursor + +def _select_object_fetchall(*args, **kwargs): + cursor = apply(_select_object, args, kwargs) + results = cursor.fetchall() + cursor.close() + return results + +def _select_object_iterate(*args, **kwargs): + cursor = apply(_select_object, args, kwargs) + while True: + result = cursor.fetchone() + if not result: return + yield result + +def _select_count_objects(*args, **kwargs): + kwargs['select_what'] = 'count(*)' + cursor = apply(_select_object, args, kwargs) + result = cursor.fetchone().values()[0] + cursor.close() + return result + +class Options: + pass + +options = Options() + +# keep up to this many objects in cache. we use a very bone-headed +# cache-management algorithm: when we reach this many objects, drop the oldest +# half. +options.OBJECT_CACHE_SIZE = 1024 + +# don't lookup referenced Ids until they are asked for. I.e., if +# evaluatedclass.instructorteamid=123, then if instructorteam#123 hasn't been +# looked up yet, don't look up evaluatedclass.instructorteam until it is +# referenced. use DEBUG_DB=1 to check out this niftiness. +options.LAZY_LOOKUPS = True + +class DatabaseTable: + def __init__(self, table, columns, extra_columns=[], + select_args = None, sort_results = False): + self.table = table + self.lcolumns = columns + self.columns = list2dict(columns) + self.extra_columns = list2dict(extra_columns) + # self.object_class = object_class + self.select_args = select_args + self.sort_results = sort_results + ## self.objects is a mapping from id->object which weakly references + ## all current objects from this table. this guarantees that if a + ## find() returns a row for which we already created an object in + ## memory, we return the same one. + self.objects = weakref.WeakValueDictionary() + ## self.object_cache is a list of the N most recently retrieved + ## objects. its values aren't really used; the list is used to ensure + ## the strong reference count for self.objects[object] is nonzero to + ## ensure is lifetime. + ## + ## This means if you look up database.Apps[1]: the first lookup does a + ## MySQL SELECT; afterwards database.Apps[1] is free (only requires a + ## lookup in database.Apps.objects). To prevent this object cache + ## from growing without bound, database.Apps.object is a weak-valued + ## dictionary. This means that if no one refers to the object, it is + ## deleted from the dictionary. database.Apps.object_cache maintains + ## a list of the OBJECT_CACHE_SIZE most recent lookups, which forces + ## their strong reference count. + self.object_cache = [] + self.defdict = {} + for key in self.lcolumns: + if key == 'id': + self.defdict[key] = None + elif key.endswith('id'): + self.defdict[key[:-2]] = None + elif key.endswith('ids'): + self.defdict[key[:-3]] = None + else: + self.defdict[key] = None + + def _cache(self, object): + """Maintain up to OBJECT_CACHE_SIZE objects in the object_cache list. + + The object's existence in this cache ensures its strong reference + count is nonzero, so that it doesn't get implicitly dropped from + self.objects.""" + if len(self.object_cache) >= options.OBJECT_CACHE_SIZE: + self.object_cache = self.object_cache[:-options.OBJECT_CACHE_SIZE/2] + self.object_cache.append(object) + + def _is_cached(self, id): + '''Returns True if object is automatically-cached, i.e. in the weak + reference cache. + + This is not the same as the manual cache invoked by _cache(), i.e. the + strong reference cache. + ''' + return id in self.objects + + def _modify_find_args(self, kwargs): + '''Derived classes can override this function to modify kwargs. + + This is only called for non-trivial find args (if there are arguments + and not just "id")''' + pass + + def count(self, **kwargs): + """Return the number of database objects matching keywords. + + Arguments are the same format as find().""" + if not kwargs: + # shortcut since this is the most common case + return _select_count_objects(self.table, {}) + if kwargs.keys() == ['id']: + # looking up by ID only, look in cache first: + id = kwargs['id'] + if not id: + return 0 + if id in self.objects: + return 1 + self._modify_find_args(kwargs) + kwargs = self.dict2database_fields(kwargs) + return _select_count_objects(self.table, kwargs, + extra_args=self.select_args) + def find(self, **kwargs): + """Return a list of database objects matching keywords. + + Allowed keywords are specified by self.columns. + + Objects are cached by ID so repeated lookups are quick. + """ + if kwargs.keys() == ['id']: + # looking up by ID only, look in cache first: + id = kwargs['id'] + if not id: + return [None] + try: + return [self.objects[id]] + except KeyError: + pass + limbo_object = self.object_class(id=None) # prevent possible id recursion + limbo_object.in_limbo = 1 + self.objects[id] = limbo_object + self._cache(limbo_object) + self._modify_find_args(kwargs) + kwargs = self.dict2database_fields(kwargs) + results = _select_object_fetchall(self.table, kwargs, + extra_args=self.select_args) + objects = self._create_objects_from_sql_results(results, kwargs) + # up to this point objects should be equivalent to list(iterate(...)) + if self.sort_results: + objects.sort() + return objects + def iterate(self, **kwargs): + """Same as find(), but using generators, and no sorting.""" + if kwargs.keys() == ['id']: + # looking up by ID only, look in cache first: + id = kwargs['id'] + if not id: + return + try: + yield self.objects[id] + return + except KeyError: + pass + limbo_object = self.object_class(id=None) # prevent possible id recursion + limbo_object.in_limbo = 1 + self.objects[id] = limbo_object + self._cache(limbo_object) + self._modify_find_args(kwargs) + kwargs = self.dict2database_fields(kwargs) + for result in _select_object_iterate(self.table, kwargs, + extra_args=self.select_args): + yield self._create_object_from_sql_result(result) + return + + def _create_objects_from_sql_results(self, results, kwargs): + return map(self._create_object_from_sql_result, results) + + def _create_object_from_sql_result(self, result): + id = result['id'] + try: + # object already exists in cache? + object = self.objects[id] + if 'in_limbo' in object.__dict__: + # earlier we set the object cache so that we don't recurse; + # update it now with real values and delete the 'in limbo' + # flag + del object.__dict__['in_limbo'] + object.do_init(result) + except KeyError: + # create the object - looking up instructors, etc + object = apply(self.object_class, [], result) + if object.id: + self.objects[object.id] = object + self._cache(object) + return object + + def find1(self, **kwargs): + '''Return a single result. Raises a DatabaseInconsistency if not + exactly 1 result returned.''' + objects = apply(self.find, [], kwargs) + if len(objects) != 1: + raise DatabaseInconsistency( + descript="find1: expected 1 result but found %d"%len(objects), + search_table = self.table, + search_kwargs = kwargs) + return objects[0] + def __getitem__(self, id): + '''Lookup (possibly cached) object by id. Returns None if id==None.''' + return id and self.find1(id=id) + def objdict2database_fields(self, indict, lazydict): + dict = {} + for key in self.columns: + if key.endswith('id'): + xkey = key[:-2] + if xkey in lazydict: + # lazydict maps 'name' (without 'id') -> (table,id) + dict[key] = lazydict[xkey][1] + else: + obj = indict[xkey] + dict[key] = obj and obj.id or 0 + else: + dict[key] = indict[key] + return dict + def _valid_query_keys(self): + return self.columns.keys()+self.extra_columns.keys()+['_join','_extra_params'] + def dict2database_fields(self, indict): + indict = indict.copy() + dict = {} + if 'id' in indict: + dict['id'] = indict['id'] + del indict['id'] + for key in self._valid_query_keys(): + if key.endswith('id'): + xkey = key[:-2] + if xkey in indict: + obj = indict[xkey] + dict[key] = obj and obj.id + del indict[xkey] + else: + if key in indict: + dict[key] = indict[key] + del indict[key] + if len(indict): + raise ValueError('Invalid key(s): %s'%indict) + return dict + +class DatabaseObject: + id_lookups = {} # set by init_table_classes + def _set_field(self, key, value): + """Set field KEY to VALUE. May be overridden by derived class. + + if options.LAZY_LOOKUPS is true, then possibly don't look up a value + yet. + """ + if key.endswith('id'): + xkey = key[:-2] + table = self.id_lookups[xkey]._table + id = value + if options.LAZY_LOOKUPS: + if table._is_cached(id): + self.__dict__[xkey] = table.objects[id] + else: + del self.__dict__[xkey] + self._lazy_lookups[xkey] = (table, id) + else: + # always lookup values + self.__dict__[xkey] = table[id] + else: + self.__dict__[key] = value + + def __getattr__(self, name): + if options.LAZY_LOOKUPS and name in self._lazy_lookups: + (table, id) = self._lazy_lookups[name] + del self._lazy_lookups[name] + object = table[id] # this probably invokes MySQL SELECTs + self.__dict__[name] = object + return object + raise AttributeError(name) + + def database_fields_to_self(self, dict): + columns = self._table.columns + self.__dict__.update(self._table.defdict) # set defaults to None + # set this first so that if we get a DatabaseInconsistency we can see + # the id + self.id = dict.get('id') + for (key, value) in dict.items(): + if key == 'id': + continue + if key or key+'id' in columns: + self._set_field(key, value) + else: + raise ValueError("database '%s' object doesn't take argument '%s'"%( + self._table.table, key)) + + def do_init(self, kwargs): + try: + self.database_fields_to_self(kwargs) + except DatabaseInconsistency, e: + e.search_tree.append(self) + raise + # if no id then object starts dirty + self._set_dirty(not self.id) + + def __init__(self, **kwargs): + self._lazy_lookups = {} + self.do_init(kwargs) + + def __eq__(self, other): + return other!=None and self.id == other.id + def __ne__(self, other): + return not (self == other) + def __hash__(self): + return self.id or 0 + + def _commit_params(self, paramdict): + """Commits the object to the dbconnection database.""" + self.id = _commit_object(self._table.table, paramdict, self.id) + + def commit(self, force=False): + if force or self._dirty: + self._commit_params(self._table.objdict2database_fields(self.__dict__, self._lazy_lookups)) + self._set_dirty(False) + + def remove(self): + """Removes the object from the dbconnection database.""" + _remove_object(self._table.table, self.id) + self.id = None + + def dset(self, key, value): + if self.__dict__[key] != value: + self.__dict__[key] = value + self._set_dirty() + def __setattr__(self, key, value): + if key in self._table.columns or key+'id' in self._table.columns: + self.dset(key, value) + else: + self.__dict__[key] = value + def _set_dirty(self, value=True): + self.__dict__['_dirty'] = value + +def do_connect(db, user, passwd, host='localhost'): + """Takes a database name, a username, and password. Connects to + SQL server and makes a new Dbconnection.""" + global dbconnection + if dbconnection: + raise 'Already connected' + dbconnection = MySQLdb.connect(db=db,host=host,user=user,passwd=passwd, + cursorclass=MySQLdb.cursors.DictCursor) +def close(): + """Closes the connection to the sql survey and deletes the Dbconnection object.""" + global dbconnection + dbconnection.close() + dbconnection = None + +def get_dbconnection(): + return dbconnection +def set_dbconnection(d): + dbconnection = d + +def init_table_classes(database_classes_, more_id_lookups = {}): + """initialize the list of database classes and tables. To be called from + database.py. + """ + + global database_classes, database_tables + database_classes = database_classes_ + for Class in database_classes: + Class._table.object_class = Class + DatabaseObject.id_lookups[Class._table.table] = Class + + DatabaseObject.id_lookups.update(more_id_lookups) + + database_tables = map(lambda c: c._table, database_classes) + +def check_database_consistency(): + '''Raises DatabaseInconsistency on error. + + Loads the entire database into memory so will take a while. + ''' + for table in database_tables: + print '\rChecking %s: [counting]' %(table.table), + sys.stdout.flush() + count = table.count() + i = 0 + j_limit = int(count / 100) # show progress every 1% + j = j_limit + print '\rChecking %s: [iterating]' %(table.table), + sys.stdout.flush() + for object in table.iterate(): + # we don't need to do anything here; just iterating through the + # database will automatically read everything into memory + i += 1 + if j == j_limit: + print '\rChecking %s: [%d/%d] %3.f%%' %(table.table, i, count, 100.0*i/count), + sys.stdout.flush() + j = 0 + j += 1 + print '\rChecking %s: all %d rows are good' %(table.table, count) diff --git a/py/Boinc/setup_project.py b/py/Boinc/setup_project.py index 20d51eee2d..3acf340b41 100644 --- a/py/Boinc/setup_project.py +++ b/py/Boinc/setup_project.py @@ -287,7 +287,9 @@ class Project: short_name, long_name, project_dir=None,key_dir=None, master_url=None, cgi_url=None, - db_name=None): + db_name=None, + production=False + ): init() self.short_name = short_name @@ -312,6 +314,8 @@ class Project: config.upload_dir = os.path.join(self.project_dir , 'upload') config.key_dir = key_dir or os.path.join(self.project_dir , 'keys') config.app_dir = os.path.join(self.project_dir, 'apps') + if production: + config.one_result_per_user_per_wu = '1' self.scheduler_url = os.path.join(config.cgi_url , 'cgi') self.project_php_file = srcdir('html_user/project.inc.sample') self.project_specific_prefs_php_file = srcdir('html_user/project_specific_prefs.inc.sample') diff --git a/tools/make_project b/tools/make_project index 7eb16e0402..3d84671cda 100755 --- a/tools/make_project +++ b/tools/make_project @@ -216,7 +216,8 @@ project = Project(project_shortname, project_longname, master_url = options.html_user_url, cgi_url = options.cgi_url, key_dir = options.key_dir, - db_name = options.db_name + db_name = options.db_name, + production = 1 ) project.install_project()