diff --git a/checkin_notes b/checkin_notes index f1289de91a..14017e7ca7 100755 --- a/checkin_notes +++ b/checkin_notes @@ -4794,6 +4794,7 @@ Karl 2003/06/17 test_uc.py test_concat.py test_1sec.py + test_backend.py version.py.in Tim June 18, 2003 diff --git a/test/boinc.py b/test/boinc.py index 5faada1f26..38d5cb4b2a 100644 --- a/test/boinc.py +++ b/test/boinc.py @@ -185,12 +185,14 @@ def generate_shmem_key(): class STARTS_WITH(str): pass -def dict_match(dict, resultdict): - '''match values in DICT against RESULTDICT''' - for key in dict.keys(): - expected = dict[key] +def dict_match(dic, resultdic): + '''match values in DIC against RESULTDIC''' + if not isinstance(dic, dict): + dic = dic.__dict__ + for key in dic.keys(): + expected = dic[key] try: - found = resultdict[key] + found = resultdic[key] except KeyError: error("Database query result didn't have key '%s'!" % key) continue @@ -199,7 +201,7 @@ def dict_match(dict, resultdict): else: match = found == expected if not match: - id = resultdict.get('id', '?') + id = resultdic.get('id', '?') if str(found).count('\n') or str(expected).count('\n'): format = """result %s: unexpected %s: @@ -212,6 +214,11 @@ def dict_match(dict, resultdict): format = "result %s: unexpected %s '%s' (expected '%s')" error( format % (id, key, found, expected)) +def _db_query(db, query): + db.query(query) + result = db.use_result() + return result and result.fetch_row(0,1) + class Platform: def __init__(self, name, user_friendly_name=None): self.name = name @@ -296,16 +303,7 @@ class Project: shell_call('sed -e s/BOINC_DB_NAME/%s/ %s | mysql' % (self.db_name, self.srcdir('db', script))) def db_open(self): - try: - self.db = MySQLdb.connect(db=self.db_name) - except e: - fatal_error('in mysql open of database "%s": %s' % (self.db_name, str(e))) - def db_query(self, query): - try: - self.db.query(query) - return self.db.use_result() - except e: - fatal_error('in mysql query "%s": %s' % (query, str(e))) + return MySQLdb.connect(db=self.db_name) def install_project(self, scheduler_file = None): verbose_echo(1, "Deleting previous test runs") @@ -374,8 +372,8 @@ class Project: verbose_echo(1, "Setting up database") map(self.run_db_script, [ 'drop.sql', 'schema.sql', 'constraints.sql' ]) - self.db_open() - self.db_query("insert into project(short_name, long_name) values('%s', '%s')" %( + db = self.db_open() + db.query("insert into project(short_name, long_name) values('%s', '%s')" %( self.short_name, self.long_name)); verbose_echo(1, "Setting up database: adding %d user(s)" % len(self.users)) @@ -389,7 +387,7 @@ class Project: else: gp = '' - self.db_query(("insert into user values (0, %d, '%s', '%s', '%s', " + + db.query(("insert into user values (0, %d, '%s', '%s', '%s', " + "'Peru', '12345', 0, 0, 0, '%s', '%s', 0, 'home', '', 0, 1)") % ( time.time(), user.email_addr, @@ -401,12 +399,14 @@ class Project: verbose_echo(1, "Setting up database: adding %d apps(s)" % len(self.apps)) for app in self.apps: check_app_executable(app.name) - self.db_query("insert into app(name, create_time) values ('%s', %d)" %( + db.query("insert into app(name, create_time) values ('%s', %d)" %( app.name, time.time())) self.platforms = unique(map(lambda a: a.platform, self.app_versions)) verbose_echo(1, "Setting up database: adding %d platform(s)" % len(self.platforms)) + db.close() + for platform in self.platforms: run_tool("add platform -db_name %s -platform_name %s -user_friendly_name '%s'" %( self.db_name, platform.name, platform.user_friendly_name)) @@ -611,7 +611,7 @@ class Project: def remove_config(self, pattern): config = self.dir('cgi/.htconfig.xml') config_old = config + '.old' - os.rename(config_old, config) + os.rename(config, config_old) f0 = open(config_old) f = open(config, 'w') for line in f0: @@ -627,20 +627,18 @@ class Project: stderr_out exit_status ''' - self.db_open() - result = self.db_query("select * from result") - rows = result.fetch_row(0, 1) + db = self.db_open() + rows = _db_query(db,"select * from result") for row in rows: dict_match(matchresult, row) + db.close() if len(rows) != ntarget: error("expected %d results, but found %d" % (ntarget, len(rows))) - def num_results_done(self): - self.db_open(self.db_name) - return self.db_query("select count(*) from result where server_state=2")[0][0] - def num_results_done(self): - self.db_open(self.db_name) - return self.db_query("select count(*) from result where server_state=4")[0][0] + def num_wus_left(self, db): + return _db_query(db, "select count(*) from result where server_state=%d"%RESULT_SERVER_STATE_UNSENT)[0]['count(*)'] + def num_results_done(self, db): + return _db_query(db, "select count(*) from result where server_state=%d"%RESULT_SERVER_STATE_OVER)[0]['count(*)'] def check_files_match(self, result, correct, count=None): '''if COUNT is specified then [0,COUNT) is mapped onto the %d in RESULT''' diff --git a/test/test_1sec.py b/test/test_1sec.py index 37ffce227a..191d3140fc 100755 --- a/test/test_1sec.py +++ b/test/test_1sec.py @@ -14,15 +14,15 @@ if __name__ == '__main__': host = Host() user = UserUC() - work = WorkUC(redundancy=5) projects = [] for i in range(2): - project = ProjectUC(users=[user], hosts=[host], works=[work], - short_name="test_1sec_%d"%i) + project = ProjectUC(users=[user], hosts=[host], + short_name="test_1sec_%d"%i, + redundancy=5) project.resource_share = [1, 5][i] projects.append(project) project.run() host.run() for project in projects: + project.check() project.stop() - project.validate() diff --git a/test/test_backend.py b/test/test_backend.py new file mode 100755 index 0000000000..8f146ab901 --- /dev/null +++ b/test/test_backend.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python + +## $Id$ + +# End to end test. Tests make_work, feeder, scheduling server, client, +# file_upload_handler, validator, assimilator, timeout_check, and file_deleter +# on a large batch of workunits. Confirms that credit is correctly granted +# and that unneeded files are deleted + +from boinc import * +from test_uc import * +import time, os + +class ProjectBackend(ProjectUC): + def __init__(self, num=500): + self.num = num + ProjectUC.__init__(self, + redundancy = 5, + short_name = 'test_backend') + + def run(self): + self.install() + self.install_make_work(work=self.work, cushion=self.num-1, redundancy=5) + self.start_servers() + + # wait for 500 results to be generated + n = 0 + db = self.db_open() + while n < self.num: + n = self.num_wus_left(db) + verbose_echo(1, "Generating results [%d/%d]" % (n,self.num)); + time.sleep(1) + db.close() + verbose_echo(1, "Generating results [%d/%d]: done." % (self.num,self.num)); + + # Stop the project, deinstall make_work, and install the normal + # backend components + + self.stop() + self.uninstall_make_work() + self.install_assimilator() + self.install_file_delete() + self.install_validate(self.app, quorum=5) + self.install_feeder() + self.install_timeout_check(self.app, nerror=5, ndet=5, nredundancy=0) + + # TODO: get PID and use wait. + verbose_echo(1, "Waiting for make_work to finish...") + while not os.system('pgrep -n make_work >/dev/null'): + time.sleep(1) + + self.start_servers() + + rpm_pid = None + def fork_result_progress_meter(self, msg): + pid = os.fork() + if pid: + self.rpm_pid = pid + return + db = self.db_open() + while True: + n = self.num_results_done(db) + verbose_echo(1, msg+" [%d/%d]"%(n,self.num)) + time.sleep(1) + + def stop_result_progress_meter(self): + if self.rpm_pid: + os.kill(self.rpm_pid, 9) + + def check(self): + # Give the server 30 seconds to finish assimilating/deleting + # TODO: use wait on all processes. + verbose_sleep("Sleeping to allow server to finish", 30) + self.check_results(self.num, ResultUC()) + +if __name__ == '__main__': + test_msg("entire backend"); + + num=None + try: + num = int(sys.argv[1]) + except: + pass + + project = ProjectBackend(num) + project.run() + project.fork_result_progress_meter("Running core client - results done: ") + project.host.run() + project.stop_result_progress_meter() + project.check() + project.stop() diff --git a/test/test_concat.py b/test/test_concat.py index b087a0c1cb..d78f951ac3 100755 --- a/test/test_concat.py +++ b/test/test_concat.py @@ -22,9 +22,9 @@ class ProjectConcat(Project): users = users, hosts = hosts) - def validate(self): + def check(self): redundancy = self.work.redundancy - Project.validate(self, redundancy) + self.validate(redundancy) result = {} result['server_state'] = RESULT_SERVER_STATE_OVER self.check_results(redundancy, result) @@ -45,5 +45,5 @@ if __name__ == '__main__': project = ProjectConcat() project.run() project.host.run() - project.validate() + project.check() project.stop() diff --git a/test/test_uc.py b/test/test_uc.py index ffa9cf669f..56185996ab 100755 --- a/test/test_uc.py +++ b/test/test_uc.py @@ -33,26 +33,29 @@ class WorkUC(Work): self.rsc_iops = 86400*1e9/2 self.rsc_disk = 10e8 +class ResultUC: + def __init__(self): + self.server_state = RESULT_SERVER_STATE_OVER + self.stderr_out = STARTS_WITH("""APP: upper_case: starting, argc 1 +APP: upper_case: argv[0] is upper_case +APP: upper_case ending, wrote """) + # self.exit_status = 0 + class ProjectUC(Project): def __init__(self, works=None, users=None, hosts=None, - short_name=None, long_name=None): + short_name=None, long_name=None, + redundancy=2): Project.__init__(self, appname = 'upper_case', - works = works or [WorkUC()], + works = works or [WorkUC(redundancy=redundancy)], users = users or [UserUC()], hosts = hosts, short_name=short_name, long_name=long_name) - def validate(self): + def check(self): redundancy = self.work.redundancy - Project.validate(self, redundancy) - result = {} - result['server_state'] = RESULT_SERVER_STATE_OVER - result['stderr_out'] = STARTS_WITH("""APP: upper_case: starting, argc 1 -APP: upper_case: argv[0] is upper_case -APP: upper_case ending, wrote """) - # result['exit_status'] = 0 - self.check_results(redundancy, result) + self.validate(redundancy) + self.check_results(redundancy, ResultUC()) self.check_files_match("upload/uc_wu_%d_0", "uc_correct_output", count=redundancy) self.assimilate() self.file_delete() @@ -70,5 +73,5 @@ if __name__ == '__main__': project = ProjectUC() project.run() project.host.run() - project.validate() + project.check() project.stop()