#!/usr/bin/env python # $Id$ """ Scans apps dir for current core client and application versions and updates the database as appropriate. config.xml must contain an which specifies the directory to search. apps/boinc/ contains core client versions. apps/APPLICATION_NAME/ contains application versions for each application. Filenames MUST be of the form NAME_VERSION_PLATFORM[.ext]. The prefix name and extensions .gz, .exe, .sit are ignored. Platform strings must match the names of platforms in the database. update_versions also handles sub-directories, which contain files to bundle as a single app-version. Sub-directory names are ignored. Non-executable files can be arbitrarily named. Example setup: apps/boinc/boinc_7.17_i686-pc-linux-gnu.gz apps/Astropulse/astropulse_7.17_windows_intelx86.exe apps/SETI@home/sah_4.20_sparc-sun-solaris2.9.gz apps/SETI@home/sah_4.20_i686-pc-linux-gnu.gz apps/SETI@Home/sah_4.21_sparc-sun-solaris2.9/sah_4.21_sparc-sun-solaris2.9.gz apps/SETI@Home/sah_4.21_sparc-sun-solaris2.9/sah_logo_1.0.jpg apps/SETI@Home/sah_4.21_i686-pc-linux-gnu/sah_4.21_i686-pc-linux-gnu.gz apps/SETI@Home/sah_4.21_i686-pc-linux-gnu/sah_logo_1.0.jpg """ import boinc_path_config from Boinc import database, db_mid, configxml, tools import sys, os, re, time def get_current_core_version(): cursor = database.get_dbconnection().cursor() cursor.execute('select max(version_num) from core_version') x = cursor.fetchall()[0].values()[0] if x == None: return 0 return int(cursor.fetchall()[0].values()[0]) verbose = 0 try: if sys.argv[1] == '-v': verbose = 1 print '(verbose mode)' except: pass config = configxml.default_config().config database.connect() ##### only allow most recent core client if True: min_core_version = get_current_core_version() else: min_core_version = 0 create_time = int(time.time()) assert(config.app_dir and config.download_dir and config.download_url ) objects_to_commit = [] def xlistdir(dir): return map(lambda file: os.path.join(dir, file), os.listdir(dir)) def add_files(app, match, exec_file, non_exec_files=[], signature_files={}, file_ref_infos = {}, min_core_version = 0, max_core_version = 0): ''' add files to app/core. EXEC_FILE is the executable, and NON_EXEC_FILES are supporting non-executable files. MATCH is the output of re_match_exec_filename(EXEC_FILE). ''' assert(match) assert(exec_file) version_major, version_minor, platform_name = match.groups() version_num = int(version_major) * 100 + int(version_minor) file_base = os.path.basename(exec_file) platforms = database.Platforms.find(name = platform_name) if not platforms: print >>sys.stderr, " Unknown platform '%s' for file %s" %(platform_name, file_base) return platform = platforms[0] if app: existing_versions = database.AppVersions.find(app=app, platform=platform, version_num=version_num) if existing_versions: if verbose: print " Skipping existing %s %3d: %s" %(app.name, version_num, file_base) return print " Found %s version %s for %s: %s" %(app, version_num, platform, file_base), if non_exec_files: print "(+ %d bundled file(s))"%len(non_exec_files) else: print xml_doc = tools.process_app_version( app = app, version_num = version_num, exec_files = [exec_file], non_exec_files = non_exec_files, signature_files = signature_files, file_ref_infos = file_ref_infos ) object = database.AppVersion(create_time = create_time, app = app, platform = platform, version_num = version_num, min_core_version = min_core_version, max_core_version = max_core_version, xml_doc = xml_doc) else: assert(not non_exec_files) # this wouldn't make sense for core clients existing_versions = database.CoreVersions.find(platform=platform, version_num=version_num) if existing_versions: if verbose: print " Skipping existing core version %s: %s" %(version_num, file_base) return print " Found core version %3d for %s: %s" %(version_num, platform, file_base) xml_doc = tools.process_executable_file(exec_file) object = database.CoreVersion(create_time = create_time, platform = platform, version_num = version_num, xml_doc = xml_doc) objects_to_commit.append(object) def re_match_exec_filename(filepath): file = os.path.basename(filepath) return re.match('[^.]+_([0-9]+)[.]([0-9]+)_([^.]+?(?:[0-9][0-9.]*[0-9])?)(?:[.]gz|[.]exe|[.]sit|[.]msi)?$', file) def find_versions(app, dir): """Find application versions/core client versions in DIR. if app==None, then dir contains core clients; else contains application versions. If directory contains sub-directories, those are scanned (non-recursively) for files. If an executable is found, the first one found (alphabetically) is the main program and other files bundled as non-executables. """ for filepath in xlistdir(dir): if os.path.islink(filepath): # ignore symlinks continue match = re_match_exec_filename(filepath) if not match: print >>sys.stderr, " Ignoring unknown file", filepath continue if os.path.isdir(filepath): find_versions__process_bundle(app, match, filepath) else: find_versions__process_single_file(app, match, filepath) def find_versions__process_single_file(app, match, filepath): '''add a single executable as app/core version''' add_files(app=app, match=match, exec_file=filepath, min_core_version = min_core_version ) def find_versions__process_bundle(app, match, dir): '''add executable + bundle as app/core version''' exec_file = None non_exec_files = [] signature_files = {} dirname = os.path.basename(dir) for filepath in xlistdir(dir): if os.path.isdir(filepath): continue if os.path.basename(filepath) == dirname: # the filename matches the folder name, so this is the executable. exec_file = filepath continue # if filename is of format 'EXECFILE.sig' treat it as signature file if filepath.endswith('.sig'): s = filepath[:-4] if re_match_exec_filename(s): signature_files[s] = filepath continue if filepath.endswith('.file_ref_info'): s = filepath[:-len('.file_ref_info')] file_ref_infos[s] = open(filepath).read() continue non_exec_files.append(filepath) if not exec_file: print >>sys.stderr, " Ignoring directory (no executable found - it has to be named the same as the directory)", dir return # check signatures, file_path_infos for filepath, signature_file in signature_files.items(): if filepath != exec_file: # and filepath not in non_exec_files: print >>sys.stderr, " Warning: signature file '%s' will be ignored because it does not match an executable file" %signature_file for filepath in file_ref_infos: file_ref_info_filepath = filepath+'.file_ref_info' if filepath != exec_file and filepath not in non_exec_files: print >>sys.stderr, " Warning: file_ref info file '%s' will be ignored because it does not match a file" %file_ref_info_filepath add_files(app = app, match = match, exec_file = exec_file, non_exec_files = non_exec_files, signature_files = signature_files, file_ref_infos = file_ref_infos, min_core_version = min_core_version ) for appdir in xlistdir(config.app_dir): if not os.path.isdir(appdir): continue dirname = os.path.basename(appdir) if dirname == 'boinc': print "Looking for core versions in", appdir find_versions(None, appdir) continue appname = os.path.basename(appdir) apps = database.Apps.find(name=appname) if apps: print "Looking for %s versions in"%apps[0], appdir find_versions(apps[0], appdir) continue print "Couldn't find app '%s'" %(appname) if not objects_to_commit: raise SystemExit("No new versions found!") print "Ready to commit %d items:" %len(objects_to_commit) for object in objects_to_commit: print " ", object if not tools.query_yesno("Continue"): raise SystemExit print "Committed:" for object in objects_to_commit: object.commit() print " ", object print "Done"