boinc/tools/update_versions

#!/usr/bin/env python

# $Id$

"""
Scans apps dir for current core client and application versions and updates
the database as appropriate.

config.xml must contain an <app_dir> which specifies the directory to search.

apps/boinc/ contains core client versions.
apps/APPLICATION_NAME/ contains application versions for each application.

Filenames MUST be of the form NAME_VERSION_PLATFORM[.ext].  The prefix name
and extensions .gz, .exe, .sit are ignored.  Platform strings must match the
names of platforms in the database.

update_versions also handles sub-directories, which contain files to bundle as
a single app-version.  Sub-directory names are ignored.  Non-executable files
can be arbitrarily named.

Example setup:

  apps/boinc/boinc_7.17_i686-pc-linux-gnu.gz
  apps/Astropulse/astropulse_7.17_windows_intelx86.exe
  apps/SETI@home/sah_4.20_sparc-sun-solaris2.9.gz
  apps/SETI@home/sah_4.20_i686-pc-linux-gnu.gz
  apps/SETI@Home/sah_4.21_sparc-sun-solaris2.9/sah_4.21_sparc-sun-solaris2.9.gz
  apps/SETI@Home/sah_4.21_sparc-sun-solaris2.9/sah_logo_1.0.jpg
  apps/SETI@Home/sah_4.21_i686-pc-linux-gnu/sah_4.21_i686-pc-linux-gnu.gz
  apps/SETI@Home/sah_4.21_i686-pc-linux-gnu/sah_logo_1.0.jpg

"""

import boinc_path_config
from Boinc import database, db_mid, configxml, tools
import sys, os, re, time

def get_current_core_version():
    cursor = database.get_dbconnection().cursor()
    cursor.execute('select max(version_num) from core_version')
    x = cursor.fetchall()[0].values()[0]
    if x == None:
        return 0
    return int(cursor.fetchall()[0].values()[0])

verbose = 0
try:
    if sys.argv[1] == '-v':
        verbose = 1
        print '(verbose mode)'
except:
    pass

config = configxml.default_config().config
database.connect()

##### only allow most recent core client
if True:
    min_core_version = get_current_core_version()
else:
    min_core_version = 0

create_time = int(time.time())

assert(config.app_dir
       and
       config.download_dir
       and
       config.download_url
       )

objects_to_commit = []

def xlistdir(dir):
    return map(lambda file: os.path.join(dir, file), os.listdir(dir))

def add_files(app, match, exec_file, non_exec_files=[],
              signature_files={}, file_ref_infos = {},
              min_core_version = 0, max_core_version = 0):
    ''' add files to app/core.

    EXEC_FILE is the executable, and NON_EXEC_FILES are supporting
    non-executable files.

    MATCH is the output of re_match_exec_filename(EXEC_FILE).
    '''
    assert(match)
    assert(exec_file)
    version_major, version_minor, platform_name = match.groups()
    version_num = int(version_major) * 100 + int(version_minor)

    file_base = os.path.basename(exec_file)
    platforms = database.Platforms.find(name = platform_name)
    if not platforms:
        print >>sys.stderr, "  Unknown platform '%s' for file %s" %(platform_name, file_base)
        return
    platform = platforms[0]

    if app:
        existing_versions = database.AppVersions.find(app=app, platform=platform, version_num=version_num)
        if existing_versions:
            if verbose:
                print "  Skipping existing %s %3d: %s" %(app.name, version_num, file_base)
            return

        print "  Found %s version %s for %s: %s" %(app, version_num, platform, file_base),
        if non_exec_files:
            print "(+ %d bundled file(s))"%len(non_exec_files)
        else:
            print

        xml_doc = tools.process_app_version(
            app = app,
            version_num = version_num,
            exec_files = [exec_file],
            non_exec_files = non_exec_files,
            signature_files = signature_files,
            file_ref_infos = file_ref_infos
            )

        object = database.AppVersion(create_time = create_time,
                                     app = app,
                                     platform = platform,
                                     version_num = version_num,
                                     min_core_version = min_core_version,
                                     max_core_version = max_core_version,
                                     xml_doc = xml_doc)
    else:
        assert(not non_exec_files)      # this wouldn't make sense for core clients
        existing_versions = database.CoreVersions.find(platform=platform, version_num=version_num)
        if existing_versions:
            if verbose:
                print "  Skipping existing core version %s: %s" %(version_num, file_base)
            return

        print "  Found core version %3d for %s: %s" %(version_num, platform, file_base)

        xml_doc = tools.process_executable_file(exec_file)

        object = database.CoreVersion(create_time = create_time,
                                      platform = platform,
                                      version_num = version_num,
                                      xml_doc = xml_doc)

    objects_to_commit.append(object)

# Return a match if the filename is a possible exec file name (or installer)

def re_match_exec_filename(filepath):
    file = os.path.basename(filepath)
    return re.match('[^.]+_([0-9]+)[.]([0-9]+)_([^.]+?(?:[0-9][0-9.]*[0-9])?)(?:[.]gz|[.]exe|[.]sit|[.]msi)?$', file)

def find_versions(app, dir):
    """Find application versions/core client versions in DIR.

    if app==None, then dir contains core clients; else contains application
    versions.

    If directory contains sub-directories, those are scanned (non-recursively)
    for files.  If an executable is found, the first one found
    (alphabetically) is the main program and other files bundled as
    non-executables.

    """

    for filepath in xlistdir(dir):
        # ignore symlinks
        if os.path.islink(filepath):
            continue

        # ignore signature files
        if filepath.endswith('.sig'):
            continue

        # look for an executable file (proper .ext at end)
        match = re_match_exec_filename(filepath)
        if not match:
            print >>sys.stderr, "  Ignoring unknown file", filepath
            continue
        if os.path.isdir(filepath):
            find_versions__process_bundle(app, match, filepath)
        else:
            find_versions__process_single_file(app, match, filepath)

# Process an app that is a single file only, possibly with
# a signature file included.  Could also be the core client.
#
def find_versions__process_single_file(app, match, filepath):
    '''add a single executable as app/core version'''
    # Find signature file, if it exists.
    signature_files={}
    sig_file = filepath + ".sig"
    if os.path.isfile(sig_file):
        signature_files[filepath] = sig_file

    add_files(app             = app,
             match            = match,
             exec_file        = filepath,
             signature_files  = signature_files,
             min_core_version = min_core_version,
             )


# Process an app that is a bundle of files in a directory of the same name,
# possibly with signature files included.
 
def find_versions__process_bundle(app, match, dir):
    '''add executable + bundle as app/core version'''
    exec_file = None
    non_exec_files = []
    signature_files = {}
    file_ref_infos = {}
    dirname = os.path.basename(dir)
    for filepath in xlistdir(dir):
        if os.path.isdir(filepath):
            continue
        if os.path.basename(filepath) == dirname:
            # the filename matches the folder name, so this is the executable.
            exec_file = filepath
            continue
        # if filename is of format 'EXECFILE.sig' treat it as signature file
        if filepath.endswith('.sig'):
            s = filepath[:-4]
            if re_match_exec_filename(s):
                signature_files[s] = filepath
                continue
        if filepath.endswith('.file_ref_info'):
            s = filepath[:-len('.file_ref_info')]
            file_ref_infos[s] = open(filepath).read()
            continue
        non_exec_files.append(filepath)
    if not exec_file:
        print >>sys.stderr, "  Ignoring directory (no executable found - it has to be named the same as the directory)", dir
        return
    # check signatures, file_path_infos
    for filepath, signature_file in signature_files.items():
        if filepath != exec_file: # and filepath not in non_exec_files:
            print >>sys.stderr, "  Warning: signature file '%s' will be ignored because it does not match an executable file" %signature_file
    for filepath in file_ref_infos:
        file_ref_info_filepath = filepath+'.file_ref_info'
        if filepath != exec_file and filepath not in non_exec_files:
            print >>sys.stderr, "  Warning: file_ref info file '%s' will be ignored because it does not match a file" %file_ref_info_filepath

    add_files(app              = app,
              match            = match,
              exec_file        = exec_file,
              non_exec_files   = non_exec_files,
              signature_files  = signature_files,
              file_ref_infos   = file_ref_infos,
              min_core_version = min_core_version
              )

####################
# BEGIN:

for appdir in xlistdir(config.app_dir):
    if not os.path.isdir(appdir): continue
    dirname = os.path.basename(appdir)
    if dirname == 'boinc':
        if verbose:
            print "Looking for core versions in", appdir
        find_versions(None, appdir)
        continue
    appname = os.path.basename(appdir)
    apps = database.Apps.find(name=appname)
    if apps:
        if verbose:
            print "Looking for %s versions in"%apps[0], appdir
        find_versions(apps[0], appdir)
        continue

    print "Couldn't find app '%s'" %(appname)

if not objects_to_commit:
    raise SystemExit("No new versions found!")

print "Ready to commit %d items:" %len(objects_to_commit)
for object in objects_to_commit:
    print "   ", object

if not tools.query_yesno("Continue"):
    raise SystemExit

print "Committed:"
for object in objects_to_commit:
    object.commit()
    print "   ", object

print "Done"