mirror of https://github.com/BOINC/boinc.git
make_project, upgrade, other tools
svn path=/trunk/boinc/; revision=2399
This commit is contained in:
Karl Chen
parent
bd07042ab8
commit
d2ed1c768f
|
|
@ -6480,3 +6480,30 @@ Eric Oct 2 2003
|
|||
api/graphics_api.h
|
||||
lib/xml_util.[Ch]
|
||||
|
||||
Karl 2003/10/02
|
||||
- rewrote make_project and related; a lot of restructuring:
|
||||
|
||||
- make_project only creates the project and database but doesn't add
|
||||
platform, app, app_version, or core_version. Use `add' for these.
|
||||
- database actions are through MySQLdb, removing need for mysql client
|
||||
binary.
|
||||
|
||||
tools/
|
||||
make_project
|
||||
add
|
||||
update_versions
|
||||
upgrade (new)
|
||||
py/Boinc/
|
||||
configxml.py
|
||||
database.py
|
||||
setup_project.py
|
||||
tools.py
|
||||
test/
|
||||
testbase.py
|
||||
test_uc.py
|
||||
sched/
|
||||
boinc_config.py (removed)
|
||||
db/
|
||||
constraints.sql
|
||||
doc/
|
||||
single_host_server.php
|
||||
|
|
|
|||
|
|
@ -631,39 +631,42 @@ def _connectp(dbname, user, passwd, host='localhost'):
|
|||
boincdb = MySQLdb.connect(db=dbname,host=host,user=user,passwd=passwd,
|
||||
cursorclass=MySQLdb.cursors.DictCursor)
|
||||
|
||||
# def _connectm(module):
|
||||
# _connectp(module.database, module.username, module.password)
|
||||
|
||||
# def connect(readonly = False):
|
||||
# """Connect if not already connected or if we're adding write permissions"""
|
||||
# global boincdb
|
||||
# if boincdb:
|
||||
# if not readonly and boincdb.readonly:
|
||||
# # re-open with write access
|
||||
# boincdb.close()
|
||||
# boincdb = None
|
||||
# else:
|
||||
# return 0
|
||||
# if readonly:
|
||||
# import password_settings_r
|
||||
# _connectm(password_settings_r)
|
||||
# else:
|
||||
# import password_settings
|
||||
# _connectm(password_settings)
|
||||
# boincdb.readonly = readonly
|
||||
# return 1
|
||||
|
||||
def connect(config = None):
|
||||
def connect(config = None, nodb = False):
|
||||
"""Connect if not already connected, using config values."""
|
||||
global boincdb
|
||||
if boincdb:
|
||||
return 0
|
||||
config = config or configxml.default_config().config
|
||||
_connectp(config.db_name,
|
||||
if nodb:
|
||||
db = ''
|
||||
else:
|
||||
db = config.db_name
|
||||
_connectp(db,
|
||||
config.__dict__.get('db_user',''),
|
||||
config.__dict__.get('db_passwd', ''))
|
||||
return 1
|
||||
|
||||
def _execute_sql_script(cursor, filename):
|
||||
for query in open(filename).read().split(';'):
|
||||
query = query.strip()
|
||||
if not query: continue
|
||||
cursor.execute(query)
|
||||
|
||||
def create_database(config = None, drop_first = False):
|
||||
''' creates a new database. '''
|
||||
global boincdb
|
||||
config = config or configxml.default_config().config
|
||||
connect(config, nodb=True)
|
||||
cursor = boincdb.cursor()
|
||||
if drop_first:
|
||||
cursor.execute("drop database if exists %s"%config.db_name)
|
||||
cursor.execute("create database %s"%config.db_name)
|
||||
cursor.execute("use %s"%config.db_name)
|
||||
schema_path = os.path.join(boinc_path_config.TOP_SOURCE_DIR, 'db')
|
||||
for file in ['schema.sql', 'constraints.sql']:
|
||||
_execute_sql_script(cursor, os.path.join(schema_path, file))
|
||||
cursor.close()
|
||||
|
||||
# alias
|
||||
connect_default_config = connect
|
||||
|
||||
|
|
|
|||
|
|
@ -1,20 +1,14 @@
|
|||
## $Id$
|
||||
|
||||
# module for setting up a new project (either a real project or a test project
|
||||
# - see testbase.py).
|
||||
#
|
||||
# (This used to be boinc/py/boinc.py.)
|
||||
# see tools/makeproject, test/testbase.py).
|
||||
|
||||
# TODO: make things work if build_dir != src_dir
|
||||
|
||||
# TODO: use database.py
|
||||
# TODO: make sure things work if build_dir != src_dir
|
||||
|
||||
import boinc_path_config
|
||||
import version
|
||||
from Boinc import database, db_mid, configxml, tools
|
||||
from Boinc.boinc_db import *
|
||||
import os, sys, glob, time, shutil, re, random
|
||||
# import MySQLdb
|
||||
|
||||
class Options:
|
||||
pass
|
||||
|
|
@ -27,7 +21,6 @@ options.have_init = False
|
|||
options.install_method = None
|
||||
options.echo_verbose = 1
|
||||
options.is_test = False
|
||||
options.client_bin_filename = version.CLIENT_BIN_FILENAME
|
||||
options.drop_db_first = False
|
||||
|
||||
def init():
|
||||
|
|
@ -164,8 +157,11 @@ def force_symlink(src, dest):
|
|||
os.unlink(dest)
|
||||
my_symlink(src, dest)
|
||||
def rmtree(dir):
|
||||
if os.path.exists(dir):
|
||||
shutil.rmtree(dir)
|
||||
# if os.path.exists(dir):
|
||||
# shutil.rmtree(dir)
|
||||
if not dir or dir == '/' or dir == '.' or ' ' in dir:
|
||||
raise Exception
|
||||
os.system("rm -rf %s"%dir)
|
||||
|
||||
def _remove_trail(s, suffix):
|
||||
if s.endswith(suffix):
|
||||
|
|
@ -240,122 +236,91 @@ def _check_vars(dict, **names):
|
|||
# result = db.use_result()
|
||||
# return result and result.fetch_row(0,1)
|
||||
|
||||
def num_results(db):
|
||||
def num_results():
|
||||
return database.Results.count()
|
||||
def num_results_unsent(db):
|
||||
def num_results_unsent():
|
||||
return database.Results.count(server_state = RESULT_SERVER_STATE_UNSENT)
|
||||
def num_results_in_progress(db):
|
||||
def num_results_in_progress():
|
||||
return database.Results.count(server_state = RESULT_SERVER_STATE_IN_PROGRESS)
|
||||
def num_results_over(db):
|
||||
def num_results_over():
|
||||
return database.Results.count(server_state = RESULT_SERVER_STATE_OVER)
|
||||
def num_wus(db):
|
||||
def num_wus():
|
||||
return database.Workunits.count()
|
||||
def num_wus_assimilated(db):
|
||||
def num_wus_assimilated():
|
||||
return database.Workunits.count(assimilate_state = ASSIMILATE_DONE)
|
||||
def num_wus_to_transition(db):
|
||||
def num_wus_to_transition():
|
||||
return database.Workunits.count(_extra_params = 'transition_time<%d'%(time.time()+30*86400))
|
||||
|
||||
def query_yesno(str):
|
||||
'''Query user; default Yes'''
|
||||
verbose_echo(0,'')
|
||||
print str, "[Y/n] ",
|
||||
return not raw_input().strip().lower().startswith('n')
|
||||
|
||||
def query_noyes(str):
|
||||
'''Query user; default No'''
|
||||
verbose_echo(0,'')
|
||||
print str, "[y/N] ",
|
||||
return raw_input().strip().lower().startswith('y')
|
||||
|
||||
def build_command_line(cmd, **kwargs):
|
||||
for (key, value) in kwargs.items():
|
||||
cmd += " -%s '%s'" %(key,value)
|
||||
return cmd
|
||||
|
||||
# class Platform:
|
||||
# def __init__(self, name, user_friendly_name=None):
|
||||
# self.name = name
|
||||
# self.user_friendly_name = user_friendly_name or name
|
||||
def install_boinc_files(dest_dir):
|
||||
def dir(*dirs):
|
||||
return apply(os.path.join,(dest_dir,)+dirs)
|
||||
|
||||
# class CoreVersion:
|
||||
# def __init__(self):
|
||||
# self.version = 1
|
||||
# self.platform = Platform(version.PLATFORM)
|
||||
# self.exec_dir = builddir('client')
|
||||
# self.exec_name = options.client_bin_filename
|
||||
install_glob(srcdir('html_user/*.php'), dir('html_user/'))
|
||||
install_glob(srcdir('html_user/*.inc'), dir('html_user/'))
|
||||
install_glob(srcdir('html_user/class/*.inc'), dir('html_user/class/'))
|
||||
install_glob(srcdir('html_user/include/*.inc'), dir('html_user/include/'))
|
||||
install_glob(srcdir('html_ops/*.php'), dir('html_ops/'))
|
||||
install_glob(srcdir('html_ops/*.inc'), dir('html_ops/'))
|
||||
install(builddir('tools/country_select'), dir('html_user/'))
|
||||
|
||||
# class App:
|
||||
# def __init__(self, name):
|
||||
# assert(name)
|
||||
# self.name = name
|
||||
# copy all the backend programs
|
||||
map(lambda (s): install(builddir('sched',s), dir('cgi-bin',s)),
|
||||
[ 'cgi', 'file_upload_handler'])
|
||||
map(lambda (s): install(builddir('sched',s), dir('bin',s)),
|
||||
[ 'make_work', 'feeder', 'transitioner', 'validate_test',
|
||||
'file_deleter', 'assimilator' ])
|
||||
map(lambda (s): install(srcdir('sched',s), dir('bin',s)),
|
||||
[ 'start', 'stop', 'status',
|
||||
'grep_logs' ])
|
||||
map(lambda (s): install(srcdir('tools',s), dir('bin',s)),
|
||||
[ 'boinc_path_config.py', 'add', 'dbcheck_files_exist',
|
||||
'upgrade' ])
|
||||
|
||||
# class AppVersion:
|
||||
# def __init__(self, app, appversion = 1, exec_names=None):
|
||||
# self.exec_names = []
|
||||
# self.exec_dir = builddir('apps')
|
||||
# self.exec_names = exec_names or [app.name]
|
||||
# self.app = app
|
||||
# self.version = appversion
|
||||
# self.platform = Platform(version.PLATFORM)
|
||||
|
||||
class Project:
|
||||
def __init__(self,
|
||||
short_name, long_name, appname=None,
|
||||
project_dir=None, master_url=None, cgi_url=None,
|
||||
core_versions=None, key_dir=None,
|
||||
apps=None, app_versions=None,
|
||||
resource_share=None):
|
||||
short_name, long_name,
|
||||
project_dir=None,key_dir=None,
|
||||
master_url=None, cgi_url=None,
|
||||
db_name=None):
|
||||
init()
|
||||
self.config_options = []
|
||||
self.config_daemons = []
|
||||
self.short_name = short_name or 'test_'+appname
|
||||
self.long_name = long_name or 'Project ' + self.short_name.replace('_',' ').capitalize()
|
||||
self.db_passwd = ''
|
||||
self.shmem_key = generate_shmem_key()
|
||||
self.resource_share = resource_share or 1
|
||||
self.output_level = 3
|
||||
|
||||
self.master_url = master_url or os.path.join(options.html_url , self.short_name , '')
|
||||
self.download_url = os.path.join(self.master_url, 'download')
|
||||
self.cgi_url = cgi_url or os.path.join(options.cgi_url, self.short_name)
|
||||
self.upload_url = os.path.join(self.cgi_url , 'file_upload_handler')
|
||||
self.scheduler_url = os.path.join(self.cgi_url , 'cgi')
|
||||
self.project_dir = project_dir or os.path.join(options.projects_dir , self.short_name)
|
||||
self.download_dir = os.path.join(self.project_dir , 'download')
|
||||
self.upload_dir = os.path.join(self.project_dir , 'upload')
|
||||
self.key_dir = key_dir or os.path.join(self.project_dir , 'keys')
|
||||
self.user_name = options.user_name
|
||||
self.db_name = self.user_name + '_' + self.short_name
|
||||
self.short_name = short_name
|
||||
self.long_name = long_name or 'Project ' + self.short_name.replace('_',' ').capitalize()
|
||||
|
||||
self.project_dir = project_dir or os.path.join(options.projects_dir, self.short_name)
|
||||
|
||||
self.config = configxml.ConfigFile(self.dir('config.xml')).init_empty()
|
||||
config = self.config.config
|
||||
|
||||
config.user_name = options.user_name
|
||||
config.db_name = db_name or config.user_name + '_' + self.short_name
|
||||
config.db_passwd = ''
|
||||
config.shmem_key = generate_shmem_key()
|
||||
config.output_level = 3
|
||||
|
||||
config.master_url = master_url or os.path.join(options.html_url , self.short_name , '')
|
||||
config.download_url = os.path.join(config.master_url, 'download')
|
||||
config.cgi_url = cgi_url or os.path.join(options.cgi_url, self.short_name)
|
||||
config.upload_url = os.path.join(config.cgi_url , 'file_upload_handler')
|
||||
self.scheduler_url = os.path.join(config.cgi_url , 'cgi')
|
||||
config.download_dir = os.path.join(self.project_dir , 'download')
|
||||
config.upload_dir = os.path.join(self.project_dir , 'upload')
|
||||
config.key_dir = key_dir or os.path.join(self.project_dir , 'keys')
|
||||
self.project_php_file = srcdir('html_user/project.inc.sample')
|
||||
self.project_specific_prefs_php_file = srcdir('html_user/project_specific_prefs.inc.sample')
|
||||
|
||||
self.core_versions = core_versions or [CoreVersion()]
|
||||
self.app_versions = app_versions or [AppVersion(App(appname))]
|
||||
self.apps = apps or unique(map(lambda av: av.app, self.app_versions))
|
||||
self.platforms = [Platform(version.PLATFORM)]
|
||||
# convenience vars:
|
||||
self.app_version = self.app_versions[0]
|
||||
self.app = self.apps[0]
|
||||
|
||||
def dir(self, *dirs):
|
||||
return apply(os.path.join,(self.project_dir,)+dirs)
|
||||
|
||||
def keydir(self, *dirs):
|
||||
return apply(os.path.join,(self.key_dir,)+dirs)
|
||||
|
||||
def run_db_script(self, script):
|
||||
shell_call('mysql %s < %s' % (self.db_name,srcdir('db', script)))
|
||||
|
||||
def drop_db_if_exists(self):
|
||||
shell_call('echo "drop database if exists %s" | mysql' % self.db_name)
|
||||
|
||||
def create_db(self):
|
||||
if options.drop_db_first:
|
||||
self.drop_db_if_exists()
|
||||
shell_call('echo "create database %s" | mysql' % self.db_name)
|
||||
|
||||
def db_open(self):
|
||||
return MySQLdb.connect(db=self.db_name)
|
||||
return apply(os.path.join,(self.config.config.key_dir,)+dirs)
|
||||
|
||||
def create_keys(self):
|
||||
if not os.path.exists(self.keydir()):
|
||||
|
|
@ -364,7 +329,7 @@ class Project:
|
|||
_gen_key(self.keydir('code_sign'))
|
||||
|
||||
def query_create_keys(self):
|
||||
return query_yesno("Keys don't exist in %s; generate them?"%self.key_dir)
|
||||
return query_yesno("Keys don't exist in %s; generate them?"%self.keydir())
|
||||
|
||||
def keys_exist(self):
|
||||
keys = ['upload_private', 'upload_public',
|
||||
|
|
@ -395,28 +360,23 @@ class Project:
|
|||
self.create_keys()
|
||||
|
||||
# copy the user and administrative PHP files to the project dir,
|
||||
verbose_echo(1, "Setting up server files: copying html directories")
|
||||
verbose_echo(1, "Setting up server files: copying files")
|
||||
|
||||
install_boinc_files(self.dir())
|
||||
|
||||
install_glob(srcdir('html_user/*.php'), self.dir('html_user/'))
|
||||
install_glob(srcdir('html_user/*.inc'), self.dir('html_user/'))
|
||||
install_glob(srcdir('html_user/class/*.inc'), self.dir('html_user/class/'))
|
||||
install_glob(srcdir('html_user/include/*.inc'), self.dir('html_user/include/'))
|
||||
install_glob(srcdir('html_user/*.txt'), self.dir('html_user/'))
|
||||
install_glob(srcdir('html_ops/*.php'), self.dir('html_ops/'))
|
||||
install_glob(srcdir('html_ops/*.inc'), self.dir('html_ops/'))
|
||||
install(builddir('tools/country_select'), self.dir('html_user/'))
|
||||
install(self.project_php_file,
|
||||
self.dir('html_user', 'project_specific', 'project.inc'))
|
||||
install(self.project_specific_prefs_php_file,
|
||||
self.dir('html_user', 'project_specific', 'project_specific_prefs.inc'))
|
||||
|
||||
my_symlink(self.download_dir, self.dir('html_user', 'download'))
|
||||
my_symlink(self.config.config.download_dir, self.dir('html_user', 'download'))
|
||||
|
||||
|
||||
# Copy the sched server in the cgi directory with the cgi names given
|
||||
# source_dir/html_usr/schedulers.txt
|
||||
#
|
||||
|
||||
verbose_echo(1, "Setting up server files: copying cgi programs");
|
||||
if scheduler_file:
|
||||
r = re.compile('<scheduler>([^<]+)</scheduler>', re.IGNORECASE)
|
||||
f = open(self.dir('html_user', scheduler_file))
|
||||
|
|
@ -432,91 +392,20 @@ class Project:
|
|||
else:
|
||||
scheduler_file = 'schedulers.txt'
|
||||
f = open(self.dir('html_user', scheduler_file), 'w')
|
||||
print >>f, "<scheduler>" + self.scheduler_url, "</scheduler>"
|
||||
print >>f, "<scheduler>" + self.scheduler_url.strip(), "</scheduler>"
|
||||
f.close()
|
||||
|
||||
# copy all the backend programs
|
||||
map(lambda (s): install(builddir('sched',s), self.dir('cgi-bin',s)),
|
||||
[ 'cgi', 'file_upload_handler'])
|
||||
map(lambda (s): install(builddir('sched',s), self.dir('bin',s)),
|
||||
[ 'make_work', 'feeder', 'transitioner', 'validate_test',
|
||||
'file_deleter', 'assimilator' ])
|
||||
map(lambda (s): install(srcdir('sched',s), self.dir('bin',s)),
|
||||
[ 'start', 'stop', 'status',
|
||||
'boinc_config.py', 'grep_logs' ])
|
||||
|
||||
verbose_echo(1, "Setting up database")
|
||||
self.create_db()
|
||||
map(self.run_db_script, [ 'schema.sql' ])
|
||||
|
||||
database.connect()
|
||||
database.create_database(config = self.config.config,
|
||||
drop_first = options.drop_db_first)
|
||||
|
||||
self.project = database.Project(short_name = self.short_name,
|
||||
long_name = self.long_name)
|
||||
self.project.commit()
|
||||
|
||||
verbose_echo(1, "Setting up database: adding %d apps(s)" % len(self.apps))
|
||||
for app in self.apps:
|
||||
db.query("insert into app(name, create_time) values ('%s', %d)" %(
|
||||
app.name, time.time()))
|
||||
verbose_echo(1, "Setting up server files: writing config files")
|
||||
|
||||
self.platforms = unique(map(lambda a: a.platform, self.app_versions))
|
||||
verbose_echo(1, "Setting up database: adding %d platform(s)" % len(self.platforms))
|
||||
|
||||
db.close()
|
||||
|
||||
for platform in self.platforms:
|
||||
cmd = build_command_line("old_add platform",
|
||||
db_name = self.db_name,
|
||||
platform_name = platform.name,
|
||||
user_friendly_name = platform.user_friendly_name)
|
||||
run_tool(cmd)
|
||||
|
||||
verbose_echo(1, "Setting up database: adding %d core version(s)" % len(self.core_versions))
|
||||
for core_version in self.core_versions:
|
||||
cmd = build_command_line("old_add core_version",
|
||||
db_name = self.db_name,
|
||||
platform_name = core_version.platform.name,
|
||||
version = core_version.version,
|
||||
download_dir = self.download_dir,
|
||||
download_url = self.download_url,
|
||||
exec_dir = core_version.exec_dir,
|
||||
exec_files = core_version.exec_name)
|
||||
run_tool(cmd)
|
||||
|
||||
verbose_echo(1, "Setting up database: adding %d app version(s)" % len(self.app_versions))
|
||||
for app_version in self.app_versions:
|
||||
app = app_version.app
|
||||
cmd = ("old_add app_version -db_name %s -app_name '%s'" +
|
||||
" -platform_name %s -version %s -download_dir %s -download_url %s" +
|
||||
" -code_sign_keyfile %s -exec_dir %s -exec_files") % (
|
||||
self.db_name, app.name, app_version.platform.name,
|
||||
app_version.version,
|
||||
self.download_dir,
|
||||
self.download_url,
|
||||
os.path.join(self.key_dir, 'code_sign_private'),
|
||||
app_version.exec_dir)
|
||||
for exec_name in app_version.exec_names:
|
||||
check_app_executable(exec_name)
|
||||
cmd += ' ' + exec_name
|
||||
run_tool(cmd)
|
||||
|
||||
verbose_echo(1, "Setting up server files: writing config files");
|
||||
|
||||
config = map_xml(self,
|
||||
[ 'db_name', 'db_passwd', 'shmem_key',
|
||||
'key_dir', 'download_url', 'download_dir',
|
||||
'upload_url', 'upload_dir', 'project_dir', 'user_name',
|
||||
'cgi_url',
|
||||
'output_level' ])
|
||||
self.config_options = config.split('\n')
|
||||
self.write_config()
|
||||
|
||||
# edit "index.php" in the user HTML directory to have the right file
|
||||
# as the source for scheduler_urls; default is schedulers.txt
|
||||
|
||||
macro_substitute_inplace('FILE_NAME', scheduler_file,
|
||||
self.dir('html_user', 'index.php'))
|
||||
self.config.write()
|
||||
|
||||
# create symbolic links to the CGI and HTML directories
|
||||
verbose_echo(1, "Setting up server files: linking cgi programs")
|
||||
|
|
@ -526,12 +415,6 @@ class Project:
|
|||
force_symlink(self.dir('html_user'), os.path.join(options.html_dir, self.short_name))
|
||||
force_symlink(self.dir('html_ops'), os.path.join(options.html_dir, self.short_name+'_admin'))
|
||||
|
||||
# show the URLs for user and admin sites
|
||||
# admin_url = os.path.join("html_user", self.short_name+'_admin/')
|
||||
|
||||
# verbose_echo(2, "Master URL: " + self.master_url)
|
||||
# verbose_echo(2, "Admin URL: " + admin_url)
|
||||
|
||||
def http_password(self, user, password):
|
||||
'Adds http password protection to the html_ops directory'
|
||||
passwd_file = self.dir('html_ops', '.htpassword')
|
||||
|
|
@ -585,11 +468,11 @@ class Project:
|
|||
self._run_sched_prog(prog, '-d 3 -one_pass '+cmdline)
|
||||
def sched_install(self, prog, **kwargs):
|
||||
for cmdline in self._build_sched_commandlines(prog, kwargs):
|
||||
self.config_daemons.append("%s -d 3 %s" %(prog, cmdline))
|
||||
self.write_config()
|
||||
def sched_uninstall(self, prog):
|
||||
self.config_daemons = filter(lambda l: l.find(prog)==-1, self.config_daemons)
|
||||
self.write_config()
|
||||
self.config.daemons.make_node_and_append("daemon").cmd = "%s -d 3 %s" %(prog, cmdline)
|
||||
self.config.write()
|
||||
# def sched_uninstall(self, prog):
|
||||
# self.config_daemons = XXX filter(lambda l: l.find(prog)==-1, self.config_daemons)
|
||||
# self.config.write()
|
||||
|
||||
def start_stripcharts(self):
|
||||
map(lambda l: self.copy(os.path.join('stripchart', l), 'cgi-bin/'),
|
||||
|
|
@ -613,15 +496,10 @@ class Project:
|
|||
def maybe_stop(self):
|
||||
if self.started: self.stop()
|
||||
|
||||
def write_config(self):
|
||||
f = open(self.dir('config.xml'), 'w')
|
||||
print >>f, '<boinc>'
|
||||
print >>f, ' <config>'
|
||||
for line in self.config_options:
|
||||
print >>f, " ", line
|
||||
print >>f, ' </config>'
|
||||
print >>f, ' <daemons>'
|
||||
for daemon in self.config_daemons:
|
||||
print >>f, " <daemon><cmd>%s</cmd></daemon>"%daemon
|
||||
print >>f, ' </daemons>'
|
||||
print >>f, '</boinc>'
|
||||
def query_noyes(str):
|
||||
verbose_echo(0,'')
|
||||
return tools.query_noyes(str)
|
||||
|
||||
def query_yesno(str):
|
||||
verbose_echo(0,'')
|
||||
return tools.query_yesno(str)
|
||||
|
|
|
|||
|
|
@ -14,10 +14,10 @@ def file_size(path):
|
|||
f.seek(0,2)
|
||||
return f.tell()
|
||||
|
||||
def sign_executable(executable_path):
|
||||
def sign_executable(executable_path, quiet=False):
|
||||
'''Returns signed text for executable'''
|
||||
config = configxml.default_config()
|
||||
print 'Signing', executable_path
|
||||
if not quiet: print 'Signing', executable_path
|
||||
code_sign_key = os.path.join(config.config.key_dir, 'code_sign_private')
|
||||
sign_executable_path = os.path.join(boinc_path_config.TOP_BUILD_DIR,
|
||||
'tools','sign_executable')
|
||||
|
|
@ -29,7 +29,7 @@ def sign_executable(executable_path):
|
|||
raise SystemExit("Couldn't sign executable %s"%executable_path)
|
||||
return signature_text
|
||||
|
||||
def process_executable_file(file, signature_text=None):
|
||||
def process_executable_file(file, signature_text=None, quiet=False):
|
||||
'''Handle a new executable file to be added to the database.
|
||||
|
||||
1. Copy file to download_dir if necessary.
|
||||
|
|
@ -42,7 +42,7 @@ def process_executable_file(file, signature_text=None):
|
|||
file_dir, file_base = os.path.split(file)
|
||||
target_path = os.path.join(config.config.download_dir, file_base)
|
||||
if file_dir != config.config.download_dir:
|
||||
print "Copying %s to %s"%(file_base, config.config.download_dir)
|
||||
if not quiet: print "Copying %s to %s"%(file_base, config.config.download_dir)
|
||||
shutil.copy(file, target_path)
|
||||
|
||||
xml = '''<file_info>
|
||||
|
|
@ -60,7 +60,7 @@ def process_executable_file(file, signature_text=None):
|
|||
xml += ' <nbytes>%f</nbytes>\n</file_info>\n' % file_size(target_path)
|
||||
return xml
|
||||
|
||||
def process_app_version(app, version_num, exec_files, signature_files={}):
|
||||
def process_app_version(app, version_num, exec_files, signature_files={}, quiet=False):
|
||||
"""Return xml for application version
|
||||
|
||||
app is an instance of database.App
|
||||
|
|
@ -80,8 +80,8 @@ def process_app_version(app, version_num, exec_files, signature_files={}):
|
|||
if signature_file:
|
||||
signature_text = open(signature_file).read()
|
||||
else:
|
||||
signature_text = sign_executable(exec_file)
|
||||
xml_doc += process_executable_file(exec_file, signature_text)
|
||||
signature_text = sign_executable(exec_file, quiet=quiet)
|
||||
xml_doc += process_executable_file(exec_file, signature_text, quiet=quiet)
|
||||
|
||||
xml_doc += ('<app_version>\n'+
|
||||
' <app_name>%s</app_name>\n'+
|
||||
|
|
@ -101,3 +101,13 @@ def process_app_version(app, version_num, exec_files, signature_files={}):
|
|||
|
||||
xml_doc += '</app_version>\n'
|
||||
return xml_doc
|
||||
|
||||
def query_yesno(str):
|
||||
'''Query user; default Yes'''
|
||||
print str, "[Y/n] ",
|
||||
return not raw_input().strip().lower().startswith('n')
|
||||
|
||||
def query_noyes(str):
|
||||
'''Query user; default No'''
|
||||
print str, "[y/N] ",
|
||||
return raw_input().strip().lower().startswith('y')
|
||||
|
|
|
|||
|
|
@ -1,200 +0,0 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
# $Id$
|
||||
|
||||
# boinc_config.py - module to read and parse config.xml, run_state.xml
|
||||
|
||||
'''
|
||||
SYNOPSIS: parses and writes config.xml and run_state.xml
|
||||
|
||||
USAGE: from boinc_config import *
|
||||
config = BoincConfig('confing.xml').read()
|
||||
run_state = BoincRunState('run_state.xml').read()
|
||||
print config.config.db_name
|
||||
print config.tasks[4].cmd
|
||||
run_state.enabled = True
|
||||
new_task = newConfigDict()
|
||||
new_task.cmd = "echo hi | mail quarl"
|
||||
config.tasks.append(new_task)
|
||||
config.write()
|
||||
run_state.write()
|
||||
|
||||
TODO: create a Boinc package and remove these Boinc prefix qualifiers
|
||||
'''
|
||||
|
||||
import sys
|
||||
import xml.dom.minidom
|
||||
|
||||
CONFIG_FILE = '../config.xml'
|
||||
RUN_STATE_FILE = '../run_state.xml'
|
||||
|
||||
def _append_new_element(parent_node, name):
|
||||
new_element = xml.dom.minidom.Element(name)
|
||||
if isinstance(parent_node,xml.dom.minidom.Document):
|
||||
new_element.ownerDocument = parent_node
|
||||
else:
|
||||
assert(parent_node.ownerDocument)
|
||||
new_element.ownerDocument = parent_node.ownerDocument
|
||||
parent_node.appendChild(new_element)
|
||||
return new_element
|
||||
|
||||
def _get_elements(node, name):
|
||||
return node.getElementsByTagName(name)
|
||||
|
||||
def _get_element(node, name, optional=True):
|
||||
try:
|
||||
return _get_elements(node,name)[0]
|
||||
except IndexError:
|
||||
if optional:
|
||||
return _append_new_element(node, name)
|
||||
raise SystemExit("ERROR: Couldn't find xml node <%s>"% name)
|
||||
|
||||
def _None2Str(object):
|
||||
if object == None:
|
||||
return ''
|
||||
else:
|
||||
return object
|
||||
|
||||
def _get_element_data(node):
|
||||
return node and _None2Str(node.firstChild and node.firstChild.data)
|
||||
|
||||
def _get_element_int(node, default=0):
|
||||
try:
|
||||
return int(_get_element_data(node))
|
||||
except:
|
||||
return default
|
||||
|
||||
def _get_child_elements(node):
|
||||
return filter(lambda node: node.nodeType == node.ELEMENT_NODE, node.childNodes)
|
||||
|
||||
def _set_element(node, new_data):
|
||||
if node.firstChild and node.firstChild.data:
|
||||
node.firstChild.data = str(new_data)
|
||||
else:
|
||||
new_data_node = xml.dom.minidom.Text()
|
||||
new_data_node.data = str(new_data)
|
||||
new_data_node.ownerDocument = node.ownerDocument
|
||||
node.appendChild(new_data_node)
|
||||
|
||||
class ConfigDict:
|
||||
def __init__(self, dom_node):
|
||||
self._node = dom_node
|
||||
self._name = self._node.nodeName
|
||||
for node in _get_child_elements(self._node):
|
||||
self.__dict__[node.nodeName] = _get_element_data(node)
|
||||
def save(self):
|
||||
for key in self.__dict__:
|
||||
if key.startswith('_'): continue
|
||||
_set_element( _get_element(self._node,key,1), str(self.__dict__[key]) )
|
||||
def debug_print(self):
|
||||
for key in self.__dict__.keys():
|
||||
print key.rjust(15), '=', self.__dict__[key]
|
||||
|
||||
class ConfigDictList(list):
|
||||
def __init__(self, dom_node, item_class=ConfigDict):
|
||||
self._node = dom_node
|
||||
list.__init__(self, map(item_class, _get_child_elements(self._node)))
|
||||
def save(self):
|
||||
map(ConfigDict.save, self)
|
||||
def make_node_and_append(self, name):
|
||||
'''Make a new ConfigDict and append it. Returns new ConfigDict.'''
|
||||
new_element = _append_new_element(self._node, name)
|
||||
new_cd = ConfigDict(new_element)
|
||||
self.append(new_cd)
|
||||
return new_cd
|
||||
|
||||
# base class for xml config files
|
||||
class XMLConfig:
|
||||
def __init__(self, filename):
|
||||
self.filename = filename
|
||||
def read(self, failopen_ok=False):
|
||||
try:
|
||||
self.xml = xml.dom.minidom.parse(self.filename)
|
||||
except IOError, e:
|
||||
if not failopen_ok:
|
||||
raise
|
||||
print >>sys.stderr, "Warning:", e
|
||||
# self.xml = xml.dom.minidom.Document()
|
||||
self.xml = xml.dom.minidom.parseString(self.default_xml)
|
||||
self._get_elements()
|
||||
return self
|
||||
def _get_elements(self):
|
||||
pass
|
||||
def write(self, output=None):
|
||||
self._set_elements()
|
||||
if not output:
|
||||
output = open(self.filename,'w')
|
||||
self.xml.writexml(output)
|
||||
print >>output
|
||||
return self
|
||||
def _set_elements(self):
|
||||
pass
|
||||
|
||||
# normal config file
|
||||
class BoincConfig(XMLConfig):
|
||||
'''
|
||||
embodies config.xml
|
||||
Public attributes:
|
||||
config - ConfigDict
|
||||
tasks - list of ConfigDict elements
|
||||
'''
|
||||
def _get_elements(self):
|
||||
self.xml_boinc = _get_element(self.xml, 'boinc', optional=False)
|
||||
self.xml_config = _get_element(self.xml_boinc, 'config', optional=False)
|
||||
self.xml_tasks = _get_element(self.xml_boinc, 'tasks')
|
||||
self.xml_daemons = _get_element(self.xml_boinc, 'daemons')
|
||||
self.config = ConfigDict(self.xml_config)
|
||||
self.daemons = ConfigDictList(self.xml_daemons)
|
||||
self.tasks = ConfigDictList(self.xml_tasks)
|
||||
def _set_elements(self):
|
||||
self.config.save()
|
||||
self.daemons.save()
|
||||
self.tasks.save()
|
||||
def debug_print_all(self):
|
||||
'''print everything to stdout.'''
|
||||
|
||||
print 'Debug dump of', self.filename
|
||||
print '-- parsed xml -------------------------------------------------------'
|
||||
self.xml.writexml(sys.stdout)
|
||||
print
|
||||
print '-- Config -----------------------------------------------------------'
|
||||
self.config.debug_print()
|
||||
print
|
||||
print '-- Daemons ------------------------------------------------------------'
|
||||
for daemon in self.daemons:
|
||||
daemon.debug_print()
|
||||
print
|
||||
print
|
||||
print '-- Tasks ------------------------------------------------------------'
|
||||
for task in self.tasks:
|
||||
task.debug_print()
|
||||
print
|
||||
default_xml = '<boinc><config></config></boinc>'
|
||||
|
||||
# keeps BoincCron's timestamp status file
|
||||
class BoincRunState(XMLConfig):
|
||||
'''
|
||||
embodies run_state.xml
|
||||
Public attributes:
|
||||
tasks - list of ConfigDict elements
|
||||
enabled - boolean
|
||||
'''
|
||||
def _get_elements(self):
|
||||
self.xml_boinc = _get_element(self.xml, 'boinc', optional=False)
|
||||
self.xml_tasks = _get_element(self.xml_boinc, 'tasks')
|
||||
self.xml_enabled = _get_element(self.xml_boinc, 'enabled')
|
||||
self.tasks = ConfigDictList(self.xml_tasks)
|
||||
self.enabled = _get_element_int(self.xml_enabled)
|
||||
def _set_elements(self):
|
||||
_set_element( self.xml_enabled, self.enabled )
|
||||
self.tasks.save()
|
||||
default_xml = '<boinc></boinc>'
|
||||
|
||||
if __name__ == '__main__':
|
||||
config = BoincConfig('config.xml')
|
||||
config.read()
|
||||
# print "setting config.enabled = True"
|
||||
# config.enabled = True
|
||||
config.debug_print_all()
|
||||
print " -- saving xml and rewriting -----------------------------------------------"
|
||||
config.write(sys.stdout)
|
||||
|
|
@ -9,16 +9,24 @@
|
|||
from testbase import *
|
||||
|
||||
class UserUC(User):
|
||||
def __init__(self):
|
||||
User.__init__(self)
|
||||
self.project_prefs = "<project_specific>\nfoobar\n</project_specific>"
|
||||
self.global_prefs = """<venue name=\"home\">
|
||||
def init(self):
|
||||
self.User.__init__()
|
||||
self.project_prefs = """<project_preferences>
|
||||
<project_specific>
|
||||
foobar
|
||||
</project_specific>
|
||||
</project_preferences>
|
||||
"""
|
||||
self.global_prefs = """<global_preferences>
|
||||
<venue name=\"home\">
|
||||
<work_buf_min_days>0</work_buf_min_days>
|
||||
<work_buf_max_days>2</work_buf_max_days>
|
||||
<disk_interval>1</disk_interval>
|
||||
<run_on_batteries/>
|
||||
<max_bytes_sec_down>400000</max_bytes_sec_down>
|
||||
</venue>"""
|
||||
</venue>
|
||||
</global_preferences>
|
||||
"""
|
||||
|
||||
class WorkUC(Work):
|
||||
def __init__(self, redundancy, **kwargs):
|
||||
|
|
@ -28,7 +36,7 @@ class WorkUC(Work):
|
|||
self.input_files = ['input']
|
||||
self.__dict__.update(kwargs)
|
||||
|
||||
class ResultUC(Result):
|
||||
class ResultUC(ExpectedResult):
|
||||
def __init__(self):
|
||||
self.stderr_out = MATCH_REGEXPS([
|
||||
"<stderr_txt>",
|
||||
|
|
@ -36,7 +44,7 @@ class ResultUC(Result):
|
|||
"APP: upper_case: argv[[]0[]] is upper_case",
|
||||
"APP: upper_case ending, wrote \\d+ chars"])
|
||||
|
||||
class ResultComputeErrorUC(ResultComputeError):
|
||||
class ResultComputeErrorUC(ExpectedResultComputeError):
|
||||
def __init__(self):
|
||||
self.stderr_out = MATCH_REGEXPS([ """<stderr_txt>
|
||||
APP: upper_case: starting, argc \\d+"""])
|
||||
|
|
|
|||
161
test/testbase.py
161
test/testbase.py
|
|
@ -19,15 +19,16 @@ import cgiserver
|
|||
|
||||
options.have_init_t = False
|
||||
options.echo_overwrite = False
|
||||
options.client_bin_filename = version.CLIENT_BIN_FILENAME
|
||||
|
||||
def test_init():
|
||||
if options.have_init_t: return
|
||||
options.have_init_t = True
|
||||
|
||||
if not os.path.exists('test_uc.py'):
|
||||
if not os.path.exists('testbase.py'):
|
||||
os.chdir(os.path.join(boinc_path_config.TOP_SOURCE_DIR,'test'))
|
||||
if not os.path.exists('test_uc.py'):
|
||||
raise SystemExit('Could not find boinc_db.py anywhere')
|
||||
if not os.path.exists('testbase.py'):
|
||||
raise SystemExit('Could not find testbase.py anywhere')
|
||||
|
||||
#options.program_path = os.path.realpath(os.path.dirname(sys.argv[0]))
|
||||
options.program_path = os.getcwd()
|
||||
|
|
@ -170,13 +171,13 @@ def dict_match(dic, resultdic):
|
|||
format = "result %s: unexpected %s '%s' (expected '%s')"
|
||||
error( format % (id, key, found, expected))
|
||||
|
||||
class Result:
|
||||
class ExpectedResult:
|
||||
def __init__(self):
|
||||
self.server_state = RESULT_SERVER_STATE_OVER
|
||||
self.client_state = RESULT_FILES_UPLOADED
|
||||
self.outcome = RESULT_OUTCOME_SUCCESS
|
||||
|
||||
class ResultComputeError:
|
||||
class ExpectedResultComputeError:
|
||||
def __init__(self):
|
||||
self.server_state = RESULT_SERVER_STATE_OVER
|
||||
self.client_state = RESULT_COMPUTE_DONE
|
||||
|
|
@ -228,24 +229,38 @@ def get_redundancy_args(num_wu = None, redundancy = None):
|
|||
return (num_wu, redundancy)
|
||||
|
||||
class TestProject(Project):
|
||||
def __init__(self, works, expected_result,
|
||||
def __init__(self, works, expected_result, appname=None,
|
||||
num_wu=None, redundancy=None,
|
||||
users=None, hosts=None,
|
||||
add_to_list=True,
|
||||
apps=None, app_versions=None, core_versions=None,
|
||||
resource_share=None,
|
||||
**kwargs):
|
||||
test_init()
|
||||
if add_to_list:
|
||||
all_projects.append(self)
|
||||
|
||||
kwargs['short_name'] = kwargs.get('short_name') or 'test_'+kwargs['appname']
|
||||
kwargs['short_name'] = kwargs.get('short_name') or 'test_'+appname
|
||||
kwargs['long_name'] = kwargs.get('long_name') or 'Project ' + kwargs['short_name'].replace('_',' ').capitalize()
|
||||
(num_wu, redundancy) = get_redundancy_args(num_wu, redundancy)
|
||||
self.resource_share = resource_share or 1
|
||||
self.num_wu = num_wu
|
||||
self.redundancy = redundancy
|
||||
self.expected_result = expected_result
|
||||
self.works = works
|
||||
self.users = users or [User()]
|
||||
self.hosts = hosts or [Host()]
|
||||
|
||||
self.platforms = [Platform()]
|
||||
self.core_versions = core_versions or [CoreVersion(self.platforms[0])]
|
||||
self.app_versions = app_versions or [AppVersion(App(appname),
|
||||
self.platforms[0],
|
||||
appname)]
|
||||
self.apps = apps or unique(map(lambda av: av.app, self.app_versions))
|
||||
# convenience vars:
|
||||
self.app_version = self.app_versions[0]
|
||||
self.app = self.apps[0]
|
||||
|
||||
# convenience vars:
|
||||
self.work = self.works[0]
|
||||
self.user = self.users[0]
|
||||
|
|
@ -263,28 +278,6 @@ class TestProject(Project):
|
|||
'''Overrides Project::query_create_keys() to always return true'''
|
||||
return True
|
||||
|
||||
def install_project_users(self):
|
||||
db = self.db_open()
|
||||
verbose_echo(1, "Setting up database: adding %d user(s)" % len(self.users))
|
||||
for user in self.users:
|
||||
if user.project_prefs:
|
||||
pp = "<project_preferences>\n%s\n</project_preferences>\n" % user.project_prefs
|
||||
else:
|
||||
pp = ''
|
||||
if user.global_prefs:
|
||||
gp = "<global_preferences>\n%s\n</global_preferences>\n" % user.global_prefs
|
||||
else:
|
||||
gp = ''
|
||||
|
||||
db.query(("insert into user values (0, %d, '%s', '%s', '%s', " +
|
||||
"'Peru', '12345', 0, 0, 0, '%s', '%s', 0, 'home', '', 0, 1, 0)") % (
|
||||
time.time(),
|
||||
user.email_addr,
|
||||
user.name,
|
||||
user.authenticator,
|
||||
gp,
|
||||
pp))
|
||||
|
||||
def install_works(self):
|
||||
for work in self.works:
|
||||
work.install(self)
|
||||
|
|
@ -295,10 +288,30 @@ class TestProject(Project):
|
|||
host.add_user(user, self)
|
||||
host.install()
|
||||
|
||||
def install_platforms_versions(self):
|
||||
def commit(list):
|
||||
for item in list: item.commit()
|
||||
|
||||
self.platforms = unique(map(lambda a: a.platform, self.app_versions))
|
||||
verbose_echo(1, "Setting up database: adding %d platform(s)" % len(self.platforms))
|
||||
commit(self.platforms)
|
||||
|
||||
verbose_echo(1, "Setting up database: adding %d core version(s)" % len(self.core_versions))
|
||||
commit(self.core_versions)
|
||||
|
||||
verbose_echo(1, "Setting up database: adding %d apps(s)" % len(self.apps))
|
||||
commit(self.apps)
|
||||
|
||||
verbose_echo(1, "Setting up database: adding %d app version(s)" % len(self.app_versions))
|
||||
commit(self.app_versions)
|
||||
|
||||
verbose_echo(1, "Setting up database: adding %d user(s)" % len(self.users))
|
||||
commit(self.users)
|
||||
|
||||
def install(self):
|
||||
self.init_install()
|
||||
self.install_project()
|
||||
self.install_project_users()
|
||||
self.install_platforms_versions()
|
||||
self.install_works()
|
||||
self.install_hosts()
|
||||
|
||||
|
|
@ -318,14 +331,12 @@ class TestProject(Project):
|
|||
|
||||
If more than X seconds have passed than just assume something is broken and return.'''
|
||||
|
||||
db = self.db_open()
|
||||
timeout = time.time() + 3*60
|
||||
while (num_wus_assimilated(db) < self.num_wu) or num_wus_to_transition(db):
|
||||
while (num_wus_assimilated() < self.num_wu) or num_wus_to_transition():
|
||||
time.sleep(.5)
|
||||
if time.time() > timeout:
|
||||
error("run_finish_wait(): timed out waiting for workunits to assimilate/transition")
|
||||
break
|
||||
db.close()
|
||||
|
||||
def check(self):
|
||||
# verbose_sleep("Sleeping to allow server daemons to finish", 5)
|
||||
|
|
@ -337,17 +348,16 @@ class TestProject(Project):
|
|||
# self.check_deleted("upload/uc_wu_%d_0", count=self.num_wu)
|
||||
|
||||
def progress_meter_ctor(self):
|
||||
self.db = self.db_open()
|
||||
pass
|
||||
def progress_meter_status(self):
|
||||
return "WUs: [%dassim/%dtotal/%dtarget] Results: [%dunsent,%dinProg,%dover/%dtotal]" % (
|
||||
num_wus_assimilated(self.db), num_wus(self.db), self.num_wu,
|
||||
num_results_unsent(self.db),
|
||||
num_results_in_progress(self.db),
|
||||
num_results_over(self.db),
|
||||
num_results(self.db))
|
||||
num_wus_assimilated(), num_wus(), self.num_wu,
|
||||
num_results_unsent(),
|
||||
num_results_in_progress(),
|
||||
num_results_over(),
|
||||
num_results())
|
||||
def progress_meter_dtor(self):
|
||||
self.db.close()
|
||||
self.db = None
|
||||
pass
|
||||
|
||||
def _disable(self, *path):
|
||||
'''Temporarily disable a file to test exponential backoff'''
|
||||
|
|
@ -382,13 +392,11 @@ class TestProject(Project):
|
|||
exit_status
|
||||
'''
|
||||
expected_count = expected_count or self.redundancy
|
||||
db = self.db_open()
|
||||
rows = db_query(db,"select * from result")
|
||||
for row in rows:
|
||||
dict_match(matchresult, row)
|
||||
db.close()
|
||||
if len(rows) != expected_count:
|
||||
error("expected %d results, but found %d" % (expected_count, len(rows)))
|
||||
results = database.Results.find()
|
||||
for result in results:
|
||||
dict_match(matchresult, result.__dict__)
|
||||
if len(results) != expected_count:
|
||||
error("expected %d results, but found %d" % (expected_count, len(results)))
|
||||
|
||||
def check_files_match(self, result, correct, count=None):
|
||||
'''if COUNT is specified then [0,COUNT) is mapped onto the %d in RESULT'''
|
||||
|
|
@ -414,16 +422,47 @@ class TestProject(Project):
|
|||
return errs
|
||||
return check_exists(self.dir(file))
|
||||
|
||||
class Platform(database.Platform):
|
||||
def __init__(self, name=None, user_friendly_name=None):
|
||||
database.Platform.__init__(self)
|
||||
self.name = name or version.PLATFORM
|
||||
self.user_friendly_name = user_friendly_name or name
|
||||
|
||||
class CoreVersion(database.CoreVersion):
|
||||
def __init__(self, platform):
|
||||
database.CoreVersion.__init__(self)
|
||||
self.version_num = 1
|
||||
self.platform = platform
|
||||
self.xml_doc = tools.process_executable_file(
|
||||
os.path.join(boinc_path_config.TOP_BUILD_DIR,'client',
|
||||
options.client_bin_filename),
|
||||
quiet=True)
|
||||
|
||||
class User:
|
||||
'''represents an account on a particular project'''
|
||||
class User(database.User):
|
||||
def __init__(self):
|
||||
database.User.__init__(self,id=None)
|
||||
self.name = 'John'
|
||||
self.email_addr = 'john@boinc.org'
|
||||
self.authenticator = "3f7b90793a0175ad0bda68684e8bd136"
|
||||
self.project_prefs = None
|
||||
self.global_prefs = None
|
||||
|
||||
class App(database.App):
|
||||
def __init__(self, name):
|
||||
database.App.__init__(self,id=None)
|
||||
self.name = name
|
||||
self.min_version = 1
|
||||
|
||||
class AppVersion(database.AppVersion):
|
||||
def __init__(self, app, platform, exec_file):
|
||||
database.AppVersion.__init__(self,id=None)
|
||||
self.app = app
|
||||
self.version_num = 1
|
||||
self.platform = platform
|
||||
self.xml_doc = tools.process_app_version(
|
||||
app, self.version_num,
|
||||
[os.path.join(boinc_path_config.TOP_BUILD_DIR,'apps',exec_file)],
|
||||
quiet=True)
|
||||
self.min_core_version = 1
|
||||
self.max_core_version = 999
|
||||
|
||||
class HostList(list):
|
||||
def run(self, asynch=False): map(lambda i: i.run(asynch=asynch), self)
|
||||
|
|
@ -457,12 +496,12 @@ class Host:
|
|||
|
||||
verbose_echo(1, "Setting up host '%s': creating account files" % self.name);
|
||||
for (user,project) in map(None,self.users,self.projects):
|
||||
filename = self.dir(account_file_name(project.master_url))
|
||||
filename = self.dir(account_file_name(project.config.config.master_url))
|
||||
verbose_echo(2, "Setting up host '%s': writing %s" % (self.name, filename))
|
||||
|
||||
f = open(filename, "w")
|
||||
print >>f, "<account>"
|
||||
print >>f, map_xml(project, ['master_url'])
|
||||
print >>f, map_xml(project.config.config, ['master_url'])
|
||||
print >>f, map_xml(user, ['authenticator'])
|
||||
if user.project_prefs:
|
||||
print >>f, user.project_prefs
|
||||
|
|
@ -509,6 +548,7 @@ class Host:
|
|||
_url_to_filename(project.master_url),
|
||||
filename))
|
||||
|
||||
# TODO: do this in Python
|
||||
class Work:
|
||||
def __init__(self, redundancy, **kwargs):
|
||||
self.input_files = []
|
||||
|
|
@ -534,7 +574,8 @@ class Work:
|
|||
self.app = project.app_versions[0].app
|
||||
for input_file in unique(self.input_files):
|
||||
install(os.path.realpath(input_file),
|
||||
os.path.join(project.download_dir,os.path.basename(input_file)))
|
||||
os.path.join(project.config.config.download_dir,
|
||||
os.path.basename(input_file)))
|
||||
|
||||
# simulate multiple data servers by making symbolic links to the
|
||||
# download directory
|
||||
|
|
@ -558,11 +599,11 @@ class Work:
|
|||
os.symlink(handler, newhandler)
|
||||
|
||||
cmd = build_command_line("create_work",
|
||||
db_name = project.db_name,
|
||||
download_dir = project.download_dir,
|
||||
upload_url = project.upload_url,
|
||||
download_url = project.download_url,
|
||||
keyfile = os.path.join(project.key_dir,'upload_private'),
|
||||
db_name = project.config.config.db_name,
|
||||
download_dir = project.config.config.download_dir,
|
||||
upload_url = project.config.config.upload_url,
|
||||
download_url = project.config.config.download_url,
|
||||
keyfile = os.path.join(project.config.config.key_dir,'upload_private'),
|
||||
appname = self.app.name,
|
||||
rsc_fpops_est = self.rsc_fpops_est,
|
||||
rsc_fpops_bound = self.rsc_fpops_bound,
|
||||
|
|
|
|||
|
|
@ -2,6 +2,9 @@
|
|||
|
||||
# $Id$
|
||||
|
||||
|
||||
#XXX TODO: add app should modify config.xml to add application-specific daemons
|
||||
|
||||
'''
|
||||
add items to the BOINC database.
|
||||
|
||||
|
|
|
|||
|
|
@ -8,17 +8,27 @@
|
|||
# TODO: create 'apps' subdirectory and set it in config.xml
|
||||
# TODO: use configxml module
|
||||
|
||||
import sys, os, getopt, re
|
||||
sys.path.append('../py')
|
||||
from version import *
|
||||
from boinc import *
|
||||
import boinc_path_config
|
||||
from Boinc.setup_project import *
|
||||
from Boinc import database, db_mid, configxml, tools
|
||||
import sys, os, getopt, re, socket
|
||||
|
||||
def gethostname():
|
||||
try:
|
||||
return socket.gethostbyaddr(socket.gethostname())[0]
|
||||
except:
|
||||
return 'localhost'
|
||||
|
||||
def isurl(s):
|
||||
return s.startswith('http://') or s.startswith('https://')
|
||||
|
||||
argv0 = sys.argv[0]
|
||||
HOME = os.path.expanduser('~')
|
||||
USER = os.environ['USER']
|
||||
NODENAME = gethostname()
|
||||
|
||||
HELP = """
|
||||
syntax: %(argv0)s [options] project-dir-name 'Project Long Name' app-exe 'AppName'
|
||||
syntax: %(argv0)s [options] project ['Project Long Name']
|
||||
|
||||
Creates a new project with given name with everything running on a single
|
||||
server.
|
||||
|
|
@ -35,15 +45,17 @@ Dir-options:
|
|||
--base default: $HOME (%(HOME)s)
|
||||
--key_dir default: BASE/keys
|
||||
--project_root default: BASE/projects/PROJECT
|
||||
--url_base REQUIRED; e.g.: http://maggie.ssl.berkeley.edu/
|
||||
--url_base default: http://$NODENAME/ (http://%(NODENAME)s/)
|
||||
|
||||
--html_user_url default: URL_BASE/PROJECT/
|
||||
--html_ops_url default: URL_BASE/PROJECT_ops/
|
||||
--cgi_url default: URL_BASE/PROJECT_cgi/
|
||||
|
||||
Other:
|
||||
--db_name default: PROJECT
|
||||
|
||||
Example command line:
|
||||
./make_project --base $HOME/boinc --url_base http://boink/ yah 'YETI @ Home' \\
|
||||
upper_case 'UpperCase'
|
||||
./make_project --base $HOME/boinc --url_base http://boink/ yah 'YETI @ Home'
|
||||
|
||||
Then upload_dir = $HOME/boinc/projects/yah/upload
|
||||
and cgi_url = http://boink/yah_cgi/
|
||||
|
|
@ -104,7 +116,7 @@ options.delete_prev_inst = False
|
|||
|
||||
for o,a in opts:
|
||||
if o == '-h' or o == '--help': usage()
|
||||
elif o == '-v': options.echo_verbose = 2
|
||||
elif o == '-v': options.echo_verbose = 2
|
||||
elif o == '--verbose': options.echo_verbose = int(a)
|
||||
elif o == '--no_query': options.no_query = True
|
||||
elif o == '--user_name': options.user_name = a
|
||||
|
|
@ -117,6 +129,7 @@ for o,a in opts:
|
|||
elif o == '--html_user_url': options.html_user_url = a
|
||||
elif o == '--html_ops_url': options.html_ops_url = a
|
||||
elif o == '--cgi_url': options.cgi_url = a
|
||||
elif o == '--db_name': options.db_name = a
|
||||
# elif o == '--bin_dir': options.bin_dir = a
|
||||
# elif o == '--cgi_bin_dir': options.cgi_bin_dir = a
|
||||
# elif o == '--html_user_dir': options.html_user_dir = a
|
||||
|
|
@ -128,43 +141,45 @@ for o,a in opts:
|
|||
else:
|
||||
raise SystemExit('internal error o=%s'%o)
|
||||
|
||||
if len(args) != 4:
|
||||
syntax_error('Need four arguments')
|
||||
|
||||
(project_shortname, project_longname, app_exe, app_name) = args
|
||||
|
||||
if not options.url_base:
|
||||
syntax_error('Need --url_base')
|
||||
|
||||
if not options.url_base.startswith('http://'):
|
||||
syntax_error('url_base needs to be an URL')
|
||||
|
||||
options.url_base = os.path.join(options.url_base, '')
|
||||
if len(args) == 2:
|
||||
(project_shortname, project_longname) = args
|
||||
elif len(args) == 1:
|
||||
(project_shortname, project_longname) = args[0], args[0].capitalize()
|
||||
else:
|
||||
syntax_error('Need one or two arguments')
|
||||
|
||||
opt_repls = {'PROJECT':project_shortname,
|
||||
'PROJECT_ops':project_shortname+'_ops',
|
||||
'PROJECT_cgi':project_shortname+'_cgi',
|
||||
'URL_BASE':options.url_base}
|
||||
def replopt(str):
|
||||
for key in opt_repls:
|
||||
str = re.compile('\\b'+key+'\\b').sub(os.path.join(opt_repls[key],'')[:-1], str)
|
||||
return str
|
||||
'PROJECT_cgi':project_shortname+'_cgi'}
|
||||
def delete_slash(str):
|
||||
return os.path.join(str,'')[:-1]
|
||||
def add_slash(str, action=True):
|
||||
if action:
|
||||
return os.path.join(str,'')
|
||||
else:
|
||||
return str
|
||||
def replopt(str):
|
||||
for key in opt_repls:
|
||||
str = str.replace(key, delete_slash(opt_repls[key]))
|
||||
return str
|
||||
def defopt(name, v, isdir=True):
|
||||
options.__dict__[name] = opt_repls[name.upper()] = add_slash(replopt(options.__dict__.get(name) or v), isdir)
|
||||
|
||||
defopt('user_name', USER, isdir=False)
|
||||
defopt('base', HOME)
|
||||
defopt('key_dir', 'BASE/keys')
|
||||
defopt('project_root', 'BASE/projects/PROJECT')
|
||||
defopt('url_base' , 'http://%s/'%NODENAME)
|
||||
|
||||
defopt('html_user_url', 'URL_BASE/PROJECT')
|
||||
defopt('html_ops_url', 'URL_BASE/PROJECT_ops')
|
||||
defopt('cgi_url', 'URL_BASE/PROJECT_cgi')
|
||||
if not isurl(options.url_base):
|
||||
syntax_error('url_base needs to be an URL')
|
||||
|
||||
defopt('html_user_url' , 'URL_BASE/PROJECT')
|
||||
defopt('html_ops_url' , 'URL_BASE/PROJECT_ops')
|
||||
defopt('cgi_url' , 'URL_BASE/PROJECT_cgi')
|
||||
|
||||
defopt('user_name' , USER, isdir=False)
|
||||
defopt('base' , HOME)
|
||||
defopt('key_dir' , 'BASE/keys')
|
||||
defopt('project_root' , 'BASE/projects/PROJECT')
|
||||
|
||||
defopt('db_name' , 'PROJECT', isdir=False)
|
||||
|
||||
print "Creating project '%s' (short name '%s'):" %(project_longname, project_shortname)
|
||||
for k in ['base',
|
||||
|
|
@ -185,6 +200,7 @@ if os.path.exists(options.project_root):
|
|||
if not options.no_query:
|
||||
if not query_noyes('Delete %s?'%options.project_root):
|
||||
raise SystemExit('Aborted')
|
||||
print "Deleting", options.project_root
|
||||
rmtree(options.project_root)
|
||||
else:
|
||||
raise SystemExit('Project root already exists! Specify --delete_prev_inst --drop_db_first to clobber')
|
||||
|
|
@ -193,34 +209,38 @@ if not options.no_query:
|
|||
if not query_yesno("Continue?"):
|
||||
raise SystemExit('Aborted')
|
||||
|
||||
app = App(app_name)
|
||||
app_version = AppVersion(app, exec_names=[app_exe])
|
||||
|
||||
options.install_method = 'copy'
|
||||
init()
|
||||
project = Project(project_shortname, project_longname,
|
||||
project_dir = options.project_root,
|
||||
master_url = options.url_base,
|
||||
master_url = options.html_user_url,
|
||||
cgi_url = options.cgi_url,
|
||||
key_dir = options.key_dir,
|
||||
apps=[app], app_versions=[app_version],
|
||||
db_name = options.db_name
|
||||
)
|
||||
|
||||
project.install_project()
|
||||
|
||||
project.sched_install('feeder')
|
||||
project.sched_install('transitioner')
|
||||
project.sched_install('validate_test')
|
||||
project.sched_install('assimilator')
|
||||
project.sched_install('file_deleter')
|
||||
# project.sched_install('feeder')
|
||||
# project.sched_install('transitioner')
|
||||
# project.sched_install('validate_test')
|
||||
# project.sched_install('assimilator')
|
||||
# project.sched_install('file_deleter')
|
||||
|
||||
print '''Done installing files.
|
||||
httpd_conf_template_filename = os.path.join(options.project_root,
|
||||
project_shortname+'.httpd.conf')
|
||||
|
||||
You need to manually edit your Apache httpd.conf to add these lines:
|
||||
proot = delete_slash(options.project_root)
|
||||
html_user_url = options.html_user_url
|
||||
html_ops_url = options.html_ops_url
|
||||
|
||||
Alias /%(project)s %(proot)s/html_user
|
||||
Alias /%(project)s_ops %(proot)s/html_ops
|
||||
ScriptAlias /%(project)s_cgi %(proot)s/cgi-bin
|
||||
print >>open(httpd_conf_template_filename,'w'), '''
|
||||
|
||||
## Settings for BOINC project %(project_longname)s
|
||||
|
||||
Alias /%(project_shortname)s %(proot)s/html_user
|
||||
Alias /%(project_shortname)s_ops %(proot)s/html_ops
|
||||
ScriptAlias /%(project_shortname)s_cgi %(proot)s/cgi-bin
|
||||
|
||||
# Note: projects/*/keys/ should NOT be readable!
|
||||
|
||||
|
|
@ -243,19 +263,47 @@ You need to manually edit your Apache httpd.conf to add these lines:
|
|||
Order allow,deny
|
||||
Allow from all
|
||||
</Directory>
|
||||
''' %locals()
|
||||
|
||||
Install this in crontab:
|
||||
print '''Done installing files.
|
||||
|
||||
Steps to complete installation:
|
||||
|
||||
1. Set permissions for Apache:
|
||||
|
||||
cat %(httpd_conf_template_filename)s >> /etc/apache/httpd.conf && apachectl restart
|
||||
|
||||
# (path to httpd.conf varies)
|
||||
|
||||
2. Add to crontab (as %(USER)s)
|
||||
(If cron cannot run "start", try using a helper script to set PATH and
|
||||
PYTHONPATH)
|
||||
|
||||
0,5,10,15,20,25,30,35,40,45,50,55 * * * * %(proot)s/bin/start --cron
|
||||
|
||||
(You may need to set PATH and/or PYTHONPATH since cron runs with manilla environment)
|
||||
|
||||
To start, show status, stop BOINC daemons run:
|
||||
|
||||
%(proot)s/bin/start
|
||||
%(proot)s/bin/status
|
||||
%(proot)s/bin/stop
|
||||
|
||||
'''%{'project':project_shortname, 'proot':os.path.join(options.project_root,'')[:-1]}
|
||||
Master URL: %(html_user_url)s
|
||||
Administration URL: %(html_ops_url)s
|
||||
|
||||
## TODO: save settings to a file.
|
||||
Tasks to do:
|
||||
|
||||
1. Add platform(s)
|
||||
%(proot)s/bin/add platform --name=c64 --user_f="Commodore 64"
|
||||
%(proot)s/bin/add platform --name=i686-pc-linux-gnu --user_f="Linux x86"
|
||||
|
||||
2. Add application(s)
|
||||
%(proot)s/bin/add app --name=SpaghettiAtHome
|
||||
|
||||
3. Add core client and application binaries
|
||||
|
||||
3a. Place compiled clients in %(proot)s/apps/boinc/ and %(proot)s/apps/APP/
|
||||
3b. Run %(proot)s/bin/update_versions
|
||||
|
||||
4. Generate work : read documentation at http://boinc.berkeley.edu/
|
||||
|
||||
'''%locals()
|
||||
|
|
|
|||
|
|
@ -36,11 +36,6 @@ assert(config.app_dir
|
|||
config.download_url
|
||||
)
|
||||
|
||||
def query_yesno(msg):
|
||||
'''Query y/n; default Y'''
|
||||
print msg, "[Y/n]? ",
|
||||
return not raw_input().lower().startswith('n')
|
||||
|
||||
objects_to_commit = []
|
||||
|
||||
def xsort(list):
|
||||
|
|
@ -131,7 +126,7 @@ print "Commit %d items:" %len(objects_to_commit)
|
|||
for object in objects_to_commit:
|
||||
print " ", object
|
||||
|
||||
if not query_yesno("Continue"):
|
||||
if not tools.query_yesno("Continue"):
|
||||
raise SystemExit
|
||||
|
||||
for object in objects_to_commit:
|
||||
|
|
|
|||
|
|
@ -0,0 +1,27 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
# $Id$
|
||||
|
||||
'''
|
||||
This program upgrades the project (created using make_project) with current
|
||||
source/build files. Useful if you are a BOINC developer or closely following
|
||||
BOINC development.
|
||||
'''
|
||||
|
||||
import boinc_path_config
|
||||
from Boinc import boinc_project_path
|
||||
from Boinc.setup_project import *
|
||||
import os
|
||||
|
||||
if os.system(os.path.join(boinc_project_path.PROGRAM_PARENT_DIR,'bin/stop')):
|
||||
raise SystemExit("Couldn't stop BOINC!")
|
||||
|
||||
print "Upgrading files... "
|
||||
|
||||
options.install_method = 'copy'
|
||||
init()
|
||||
install_boinc_files(boinc_project_path.PROGRAM_PARENT_DIR)
|
||||
|
||||
print "Upgrading files... done"
|
||||
print
|
||||
print "Run `start' to resume BOINC."
|
||||
Loading…
Reference in New Issue