mirror of https://github.com/BOINC/boinc.git
- Add web interface for creating BUDA apps and variants
variants are stored in <project>/buda_apps/<app>/<variant>.
This includes app files (copied from sandbox) and templates
(generated by handler)
- Add web interface for submitting BUDA jobs (not finished)
- Change implementation of user file sandbox
old: sandbox dir had 'link files' containing md5 and size;
actual file is in download hierarchy with sb_md5 name
new: sandbox dir has actual files.
parallel .md5/ dir has 'info files' (md5 size)
Files are not stored in download hierarchy.
New philosophy: names in the download hierarchy include
not only an MD5 (for uniqueness)
but also text describing the use of the file
(input file for a batch, part of a BUDA app, etc.).
This may allow duplicate files,
but it makes it possible to always clean up unused files.
- use readdir() instead of opendir()/scandir()
This commit is contained in:
David Anderson
parent
dd16cab2c8
commit
bc4cb3bb68
|
|
@ -116,8 +116,8 @@ class DbConn {
|
|||
|
||||
function enum_general($classname, $query) {
|
||||
$result = $this->do_query($query);
|
||||
if (!$result) return null;
|
||||
$x = array();
|
||||
if (!$result) return [];
|
||||
$x = [];
|
||||
while ($obj = $result->fetch_object($classname)) {
|
||||
$x[] = $obj;
|
||||
}
|
||||
|
|
@ -133,7 +133,7 @@ class DbConn {
|
|||
$where_clause = "where $where_clause";
|
||||
}
|
||||
$query = "select $fields from DBNAME.$table $where_clause $order_clause";
|
||||
return $this->enum_general($classname,$query);
|
||||
return $this->enum_general($classname, $query);
|
||||
}
|
||||
|
||||
function enum($table, $classname, $where_clause=null, $order_clause=null) {
|
||||
|
|
|
|||
|
|
@ -120,9 +120,7 @@ function check_download_file($path, $dl_path) {
|
|||
}
|
||||
}
|
||||
|
||||
// Stage the given file, which is assumed to not be in download dir already.
|
||||
//
|
||||
function stage_file_basic($dir, $fname) {
|
||||
function get_hier_info() {
|
||||
static $dl_dir = null;
|
||||
static $fanout = null;
|
||||
if (!$dl_dir) {
|
||||
|
|
@ -130,6 +128,13 @@ function stage_file_basic($dir, $fname) {
|
|||
$dl_dir = parse_config($conf, "<download_dir>");
|
||||
$fanout = parse_config($conf, "<uldl_dir_fanout>");
|
||||
}
|
||||
return [$dl_dir, $fanout];
|
||||
}
|
||||
|
||||
// Stage the given file, which is assumed to not be in download dir already.
|
||||
//
|
||||
function stage_file_basic($dir, $fname) {
|
||||
[$dl_dir, $fanout] = get_hier_info();
|
||||
$old_path = "$dir/$fname";
|
||||
$new_path = dir_hier_path($fname, $dl_dir, $fanout);
|
||||
$md5 = md5_file($old_path);
|
||||
|
|
@ -138,4 +143,14 @@ function stage_file_basic($dir, $fname) {
|
|||
rename($old_path, $new_path);
|
||||
}
|
||||
|
||||
// copy the given file (with known size/md5)
|
||||
// to the download dir w/ given phys name
|
||||
//
|
||||
function stage_file_aux($path, $md5, $size, $phys_name) {
|
||||
[$dl_dir, $fanout] = get_hier_info();
|
||||
$phys_path = dir_hier_path($phys_name, $dl_dir, $fanout);
|
||||
copy($path, $phys_path);
|
||||
file_put_contents("$phys_path.md5", "$md5 $size\n");
|
||||
}
|
||||
|
||||
?>
|
||||
|
|
|
|||
|
|
@ -19,14 +19,23 @@
|
|||
// Utility functions for user file sandbox feature
|
||||
//
|
||||
// In this system:
|
||||
// - sandbox files live in the download hierarchy,
|
||||
// with names of the form sb_userid_md5
|
||||
// - each user has a "sandbox dir" project/sandbox/userid.
|
||||
// The files in this have user-visible names, and contents of the form
|
||||
// sb file_size file_md5
|
||||
// - each user (job submitter) has a 'sandbox' where they can store files
|
||||
// on the BOINC server, via a web interface.
|
||||
// These files are mutable; you can modify a file w/ a given name.
|
||||
// - files are stored in a dir project/sandbox/<userid>
|
||||
// - When a file is uploaded, its size and MD5 are computed and stored
|
||||
// in an 'info file' in a parallel dir, project/sandbox/<userid>/.md5
|
||||
//
|
||||
// Sandbox files can be used for web-based job submissions systems
|
||||
// like BUDA and autodock on BOINC Central.
|
||||
// Typically they are used as job input files or app files,
|
||||
// in which case they are downloadable.
|
||||
// When a file is used in this way,
|
||||
// it must be copied to the download hierarchy,
|
||||
// and assigned a physical name that includes its MD5.
|
||||
// The name depends on the role of the file.
|
||||
|
||||
require_once("../inc/util.inc");
|
||||
require_once("../inc/submit_db.inc");
|
||||
require_once("../inc/dir_hier.inc");
|
||||
|
||||
// Return path of sandbox directory for the given user.
|
||||
|
|
@ -44,75 +53,53 @@ function sandbox_dir($user) {
|
|||
if (!is_dir($d)) {
|
||||
mkdir($d);
|
||||
}
|
||||
if (!is_dir("$d/.md5")) {
|
||||
mkdir("$d/.md5");
|
||||
}
|
||||
return $d;
|
||||
}
|
||||
|
||||
function sandbox_write_link_file($path, $size, $md5) {
|
||||
file_put_contents($path, "sb $size $md5");
|
||||
}
|
||||
|
||||
// check if a link with the given md5 exists
|
||||
// parse a sandbox file's info file.
|
||||
// If missing, create it.
|
||||
//
|
||||
function sandbox_lf_exists($user, $md5) {
|
||||
$exist = false;
|
||||
$elf = "";
|
||||
function sandbox_parse_info_file($user, $name) {
|
||||
$dir = sandbox_dir($user);
|
||||
$files = sandbox_file_names($user);
|
||||
foreach ($files as $file) {
|
||||
$path = "$dir/$file";
|
||||
[$err, $file_size, $file_md5] = sandbox_parse_link_file($path);
|
||||
if (!$err){
|
||||
if (strcmp($md5, $file_md5) == 0) {
|
||||
$exist = true;
|
||||
$elf = $file;
|
||||
break;
|
||||
}
|
||||
}
|
||||
$info_path = "$dir/.md5/$name";
|
||||
$info = parse_info_file($info_path);
|
||||
if ($info) {
|
||||
return $info;
|
||||
}
|
||||
return array($exist, $elf);
|
||||
}
|
||||
|
||||
// parse a link file and return (error, size, md5)
|
||||
//
|
||||
function sandbox_parse_link_file($path) {
|
||||
if (!file_exists($path)) { return array(true, null, null); }
|
||||
$x = file_get_contents($path);
|
||||
$n = sscanf($x, "%s %d %s", $s, $size, $md5);
|
||||
if ($n != 3) return array(true, null, null);
|
||||
if ($s != 'sb') return array(true, null, null);
|
||||
return array(false, $size, $md5);
|
||||
}
|
||||
|
||||
$fanout = parse_config(get_config(), "<uldl_dir_fanout>");
|
||||
|
||||
// return the physical name of the file
|
||||
//
|
||||
function sandbox_file_name($user, $md5) {
|
||||
return "sb_".$user->id."_".$md5;
|
||||
}
|
||||
|
||||
// return the path of the file in the download directory
|
||||
//
|
||||
function sandbox_physical_path($user, $md5) {
|
||||
global $fanout;
|
||||
$f = sandbox_file_name($user, $md5);
|
||||
return dir_hier_path($f, parse_config(get_config(), "<download_dir>"), $fanout);
|
||||
[$size, $md5] = get_file_info("$dir/$name");
|
||||
write_info_file($info_path, $md5, $size);
|
||||
return [$md5, $size];
|
||||
}
|
||||
|
||||
// return list of files in sandbox
|
||||
//
|
||||
function sandbox_file_names($user) {
|
||||
$d = opendir(sandbox_dir($user));
|
||||
$files = scandir(sandbox_dir($user));
|
||||
$names = array();
|
||||
while (($f = readdir($d)) !== false) {
|
||||
if ($f == '.') continue;
|
||||
if ($f == '..') continue;
|
||||
foreach ($files as $f) {
|
||||
if ($f[0] == '.') continue;
|
||||
$names[] = $f;
|
||||
}
|
||||
natsort($names);
|
||||
return $names;
|
||||
}
|
||||
|
||||
// return list of files matching given pattern,
|
||||
// in the format used for form_select() and form_select_multiple()
|
||||
//
|
||||
function sandbox_select_items($user, $pattern=null) {
|
||||
$sbfiles = sandbox_file_names($user);
|
||||
$sbitems = [];
|
||||
foreach ($sbfiles as $f) {
|
||||
if ($pattern && !preg_match($pattern, $f)) continue;
|
||||
$sbitems[] = [$f, $f];
|
||||
}
|
||||
return $sbitems;
|
||||
}
|
||||
|
||||
// return a <select> for files in sandbox
|
||||
//
|
||||
function sandbox_file_select(
|
||||
|
|
@ -131,66 +118,13 @@ function sandbox_file_select(
|
|||
return $x;
|
||||
}
|
||||
|
||||
// convert sandbox (link) name to physical path
|
||||
// copy file and info file from sandbox to $dir
|
||||
// (which must have a subdir .md5/)
|
||||
//
|
||||
function sandbox_log_to_phys($user, $log_name) {
|
||||
$dir = sandbox_dir($user);
|
||||
[$err, $file_size, $file_md5] = sandbox_parse_link_file("$dir/$log_name");
|
||||
if ($err) return null;
|
||||
return sandbox_physical_path($user, $file_md5);
|
||||
}
|
||||
|
||||
// convert sandbox name to physical name
|
||||
//
|
||||
function sandbox_name_to_phys_name($user, $log_name) {
|
||||
$dir = sandbox_dir($user);
|
||||
[$err, $file_size, $file_md5] = sandbox_parse_link_file("$dir/$log_name");
|
||||
if ($err) return null;
|
||||
return sandbox_file_name($user, $file_md5);
|
||||
}
|
||||
|
||||
// check if a file is still being used by a unfinished batch
|
||||
//
|
||||
// TODO: this is a kludge.
|
||||
// Should we use the job_file and batch_file_assoc tables instead?
|
||||
//
|
||||
function sandbox_file_in_use($user, $file){
|
||||
$ufiles = array();
|
||||
|
||||
$pbatches = BoincBatch::enum(
|
||||
sprintf('user_id=%d and state!=%d and state!=%d',
|
||||
$user->id, BATCH_STATE_COMPLETE, BATCH_STATE_ABORTED
|
||||
)
|
||||
);
|
||||
if (!$pbatches) return false;
|
||||
|
||||
foreach ($pbatches as $batch){
|
||||
$wus = BoincWorkUnit::enum("batch = $batch->id limit 1" );
|
||||
if ($wus == null){
|
||||
continue;
|
||||
}
|
||||
foreach($wus as $wu){
|
||||
$x = "<in>".$wu->xml_doc."</in>";
|
||||
$x = simplexml_load_string($x);
|
||||
global $fanout;
|
||||
foreach($x->workunit->file_ref as $fr){
|
||||
$pname = (string)$fr->file_name;
|
||||
$ufiles[] = $pname;
|
||||
}
|
||||
}
|
||||
}
|
||||
$dir = sandbox_dir($user);
|
||||
$path = $dir."/".$file;
|
||||
list($err, $size, $md5) = sandbox_parse_link_file($path);
|
||||
if (!$err){
|
||||
$f = sandbox_file_name($user, $md5);
|
||||
foreach($ufiles as $uf) {
|
||||
if (strcmp($f,$uf) == 0){
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
function copy_sandbox_file($user, $fname, $dir) {
|
||||
$sbdir = sandbox_dir($user);
|
||||
copy("$sbdir/$fname", "$dir/$fname");
|
||||
copy("$sbdir/.md5/$fname", "$dir/.md5/$fname");
|
||||
}
|
||||
|
||||
?>
|
||||
|
|
|
|||
|
|
@ -288,4 +288,40 @@ function boinc_get_wu_output_files_url($user, $wu_id) {
|
|||
return "get_output.php?cmd=workunit_files&wu_id=$wu_id&auth_str=$auth_str";
|
||||
}
|
||||
|
||||
////////////////// FILE INFO FILES //////////////
|
||||
|
||||
// these are used:
|
||||
// 1) in user file sandbox
|
||||
// 2) in BUDA app variant dirs
|
||||
// in each case a file dir/foo has an info file dir/.md5/foo
|
||||
// containing its md5 and size
|
||||
// (same format as .md5 files in download hierarchy)
|
||||
|
||||
// get the MD5 and size of a file
|
||||
//
|
||||
function get_file_info($path) {
|
||||
$md5 = md5_file($path);
|
||||
$s = stat($path);
|
||||
$size = $s['size'];
|
||||
return [$md5, $size];
|
||||
}
|
||||
|
||||
// write a "info file" containing MD5 and size
|
||||
//
|
||||
function write_info_file($path, $md5, $size) {
|
||||
file_put_contents($path, "$md5 $size");
|
||||
}
|
||||
|
||||
// parse info file and return [md5, size]
|
||||
//
|
||||
function parse_info_file($path) {
|
||||
if (!file_exists($path)) return null;
|
||||
$x = file_get_contents($path);
|
||||
$n = sscanf($x, "%s %d", $md5, $size);
|
||||
if ($n != 2 || strlen($md5)!=32) {
|
||||
return null;
|
||||
}
|
||||
return [$md5, $size];
|
||||
}
|
||||
|
||||
?>
|
||||
|
|
|
|||
|
|
@ -1133,10 +1133,8 @@ function credit_to_gflop_hours($c) {
|
|||
return $c/(200/24);
|
||||
}
|
||||
|
||||
function do_download($path,$name="") {
|
||||
if (strcmp($name,"") == 0) {
|
||||
$name=basename($path);
|
||||
}
|
||||
function do_download($path) {
|
||||
$name=basename($path);
|
||||
header('Content-Description: File Transfer');
|
||||
header('Content-Type: application/octet-stream');
|
||||
header('Content-Disposition: attachment; filename='.$name);
|
||||
|
|
|
|||
|
|
@ -0,0 +1,330 @@
|
|||
<?php
|
||||
// This file is part of BOINC.
|
||||
// https://boinc.berkeley.edu
|
||||
// Copyright (C) 2024 University of California
|
||||
//
|
||||
// BOINC is free software; you can redistribute it and/or modify it
|
||||
// under the terms of the GNU Lesser General Public License
|
||||
// as published by the Free Software Foundation,
|
||||
// either version 3 of the License, or (at your option) any later version.
|
||||
//
|
||||
// BOINC is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
|
||||
// See the GNU Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with BOINC. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
// web interface for managing BUDA science apps
|
||||
//
|
||||
// in the following, 'app' means BUDA science app
|
||||
// and 'variant' means a variant of one of these
|
||||
//
|
||||
// files can have 3 names
|
||||
// - sandbox name
|
||||
// - physical name (e.g. sb_<md5>)
|
||||
// - logical name (how the science app refers to it)
|
||||
|
||||
require_once('../inc/util.inc');
|
||||
require_once('../inc/sandbox.inc');
|
||||
require_once('../inc/submit_util.inc');
|
||||
|
||||
display_errors();
|
||||
|
||||
$buda_root = "../../buda_apps";
|
||||
|
||||
// show list of BUDA apps and variants,
|
||||
// w/ buttons for adding and deleting
|
||||
//
|
||||
function app_list($notice=null) {
|
||||
global $buda_root;
|
||||
if (!is_dir($buda_root)) {
|
||||
mkdir($buda_root);
|
||||
}
|
||||
page_head('Docker apps');
|
||||
if ($notice) {
|
||||
echo "$notice <p>\n";
|
||||
}
|
||||
$dirs = scandir($buda_root);
|
||||
foreach ($dirs as $dir) {
|
||||
if ($dir[0] == '.') continue;
|
||||
show_app($dir);
|
||||
}
|
||||
echo '<hr>';
|
||||
show_button_small('buda.php?action=app_form', 'Add app');
|
||||
page_tail();
|
||||
}
|
||||
|
||||
function show_app($dir) {
|
||||
global $buda_root;
|
||||
$indent = "    ";
|
||||
echo "<hr><font size=+3>$dir</font>\n";
|
||||
echo "<p>$indent Variants (click for details):<ul>";
|
||||
$pcs = scandir("$buda_root/$dir");
|
||||
foreach ($pcs as $pc) {
|
||||
if ($pc[0] == '.') continue;
|
||||
echo "<li><a href=buda.php?action=variant_view&app=$dir&variant=$pc>$pc</href>";
|
||||
show_button_small("buda.php?action=variant_delete&app=$dir&variant=$pc", 'Delete variant');
|
||||
show_button_small(
|
||||
"buda_submit.php?app=$dir&variant=$pc", "Submit jobs"
|
||||
);
|
||||
}
|
||||
echo '</ul>';
|
||||
echo $indent;
|
||||
show_button_small("buda.php?action=variant_form&app=$dir", 'Add variant');
|
||||
echo "<p>";
|
||||
echo "<p>";
|
||||
show_button_small(
|
||||
"buda.php?action=app_delete&app=$dir", "Delete app '$dir'"
|
||||
);
|
||||
}
|
||||
|
||||
function variant_view() {
|
||||
global $buda_root;
|
||||
$app = get_str('app');
|
||||
$variant = get_str('variant');
|
||||
page_head("App $app variant $variant");
|
||||
$dir = "$buda_root/$app/$variant";
|
||||
start_table();
|
||||
table_header('name', 'size', 'md5');
|
||||
foreach(scandir($dir) as $f) {
|
||||
if ($f[0] == '.') continue;
|
||||
[$size, $md5] = parse_info_file("$dir/.md5/$f");
|
||||
table_row(
|
||||
"<a href=buda.php?action=view_file&app=$app&variant=$variant&fname=$f>$f</a>",
|
||||
$size,
|
||||
$md5
|
||||
);
|
||||
}
|
||||
end_table();
|
||||
page_tail();
|
||||
}
|
||||
|
||||
// form for creating app variant.
|
||||
// Currently doesn't support indirect files.
|
||||
// doing this would require checkboxes for indirect
|
||||
//
|
||||
// Could have other stuff like
|
||||
// - min_quorum, max_total_results
|
||||
// - rsc_disk_bound, rsc_memory_bound
|
||||
// or does this belong in job submission?
|
||||
//
|
||||
function variant_form($user) {
|
||||
$sbitems = sandbox_select_items($user);
|
||||
$app = get_str('app');
|
||||
|
||||
page_head("Create variant of Docker app $app");
|
||||
form_start('buda.php');
|
||||
form_input_hidden('app', $app);
|
||||
form_input_hidden('action', 'variant_action');
|
||||
form_input_text('Plan class', 'plan_class');
|
||||
form_select('Dockerfile', 'dockerfile', $sbitems);
|
||||
form_select('Main program', 'main_prog', $sbitems);
|
||||
form_select_multiple('Other application files', 'others', $sbitems);
|
||||
form_input_text('Input file names', 'input_file_names');
|
||||
form_input_text('Output file names', 'output_file_names');
|
||||
form_submit('OK');
|
||||
form_end();
|
||||
page_tail();
|
||||
}
|
||||
|
||||
// copy file from sandbox to variant dir, and stage to download hier
|
||||
//
|
||||
function copy_and_stage_file($user, $fname, $dir, $variant) {
|
||||
copy_sandbox_file($user, $fname, $dir);
|
||||
[$md5, $size] = parse_info_file("$dir/.md5/$fname");
|
||||
$phys_name = sprintf('buda_%s_%s_%s', $app, $variant, $md5);
|
||||
stage_file_aux("$dir/$fname", $md5, $size, $phys_name);
|
||||
}
|
||||
|
||||
// create variant
|
||||
//
|
||||
function variant_action($user) {
|
||||
global $buda_root;
|
||||
$plan_class = get_str('plan_class');
|
||||
$app = get_str('app');
|
||||
$dockerfile = get_str('dockerfile');
|
||||
$main_prog = get_str('main_prog');
|
||||
$others = get_array('others');
|
||||
$input_file_names = explode(' ', get_str('input_file_names'));
|
||||
$output_file_names = explode(' ', get_str('output_file_names'));
|
||||
|
||||
if (file_exists("$buda_root/$app/$plan_class")) {
|
||||
error_page("Variant '$plan_class' already exists.");
|
||||
}
|
||||
$dir = "$buda_root/$app/$plan_class";
|
||||
mkdir($dir);
|
||||
mkdir("$dir/.md5");
|
||||
|
||||
// copy files from sandbox to variant dir
|
||||
//
|
||||
copy_and_stage_file($user, $dockerfile, $dir, $app, $variant);
|
||||
copy_and_stage_file($user, $main_prog, $dir, $app, $variant);
|
||||
foreach ($others as $fname) {
|
||||
copy_and_stage_file($user, $fname, $dir, $app, $variant);
|
||||
}
|
||||
|
||||
// create input template
|
||||
//
|
||||
$x = "<input_template>\n";
|
||||
$ninfiles = 2 + count($input_file_names) + count($others);
|
||||
for ($i=0; $i<$ninfiles; $i++) {
|
||||
$x .= " <file_info>\n </file_info>\n";
|
||||
}
|
||||
$x .= " <workunit>\n";
|
||||
$x .= file_ref_in('Dockerfile');
|
||||
$x .= file_ref_in($main_prog);
|
||||
foreach ($others as $fname) {
|
||||
$x .= file_ref_in($fname);
|
||||
}
|
||||
foreach ($input_file_names as $fname) {
|
||||
$x .= file_ref_in($fname);
|
||||
}
|
||||
$x .= " </workunit>\n<input_template>\n";
|
||||
file_put_contents("$dir/template_in", $x);
|
||||
|
||||
// create output template
|
||||
//
|
||||
$x = "<output_template>\n";
|
||||
$i = 0;
|
||||
foreach ($output_file_names as $fname) {
|
||||
$x .= file_info_out($i++);
|
||||
}
|
||||
$x .= " <result>\n";
|
||||
$i = 0;
|
||||
foreach ($output_file_names as $fname) {
|
||||
$x .= file_ref_out($i++, $fname);
|
||||
}
|
||||
$x .= " </result>\n</output_template>\n";
|
||||
file_put_contents("$dir/template_out", $x);
|
||||
|
||||
// Note: we don't currently allow indirect file access.
|
||||
// If we did, we'd need to create job.toml to mount project dir
|
||||
|
||||
app_list("Variant $plan_class added for app $app.");
|
||||
}
|
||||
|
||||
function file_ref_in($fname) {
|
||||
return(sprintf(
|
||||
' <file_ref>
|
||||
<open_name>%s</open_name>
|
||||
</file_ref>
|
||||
',
|
||||
$fname
|
||||
));
|
||||
}
|
||||
function file_info_out($i) {
|
||||
return sprintf(
|
||||
' <file_info>
|
||||
<name><OUTFILE_%d/></name>
|
||||
<generated_locally/>
|
||||
<upload_when_present/>
|
||||
<max_nbytes>5000000</max_nbytes>
|
||||
<url><UPLOAD_URL/></url>
|
||||
</file_info>
|
||||
',
|
||||
$i
|
||||
);
|
||||
}
|
||||
|
||||
function file_ref_out($i, $fname) {
|
||||
return sprintf(
|
||||
' <file_ref>
|
||||
<file_name><OUTFILE_%d/></file_name>
|
||||
<open_name>%s</open_name>
|
||||
</file_ref>
|
||||
', $i, $fname
|
||||
);
|
||||
}
|
||||
|
||||
function variant_delete() {
|
||||
global $buda_root;
|
||||
$app = get_str('app');
|
||||
$variant = get_str('variant');
|
||||
$confirmed = get_str('confirmed', true);
|
||||
if ($confirmed) {
|
||||
$dir = "$buda_root/$app/$variant";
|
||||
// delete staged files
|
||||
//
|
||||
foreach (scandir("$dir/.md5") as $fname) {
|
||||
if ($fname[0] == '.') continue;
|
||||
[$size, $md5] = parse_file_info("$dir/$fname");
|
||||
$phys_name = buda_app_file_phys_name($app, $variant, $md5);
|
||||
$phys_path = download_path($phys_name);
|
||||
unlink($phys_path);
|
||||
}
|
||||
system("rm -r $buda_root/$app/$variant", $ret);
|
||||
if ($ret) {
|
||||
error_page("delete failed");
|
||||
}
|
||||
$notice = "Variant $variant of app $app removed.";
|
||||
app_list($notice);
|
||||
} else {
|
||||
page_head("Confirm");
|
||||
echo "Are you sure want to delete variant $variant of app $app?
|
||||
<p>
|
||||
";
|
||||
show_button(
|
||||
"buda.php?action=variant_delete&app=$app&variant=$variant&confirmed=yes",
|
||||
"Yes"
|
||||
);
|
||||
page_tail();
|
||||
}
|
||||
}
|
||||
|
||||
function app_form() {
|
||||
page_head("Create Docker app");
|
||||
form_start();
|
||||
form_input_text('Name', 'name');
|
||||
form_submit('OK');
|
||||
form_end();
|
||||
page_tail();
|
||||
}
|
||||
|
||||
function app_action() {
|
||||
global $buda_root;
|
||||
$name = get_str('name');
|
||||
$dir = "$buda_root/$name";
|
||||
if (file_exists($dir)) {
|
||||
error_page("App $name already exists.");
|
||||
}
|
||||
mkdir($dir);
|
||||
header("Location: buda.php");
|
||||
}
|
||||
|
||||
function view_file() {
|
||||
global $buda_root;
|
||||
$app = get_str('app');
|
||||
$variant = get_str('variant');
|
||||
$fname = get_str('fname');
|
||||
echo "<pre>\n";
|
||||
readfile("$buda_root/$app/$variant/$fname");
|
||||
}
|
||||
|
||||
$user = get_logged_in_user();
|
||||
$action = get_str('action', true);
|
||||
switch ($action) {
|
||||
case 'app_form':
|
||||
app_form(); break;
|
||||
case 'app_action':
|
||||
app_action(); break;
|
||||
case 'app_delete':
|
||||
app_delete(); break;
|
||||
case 'variant_view':
|
||||
variant_view($user); break;
|
||||
case 'variant_form':
|
||||
variant_form($user); break;
|
||||
case 'variant_action':
|
||||
variant_action($user); break;
|
||||
case 'variant_delete':
|
||||
variant_delete(); break;
|
||||
case 'view_file':
|
||||
view_file(); break;
|
||||
case null:
|
||||
app_list(); break;
|
||||
default:
|
||||
error_page("unknown action $action");
|
||||
}
|
||||
|
||||
?>
|
||||
|
|
@ -0,0 +1,73 @@
|
|||
<?php
|
||||
// This file is part of BOINC.
|
||||
// https://boinc.berkeley.edu
|
||||
// Copyright (C) 2024 University of California
|
||||
//
|
||||
// BOINC is free software; you can redistribute it and/or modify it
|
||||
// under the terms of the GNU Lesser General Public License
|
||||
// as published by the Free Software Foundation,
|
||||
// either version 3 of the License, or (at your option) any later version.
|
||||
//
|
||||
// BOINC is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
|
||||
// See the GNU Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with BOINC. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
// web interface for submitting BUDA jobs
|
||||
|
||||
require_once('../inc/util.inc');
|
||||
require_once('../inc/sandbox.inc');
|
||||
|
||||
display_errors();
|
||||
|
||||
function submit_form($user) {
|
||||
$sbitems_zip = sandbox_select_items($user, '/.zip$/');
|
||||
if (!$sbitems_zip) {
|
||||
error_page("No .zip files in your sandbox.");
|
||||
}
|
||||
$app = get_str('app');
|
||||
$variant = get_str('variant');
|
||||
|
||||
page_head("Submit jobs to $app ($variant)");
|
||||
form_start('buda_submit.php');
|
||||
form_input_hidden('action', 'submit');
|
||||
form_input_hidden('app', $app);
|
||||
form_input_hidden('variant', $variant);
|
||||
form_input_text('Batch name', 'batch_name');
|
||||
form_select('Job file', 'job_file', $sbitems_zip);
|
||||
form_submit('OK');
|
||||
form_end();
|
||||
page_tail();
|
||||
}
|
||||
|
||||
function handle_submit() {
|
||||
// stage app files if not already staged
|
||||
//
|
||||
|
||||
// create batch
|
||||
//
|
||||
|
||||
// unzip batch file
|
||||
//
|
||||
|
||||
// stage top-level input files
|
||||
//
|
||||
|
||||
// scan jobs; stage per-job input files and make create_work input
|
||||
//
|
||||
|
||||
// create jobs
|
||||
}
|
||||
|
||||
$user = get_logged_in_user();
|
||||
$action = get_str('action', true);
|
||||
if ($action == 'submit') {
|
||||
handle_submit();
|
||||
} else {
|
||||
submit_form($user);
|
||||
}
|
||||
|
||||
?>
|
||||
|
|
@ -16,14 +16,10 @@
|
|||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with BOINC. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
// Per-user "file sandboxes" for job submission.
|
||||
// These are stored in project-root/sandbox/USERID/
|
||||
//
|
||||
// The entries in a sandbox directory have contents
|
||||
// size md5
|
||||
//
|
||||
// The actual files are stored in the download hierarchy,
|
||||
// with sb_userid_MD5 as the physical name
|
||||
// Per-user "file sandboxes".
|
||||
// Files are stored in <project>/sandbox/<userid>/
|
||||
// File infos (md5/size) are scored in a parallel dir
|
||||
// <project>/sandbox/<userid>/.md5/
|
||||
|
||||
// NOTE: PHP's default max file upload size is 2MB.
|
||||
// To increase this, edit /etc/php.ini, and change, e.g.
|
||||
|
|
@ -32,33 +28,31 @@
|
|||
// post_max_size = 64M
|
||||
|
||||
require_once("../inc/sandbox.inc");
|
||||
require_once("../inc/submit_db.inc");
|
||||
require_once("../inc/submit_util.inc");
|
||||
|
||||
display_errors();
|
||||
|
||||
function list_files($user, $notice) {
|
||||
$dir = sandbox_dir($user);
|
||||
$d = opendir($dir);
|
||||
if (!$d) error_page("Can't open sandbox directory");
|
||||
if (!is_dir($dir)) error_page("Can't open sandbox directory");
|
||||
page_head("File sandbox");
|
||||
if ($notice) {
|
||||
echo "<p>$notice<hr>";
|
||||
}
|
||||
echo "
|
||||
<p>
|
||||
<h3>Upload files</h3>
|
||||
<form action=sandbox.php method=post ENCTYPE=\"multipart/form-data\">
|
||||
<input type=hidden name=action value=upload_file>
|
||||
Upload files to your sandbox:
|
||||
<p><p><input size=80 type=file name=\"new_file[]\" multiple=\"multiple\">
|
||||
<p> <input class=\"btn btn-success\" type=submit value=Upload>
|
||||
</form>
|
||||
<hr>
|
||||
<h3>Sandbox contents</h3>
|
||||
";
|
||||
$files = array();
|
||||
while (($f = readdir($d)) !== false) {
|
||||
if ($f == '.') continue;
|
||||
if ($f == '..') continue;
|
||||
foreach (scandir($dir) as $f) {
|
||||
if ($f[0] == '.') continue;
|
||||
$files[] = $f;
|
||||
}
|
||||
if (count($files) == 0) {
|
||||
|
|
@ -66,19 +60,10 @@ function list_files($user, $notice) {
|
|||
} else {
|
||||
sort($files);
|
||||
start_table();
|
||||
table_header("Name<br><p class=\"text-muted\">(click to view)</p>", "Modified", "Size (bytes)", "MD5", "Delete","Download");
|
||||
table_header("Name<br><p class=\"text-muted\">(click to view text files)</p>", "Modified", "Size (bytes)", "MD5", "Delete","Download");
|
||||
foreach ($files as $f) {
|
||||
[$md5, $size] = sandbox_parse_info_file($user, $f);
|
||||
$path = "$dir/$f";
|
||||
list($error, $size, $md5) = sandbox_parse_link_file($path);
|
||||
if ($error) {
|
||||
table_row($f, "Can't parse link file", "", "<a href=sandbox.php?action=delete_files&name=$f>delete</a>");
|
||||
continue;
|
||||
}
|
||||
$p = sandbox_physical_path($user, $md5);
|
||||
if (!is_file($p)) {
|
||||
table_row($f, "Physical file not found", "", "");
|
||||
continue;
|
||||
}
|
||||
$ct = time_str(filemtime($path));
|
||||
table_row(
|
||||
"<a href=sandbox.php?action=view_file&name=$f>$f</a>",
|
||||
|
|
@ -100,10 +85,13 @@ function list_files($user, $notice) {
|
|||
page_tail();
|
||||
}
|
||||
|
||||
// upload one or more files
|
||||
|
||||
function upload_file($user) {
|
||||
$notice = "";
|
||||
$dir = sandbox_dir($user);
|
||||
$count = count($_FILES['new_file']['tmp_name']);
|
||||
for ($i = 0; $i < $count; $i++) {
|
||||
for ($i=0; $i<$count; $i++) {
|
||||
$tmp_name = $_FILES['new_file']['tmp_name'][$i];
|
||||
if (!is_uploaded_file($tmp_name)) {
|
||||
error_page("$tmp_name is not uploaded file");
|
||||
|
|
@ -112,79 +100,48 @@ function upload_file($user) {
|
|||
if (strstr($name, "/")) {
|
||||
error_page("no / allowed");
|
||||
}
|
||||
$md5 = md5_file($tmp_name);
|
||||
$s = stat($tmp_name);
|
||||
$size = $s['size'];
|
||||
[$exists, $elf] = sandbox_lf_exists($user, $md5);
|
||||
if (!$exists){
|
||||
// move file to download dir
|
||||
//
|
||||
$phys_path = sandbox_physical_path($user, $md5);
|
||||
move_uploaded_file($tmp_name, $phys_path);
|
||||
if (file_exists("$dir/$name")) {
|
||||
$notice .= "can't upload $name; file exists.<br>";
|
||||
continue;
|
||||
}
|
||||
move_uploaded_file($tmp_name, "$dir/$name");
|
||||
|
||||
// write link file
|
||||
// write info file
|
||||
//
|
||||
$dir = sandbox_dir($user);
|
||||
$link_path = "$dir/$name";
|
||||
sandbox_write_link_file($link_path, $size, $md5);
|
||||
[$md5, $size] = get_file_info("$dir/$name");
|
||||
write_info_file("$dir/.md5/$name", $md5, $size);
|
||||
|
||||
$notice .= "Uploaded file <strong>$name</strong><br/>";
|
||||
}
|
||||
list_files($user, $notice);
|
||||
}
|
||||
|
||||
// delete a link to a file.
|
||||
// check if currently being used by a batch.
|
||||
// If the last link w/ that contents, delete the file itself
|
||||
// delete a sandbox file.
|
||||
//
|
||||
function delete_file($user) {
|
||||
$name = get_str('name');
|
||||
$dir = sandbox_dir($user);
|
||||
list($error, $size, $md5) = sandbox_parse_link_file("$dir/$name");
|
||||
if ($error) {
|
||||
error_page("can't parse link file");
|
||||
}
|
||||
$p = sandbox_physical_path($user, $md5);
|
||||
if (!is_file($p)) {
|
||||
error_page("physical file is missing");
|
||||
}
|
||||
$bused = sandbox_file_in_use($user, $name);
|
||||
if ($bused){
|
||||
$notice = "<strong>$name</strong> is being used by batch(es), you can not delete it now!<br/>";
|
||||
} else{
|
||||
unlink("$dir/$name");
|
||||
$notice = "<strong>$name</strong> was deleted from your sandbox<br/>";
|
||||
[$exists, $elf] = sandbox_lf_exists($user, $md5);
|
||||
if (!$exists) {
|
||||
unlink($p);
|
||||
}
|
||||
|
||||
}
|
||||
unlink("$dir/$name");
|
||||
unlink("$dir/.md5/$name");
|
||||
$notice = "<strong>$name</strong> was deleted from your sandbox<br/>";
|
||||
list_files($user, $notice);
|
||||
//Header("Location: sandbox.php");
|
||||
}
|
||||
|
||||
function download_file($user) {
|
||||
$name = get_str('name');
|
||||
$dir = sandbox_dir($user);
|
||||
list($err, $size, $md5) = sandbox_parse_link_file("$dir/$name");
|
||||
if ($err) {
|
||||
error_page("can't parse link file");
|
||||
}
|
||||
$p = sandbox_physical_path($user, $md5);
|
||||
if (!is_file($p)) {
|
||||
error_page("$p does not exist!");
|
||||
}
|
||||
do_download($p, $name);
|
||||
do_download("$dir/$name");
|
||||
}
|
||||
|
||||
function view_file($user) {
|
||||
$name = get_str('name');
|
||||
$dir = sandbox_dir($user);
|
||||
list($error, $size, $md5) = sandbox_parse_link_file("$dir/$name");
|
||||
if ($error) error_page("no such link file");
|
||||
$p = sandbox_physical_path($user, $md5);
|
||||
if (!is_file($p)) error_page("no such physical file");
|
||||
$path = "$dir/$name";
|
||||
if (!is_file($path)) {
|
||||
error_path("no such file $name");
|
||||
}
|
||||
echo "<pre>\n";
|
||||
readfile($p);
|
||||
readfile($path);
|
||||
echo "</pre>\n";
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,35 +1,40 @@
|
|||
#! /usr/bin/env php
|
||||
<?php
|
||||
|
||||
// submit a job for the Docker universal app (which must be named 'buda')
|
||||
// submit a job for a BUDA (BOINC Universal Docker app) science app
|
||||
//
|
||||
// submit_buda [--buda_dir dir] [--verbose] infile ...
|
||||
// submit_buda [options] sci_app infile ...
|
||||
//
|
||||
// the buda dir (default buda_dir/) must contain
|
||||
// - buda_in, buda_out: input, output templates
|
||||
// - other files; typically
|
||||
// a Dockerfile
|
||||
// job.toml
|
||||
// main.sh script
|
||||
// science executable
|
||||
// - file_list
|
||||
// a list of the other filenames (one per line)
|
||||
// in the order of the input template
|
||||
// --buda_root dir root of BUDA app hierarchy, default 'buda_apps'
|
||||
// --variant x variant, default 'cpu'
|
||||
// --verbose
|
||||
|
||||
$buda_dir = 'buda_dir';
|
||||
$buda_root = 'buda_apps';
|
||||
$infiles = [];
|
||||
$verbose = false;
|
||||
$variant = 'cpu';
|
||||
$sci_app = null;
|
||||
|
||||
for ($i=1; $i<$argc; $i++) {
|
||||
if ($argv[$i] == '--buda_dir') {
|
||||
$buda_dir = $argv[++$i];
|
||||
if ($argv[$i] == '--buda_root') {
|
||||
$buda_root = $argv[++$i];
|
||||
} else if ($argv[$i] == '--variant') {
|
||||
$variant = $argv[++$i];
|
||||
} else if ($argv[$i] == '--verbose') {
|
||||
$verbose = true;
|
||||
} else {
|
||||
$infiles[] = $argv[$i];
|
||||
if (!$sci_app) {
|
||||
$sci_app = $argv[$i];
|
||||
} else {
|
||||
$infiles[] = $argv[$i];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$buda_dir = "$buda_root/$sci_app/$variant";
|
||||
if (!is_dir($buda_dir)) {
|
||||
die("No version dir $buda_dir\n");
|
||||
}
|
||||
if (!file_exists("$buda_dir/template_in")) {
|
||||
die("no input template\n");
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue