diff --git a/HACKING b/HACKING index 64c3e1384..60878e9ba 100644 --- a/HACKING +++ b/HACKING @@ -35,6 +35,15 @@ $ ./dev-camput $ ./dev-camget ... etc. +We are in the process of migrating those scripts to a go command +with subcommands: devcam. To install (assuming your GOPATH is set): + +$ go install ./dev/devcam + +./dev/server has already been replaced by: + +$ devcam server + e.g. - Upload a file (make sure this works): ~/camlistore/dev-camput file ~/camlistore/COPYING diff --git a/dev-server b/dev-server deleted file mode 100755 index 434c90dc8..000000000 --- a/dev-server +++ /dev/null @@ -1,199 +0,0 @@ -#!/usr/bin/perl - -use strict; -use FindBin qw($Bin); -use Getopt::Long; -require "$Bin/misc/devlib.pl"; - -sub usage { - die "Usage: dev-server [OPTS] [] -- [other_blobserver_opts]\n\nWhere OPTS include:\n [--wipe] [--mongo|--mysql|--postgres] [--tls] [--fullclosure] " . - "[--all] [--nobuild] [--staticres] [--KBps=] " . - "[--latency_ms=] [--slow] [--verbose] [--hostname=]\n"; -} - -chdir $Bin or die; - -my $opt_slow; # add latency -my $opt_KBps = 150; # if non-zero, kBps to throttle connections -my $opt_latency_ms = 90; # added latency in millisecond -my $opt_fast; # (no longer does anything) - -my $opt_all; # listen on all interfaces -my $opt_hostname; # hostname to advertise, else `hostname` is used -my $opt_nobuild; -my $opt_fullclosure; # make all of closure available -my $opt_staticres; # use static resources, not those on disk -my $opt_tls; -my $opt_wipe; -my $opt_verbose; - -# keep indexes in memory only. often used with --wipe, but not -# necessarily. if --wipe isn't used, all blobs are re-indexed -# on start-up. -my $opt_memory; -my $opt_mongo; -my $opt_postgres; -my $opt_mysql; -GetOptions("wipe" => \$opt_wipe, - "tls" => \$opt_tls, - "all" => \$opt_all, - "nobuild" => \$opt_nobuild, - "memory" => \$opt_memory, - "mongo" => \$opt_mongo, - "postgres" => \$opt_postgres, - "mysql" => \$opt_mysql, - "staticres" => \$opt_staticres, - "KBps=i" => \$opt_KBps, - "latency_ms=i" => \$opt_latency_ms, - "fast" => \$opt_fast, - "slow" => \$opt_slow, - "verbose" => \$opt_verbose, - "hostname=s" => \$opt_hostname, - "fullclosure" => \$opt_fullclosure, - ) - or usage(); - -$opt_memory = 1 unless $opt_memory || $opt_mongo || $opt_postgres || $opt_mysql; - -my $port = shift; -$port = "3179" unless defined($port); -usage() unless $port =~ /^\d+$/; - -if ($opt_fast) { - warn "# Btw, --fast is now the default. Use --slow if you want to simulate slowness.\n"; -} - -if ($opt_slow) { - $ENV{DEV_THROTTLE_KBPS} = $opt_KBps; - $ENV{DEV_THROTTLE_LATENCY_MS} = $opt_latency_ms; -} - -$ENV{CAMLI_HTTP_DEBUG} = 1 if $opt_verbose; - -my $camlistored; -my $camtool; # closure to return path -if ($opt_nobuild) { - $camlistored = find_bin("./server/camlistored"); - $camtool = sub { scalar find_bin("./cmd/camtool") }; -} else { - $camlistored = build_bin("./server/camlistored"); - $camtool = sub { scalar build_bin("./cmd/camtool") }; -} - -my $root = "/tmp/camliroot-$ENV{USER}/port$port/"; -if ($opt_wipe && -d $root) { - print "Wiping $root\n"; - system("rm", "-rf", $root) and die "Failed to wipe $root.\n"; -} - -my $suffixdir = sub { - my $suffix = shift; - my $root = "$root/$suffix"; - unless (-d $root) { - system("mkdir", "-p", $root) and die "Failed to create $root.\n"; - } - return $root; -}; - -my $DBNAME = "devcamli$ENV{USER}"; -my @opts; -if ($opt_wipe) { - push @opts, "-wipe"; -} else { - push @opts, "-ignoreexists"; -} - -$ENV{"CAMLI_MYSQL_ENABLED"} = "false"; -$ENV{"CAMLI_MONGO_ENABLED"} = "false"; -$ENV{"CAMLI_POSTGRES_ENABLED"} = "false"; -if ($opt_memory) { - $ENV{"CAMLI_INDEXER_PATH"} = "/index-mem/"; -} elsif ($opt_mongo) { - $ENV{"CAMLI_MONGO_ENABLED"} = "true"; - $ENV{"CAMLI_INDEXER_PATH"} = "/index-mongo/"; - if ($opt_wipe) { - $ENV{"CAMLI_MONGO_WIPE"} = "true"; - } else { - $ENV{"CAMLI_MONGO_WIPE"} = "false"; - } -} elsif ($opt_postgres) { - $ENV{"CAMLI_POSTGRES_ENABLED"} = "true"; - $ENV{"CAMLI_INDEXER_PATH"} = "/index-postgres/"; - system($camtool->(), - "dbinit", - "-dbtype=postgres", - "-user=postgres", - "-password=postgres", - "-host=localhost", - "-dbname=$DBNAME", - @opts) and die "Failed to run camtool dbinit.\n"; -} else { - $ENV{"CAMLI_MYSQL_ENABLED"} = "true"; - $ENV{"CAMLI_INDEXER_PATH"} = "/index-mysql/"; - system($camtool->(), - "dbinit", - "-user=root", - "-password=root", - "-host=localhost", - "-dbname=$DBNAME", - @opts) and die "Failed to run camtool dbinit.\n"; -} - -my $base = "http://localhost:$port"; -my $listen = "127.0.0.1:$port"; -if ($opt_all) { - $listen = "0.0.0.0:$port"; - my $host = $opt_hostname || `hostname`; - chomp $host; - $base = "http://$host:$port"; -} -if ($opt_tls) { - $base =~ s/^http/https/; -} - -$ENV{CAMLI_TLS} = "false"; -if ($opt_tls) { - $ENV{CAMLI_TLS} = "true"; -} -$ENV{CAMLI_DEV_CAMLI_ROOT} = $Bin; -$ENV{CAMLI_BASEURL} = $base; -$ENV{CAMLI_AUTH} = "userpass:camlistore:pass$port:+localhost"; -$ENV{CAMLI_ADVERTISED_PASSWORD} = "pass$port"; # public password -$ENV{CAMLI_ROOT} = $suffixdir->("bs"); -$ENV{CAMLI_ROOT_SHARD1} = $suffixdir->("s1"); -$ENV{CAMLI_ROOT_SHARD2} = $suffixdir->("s2"); -$ENV{CAMLI_ROOT_REPLICA1} = $suffixdir->("r1"); -$ENV{CAMLI_ROOT_REPLICA2} = $suffixdir->("r2"); -$ENV{CAMLI_ROOT_REPLICA3} = $suffixdir->("r3"); -$ENV{CAMLI_ROOT_CACHE} = $suffixdir->("cache"); -$ENV{CAMLI_ROOT_ENCMETA} = $suffixdir->("encmeta"); -$ENV{CAMLI_ROOT_ENCBLOB} = $suffixdir->("encblob"); -$ENV{CAMLI_PORT} = $port; -$ENV{CAMLI_SECRET_RING} = "$Bin/pkg/jsonsign/testdata/test-secring.gpg"; -$ENV{CAMLI_DBNAME} = $DBNAME; - -my $templatedir = "$Bin/dev-server-template"; -if ($opt_wipe && -d $templatedir) { - my $blobs = "$ENV{CAMLI_ROOT}/sha1"; - system("cp", "-a", $templatedir, $blobs) and die "Failed to cp template blobs.\n"; -} - -# To use resources from disk, instead of the copies linked into the -# binary: -if ($opt_fullclosure) { - if (-e "$Bin/tmp/closure-lib/.svn") { - system("rm", "-rf", "$Bin/tmp/closure-lib") and die "Failed to remove the svn checkout of the closure-lib.\n"; - } - print STDERR "Updating closure library ...\n"; - system("go", "run", "third_party/closure/updatelibrary.go", "-verbose") - and die "go run third_party/closure/updatelibrary.go failed"; - $ENV{CAMLI_DEV_CLOSURE_DIR} = "third_party/closure/lib/closure"; -} - -print "Starting dev server on $base/ui/ with password \"pass$port\"\n"; - -exec("$camlistored", - "-configfile=$Bin/config/dev-server-config.json", - "-listen=$listen", - @ARGV); -die "exec failure: $!\n"; diff --git a/dev/devcam/server.go b/dev/devcam/server.go index 2476880f1..4428ffd0a 100644 --- a/dev/devcam/server.go +++ b/dev/devcam/server.go @@ -25,9 +25,11 @@ import ( "log" "os" "os/exec" + "os/signal" "path/filepath" "strconv" "strings" + "syscall" "camlistore.org/pkg/cmdmain" "camlistore.org/pkg/osutil" @@ -313,6 +315,28 @@ func (c *serverCmd) setFullClosure() error { return nil } +func handleKillCamliSignal(camliProc *os.Process) { + c := make(chan os.Signal, 1) + signal.Notify(c, syscall.SIGTERM, syscall.SIGINT) + for { + sig := <-c + sysSig, ok := sig.(syscall.Signal) + if !ok { + log.Fatal("Not a unix signal") + } + switch sysSig { + case syscall.SIGTERM, syscall.SIGINT: + log.Print("Received kill signal, terminating.") + err := camliProc.Kill() + if err != nil { + log.Fatalf("Failed to kill camli: %v ", err) + } + default: + log.Fatal("Received another signal, should not happen.") + } + } +} + func (c *serverCmd) RunCommand(args []string) error { err := c.checkFlags(args) if err != nil { @@ -363,5 +387,7 @@ func (c *serverCmd) RunCommand(args []string) error { if err := cmd.Start(); err != nil { return fmt.Errorf("Could not start camlistored: %v", err) } + go handleKillCamliSignal(cmd.Process) + cmd.Wait() return nil } diff --git a/dev/make-release b/dev/make-release index 58d51e10e..5c6b2c8c8 100755 --- a/dev/make-release +++ b/dev/make-release @@ -40,7 +40,6 @@ foreach my $d (qw{ dev-camtool dev-db dev-pass-camput - dev-server }) { system("git", "rm", "-r", $d) and die "Failed to git rm -r $d"; } diff --git a/misc/buildbot/bot.go b/misc/buildbot/bot.go index f3206ea80..be083292b 100644 --- a/misc/buildbot/bot.go +++ b/misc/buildbot/bot.go @@ -40,7 +40,7 @@ import ( const ( interval = 60 * time.Second // polling frequency - warmup = 60 * time.Second // duration before we test if dev-server has started properly + warmup = 60 * time.Second // duration before we test if devcam server has started properly historySize = 30 ) @@ -98,8 +98,9 @@ var NameToCmd = map[string]string{ "prepRepo8": "git rev-parse HEAD", "buildGoTip1": "./make.bash", "buildCamli1": "make forcefull", - "buildCamli2": "make presubmit", - "runCamli": "./dev-server --fast --wipe --mysql", + "buildCamli2": "go build -o devcam ./dev/devcam/", + "buildCamli3": "make presubmit", + "runCamli": "./devcam server --wipe --mysql", "hitCamliUi1": "http://localhost:3179/ui/", "camget": "./dev-camget ", "camput1": "./dev-camput file --permanode " + testFile[0], @@ -575,9 +576,9 @@ func buildCamli(isTip bool) error { tasks := []string{} if doBuildCamli0 || doBuildCamli1 || (doBuildGo && isTip) { - tasks = append(tasks, NameToCmd["buildCamli1"]) + tasks = append(tasks, NameToCmd["buildCamli1"], NameToCmd["buildCamli2"]) } - tasks = append(tasks, NameToCmd["buildCamli2"]) + tasks = append(tasks, NameToCmd["buildCamli3"]) for _, v := range tasks { setCurrentTask(v) tsk := getCurrentTask() @@ -635,7 +636,7 @@ func runCamli() (*os.Process, error) { addRun(tsk, err) return nil, fmt.Errorf("%v: server failed to start\n", tsk.Cmd) case <-time.After(warmup): - dbg.Println("dev server OK") + dbg.Println("devcam server OK") addRun(getCurrentTask(), nil) } return cmd.Process, nil @@ -646,9 +647,9 @@ func killCamli(proc *os.Process) { return } dbg.Println("killing dev server") - err := proc.Kill() + err := proc.Signal(os.Interrupt) if err != nil { - log.Fatalf("Could not kill dev-server: %v", err) + log.Fatalf("Could not kill server: %v", err) } dbg.Println("") } diff --git a/pkg/index/mysql/mysqlindexer.go b/pkg/index/mysql/mysqlindexer.go index 4dabe272a..2b6fabd24 100644 --- a/pkg/index/mysql/mysqlindexer.go +++ b/pkg/index/mysql/mysqlindexer.go @@ -101,9 +101,9 @@ func newFromConfig(ld blobserver.Loader, config jsonconfig.Obj) (blobserver.Stor fmt.Fprintf(os.Stderr, fixSchema20to21) } if os.Getenv("CAMLI_ADVERTISED_PASSWORD") != "" { - // Good signal that we're using the dev-server script, so help out + // Good signal that we're using the devcam server, so help out // the user with a more useful tip: - return nil, fmt.Errorf("database schema version is %d; expect %d (run \"./dev-server --wipe\" to wipe both your blobs and re-populate the database schema)", version, requiredSchemaVersion) + return nil, fmt.Errorf("database schema version is %d; expect %d (run \"devcam server --wipe\" to wipe both your blobs and re-populate the database schema)", version, requiredSchemaVersion) } return nil, fmt.Errorf("database schema version is %d; expect %d (need to re-init/upgrade database?)", version, requiredSchemaVersion) diff --git a/pkg/index/postgres/postgresindexer.go b/pkg/index/postgres/postgresindexer.go index 32c2d480b..0824a5f3a 100644 --- a/pkg/index/postgres/postgresindexer.go +++ b/pkg/index/postgres/postgresindexer.go @@ -126,9 +126,9 @@ func newFromConfig(ld blobserver.Loader, config jsonconfig.Obj) (blobserver.Stor } if version != requiredSchemaVersion { if os.Getenv("CAMLI_ADVERTISED_PASSWORD") != "" { - // Good signal that we're using the dev-server script, so help out + // Good signal that we're using the devcam server, so help out // the user with a more useful tip: - return nil, fmt.Errorf("database schema version is %d; expect %d (run \"./dev-server --wipe\" to wipe both your blobs and re-populate the database schema)", version, requiredSchemaVersion) + return nil, fmt.Errorf("database schema version is %d; expect %d (run \"devcam server --wipe\" to wipe both your blobs and re-populate the database schema)", version, requiredSchemaVersion) } return nil, fmt.Errorf("database schema version is %d; expect %d (need to re-init/upgrade database?)", version, requiredSchemaVersion) diff --git a/pkg/index/sqlite/sqlite.go b/pkg/index/sqlite/sqlite.go index a7e9219fa..82f88b277 100644 --- a/pkg/index/sqlite/sqlite.go +++ b/pkg/index/sqlite/sqlite.go @@ -70,7 +70,7 @@ func NewStorage(file string) (index.Storage, error) { file: file, db: db, Storage: &sqlindex.Storage{ - DB: db, + DB: db, Serial: true, }, }, nil @@ -109,9 +109,9 @@ func newFromConfig(ld blobserver.Loader, config jsonconfig.Obj) (blobserver.Stor if version != requiredSchemaVersion { if os.Getenv("CAMLI_ADVERTISED_PASSWORD") != "" { - // Good signal that we're using the dev-server script, so help out + // Good signal that we're using the devcam server, so help out // the user with a more useful tip: - return nil, fmt.Errorf("database schema version is %d; expect %d (run \"./dev-server --wipe\" to wipe both your blobs and re-populate the database schema)", version, requiredSchemaVersion) + return nil, fmt.Errorf("database schema version is %d; expect %d (run \"devcam server --wipe\" to wipe both your blobs and re-populate the database schema)", version, requiredSchemaVersion) } return nil, fmt.Errorf("database schema version is %d; expect %d (need to re-init/upgrade database?)", version, requiredSchemaVersion) diff --git a/pkg/server/ui.go b/pkg/server/ui.go index ed6b56e21..18b883e94 100644 --- a/pkg/server/ui.go +++ b/pkg/server/ui.go @@ -201,9 +201,9 @@ func (ui *UIHandler) makeClosureHandler(root string) (http.Handler, error) { // 3) a path on disk to the root of camlistore's source (which // contains the necessary subset of Closure files) func makeClosureHandler(root, handlerName string) (http.Handler, error) { - // dev-server environment variable takes precendence: + // devcam server environment variable takes precendence: if d := os.Getenv("CAMLI_DEV_CLOSURE_DIR"); d != "" { - log.Printf("%v: serving Closure from dev-server's $CAMLI_DEV_CLOSURE_DIR: %v", handlerName, d) + log.Printf("%v: serving Closure from devcam server's $CAMLI_DEV_CLOSURE_DIR: %v", handlerName, d) return http.FileServer(http.Dir(d)), nil } if root == "" { diff --git a/server/camlistored/ui/closure/closure.go b/server/camlistored/ui/closure/closure.go index 3bf15fe2d..70b4d4052 100644 --- a/server/camlistored/ui/closure/closure.go +++ b/server/camlistored/ui/closure/closure.go @@ -34,7 +34,7 @@ import ( ) // ZipData is either the empty string (when compiling with "go get", -// or the dev-server), or is initialized to a base64-encoded zip file +// or the devcam server), or is initialized to a base64-encoded zip file // of the Closure library (when using make.go, which puts an extra // file in this package containing an init function to set ZipData). var ZipData string diff --git a/website/content/docs/server-config b/website/content/docs/server-config index 324312abd..7294c140d 100644 --- a/website/content/docs/server-config +++ b/website/content/docs/server-config @@ -61,7 +61,7 @@ to pkg/genconfig welcome.

Additionally, mongo, mysql, and postgres require the dbname value set. Initialize your database with camtool dbinit.

There's also an in-memory index type, but only in the low-level -config, as used by the dev-server script.

+config, as used by devcam server.

Publishing options

Although limited, publishing can be configured through the publish key. There is only support for an image gallery view (even though it will display thumbnails for other kinds of items), which is not really customizable. Here is an example of a value if one wanted to publish some items under /pics/: