diff --git a/pkg/cloudlaunch/cloudlaunch.go b/pkg/cloudlaunch/cloudlaunch.go index fa581f87e..0b4169b50 100644 --- a/pkg/cloudlaunch/cloudlaunch.go +++ b/pkg/cloudlaunch/cloudlaunch.go @@ -22,11 +22,13 @@ import ( "encoding/json" "errors" "flag" + "fmt" "io" "io/ioutil" "log" "net/http" "os" + "path" "path/filepath" "runtime" "strings" @@ -74,12 +76,26 @@ coreos: WantedBy=network-online.target ` +// RestartPolicy controls whether the binary automatically restarts. +type RestartPolicy int + +const ( + RestartOnUpdates RestartPolicy = iota + RestartNever + // TODO: more graceful restarts; make systemd own listening on network sockets, + // don't break connections. +) + type Config struct { // Name is the name of a service to run. // This is the name of the systemd service (without .service) // and the name of the GCE instance. Name string + // RestartPolicy controls whether the binary automatically restarts + // on updates. The zero value means automatic. + RestartPolicy RestartPolicy + // BinaryBucket and BinaryObject are the GCS bucket and object // within that bucket containing the Linux binary to download // on boot and occasionally run. This binary must be public @@ -134,6 +150,7 @@ var ( func (c *Config) MaybeDeploy() { flag.Parse() if !*doLaunch { + go c.restartLoop() return } defer os.Exit(1) // backup, in case we return without Fatal or os.Exit later @@ -167,6 +184,36 @@ func (c *Config) MaybeDeploy() { os.Exit(0) } +func (c *Config) restartLoop() { + if c.RestartPolicy == RestartNever { + return + } + url := "https://storage.googleapis.com/" + c.BinaryBucket + "/" + c.binaryObject() + var lastEtag string + for { + res, err := http.Head(url + "?" + fmt.Sprint(time.Now().Unix())) + if err != nil { + log.Printf("Warning: %v", err) + time.Sleep(15 * time.Second) + continue + } + etag := res.Header.Get("Etag") + if etag == "" { + log.Printf("Warning, no ETag in response: %v", res) + time.Sleep(15 * time.Second) + continue + } + if lastEtag != "" && etag != lastEtag { + log.Printf("Binary updated; restarting.") + // TODO: more graceful restart, letting systemd own the network connections. + // Then we can finish up requests here. + os.Exit(0) + } + lastEtag = etag + time.Sleep(15 * time.Second) + } +} + // uploadBinary uploads the currently-running Linux binary. // It crashes if it fails. func (cl *cloudLaunch) uploadBinary() { @@ -219,6 +266,46 @@ func getSelfPath() string { return v } +func zoneInRegion(zone, regionURL string) bool { + if zone == "" { + panic("empty zone") + } + if regionURL == "" { + panic("empty regionURL") + } + // zone is like "us-central1-f" + // regionURL is like "https://www.googleapis.com/compute/v1/projects/camlistore-website/regions/us-central1" + region := path.Base(regionURL) // "us-central1" + if region == "" { + panic("empty region") + } + return strings.HasPrefix(zone, region) +} + +// findIP finds an IP address to use, or returns the empty string if none is found. +// It tries to find a reserved one in the same region where the name of the reserved IP +// is "NAME-ip" and the IP is not in use. +func (cl *cloudLaunch) findIP() string { + // Try to find it by name. + aggAddrList, err := cl.computeService.Addresses.AggregatedList(cl.GCEProjectID).Do() + if err != nil { + log.Fatal(err) + } + // https://godoc.org/google.golang.org/api/compute/v1#AddressAggregatedList + var ip string +IPLoop: + for _, asl := range aggAddrList.Items { + for _, addr := range asl.Addresses { + log.Printf(" addr: %#v", addr) + if addr.Name == cl.Name+"-ip" && addr.Status == "RESERVED" && zoneInRegion(cl.zone(), addr.Region) { + ip = addr.Address + break IPLoop + } + } + } + return ip +} + func (cl *cloudLaunch) createInstance() { inst := cl.lookupInstance() if inst != nil { @@ -228,35 +315,14 @@ func (cl *cloudLaunch) createInstance() { log.Printf("Instance doesn't exist; creating...") + ip := cl.findIP() + log.Printf("Found IP: %v", ip) + cloudConfig := strings.NewReplacer( "$NAME", cl.Name, "$URL", cl.binaryURL(), ).Replace(baseConfig) - /* - // Try to find it by name. - aggAddrList, err := computeService.Addresses.AggregatedList(c.GCEProjectID).Do() - if err != nil { - log.Fatal(err) - } - // https://godoc.org/google.golang.org/api/compute/v1#AddressAggregatedList - log.Printf("Addr list: %v", aggAddrList.Items) - var ip string - IPLoop: - for _, asl := range aggAddrList.Items { - for _, addr := range asl.Addresses { - log.Printf(" addr: %#v", addr) - if addr.Name == c.Name+"-ip" && addr.Status == "RESERVED" { - ip = addr.Address - break IPLoop - } - } - } - log.Printf("Found IP: %v", ip) - */ - - natIP := "" - instance := &compute.Instance{ Name: cl.instName(), Description: cl.Name, @@ -279,7 +345,7 @@ func (cl *cloudLaunch) createInstance() { &compute.AccessConfig{ Type: "ONE_TO_ONE_NAT", Name: "External NAT", - NatIP: natIP, + NatIP: ip, }, }, Network: cl.projectAPIURL() + "/global/networks/default", diff --git a/website/camweb.go b/website/camweb.go index e4857f88d..28894cbeb 100644 --- a/website/camweb.go +++ b/website/camweb.go @@ -18,6 +18,7 @@ package main import ( "bytes" + "crypto/rand" "crypto/tls" "flag" "fmt" @@ -28,11 +29,13 @@ import ( "net" "net/http" "net/http/httputil" + "net/smtp" "net/url" "os" "os/exec" "path/filepath" "regexp" + "runtime" "strings" txttemplate "text/template" "time" @@ -41,6 +44,7 @@ import ( "camlistore.org/pkg/deploy/gce" "camlistore.org/pkg/googlestorage" "camlistore.org/pkg/netutil" + "camlistore.org/pkg/osutil" "camlistore.org/pkg/types/camtypes" "camlistore.org/third_party/github.com/russross/blackfriday" "golang.org/x/net/context" @@ -73,6 +77,10 @@ var ( gceLogName = flag.String("gce_log_name", "", "GCE Cloud Logging log name; if non-empty, logs go to Cloud Logging instead of Apache-style local disk log files") gceJWTFile = flag.String("gce_jwt_file", "", "If non-empty, a filename to the GCE Service Account's JWT (JSON) config file.") gitContainer = flag.Bool("git_container", false, "Use git in the camlistore/git Docker container.") +) + +var ( + inProd bool pageHTML, errorHTML, camliErrorHTML *template.Template packageHTML *txttemplate.Template @@ -396,7 +404,7 @@ var launchConfig = &cloudlaunch.Config{ }, } -func inProduction() bool { +func checkInProduction() bool { if !metadata.OnGCE() { return false } @@ -409,7 +417,8 @@ func inProduction() bool { const prodSrcDir = "/var/camweb/camsrc" func setProdFlags() { - if !inProduction() { + inProd = checkInProduction() + if !inProd { return } log.Printf("Running in production; configuring prod flags & containers") @@ -417,24 +426,111 @@ func setProdFlags() { *httpsAddr = ":443" *gceLogName = "camweb-access-log" *root = filepath.Join(prodSrcDir, "website") - *emailsTo = "" // TODO: renable emails on new commits *gitContainer = true + + *emailsTo = "camlistore-commits@googlegroups.com" + *smtpServer = "50.19.239.94:2500" // double firewall: rinetd allow + AWS + os.RemoveAll(prodSrcDir) if err := os.MkdirAll(prodSrcDir, 0755); err != nil { log.Fatal(err) } + log.Printf("fetching git docker image...") getDockerImage("camlistore/git", "docker-git.tar.gz") + getDockerImage("camlistore/demoblobserver", "docker-demoblobserver.tar.gz") + + log.Printf("cloning camlistore git tree...") out, err := exec.Command("docker", "run", + "--rm", "-v", "/var/camweb:/var/camweb", "camlistore/git", "git", "clone", + "--depth=1", "https://camlistore.googlesource.com/camlistore", prodSrcDir).CombinedOutput() if err != nil { log.Fatalf("git clone: %v, %s", err, out) } os.Chdir(*root) + log.Printf("Starting.") + sendStartingEmail() +} + +func randHex(n int) string { + buf := make([]byte, n/2+1) + rand.Read(buf) + return fmt.Sprintf("%x", buf)[:n] +} + +func runDemoBlobserverLoop() { + if runtime.GOOS != "linux" { + return + } + if _, err := exec.LookPath("docker"); err != nil { + return + } + for { + cmd := exec.Command("docker", "run", + "--rm", + "-e", "CAMLI_ROOT=/var/camweb/camsrc/website/blobserver-example/root", + "-e", "CAMLI_PASSWORD="+randHex(20), + "-v", camSrcDir()+":/var/camweb/camsrc", + "--net=host", + "--workdir=/var/camweb/camsrc", + "camlistore/demoblobserver", + "camlistored", + "--openbrowser=false", + "--listen=:3179", + "--configfile=/var/camweb/camsrc/website/blobserver-example/example-blobserver-config.json") + err := cmd.Run() + if err != nil { + log.Printf("Failed to run demo blob server: %v", err) + } + if !inProd { + return + } + time.Sleep(10 * time.Second) + } +} + +func sendStartingEmail() { + contentRev, err := exec.Command("docker", "run", + "--rm", + "-v", "/var/camweb:/var/camweb", + "-w", "/var/camweb/camsrc", + "camlistore/git", + "/bin/bash", "-c", + "git show --pretty=format:'%ad-%h' --abbrev-commit --date=short | head -1").Output() + + cl, err := smtp.Dial(*smtpServer) + if err != nil { + log.Printf("Failed to connect to SMTP server: %v", err) + } + defer cl.Quit() + if err = cl.Mail("noreply@camlistore.org"); err != nil { + return + } + if err = cl.Rcpt("brad@danga.com"); err != nil { + return + } + if err = cl.Rcpt("mathieu.lonjaret@gmail.com"); err != nil { + return + } + wc, err := cl.Data() + if err != nil { + return + } + _, err = fmt.Fprintf(wc, `From: noreply@camlistore.org (Camlistore Website) +To: brad@danga.com, mathieu.lonjaret@gmail.com +Subject: Camlistore camweb restarting + +Camlistore website starting with binary XXXXTODO and content at git rev %s +`, contentRev) + if err != nil { + return + } + wc.Close() } func getDockerImage(tag, file string) { @@ -462,6 +558,7 @@ func main() { } } readTemplates() + go runDemoBlobserverLoop() mux := http.DefaultServeMux mux.Handle("/favicon.ico", http.FileServer(http.Dir(filepath.Join(*root, "static")))) @@ -474,7 +571,8 @@ func main() { mux.HandleFunc("/r/", gerritRedirect) mux.HandleFunc("/dl/", releaseRedirect) - mux.HandleFunc("/debugz/ip", ipHandler) + mux.HandleFunc("/debug/ip", ipHandler) + mux.HandleFunc("/debug/uptime", uptimeHandler) mux.Handle("/docs/contributing", redirTo("/code#contributing")) mux.Handle("/lists", redirTo("/community")) @@ -574,10 +672,9 @@ func serveHTTPS(httpServer *http.Server) error { httpsServer := new(http.Server) *httpsServer = *httpServer httpsServer.Addr = *httpsAddr - if !inProduction() { + if !inProd { return httpsServer.ListenAndServeTLS(*tlsCertFile, *tlsKeyFile) } - cert, err := tlsCertFromGCS() if err != nil { return err @@ -713,6 +810,12 @@ func ipHandler(w http.ResponseWriter, r *http.Request) { w.Write([]byte(str)) } +var startTime = time.Now() + +func uptimeHandler(w http.ResponseWriter, r *http.Request) { + fmt.Fprintf(w, "%v", time.Now().Sub(startTime)) +} + const ( errPattern = "/err/" toHyperlink = `$1$2` @@ -740,3 +843,14 @@ func errHandler(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusFound) servePage(w, errString, "", contents) } + +func camSrcDir() string { + if inProd { + return prodSrcDir + } + dir, err := osutil.GoPackagePath("camlistore.org") + if err != nil { + log.Fatalf("Failed to find the root of the Camlistore source code via osutil.GoPackagePath: %v", err) + } + return dir +} diff --git a/website/containers/demoblobserver/Dockerfile b/website/containers/demoblobserver/Dockerfile new file mode 100644 index 000000000..4303496ab --- /dev/null +++ b/website/containers/demoblobserver/Dockerfile @@ -0,0 +1,10 @@ +# Copyright 2015 The Camlistore Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +FROM debian:wheezy + +ENV DEBIAN_FRONTEND noninteractive + +ADD build.sh /scripts/build.sh +RUN /scripts/build.sh diff --git a/website/containers/demoblobserver/Makefile b/website/containers/demoblobserver/Makefile new file mode 100644 index 000000000..baf211845 --- /dev/null +++ b/website/containers/demoblobserver/Makefile @@ -0,0 +1,7 @@ +container: + docker build -t camlistore/demoblobserver . + +upload: + go get golang.org/x/build/cmd/upload + docker save camlistore/demoblobserver | gzip | upload --public --project=camlistore-website camlistore-website-resource/docker-demoblobserver.tar.gz + diff --git a/website/containers/demoblobserver/build.sh b/website/containers/demoblobserver/build.sh new file mode 100755 index 000000000..8a59bff73 --- /dev/null +++ b/website/containers/demoblobserver/build.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +set -e + +apt-get update +apt-get install -y --no-install-recommends curl git-core ca-certificates + +curl --silent https://storage.googleapis.com/golang/go1.5.1.linux-amd64.tar.gz | tar -C /usr/local -zxv +mkdir -p /gopath/src +git clone --depth=1 https://camlistore.googlesource.com/camlistore /gopath/src/camlistore.org + +export GOPATH=/gopath +export GOBIN=/usr/local/bin +export GO15VENDOREXPERIMENT=1 +export CGO_ENABLED=0 +/usr/local/go/bin/go install -v camlistore.org/server/camlistored + +rm -rf /usr/local/go +apt-get remove --yes curl git-core +apt-get clean +rm -rf /var/cache/apt/ +rm -fr /var/lib/apt/lists diff --git a/website/contributors.go b/website/contributors.go index 872a7c443..1abf92ba2 100644 --- a/website/contributors.go +++ b/website/contributors.go @@ -76,8 +76,8 @@ func gitShortlog() *exec.Cmd { if !*gitContainer { return exec.Command("/bin/bash", "-c", "git log | git shortlog -sen") } - args := []string{"run"} - if inProduction() { + args := []string{"run", "--rm"} + if inProd { args = append(args, "-v", "/var/camweb:/var/camweb", "--workdir="+prodSrcDir, diff --git a/website/email.go b/website/email.go index c17468faf..7ad9a1d74 100644 --- a/website/email.go +++ b/website/email.go @@ -71,8 +71,7 @@ var knownCommit = map[string]bool{} // commit -> true var diffMarker = []byte("diff --git a/") func emailCommit(dir, hash string) (err error) { - cmd := exec.Command("git", "show", hash) - cmd.Dir = dir + cmd := execGit(dir, "show", hash) body, err := cmd.CombinedOutput() if err != nil { return fmt.Errorf("Error runnning git show: %v\n%s", err, body) @@ -82,8 +81,7 @@ func emailCommit(dir, hash string) (err error) { return nil } - cmd = exec.Command("git", "show", "--pretty=oneline", hash) - cmd.Dir = dir + cmd = execGit(dir, "show", "--pretty=oneline", hash) out, err := cmd.Output() if err != nil { return @@ -143,10 +141,7 @@ func commitEmailLoop() error { } }() - dir, err := osutil.GoPackagePath("camlistore.org") - if err != nil { - return err - } + dir := camSrcDir() hashes, err := recentCommits(dir) if err != nil { @@ -173,9 +168,27 @@ func commitEmailLoop() error { } } +func execGit(dir string, gitArgs ...string) *exec.Cmd { + var cmd *exec.Cmd + if *gitContainer { + args := append([]string{ + "run", + "--rm", + "-v", dir + ":" + dir, + "--workdir=" + dir, + "camlistore/git", + "git", + }, gitArgs...) + cmd = exec.Command("docker", args...) + } else { + cmd = exec.Command("git", gitArgs...) + cmd.Dir = dir + } + return cmd +} + func pollCommits(dir string) { - cmd := exec.Command("git", "fetch", "origin") - cmd.Dir = dir + cmd := execGit(dir, "fetch", "origin") out, err := cmd.CombinedOutput() if err != nil { log.Printf("Error running git fetch origin master in %s: %v\n%s", dir, err, out) @@ -204,8 +217,7 @@ func pollCommits(dir string) { } func recentCommits(dir string) (hashes []string, err error) { - cmd := exec.Command("git", "log", "--since=1 month ago", "--pretty=oneline", "origin/master") - cmd.Dir = dir + cmd := execGit(dir, "log", "--since=1 month ago", "--pretty=oneline", "origin/master") out, err := cmd.CombinedOutput() if err != nil { return nil, fmt.Errorf("Error running git log in %s: %v\n%s", dir, err, out) diff --git a/website/run.pl b/website/run.pl deleted file mode 100755 index 61f3707e5..000000000 --- a/website/run.pl +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/perl - -use strict; -use FindBin qw($Bin); - -my $logdir = "$Bin/../logs"; - -unless (-d $logdir) { - mkdir $logdir, 0700 or die "mkdir: $!"; -} - -my $HOME = $ENV{HOME}; -chdir $Bin or die; - -print STDERR "Running camweb in $Bin on port 8080\n"; - -my $in_prod = -e "$HOME/etc/ssl.key"; # heuristic. good enough. - -my @args; -push @args, "go", "run", "camweb.go", "logging.go", "contributors.go", "godoc.go", "format.go", "dirtrees.go", "email.go"; -push @args, "--root=$Bin"; -push @args, "--logdir=$logdir"; -push @args, "--buildbot_host=build.camlistore.org"; -push @args, "--buildbot_backend=http://c1.danga.com:8080"; -push @args, "--also_run=$Bin/scripts/run-blobserver"; -if ($in_prod) { - push @args, "--email_dest=camlistore-commits\@googlegroups.com"; - push @args, "--http=:8080"; - push @args, "--https=:4430"; - push @args, "--tlscert=$HOME/etc/ssl.crt"; - push @args, "--tlskey=$HOME/etc/ssl.key"; - while (1) { - system(@args); - print STDERR "Exit: $?; sleeping/restarting...\n"; - sleep 5; - } -} else { - my $pass_file = "$ENV{HOME}/.config/camlistore/camorg-blobserver.pass"; - unless (-s $pass_file) { - `mkdir -p $ENV{HOME}/.config/camlistore/`; - open (my $fh, ">$pass_file"); - print $fh "foo\n"; - close($fh); - } - # These https certificate and key are the default ones used by devcam server. - die "TLS cert or key not initialized; run devcam server --tls" unless -e "$Bin/../config/tls.crt" && -e "$Bin/../config/tls.key"; - push @args, "--tlscert=$Bin/../config/tls.crt"; - push @args, "--tlskey=$Bin/../config/tls.key"; - push @args, "--http=127.0.0.1:8080"; - push @args, "--https=127.0.0.1:4430"; - push @args, @ARGV; - exec(@args); - die "Failed to exec: $!"; -}