website: run on GCE under CoreOS. Containerize demo blob server.

pkg/cloudlaunch/cloudlaunch.go

@@ -22,11 +22,13 @@ import (
 	"encoding/json"
 	"errors"
 	"flag"
+	"fmt"
 	"io"
 	"io/ioutil"
 	"log"
 	"net/http"
 	"os"
+	"path"
 	"path/filepath"
 	"runtime"
 	"strings"
@@ -74,12 +76,26 @@ coreos:
         WantedBy=network-online.target
 `
 
+// RestartPolicy controls whether the binary automatically restarts.
+type RestartPolicy int
+
+const (
+	RestartOnUpdates RestartPolicy = iota
+	RestartNever
+	// TODO: more graceful restarts; make systemd own listening on network sockets,
+	// don't break connections.
+)
+
 type Config struct {
 	// Name is the name of a service to run.
 	// This is the name of the systemd service (without .service)
 	// and the name of the GCE instance.
 	Name string
 
+	// RestartPolicy controls whether the binary automatically restarts
+	// on updates. The zero value means automatic.
+	RestartPolicy RestartPolicy
+
 	// BinaryBucket and BinaryObject are the GCS bucket and object
 	// within that bucket containing the Linux binary to download
 	// on boot and occasionally run. This binary must be public
@@ -134,6 +150,7 @@ var (
 func (c *Config) MaybeDeploy() {
 	flag.Parse()
 	if !*doLaunch {
+		go c.restartLoop()
 		return
 	}
 	defer os.Exit(1) // backup, in case we return without Fatal or os.Exit later
@@ -167,6 +184,36 @@ func (c *Config) MaybeDeploy() {
 	os.Exit(0)
 }
 
+func (c *Config) restartLoop() {
+	if c.RestartPolicy == RestartNever {
+		return
+	}
+	url := "https://storage.googleapis.com/" + c.BinaryBucket + "/" + c.binaryObject()
+	var lastEtag string
+	for {
+		res, err := http.Head(url + "?" + fmt.Sprint(time.Now().Unix()))
+		if err != nil {
+			log.Printf("Warning: %v", err)
+			time.Sleep(15 * time.Second)
+			continue
+		}
+		etag := res.Header.Get("Etag")
+		if etag == "" {
+			log.Printf("Warning, no ETag in response: %v", res)
+			time.Sleep(15 * time.Second)
+			continue
+		}
+		if lastEtag != "" && etag != lastEtag {
+			log.Printf("Binary updated; restarting.")
+			// TODO: more graceful restart, letting systemd own the network connections.
+			// Then we can finish up requests here.
+			os.Exit(0)
+		}
+		lastEtag = etag
+		time.Sleep(15 * time.Second)
+	}
+}
+
 // uploadBinary uploads the currently-running Linux binary.
 // It crashes if it fails.
 func (cl *cloudLaunch) uploadBinary() {
@@ -219,6 +266,46 @@ func getSelfPath() string {
 	return v
 }
 
+func zoneInRegion(zone, regionURL string) bool {
+	if zone == "" {
+		panic("empty zone")
+	}
+	if regionURL == "" {
+		panic("empty regionURL")
+	}
+	// zone is like "us-central1-f"
+	// regionURL is like "https://www.googleapis.com/compute/v1/projects/camlistore-website/regions/us-central1"
+	region := path.Base(regionURL) // "us-central1"
+	if region == "" {
+		panic("empty region")
+	}
+	return strings.HasPrefix(zone, region)
+}
+
+// findIP finds an IP address to use, or returns the empty string if none is found.
+// It tries to find a reserved one in the same region where the name of the reserved IP
+// is "NAME-ip" and the IP is not in use.
+func (cl *cloudLaunch) findIP() string {
+	// Try to find it by name.
+	aggAddrList, err := cl.computeService.Addresses.AggregatedList(cl.GCEProjectID).Do()
+	if err != nil {
+		log.Fatal(err)
+	}
+	// https://godoc.org/google.golang.org/api/compute/v1#AddressAggregatedList
+	var ip string
+IPLoop:
+	for _, asl := range aggAddrList.Items {
+		for _, addr := range asl.Addresses {
+			log.Printf("  addr: %#v", addr)
+			if addr.Name == cl.Name+"-ip" && addr.Status == "RESERVED" && zoneInRegion(cl.zone(), addr.Region) {
+				ip = addr.Address
+				break IPLoop
+			}
+		}
+	}
+	return ip
+}
+
 func (cl *cloudLaunch) createInstance() {
 	inst := cl.lookupInstance()
 	if inst != nil {
@@ -228,35 +315,14 @@ func (cl *cloudLaunch) createInstance() {
 
 	log.Printf("Instance doesn't exist; creating...")
 
+	ip := cl.findIP()
+	log.Printf("Found IP: %v", ip)
+
 	cloudConfig := strings.NewReplacer(
 		"$NAME", cl.Name,
 		"$URL", cl.binaryURL(),
 	).Replace(baseConfig)
 
-	/*
-	   	// Try to find it by name.
-	   	aggAddrList, err := computeService.Addresses.AggregatedList(c.GCEProjectID).Do()
-	   	if err != nil {
-	   		log.Fatal(err)
-	   	}
-	   	// https://godoc.org/google.golang.org/api/compute/v1#AddressAggregatedList
-	   	log.Printf("Addr list: %v", aggAddrList.Items)
-	   	var ip string
-	   IPLoop:
-	   	for _, asl := range aggAddrList.Items {
-	   		for _, addr := range asl.Addresses {
-	   			log.Printf("  addr: %#v", addr)
-	   			if addr.Name == c.Name+"-ip" && addr.Status == "RESERVED" {
-	   				ip = addr.Address
-	   				break IPLoop
-	   			}
-	   		}
-	   	}
-	   	log.Printf("Found IP: %v", ip)
-	*/
-
-	natIP := ""
-
 	instance := &compute.Instance{
 		Name:        cl.instName(),
 		Description: cl.Name,
@@ -279,7 +345,7 @@ func (cl *cloudLaunch) createInstance() {
 					&compute.AccessConfig{
 						Type:  "ONE_TO_ONE_NAT",
 						Name:  "External NAT",
-						NatIP: natIP,
+						NatIP: ip,
 					},
 				},
 				Network: cl.projectAPIURL() + "/global/networks/default",

website/camweb.go

@@ -18,6 +18,7 @@ package main
 
 import (
 	"bytes"
+	"crypto/rand"
 	"crypto/tls"
 	"flag"
 	"fmt"
@@ -28,11 +29,13 @@ import (
 	"net"
 	"net/http"
 	"net/http/httputil"
+	"net/smtp"
 	"net/url"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"regexp"
+	"runtime"
 	"strings"
 	txttemplate "text/template"
 	"time"
@@ -41,6 +44,7 @@ import (
 	"camlistore.org/pkg/deploy/gce"
 	"camlistore.org/pkg/googlestorage"
 	"camlistore.org/pkg/netutil"
+	"camlistore.org/pkg/osutil"
 	"camlistore.org/pkg/types/camtypes"
 	"camlistore.org/third_party/github.com/russross/blackfriday"
 	"golang.org/x/net/context"
@@ -73,6 +77,10 @@ var (
 	gceLogName   = flag.String("gce_log_name", "", "GCE Cloud Logging log name; if non-empty, logs go to Cloud Logging instead of Apache-style local disk log files")
 	gceJWTFile   = flag.String("gce_jwt_file", "", "If non-empty, a filename to the GCE Service Account's JWT (JSON) config file.")
 	gitContainer = flag.Bool("git_container", false, "Use git in the camlistore/git Docker container.")
+)
+
+var (
+	inProd bool
 
 	pageHTML, errorHTML, camliErrorHTML *template.Template
 	packageHTML                         *txttemplate.Template
@@ -396,7 +404,7 @@ var launchConfig = &cloudlaunch.Config{
 	},
 }
 
-func inProduction() bool {
+func checkInProduction() bool {
 	if !metadata.OnGCE() {
 		return false
 	}
@@ -409,7 +417,8 @@ func inProduction() bool {
 const prodSrcDir = "/var/camweb/camsrc"
 
 func setProdFlags() {
-	if !inProduction() {
+	inProd = checkInProduction()
+	if !inProd {
 		return
 	}
 	log.Printf("Running in production; configuring prod flags & containers")
@@ -417,24 +426,111 @@ func setProdFlags() {
 	*httpsAddr = ":443"
 	*gceLogName = "camweb-access-log"
 	*root = filepath.Join(prodSrcDir, "website")
-	*emailsTo = "" // TODO: renable emails on new commits
 	*gitContainer = true
+
+	*emailsTo = "camlistore-commits@googlegroups.com"
+	*smtpServer = "50.19.239.94:2500" // double firewall: rinetd allow + AWS
+
 	os.RemoveAll(prodSrcDir)
 	if err := os.MkdirAll(prodSrcDir, 0755); err != nil {
 		log.Fatal(err)
 	}
+	log.Printf("fetching git docker image...")
 	getDockerImage("camlistore/git", "docker-git.tar.gz")
+	getDockerImage("camlistore/demoblobserver", "docker-demoblobserver.tar.gz")
+
+	log.Printf("cloning camlistore git tree...")
 	out, err := exec.Command("docker", "run",
+		"--rm",
 		"-v", "/var/camweb:/var/camweb",
 		"camlistore/git",
 		"git",
 		"clone",
+		"--depth=1",
 		"https://camlistore.googlesource.com/camlistore",
 		prodSrcDir).CombinedOutput()
 	if err != nil {
 		log.Fatalf("git clone: %v, %s", err, out)
 	}
 	os.Chdir(*root)
+	log.Printf("Starting.")
+	sendStartingEmail()
+}
+
+func randHex(n int) string {
+	buf := make([]byte, n/2+1)
+	rand.Read(buf)
+	return fmt.Sprintf("%x", buf)[:n]
+}
+
+func runDemoBlobserverLoop() {
+	if runtime.GOOS != "linux" {
+		return
+	}
+	if _, err := exec.LookPath("docker"); err != nil {
+		return
+	}
+	for {
+		cmd := exec.Command("docker", "run",
+			"--rm",
+			"-e", "CAMLI_ROOT=/var/camweb/camsrc/website/blobserver-example/root",
+			"-e", "CAMLI_PASSWORD="+randHex(20),
+			"-v", camSrcDir()+":/var/camweb/camsrc",
+			"--net=host",
+			"--workdir=/var/camweb/camsrc",
+			"camlistore/demoblobserver",
+			"camlistored",
+			"--openbrowser=false",
+			"--listen=:3179",
+			"--configfile=/var/camweb/camsrc/website/blobserver-example/example-blobserver-config.json")
+		err := cmd.Run()
+		if err != nil {
+			log.Printf("Failed to run demo blob server: %v", err)
+		}
+		if !inProd {
+			return
+		}
+		time.Sleep(10 * time.Second)
+	}
+}
+
+func sendStartingEmail() {
+	contentRev, err := exec.Command("docker", "run",
+		"--rm",
+		"-v", "/var/camweb:/var/camweb",
+		"-w", "/var/camweb/camsrc",
+		"camlistore/git",
+		"/bin/bash", "-c",
+		"git show --pretty=format:'%ad-%h' --abbrev-commit --date=short | head -1").Output()
+
+	cl, err := smtp.Dial(*smtpServer)
+	if err != nil {
+		log.Printf("Failed to connect to SMTP server: %v", err)
+	}
+	defer cl.Quit()
+	if err = cl.Mail("noreply@camlistore.org"); err != nil {
+		return
+	}
+	if err = cl.Rcpt("brad@danga.com"); err != nil {
+		return
+	}
+	if err = cl.Rcpt("mathieu.lonjaret@gmail.com"); err != nil {
+		return
+	}
+	wc, err := cl.Data()
+	if err != nil {
+		return
+	}
+	_, err = fmt.Fprintf(wc, `From: noreply@camlistore.org (Camlistore Website)
+To: brad@danga.com, mathieu.lonjaret@gmail.com
+Subject: Camlistore camweb restarting
+
+Camlistore website starting with binary XXXXTODO and content at git rev %s
+`, contentRev)
+	if err != nil {
+		return
+	}
+	wc.Close()
 }
 
 func getDockerImage(tag, file string) {
@@ -462,6 +558,7 @@ func main() {
 		}
 	}
 	readTemplates()
+	go runDemoBlobserverLoop()
 
 	mux := http.DefaultServeMux
 	mux.Handle("/favicon.ico", http.FileServer(http.Dir(filepath.Join(*root, "static"))))
@@ -474,7 +571,8 @@ func main() {
 
 	mux.HandleFunc("/r/", gerritRedirect)
 	mux.HandleFunc("/dl/", releaseRedirect)
-	mux.HandleFunc("/debugz/ip", ipHandler)
+	mux.HandleFunc("/debug/ip", ipHandler)
+	mux.HandleFunc("/debug/uptime", uptimeHandler)
 	mux.Handle("/docs/contributing", redirTo("/code#contributing"))
 	mux.Handle("/lists", redirTo("/community"))
 
@@ -574,10 +672,9 @@ func serveHTTPS(httpServer *http.Server) error {
 	httpsServer := new(http.Server)
 	*httpsServer = *httpServer
 	httpsServer.Addr = *httpsAddr
-	if !inProduction() {
+	if !inProd {
 		return httpsServer.ListenAndServeTLS(*tlsCertFile, *tlsKeyFile)
 	}
-
 	cert, err := tlsCertFromGCS()
 	if err != nil {
 		return err
@@ -713,6 +810,12 @@ func ipHandler(w http.ResponseWriter, r *http.Request) {
 	w.Write([]byte(str))
 }
 
+var startTime = time.Now()
+
+func uptimeHandler(w http.ResponseWriter, r *http.Request) {
+	fmt.Fprintf(w, "%v", time.Now().Sub(startTime))
+}
+
 const (
 	errPattern  = "/err/"
 	toHyperlink = `<a href="$1$2">$1$2</a>`
@@ -740,3 +843,14 @@ func errHandler(w http.ResponseWriter, r *http.Request) {
 	w.WriteHeader(http.StatusFound)
 	servePage(w, errString, "", contents)
 }
+
+func camSrcDir() string {
+	if inProd {
+		return prodSrcDir
+	}
+	dir, err := osutil.GoPackagePath("camlistore.org")
+	if err != nil {
+		log.Fatalf("Failed to find the root of the Camlistore source code via osutil.GoPackagePath: %v", err)
+	}
+	return dir
+}

website/containers/demoblobserver/Dockerfile

@@ -0,0 +1,10 @@
+# Copyright 2015 The Camlistore Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+FROM debian:wheezy
+
+ENV DEBIAN_FRONTEND noninteractive
+
+ADD build.sh /scripts/build.sh
+RUN /scripts/build.sh

website/containers/demoblobserver/Makefile

@@ -0,0 +1,7 @@
+container:
+	docker build -t camlistore/demoblobserver .
+
+upload:
+	go get golang.org/x/build/cmd/upload
+	docker save camlistore/demoblobserver | gzip | upload --public  --project=camlistore-website camlistore-website-resource/docker-demoblobserver.tar.gz
+

website/containers/demoblobserver/build.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+set -e
+
+apt-get update
+apt-get install -y --no-install-recommends curl git-core ca-certificates
+
+curl --silent https://storage.googleapis.com/golang/go1.5.1.linux-amd64.tar.gz | tar -C /usr/local -zxv
+mkdir -p /gopath/src
+git clone --depth=1 https://camlistore.googlesource.com/camlistore /gopath/src/camlistore.org
+
+export GOPATH=/gopath
+export GOBIN=/usr/local/bin
+export GO15VENDOREXPERIMENT=1
+export CGO_ENABLED=0
+/usr/local/go/bin/go install -v camlistore.org/server/camlistored
+
+rm -rf /usr/local/go
+apt-get remove --yes curl git-core
+apt-get clean
+rm -rf /var/cache/apt/
+rm -fr /var/lib/apt/lists

website/contributors.go

@@ -76,8 +76,8 @@ func gitShortlog() *exec.Cmd {
 	if !*gitContainer {
 		return exec.Command("/bin/bash", "-c", "git log | git shortlog -sen")
 	}
-	args := []string{"run"}
+	args := []string{"run", "--rm"}
-	if inProduction() {
+	if inProd {
 		args = append(args,
 			"-v", "/var/camweb:/var/camweb",
 			"--workdir="+prodSrcDir,

website/email.go

@@ -71,8 +71,7 @@ var knownCommit = map[string]bool{} // commit -> true
 var diffMarker = []byte("diff --git a/")
 
 func emailCommit(dir, hash string) (err error) {
-	cmd := exec.Command("git", "show", hash)
+	cmd := execGit(dir, "show", hash)
-	cmd.Dir = dir
 	body, err := cmd.CombinedOutput()
 	if err != nil {
 		return fmt.Errorf("Error runnning git show: %v\n%s", err, body)
@@ -82,8 +81,7 @@ func emailCommit(dir, hash string) (err error) {
 		return nil
 	}
 
-	cmd = exec.Command("git", "show", "--pretty=oneline", hash)
+	cmd = execGit(dir, "show", "--pretty=oneline", hash)
-	cmd.Dir = dir
 	out, err := cmd.Output()
 	if err != nil {
 		return
@@ -143,10 +141,7 @@ func commitEmailLoop() error {
 		}
 	}()
 
-	dir, err := osutil.GoPackagePath("camlistore.org")
+	dir := camSrcDir()
-	if err != nil {
-		return err
-	}
 
 	hashes, err := recentCommits(dir)
 	if err != nil {
@@ -173,9 +168,27 @@ func commitEmailLoop() error {
 	}
 }
 
+func execGit(dir string, gitArgs ...string) *exec.Cmd {
+	var cmd *exec.Cmd
+	if *gitContainer {
+		args := append([]string{
+			"run",
+			"--rm",
+			"-v", dir + ":" + dir,
+			"--workdir=" + dir,
+			"camlistore/git",
+			"git",
+		}, gitArgs...)
+		cmd = exec.Command("docker", args...)
+	} else {
+		cmd = exec.Command("git", gitArgs...)
+		cmd.Dir = dir
+	}
+	return cmd
+}
+
 func pollCommits(dir string) {
-	cmd := exec.Command("git", "fetch", "origin")
+	cmd := execGit(dir, "fetch", "origin")
-	cmd.Dir = dir
 	out, err := cmd.CombinedOutput()
 	if err != nil {
 		log.Printf("Error running git fetch origin master in %s: %v\n%s", dir, err, out)
@@ -204,8 +217,7 @@ func pollCommits(dir string) {
 }
 
 func recentCommits(dir string) (hashes []string, err error) {
-	cmd := exec.Command("git", "log", "--since=1 month ago", "--pretty=oneline", "origin/master")
+	cmd := execGit(dir, "log", "--since=1 month ago", "--pretty=oneline", "origin/master")
-	cmd.Dir = dir
 	out, err := cmd.CombinedOutput()
 	if err != nil {
 		return nil, fmt.Errorf("Error running git log in %s: %v\n%s", dir, err, out)

website/run.pl

@@ -1,54 +0,0 @@
-#!/usr/bin/perl
-
-use strict;
-use FindBin qw($Bin);
-
-my $logdir = "$Bin/../logs";
-
-unless (-d $logdir) {
-    mkdir $logdir, 0700 or die "mkdir: $!";
-}
-
-my $HOME = $ENV{HOME};
-chdir $Bin or die;
-
-print STDERR "Running camweb in $Bin on port 8080\n";
-
-my $in_prod = -e "$HOME/etc/ssl.key"; # heuristic. good enough.
-
-my @args;
-push @args, "go", "run", "camweb.go", "logging.go", "contributors.go", "godoc.go", "format.go", "dirtrees.go", "email.go";
-push @args, "--root=$Bin";
-push @args, "--logdir=$logdir";
-push @args, "--buildbot_host=build.camlistore.org";
-push @args, "--buildbot_backend=http://c1.danga.com:8080";
-push @args, "--also_run=$Bin/scripts/run-blobserver";
-if ($in_prod) {
-    push @args, "--email_dest=camlistore-commits\@googlegroups.com";
-    push @args, "--http=:8080";
-    push @args, "--https=:4430";
-    push @args, "--tlscert=$HOME/etc/ssl.crt";
-    push @args, "--tlskey=$HOME/etc/ssl.key";
-    while (1) {
-        system(@args);
-        print STDERR "Exit: $?; sleeping/restarting...\n";
-        sleep 5;
-    }
-} else {
-    my $pass_file = "$ENV{HOME}/.config/camlistore/camorg-blobserver.pass";
-    unless (-s $pass_file) {
-        `mkdir -p $ENV{HOME}/.config/camlistore/`;
-        open (my $fh, ">$pass_file");
-        print $fh "foo\n";
-        close($fh);
-    }
-    # These https certificate and key are the default ones used by devcam server.
-    die "TLS cert or key not initialized; run devcam server --tls" unless -e "$Bin/../config/tls.crt" && -e "$Bin/../config/tls.key";
-    push @args, "--tlscert=$Bin/../config/tls.crt";
-    push @args, "--tlskey=$Bin/../config/tls.key";
-    push @args, "--http=127.0.0.1:8080";
-    push @args, "--https=127.0.0.1:4430";
-    push @args, @ARGV;
-    exec(@args);
-    die "Failed to exec: $!";
-}