website: run on GCE under CoreOS. Containerize demo blob server.

This commit is contained in:
Brad Fitzpatrick 2015-11-11 12:35:31 +00:00
parent b4ef019bc2
commit 96dc004af7
8 changed files with 276 additions and 99 deletions

View File

@ -22,11 +22,13 @@ import (
"encoding/json"
"errors"
"flag"
"fmt"
"io"
"io/ioutil"
"log"
"net/http"
"os"
"path"
"path/filepath"
"runtime"
"strings"
@ -74,12 +76,26 @@ coreos:
WantedBy=network-online.target
`
// RestartPolicy controls whether the binary automatically restarts.
type RestartPolicy int
const (
RestartOnUpdates RestartPolicy = iota
RestartNever
// TODO: more graceful restarts; make systemd own listening on network sockets,
// don't break connections.
)
type Config struct {
// Name is the name of a service to run.
// This is the name of the systemd service (without .service)
// and the name of the GCE instance.
Name string
// RestartPolicy controls whether the binary automatically restarts
// on updates. The zero value means automatic.
RestartPolicy RestartPolicy
// BinaryBucket and BinaryObject are the GCS bucket and object
// within that bucket containing the Linux binary to download
// on boot and occasionally run. This binary must be public
@ -134,6 +150,7 @@ var (
func (c *Config) MaybeDeploy() {
flag.Parse()
if !*doLaunch {
go c.restartLoop()
return
}
defer os.Exit(1) // backup, in case we return without Fatal or os.Exit later
@ -167,6 +184,36 @@ func (c *Config) MaybeDeploy() {
os.Exit(0)
}
func (c *Config) restartLoop() {
if c.RestartPolicy == RestartNever {
return
}
url := "https://storage.googleapis.com/" + c.BinaryBucket + "/" + c.binaryObject()
var lastEtag string
for {
res, err := http.Head(url + "?" + fmt.Sprint(time.Now().Unix()))
if err != nil {
log.Printf("Warning: %v", err)
time.Sleep(15 * time.Second)
continue
}
etag := res.Header.Get("Etag")
if etag == "" {
log.Printf("Warning, no ETag in response: %v", res)
time.Sleep(15 * time.Second)
continue
}
if lastEtag != "" && etag != lastEtag {
log.Printf("Binary updated; restarting.")
// TODO: more graceful restart, letting systemd own the network connections.
// Then we can finish up requests here.
os.Exit(0)
}
lastEtag = etag
time.Sleep(15 * time.Second)
}
}
// uploadBinary uploads the currently-running Linux binary.
// It crashes if it fails.
func (cl *cloudLaunch) uploadBinary() {
@ -219,6 +266,46 @@ func getSelfPath() string {
return v
}
func zoneInRegion(zone, regionURL string) bool {
if zone == "" {
panic("empty zone")
}
if regionURL == "" {
panic("empty regionURL")
}
// zone is like "us-central1-f"
// regionURL is like "https://www.googleapis.com/compute/v1/projects/camlistore-website/regions/us-central1"
region := path.Base(regionURL) // "us-central1"
if region == "" {
panic("empty region")
}
return strings.HasPrefix(zone, region)
}
// findIP finds an IP address to use, or returns the empty string if none is found.
// It tries to find a reserved one in the same region where the name of the reserved IP
// is "NAME-ip" and the IP is not in use.
func (cl *cloudLaunch) findIP() string {
// Try to find it by name.
aggAddrList, err := cl.computeService.Addresses.AggregatedList(cl.GCEProjectID).Do()
if err != nil {
log.Fatal(err)
}
// https://godoc.org/google.golang.org/api/compute/v1#AddressAggregatedList
var ip string
IPLoop:
for _, asl := range aggAddrList.Items {
for _, addr := range asl.Addresses {
log.Printf(" addr: %#v", addr)
if addr.Name == cl.Name+"-ip" && addr.Status == "RESERVED" && zoneInRegion(cl.zone(), addr.Region) {
ip = addr.Address
break IPLoop
}
}
}
return ip
}
func (cl *cloudLaunch) createInstance() {
inst := cl.lookupInstance()
if inst != nil {
@ -228,35 +315,14 @@ func (cl *cloudLaunch) createInstance() {
log.Printf("Instance doesn't exist; creating...")
ip := cl.findIP()
log.Printf("Found IP: %v", ip)
cloudConfig := strings.NewReplacer(
"$NAME", cl.Name,
"$URL", cl.binaryURL(),
).Replace(baseConfig)
/*
// Try to find it by name.
aggAddrList, err := computeService.Addresses.AggregatedList(c.GCEProjectID).Do()
if err != nil {
log.Fatal(err)
}
// https://godoc.org/google.golang.org/api/compute/v1#AddressAggregatedList
log.Printf("Addr list: %v", aggAddrList.Items)
var ip string
IPLoop:
for _, asl := range aggAddrList.Items {
for _, addr := range asl.Addresses {
log.Printf(" addr: %#v", addr)
if addr.Name == c.Name+"-ip" && addr.Status == "RESERVED" {
ip = addr.Address
break IPLoop
}
}
}
log.Printf("Found IP: %v", ip)
*/
natIP := ""
instance := &compute.Instance{
Name: cl.instName(),
Description: cl.Name,
@ -279,7 +345,7 @@ func (cl *cloudLaunch) createInstance() {
&compute.AccessConfig{
Type: "ONE_TO_ONE_NAT",
Name: "External NAT",
NatIP: natIP,
NatIP: ip,
},
},
Network: cl.projectAPIURL() + "/global/networks/default",

View File

@ -18,6 +18,7 @@ package main
import (
"bytes"
"crypto/rand"
"crypto/tls"
"flag"
"fmt"
@ -28,11 +29,13 @@ import (
"net"
"net/http"
"net/http/httputil"
"net/smtp"
"net/url"
"os"
"os/exec"
"path/filepath"
"regexp"
"runtime"
"strings"
txttemplate "text/template"
"time"
@ -41,6 +44,7 @@ import (
"camlistore.org/pkg/deploy/gce"
"camlistore.org/pkg/googlestorage"
"camlistore.org/pkg/netutil"
"camlistore.org/pkg/osutil"
"camlistore.org/pkg/types/camtypes"
"camlistore.org/third_party/github.com/russross/blackfriday"
"golang.org/x/net/context"
@ -73,6 +77,10 @@ var (
gceLogName = flag.String("gce_log_name", "", "GCE Cloud Logging log name; if non-empty, logs go to Cloud Logging instead of Apache-style local disk log files")
gceJWTFile = flag.String("gce_jwt_file", "", "If non-empty, a filename to the GCE Service Account's JWT (JSON) config file.")
gitContainer = flag.Bool("git_container", false, "Use git in the camlistore/git Docker container.")
)
var (
inProd bool
pageHTML, errorHTML, camliErrorHTML *template.Template
packageHTML *txttemplate.Template
@ -396,7 +404,7 @@ var launchConfig = &cloudlaunch.Config{
},
}
func inProduction() bool {
func checkInProduction() bool {
if !metadata.OnGCE() {
return false
}
@ -409,7 +417,8 @@ func inProduction() bool {
const prodSrcDir = "/var/camweb/camsrc"
func setProdFlags() {
if !inProduction() {
inProd = checkInProduction()
if !inProd {
return
}
log.Printf("Running in production; configuring prod flags & containers")
@ -417,24 +426,111 @@ func setProdFlags() {
*httpsAddr = ":443"
*gceLogName = "camweb-access-log"
*root = filepath.Join(prodSrcDir, "website")
*emailsTo = "" // TODO: renable emails on new commits
*gitContainer = true
*emailsTo = "camlistore-commits@googlegroups.com"
*smtpServer = "50.19.239.94:2500" // double firewall: rinetd allow + AWS
os.RemoveAll(prodSrcDir)
if err := os.MkdirAll(prodSrcDir, 0755); err != nil {
log.Fatal(err)
}
log.Printf("fetching git docker image...")
getDockerImage("camlistore/git", "docker-git.tar.gz")
getDockerImage("camlistore/demoblobserver", "docker-demoblobserver.tar.gz")
log.Printf("cloning camlistore git tree...")
out, err := exec.Command("docker", "run",
"--rm",
"-v", "/var/camweb:/var/camweb",
"camlistore/git",
"git",
"clone",
"--depth=1",
"https://camlistore.googlesource.com/camlistore",
prodSrcDir).CombinedOutput()
if err != nil {
log.Fatalf("git clone: %v, %s", err, out)
}
os.Chdir(*root)
log.Printf("Starting.")
sendStartingEmail()
}
func randHex(n int) string {
buf := make([]byte, n/2+1)
rand.Read(buf)
return fmt.Sprintf("%x", buf)[:n]
}
func runDemoBlobserverLoop() {
if runtime.GOOS != "linux" {
return
}
if _, err := exec.LookPath("docker"); err != nil {
return
}
for {
cmd := exec.Command("docker", "run",
"--rm",
"-e", "CAMLI_ROOT=/var/camweb/camsrc/website/blobserver-example/root",
"-e", "CAMLI_PASSWORD="+randHex(20),
"-v", camSrcDir()+":/var/camweb/camsrc",
"--net=host",
"--workdir=/var/camweb/camsrc",
"camlistore/demoblobserver",
"camlistored",
"--openbrowser=false",
"--listen=:3179",
"--configfile=/var/camweb/camsrc/website/blobserver-example/example-blobserver-config.json")
err := cmd.Run()
if err != nil {
log.Printf("Failed to run demo blob server: %v", err)
}
if !inProd {
return
}
time.Sleep(10 * time.Second)
}
}
func sendStartingEmail() {
contentRev, err := exec.Command("docker", "run",
"--rm",
"-v", "/var/camweb:/var/camweb",
"-w", "/var/camweb/camsrc",
"camlistore/git",
"/bin/bash", "-c",
"git show --pretty=format:'%ad-%h' --abbrev-commit --date=short | head -1").Output()
cl, err := smtp.Dial(*smtpServer)
if err != nil {
log.Printf("Failed to connect to SMTP server: %v", err)
}
defer cl.Quit()
if err = cl.Mail("noreply@camlistore.org"); err != nil {
return
}
if err = cl.Rcpt("brad@danga.com"); err != nil {
return
}
if err = cl.Rcpt("mathieu.lonjaret@gmail.com"); err != nil {
return
}
wc, err := cl.Data()
if err != nil {
return
}
_, err = fmt.Fprintf(wc, `From: noreply@camlistore.org (Camlistore Website)
To: brad@danga.com, mathieu.lonjaret@gmail.com
Subject: Camlistore camweb restarting
Camlistore website starting with binary XXXXTODO and content at git rev %s
`, contentRev)
if err != nil {
return
}
wc.Close()
}
func getDockerImage(tag, file string) {
@ -462,6 +558,7 @@ func main() {
}
}
readTemplates()
go runDemoBlobserverLoop()
mux := http.DefaultServeMux
mux.Handle("/favicon.ico", http.FileServer(http.Dir(filepath.Join(*root, "static"))))
@ -474,7 +571,8 @@ func main() {
mux.HandleFunc("/r/", gerritRedirect)
mux.HandleFunc("/dl/", releaseRedirect)
mux.HandleFunc("/debugz/ip", ipHandler)
mux.HandleFunc("/debug/ip", ipHandler)
mux.HandleFunc("/debug/uptime", uptimeHandler)
mux.Handle("/docs/contributing", redirTo("/code#contributing"))
mux.Handle("/lists", redirTo("/community"))
@ -574,10 +672,9 @@ func serveHTTPS(httpServer *http.Server) error {
httpsServer := new(http.Server)
*httpsServer = *httpServer
httpsServer.Addr = *httpsAddr
if !inProduction() {
if !inProd {
return httpsServer.ListenAndServeTLS(*tlsCertFile, *tlsKeyFile)
}
cert, err := tlsCertFromGCS()
if err != nil {
return err
@ -713,6 +810,12 @@ func ipHandler(w http.ResponseWriter, r *http.Request) {
w.Write([]byte(str))
}
var startTime = time.Now()
func uptimeHandler(w http.ResponseWriter, r *http.Request) {
fmt.Fprintf(w, "%v", time.Now().Sub(startTime))
}
const (
errPattern = "/err/"
toHyperlink = `<a href="$1$2">$1$2</a>`
@ -740,3 +843,14 @@ func errHandler(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusFound)
servePage(w, errString, "", contents)
}
func camSrcDir() string {
if inProd {
return prodSrcDir
}
dir, err := osutil.GoPackagePath("camlistore.org")
if err != nil {
log.Fatalf("Failed to find the root of the Camlistore source code via osutil.GoPackagePath: %v", err)
}
return dir
}

View File

@ -0,0 +1,10 @@
# Copyright 2015 The Camlistore Authors. All rights reserved.
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file.
FROM debian:wheezy
ENV DEBIAN_FRONTEND noninteractive
ADD build.sh /scripts/build.sh
RUN /scripts/build.sh

View File

@ -0,0 +1,7 @@
container:
docker build -t camlistore/demoblobserver .
upload:
go get golang.org/x/build/cmd/upload
docker save camlistore/demoblobserver | gzip | upload --public --project=camlistore-website camlistore-website-resource/docker-demoblobserver.tar.gz

View File

@ -0,0 +1,22 @@
#!/bin/bash
set -e
apt-get update
apt-get install -y --no-install-recommends curl git-core ca-certificates
curl --silent https://storage.googleapis.com/golang/go1.5.1.linux-amd64.tar.gz | tar -C /usr/local -zxv
mkdir -p /gopath/src
git clone --depth=1 https://camlistore.googlesource.com/camlistore /gopath/src/camlistore.org
export GOPATH=/gopath
export GOBIN=/usr/local/bin
export GO15VENDOREXPERIMENT=1
export CGO_ENABLED=0
/usr/local/go/bin/go install -v camlistore.org/server/camlistored
rm -rf /usr/local/go
apt-get remove --yes curl git-core
apt-get clean
rm -rf /var/cache/apt/
rm -fr /var/lib/apt/lists

View File

@ -76,8 +76,8 @@ func gitShortlog() *exec.Cmd {
if !*gitContainer {
return exec.Command("/bin/bash", "-c", "git log | git shortlog -sen")
}
args := []string{"run"}
if inProduction() {
args := []string{"run", "--rm"}
if inProd {
args = append(args,
"-v", "/var/camweb:/var/camweb",
"--workdir="+prodSrcDir,

View File

@ -71,8 +71,7 @@ var knownCommit = map[string]bool{} // commit -> true
var diffMarker = []byte("diff --git a/")
func emailCommit(dir, hash string) (err error) {
cmd := exec.Command("git", "show", hash)
cmd.Dir = dir
cmd := execGit(dir, "show", hash)
body, err := cmd.CombinedOutput()
if err != nil {
return fmt.Errorf("Error runnning git show: %v\n%s", err, body)
@ -82,8 +81,7 @@ func emailCommit(dir, hash string) (err error) {
return nil
}
cmd = exec.Command("git", "show", "--pretty=oneline", hash)
cmd.Dir = dir
cmd = execGit(dir, "show", "--pretty=oneline", hash)
out, err := cmd.Output()
if err != nil {
return
@ -143,10 +141,7 @@ func commitEmailLoop() error {
}
}()
dir, err := osutil.GoPackagePath("camlistore.org")
if err != nil {
return err
}
dir := camSrcDir()
hashes, err := recentCommits(dir)
if err != nil {
@ -173,9 +168,27 @@ func commitEmailLoop() error {
}
}
func execGit(dir string, gitArgs ...string) *exec.Cmd {
var cmd *exec.Cmd
if *gitContainer {
args := append([]string{
"run",
"--rm",
"-v", dir + ":" + dir,
"--workdir=" + dir,
"camlistore/git",
"git",
}, gitArgs...)
cmd = exec.Command("docker", args...)
} else {
cmd = exec.Command("git", gitArgs...)
cmd.Dir = dir
}
return cmd
}
func pollCommits(dir string) {
cmd := exec.Command("git", "fetch", "origin")
cmd.Dir = dir
cmd := execGit(dir, "fetch", "origin")
out, err := cmd.CombinedOutput()
if err != nil {
log.Printf("Error running git fetch origin master in %s: %v\n%s", dir, err, out)
@ -204,8 +217,7 @@ func pollCommits(dir string) {
}
func recentCommits(dir string) (hashes []string, err error) {
cmd := exec.Command("git", "log", "--since=1 month ago", "--pretty=oneline", "origin/master")
cmd.Dir = dir
cmd := execGit(dir, "log", "--since=1 month ago", "--pretty=oneline", "origin/master")
out, err := cmd.CombinedOutput()
if err != nil {
return nil, fmt.Errorf("Error running git log in %s: %v\n%s", dir, err, out)

View File

@ -1,54 +0,0 @@
#!/usr/bin/perl
use strict;
use FindBin qw($Bin);
my $logdir = "$Bin/../logs";
unless (-d $logdir) {
mkdir $logdir, 0700 or die "mkdir: $!";
}
my $HOME = $ENV{HOME};
chdir $Bin or die;
print STDERR "Running camweb in $Bin on port 8080\n";
my $in_prod = -e "$HOME/etc/ssl.key"; # heuristic. good enough.
my @args;
push @args, "go", "run", "camweb.go", "logging.go", "contributors.go", "godoc.go", "format.go", "dirtrees.go", "email.go";
push @args, "--root=$Bin";
push @args, "--logdir=$logdir";
push @args, "--buildbot_host=build.camlistore.org";
push @args, "--buildbot_backend=http://c1.danga.com:8080";
push @args, "--also_run=$Bin/scripts/run-blobserver";
if ($in_prod) {
push @args, "--email_dest=camlistore-commits\@googlegroups.com";
push @args, "--http=:8080";
push @args, "--https=:4430";
push @args, "--tlscert=$HOME/etc/ssl.crt";
push @args, "--tlskey=$HOME/etc/ssl.key";
while (1) {
system(@args);
print STDERR "Exit: $?; sleeping/restarting...\n";
sleep 5;
}
} else {
my $pass_file = "$ENV{HOME}/.config/camlistore/camorg-blobserver.pass";
unless (-s $pass_file) {
`mkdir -p $ENV{HOME}/.config/camlistore/`;
open (my $fh, ">$pass_file");
print $fh "foo\n";
close($fh);
}
# These https certificate and key are the default ones used by devcam server.
die "TLS cert or key not initialized; run devcam server --tls" unless -e "$Bin/../config/tls.crt" && -e "$Bin/../config/tls.key";
push @args, "--tlscert=$Bin/../config/tls.crt";
push @args, "--tlskey=$Bin/../config/tls.key";
push @args, "--http=127.0.0.1:8080";
push @args, "--https=127.0.0.1:4430";
push @args, @ARGV;
exec(@args);
die "Failed to exec: $!";
}