misc/docker/server: build with make.go

So far, when building camlistored docker image for CoreOS, we were not
using make.go, and we were neither running gopherjs nor embedding the
resources (but rather provide the UI resources at their default
filesystem location).

Now that we're using gopherjs for the web UI, it is a hard dependency
for the camlistore server.
We could reproduce the steps in make.go to build gopherjs, run it to
build the web ui resources, and then move the resources at the right
place, but since make.go already does the equivalent work it seems
to make more sense to use it, which is the main point of this CL.

Similarly, it seems to make more sense to now build a binary with the
resources embedded, which is the default make.go behaviour, instead of
building a "raw" camlistored, and provide the resources as additional
directories in the container image, so this CL takes that approach too.

Finally, it was necessary to add the "-static" flag to make.go, so we
can keep on building a static camlistored binary, that does not rely on
libc for DNS. Because our container image is FROM SCRATCH, with just the
necessary binaries, in order to get a container image of a reasonable
size.

One drawback of now using make.go in
misc/docker/server/build-camlistore-server.go is we're doing some
unnecessary (since we're already running in the isolation of a
container) copying to the virtual gopath, but that seems a very tiny
price to pay. Especially considering how rarely we run that code.

Change-Id: I416c86d366cd4ed2d3b8b1636a6a65a83b9f15d7

make.go

@@ -65,6 +65,7 @@ var (
 	stampVersion   = flag.Bool("stampversion", true, "Stamp version into buildinfo.GitInfo")
 	website        = flag.Bool("website", false, "Just build the website.")
 	camnetdns      = flag.Bool("camnetdns", false, "Just build camlistore.org/server/camnetdns.")
+	static         = flag.Bool("static", false, "Build a static binary, so it can run in an empty container.")
 
 	// Use GOPATH from the environment and work from there. Do not create a temporary source tree with a new GOPATH in it.
 	// It is set through CAMLI_MAKE_USEGOPATH for integration tests that call 'go run make.go', and which are already in
@@ -207,6 +208,7 @@ func main() {
 
 	withCamlistored := stringListContains(targs, "camlistore.org/server/camlistored")
 
+	// TODO(mpl): no need to build publisher.js if we're not building the publisher app.
 	if withCamlistored {
 		// gopherjs has to run before doEmbed since we need all the javascript
 		// to be generated before embedding happens.
@@ -224,6 +226,9 @@ func main() {
 	}
 
 	tags := []string{"purego"} // for cznic/zappy
+	if *static {
+		tags = append(tags, "netgo")
+	}
 	if sql {
 		tags = append(tags, "with_sqlite")
 	}
@@ -238,8 +243,14 @@ func main() {
 		log.Printf("version to stamp is %q", version)
 	}
 	var ldFlags string
+	if *static {
+		ldFlags = "-w -d -linkmode internal"
+	}
 	if *stampVersion {
-		ldFlags = "-X \"camlistore.org/pkg/buildinfo.GitInfo=" + version + "\""
+		if ldFlags != "" {
+			ldFlags += " "
+		}
+		ldFlags += "-X \"camlistore.org/pkg/buildinfo.GitInfo=" + version + "\""
 	}
 	baseArgs = append(baseArgs, "--ldflags="+ldFlags, "--tags="+strings.Join(tags, " "))
 
@@ -260,6 +271,9 @@ func main() {
 	cmd.Env = append(cleanGoEnv(),
 		"GOPATH="+buildGoPath,
 	)
+	if *static {
+		cmd.Env = append(cmd.Env, "CGO_ENABLED=0")
+	}
 
 	if *verbose {
 		log.Printf("Running go %q with Env %q", args, cmd.Env)

misc/docker/dock.go

@@ -230,7 +230,7 @@ func genZoneinfo(ctxDir string) {
 
 func buildServer(ctxDir string) {
 	copyFinalDockerfile(ctxDir)
-	cmd := exec.Command("docker", "build", "-t", serverImage, ".")
+	cmd := exec.Command("docker", "build", "--no-cache", "-t", serverImage, ".")
 	cmd.Dir = ctxDir
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr

misc/docker/server/build-camlistore-server.go

@@ -28,12 +28,12 @@ import (
 	"flag"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"log"
 	"net/http"
 	"os"
 	"os/exec"
 	"path"
-	"path/filepath"
 	"strings"
 )
 
@@ -81,9 +81,13 @@ func rev() string {
 func getCamliSrc() {
 	if localCamliSource() != "" {
 		mirrorCamliSrc(localCamliSource())
-		return
+	} else {
+		fetchCamliSrc()
 	}
-	fetchCamliSrc()
+	// we insert the version in the VERSION file, so make.go does no need git
+	// in the container to detect the Camlistore version.
+	check(os.Chdir("/gopath/src/camlistore.org"))
+	check(ioutil.WriteFile("VERSION", []byte(rev()), 0777))
 }
 
 func mirrorCamliSrc(srcDir string) {
@@ -131,47 +135,25 @@ func fetchCamliSrc() {
 }
 
 func buildCamlistored() {
-	check(os.MkdirAll(path.Join(*outDir, "/bin"), 0777))
-	check(os.MkdirAll(path.Join(*outDir, "/server/camlistored"), 0777))
 	oldPath := os.Getenv("PATH")
 	os.Setenv("GOPATH", "/gopath")
 	os.Setenv("PATH", "/usr/local/go/bin:"+oldPath)
-	os.Setenv("CGO_ENABLED", "0")
+	check(os.Chdir("/gopath/src/camlistore.org"))
-	os.Setenv("GO15VENDOREXPERIMENT", "1")
+	cmd := exec.Command("go", "run", "make.go",
-	cmd := exec.Command("go", "build",
+		"-static", "true",
-		"-o", path.Join(*outDir, "/bin/camlistored"),
+		"-targets", "camlistore.org/server/camlistored")
-		`--ldflags`, "-w -d -linkmode internal -X camlistore.org/pkg/buildinfo.GitInfo="+rev(),
-		"--tags=netgo", "camlistore.org/server/camlistored")
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
 	if err := cmd.Run(); err != nil {
 		log.Fatalf("Error building camlistored in go container: %v", err)
 	}
-}
 
-func setUIResources() {
+	// And move it to the output dir
-	cmd := exec.Command("mv", "/gopath/src/camlistore.org/server/camlistored/ui", path.Join(*outDir, "/server/camlistored/ui"))
+	check(os.MkdirAll(path.Join(*outDir, "/bin"), 0777))
+	cmd = exec.Command("mv", "/gopath/src/camlistore.org/bin/camlistored", path.Join(*outDir, "/bin"))
 	if err := cmd.Run(); err != nil {
-		log.Fatalf("Error moving UI dir %v in output dir %v: %v",
+		log.Fatalf("Error moving camlistored binary %v in output dir %v: %v",
-			"/gopath/src/camlistore.org/server/camlistored/ui", path.Join(*outDir, "/server/camlistored/ui"), err)
+			"/gopath/src/camlistore.org/bin/camlistored", path.Join(*outDir, "/bin"), err)
-	}
-	filepath.Walk("/gopath/src/camlistore.org/vendor/embed", func(path string, fi os.FileInfo, err error) error {
-		if err != nil {
-			log.Fatalf("Error stating while cleaning %s: %v", path, err)
-		}
-		if fi.IsDir() {
-			return nil
-		}
-		if strings.HasSuffix(path, ".go") {
-			check(os.Remove(path))
-		}
-		return nil
-	})
-	check(os.MkdirAll(path.Join(*outDir, "/vendor"), 0777))
-	cmd = exec.Command("mv", "/gopath/src/camlistore.org/vendor/embed", path.Join(*outDir, "/vendor/embed"))
-	if err := cmd.Run(); err != nil {
-		log.Fatalf("Error moving vendor/embed dir %v in output dir %v: %v",
-			"/gopath/src/camlistore.org/vendor/embed", path.Join(*outDir, "/vendor/embed"), err)
 	}
 }
 
@@ -219,7 +201,6 @@ func main() {
 
 	getCamliSrc()
 	buildCamlistored()
-	setUIResources()
 }
 
 func check(err error) {

pkg/serverinit/env.go

@@ -66,10 +66,6 @@ func DefaultEnvConfig() (*Config, error) {
 		GoogleCloudStorage: ":" + strings.TrimPrefix(blobBucket, "gs://"),
 		DBNames:            map[string]string{},
 		PackRelated:        true,
-
-		// SourceRoot is where we look for the UI js/css/html files, and the Closure resources.
-		// Must be in sync with misc/docker/server/Dockerfile.
-		SourceRoot: "/camlistore",
 	}
 
 	externalIP, _ := metadata.ExternalIP()