From 753bd38ca919c8f39df1b122ae8cc7051568b21a Mon Sep 17 00:00:00 2001 From: mpl Date: Tue, 4 Apr 2017 17:58:03 +0200 Subject: [PATCH] misc/docker/server: build with make.go So far, when building camlistored docker image for CoreOS, we were not using make.go, and we were neither running gopherjs nor embedding the resources (but rather provide the UI resources at their default filesystem location). Now that we're using gopherjs for the web UI, it is a hard dependency for the camlistore server. We could reproduce the steps in make.go to build gopherjs, run it to build the web ui resources, and then move the resources at the right place, but since make.go already does the equivalent work it seems to make more sense to use it, which is the main point of this CL. Similarly, it seems to make more sense to now build a binary with the resources embedded, which is the default make.go behaviour, instead of building a "raw" camlistored, and provide the resources as additional directories in the container image, so this CL takes that approach too. Finally, it was necessary to add the "-static" flag to make.go, so we can keep on building a static camlistored binary, that does not rely on libc for DNS. Because our container image is FROM SCRATCH, with just the necessary binaries, in order to get a container image of a reasonable size. One drawback of now using make.go in misc/docker/server/build-camlistore-server.go is we're doing some unnecessary (since we're already running in the isolation of a container) copying to the virtual gopath, but that seems a very tiny price to pay. Especially considering how rarely we run that code. Change-Id: I416c86d366cd4ed2d3b8b1636a6a65a83b9f15d7 --- make.go | 16 +++++- misc/docker/dock.go | 2 +- misc/docker/server/build-camlistore-server.go | 51 ++++++------------- pkg/serverinit/env.go | 4 -- 4 files changed, 32 insertions(+), 41 deletions(-) diff --git a/make.go b/make.go index a83610484..22ec8ada6 100644 --- a/make.go +++ b/make.go @@ -65,6 +65,7 @@ var ( stampVersion = flag.Bool("stampversion", true, "Stamp version into buildinfo.GitInfo") website = flag.Bool("website", false, "Just build the website.") camnetdns = flag.Bool("camnetdns", false, "Just build camlistore.org/server/camnetdns.") + static = flag.Bool("static", false, "Build a static binary, so it can run in an empty container.") // Use GOPATH from the environment and work from there. Do not create a temporary source tree with a new GOPATH in it. // It is set through CAMLI_MAKE_USEGOPATH for integration tests that call 'go run make.go', and which are already in @@ -207,6 +208,7 @@ func main() { withCamlistored := stringListContains(targs, "camlistore.org/server/camlistored") + // TODO(mpl): no need to build publisher.js if we're not building the publisher app. if withCamlistored { // gopherjs has to run before doEmbed since we need all the javascript // to be generated before embedding happens. @@ -224,6 +226,9 @@ func main() { } tags := []string{"purego"} // for cznic/zappy + if *static { + tags = append(tags, "netgo") + } if sql { tags = append(tags, "with_sqlite") } @@ -238,8 +243,14 @@ func main() { log.Printf("version to stamp is %q", version) } var ldFlags string + if *static { + ldFlags = "-w -d -linkmode internal" + } if *stampVersion { - ldFlags = "-X \"camlistore.org/pkg/buildinfo.GitInfo=" + version + "\"" + if ldFlags != "" { + ldFlags += " " + } + ldFlags += "-X \"camlistore.org/pkg/buildinfo.GitInfo=" + version + "\"" } baseArgs = append(baseArgs, "--ldflags="+ldFlags, "--tags="+strings.Join(tags, " ")) @@ -260,6 +271,9 @@ func main() { cmd.Env = append(cleanGoEnv(), "GOPATH="+buildGoPath, ) + if *static { + cmd.Env = append(cmd.Env, "CGO_ENABLED=0") + } if *verbose { log.Printf("Running go %q with Env %q", args, cmd.Env) diff --git a/misc/docker/dock.go b/misc/docker/dock.go index 9b7731715..6c7fa219a 100644 --- a/misc/docker/dock.go +++ b/misc/docker/dock.go @@ -230,7 +230,7 @@ func genZoneinfo(ctxDir string) { func buildServer(ctxDir string) { copyFinalDockerfile(ctxDir) - cmd := exec.Command("docker", "build", "-t", serverImage, ".") + cmd := exec.Command("docker", "build", "--no-cache", "-t", serverImage, ".") cmd.Dir = ctxDir cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr diff --git a/misc/docker/server/build-camlistore-server.go b/misc/docker/server/build-camlistore-server.go index 33e1711f1..3ee2269f0 100644 --- a/misc/docker/server/build-camlistore-server.go +++ b/misc/docker/server/build-camlistore-server.go @@ -28,12 +28,12 @@ import ( "flag" "fmt" "io" + "io/ioutil" "log" "net/http" "os" "os/exec" "path" - "path/filepath" "strings" ) @@ -81,9 +81,13 @@ func rev() string { func getCamliSrc() { if localCamliSource() != "" { mirrorCamliSrc(localCamliSource()) - return + } else { + fetchCamliSrc() } - fetchCamliSrc() + // we insert the version in the VERSION file, so make.go does no need git + // in the container to detect the Camlistore version. + check(os.Chdir("/gopath/src/camlistore.org")) + check(ioutil.WriteFile("VERSION", []byte(rev()), 0777)) } func mirrorCamliSrc(srcDir string) { @@ -131,47 +135,25 @@ func fetchCamliSrc() { } func buildCamlistored() { - check(os.MkdirAll(path.Join(*outDir, "/bin"), 0777)) - check(os.MkdirAll(path.Join(*outDir, "/server/camlistored"), 0777)) oldPath := os.Getenv("PATH") os.Setenv("GOPATH", "/gopath") os.Setenv("PATH", "/usr/local/go/bin:"+oldPath) - os.Setenv("CGO_ENABLED", "0") - os.Setenv("GO15VENDOREXPERIMENT", "1") - cmd := exec.Command("go", "build", - "-o", path.Join(*outDir, "/bin/camlistored"), - `--ldflags`, "-w -d -linkmode internal -X camlistore.org/pkg/buildinfo.GitInfo="+rev(), - "--tags=netgo", "camlistore.org/server/camlistored") + check(os.Chdir("/gopath/src/camlistore.org")) + cmd := exec.Command("go", "run", "make.go", + "-static", "true", + "-targets", "camlistore.org/server/camlistored") cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr if err := cmd.Run(); err != nil { log.Fatalf("Error building camlistored in go container: %v", err) } -} -func setUIResources() { - cmd := exec.Command("mv", "/gopath/src/camlistore.org/server/camlistored/ui", path.Join(*outDir, "/server/camlistored/ui")) + // And move it to the output dir + check(os.MkdirAll(path.Join(*outDir, "/bin"), 0777)) + cmd = exec.Command("mv", "/gopath/src/camlistore.org/bin/camlistored", path.Join(*outDir, "/bin")) if err := cmd.Run(); err != nil { - log.Fatalf("Error moving UI dir %v in output dir %v: %v", - "/gopath/src/camlistore.org/server/camlistored/ui", path.Join(*outDir, "/server/camlistored/ui"), err) - } - filepath.Walk("/gopath/src/camlistore.org/vendor/embed", func(path string, fi os.FileInfo, err error) error { - if err != nil { - log.Fatalf("Error stating while cleaning %s: %v", path, err) - } - if fi.IsDir() { - return nil - } - if strings.HasSuffix(path, ".go") { - check(os.Remove(path)) - } - return nil - }) - check(os.MkdirAll(path.Join(*outDir, "/vendor"), 0777)) - cmd = exec.Command("mv", "/gopath/src/camlistore.org/vendor/embed", path.Join(*outDir, "/vendor/embed")) - if err := cmd.Run(); err != nil { - log.Fatalf("Error moving vendor/embed dir %v in output dir %v: %v", - "/gopath/src/camlistore.org/vendor/embed", path.Join(*outDir, "/vendor/embed"), err) + log.Fatalf("Error moving camlistored binary %v in output dir %v: %v", + "/gopath/src/camlistore.org/bin/camlistored", path.Join(*outDir, "/bin"), err) } } @@ -219,7 +201,6 @@ func main() { getCamliSrc() buildCamlistored() - setUIResources() } func check(err error) { diff --git a/pkg/serverinit/env.go b/pkg/serverinit/env.go index 685bf145d..b18212805 100644 --- a/pkg/serverinit/env.go +++ b/pkg/serverinit/env.go @@ -66,10 +66,6 @@ func DefaultEnvConfig() (*Config, error) { GoogleCloudStorage: ":" + strings.TrimPrefix(blobBucket, "gs://"), DBNames: map[string]string{}, PackRelated: true, - - // SourceRoot is where we look for the UI js/css/html files, and the Closure resources. - // Must be in sync with misc/docker/server/Dockerfile. - SourceRoot: "/camlistore", } externalIP, _ := metadata.ExternalIP()