diff --git a/misc/docker/dock.go b/misc/docker/dock.go index dee453020..43d162f73 100644 --- a/misc/docker/dock.go +++ b/misc/docker/dock.go @@ -26,9 +26,9 @@ import ( "log" "os" "os/exec" + "path" "path/filepath" "runtime" - "strings" "camlistore.org/pkg/osutil" "camlistore.org/third_party/golang.org/x/oauth2" @@ -38,7 +38,8 @@ import ( ) var ( - rev = flag.String("rev", "4e8413c5012c", "Camlistore revision to build (tag or commit hash") + rev = flag.String("rev", "4e8413c5012c", "Camlistore revision to build (tag or commit hash)") + localSrc = flag.String("camlisource", "", "(dev flag) Path to a local Camlistore source tree from which to build. This flag is ignored unless -rev=WORKINPROGRESS") doBuildServer = flag.Bool("build_server", true, "build the server") doUpload = flag.Bool("upload", false, "upload a snapshot of the server tarball to http://storage.googleapis.com/camlistore-release/docker/camlistored[-VERSION].tar.gz") @@ -65,44 +66,27 @@ var dockDir string const ( goDockerImage = "camlistore/go" djpegDockerImage = "camlistore/djpeg" + goCmd = "/usr/local/go/bin/go" + // Path to where the Camlistore builder is mounted on the camlistore/go image. + genCamliProgram = "/usr/local/bin/build-camlistore-server.go" ) func genCamlistore(ctxDir string) { - repl := strings.NewReplacer( - "[[REV]]", *rev, - ) check(os.Mkdir(filepath.Join(ctxDir, "/camlistore.org"), 0755)) - cmd := exec.Command("docker", "run", + args := []string{ + "run", "--rm", - "--volume="+ctxDir+"/camlistore.org:/OUT", - goDockerImage, "/bin/bash", "-c", repl.Replace(` - -# TODO(bradfitz,mpl): rewrite this shell into a Go program that's -# baked into the camlistore/go image, and then all this shell becomes: -# /usr/local/bin/build-camlistore-server $REV -# (and it would still write to /OUT) - - set -e - set -x - export GOPATH=/gopath; - export PATH=/usr/local/go/bin:$PATH; - mkdir -p /OUT/bin && - mkdir -p /OUT/server/camlistored && - mkdir -p /gopath/src/camlistore.org && - cd /gopath/src/camlistore.org && - curl --silent https://camlistore.googlesource.com/camlistore/+archive/[[REV]].tar.gz | - tar -zxv && - CGO_ENABLED=0 go build \ - -o /OUT/bin/camlistored \ - --ldflags="-w -d -linkmode internal -X camlistore.org/pkg/buildinfo.GitInfo [[REV]]" \ - --tags=netgo \ - camlistore.org/server/camlistored && - mv /gopath/src/camlistore.org/server/camlistored/ui /OUT/server/camlistored/ui && - find /gopath/src/camlistore.org/third_party -type f -name '*.go' -exec rm {} \; && - mv /gopath/src/camlistore.org/third_party /OUT/third_party && - echo DONE -`)) + "--volume=" + ctxDir + "/camlistore.org:/OUT", + "--volume=" + path.Join(dockDir, "server/build-camlistore-server.go") + ":" + genCamliProgram + ":ro", + } + if *rev == "WORKINPROGRESS" { + args = append(args, "--volume="+*localSrc+":/IN:ro", + goDockerImage, goCmd, "run", genCamliProgram, "--rev="+*rev, "--camlisource=/IN") + } else { + args = append(args, goDockerImage, goCmd, "run", genCamliProgram, "--rev="+*rev) + } + cmd := exec.Command("docker", args...) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr if err := cmd.Run(); err != nil { @@ -197,11 +181,36 @@ func uploadDockerImage() { log.Printf("Uploaded tarball to %s", object) } -func main() { - flag.Parse() +func usage() { + fmt.Fprintf(os.Stderr, "Usage:\n") + fmt.Fprintf(os.Stderr, "%s [-rev camlistore_revision | -rev WORKINPROGRESS -camlisource dir]\n", os.Args[0]) + flag.PrintDefaults() + os.Exit(1) +} + +func checkFlags() { if flag.NArg() != 0 { - log.Fatalf("Bogus usage. dock does not currently take any arguments.") + usage() } + if *rev == "" { + usage() + } + if *rev == "WORKINPROGRESS" { + if *localSrc == "" { + usage() + } + return + } + if *localSrc != "" { + fmt.Fprintf(os.Stderr, "Usage error: --camlisource can only be used with --rev WORKINPROGRESS.\n") + usage() + } +} + +func main() { + flag.Usage = usage + flag.Parse() + checkFlags() camDir, err := osutil.GoPackagePath("camlistore.org") if err != nil { @@ -222,9 +231,7 @@ func main() { defer os.RemoveAll(ctxDir) genCamlistore(ctxDir) - genDjpeg(ctxDir) - buildServer(ctxDir) } diff --git a/misc/docker/server/build-camlistore-server.go b/misc/docker/server/build-camlistore-server.go new file mode 100644 index 000000000..1d8926fef --- /dev/null +++ b/misc/docker/server/build-camlistore-server.go @@ -0,0 +1,190 @@ +/* +Copyright 2015 The Camlistore Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Command build-camlistore-server builds camlistored and bundles all the +// necessary resources for a Camlistore server in docker. It should be run in a +// docker container. +package main + +import ( + "archive/tar" + "compress/gzip" + "flag" + "fmt" + "io" + "log" + "net/http" + "os" + "os/exec" + "path" + "path/filepath" + "strings" +) + +var ( + rev = flag.String("rev", "", "Camlistore revision to build (tag or commit hash)") + localSrc = flag.String("camlisource", "", "(dev flag) Path to a local Camlistore source tree from which to build. It is ignored unless -rev=WORKINPROGRESS") + outDir = flag.String("outdir", "/OUT/", "Output directory, where camlistored and all the resources will be written") +) + +func usage() { + fmt.Fprintf(os.Stderr, "Usage:\n") + fmt.Fprintf(os.Stderr, "%s --rev=camlistore_revision\n", os.Args[0]) + fmt.Fprintf(os.Stderr, "%s --rev=WORKINPROGRESS --camlisource=/path/to/camli/source/dir\n", os.Args[0]) + flag.PrintDefaults() + example(os.Args[0]) + os.Exit(1) +} + +func example(program string) { + fmt.Fprintf(os.Stderr, "Examples:\n") + fmt.Fprintf(os.Stderr, "\tdocker run --rm --volume=/tmp/camli-build/camlistore.org:/OUT camlistore/go %s --rev=4e8413c5012c\n", program) + fmt.Fprintf(os.Stderr, "\tdocker run --rm --volume=/tmp/camli-build/camlistore.org:/OUT --volume=~/camlistore.org:/IN camlistore/go %s --rev=WORKINPROGRESS --camlisource=/IN\n", program) +} + +func getCamliSrc() { + if *localSrc != "" { + mirrorCamliSrc(*localSrc) + return + } + fetchCamliSrc() +} + +func mirrorCamliSrc(srcDir string) { + check(os.MkdirAll("/gopath/src", 0777)) + cmd := exec.Command("cp", "-a", srcDir, "/gopath/src/camlistore.org") + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + log.Fatalf("Error mirroring camlistore source from %v: %v", srcDir, err) + } +} + +func fetchCamliSrc() { + check(os.MkdirAll("/gopath/src/camlistore.org", 0777)) + check(os.Chdir("/gopath/src/camlistore.org")) + + res, err := http.Get("https://camlistore.googlesource.com/camlistore/+archive/" + *rev + ".tar.gz") + check(err) + defer res.Body.Close() + gz, err := gzip.NewReader(res.Body) + check(err) + defer gz.Close() + tr := tar.NewReader(gz) + for { + h, err := tr.Next() + if err == io.EOF { + break + } + check(err) + if h.Typeflag == tar.TypeDir { + check(os.MkdirAll(h.Name, os.FileMode(h.Mode))) + continue + } + f, err := os.Create(h.Name) + check(err) + n, err := io.Copy(f, tr) + if err != nil && err != io.EOF { + log.Fatal(err) + } + if n != h.Size { + log.Fatalf("Error when creating %v: wanted %v bytes, got %v bytes", h.Name, h.Size, n) + } + check(f.Close()) + } +} + +func buildCamlistored() { + check(os.MkdirAll(path.Join(*outDir, "/bin"), 0777)) + check(os.MkdirAll(path.Join(*outDir, "/server/camlistored"), 0777)) + oldPath := os.Getenv("PATH") + os.Setenv("GOPATH", "/gopath") + os.Setenv("PATH", "/usr/local/go/bin:"+oldPath) + os.Setenv("CGO_ENABLED", "0") + cmd := exec.Command("go", "build", + "-o", path.Join(*outDir, "/bin/camlistored"), + `--ldflags`, "-w -d -linkmode internal -X camlistore.org/pkg/buildinfo.GitInfo "+*rev, + "--tags=netgo", "camlistore.org/server/camlistored") + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + log.Fatalf("Error building camlistored in go container: %v", err) + } +} + +func setUIResources() { + cmd := exec.Command("mv", "/gopath/src/camlistore.org/server/camlistored/ui", path.Join(*outDir, "/server/camlistored/ui")) + if err := cmd.Run(); err != nil { + log.Fatalf("Error moving UI dir %v in output dir %v: %v", + "/gopath/src/camlistore.org/server/camlistored/ui", path.Join(*outDir, "/server/camlistored/ui"), err) + } + filepath.Walk("/gopath/src/camlistore.org/third_party", func(path string, fi os.FileInfo, err error) error { + if err != nil { + log.Fatalf("Error stating while cleaning %s: %v", path, err) + } + if fi.IsDir() { + return nil + } + if strings.HasSuffix(path, ".go") { + check(os.Remove(path)) + } + return nil + }) + cmd = exec.Command("mv", "/gopath/src/camlistore.org/third_party", path.Join(*outDir, "/third_party")) + if err := cmd.Run(); err != nil { + log.Fatalf("Error moving third_party dir %v in output dir %v: %v", + "/gopath/src/camlistore.org/third_party", path.Join(*outDir, "/third_party"), err) + } +} + +func checkArgs() { + if flag.NArg() != 0 { + usage() + } + if *rev == "" { + usage() + } + if *rev == "WORKINPROGRESS" { + if *localSrc == "" { + usage() + } + return + } + if *localSrc != "" { + fmt.Fprintf(os.Stderr, "Usage error: --camlisource can only be used with --rev WORKINPROGRESS.\n") + usage() + } +} + +func main() { + flag.Usage = usage + flag.Parse() + if _, err := os.Stat("/.dockerinit"); err != nil { + fmt.Fprintf(os.Stderr, "Usage error: this program should be run within a docker container, and is meant to be called from misc/docker/dock.go\n") + usage() + } + checkArgs() + + getCamliSrc() + buildCamlistored() + setUIResources() +} + +func check(err error) { + if err != nil { + log.Fatal(err) + } +}