From e27e2302a48be5e6c769e4101fde12843d6126df Mon Sep 17 00:00:00 2001 From: mpl Date: Wed, 9 May 2018 02:33:02 +0200 Subject: [PATCH] misc/release: update release tools misc/docker/release moved to misc/release because it did not really have anything to do with making docker images, it just happens to use them. misc/monthly.go has been moved to misc/release/make-release.go, and reworked to be more generic. misc/docker/dock.go has been trimmed down to only deal with creating docker images Fixes #1153 Change-Id: I4cb566551007300aefa6cb23714b90461f0e3e51 --- .gitignore | 1 + doc/release.txt | 25 +- misc/docker/dock.go | 346 +---------- misc/{docker => }/release/.gitignore | 0 misc/{docker => }/release/build-binaries.go | 7 +- misc/{monthly.go => release/make-release.go} | 611 +++++++++++++------ misc/{docker => }/release/zip-source.go | 32 +- 7 files changed, 457 insertions(+), 565 deletions(-) rename misc/{docker => }/release/.gitignore (100%) rename misc/{docker => }/release/build-binaries.go (88%) rename misc/{monthly.go => release/make-release.go} (55%) rename misc/{docker => }/release/zip-source.go (87%) diff --git a/.gitignore b/.gitignore index a07c87861..8a147fe48 100644 --- a/.gitignore +++ b/.gitignore @@ -41,6 +41,7 @@ clients/web/embed/opensans/zembed_*.go clients/web/embed/react/zembed_*.go config/tls.* misc/docker/djpeg-static/djpeg +misc/docker/djpeg-static/djpeg.tar.gz misc/docker/perkeepd/perkeepd* misc/docker/perkeepd/djpeg pkg/server/zembed_favicon.ico.go diff --git a/doc/release.txt b/doc/release.txt index 46a40cdb6..6ccacec9f 100644 --- a/doc/release.txt +++ b/doc/release.txt @@ -1,22 +1,17 @@ +** How to build the Perkeep server docker image (for the GCE launcher): ** + +$ go run ./misc/docker/dock.go -rev=$GIT_REVISION -upload=true + ** How to build a release tarball for binaries: ** -$ go run ./misc/docker/dock.go -build_image=false -build_release=true -rev=$GIT_REVISION -tarball_version=0.9 +$ go run ./misc/release/make-release.go -rev=$GIT_REVISION -kind=darwin -will generate ./misc/docker/release/camlistore0.9-linux.tar.gz +will generate ./misc/release/perkeep-darwin.tar.gz -use -os to build the binaries for another OS: windows or darwin. +use -upload=true to directly upload the tarball to the perkeep-release Google Cloud bucket. -use -upload=true to directly upload the tarball to the camlistore-release/0.9/ Google Cloud bucket. +** How to generate a release, i.e. all the tarballs/zips, and the release page: ** -** How to build a release zip for source: ** +go run ./misc/release/make-release.go -rev=$GIT_REVISION -version=0.10.1 -stats_from=d6fb092e69ebf96faa68b3c2379aeb3563840c1b -$ go run ./misc/docker/dock.go -build_image=false -zip_source=true -rev=$GIT_REVISION -tarball_version=0.10 [-sanity=false] - -will generate ./misc/docker/release/camlistore0.10-src.zip - -use -upload=true to directly upload the zip file to the camlistore-release/0.10/ Google Cloud bucket. - -** How to generate a monthly release: ** - -go run ./misc/monthly.go -rev=$GIT_REVISION -stats_from=9e34d14ef5f240f35bd88d71495da0f6cbf99600 -git commit -m 'monthly release' doc/release/monthly.html +It will create the archives in ./misc/release, upload them (with a versioned name) to the perkeep-release bucket, and it will create the ./doc/release/release.html page. diff --git a/misc/docker/dock.go b/misc/docker/dock.go index 76d04df1b..c69b44cc3 100644 --- a/misc/docker/dock.go +++ b/misc/docker/dock.go @@ -19,8 +19,6 @@ limitations under the License. package main // import "perkeep.org/misc/docker" import ( - "archive/tar" - "archive/zip" "compress/gzip" "context" "flag" @@ -44,16 +42,8 @@ import ( ) var ( - flagRev = flag.String("rev", "", "Perkeep revision to build (tag or commit hash). For development purposes, you can instead specify the path to a local Perkeep source tree from which to build, with the form \"WIP:/path/to/dir\".") - flagVersion = flag.String("tarball_version", "", "For --build_release mode, the version number (e.g. 0.9) used for the release tarball name. It also defines the destination directory where the release tarball is uploaded.") - buildOS = flag.String("os", runtime.GOOS, "Operating system to build for. Requires --build_release.") - - doImage = flag.Bool("build_image", true, "build the Perkeep server as a docker image. Conflicts with --build_release.") - doUpload = flag.Bool("upload", false, "With build_image, upload a snapshot of the server in docker as a tarball to https://storage.googleapis.com/camlistore-release/docker/. With build_release, upload the generated tarball at https://storage.googleapis.com/camlistore-release/dl/VERSION/.") - doBinaries = flag.Bool("build_release", false, "build the Perkeep server and tools as standalone binaries to a tarball in misc/docker/release. Requires --build_image=false.") - - doZipSource = flag.Bool("zip_source", false, "pack the Perkeep source for a release in a zip file in misc/docker/release. Requires --build_image=false.") - flagSanity = flag.Bool("sanity", true, "When doing --zip_source, check the source used is buildable with \"go run make.go\".") + flagRev = flag.String("rev", "", "Perkeep revision to build (tag or commit hash). For development purposes, you can instead specify the path to a local Perkeep source tree from which to build, with the form \"WIP:/path/to/dir\".") + flagUpload = flag.Bool("upload", false, "Whether to pload a snapshot of the server in docker as a tarball to https://storage.googleapis.com/camlistore-release/docker/.") asCamlistore = flag.Bool("as_camli", false, `generate and upload things using the old "camlistore" based names. This exists in order to migrate users on the camlistore named image/systemd service, to the new perkeed named ones.`) ) @@ -75,9 +65,8 @@ func buildDockerImage(imageDir, imageName string) { } var ( - dockDir string - releaseTarball string // file path to the tarball generated with -build_release or -zip_source - serverImage = "perkeep/server" + dockDir string + serverImage = "perkeep/server" ) const ( @@ -86,9 +75,7 @@ const ( zoneinfoDockerImage = "perkeep/zoneinfo" goCmd = "/usr/local/go/bin/go" // Path to where the Perkeep builder is mounted on the perkeep/go image. - genCamliProgram = "/usr/local/bin/build-perkeep-server.go" - genBinariesProgram = "/usr/local/bin/build-binaries.go" - zipSourceProgram = "/usr/local/bin/zip-source.go" + genPkProgram = "/usr/local/bin/build-perkeep-server.go" ) func isWIP() bool { @@ -112,20 +99,20 @@ func rev() string { return *flagRev } -func genCamlistore(ctxDir string) { +func genPerkeep(ctxDir string) { check(os.Mkdir(filepath.Join(ctxDir, "/perkeep.org"), 0755)) args := []string{ "run", "--rm", "--volume=" + ctxDir + "/perkeep.org:/OUT", - "--volume=" + path.Join(dockDir, "server/build-perkeep-server.go") + ":" + genCamliProgram + ":ro", + "--volume=" + path.Join(dockDir, "server/build-perkeep-server.go") + ":" + genPkProgram + ":ro", } if isWIP() { args = append(args, "--volume="+localCamliSource()+":/IN:ro", - goDockerImage, goCmd, "run", genCamliProgram, "--rev=WIP:/IN") + goDockerImage, goCmd, "run", genPkProgram, "--rev=WIP:/IN") } else { - args = append(args, goDockerImage, goCmd, "run", genCamliProgram, "--rev="+rev()) + args = append(args, goDockerImage, goCmd, "run", genPkProgram, "--rev="+rev()) } cmd := exec.Command("docker", args...) cmd.Stdout = os.Stdout @@ -135,70 +122,6 @@ func genCamlistore(ctxDir string) { } } -func genBinaries(ctxDir string) { - check(os.Mkdir(filepath.Join(ctxDir, "/perkeep.org"), 0755)) - image := goDockerImage - args := []string{ - "run", - "--rm", - "--volume=" + ctxDir + "/perkeep.org:/OUT", - "--volume=" + path.Join(dockDir, "release/build-binaries.go") + ":" + genBinariesProgram + ":ro", - } - if isWIP() { - args = append(args, "--volume="+localCamliSource()+":/IN:ro", - image, goCmd, "run", genBinariesProgram, "--rev=WIP:/IN", "--os="+*buildOS) - } else { - args = append(args, image, goCmd, "run", genBinariesProgram, "--rev="+rev(), "--os="+*buildOS) - } - if *flagVersion != "" { - args = append(args, "--version="+*flagVersion) - } - cmd := exec.Command("docker", args...) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - if err := cmd.Run(); err != nil { - log.Fatalf("Error building binaries in go container: %v", err) - } - fmt.Printf("Perkeep binaries successfully generated in %v\n", filepath.Join(ctxDir, "perkeep.org", "bin")) -} - -func zipSource(ctxDir string) { - image := goDockerImage - args := []string{ - "run", - "--rm", - "--volume=" + ctxDir + ":/OUT", - "--volume=" + path.Join(dockDir, "release/zip-source.go") + ":" + zipSourceProgram + ":ro", - } - if isWIP() { - args = append(args, "--volume="+localCamliSource()+":/IN:ro", - image, goCmd, "run", zipSourceProgram, "--rev=WIP:/IN") - } else { - args = append(args, image, goCmd, "run", zipSourceProgram, "--rev="+rev()) - } - if *flagVersion != "" { - args = append(args, "--version="+*flagVersion) - } - if !*flagSanity { - args = append(args, "--sanity=false") - } - cmd := exec.Command("docker", args...) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - if err := cmd.Run(); err != nil { - log.Fatalf("Error zipping Perkeep source in go container: %v", err) - } - setReleaseTarballName() - // can't use os.Rename because invalid cross-device link error likely - cmd = exec.Command("mv", filepath.Join(ctxDir, "perkeep-src.zip"), releaseTarball) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - if err := cmd.Run(); err != nil { - log.Fatalf("Error moving source zip from %v to %v: %v", filepath.Join(ctxDir, "perkeep-src.zip"), releaseTarball, err) - } - fmt.Printf("Perkeep source successfully zipped in %v\n", releaseTarball) -} - func copyFinalDockerfile(ctxDir string) { // Copy Dockerfile into the temp dir. serverDockerFile, err := ioutil.ReadFile(filepath.Join(dockDir, "server", "Dockerfile")) @@ -252,63 +175,6 @@ func publicACL(proj string) []storage.ACLRule { } } -// uploadReleaseTarball uploads the generated tarball of binaries in -// camlistore-release/VERSION/perkeepVERSION-REV-CONTENTS.EXT. It then makes a copy in -// the same bucket and path, as perkeepVERSION-CONTENTS.EXT. -func uploadReleaseTarball() { - proj := "camlistore-website" - bucket := "camlistore-release" - tarball := *flagVersion + "/" + filepath.Base(releaseTarball) - versionedTarball := strings.Replace(tarball, "perkeep"+*flagVersion, "perkeep"+*flagVersion+"-"+rev(), 1) - - log.Printf("Uploading %s/%s ...", bucket, versionedTarball) - - ts, err := tokenSource(bucket) - if err != nil { - log.Fatal(err) - } - ctx := context.Background() - stoClient, err := storage.NewClient(ctx, option.WithTokenSource(ts), option.WithHTTPClient(oauth2.NewClient(ctx, ts))) - if err != nil { - log.Fatal(err) - } - w := stoClient.Bucket(bucket).Object(versionedTarball).NewWriter(ctx) - w.ACL = publicACL(proj) - w.CacheControl = "no-cache" // TODO: remove for non-tip releases? set expirations? - contentType := "application/x-gtar" - if *buildOS == "windows" { - contentType = "application/zip" - } - w.ContentType = contentType - - src, err := os.Open(releaseTarball) - if err != nil { - log.Fatal(err) - } - defer src.Close() - - if _, err := io.Copy(w, src); err != nil { - log.Fatalf("io.Copy: %v", err) - } - if err := w.Close(); err != nil { - log.Fatalf("closing GCS storage writer: %v", err) - } - log.Printf("Uploaded tarball to %s", versionedTarball) - if !isWIP() { - log.Printf("Copying tarball to %s/%s ...", bucket, tarball) - dest := stoClient.Bucket(bucket).Object(tarball) - cpier := dest.CopierFrom(stoClient.Bucket(bucket).Object(versionedTarball)) - cpier.ObjectAttrs = storage.ObjectAttrs{ - ACL: publicACL(proj), - ContentType: contentType, - } - if _, err := cpier.Run(ctx); err != nil { - log.Fatalf("Error uploading %v: %v", tarball, err) - } - log.Printf("Uploaded tarball to %s", tarball) - } -} - // uploadDockerImage makes a tar.gz snapshot of the perkeepd docker image, // and uploads it at camlistore-release/docker/perkeepd-REV.tar.gz. It then // makes a copy in the same bucket and path as perkeepd.tar.gz. @@ -399,136 +265,6 @@ func uploadDockerImage() { } } -func exeName(s string) string { - if *buildOS == "windows" { - return s + ".exe" - } - return s -} - -// setReleaseTarballName sets releaseTarball. -func setReleaseTarballName() { - var filename, extension, contents string - if *doZipSource { - contents = "src" - } else { - contents = *buildOS - } - if *buildOS == "windows" || contents == "src" { - extension = ".zip" - } else { - extension = ".tar.gz" - } - if *flagVersion != "" { - filename = "perkeep" + *flagVersion + "-" + contents + extension - } else { - filename = "perkeep-" + contents + extension - } - releaseTarball = path.Join(dockDir, "release", filename) -} - -func packBinaries(ctxDir string) { - binaries := map[string]bool{ - exeName("perkeepd"): false, - exeName("pk-get"): false, - exeName("pk-put"): false, - exeName("pk"): false, - exeName("publisher"): false, - } - switch *buildOS { - case "linux", "darwin": - binaries["pk-mount"] = false - } - toPack := func(bin string) bool { - for k := range binaries { - if bin == k { - binaries[k] = true - return true - } - } - return false - } - defer func() { - for name, found := range binaries { - if !found { - log.Fatalf("%v was not packed in tarball", name) - } - } - fmt.Printf("Perkeep binaries successfully packed in %v\n", releaseTarball) - }() - - binDir := path.Join(ctxDir, "perkeep.org", "bin") - check(os.Chdir(binDir)) - dir, err := os.Open(binDir) - check(err) - defer dir.Close() - - setReleaseTarballName() - if *buildOS == "windows" { - fw, err := os.Create(releaseTarball) - check(err) - defer func() { - check(fw.Close()) - }() - w := zip.NewWriter(fw) - defer func() { - check(w.Close()) - }() - names, err := dir.Readdirnames(-1) - check(err) - for _, name := range names { - if !toPack(name) { - continue - } - b, err := ioutil.ReadFile(path.Join(binDir, name)) - check(err) - f, err := w.Create(name) - check(err) - _, err = f.Write(b) - check(err) - } - return - } - - fw, err := os.Create(releaseTarball) - check(err) - defer func() { - check(fw.Close()) - }() - pr, pw := io.Pipe() - go func() { - tw := tar.NewWriter(pw) - fis, err := dir.Readdir(-1) - check(err) - for _, file := range fis { - if !toPack(file.Name()) { - continue - } - hdr, err := tar.FileInfoHeader(file, "") - check(err) - check(tw.WriteHeader(hdr)) - fr, err := os.Open(file.Name()) - check(err) - n, err := io.Copy(tw, fr) - check(err) - fr.Close() - if n != file.Size() { - log.Fatalf("failed to tar all of %v; got %v, wanted %v", file.Name(), n, file.Size()) - } - } - check(tw.Close()) - check(pw.CloseWithError(io.EOF)) - }() - zw := gzip.NewWriter(fw) - n, err := io.Copy(zw, pr) - if err != nil { - log.Fatalf("Error copying to gzip writer: after %d bytes, %v", n, err) - } - if err := zw.Close(); err != nil { - log.Fatalf("gzip.Close: %v", err) - } -} - func usage() { fmt.Fprintf(os.Stderr, "Usage:\n") fmt.Fprintf(os.Stderr, "%s [-rev perkeep_revision | -rev WIP:/path/to/perkeep/source]\n", os.Args[0]) @@ -536,26 +272,6 @@ func usage() { os.Exit(1) } -// TODO(mpl): I copied numSet from genconfig.go. Move it to some *util package? go4.org? - -func numSet(vv ...interface{}) (num int) { - for _, vi := range vv { - switch v := vi.(type) { - case string: - if v != "" { - num++ - } - case bool: - if v { - num++ - } - default: - panic("unknown type") - } - } - return -} - func checkFlags() { if flag.NArg() != 0 { usage() @@ -564,19 +280,6 @@ func checkFlags() { fmt.Fprintf(os.Stderr, "Usage error: --rev is required.\n") usage() } - numModes := numSet(*doBinaries, *doImage, *doZipSource) - if numModes != 1 { - fmt.Fprintf(os.Stderr, "Usage error: --build_release, --build_image, and --zip_source are mutually exclusive.\n") - usage() - } - if (*doBinaries || *doZipSource) && *doUpload && *flagVersion == "" { - fmt.Fprintf(os.Stderr, "Usage error: --tarball_version required for uploading the release tarball.\n") - usage() - } - if *doImage && *flagVersion != "" { - fmt.Fprintf(os.Stderr, "Usage error: --tarball_version not applicable in --build_image mode.\n") - usage() - } if isWIP() { if _, err := os.Stat(localCamliSource()); err != nil { fmt.Fprintf(os.Stderr, "Usage error: could not stat path %q provided with --rev: %v", localCamliSource(), err) @@ -603,34 +306,23 @@ func main() { buildDockerImage("go", goDockerImage) // ctxDir is where we run "docker build" to produce the final // "FROM scratch" Docker image. - ctxDir, err := ioutil.TempDir("", "camli-build") + ctxDir, err := ioutil.TempDir("", "pk-build_docker_image") if err != nil { log.Fatal(err) } defer os.RemoveAll(ctxDir) - switch { - case *doImage: - buildDockerImage("djpeg-static", djpegDockerImage) - buildDockerImage("zoneinfo", zoneinfoDockerImage) - genCamlistore(ctxDir) - genDjpeg(ctxDir) - genZoneinfo(ctxDir) - buildServer(ctxDir) - case *doBinaries: - genBinaries(ctxDir) - packBinaries(ctxDir) - case *doZipSource: - zipSource(ctxDir) - } - if !*doUpload { + buildDockerImage("djpeg-static", djpegDockerImage) + buildDockerImage("zoneinfo", zoneinfoDockerImage) + genPerkeep(ctxDir) + genDjpeg(ctxDir) + genZoneinfo(ctxDir) + buildServer(ctxDir) + + if !*flagUpload { return } - if *doImage { - uploadDockerImage() - } else { - uploadReleaseTarball() - } + uploadDockerImage() } func check(err error) { diff --git a/misc/docker/release/.gitignore b/misc/release/.gitignore similarity index 100% rename from misc/docker/release/.gitignore rename to misc/release/.gitignore diff --git a/misc/docker/release/build-binaries.go b/misc/release/build-binaries.go similarity index 88% rename from misc/docker/release/build-binaries.go rename to misc/release/build-binaries.go index 516cad690..d16926ed3 100644 --- a/misc/docker/release/build-binaries.go +++ b/misc/release/build-binaries.go @@ -33,10 +33,9 @@ import ( ) var ( - flagRev = flag.String("rev", "", "Perkeep revision to build (tag or commit hash). For development purposes, you can instead specify the path to a local Perkeep source tree from which to build, with the form \"WIP:/path/to/dir\".") - flagVersion = flag.String("version", "", "The optional version number (e.g. 0.9) that will be stamped into the binaries, in addition to the revision.") - outDir = flag.String("outdir", "/OUT/", "Output directory, where the binaries will be written") - buildOS = flag.String("os", runtime.GOOS, "Operating system to build for.") + flagRev = flag.String("rev", "", "Perkeep revision to build (tag or commit hash). For development purposes, you can instead specify the path to a local Perkeep source tree from which to build, with the form \"WIP:/path/to/dir\".") + outDir = flag.String("outdir", "/OUT/", "Output directory, where the binaries will be written") + buildOS = flag.String("os", runtime.GOOS, "Operating system to build for.") ) func usage() { diff --git a/misc/monthly.go b/misc/release/make-release.go similarity index 55% rename from misc/monthly.go rename to misc/release/make-release.go index 2a068b778..1ba51ae37 100644 --- a/misc/monthly.go +++ b/misc/release/make-release.go @@ -1,5 +1,5 @@ /* -Copyright 2016 The Perkeep Authors +Copyright 2018 The Perkeep Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,16 +14,19 @@ See the License for the specific language governing permissions and limitations under the License. */ -// Command monthly builds the tarballs and zip archives for all the monthly -// released Perkeep downloads. That is: source zip, linux and darwin tarballs, +// Command make-release builds the tarballs and zip archives for the Perkeep +// release downloads. That is: source zip, linux and darwin tarballs, // and windows zip. These files are then uploaded to the dedicated repository, as // well as a file with their checksum, for each of them. Finally, the template page -// to serve these downloads with camweb is generated. +// to serve these downloads with pk-web is generated. package main import ( + "archive/tar" + "archive/zip" "bufio" "bytes" + "compress/gzip" "context" "crypto/sha256" "flag" @@ -34,13 +37,13 @@ import ( "log" "os" "os/exec" + "path" "path/filepath" "regexp" "runtime" "sort" "strconv" "strings" - "sync" "time" "perkeep.org/internal/osutil" @@ -53,106 +56,358 @@ import ( ) var ( - flagRev = flag.String("rev", "", "Perkeep revision to build (tag or commit hash). For development purposes, you can instead specify the path to a local Perkeep source tree from which to build, with the form \"WIP:/path/to/dir\".") - flagDate = flag.String("date", "", "The release date to use in the file names to be uploaded, in the YYYYMMDD format. Defaults to today's date.") - flagUpload = flag.Bool("upload", true, "Upload all the generated tarballs and zip archives.") - flagSkipGen = flag.Bool("skipgen", false, "Do not recreate the release tarballs, and directly use the ones found in perkeep.org/misc/docker/release. Use -upload=false and -skipgen=true to only generate the monthly release page.") - flagStatsFrom = flag.String("stats_from", "", "Also generate commit statistics on the release page, starting from the given commit, and ending at the one given as -rev.") - // TODO(mpl): make sanity run the tests too, once they're more reliable. - flagSanity = flag.Bool("sanity", true, "Verify 'go run make.go' succeeds when building the source tarball. Abort everything if not.") + flagRev = flag.String("rev", "", "Perkeep revision to build (tag or commit hash). For development purposes, you can instead specify the path to a local Perkeep source tree from which to build, with the form \"WIP:/path/to/dir\".") + flagVersion = flag.String("version", "", "The name of the release, (e.g. 0.10.1, or 20180512) used as part of the name for the file downloads. Defaults to today's date in YYYYMMDD format.") + flagArchiveType = flag.String("kind", "all", `The kind of archive to build for the release. Possible values are: "darwin", "linux", "windows", "src" (zip of all the source code), or "all" (for all the previous values).`) + flagUpload = flag.Bool("upload", true, "Upload all the generated tarballs and zip archives.") + flagSkipGen = flag.Bool("skipgen", false, "Do not recreate the release tarballs, and directly use the ones found in perkeep.org/misc/release. Use -upload=false and -skipgen=true to only generate the release page.") + flagStatsFrom = flag.String("stats_from", "", "Also generate commit statistics on the release page, starting from the given commit, and ending at the one given as -rev.") ) var ( - camDir string - releaseDate time.Time + pkDir string + releaseDir string + workDir string + goVersion string + pkVersion string ) const ( - titleDateFormat = "2006-01-02" - fileDateFormat = "20060102" - project = "camlistore-website" - bucket = "camlistore-release" + goDockerImage = "perkeep/go" + goCmd = "/usr/local/go/bin/go" + genBinariesProgram = "/usr/local/bin/build-binaries.go" + zipSourceProgram = "/usr/local/bin/zip-source.go" + titleDateFormat = "2006-01-02" + fileDateFormat = "20060102" + project = "camlistore-website" + bucket = "perkeep-release" ) -func isWIP() bool { - return strings.HasPrefix(*flagRev, "WIP") +func usage() { + fmt.Fprintf(os.Stderr, "Usage:\n") + fmt.Fprintf(os.Stderr, "%s [-rev perkeep_revision | -rev WIP:/path/to/perkeep/source]\n", os.Args[0]) + flag.PrintDefaults() + os.Exit(1) } -func rev() string { - if isWIP() { - return "WORKINPROGRESS" +func checkFlags() { + if flag.NArg() != 0 { + usage() + } + if *flagRev == "" { + fmt.Fprintf(os.Stderr, "Usage error: --rev is required.\n") + usage() } - return (*flagRev)[0:10] } -// genDownloads creates all the zips and tarballs, and uploads them. -func genDownloads() error { - dockDotGo := filepath.Join(camDir, "misc", "docker", "dock.go") - releaseDir := filepath.Join(camDir, "misc", "docker", "release") - var wg sync.WaitGroup +func main() { + flag.Usage = usage + flag.Parse() + checkFlags() + + var err error + pkDir, err = osutil.GoPackagePath("perkeep.org") + if err != nil { + log.Fatalf("Error looking up perkeep.org dir: %v", err) + } + releaseDir = filepath.Join(pkDir, "misc", "release") + + workDir, err = ioutil.TempDir("", "pk-build_release") + if err != nil { + log.Fatal(err) + } + defer os.RemoveAll(workDir) + + var archives []string if !*flagSkipGen { - // Gen the source zip: - args := []string{ - "run", - dockDotGo, - "-build_image=false", - "-zip_source=true", - "-rev=" + *flagRev, - "-sanity=" + fmt.Sprintf("%t", *flagSanity), - } - cmd := exec.Command("go", args...) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - if err := cmd.Run(); err != nil { - return err + archives, err = genArchive() + if err != nil { + log.Fatal(err) } } - wg.Add(1) - go func() { - defer wg.Done() - upload(filepath.Join(releaseDir, "perkeep-src.zip")) + if *flagUpload { + for _, v := range archives { + upload(filepath.Join(releaseDir, v)) + } + } + + if *flagArchiveType != "all" { + // do not bother generating the release page if we're not doing a full blown release build. + return + } + + releaseData, err := listDownloads() + if err != nil { + if *flagSkipGen { + // Most likely we're failing because we can't reach the + // bucket (working offline), annd we're working on this + // program and testing things out, so make this error + // non-critical so we can still generate the release notes + // and stats. + log.Print(err) + releaseData = &ReleaseData{} + } else { + log.Fatal(err) + } + } + + if *flagStatsFrom != "" { + commitStats, err := genCommitStats() + if err != nil { + log.Fatal(err) + } + releaseData.Stats = commitStats + + notes, err := genReleaseNotes() + if err != nil { + log.Fatal(err) + } + releaseData.ReleaseNotes = notes + } + + if err := genReleasePage(releaseData); err != nil { + log.Fatal(err) + } +} + +// genArchive generates the requested tarball(s) and zip archive(s), and returns +// their filenames. +func genArchive() ([]string, error) { + switch *flagArchiveType { + case "linux", "darwin", "windows": + genBinaries(*flagArchiveType) + return []string{packBinaries(*flagArchiveType)}, nil + case "src": + return []string{zipSource()}, nil + default: + return genAll() + } +} + +// genAll creates all the zips and tarballs, and returns their filenames. +func genAll() ([]string, error) { + zipSource() + for _, platform := range []string{"linux", "darwin", "windows"} { + genBinaries(platform) + packBinaries(platform) + } + getVersions() + return []string{ + "perkeep-darwin.tar.gz", + "perkeep-linux.tar.gz", + "perkeep-src.zip", + "perkeep-windows.zip", + }, nil +} + +// getVersions uses the freshly built perkeepd binary to get the Perkeep and Go +// versions used to build the release. +func getVersions() { + pkBin := filepath.Join(workDir, runtime.GOOS, "bin", "perkeepd") + cmd := exec.Command(pkBin, "-version") + var buf bytes.Buffer + cmd.Stderr = &buf + if err := cmd.Run(); err != nil { + log.Fatalf("Error getting version from perkeepd: %v, %v", err, buf.String()) + } + sc := bufio.NewScanner(&buf) + for sc.Scan() { + l := sc.Text() + fields := strings.Fields(l) + if pkVersion == "" { + if len(fields) != 4 || fields[0] != "perkeepd" { + log.Fatalf("Unexpected perkeepd -version output: %q", l) + } + pkVersion = fields[3] + continue + } + if len(fields) != 4 || fields[0] != "Go" { + log.Fatalf("Unexpected perkeepd -version output: %q", l) + } + goVersion = fields[2] + break + } + if err := sc.Err(); err != nil { + log.Fatal(err) + } +} + +// genBinaries runs go run make.go for the given osType in a docker container. +func genBinaries(osType string) { + cwd := filepath.Join(workDir, osType) + check(os.MkdirAll(cwd, 0755)) + image := goDockerImage + args := []string{ + "run", + "--rm", + "--volume=" + cwd + ":/OUT", + "--volume=" + filepath.Join(releaseDir, "build-binaries.go") + ":" + genBinariesProgram + ":ro", + } + if isWIP() { + args = append(args, "--volume="+localCamliSource()+":/IN:ro", + image, goCmd, "run", genBinariesProgram, "--rev=WIP:/IN", "--os="+osType) + } else { + args = append(args, image, goCmd, "run", genBinariesProgram, "--rev="+rev(), "--os="+osType) + } + cmd := exec.Command("docker", args...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + log.Fatalf("Error building binaries in go container: %v", err) + } + fmt.Printf("Perkeep binaries successfully generated in %v\n", filepath.Join(cwd, "bin")) +} + +// packBinaries builds the archive that contains the binaries built by +// genBinaries. +func packBinaries(osType string) string { + archiveName := "perkeep-" + osType + ".tar.gz" + if osType == "windows" { + archiveName = strings.Replace(archiveName, ".tar.gz", ".zip", 1) + } + binaries := map[string]bool{ + exeName("perkeepd", osType): false, + exeName("pk-get", osType): false, + exeName("pk-put", osType): false, + exeName("pk", osType): false, + exeName("publisher", osType): false, + exeName("scancab", osType): false, + exeName("scanningcabinet", osType): false, + } + switch osType { + case "linux", "darwin": + binaries["pk-mount"] = false + } + toPack := func(bin string) bool { + for k := range binaries { + if bin == k { + binaries[k] = true + return true + } + } + return false + } + archivePath := filepath.Join(releaseDir, archiveName) + defer func() { + for name, found := range binaries { + if !found { + log.Fatalf("%v was not packed in tarball", name) + } + } + fmt.Printf("Perkeep binaries successfully packed in %v\n", archivePath) }() - // gen the binaries tarballs: - for _, platform := range []string{"linux", "darwin", "windows"} { - if !*flagSkipGen { - args := []string{ - "run", - dockDotGo, - "-build_image=false", - "-build_release=true", - "-rev=" + *flagRev, - "-os=" + platform, + binDir := path.Join(workDir, osType, "bin") + check(os.Chdir(binDir)) + dir, err := os.Open(binDir) + check(err) + defer dir.Close() + + if osType == "windows" { + fw, err := os.Create(archivePath) + check(err) + defer func() { + check(fw.Close()) + }() + w := zip.NewWriter(fw) + defer func() { + check(w.Close()) + }() + names, err := dir.Readdirnames(-1) + check(err) + for _, name := range names { + if !toPack(name) { + continue } - cmd := exec.Command("go", args...) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - if err := cmd.Run(); err != nil { - return err + b, err := ioutil.ReadFile(path.Join(binDir, name)) + check(err) + f, err := w.Create(name) + check(err) + _, err = f.Write(b) + check(err) + } + return archiveName + } + + fw, err := os.Create(archivePath) + check(err) + defer func() { + check(fw.Close()) + }() + pr, pw := io.Pipe() + go func() { + tw := tar.NewWriter(pw) + fis, err := dir.Readdir(-1) + check(err) + for _, file := range fis { + if !toPack(file.Name()) { + continue + } + hdr, err := tar.FileInfoHeader(file, "") + check(err) + check(tw.WriteHeader(hdr)) + fr, err := os.Open(file.Name()) + check(err) + n, err := io.Copy(tw, fr) + check(err) + fr.Close() + if n != file.Size() { + log.Fatalf("failed to tar all of %v; got %v, wanted %v", file.Name(), n, file.Size()) } } - wg.Add(1) - go func(osType string) { - defer wg.Done() - filename := "perkeep-" + osType + ".tar.gz" - if osType == "windows" { - filename = strings.Replace(filename, ".tar.gz", ".zip", 1) - } - upload(filepath.Join(releaseDir, filename)) - }(platform) + check(tw.Close()) + check(pw.CloseWithError(io.EOF)) + }() + zw := gzip.NewWriter(fw) + n, err := io.Copy(zw, pr) + if err != nil { + log.Fatalf("Error copying to gzip writer: after %d bytes, %v", n, err) } - wg.Wait() - return nil + if err := zw.Close(); err != nil { + log.Fatalf("gzip.Close: %v", err) + } + return archiveName +} + +// zipSource builds the zip archive that contains the source code of Perkeep. +func zipSource() string { + cwd := filepath.Join(workDir, "src") + check(os.MkdirAll(cwd, 0755)) + image := goDockerImage + args := []string{ + "run", + "--rm", + "--volume=" + cwd + ":/OUT", + "--volume=" + path.Join(releaseDir, "zip-source.go") + ":" + zipSourceProgram + ":ro", + } + if isWIP() { + args = append(args, "--volume="+localCamliSource()+":/IN:ro", + image, goCmd, "run", zipSourceProgram, "--rev=WIP:/IN") + } else { + args = append(args, image, goCmd, "run", zipSourceProgram, "--rev="+rev()) + } + cmd := exec.Command("docker", args...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + log.Fatalf("Error zipping Perkeep source in go container: %v", err) + } + archiveName := "perkeep-src.zip" + // can't use os.Rename because invalid cross-device link error likely + cmd = exec.Command("mv", filepath.Join(cwd, archiveName), releaseDir) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + log.Fatalf("Error moving source zip from %v to %v: %v", filepath.Join(cwd, archiveName), releaseDir, err) + } + fmt.Printf("Perkeep source successfully zipped in %v\n", releaseDir) + return archiveName } func upload(srcPath string) { - if !*flagUpload { - return - } - destName := strings.Replace(filepath.Base(srcPath), "perkeep", "perkeep-"+releaseDate.Format(fileDateFormat), 1) - versionedTarball := "monthly/" + destName + uploadName := strings.Replace(filepath.Base(srcPath), "perkeep", "perkeep-"+version(), 1) - log.Printf("Uploading %s/%s ...", bucket, versionedTarball) + log.Printf("Uploading %s/%s ...", bucket, uploadName) ts, err := tokenSource(bucket) if err != nil { @@ -163,11 +418,11 @@ func upload(srcPath string) { if err != nil { log.Fatal(err) } - w := stoClient.Bucket(bucket).Object(versionedTarball).NewWriter(ctx) + w := stoClient.Bucket(bucket).Object(uploadName).NewWriter(ctx) w.ACL = publicACL(project) w.CacheControl = "no-cache" // TODO: remove for non-tip releases? set expirations? contentType := "application/x-gtar" - if strings.HasSuffix(versionedTarball, ".zip") { + if strings.HasSuffix(uploadName, ".zip") { contentType = "application/zip" } w.ContentType = contentType @@ -186,14 +441,14 @@ func upload(srcPath string) { if err := w.Close(); err != nil { log.Fatalf("closing GCS storage writer: %v", err) } - log.Printf("Uploaded monthly tarball to %s", versionedTarball) + log.Printf("Uploaded archive to %s/%s", bucket, uploadName) // And upload the corresponding checksum - checkSumFile := versionedTarball + ".sha256" + checkSumFile := uploadName + ".sha256" sum := fmt.Sprintf("%x", csw.Sum(nil)) w = stoClient.Bucket(bucket).Object(checkSumFile).NewWriter(ctx) w.ACL = publicACL(project) - w.CacheControl = "no-cache" // TODO: remove for non-tip releases? set expirations? + w.CacheControl = "no-cache" w.ContentType = "text/plain" if _, err := io.Copy(w, strings.NewReader(sum)); err != nil { log.Fatalf("error uploading checksum %v: %v", checkSumFile, err) @@ -201,7 +456,7 @@ func upload(srcPath string) { if err := w.Close(); err != nil { log.Fatalf("closing GCS storage writer: %v", err) } - log.Printf("Uploaded monthly tarball checksum to %s", checkSumFile) + log.Printf("Uploaded archive checksum to %s", checkSumFile) } type DownloadData struct { @@ -211,9 +466,9 @@ type DownloadData struct { } type ReleaseData struct { - Date string + Name string Download []DownloadData - CamliVersion string + PkVersion string GoVersion string Stats *stats ReleaseNotes map[string][]string @@ -222,18 +477,18 @@ type ReleaseData struct { // Note: the space trimming in the range loop is important. Since all of our // html still goes through a markdown engine, newlines in between items would make // markdown wrap the items in

, which breaks the page's style. -var monthlyTemplate = ` -

Monthly Release: {{.Date}}

+var releaseTemplate = ` +

Perkeep Release: {{.Name}}

-Perkeep version {{.CamliVersion}} built with Go {{.GoVersion}}. +Perkeep version {{.PkVersion}} built with Go {{.GoVersion}}.

Downloads

{{- range $d := .Download}} - +
{{$d.Platform}}
{{$d.Filename}} @@ -275,14 +530,9 @@ Perkeep version