mirror of https://github.com/perkeep/perkeep.git
misc/release: update release tools
misc/docker/release moved to misc/release because it did not really have anything to do with making docker images, it just happens to use them. misc/monthly.go has been moved to misc/release/make-release.go, and reworked to be more generic. misc/docker/dock.go has been trimmed down to only deal with creating docker images Fixes #1153 Change-Id: I4cb566551007300aefa6cb23714b90461f0e3e51
This commit is contained in:
mpl
parent
2e052c5fe5
commit
e27e2302a4
|
|
@ -41,6 +41,7 @@ clients/web/embed/opensans/zembed_*.go
|
|||
clients/web/embed/react/zembed_*.go
|
||||
config/tls.*
|
||||
misc/docker/djpeg-static/djpeg
|
||||
misc/docker/djpeg-static/djpeg.tar.gz
|
||||
misc/docker/perkeepd/perkeepd*
|
||||
misc/docker/perkeepd/djpeg
|
||||
pkg/server/zembed_favicon.ico.go
|
||||
|
|
|
|||
|
|
@ -1,22 +1,17 @@
|
|||
** How to build the Perkeep server docker image (for the GCE launcher): **
|
||||
|
||||
$ go run ./misc/docker/dock.go -rev=$GIT_REVISION -upload=true
|
||||
|
||||
** How to build a release tarball for binaries: **
|
||||
|
||||
$ go run ./misc/docker/dock.go -build_image=false -build_release=true -rev=$GIT_REVISION -tarball_version=0.9
|
||||
$ go run ./misc/release/make-release.go -rev=$GIT_REVISION -kind=darwin
|
||||
|
||||
will generate ./misc/docker/release/camlistore0.9-linux.tar.gz
|
||||
will generate ./misc/release/perkeep-darwin.tar.gz
|
||||
|
||||
use -os to build the binaries for another OS: windows or darwin.
|
||||
use -upload=true to directly upload the tarball to the perkeep-release Google Cloud bucket.
|
||||
|
||||
use -upload=true to directly upload the tarball to the camlistore-release/0.9/ Google Cloud bucket.
|
||||
** How to generate a release, i.e. all the tarballs/zips, and the release page: **
|
||||
|
||||
** How to build a release zip for source: **
|
||||
go run ./misc/release/make-release.go -rev=$GIT_REVISION -version=0.10.1 -stats_from=d6fb092e69ebf96faa68b3c2379aeb3563840c1b
|
||||
|
||||
$ go run ./misc/docker/dock.go -build_image=false -zip_source=true -rev=$GIT_REVISION -tarball_version=0.10 [-sanity=false]
|
||||
|
||||
will generate ./misc/docker/release/camlistore0.10-src.zip
|
||||
|
||||
use -upload=true to directly upload the zip file to the camlistore-release/0.10/ Google Cloud bucket.
|
||||
|
||||
** How to generate a monthly release: **
|
||||
|
||||
go run ./misc/monthly.go -rev=$GIT_REVISION -stats_from=9e34d14ef5f240f35bd88d71495da0f6cbf99600
|
||||
git commit -m 'monthly release' doc/release/monthly.html
|
||||
It will create the archives in ./misc/release, upload them (with a versioned name) to the perkeep-release bucket, and it will create the ./doc/release/release.html page.
|
||||
|
|
|
|||
|
|
@ -19,8 +19,6 @@ limitations under the License.
|
|||
package main // import "perkeep.org/misc/docker"
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"archive/zip"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"flag"
|
||||
|
|
@ -44,16 +42,8 @@ import (
|
|||
)
|
||||
|
||||
var (
|
||||
flagRev = flag.String("rev", "", "Perkeep revision to build (tag or commit hash). For development purposes, you can instead specify the path to a local Perkeep source tree from which to build, with the form \"WIP:/path/to/dir\".")
|
||||
flagVersion = flag.String("tarball_version", "", "For --build_release mode, the version number (e.g. 0.9) used for the release tarball name. It also defines the destination directory where the release tarball is uploaded.")
|
||||
buildOS = flag.String("os", runtime.GOOS, "Operating system to build for. Requires --build_release.")
|
||||
|
||||
doImage = flag.Bool("build_image", true, "build the Perkeep server as a docker image. Conflicts with --build_release.")
|
||||
doUpload = flag.Bool("upload", false, "With build_image, upload a snapshot of the server in docker as a tarball to https://storage.googleapis.com/camlistore-release/docker/. With build_release, upload the generated tarball at https://storage.googleapis.com/camlistore-release/dl/VERSION/.")
|
||||
doBinaries = flag.Bool("build_release", false, "build the Perkeep server and tools as standalone binaries to a tarball in misc/docker/release. Requires --build_image=false.")
|
||||
|
||||
doZipSource = flag.Bool("zip_source", false, "pack the Perkeep source for a release in a zip file in misc/docker/release. Requires --build_image=false.")
|
||||
flagSanity = flag.Bool("sanity", true, "When doing --zip_source, check the source used is buildable with \"go run make.go\".")
|
||||
flagRev = flag.String("rev", "", "Perkeep revision to build (tag or commit hash). For development purposes, you can instead specify the path to a local Perkeep source tree from which to build, with the form \"WIP:/path/to/dir\".")
|
||||
flagUpload = flag.Bool("upload", false, "Whether to pload a snapshot of the server in docker as a tarball to https://storage.googleapis.com/camlistore-release/docker/.")
|
||||
|
||||
asCamlistore = flag.Bool("as_camli", false, `generate and upload things using the old "camlistore" based names. This exists in order to migrate users on the camlistore named image/systemd service, to the new perkeed named ones.`)
|
||||
)
|
||||
|
|
@ -75,9 +65,8 @@ func buildDockerImage(imageDir, imageName string) {
|
|||
}
|
||||
|
||||
var (
|
||||
dockDir string
|
||||
releaseTarball string // file path to the tarball generated with -build_release or -zip_source
|
||||
serverImage = "perkeep/server"
|
||||
dockDir string
|
||||
serverImage = "perkeep/server"
|
||||
)
|
||||
|
||||
const (
|
||||
|
|
@ -86,9 +75,7 @@ const (
|
|||
zoneinfoDockerImage = "perkeep/zoneinfo"
|
||||
goCmd = "/usr/local/go/bin/go"
|
||||
// Path to where the Perkeep builder is mounted on the perkeep/go image.
|
||||
genCamliProgram = "/usr/local/bin/build-perkeep-server.go"
|
||||
genBinariesProgram = "/usr/local/bin/build-binaries.go"
|
||||
zipSourceProgram = "/usr/local/bin/zip-source.go"
|
||||
genPkProgram = "/usr/local/bin/build-perkeep-server.go"
|
||||
)
|
||||
|
||||
func isWIP() bool {
|
||||
|
|
@ -112,20 +99,20 @@ func rev() string {
|
|||
return *flagRev
|
||||
}
|
||||
|
||||
func genCamlistore(ctxDir string) {
|
||||
func genPerkeep(ctxDir string) {
|
||||
check(os.Mkdir(filepath.Join(ctxDir, "/perkeep.org"), 0755))
|
||||
|
||||
args := []string{
|
||||
"run",
|
||||
"--rm",
|
||||
"--volume=" + ctxDir + "/perkeep.org:/OUT",
|
||||
"--volume=" + path.Join(dockDir, "server/build-perkeep-server.go") + ":" + genCamliProgram + ":ro",
|
||||
"--volume=" + path.Join(dockDir, "server/build-perkeep-server.go") + ":" + genPkProgram + ":ro",
|
||||
}
|
||||
if isWIP() {
|
||||
args = append(args, "--volume="+localCamliSource()+":/IN:ro",
|
||||
goDockerImage, goCmd, "run", genCamliProgram, "--rev=WIP:/IN")
|
||||
goDockerImage, goCmd, "run", genPkProgram, "--rev=WIP:/IN")
|
||||
} else {
|
||||
args = append(args, goDockerImage, goCmd, "run", genCamliProgram, "--rev="+rev())
|
||||
args = append(args, goDockerImage, goCmd, "run", genPkProgram, "--rev="+rev())
|
||||
}
|
||||
cmd := exec.Command("docker", args...)
|
||||
cmd.Stdout = os.Stdout
|
||||
|
|
@ -135,70 +122,6 @@ func genCamlistore(ctxDir string) {
|
|||
}
|
||||
}
|
||||
|
||||
func genBinaries(ctxDir string) {
|
||||
check(os.Mkdir(filepath.Join(ctxDir, "/perkeep.org"), 0755))
|
||||
image := goDockerImage
|
||||
args := []string{
|
||||
"run",
|
||||
"--rm",
|
||||
"--volume=" + ctxDir + "/perkeep.org:/OUT",
|
||||
"--volume=" + path.Join(dockDir, "release/build-binaries.go") + ":" + genBinariesProgram + ":ro",
|
||||
}
|
||||
if isWIP() {
|
||||
args = append(args, "--volume="+localCamliSource()+":/IN:ro",
|
||||
image, goCmd, "run", genBinariesProgram, "--rev=WIP:/IN", "--os="+*buildOS)
|
||||
} else {
|
||||
args = append(args, image, goCmd, "run", genBinariesProgram, "--rev="+rev(), "--os="+*buildOS)
|
||||
}
|
||||
if *flagVersion != "" {
|
||||
args = append(args, "--version="+*flagVersion)
|
||||
}
|
||||
cmd := exec.Command("docker", args...)
|
||||
cmd.Stdout = os.Stdout
|
||||
cmd.Stderr = os.Stderr
|
||||
if err := cmd.Run(); err != nil {
|
||||
log.Fatalf("Error building binaries in go container: %v", err)
|
||||
}
|
||||
fmt.Printf("Perkeep binaries successfully generated in %v\n", filepath.Join(ctxDir, "perkeep.org", "bin"))
|
||||
}
|
||||
|
||||
func zipSource(ctxDir string) {
|
||||
image := goDockerImage
|
||||
args := []string{
|
||||
"run",
|
||||
"--rm",
|
||||
"--volume=" + ctxDir + ":/OUT",
|
||||
"--volume=" + path.Join(dockDir, "release/zip-source.go") + ":" + zipSourceProgram + ":ro",
|
||||
}
|
||||
if isWIP() {
|
||||
args = append(args, "--volume="+localCamliSource()+":/IN:ro",
|
||||
image, goCmd, "run", zipSourceProgram, "--rev=WIP:/IN")
|
||||
} else {
|
||||
args = append(args, image, goCmd, "run", zipSourceProgram, "--rev="+rev())
|
||||
}
|
||||
if *flagVersion != "" {
|
||||
args = append(args, "--version="+*flagVersion)
|
||||
}
|
||||
if !*flagSanity {
|
||||
args = append(args, "--sanity=false")
|
||||
}
|
||||
cmd := exec.Command("docker", args...)
|
||||
cmd.Stdout = os.Stdout
|
||||
cmd.Stderr = os.Stderr
|
||||
if err := cmd.Run(); err != nil {
|
||||
log.Fatalf("Error zipping Perkeep source in go container: %v", err)
|
||||
}
|
||||
setReleaseTarballName()
|
||||
// can't use os.Rename because invalid cross-device link error likely
|
||||
cmd = exec.Command("mv", filepath.Join(ctxDir, "perkeep-src.zip"), releaseTarball)
|
||||
cmd.Stdout = os.Stdout
|
||||
cmd.Stderr = os.Stderr
|
||||
if err := cmd.Run(); err != nil {
|
||||
log.Fatalf("Error moving source zip from %v to %v: %v", filepath.Join(ctxDir, "perkeep-src.zip"), releaseTarball, err)
|
||||
}
|
||||
fmt.Printf("Perkeep source successfully zipped in %v\n", releaseTarball)
|
||||
}
|
||||
|
||||
func copyFinalDockerfile(ctxDir string) {
|
||||
// Copy Dockerfile into the temp dir.
|
||||
serverDockerFile, err := ioutil.ReadFile(filepath.Join(dockDir, "server", "Dockerfile"))
|
||||
|
|
@ -252,63 +175,6 @@ func publicACL(proj string) []storage.ACLRule {
|
|||
}
|
||||
}
|
||||
|
||||
// uploadReleaseTarball uploads the generated tarball of binaries in
|
||||
// camlistore-release/VERSION/perkeepVERSION-REV-CONTENTS.EXT. It then makes a copy in
|
||||
// the same bucket and path, as perkeepVERSION-CONTENTS.EXT.
|
||||
func uploadReleaseTarball() {
|
||||
proj := "camlistore-website"
|
||||
bucket := "camlistore-release"
|
||||
tarball := *flagVersion + "/" + filepath.Base(releaseTarball)
|
||||
versionedTarball := strings.Replace(tarball, "perkeep"+*flagVersion, "perkeep"+*flagVersion+"-"+rev(), 1)
|
||||
|
||||
log.Printf("Uploading %s/%s ...", bucket, versionedTarball)
|
||||
|
||||
ts, err := tokenSource(bucket)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
ctx := context.Background()
|
||||
stoClient, err := storage.NewClient(ctx, option.WithTokenSource(ts), option.WithHTTPClient(oauth2.NewClient(ctx, ts)))
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
w := stoClient.Bucket(bucket).Object(versionedTarball).NewWriter(ctx)
|
||||
w.ACL = publicACL(proj)
|
||||
w.CacheControl = "no-cache" // TODO: remove for non-tip releases? set expirations?
|
||||
contentType := "application/x-gtar"
|
||||
if *buildOS == "windows" {
|
||||
contentType = "application/zip"
|
||||
}
|
||||
w.ContentType = contentType
|
||||
|
||||
src, err := os.Open(releaseTarball)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer src.Close()
|
||||
|
||||
if _, err := io.Copy(w, src); err != nil {
|
||||
log.Fatalf("io.Copy: %v", err)
|
||||
}
|
||||
if err := w.Close(); err != nil {
|
||||
log.Fatalf("closing GCS storage writer: %v", err)
|
||||
}
|
||||
log.Printf("Uploaded tarball to %s", versionedTarball)
|
||||
if !isWIP() {
|
||||
log.Printf("Copying tarball to %s/%s ...", bucket, tarball)
|
||||
dest := stoClient.Bucket(bucket).Object(tarball)
|
||||
cpier := dest.CopierFrom(stoClient.Bucket(bucket).Object(versionedTarball))
|
||||
cpier.ObjectAttrs = storage.ObjectAttrs{
|
||||
ACL: publicACL(proj),
|
||||
ContentType: contentType,
|
||||
}
|
||||
if _, err := cpier.Run(ctx); err != nil {
|
||||
log.Fatalf("Error uploading %v: %v", tarball, err)
|
||||
}
|
||||
log.Printf("Uploaded tarball to %s", tarball)
|
||||
}
|
||||
}
|
||||
|
||||
// uploadDockerImage makes a tar.gz snapshot of the perkeepd docker image,
|
||||
// and uploads it at camlistore-release/docker/perkeepd-REV.tar.gz. It then
|
||||
// makes a copy in the same bucket and path as perkeepd.tar.gz.
|
||||
|
|
@ -399,136 +265,6 @@ func uploadDockerImage() {
|
|||
}
|
||||
}
|
||||
|
||||
func exeName(s string) string {
|
||||
if *buildOS == "windows" {
|
||||
return s + ".exe"
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// setReleaseTarballName sets releaseTarball.
|
||||
func setReleaseTarballName() {
|
||||
var filename, extension, contents string
|
||||
if *doZipSource {
|
||||
contents = "src"
|
||||
} else {
|
||||
contents = *buildOS
|
||||
}
|
||||
if *buildOS == "windows" || contents == "src" {
|
||||
extension = ".zip"
|
||||
} else {
|
||||
extension = ".tar.gz"
|
||||
}
|
||||
if *flagVersion != "" {
|
||||
filename = "perkeep" + *flagVersion + "-" + contents + extension
|
||||
} else {
|
||||
filename = "perkeep-" + contents + extension
|
||||
}
|
||||
releaseTarball = path.Join(dockDir, "release", filename)
|
||||
}
|
||||
|
||||
func packBinaries(ctxDir string) {
|
||||
binaries := map[string]bool{
|
||||
exeName("perkeepd"): false,
|
||||
exeName("pk-get"): false,
|
||||
exeName("pk-put"): false,
|
||||
exeName("pk"): false,
|
||||
exeName("publisher"): false,
|
||||
}
|
||||
switch *buildOS {
|
||||
case "linux", "darwin":
|
||||
binaries["pk-mount"] = false
|
||||
}
|
||||
toPack := func(bin string) bool {
|
||||
for k := range binaries {
|
||||
if bin == k {
|
||||
binaries[k] = true
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
defer func() {
|
||||
for name, found := range binaries {
|
||||
if !found {
|
||||
log.Fatalf("%v was not packed in tarball", name)
|
||||
}
|
||||
}
|
||||
fmt.Printf("Perkeep binaries successfully packed in %v\n", releaseTarball)
|
||||
}()
|
||||
|
||||
binDir := path.Join(ctxDir, "perkeep.org", "bin")
|
||||
check(os.Chdir(binDir))
|
||||
dir, err := os.Open(binDir)
|
||||
check(err)
|
||||
defer dir.Close()
|
||||
|
||||
setReleaseTarballName()
|
||||
if *buildOS == "windows" {
|
||||
fw, err := os.Create(releaseTarball)
|
||||
check(err)
|
||||
defer func() {
|
||||
check(fw.Close())
|
||||
}()
|
||||
w := zip.NewWriter(fw)
|
||||
defer func() {
|
||||
check(w.Close())
|
||||
}()
|
||||
names, err := dir.Readdirnames(-1)
|
||||
check(err)
|
||||
for _, name := range names {
|
||||
if !toPack(name) {
|
||||
continue
|
||||
}
|
||||
b, err := ioutil.ReadFile(path.Join(binDir, name))
|
||||
check(err)
|
||||
f, err := w.Create(name)
|
||||
check(err)
|
||||
_, err = f.Write(b)
|
||||
check(err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
fw, err := os.Create(releaseTarball)
|
||||
check(err)
|
||||
defer func() {
|
||||
check(fw.Close())
|
||||
}()
|
||||
pr, pw := io.Pipe()
|
||||
go func() {
|
||||
tw := tar.NewWriter(pw)
|
||||
fis, err := dir.Readdir(-1)
|
||||
check(err)
|
||||
for _, file := range fis {
|
||||
if !toPack(file.Name()) {
|
||||
continue
|
||||
}
|
||||
hdr, err := tar.FileInfoHeader(file, "")
|
||||
check(err)
|
||||
check(tw.WriteHeader(hdr))
|
||||
fr, err := os.Open(file.Name())
|
||||
check(err)
|
||||
n, err := io.Copy(tw, fr)
|
||||
check(err)
|
||||
fr.Close()
|
||||
if n != file.Size() {
|
||||
log.Fatalf("failed to tar all of %v; got %v, wanted %v", file.Name(), n, file.Size())
|
||||
}
|
||||
}
|
||||
check(tw.Close())
|
||||
check(pw.CloseWithError(io.EOF))
|
||||
}()
|
||||
zw := gzip.NewWriter(fw)
|
||||
n, err := io.Copy(zw, pr)
|
||||
if err != nil {
|
||||
log.Fatalf("Error copying to gzip writer: after %d bytes, %v", n, err)
|
||||
}
|
||||
if err := zw.Close(); err != nil {
|
||||
log.Fatalf("gzip.Close: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func usage() {
|
||||
fmt.Fprintf(os.Stderr, "Usage:\n")
|
||||
fmt.Fprintf(os.Stderr, "%s [-rev perkeep_revision | -rev WIP:/path/to/perkeep/source]\n", os.Args[0])
|
||||
|
|
@ -536,26 +272,6 @@ func usage() {
|
|||
os.Exit(1)
|
||||
}
|
||||
|
||||
// TODO(mpl): I copied numSet from genconfig.go. Move it to some *util package? go4.org?
|
||||
|
||||
func numSet(vv ...interface{}) (num int) {
|
||||
for _, vi := range vv {
|
||||
switch v := vi.(type) {
|
||||
case string:
|
||||
if v != "" {
|
||||
num++
|
||||
}
|
||||
case bool:
|
||||
if v {
|
||||
num++
|
||||
}
|
||||
default:
|
||||
panic("unknown type")
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func checkFlags() {
|
||||
if flag.NArg() != 0 {
|
||||
usage()
|
||||
|
|
@ -564,19 +280,6 @@ func checkFlags() {
|
|||
fmt.Fprintf(os.Stderr, "Usage error: --rev is required.\n")
|
||||
usage()
|
||||
}
|
||||
numModes := numSet(*doBinaries, *doImage, *doZipSource)
|
||||
if numModes != 1 {
|
||||
fmt.Fprintf(os.Stderr, "Usage error: --build_release, --build_image, and --zip_source are mutually exclusive.\n")
|
||||
usage()
|
||||
}
|
||||
if (*doBinaries || *doZipSource) && *doUpload && *flagVersion == "" {
|
||||
fmt.Fprintf(os.Stderr, "Usage error: --tarball_version required for uploading the release tarball.\n")
|
||||
usage()
|
||||
}
|
||||
if *doImage && *flagVersion != "" {
|
||||
fmt.Fprintf(os.Stderr, "Usage error: --tarball_version not applicable in --build_image mode.\n")
|
||||
usage()
|
||||
}
|
||||
if isWIP() {
|
||||
if _, err := os.Stat(localCamliSource()); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Usage error: could not stat path %q provided with --rev: %v", localCamliSource(), err)
|
||||
|
|
@ -603,34 +306,23 @@ func main() {
|
|||
buildDockerImage("go", goDockerImage)
|
||||
// ctxDir is where we run "docker build" to produce the final
|
||||
// "FROM scratch" Docker image.
|
||||
ctxDir, err := ioutil.TempDir("", "camli-build")
|
||||
ctxDir, err := ioutil.TempDir("", "pk-build_docker_image")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(ctxDir)
|
||||
switch {
|
||||
case *doImage:
|
||||
buildDockerImage("djpeg-static", djpegDockerImage)
|
||||
buildDockerImage("zoneinfo", zoneinfoDockerImage)
|
||||
genCamlistore(ctxDir)
|
||||
genDjpeg(ctxDir)
|
||||
genZoneinfo(ctxDir)
|
||||
buildServer(ctxDir)
|
||||
case *doBinaries:
|
||||
genBinaries(ctxDir)
|
||||
packBinaries(ctxDir)
|
||||
case *doZipSource:
|
||||
zipSource(ctxDir)
|
||||
}
|
||||
|
||||
if !*doUpload {
|
||||
buildDockerImage("djpeg-static", djpegDockerImage)
|
||||
buildDockerImage("zoneinfo", zoneinfoDockerImage)
|
||||
genPerkeep(ctxDir)
|
||||
genDjpeg(ctxDir)
|
||||
genZoneinfo(ctxDir)
|
||||
buildServer(ctxDir)
|
||||
|
||||
if !*flagUpload {
|
||||
return
|
||||
}
|
||||
if *doImage {
|
||||
uploadDockerImage()
|
||||
} else {
|
||||
uploadReleaseTarball()
|
||||
}
|
||||
uploadDockerImage()
|
||||
}
|
||||
|
||||
func check(err error) {
|
||||
|
|
|
|||
|
|
@ -33,10 +33,9 @@ import (
|
|||
)
|
||||
|
||||
var (
|
||||
flagRev = flag.String("rev", "", "Perkeep revision to build (tag or commit hash). For development purposes, you can instead specify the path to a local Perkeep source tree from which to build, with the form \"WIP:/path/to/dir\".")
|
||||
flagVersion = flag.String("version", "", "The optional version number (e.g. 0.9) that will be stamped into the binaries, in addition to the revision.")
|
||||
outDir = flag.String("outdir", "/OUT/", "Output directory, where the binaries will be written")
|
||||
buildOS = flag.String("os", runtime.GOOS, "Operating system to build for.")
|
||||
flagRev = flag.String("rev", "", "Perkeep revision to build (tag or commit hash). For development purposes, you can instead specify the path to a local Perkeep source tree from which to build, with the form \"WIP:/path/to/dir\".")
|
||||
outDir = flag.String("outdir", "/OUT/", "Output directory, where the binaries will be written")
|
||||
buildOS = flag.String("os", runtime.GOOS, "Operating system to build for.")
|
||||
)
|
||||
|
||||
func usage() {
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
/*
|
||||
Copyright 2016 The Perkeep Authors
|
||||
Copyright 2018 The Perkeep Authors
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
|
|
@ -14,16 +14,19 @@ See the License for the specific language governing permissions and
|
|||
limitations under the License.
|
||||
*/
|
||||
|
||||
// Command monthly builds the tarballs and zip archives for all the monthly
|
||||
// released Perkeep downloads. That is: source zip, linux and darwin tarballs,
|
||||
// Command make-release builds the tarballs and zip archives for the Perkeep
|
||||
// release downloads. That is: source zip, linux and darwin tarballs,
|
||||
// and windows zip. These files are then uploaded to the dedicated repository, as
|
||||
// well as a file with their checksum, for each of them. Finally, the template page
|
||||
// to serve these downloads with camweb is generated.
|
||||
// to serve these downloads with pk-web is generated.
|
||||
package main
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"archive/zip"
|
||||
"bufio"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"flag"
|
||||
|
|
@ -34,13 +37,13 @@ import (
|
|||
"log"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"runtime"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"perkeep.org/internal/osutil"
|
||||
|
|
@ -53,106 +56,358 @@ import (
|
|||
)
|
||||
|
||||
var (
|
||||
flagRev = flag.String("rev", "", "Perkeep revision to build (tag or commit hash). For development purposes, you can instead specify the path to a local Perkeep source tree from which to build, with the form \"WIP:/path/to/dir\".")
|
||||
flagDate = flag.String("date", "", "The release date to use in the file names to be uploaded, in the YYYYMMDD format. Defaults to today's date.")
|
||||
flagUpload = flag.Bool("upload", true, "Upload all the generated tarballs and zip archives.")
|
||||
flagSkipGen = flag.Bool("skipgen", false, "Do not recreate the release tarballs, and directly use the ones found in perkeep.org/misc/docker/release. Use -upload=false and -skipgen=true to only generate the monthly release page.")
|
||||
flagStatsFrom = flag.String("stats_from", "", "Also generate commit statistics on the release page, starting from the given commit, and ending at the one given as -rev.")
|
||||
// TODO(mpl): make sanity run the tests too, once they're more reliable.
|
||||
flagSanity = flag.Bool("sanity", true, "Verify 'go run make.go' succeeds when building the source tarball. Abort everything if not.")
|
||||
flagRev = flag.String("rev", "", "Perkeep revision to build (tag or commit hash). For development purposes, you can instead specify the path to a local Perkeep source tree from which to build, with the form \"WIP:/path/to/dir\".")
|
||||
flagVersion = flag.String("version", "", "The name of the release, (e.g. 0.10.1, or 20180512) used as part of the name for the file downloads. Defaults to today's date in YYYYMMDD format.")
|
||||
flagArchiveType = flag.String("kind", "all", `The kind of archive to build for the release. Possible values are: "darwin", "linux", "windows", "src" (zip of all the source code), or "all" (for all the previous values).`)
|
||||
flagUpload = flag.Bool("upload", true, "Upload all the generated tarballs and zip archives.")
|
||||
flagSkipGen = flag.Bool("skipgen", false, "Do not recreate the release tarballs, and directly use the ones found in perkeep.org/misc/release. Use -upload=false and -skipgen=true to only generate the release page.")
|
||||
flagStatsFrom = flag.String("stats_from", "", "Also generate commit statistics on the release page, starting from the given commit, and ending at the one given as -rev.")
|
||||
)
|
||||
|
||||
var (
|
||||
camDir string
|
||||
releaseDate time.Time
|
||||
pkDir string
|
||||
releaseDir string
|
||||
workDir string
|
||||
goVersion string
|
||||
pkVersion string
|
||||
)
|
||||
|
||||
const (
|
||||
titleDateFormat = "2006-01-02"
|
||||
fileDateFormat = "20060102"
|
||||
project = "camlistore-website"
|
||||
bucket = "camlistore-release"
|
||||
goDockerImage = "perkeep/go"
|
||||
goCmd = "/usr/local/go/bin/go"
|
||||
genBinariesProgram = "/usr/local/bin/build-binaries.go"
|
||||
zipSourceProgram = "/usr/local/bin/zip-source.go"
|
||||
titleDateFormat = "2006-01-02"
|
||||
fileDateFormat = "20060102"
|
||||
project = "camlistore-website"
|
||||
bucket = "perkeep-release"
|
||||
)
|
||||
|
||||
func isWIP() bool {
|
||||
return strings.HasPrefix(*flagRev, "WIP")
|
||||
func usage() {
|
||||
fmt.Fprintf(os.Stderr, "Usage:\n")
|
||||
fmt.Fprintf(os.Stderr, "%s [-rev perkeep_revision | -rev WIP:/path/to/perkeep/source]\n", os.Args[0])
|
||||
flag.PrintDefaults()
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
func rev() string {
|
||||
if isWIP() {
|
||||
return "WORKINPROGRESS"
|
||||
func checkFlags() {
|
||||
if flag.NArg() != 0 {
|
||||
usage()
|
||||
}
|
||||
if *flagRev == "" {
|
||||
fmt.Fprintf(os.Stderr, "Usage error: --rev is required.\n")
|
||||
usage()
|
||||
}
|
||||
return (*flagRev)[0:10]
|
||||
}
|
||||
|
||||
// genDownloads creates all the zips and tarballs, and uploads them.
|
||||
func genDownloads() error {
|
||||
dockDotGo := filepath.Join(camDir, "misc", "docker", "dock.go")
|
||||
releaseDir := filepath.Join(camDir, "misc", "docker", "release")
|
||||
var wg sync.WaitGroup
|
||||
func main() {
|
||||
flag.Usage = usage
|
||||
flag.Parse()
|
||||
checkFlags()
|
||||
|
||||
var err error
|
||||
pkDir, err = osutil.GoPackagePath("perkeep.org")
|
||||
if err != nil {
|
||||
log.Fatalf("Error looking up perkeep.org dir: %v", err)
|
||||
}
|
||||
releaseDir = filepath.Join(pkDir, "misc", "release")
|
||||
|
||||
workDir, err = ioutil.TempDir("", "pk-build_release")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(workDir)
|
||||
|
||||
var archives []string
|
||||
if !*flagSkipGen {
|
||||
// Gen the source zip:
|
||||
args := []string{
|
||||
"run",
|
||||
dockDotGo,
|
||||
"-build_image=false",
|
||||
"-zip_source=true",
|
||||
"-rev=" + *flagRev,
|
||||
"-sanity=" + fmt.Sprintf("%t", *flagSanity),
|
||||
}
|
||||
cmd := exec.Command("go", args...)
|
||||
cmd.Stdout = os.Stdout
|
||||
cmd.Stderr = os.Stderr
|
||||
if err := cmd.Run(); err != nil {
|
||||
return err
|
||||
archives, err = genArchive()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
upload(filepath.Join(releaseDir, "perkeep-src.zip"))
|
||||
if *flagUpload {
|
||||
for _, v := range archives {
|
||||
upload(filepath.Join(releaseDir, v))
|
||||
}
|
||||
}
|
||||
|
||||
if *flagArchiveType != "all" {
|
||||
// do not bother generating the release page if we're not doing a full blown release build.
|
||||
return
|
||||
}
|
||||
|
||||
releaseData, err := listDownloads()
|
||||
if err != nil {
|
||||
if *flagSkipGen {
|
||||
// Most likely we're failing because we can't reach the
|
||||
// bucket (working offline), annd we're working on this
|
||||
// program and testing things out, so make this error
|
||||
// non-critical so we can still generate the release notes
|
||||
// and stats.
|
||||
log.Print(err)
|
||||
releaseData = &ReleaseData{}
|
||||
} else {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
if *flagStatsFrom != "" {
|
||||
commitStats, err := genCommitStats()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
releaseData.Stats = commitStats
|
||||
|
||||
notes, err := genReleaseNotes()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
releaseData.ReleaseNotes = notes
|
||||
}
|
||||
|
||||
if err := genReleasePage(releaseData); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// genArchive generates the requested tarball(s) and zip archive(s), and returns
|
||||
// their filenames.
|
||||
func genArchive() ([]string, error) {
|
||||
switch *flagArchiveType {
|
||||
case "linux", "darwin", "windows":
|
||||
genBinaries(*flagArchiveType)
|
||||
return []string{packBinaries(*flagArchiveType)}, nil
|
||||
case "src":
|
||||
return []string{zipSource()}, nil
|
||||
default:
|
||||
return genAll()
|
||||
}
|
||||
}
|
||||
|
||||
// genAll creates all the zips and tarballs, and returns their filenames.
|
||||
func genAll() ([]string, error) {
|
||||
zipSource()
|
||||
for _, platform := range []string{"linux", "darwin", "windows"} {
|
||||
genBinaries(platform)
|
||||
packBinaries(platform)
|
||||
}
|
||||
getVersions()
|
||||
return []string{
|
||||
"perkeep-darwin.tar.gz",
|
||||
"perkeep-linux.tar.gz",
|
||||
"perkeep-src.zip",
|
||||
"perkeep-windows.zip",
|
||||
}, nil
|
||||
}
|
||||
|
||||
// getVersions uses the freshly built perkeepd binary to get the Perkeep and Go
|
||||
// versions used to build the release.
|
||||
func getVersions() {
|
||||
pkBin := filepath.Join(workDir, runtime.GOOS, "bin", "perkeepd")
|
||||
cmd := exec.Command(pkBin, "-version")
|
||||
var buf bytes.Buffer
|
||||
cmd.Stderr = &buf
|
||||
if err := cmd.Run(); err != nil {
|
||||
log.Fatalf("Error getting version from perkeepd: %v, %v", err, buf.String())
|
||||
}
|
||||
sc := bufio.NewScanner(&buf)
|
||||
for sc.Scan() {
|
||||
l := sc.Text()
|
||||
fields := strings.Fields(l)
|
||||
if pkVersion == "" {
|
||||
if len(fields) != 4 || fields[0] != "perkeepd" {
|
||||
log.Fatalf("Unexpected perkeepd -version output: %q", l)
|
||||
}
|
||||
pkVersion = fields[3]
|
||||
continue
|
||||
}
|
||||
if len(fields) != 4 || fields[0] != "Go" {
|
||||
log.Fatalf("Unexpected perkeepd -version output: %q", l)
|
||||
}
|
||||
goVersion = fields[2]
|
||||
break
|
||||
}
|
||||
if err := sc.Err(); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// genBinaries runs go run make.go for the given osType in a docker container.
|
||||
func genBinaries(osType string) {
|
||||
cwd := filepath.Join(workDir, osType)
|
||||
check(os.MkdirAll(cwd, 0755))
|
||||
image := goDockerImage
|
||||
args := []string{
|
||||
"run",
|
||||
"--rm",
|
||||
"--volume=" + cwd + ":/OUT",
|
||||
"--volume=" + filepath.Join(releaseDir, "build-binaries.go") + ":" + genBinariesProgram + ":ro",
|
||||
}
|
||||
if isWIP() {
|
||||
args = append(args, "--volume="+localCamliSource()+":/IN:ro",
|
||||
image, goCmd, "run", genBinariesProgram, "--rev=WIP:/IN", "--os="+osType)
|
||||
} else {
|
||||
args = append(args, image, goCmd, "run", genBinariesProgram, "--rev="+rev(), "--os="+osType)
|
||||
}
|
||||
cmd := exec.Command("docker", args...)
|
||||
cmd.Stdout = os.Stdout
|
||||
cmd.Stderr = os.Stderr
|
||||
if err := cmd.Run(); err != nil {
|
||||
log.Fatalf("Error building binaries in go container: %v", err)
|
||||
}
|
||||
fmt.Printf("Perkeep binaries successfully generated in %v\n", filepath.Join(cwd, "bin"))
|
||||
}
|
||||
|
||||
// packBinaries builds the archive that contains the binaries built by
|
||||
// genBinaries.
|
||||
func packBinaries(osType string) string {
|
||||
archiveName := "perkeep-" + osType + ".tar.gz"
|
||||
if osType == "windows" {
|
||||
archiveName = strings.Replace(archiveName, ".tar.gz", ".zip", 1)
|
||||
}
|
||||
binaries := map[string]bool{
|
||||
exeName("perkeepd", osType): false,
|
||||
exeName("pk-get", osType): false,
|
||||
exeName("pk-put", osType): false,
|
||||
exeName("pk", osType): false,
|
||||
exeName("publisher", osType): false,
|
||||
exeName("scancab", osType): false,
|
||||
exeName("scanningcabinet", osType): false,
|
||||
}
|
||||
switch osType {
|
||||
case "linux", "darwin":
|
||||
binaries["pk-mount"] = false
|
||||
}
|
||||
toPack := func(bin string) bool {
|
||||
for k := range binaries {
|
||||
if bin == k {
|
||||
binaries[k] = true
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
archivePath := filepath.Join(releaseDir, archiveName)
|
||||
defer func() {
|
||||
for name, found := range binaries {
|
||||
if !found {
|
||||
log.Fatalf("%v was not packed in tarball", name)
|
||||
}
|
||||
}
|
||||
fmt.Printf("Perkeep binaries successfully packed in %v\n", archivePath)
|
||||
}()
|
||||
|
||||
// gen the binaries tarballs:
|
||||
for _, platform := range []string{"linux", "darwin", "windows"} {
|
||||
if !*flagSkipGen {
|
||||
args := []string{
|
||||
"run",
|
||||
dockDotGo,
|
||||
"-build_image=false",
|
||||
"-build_release=true",
|
||||
"-rev=" + *flagRev,
|
||||
"-os=" + platform,
|
||||
binDir := path.Join(workDir, osType, "bin")
|
||||
check(os.Chdir(binDir))
|
||||
dir, err := os.Open(binDir)
|
||||
check(err)
|
||||
defer dir.Close()
|
||||
|
||||
if osType == "windows" {
|
||||
fw, err := os.Create(archivePath)
|
||||
check(err)
|
||||
defer func() {
|
||||
check(fw.Close())
|
||||
}()
|
||||
w := zip.NewWriter(fw)
|
||||
defer func() {
|
||||
check(w.Close())
|
||||
}()
|
||||
names, err := dir.Readdirnames(-1)
|
||||
check(err)
|
||||
for _, name := range names {
|
||||
if !toPack(name) {
|
||||
continue
|
||||
}
|
||||
cmd := exec.Command("go", args...)
|
||||
cmd.Stdout = os.Stdout
|
||||
cmd.Stderr = os.Stderr
|
||||
if err := cmd.Run(); err != nil {
|
||||
return err
|
||||
b, err := ioutil.ReadFile(path.Join(binDir, name))
|
||||
check(err)
|
||||
f, err := w.Create(name)
|
||||
check(err)
|
||||
_, err = f.Write(b)
|
||||
check(err)
|
||||
}
|
||||
return archiveName
|
||||
}
|
||||
|
||||
fw, err := os.Create(archivePath)
|
||||
check(err)
|
||||
defer func() {
|
||||
check(fw.Close())
|
||||
}()
|
||||
pr, pw := io.Pipe()
|
||||
go func() {
|
||||
tw := tar.NewWriter(pw)
|
||||
fis, err := dir.Readdir(-1)
|
||||
check(err)
|
||||
for _, file := range fis {
|
||||
if !toPack(file.Name()) {
|
||||
continue
|
||||
}
|
||||
hdr, err := tar.FileInfoHeader(file, "")
|
||||
check(err)
|
||||
check(tw.WriteHeader(hdr))
|
||||
fr, err := os.Open(file.Name())
|
||||
check(err)
|
||||
n, err := io.Copy(tw, fr)
|
||||
check(err)
|
||||
fr.Close()
|
||||
if n != file.Size() {
|
||||
log.Fatalf("failed to tar all of %v; got %v, wanted %v", file.Name(), n, file.Size())
|
||||
}
|
||||
}
|
||||
wg.Add(1)
|
||||
go func(osType string) {
|
||||
defer wg.Done()
|
||||
filename := "perkeep-" + osType + ".tar.gz"
|
||||
if osType == "windows" {
|
||||
filename = strings.Replace(filename, ".tar.gz", ".zip", 1)
|
||||
}
|
||||
upload(filepath.Join(releaseDir, filename))
|
||||
}(platform)
|
||||
check(tw.Close())
|
||||
check(pw.CloseWithError(io.EOF))
|
||||
}()
|
||||
zw := gzip.NewWriter(fw)
|
||||
n, err := io.Copy(zw, pr)
|
||||
if err != nil {
|
||||
log.Fatalf("Error copying to gzip writer: after %d bytes, %v", n, err)
|
||||
}
|
||||
wg.Wait()
|
||||
return nil
|
||||
if err := zw.Close(); err != nil {
|
||||
log.Fatalf("gzip.Close: %v", err)
|
||||
}
|
||||
return archiveName
|
||||
}
|
||||
|
||||
// zipSource builds the zip archive that contains the source code of Perkeep.
|
||||
func zipSource() string {
|
||||
cwd := filepath.Join(workDir, "src")
|
||||
check(os.MkdirAll(cwd, 0755))
|
||||
image := goDockerImage
|
||||
args := []string{
|
||||
"run",
|
||||
"--rm",
|
||||
"--volume=" + cwd + ":/OUT",
|
||||
"--volume=" + path.Join(releaseDir, "zip-source.go") + ":" + zipSourceProgram + ":ro",
|
||||
}
|
||||
if isWIP() {
|
||||
args = append(args, "--volume="+localCamliSource()+":/IN:ro",
|
||||
image, goCmd, "run", zipSourceProgram, "--rev=WIP:/IN")
|
||||
} else {
|
||||
args = append(args, image, goCmd, "run", zipSourceProgram, "--rev="+rev())
|
||||
}
|
||||
cmd := exec.Command("docker", args...)
|
||||
cmd.Stdout = os.Stdout
|
||||
cmd.Stderr = os.Stderr
|
||||
if err := cmd.Run(); err != nil {
|
||||
log.Fatalf("Error zipping Perkeep source in go container: %v", err)
|
||||
}
|
||||
archiveName := "perkeep-src.zip"
|
||||
// can't use os.Rename because invalid cross-device link error likely
|
||||
cmd = exec.Command("mv", filepath.Join(cwd, archiveName), releaseDir)
|
||||
cmd.Stdout = os.Stdout
|
||||
cmd.Stderr = os.Stderr
|
||||
if err := cmd.Run(); err != nil {
|
||||
log.Fatalf("Error moving source zip from %v to %v: %v", filepath.Join(cwd, archiveName), releaseDir, err)
|
||||
}
|
||||
fmt.Printf("Perkeep source successfully zipped in %v\n", releaseDir)
|
||||
return archiveName
|
||||
}
|
||||
|
||||
func upload(srcPath string) {
|
||||
if !*flagUpload {
|
||||
return
|
||||
}
|
||||
destName := strings.Replace(filepath.Base(srcPath), "perkeep", "perkeep-"+releaseDate.Format(fileDateFormat), 1)
|
||||
versionedTarball := "monthly/" + destName
|
||||
uploadName := strings.Replace(filepath.Base(srcPath), "perkeep", "perkeep-"+version(), 1)
|
||||
|
||||
log.Printf("Uploading %s/%s ...", bucket, versionedTarball)
|
||||
log.Printf("Uploading %s/%s ...", bucket, uploadName)
|
||||
|
||||
ts, err := tokenSource(bucket)
|
||||
if err != nil {
|
||||
|
|
@ -163,11 +418,11 @@ func upload(srcPath string) {
|
|||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
w := stoClient.Bucket(bucket).Object(versionedTarball).NewWriter(ctx)
|
||||
w := stoClient.Bucket(bucket).Object(uploadName).NewWriter(ctx)
|
||||
w.ACL = publicACL(project)
|
||||
w.CacheControl = "no-cache" // TODO: remove for non-tip releases? set expirations?
|
||||
contentType := "application/x-gtar"
|
||||
if strings.HasSuffix(versionedTarball, ".zip") {
|
||||
if strings.HasSuffix(uploadName, ".zip") {
|
||||
contentType = "application/zip"
|
||||
}
|
||||
w.ContentType = contentType
|
||||
|
|
@ -186,14 +441,14 @@ func upload(srcPath string) {
|
|||
if err := w.Close(); err != nil {
|
||||
log.Fatalf("closing GCS storage writer: %v", err)
|
||||
}
|
||||
log.Printf("Uploaded monthly tarball to %s", versionedTarball)
|
||||
log.Printf("Uploaded archive to %s/%s", bucket, uploadName)
|
||||
|
||||
// And upload the corresponding checksum
|
||||
checkSumFile := versionedTarball + ".sha256"
|
||||
checkSumFile := uploadName + ".sha256"
|
||||
sum := fmt.Sprintf("%x", csw.Sum(nil))
|
||||
w = stoClient.Bucket(bucket).Object(checkSumFile).NewWriter(ctx)
|
||||
w.ACL = publicACL(project)
|
||||
w.CacheControl = "no-cache" // TODO: remove for non-tip releases? set expirations?
|
||||
w.CacheControl = "no-cache"
|
||||
w.ContentType = "text/plain"
|
||||
if _, err := io.Copy(w, strings.NewReader(sum)); err != nil {
|
||||
log.Fatalf("error uploading checksum %v: %v", checkSumFile, err)
|
||||
|
|
@ -201,7 +456,7 @@ func upload(srcPath string) {
|
|||
if err := w.Close(); err != nil {
|
||||
log.Fatalf("closing GCS storage writer: %v", err)
|
||||
}
|
||||
log.Printf("Uploaded monthly tarball checksum to %s", checkSumFile)
|
||||
log.Printf("Uploaded archive checksum to %s", checkSumFile)
|
||||
}
|
||||
|
||||
type DownloadData struct {
|
||||
|
|
@ -211,9 +466,9 @@ type DownloadData struct {
|
|||
}
|
||||
|
||||
type ReleaseData struct {
|
||||
Date string
|
||||
Name string
|
||||
Download []DownloadData
|
||||
CamliVersion string
|
||||
PkVersion string
|
||||
GoVersion string
|
||||
Stats *stats
|
||||
ReleaseNotes map[string][]string
|
||||
|
|
@ -222,18 +477,18 @@ type ReleaseData struct {
|
|||
// Note: the space trimming in the range loop is important. Since all of our
|
||||
// html still goes through a markdown engine, newlines in between items would make
|
||||
// markdown wrap the items in <p></p>, which breaks the page's style.
|
||||
var monthlyTemplate = `
|
||||
<h1>Monthly Release: {{.Date}}</h1>
|
||||
var releaseTemplate = `
|
||||
<h1>Perkeep Release: {{.Name}}</h1>
|
||||
|
||||
<p>
|
||||
Perkeep version <a href='https://github.com/perkeep/perkeep/commit/{{.CamliVersion}}'>{{.CamliVersion}}</a> built with Go {{.GoVersion}}.
|
||||
Perkeep version <a href='https://github.com/perkeep/perkeep/commit/{{.PkVersion}}'>{{.PkVersion}}</a> built with Go {{.GoVersion}}.
|
||||
</p>
|
||||
|
||||
<h2>Downloads</h2>
|
||||
|
||||
<center>
|
||||
{{- range $d := .Download}}
|
||||
<a class="downloadBox" href="/dl/monthly/{{$d.Filename}}">
|
||||
<a class="downloadBox" href="/dl/{{$d.Filename}}">
|
||||
<div class="platform">{{$d.Platform}}</div>
|
||||
<div>
|
||||
<span class="filename">{{$d.Filename}}</span>
|
||||
|
|
@ -275,14 +530,9 @@ Perkeep version <a href='https://github.com/perkeep/perkeep/commit/{{.CamliVersi
|
|||
{{end}}
|
||||
`
|
||||
|
||||
// TODO(mpl): keep goVersion automatically in sync with version in
|
||||
// misc/docker/go. Or guess it from somewhere else.
|
||||
|
||||
const goVersion = "1.10"
|
||||
|
||||
// listDownloads lists all the files found in the monthly repo, and from them,
|
||||
// builds the data that we'll feed to the template to generate the monthly
|
||||
// downloads camweb page.
|
||||
// listDownloads lists all the files found in the release bucket, and from them,
|
||||
// builds the data that we'll feed to the template to generate the release
|
||||
// downloads page for pk-web.
|
||||
func listDownloads() (*ReleaseData, error) {
|
||||
ts, err := tokenSource(bucket)
|
||||
if err != nil {
|
||||
|
|
@ -320,6 +570,7 @@ func listDownloads() (*ReleaseData, error) {
|
|||
return buf.String(), nil
|
||||
}
|
||||
var date time.Time
|
||||
fileVersion := version()
|
||||
checkDate := func(objDate time.Time) error {
|
||||
if date.IsZero() {
|
||||
date = objDate
|
||||
|
|
@ -332,25 +583,26 @@ func listDownloads() (*ReleaseData, error) {
|
|||
if d < 24*time.Hour {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("objects in monthly have not been uploaded or updated the same day")
|
||||
return fmt.Errorf("archives for version %s have not been uploaded or updated the same day", fileVersion)
|
||||
}
|
||||
|
||||
var (
|
||||
downloadData []DownloadData
|
||||
nameToSum = make(map[string]string)
|
||||
)
|
||||
fileDate := releaseDate.Format(fileDateFormat)
|
||||
log.Printf("Now looking for monthly/perkeep-%s-* files in bucket", fileDate)
|
||||
objIt := stoClient.Bucket(bucket).Objects(ctx, &storage.Query{Prefix: "monthly/"})
|
||||
|
||||
log.Printf("Now looking for perkeep-%s-* files in bucket %s", fileVersion, bucket)
|
||||
objPrefix := "perkeep-" + fileVersion
|
||||
objIt := stoClient.Bucket(bucket).Objects(ctx, &storage.Query{Prefix: objPrefix})
|
||||
for {
|
||||
attrs, err := objIt.Next()
|
||||
if err == iterator.Done {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error listing objects in \"monthly\": %v", err)
|
||||
return nil, fmt.Errorf("error listing objects in %s: %v", bucket, err)
|
||||
}
|
||||
if !strings.Contains(attrs.Name, fileDate) {
|
||||
if !strings.Contains(attrs.Name, fileVersion) {
|
||||
continue
|
||||
}
|
||||
if err := checkDate(attrs.Updated); err != nil {
|
||||
|
|
@ -365,16 +617,16 @@ func listDownloads() (*ReleaseData, error) {
|
|||
}
|
||||
nameToSum[strings.TrimSuffix(attrs.Name, ".sha256")] = sum
|
||||
}
|
||||
objIt = stoClient.Bucket(bucket).Objects(ctx, &storage.Query{Prefix: "monthly/"})
|
||||
objIt = stoClient.Bucket(bucket).Objects(ctx, &storage.Query{Prefix: objPrefix})
|
||||
for {
|
||||
attrs, err := objIt.Next()
|
||||
if err == iterator.Done {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error listing objects in \"monthly\": %v", err)
|
||||
return nil, fmt.Errorf("error listing objects in %s: %v", bucket, err)
|
||||
}
|
||||
if !strings.Contains(attrs.Name, fileDate) {
|
||||
if !strings.Contains(attrs.Name, fileVersion) {
|
||||
continue
|
||||
}
|
||||
if strings.HasSuffix(attrs.Name, ".sha256") {
|
||||
|
|
@ -392,61 +644,35 @@ func listDownloads() (*ReleaseData, error) {
|
|||
}
|
||||
|
||||
return &ReleaseData{
|
||||
Date: releaseDate.Format(titleDateFormat),
|
||||
Download: downloadData,
|
||||
CamliVersion: rev(),
|
||||
GoVersion: goVersion,
|
||||
Name: version(),
|
||||
Download: downloadData,
|
||||
PkVersion: pkVersion,
|
||||
GoVersion: goVersion,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func genMonthlyPage(releaseData *ReleaseData) error {
|
||||
tpl, err := template.New("monthly").Parse(monthlyTemplate)
|
||||
func genReleasePage(releaseData *ReleaseData) error {
|
||||
tpl, err := template.New("release").Parse(releaseTemplate)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not parse template: %v", err)
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
if err := tpl.ExecuteTemplate(&buf, "monthly", releaseData); err != nil {
|
||||
if err := tpl.ExecuteTemplate(&buf, "release", releaseData); err != nil {
|
||||
return fmt.Errorf("could not execute template: %v", err)
|
||||
}
|
||||
|
||||
monthlyDocDir := filepath.Join(camDir, filepath.FromSlash("doc/release"))
|
||||
if err := os.MkdirAll(monthlyDocDir, 0755); err != nil {
|
||||
releaseDocDir := filepath.Join(pkDir, filepath.FromSlash("doc/release"))
|
||||
if err := os.MkdirAll(releaseDocDir, 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
monthlyDocPage := filepath.Join(monthlyDocDir, "monthly.html")
|
||||
if err := ioutil.WriteFile(monthlyDocPage, buf.Bytes(), 0700); err != nil {
|
||||
return fmt.Errorf("could not write template to file %v: %v", monthlyDocPage, err)
|
||||
releaseDocPage := filepath.Join(releaseDocDir, "release.html")
|
||||
if err := ioutil.WriteFile(releaseDocPage, buf.Bytes(), 0700); err != nil {
|
||||
return fmt.Errorf("could not write template to file %v: %v", releaseDocPage, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func usage() {
|
||||
fmt.Fprintf(os.Stderr, "Usage:\n")
|
||||
fmt.Fprintf(os.Stderr, "%s [-rev perkeep_revision | -rev WIP:/path/to/camli/source]\n", os.Args[0])
|
||||
flag.PrintDefaults()
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
func checkFlags() {
|
||||
if flag.NArg() != 0 {
|
||||
usage()
|
||||
}
|
||||
if *flagRev == "" {
|
||||
fmt.Fprintf(os.Stderr, "Usage error: --rev is required.\n")
|
||||
usage()
|
||||
}
|
||||
releaseDate = time.Now()
|
||||
if *flagDate != "" {
|
||||
var err error
|
||||
releaseDate, err = time.Parse(fileDateFormat, *flagDate)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Incorrect date format: %v", err)
|
||||
usage()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type stats struct {
|
||||
FromRev string
|
||||
TotalCommitters int
|
||||
|
|
@ -457,7 +683,7 @@ type stats struct {
|
|||
// returns commiters names mapped by e-mail, uniqued first by e-mail, then by name.
|
||||
// When uniquing, higher count of commits wins.
|
||||
func committers() (map[string]string, error) {
|
||||
cmd := exec.Command("git", "shortlog", "-n", "-e", "-s", *flagStatsFrom+".."+rev())
|
||||
cmd := exec.Command("git", "shortlog", "-n", "-e", "-s", *flagStatsFrom+".."+revOrHEAD())
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%v; %v", err, string(out))
|
||||
|
|
@ -508,7 +734,7 @@ func committers() (map[string]string, error) {
|
|||
}
|
||||
|
||||
func countCommits() (int, error) {
|
||||
cmd := exec.Command("git", "log", "--format=oneline", *flagStatsFrom+".."+rev())
|
||||
cmd := exec.Command("git", "log", "--format=oneline", *flagStatsFrom+".."+revOrHEAD())
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("%v; %v", err, string(out))
|
||||
|
|
@ -547,7 +773,7 @@ func genCommitStats() (*stats, error) {
|
|||
}
|
||||
|
||||
func genReleaseNotes() (map[string][]string, error) {
|
||||
cmd := exec.Command("git", "log", "--format=oneline", "--no-merges", *flagStatsFrom+".."+rev())
|
||||
cmd := exec.Command("git", "log", "--format=oneline", "--no-merges", *flagStatsFrom+".."+revOrHEAD())
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%v; %v", err, string(out))
|
||||
|
|
@ -608,56 +834,53 @@ func genReleaseNotes() (map[string][]string, error) {
|
|||
return commitByContext, nil
|
||||
}
|
||||
|
||||
func main() {
|
||||
flag.Usage = usage
|
||||
flag.Parse()
|
||||
checkFlags()
|
||||
func isWIP() bool {
|
||||
return strings.HasPrefix(*flagRev, "WIP")
|
||||
}
|
||||
|
||||
var err error
|
||||
camDir, err = osutil.GoPackagePath("perkeep.org")
|
||||
// localCamliSource returns the path to the local Perkeep source tree
|
||||
// that should be specified in *flagRev if *flagRev starts with "WIP:",
|
||||
// empty string otherwise.
|
||||
func localCamliSource() string {
|
||||
if !isWIP() {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimPrefix(*flagRev, "WIP:")
|
||||
}
|
||||
|
||||
func rev() string {
|
||||
if isWIP() {
|
||||
return "WORKINPROGRESS"
|
||||
}
|
||||
return (*flagRev)[0:10]
|
||||
}
|
||||
|
||||
func revOrHEAD() string {
|
||||
if isWIP() {
|
||||
return "HEAD"
|
||||
}
|
||||
return (*flagRev)[0:10]
|
||||
}
|
||||
|
||||
func version() string {
|
||||
if *flagVersion != "" {
|
||||
return *flagVersion
|
||||
}
|
||||
return time.Now().Format(fileDateFormat)
|
||||
}
|
||||
|
||||
func check(err error) {
|
||||
if err != nil {
|
||||
log.Fatalf("Error looking up perkeep.org dir: %v", err)
|
||||
}
|
||||
|
||||
if err := genDownloads(); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
releaseData, err := listDownloads()
|
||||
if err != nil {
|
||||
if *flagSkipGen {
|
||||
// Most likely we're failing because we can't reach the
|
||||
// bucket (working offline), annd we're working on this
|
||||
// program and testing things out, so make this error
|
||||
// non-critical so we can still generate the release notes
|
||||
// and stats.
|
||||
log.Print(err)
|
||||
releaseData = &ReleaseData{}
|
||||
} else {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
if *flagStatsFrom != "" && !isWIP() {
|
||||
commitStats, err := genCommitStats()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
releaseData.Stats = commitStats
|
||||
|
||||
notes, err := genReleaseNotes()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
releaseData.ReleaseNotes = notes
|
||||
}
|
||||
|
||||
if err := genMonthlyPage(releaseData); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// TODO(mpl): refactor in a common place so that dock.go and this program here can use the helpers below.
|
||||
func exeName(s, osType string) string {
|
||||
if osType == "windows" {
|
||||
return s + ".exe"
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func homedir() string {
|
||||
if runtime.GOOS == "windows" {
|
||||
|
|
@ -689,7 +912,7 @@ func ProjectTokenSource(proj string, scopes ...string) (oauth2.TokenSource, erro
|
|||
}
|
||||
|
||||
var bucketProject = map[string]string{
|
||||
"camlistore-release": "camlistore-website",
|
||||
"perkeep-release": "camlistore-website",
|
||||
}
|
||||
|
||||
func tokenSource(bucket string) (oauth2.TokenSource, error) {
|
||||
|
|
@ -39,10 +39,9 @@ import (
|
|||
)
|
||||
|
||||
var (
|
||||
flagRev = flag.String("rev", "", "Perkeep revision to ship (tag or commit hash). For development purposes, you can instead specify the path to a local Perkeep source tree from which to build, with the form \"WIP:/path/to/dir\".")
|
||||
flagVersion = flag.String("version", "", "The version number that is used in the zip file name, and in the VERSION file, e.g. 0.10")
|
||||
flagOutDir = flag.String("outdir", "/OUT/", "Directory where to write the zip file.")
|
||||
flagSanity = flag.Bool("sanity", true, "Check before making the zip that its contents pass the \"go run make.go\" test.")
|
||||
flagRev = flag.String("rev", "", "Perkeep revision to ship (tag or commit hash). For development purposes, you can instead specify the path to a local Perkeep source tree from which to build, with the form \"WIP:/path/to/dir\".")
|
||||
flagOutDir = flag.String("outdir", "/OUT/", "Directory where to write the zip file.")
|
||||
flagSanity = flag.Bool("sanity", true, "Check before making the zip that its contents pass the \"go run make.go\" test.")
|
||||
)
|
||||
|
||||
const tmpSource = "/tmp/perkeep.org"
|
||||
|
|
@ -66,7 +65,6 @@ var (
|
|||
"Gopkg.lock": false,
|
||||
"Gopkg.toml": false,
|
||||
"internal": true,
|
||||
"lib": true,
|
||||
"Makefile": false,
|
||||
"make.go": false,
|
||||
"misc": true,
|
||||
|
|
@ -108,20 +106,6 @@ func localCamliSource() string {
|
|||
return strings.TrimPrefix(*flagRev, "WIP:")
|
||||
}
|
||||
|
||||
func rev() string {
|
||||
if isWIP() {
|
||||
return "WORKINPROGRESS"
|
||||
}
|
||||
return *flagRev
|
||||
}
|
||||
|
||||
func version() string {
|
||||
if *flagVersion != "" {
|
||||
return fmt.Sprintf("%v (git rev %v)", *flagVersion, rev())
|
||||
}
|
||||
return rev()
|
||||
}
|
||||
|
||||
func getCamliSrc() {
|
||||
// TODO(mpl): we could filter right within mirrorCamliSrc and
|
||||
// fetchCamliSrc so we end up directly only with what we want as source.
|
||||
|
|
@ -247,10 +231,6 @@ func filter() {
|
|||
log.Fatalf("file (or directory) %v should be included in release, but not found in source", name)
|
||||
}
|
||||
}
|
||||
// we insert the version in the VERSION file, so make.go does no need git
|
||||
// in the container to detect the Perkeep version.
|
||||
check(os.Chdir(destDir))
|
||||
check(ioutil.WriteFile("VERSION", []byte(version()), 0777))
|
||||
}
|
||||
|
||||
func checkBuild() {
|
||||
|
|
@ -276,7 +256,7 @@ func pack() {
|
|||
check(err)
|
||||
w := zip.NewWriter(fw)
|
||||
|
||||
check(filepath.Walk("perkeep.org", func(filePath string, fi os.FileInfo, err error) error {
|
||||
if err := filepath.Walk("perkeep.org", func(filePath string, fi os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -301,7 +281,9 @@ func pack() {
|
|||
return err
|
||||
}
|
||||
return nil
|
||||
}))
|
||||
}); err != nil {
|
||||
log.Fatalf("Error while walking the source tree for zipping: %v", err)
|
||||
}
|
||||
check(w.Close())
|
||||
check(fw.Close())
|
||||
fmt.Printf("Perkeep source successfully packed in %v\n", zipFile)
|
||||
Loading…
Reference in New Issue