Merge "misc/docker: bucket.List + CAMLI_GOPHERJS_GOROOT fixes"

doc/release.txt

@@ -18,5 +18,5 @@ use -upload=true to directly upload the zip file to the camlistore-release/0.10/
 
 ** How to generate a monthly release: **
 
-go run ./misc/monthly.go -rev=$GIT_REVISION
+go run ./misc/monthly.go -rev=$GIT_REVISION -stats_from=cf0c847fbf7e3e48ef463e86674520a4da9aedbb
 git commit -m 'monthly release' doc/release/monthly.html

misc/docker/go/Dockerfile

@@ -10,3 +10,13 @@ WORKDIR /tmp
 RUN curl -O https://storage.googleapis.com/golang/go1.8rc2.linux-amd64.tar.gz
 RUN echo 'd62c2d44d0c6b434e3cda12505f3c9fb880757e3396af1e9ba861f7b547cc864  go1.8rc2.linux-amd64.tar.gz' | sha256sum -c
 RUN tar -C /usr/local -xzf go1.8rc2.linux-amd64.tar.gz
+
+# Adding Go 1.7 as well for gopherjs. Not all our docker images need to do a full
+# 'go run make.go', but I feel maintaining two different go docker images would
+# not be worth the trouble. We can just remove that as soon as gopherjs supports
+# Go 1.8.
+WORKDIR /tmp
+RUN curl -O https://storage.googleapis.com/golang/go1.7.4.linux-amd64.tar.gz
+RUN echo '47fda42e46b4c3ec93fa5d4d4cc6a748aa3f9411a2a2b7e08e3a6d80d753ec8b  go1.7.4.linux-amd64.tar.gz' | sha256sum -c
+RUN tar -xzf go1.7.4.linux-amd64.tar.gz
+RUN mv go /usr/local/go1.7

misc/docker/release/build-binaries.go

@@ -148,6 +148,7 @@ func build() {
 	oldPath := os.Getenv("PATH")
 	os.Setenv("GOPATH", "/gopath")
 	os.Setenv("PATH", "/usr/local/go/bin:"+oldPath)
+	check(os.Setenv("CAMLI_GOPHERJS_GOROOT", "/usr/local/go1.7"))
 	cmd := exec.Command("go", "run", "make.go", "--os", *buildOS)
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr

misc/docker/release/zip-source.go

@@ -259,6 +259,7 @@ func checkBuild() {
 	}
 	check(os.Chdir(tarballSrc))
 	check(os.Setenv("PATH", os.Getenv("PATH")+":/usr/local/go/bin/"))
+	check(os.Setenv("CAMLI_GOPHERJS_GOROOT", "/usr/local/go1.7"))
 	cmd := exec.Command("go", "run", "make.go")
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr

misc/monthly.go

@@ -48,6 +48,7 @@ import (
 	"golang.org/x/net/context"
 	"golang.org/x/oauth2"
 	"golang.org/x/oauth2/google"
+	"google.golang.org/api/iterator"
 	"google.golang.org/api/option"
 )
 
@@ -55,7 +56,7 @@ var (
 	flagRev       = flag.String("rev", "", "Camlistore revision to build (tag or commit hash). For development purposes, you can instead specify the path to a local Camlistore source tree from which to build, with the form \"WIP:/path/to/dir\".")
 	flagDate      = flag.String("date", "", "The release date to use in the file names to be uploaded, in the YYYYMMDD format. Defaults to today's date.")
 	flagUpload    = flag.Bool("upload", true, "Upload all the generated tarballs and zip archives.")
-	flagSkipGen   = flag.Bool("skipgen", false, "Do not recreate the release tarballs, and directly use the ones found in camlistore.org/misc/docker/release. Use -upload=true and -skipgen=true to only generate the monthly release page.")
+	flagSkipGen   = flag.Bool("skipgen", false, "Do not recreate the release tarballs, and directly use the ones found in camlistore.org/misc/docker/release. Use -upload=false and -skipgen=true to only generate the monthly release page.")
 	flagStatsFrom = flag.String("stats_from", "", "Also generate commit statistics on the release page, starting from the given commit, and ending at the one given as -rev.")
 	// TODO(mpl): make sanity run the tests too, once they're more reliable.
 	flagSanity = flag.Bool("sanity", true, "Verify 'go run make.go' succeeds when building the source tarball. Abort everything if not.")
@@ -275,7 +276,7 @@ Camlistore version <a href='https://github.com/camlistore/camlistore/commit/{{.C
 // TODO(mpl): keep goVersion automatically in sync with version in
 // misc/docker/go. Or guess it from somewhere else.
 
-const goVersion = "1.7"
+const goVersion = "1.8rc2"
 
 // listDownloads lists all the files found in the monthly repo, and from them,
 // builds the data that we'll feed to the template to generate the monthly
@@ -290,10 +291,6 @@ func listDownloads() (*ReleaseData, error) {
 	if err != nil {
 		return nil, err
 	}
-	objList, err := stoClient.Bucket(bucket).List(ctx, &storage.Query{Prefix: "monthly/"})
-	if err != nil {
-		return nil, err
-	}
 
 	platformBySuffix := map[string]string{
 		"src.zip":       "Source",
@@ -341,7 +338,16 @@ func listDownloads() (*ReleaseData, error) {
 		nameToSum    = make(map[string]string)
 	)
 	fileDate := releaseDate.Format(fileDateFormat)
-	for _, attrs := range objList.Results {
+	log.Printf("Now looking for monthly/camlistore-%s-* files in bucket", fileDate)
+	objIt := stoClient.Bucket(bucket).Objects(ctx, &storage.Query{Prefix: "monthly/"})
+	for {
+		attrs, err := objIt.Next()
+		if err == iterator.Done {
+			break
+		}
+		if err != nil {
+			return nil, fmt.Errorf("error listing objects in \"monthly\": %v", err)
+		}
 		if !strings.Contains(attrs.Name, fileDate) {
 			continue
 		}
@@ -357,7 +363,15 @@ func listDownloads() (*ReleaseData, error) {
 		}
 		nameToSum[strings.TrimSuffix(attrs.Name, ".sha256")] = sum
 	}
-	for _, attrs := range objList.Results {
+	objIt = stoClient.Bucket(bucket).Objects(ctx, &storage.Query{Prefix: "monthly/"})
+	for {
+		attrs, err := objIt.Next()
+		if err == iterator.Done {
+			break
+		}
+		if err != nil {
+			return nil, fmt.Errorf("error listing objects in \"monthly\": %v", err)
+		}
 		if !strings.Contains(attrs.Name, fileDate) {
 			continue
 		}