clients/web/embed/closure: use modern go:embed instead of old bespoke make.go hack

Signed-off-by: Brad Fitzpatrick <brad@danga.com>

.gitignore

@@ -36,7 +36,6 @@ misc/docker/djpeg-static/djpeg
 misc/docker/djpeg-static/djpeg.tar.gz
 misc/docker/perkeepd/perkeepd*
 misc/docker/perkeepd/djpeg
-server/perkeepd/ui/closure/z_data.go
 
 # not vendored in, because we build with the system's libsqlite3
 /vendor/github.com/mattn/go-sqlite3/sqlite3-binding.c

clients/web/embed/closure/lib/closure_embed.go

@@ -0,0 +1,6 @@
+package closurestatic
+
+import "embed"
+
+//go:embed closure
+var Closure embed.FS

dev/devcam/devcam.go

@@ -252,7 +252,6 @@ func build(targets ...string) error {
 		"run", "make.go",
 		"--quiet",
 		"--race=" + strconv.FormatBool(*race),
-		"--embed_static=false",
 		"--targets=" + targetsComma,
 	}
 	cmd := exec.Command("go", args...)

make.go

@@ -27,7 +27,6 @@ limitations under the License.
 package main
 
 import (
-	"archive/zip"
 	"bytes"
 	"crypto/sha256"
 	"errors"
@@ -42,13 +41,12 @@ import (
 	"path/filepath"
 	"regexp"
 	"runtime"
+	"slices"
 	"strconv"
 	"strings"
-	"time"
 )
 
 var (
-	embedResources   = flag.Bool("embed_static", true, "Whether to embed resources needed by the UI such as images, css, and javascript.")
 	race             = flag.Bool("race", false, "Build race-detector version of binaries (they will run slowly)")
 	verbose          = flag.Bool("v", strings.Contains(os.Getenv("CAMLI_DEBUG_X"), "makego"), "Verbose mode")
 	targets          = flag.String("targets", "", "Optional comma-separated list of targets (i.e go packages) to build and install. '*' builds everything.  Empty builds defaults for this platform. Example: perkeep.org/server/perkeepd,perkeep.org/cmd/pk-put")
@@ -138,25 +136,17 @@ func main() {
 		}
 	}
 
-	withPublisher := stringListContains(targs, "perkeep.org/app/publisher")
+	withPublisher := slices.Contains(targs, "perkeep.org/app/publisher")
 	if withPublisher {
 		if err := doPublisherUI(); err != nil {
 			log.Fatal(err)
 		}
 	}
 
-	withPerkeepd := stringListContains(targs, "perkeep.org/server/perkeepd")
-	if *embedResources && withPerkeepd {
-		doEmbed()
-	}
-
 	tags := []string{"purego"} // for cznic/zappy
 	if *static {
 		tags = append(tags, "netgo", "osusergo")
 	}
-	if *embedResources {
-		tags = append(tags, "with_embed")
-	}
 	baseArgs := []string{"install", "-v"}
 	if *race {
 		baseArgs = append(baseArgs, "-race")
@@ -339,10 +329,6 @@ func genJS(pkg, output string) error {
 
 func runGopherJS(pkg string) error {
 	args := []string{"run", "-mod=readonly", "github.com/goplusjs/gopherjs", "install", pkg, "-v", "--tags", "nocgo noReactBundle"}
-	if *embedResources {
-		// when embedding for "production", use -m to minify the javascript output
-		args = append(args, "-m")
-	}
 	cmd := exec.Command("go", args...)
 	cmd.Env = os.Environ()
 	// Pretend we're on linux regardless of the actual host, because recommended
@@ -447,8 +433,6 @@ func doPublisherUI() error {
 		return err
 	}
 
-	// gopherjs has to run before doEmbed since we need all the javascript
-	// to be generated before embedding happens.
 	return genPublisherJS()
 }
 
@@ -509,15 +493,6 @@ Env:
 	return
 }
 
-func stringListContains(strs []string, str string) bool {
-	for _, s := range strs {
-		if s == str {
-			return true
-		}
-	}
-	return false
-}
-
 // fullSrcPath returns the full path concatenation
 // of pkRoot with fromSrc.
 func fullSrcPath(fromSrc string) string {
@@ -718,147 +693,6 @@ func verifyGoVersion() {
 
 }
 
-func doEmbed() {
-	if *verbose {
-		log.Printf("Embedding resources...")
-	}
-	closureEmbed := fullSrcPath("server/perkeepd/ui/closure/z_data.go")
-	closureSrcDir := filepath.Join(pkRoot, filepath.FromSlash("clients/web/embed/closure/lib"))
-	err := embedClosure(closureSrcDir, closureEmbed)
-	if err != nil {
-		log.Fatal(err)
-	}
-}
-
-func embedClosure(closureDir, embedFile string) error {
-	if _, err := os.Stat(closureDir); err != nil {
-		return fmt.Errorf("Could not stat %v: %v", closureDir, err)
-	}
-
-	// first collect the files and modTime
-	var modTime time.Time
-	type pathAndSuffix struct {
-		path, suffix string
-	}
-	var files []pathAndSuffix
-	err := filepath.Walk(closureDir, func(path string, fi os.FileInfo, err error) error {
-		if err != nil {
-			return err
-		}
-		suffix, err := filepath.Rel(closureDir, path)
-		if err != nil {
-			return fmt.Errorf("Failed to find Rel(%q, %q): %v", closureDir, path, err)
-		}
-		if fi.IsDir() {
-			return nil
-		}
-		if mt := fi.ModTime(); mt.After(modTime) {
-			modTime = mt
-		}
-		files = append(files, pathAndSuffix{path, suffix})
-		return nil
-	})
-	if err != nil {
-		return err
-	}
-	// do not regenerate the whole embedFile if it exists and newer than modTime.
-	if fi, err := os.Stat(embedFile); err == nil && fi.Size() > 0 && fi.ModTime().After(modTime) {
-		if *verbose {
-			log.Printf("skipping regeneration of %s", embedFile)
-		}
-		return nil
-	}
-
-	// second, zip it
-	var zipbuf bytes.Buffer
-	var zipdest io.Writer = &zipbuf
-	if os.Getenv("CAMLI_WRITE_TMP_ZIP") != "" {
-		f, _ := os.Create("/tmp/camli-closure.zip")
-		zipdest = io.MultiWriter(zipdest, f)
-		defer f.Close()
-	}
-	w := zip.NewWriter(zipdest)
-	for _, elt := range files {
-		b, err := os.ReadFile(elt.path)
-		if err != nil {
-			return err
-		}
-		f, err := w.Create(filepath.ToSlash(elt.suffix))
-		if err != nil {
-			return err
-		}
-		if _, err = f.Write(b); err != nil {
-			return err
-		}
-	}
-	if err = w.Close(); err != nil {
-		return err
-	}
-
-	// then embed it as a quoted string
-	var qb bytes.Buffer
-	fmt.Fprint(&qb, "//go:build with_embed\n\n")
-	fmt.Fprint(&qb, "package closure\n\n")
-	fmt.Fprint(&qb, "import \"time\"\n\n")
-	fmt.Fprint(&qb, "func init() {\n")
-	fmt.Fprintf(&qb, "\tZipModTime = time.Unix(%d, 0)\n", modTime.Unix())
-	fmt.Fprint(&qb, "\tZipData = ")
-	quote(&qb, zipbuf.Bytes())
-	fmt.Fprint(&qb, "\n}\n")
-
-	// and write to a .go file
-	if err := writeFileIfDifferent(embedFile, qb.Bytes()); err != nil {
-		return err
-	}
-	return nil
-
-}
-
-func writeFileIfDifferent(filename string, contents []byte) error {
-	fi, err := os.Stat(filename)
-	if err == nil && fi.Size() == int64(len(contents)) && contentsEqual(filename, contents) {
-		return nil
-	}
-	return os.WriteFile(filename, contents, 0644)
-}
-
-func contentsEqual(filename string, contents []byte) bool {
-	got, err := os.ReadFile(filename)
-	if os.IsNotExist(err) {
-		return false
-	}
-	if err != nil {
-		log.Fatalf("Error reading %v: %v", filename, err)
-	}
-	return bytes.Equal(got, contents)
-}
-
-// quote escapes and quotes the bytes from bs and writes
-// them to dest.
-func quote(dest *bytes.Buffer, bs []byte) {
-	dest.WriteByte('"')
-	for _, b := range bs {
-		if b == '\n' {
-			dest.WriteString(`\n`)
-			continue
-		}
-		if b == '\\' {
-			dest.WriteString(`\\`)
-			continue
-		}
-		if b == '"' {
-			dest.WriteString(`\"`)
-			continue
-		}
-		if (b >= 32 && b <= 126) || b == '\t' {
-			dest.WriteByte(b)
-			continue
-		}
-		fmt.Fprintf(dest, "\\x%02x", b)
-	}
-	dest.WriteByte('"')
-}
-
 // hostExeName returns the executable name
 // for s on the currently running host OS.
 func hostExeName(s string) string {

pkg/server/ui.go

@@ -32,6 +32,7 @@ import (
 	"strings"
 	"time"
 
+	closurestatic "perkeep.org/clients/web/embed/closure/lib"
 	fontawesomestatic "perkeep.org/clients/web/embed/fontawesome"
 	keepystatic "perkeep.org/clients/web/embed/keepy"
 	leafletstatic "perkeep.org/clients/web/embed/leaflet"
@@ -53,7 +54,6 @@ import (
 	"perkeep.org/pkg/sorted"
 	"perkeep.org/pkg/types/camtypes"
 	uistatic "perkeep.org/server/perkeepd/ui"
-	closurestatic "perkeep.org/server/perkeepd/ui/closure"
 	"rsc.io/qr"
 )
 
@@ -62,7 +62,7 @@ var (
 	identOrDotPattern  = regexp.MustCompile(`^[a-zA-Z\_]+(\.[a-zA-Z\_]+)*$`)
 	thumbnailPattern   = regexp.MustCompile(`^thumbnail/([^/]+)(/.*)?$`)
 	treePattern        = regexp.MustCompile(`^tree/([^/]+)(/.*)?$`)
-	closurePattern     = regexp.MustCompile(`^closure/(([^/]+)(/.*)?)$`)
+	closurePattern     = regexp.MustCompile(`^(closure/([^/]+)(/.*)?)$`)
 	lessPattern        = regexp.MustCompile(`^less/(.+)$`)
 	reactPattern       = regexp.MustCompile(`^react/(.+)$`)
 	leafletPattern     = regexp.MustCompile(`^leaflet/(.+)$`)
@@ -344,16 +344,9 @@ func makeClosureHandler(root, handlerName string) (http.Handler, error) {
 		return http.FileServer(http.Dir(d)), nil
 	}
 	if root == "" {
-		fs, err := closurestatic.FileSystem()
-		if err == os.ErrNotExist {
-			log.Printf("%v: no configured setting or embedded resources; serving Closure via %v", handlerName, closureBaseURL)
-			return closureBaseURL, nil
-		}
-		if err != nil {
-			return nil, fmt.Errorf("error loading embedded Closure zip file: %v", err)
-		}
+		fs := closurestatic.Closure
 		log.Printf("%v: serving Closure from embedded resources", handlerName)
-		return http.FileServer(fs), nil
+		return http.FileServer(http.FS(fs)), nil
 	}
 	if strings.HasPrefix(root, "http") {
 		log.Printf("%v: serving Closure using redirects to %v", handlerName, root)
@@ -380,8 +373,6 @@ func makeFileServer(sourceRoot string, pathToServe string, expectedContentPath s
 	return http.FileServer(http.Dir(dirToServe)), nil
 }
 
-const closureBaseURL closureRedirector = "https://closure-library.googlecode.com/git"
-
 // closureRedirector is a hack to redirect requests for Closure's million *.js files
 // to https://closure-library.googlecode.com/git.
 // TODO: this doesn't work when offline. We need to run genjsdeps over all of the Perkeep

pkg/test/world.go

@@ -108,7 +108,6 @@ func (w *World) Build() error {
 	// Build.
 	{
 		cmd := exec.Command("go", "run", "make.go",
-			"--embed_static=false",
 			"--stampversion=false",
 			"--buildPublisherUI=false",
 			"--targets="+strings.Join([]string{

server/perkeepd/ui/closure/closure.go

@@ -1,142 +0,0 @@
-/*
-Copyright 2013 The Perkeep Authors
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-	http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package closure // import "perkeep.org/server/perkeepd/ui/closure"
-
-import (
-	"archive/zip"
-	"bytes"
-	"errors"
-	"fmt"
-	"io"
-	"net/http"
-	"os"
-	"path"
-	"strings"
-	"sync"
-	"time"
-)
-
-// ZipData is either the empty string (when compiling with "go get",
-// or the devcam server), or is initialized to a base64-encoded zip file
-// of the Closure library (when using make.go, which puts an extra
-// file in this package containing an init function to set ZipData).
-var ZipData string
-var ZipModTime time.Time
-
-func FileSystem() (http.FileSystem, error) {
-	if ZipData == "" {
-		return nil, os.ErrNotExist
-	}
-	zr, err := zip.NewReader(strings.NewReader(ZipData), int64(len(ZipData)))
-	if err != nil {
-		return nil, err
-	}
-	m := make(map[string]*fileInfo)
-	for _, zf := range zr.File {
-		if !strings.HasPrefix(zf.Name, "closure/") {
-			continue
-		}
-		fi, err := newFileInfo(zf)
-		if err != nil {
-			return nil, fmt.Errorf("Error reading zip file %q: %v", zf.Name, err)
-		}
-		m[strings.TrimPrefix(zf.Name, "closure")] = fi
-	}
-	return &fs{zr, m}, nil
-
-}
-
-type fs struct {
-	zr *zip.Reader
-	m  map[string]*fileInfo // keyed by what Open gets. see Open's comment.
-}
-
-var nopCloser = io.NopCloser(nil)
-
-// Open is called with names like "/goog/base.js", but the zip contains Files named like "closure/goog/base.js".
-func (s *fs) Open(name string) (http.File, error) {
-	fi, ok := s.m[name]
-	if !ok {
-		return nil, os.ErrNotExist
-	}
-	return &file{fileInfo: fi}, nil
-}
-
-// a file is an http.File, wrapping a *fileInfo with a lazily-constructed SectionReader.
-type file struct {
-	*fileInfo
-	once sync.Once // for making the SectionReader
-	sr   *io.SectionReader
-}
-
-func (f *file) Read(p []byte) (n int, err error) {
-	f.once.Do(f.initReader)
-	return f.sr.Read(p)
-}
-
-func (f *file) Seek(offset int64, whence int) (ret int64, err error) {
-	f.once.Do(f.initReader)
-	return f.sr.Seek(offset, whence)
-}
-
-func (f *file) initReader() {
-	f.sr = io.NewSectionReader(f.fileInfo.ra, 0, f.Size())
-}
-
-func newFileInfo(zf *zip.File) (*fileInfo, error) {
-	rc, err := zf.Open()
-	if err != nil {
-		return nil, err
-	}
-	all, err := io.ReadAll(rc)
-	if err != nil {
-		return nil, err
-	}
-	rc.Close()
-	return &fileInfo{
-		fullName: zf.Name,
-		regdata:  all,
-		Closer:   nopCloser,
-		ra:       bytes.NewReader(all),
-	}, nil
-}
-
-type fileInfo struct {
-	fullName string
-	regdata  []byte      // non-nil if regular file
-	ra       io.ReaderAt // over regdata
-	io.Closer
-}
-
-func (f *fileInfo) IsDir() bool                { return f.regdata == nil }
-func (f *fileInfo) Size() int64                { return int64(len(f.regdata)) }
-func (f *fileInfo) ModTime() time.Time         { return ZipModTime }
-func (f *fileInfo) Name() string               { return path.Base(f.fullName) }
-func (f *fileInfo) Stat() (os.FileInfo, error) { return f, nil }
-func (f *fileInfo) Sys() interface{}           { return nil }
-
-func (f *fileInfo) Readdir(count int) ([]os.FileInfo, error) {
-	// TODO: implement.
-	return nil, errors.New("TODO")
-}
-
-func (f *fileInfo) Mode() os.FileMode {
-	if f.IsDir() {
-		return 0755 | os.ModeDir
-	}
-	return 0644
-}

website/scripts/run-blobserver

@@ -8,7 +8,6 @@ export CAMLI_ROOT=$Bin/blobserver-example/root
 
 cd $Bin/..
 go run make.go \
-	-embed_static=false \
 	-targets=camlistore.org/server/perkeepd && \
 ./bin/perkeepd \
 	-openbrowser=false \