make.go, genfileembed: track genfileembed output files and delete stale ones

Don't just trust all zembed_*.go files to be wanted. Instead, we now
ask genfileembed to write out its output files to stderr, parse that,
and add those to our set of wanted files. Then when we look for
unwanted files that aren't known outputs that now also potentially
includes zembed_*.go files.

This fixes a JavaScript breakage from when Aaron renamed some
JavaScript files recently.

Turns out I had 4 such files that were stale outputs.

In other news, I hereby declare a moratorium on updates to make.go.
It has become disgusting and is need of cleanup love before it is
disgraced further.

Change-Id: Ia1432d21ce486bfbca9e3c9db6f1852b859ebfc0

make.go

@@ -28,6 +28,7 @@ package main
 
 import (
 	"archive/zip"
+	"bufio"
 	"bytes"
 	"flag"
 	"fmt"
@@ -390,33 +391,41 @@ func genEmbeds() error {
 	cmdName := exeName(filepath.Join(buildGoPath, "bin", "genfileembed"))
 	for _, embeds := range []string{"server/camlistored/ui", "pkg/server", "third_party/react", "third_party/glitch", "third_party/fontawesome", "app/publisher"} {
 		embeds := buildSrcPath(embeds)
-		args := []string{embeds}
+		args := []string{"--output-files-stderr", embeds}
 		cmd := exec.Command(cmdName, args...)
 		cmd.Env = append(cleanGoEnv(),
 			"GOPATH="+buildGoPath,
 		)
 		cmd.Stdout = os.Stdout
-		cmd.Stderr = os.Stderr
+		stderr, err := cmd.StderrPipe()
+		if err != nil {
+			log.Fatal(err)
+		}
 		if *verbose {
 			log.Printf("Running %s %s", cmdName, embeds)
 		}
-		if err := cmd.Run(); err != nil {
+		if err := cmd.Start(); err != nil {
-			return fmt.Errorf("Error running %s %s: %v", cmdName, embeds, err)
+			return fmt.Errorf("Error starting %s %s: %v", cmdName, embeds, err)
 		}
-		// We mark all the zembeds in builddir as wanted, so that we do not
+		parseGenEmbedOutputLines(stderr)
-		// have to regen them next time, unless they need updating.
+		if err := cmd.Wait(); err != nil {
-		if err := filepath.Walk(embeds, func(path string, _ os.FileInfo, err error) error {
+			return fmt.Errorf("Error running %s %s: %v", cmdName, embeds, err)
-			if strings.HasPrefix(filepath.Base(path), "zembed_") {
-				wantDestFile[path] = true
-			}
-			return err
-		}); err != nil {
-			return err
 		}
 	}
 	return nil
 }
 
+func parseGenEmbedOutputLines(r io.Reader) {
+	sc := bufio.NewScanner(r)
+	for sc.Scan() {
+		ln := sc.Text()
+		if !strings.HasPrefix(ln, "OUTPUT:") {
+			continue
+		}
+		wantDestFile[strings.TrimSpace(strings.TrimPrefix(ln, "OUTPUT:"))] = true
+	}
+}
+
 func buildGenfileembed() error {
 	args := []string{"install", "-v"}
 	if *all {
@@ -630,8 +639,9 @@ func deleteUnwantedOldMirrorFiles(dir string, withCamlistored bool) {
 		if fi.IsDir() {
 			return nil
 		}
+		base := filepath.Base(path)
 		if !wantDestFile[path] {
-			if !withCamlistored && (strings.Contains(path, "zembed_") || strings.Contains(path, "z_data.go")) {
+			if !withCamlistored && (strings.HasPrefix(base, "zembed_") || strings.Contains(path, "z_data.go")) {
 				// If we're not building the camlistored binary,
 				// no need to clean up the embedded Closure, JS,
 				// CSS, HTML, etc. Doing so would just mean we'd

pkg/fileembed/genfileembed/genfileembed.go

@@ -47,6 +47,8 @@ var (
 
 	chunkThreshold = flag.Int64("chunk-threshold", 0, "If non-zero, the maximum size of a file before it's cut up into content-addressable chunks with a rolling checksum")
 	chunkPackage   = flag.String("chunk-package", "", "Package to hold chunks")
+
+	destFilesStderr = flag.Bool("output-files-stderr", false, "Write the absolute path of all output files to stderr prefixed with OUTPUT:")
 )
 
 const (
@@ -66,6 +68,10 @@ func main() {
 	flag.Usage = usage
 	flag.Parse()
 
+	absPath, err := os.Getwd() // absolute path of output directory
+	if err != nil {
+		log.Fatal(err)
+	}
 	dir := "."
 	switch flag.NArg() {
 	case 0:
@@ -74,6 +80,11 @@ func main() {
 		if err := os.Chdir(dir); err != nil {
 			log.Fatalf("chdir(%q) = %v", dir, err)
 		}
+		if filepath.IsAbs(dir) {
+			absPath = dir
+		} else {
+			absPath = filepath.Join(absPath, dir)
+		}
 	default:
 		flag.Usage()
 	}
@@ -90,6 +101,9 @@ func main() {
 		}
 
 		embedName := "zembed_" + strings.Replace(fileName, string(filepath.Separator), "_", -1) + ".go"
+		if *destFilesStderr {
+			fmt.Fprintf(os.Stderr, "OUTPUT:%s\n", filepath.Join(absPath, embedName))
+		}
 		zfi, zerr := os.Stat(embedName)
 		genFile := func() bool {
 			if *processAll || zerr != nil {