From d953dee008b76225d8ca4f3f2fac91e7f6b6162c Mon Sep 17 00:00:00 2001 From: mpl Date: Wed, 6 Aug 2014 18:47:42 +0200 Subject: [PATCH] devcam test: do not "recurse" temp GOPATH, docs, couple more options. Problem: make.go creates an isolated temp gopath ./tmp/build-gopath. The integration tests make use of that gopath (by running make.go) to build the tools, and run the test world in it. Similarly, devcam test uses make.go to setup that temp gopath, and runs the tests from the source files in that gopath. Consequently, when the integration tests are run through devcam test, even though they're run from the temp gopath, they would use the make.go in it, which would create a nested temp gopath (CAMLIROOT/tmp/build-gopath/src/camlistore.org/tmp/build-gopath) in which to run the tests. This patch addresses this issue by creating a new flag (-envGoPath), and the corresponding env var (CAMLI_MAKE_USEGOPATH), which tells make.go not to create a new temporary gopath (and hence not to mirror any files), and to rely on the already set GOPATH env var instead. Also refactored make.go a bit, and added a couple options and doc to devcam test. Change-Id: Ia8a5d7a31e6e317f05218d9e18fb886001cd19cb --- dev/devcam/test.go | 17 ++- doc/environment-vars.txt | 3 + make.go | 253 ++++++++++++++++++++++++--------------- pkg/test/world.go | 5 +- 4 files changed, 177 insertions(+), 101 deletions(-) diff --git a/dev/devcam/test.go b/dev/devcam/test.go index 0471898ff..7523e0afa 100644 --- a/dev/devcam/test.go +++ b/dev/devcam/test.go @@ -31,7 +31,9 @@ import ( type testCmd struct { // start of flag vars - short bool + verbose bool + short bool + run string // end of flag vars // buildGoPath becomes our child "go" processes' GOPATH environment variable @@ -42,16 +44,18 @@ func init() { cmdmain.RegisterCommand("test", func(flags *flag.FlagSet) cmdmain.CommandRunner { cmd := new(testCmd) flags.BoolVar(&cmd.short, "short", false, "Use '-short' with go test.") + flags.BoolVar(&cmd.verbose, "v", false, "Use '-v' (for verbose) with go test.") + flags.StringVar(&cmd.run, "run", "", "Use '-run' with go test.") return cmd }) } func (c *testCmd) Usage() { - fmt.Fprintf(cmdmain.Stderr, "Usage: devcam test\n") + fmt.Fprintf(cmdmain.Stderr, "Usage: devcam test [test_opts] [targets]\n") } func (c *testCmd) Describe() string { - return "run the full test suite." + return "run the full test suite, or the tests in the specified target packages." } func (c *testCmd) RunCommand(args []string) error { @@ -79,6 +83,7 @@ func (c *testCmd) env() *Env { env := NewCopyEnv() env.NoGo() env.Set("GOPATH", c.buildGoPath) + env.Set("CAMLI_MAKE_USEGOPATH", "true") return env } @@ -123,6 +128,12 @@ func (c *testCmd) runTests(args []string) error { if c.short { targs = append(targs, "-short") } + if c.verbose { + targs = append(targs, "-v") + } + if c.run != "" { + targs = append(targs, "-run="+c.run) + } if len(args) > 0 { targs = append(targs, args...) } else { diff --git a/doc/environment-vars.txt b/doc/environment-vars.txt index bbecc07f7..e6888a8b0 100644 --- a/doc/environment-vars.txt +++ b/doc/environment-vars.txt @@ -98,6 +98,9 @@ CAMLI_KV_VERIFY (bool): CAMLI_MONGO_WIPE (bool): Wipe out mongo based index on startup. +CAMLI_MAKE_USEGOPATH (bool): + When running make.go, overrides the -use_gopath flag. + CAMLI_NO_FILE_DUP_SEARCH (bool): This will cause the search-for-exists-before-upload step to be skipped when camput is uploading files. diff --git a/make.go b/make.go index 87ae1653f..719c18af2 100644 --- a/make.go +++ b/make.go @@ -49,20 +49,23 @@ var haveSQLite = checkHaveSQLite() var ( embedResources = flag.Bool("embed_static", true, "Whether to embed resources needed by the UI such as images, css, and javascript.") - sqlFlag = flag.String("sqlite", "auto", "Whether you want SQLite in your build: yes, no, or auto.") + sqlFlag = flag.String("sqlite", "auto", "Whether you want SQLite in your build: true, false, or auto.") all = flag.Bool("all", false, "Force rebuild of everything (go install -a)") race = flag.Bool("race", false, "Build race-detector version of binaries (they will run slowly)") verbose = flag.Bool("v", false, "Verbose mode") targets = flag.String("targets", "", "Optional comma-separated list of targets (i.e go packages) to build and install. '*' builds everything. Empty builds defaults for this platform. Example: camlistore.org/server/camlistored,camlistore.org/cmd/camput") quiet = flag.Bool("quiet", false, "Don't print anything unless there's a failure.") onlysync = flag.Bool("onlysync", false, "Only populate the temporary source/build tree and output its full path. It is meant to prepare the environment for running the full test suite with 'devcam test'.") - // TODO(mpl): looks like ifModsSince is not used anywhere? - ifModsSince = flag.Int64("if_mods_since", 0, "If non-zero return immediately without building if there aren't any filesystem modifications past this time (in unix seconds)") + useGoPath = flag.Bool("use_gopath", false, "Use GOPATH from the environment and work from there. Do not create a temporary source tree with a new GOPATH in it.") + // TODO(mpl): implement ifModsSince, or remove it from all places where we use make.go, because we shouldn't lie. + ifModsSince = flag.Int64("if_mods_since", 0, "Not implemented yet. If non-zero return immediately without building if there aren't any filesystem modifications past this time (in unix seconds)") buildARCH = flag.String("arch", runtime.GOARCH, "Architecture to build for.") buildOS = flag.String("os", runtime.GOOS, "Operating system to build for.") ) var ( + // camRoot is the original Camlistore project root, from where the source files are mirrored. + camRoot string // buildGoPath becomes our child "go" processes' GOPATH environment variable buildGoPath string // Our temporary source tree root and build dir, i.e: buildGoPath + "src/camlistore.org" @@ -75,53 +78,36 @@ func main() { log.SetFlags(0) flag.Parse() - camRoot, err := os.Getwd() - if err != nil { - log.Fatalf("Failed to get current directory: %v", err) - } - verifyCamlistoreRoot(camRoot) + verifyGoVersion() - cross := runtime.GOOS != *buildOS || runtime.GOARCH != *buildARCH - var sql bool - if *sqlFlag == "auto" { - sql = !cross && haveSQLite - } else { - sql, err = strconv.ParseBool(*sqlFlag) + sql := withSQLite() + if useEnvGoPath, _ := strconv.ParseBool(os.Getenv("CAMLI_MAKE_USEGOPATH")); useEnvGoPath { + *useGoPath = true + } + if *useGoPath { + buildGoPath = os.Getenv("GOPATH") + var err error + camRoot, err = goPackagePath("camlistore.org") if err != nil { - log.Fatalf("Bad boolean --sql flag %q", *sqlFlag) + log.Fatalf("Cannot run make.go with --use_gopath: %v (is GOPATH not set?)", err) + } + buildSrcDir = camRoot + } else { + var err error + camRoot, err = os.Getwd() + if err != nil { + log.Fatalf("Failed to get current directory: %v", err) + } + mirror(sql) + if *onlysync { + mirrorFile("make.go", filepath.Join(buildSrcDir, "make.go")) + deleteUnwantedOldMirrorFiles(buildSrcDir, true) + fmt.Println(buildGoPath) + return } } - - if cross && sql { - log.Fatalf("SQLite isn't available when cross-compiling to another OS. Set --sqlite=false.") - } - if sql && !haveSQLite { - log.Printf("SQLite not found. Either install it, or run make.go with --sqlite=false See https://code.google.com/p/camlistore/wiki/SQLite") - switch runtime.GOOS { - case "darwin": - log.Printf("On OS X, run 'brew install sqlite3 pkg-config'. Get brew from http://mxcl.github.io/homebrew/") - case "linux": - log.Printf("On Linux, run 'sudo apt-get install libsqlite3-dev' or equivalent.") - case "windows": - log.Printf("SQLite is not easy on windows. Please see http://camlistore.org/docs/server-config#windows") - } - os.Exit(2) - } - - buildBaseDir := "build-gopath" - if !sql { - buildBaseDir += "-nosqlite" - } - - buildGoPath = filepath.Join(camRoot, "tmp", buildBaseDir) binDir := filepath.Join(camRoot, "bin") - buildSrcDir = filepath.Join(buildGoPath, "src", "camlistore.org") - - if err := os.MkdirAll(buildSrcDir, 0755); err != nil { - log.Fatal(err) - } - - version := getVersion(camRoot) + version := getVersion() if *verbose { log.Printf("Camlistore version = %s", version) @@ -130,37 +116,6 @@ func main() { log.Printf("Output binaries: %s", binDir) } - // TODO(mpl): main is getting long. We could probably move all the mirroring - // dance to its own func. - // We copy all *.go files from camRoot's goDirs to buildSrcDir. - goDirs := []string{"app", "cmd", "pkg", "dev", "server/camlistored", "third_party"} - if *onlysync { - goDirs = append(goDirs, "server/appengine", "config") - } - // Copy files we do want in our mirrored GOPATH. This has the side effect of - // populating wantDestFile, populated by mirrorFile. - var latestSrcMod time.Time - for _, dir := range goDirs { - oriPath := filepath.Join(camRoot, filepath.FromSlash(dir)) - dstPath := buildSrcPath(dir) - if maxMod, err := mirrorDir(oriPath, dstPath, mirrorOpts{sqlite: sql}); err != nil { - log.Fatalf("Error while mirroring %s to %s: %v", oriPath, dstPath, err) - } else { - if maxMod.After(latestSrcMod) { - latestSrcMod = maxMod - } - } - } - - verifyGoVersion() - - if *onlysync { - mirrorFile("make.go", filepath.Join(buildSrcDir, "make.go")) - deleteUnwantedOldMirrorFiles(buildSrcDir, true) - fmt.Println(buildGoPath) - return - } - buildAll := false targs := []string{ "camlistore.org/dev/devcam", @@ -188,25 +143,15 @@ func main() { withCamlistored := stringListContains(targs, "camlistore.org/server/camlistored") if *embedResources && withCamlistored { - if *verbose { - log.Printf("Embedding resources...") - } - closureEmbed := buildSrcPath("server/camlistored/ui/closure/z_data.go") - closureSrcDir := filepath.Join(camRoot, filepath.FromSlash("third_party/closure/lib")) - err := embedClosure(closureSrcDir, closureEmbed) - if err != nil { - log.Fatal(err) - } - wantDestFile[closureEmbed] = true - if err = buildGenfileembed(); err != nil { - log.Fatal(err) - } - if err = genEmbeds(); err != nil { - log.Fatal(err) - } + // TODO(mpl): in doEmbed, it looks like we not only always recreate the closure + // z_data.go, but we also always regenerate the zembed.*.go, at least for the + // integration tests. I'll look into it. + doEmbed() } - deleteUnwantedOldMirrorFiles(buildSrcDir, withCamlistored) + if !*useGoPath { + deleteUnwantedOldMirrorFiles(buildSrcDir, withCamlistored) + } tags := "" if sql { @@ -273,6 +218,43 @@ func main() { } } +// create the tmp GOPATH, and mirror to it from camRoot. +func mirror(sql bool) { + verifyCamlistoreRoot(camRoot) + + buildBaseDir := "build-gopath" + if !sql { + buildBaseDir += "-nosqlite" + } + + buildGoPath = filepath.Join(camRoot, "tmp", buildBaseDir) + buildSrcDir = filepath.Join(buildGoPath, "src", "camlistore.org") + + if err := os.MkdirAll(buildSrcDir, 0755); err != nil { + log.Fatal(err) + } + + // We copy all *.go files from camRoot's goDirs to buildSrcDir. + goDirs := []string{"app", "cmd", "pkg", "dev", "server/camlistored", "third_party"} + if *onlysync { + goDirs = append(goDirs, "server/appengine", "config") + } + // Copy files we do want in our mirrored GOPATH. This has the side effect of + // populating wantDestFile, populated by mirrorFile. + var latestSrcMod time.Time + for _, dir := range goDirs { + srcPath := filepath.Join(camRoot, filepath.FromSlash(dir)) + dstPath := buildSrcPath(dir) + if maxMod, err := mirrorDir(srcPath, dstPath, mirrorOpts{sqlite: sql}); err != nil { + log.Fatalf("Error while mirroring %s to %s: %v", srcPath, dstPath, err) + } else { + if maxMod.After(latestSrcMod) { + latestSrcMod = maxMod + } + } + } +} + func actualBinDir(dir string) string { if *buildARCH == runtime.GOARCH && *buildOS == runtime.GOOS { return dir @@ -410,12 +392,12 @@ func buildGenfileembed() error { // getVersion returns the version of Camlistore. Either from a VERSION file at the root, // or from git. -func getVersion(camRoot string) string { +func getVersion() string { slurp, err := ioutil.ReadFile(filepath.Join(camRoot, "VERSION")) if err == nil { return strings.TrimSpace(string(slurp)) } - return gitVersion(camRoot) + return gitVersion() } var gitVersionRx = regexp.MustCompile(`\b\d\d\d\d-\d\d-\d\d-[0-9a-f]{7,7}\b`) @@ -423,7 +405,7 @@ var gitVersionRx = regexp.MustCompile(`\b\d\d\d\d-\d\d-\d\d-[0-9a-f]{7,7}\b`) // gitVersion returns the git version of the git repo at camRoot as a // string of the form "yyyy-mm-dd-xxxxxxx", with an optional trailing // '+' if there are any local uncomitted modifications to the tree. -func gitVersion(camRoot string) string { +func gitVersion() string { cmd := exec.Command("git", "rev-list", "--max-count=1", "--pretty=format:'%ad-%h'", "--date=short", "HEAD") cmd.Dir = camRoot out, err := cmd.Output() @@ -576,7 +558,7 @@ func deleteUnwantedOldMirrorFiles(dir string, withCamlistored bool) { // have to put it back into place later. return nil } - if !*quiet { + if *verbose { log.Printf("Deleting old file from temp build dir: %s", path) } return os.Remove(path) @@ -585,6 +567,37 @@ func deleteUnwantedOldMirrorFiles(dir string, withCamlistored bool) { }) } +func withSQLite() bool { + cross := runtime.GOOS != *buildOS || runtime.GOARCH != *buildARCH + var sql bool + var err error + if *sqlFlag == "auto" { + sql = !cross && haveSQLite + } else { + sql, err = strconv.ParseBool(*sqlFlag) + if err != nil { + log.Fatalf("Bad boolean --sql flag %q", *sqlFlag) + } + } + + if cross && sql { + log.Fatalf("SQLite isn't available when cross-compiling to another OS. Set --sqlite=false.") + } + if sql && !haveSQLite { + log.Printf("SQLite not found. Either install it, or run make.go with --sqlite=false See https://code.google.com/p/camlistore/wiki/SQLite") + switch runtime.GOOS { + case "darwin": + log.Printf("On OS X, run 'brew install sqlite3 pkg-config'. Get brew from http://mxcl.github.io/homebrew/") + case "linux": + log.Printf("On Linux, run 'sudo apt-get install libsqlite3-dev' or equivalent.") + case "windows": + log.Printf("SQLite is not easy on windows. Please see http://camlistore.org/docs/server-config#windows") + } + os.Exit(2) + } + return sql +} + func checkHaveSQLite() bool { if runtime.GOOS == "windows" { // TODO: Find some other non-pkg-config way to test, like @@ -612,6 +625,25 @@ func checkHaveSQLite() bool { return strings.TrimSpace(string(out)) != "" } +func doEmbed() { + if *verbose { + log.Printf("Embedding resources...") + } + closureEmbed := buildSrcPath("server/camlistored/ui/closure/z_data.go") + closureSrcDir := filepath.Join(camRoot, filepath.FromSlash("third_party/closure/lib")) + err := embedClosure(closureSrcDir, closureEmbed) + if err != nil { + log.Fatal(err) + } + wantDestFile[closureEmbed] = true + if err = buildGenfileembed(); err != nil { + log.Fatal(err) + } + if err = genEmbeds(); err != nil { + log.Fatal(err) + } +} + func embedClosure(closureDir, embedFile string) error { if _, err := os.Stat(closureDir); err != nil { return fmt.Errorf("Could not stat %v: %v", closureDir, err) @@ -728,3 +760,30 @@ func exeName(s string) string { } return s } + +// goPackagePath returns the path to the provided Go package's +// source directory. +// pkg may be a path prefix without any *.go files. +// The error is os.ErrNotExist if GOPATH is unset or the directory +// doesn't exist in any GOPATH component. +func goPackagePath(pkg string) (path string, err error) { + gp := os.Getenv("GOPATH") + if gp == "" { + return path, os.ErrNotExist + } + for _, p := range filepath.SplitList(gp) { + dir := filepath.Join(p, "src", filepath.FromSlash(pkg)) + fi, err := os.Stat(dir) + if os.IsNotExist(err) { + continue + } + if err != nil { + return "", err + } + if !fi.IsDir() { + continue + } + return dir, nil + } + return path, os.ErrNotExist +} diff --git a/pkg/test/world.go b/pkg/test/world.go index e5cac5f47..3373b1269 100644 --- a/pkg/test/world.go +++ b/pkg/test/world.go @@ -94,6 +94,7 @@ func (w *World) Start() error { // Build. { + // TODO(mpl): when running with -v (either with go test or devcam test), append it for make.go as well cmd := exec.Command("go", "run", "make.go") cmd.Dir = w.camRoot log.Print("Running make.go to build camlistore binaries for testing...") @@ -184,7 +185,9 @@ func (w *World) Stop() { if w == nil { return } - w.server.Process.Kill() + if err := w.server.Process.Kill(); err != nil { + log.Fatalf("killed failed: %v", err) + } if d := w.tempDir; d != "" { os.RemoveAll(d)