diff --git a/dev/devcam/test.go b/dev/devcam/test.go index 0471898ff..7523e0afa 100644 --- a/dev/devcam/test.go +++ b/dev/devcam/test.go @@ -31,7 +31,9 @@ import ( type testCmd struct { // start of flag vars - short bool + verbose bool + short bool + run string // end of flag vars // buildGoPath becomes our child "go" processes' GOPATH environment variable @@ -42,16 +44,18 @@ func init() { cmdmain.RegisterCommand("test", func(flags *flag.FlagSet) cmdmain.CommandRunner { cmd := new(testCmd) flags.BoolVar(&cmd.short, "short", false, "Use '-short' with go test.") + flags.BoolVar(&cmd.verbose, "v", false, "Use '-v' (for verbose) with go test.") + flags.StringVar(&cmd.run, "run", "", "Use '-run' with go test.") return cmd }) } func (c *testCmd) Usage() { - fmt.Fprintf(cmdmain.Stderr, "Usage: devcam test\n") + fmt.Fprintf(cmdmain.Stderr, "Usage: devcam test [test_opts] [targets]\n") } func (c *testCmd) Describe() string { - return "run the full test suite." + return "run the full test suite, or the tests in the specified target packages." } func (c *testCmd) RunCommand(args []string) error { @@ -79,6 +83,7 @@ func (c *testCmd) env() *Env { env := NewCopyEnv() env.NoGo() env.Set("GOPATH", c.buildGoPath) + env.Set("CAMLI_MAKE_USEGOPATH", "true") return env } @@ -123,6 +128,12 @@ func (c *testCmd) runTests(args []string) error { if c.short { targs = append(targs, "-short") } + if c.verbose { + targs = append(targs, "-v") + } + if c.run != "" { + targs = append(targs, "-run="+c.run) + } if len(args) > 0 { targs = append(targs, args...) } else { diff --git a/doc/environment-vars.txt b/doc/environment-vars.txt index bbecc07f7..e6888a8b0 100644 --- a/doc/environment-vars.txt +++ b/doc/environment-vars.txt @@ -98,6 +98,9 @@ CAMLI_KV_VERIFY (bool): CAMLI_MONGO_WIPE (bool): Wipe out mongo based index on startup. +CAMLI_MAKE_USEGOPATH (bool): + When running make.go, overrides the -use_gopath flag. + CAMLI_NO_FILE_DUP_SEARCH (bool): This will cause the search-for-exists-before-upload step to be skipped when camput is uploading files. diff --git a/make.go b/make.go index bec6aac24..951dfa5de 100644 --- a/make.go +++ b/make.go @@ -49,20 +49,23 @@ var haveSQLite = checkHaveSQLite() var ( embedResources = flag.Bool("embed_static", true, "Whether to embed resources needed by the UI such as images, css, and javascript.") - sqlFlag = flag.String("sqlite", "auto", "Whether you want SQLite in your build: yes, no, or auto.") + sqlFlag = flag.String("sqlite", "auto", "Whether you want SQLite in your build: true, false, or auto.") all = flag.Bool("all", false, "Force rebuild of everything (go install -a)") race = flag.Bool("race", false, "Build race-detector version of binaries (they will run slowly)") verbose = flag.Bool("v", false, "Verbose mode") targets = flag.String("targets", "", "Optional comma-separated list of targets (i.e go packages) to build and install. '*' builds everything. Empty builds defaults for this platform. Example: camlistore.org/server/camlistored,camlistore.org/cmd/camput") quiet = flag.Bool("quiet", false, "Don't print anything unless there's a failure.") onlysync = flag.Bool("onlysync", false, "Only populate the temporary source/build tree and output its full path. It is meant to prepare the environment for running the full test suite with 'devcam test'.") - // TODO(mpl): looks like ifModsSince is not used anywhere? - ifModsSince = flag.Int64("if_mods_since", 0, "If non-zero return immediately without building if there aren't any filesystem modifications past this time (in unix seconds)") + useGoPath = flag.Bool("use_gopath", false, "Use GOPATH from the environment and work from there. Do not create a temporary source tree with a new GOPATH in it.") + // TODO(mpl): implement ifModsSince, or remove it from all places where we use make.go, because we shouldn't lie. + ifModsSince = flag.Int64("if_mods_since", 0, "Not implemented yet. If non-zero return immediately without building if there aren't any filesystem modifications past this time (in unix seconds)") buildARCH = flag.String("arch", runtime.GOARCH, "Architecture to build for.") buildOS = flag.String("os", runtime.GOOS, "Operating system to build for.") ) var ( + // camRoot is the original Camlistore project root, from where the source files are mirrored. + camRoot string // buildGoPath becomes our child "go" processes' GOPATH environment variable buildGoPath string // Our temporary source tree root and build dir, i.e: buildGoPath + "src/camlistore.org" @@ -75,53 +78,36 @@ func main() { log.SetFlags(0) flag.Parse() - camRoot, err := os.Getwd() - if err != nil { - log.Fatalf("Failed to get current directory: %v", err) - } - verifyCamlistoreRoot(camRoot) + verifyGoVersion() - cross := runtime.GOOS != *buildOS || runtime.GOARCH != *buildARCH - var sql bool - if *sqlFlag == "auto" { - sql = !cross && haveSQLite - } else { - sql, err = strconv.ParseBool(*sqlFlag) + sql := withSQLite() + if useEnvGoPath, _ := strconv.ParseBool(os.Getenv("CAMLI_MAKE_USEGOPATH")); useEnvGoPath { + *useGoPath = true + } + if *useGoPath { + buildGoPath = os.Getenv("GOPATH") + var err error + camRoot, err = goPackagePath("camlistore.org") if err != nil { - log.Fatalf("Bad boolean --sql flag %q", *sqlFlag) + log.Fatalf("Cannot run make.go with --use_gopath: %v (is GOPATH not set?)", err) + } + buildSrcDir = camRoot + } else { + var err error + camRoot, err = os.Getwd() + if err != nil { + log.Fatalf("Failed to get current directory: %v", err) + } + mirror(sql) + if *onlysync { + mirrorFile("make.go", filepath.Join(buildSrcDir, "make.go")) + deleteUnwantedOldMirrorFiles(buildSrcDir, true) + fmt.Println(buildGoPath) + return } } - - if cross && sql { - log.Fatalf("SQLite isn't available when cross-compiling to another OS. Set --sqlite=false.") - } - if sql && !haveSQLite { - log.Printf("SQLite not found. Either install it, or run make.go with --sqlite=false See https://code.google.com/p/camlistore/wiki/SQLite") - switch runtime.GOOS { - case "darwin": - log.Printf("On OS X, run 'brew install sqlite3 pkg-config'. Get brew from http://mxcl.github.io/homebrew/") - case "linux": - log.Printf("On Linux, run 'sudo apt-get install libsqlite3-dev' or equivalent.") - case "windows": - log.Printf("SQLite is not easy on windows. Please see http://camlistore.org/docs/server-config#windows") - } - os.Exit(2) - } - - buildBaseDir := "build-gopath" - if !sql { - buildBaseDir += "-nosqlite" - } - - buildGoPath = filepath.Join(camRoot, "tmp", buildBaseDir) binDir := filepath.Join(camRoot, "bin") - buildSrcDir = filepath.Join(buildGoPath, "src", "camlistore.org") - - if err := os.MkdirAll(buildSrcDir, 0755); err != nil { - log.Fatal(err) - } - - version := getVersion(camRoot) + version := getVersion() if *verbose { log.Printf("Camlistore version = %s", version) @@ -130,37 +116,6 @@ func main() { log.Printf("Output binaries: %s", binDir) } - // TODO(mpl): main is getting long. We could probably move all the mirroring - // dance to its own func. - // We copy all *.go files from camRoot's goDirs to buildSrcDir. - goDirs := []string{"app", "cmd", "pkg", "dev", "server/camlistored", "third_party"} - if *onlysync { - goDirs = append(goDirs, "server/appengine", "config") - } - // Copy files we do want in our mirrored GOPATH. This has the side effect of - // populating wantDestFile, populated by mirrorFile. - var latestSrcMod time.Time - for _, dir := range goDirs { - oriPath := filepath.Join(camRoot, filepath.FromSlash(dir)) - dstPath := buildSrcPath(dir) - if maxMod, err := mirrorDir(oriPath, dstPath, mirrorOpts{sqlite: sql}); err != nil { - log.Fatalf("Error while mirroring %s to %s: %v", oriPath, dstPath, err) - } else { - if maxMod.After(latestSrcMod) { - latestSrcMod = maxMod - } - } - } - - verifyGoVersion() - - if *onlysync { - mirrorFile("make.go", filepath.Join(buildSrcDir, "make.go")) - deleteUnwantedOldMirrorFiles(buildSrcDir, true) - fmt.Println(buildGoPath) - return - } - buildAll := false targs := []string{ "camlistore.org/dev/devcam", @@ -188,25 +143,15 @@ func main() { withCamlistored := stringListContains(targs, "camlistore.org/server/camlistored") if *embedResources && withCamlistored { - if *verbose { - log.Printf("Embedding resources...") - } - closureEmbed := buildSrcPath("server/camlistored/ui/closure/z_data.go") - closureSrcDir := filepath.Join(camRoot, filepath.FromSlash("third_party/closure/lib")) - err := embedClosure(closureSrcDir, closureEmbed) - if err != nil { - log.Fatal(err) - } - wantDestFile[closureEmbed] = true - if err = buildGenfileembed(); err != nil { - log.Fatal(err) - } - if err = genEmbeds(); err != nil { - log.Fatal(err) - } + // TODO(mpl): in doEmbed, it looks like we not only always recreate the closure + // z_data.go, but we also always regenerate the zembed.*.go, at least for the + // integration tests. I'll look into it. + doEmbed() } - deleteUnwantedOldMirrorFiles(buildSrcDir, withCamlistored) + if !*useGoPath { + deleteUnwantedOldMirrorFiles(buildSrcDir, withCamlistored) + } tags := "" if sql { @@ -273,6 +218,43 @@ func main() { } } +// create the tmp GOPATH, and mirror to it from camRoot. +func mirror(sql bool) { + verifyCamlistoreRoot(camRoot) + + buildBaseDir := "build-gopath" + if !sql { + buildBaseDir += "-nosqlite" + } + + buildGoPath = filepath.Join(camRoot, "tmp", buildBaseDir) + buildSrcDir = filepath.Join(buildGoPath, "src", "camlistore.org") + + if err := os.MkdirAll(buildSrcDir, 0755); err != nil { + log.Fatal(err) + } + + // We copy all *.go files from camRoot's goDirs to buildSrcDir. + goDirs := []string{"app", "cmd", "pkg", "dev", "server/camlistored", "third_party"} + if *onlysync { + goDirs = append(goDirs, "server/appengine", "config") + } + // Copy files we do want in our mirrored GOPATH. This has the side effect of + // populating wantDestFile, populated by mirrorFile. + var latestSrcMod time.Time + for _, dir := range goDirs { + srcPath := filepath.Join(camRoot, filepath.FromSlash(dir)) + dstPath := buildSrcPath(dir) + if maxMod, err := mirrorDir(srcPath, dstPath, mirrorOpts{sqlite: sql}); err != nil { + log.Fatalf("Error while mirroring %s to %s: %v", srcPath, dstPath, err) + } else { + if maxMod.After(latestSrcMod) { + latestSrcMod = maxMod + } + } + } +} + func actualBinDir(dir string) string { if *buildARCH == runtime.GOARCH && *buildOS == runtime.GOOS { return dir @@ -410,12 +392,12 @@ func buildGenfileembed() error { // getVersion returns the version of Camlistore. Either from a VERSION file at the root, // or from git. -func getVersion(camRoot string) string { +func getVersion() string { slurp, err := ioutil.ReadFile(filepath.Join(camRoot, "VERSION")) if err == nil { return strings.TrimSpace(string(slurp)) } - return gitVersion(camRoot) + return gitVersion() } var gitVersionRx = regexp.MustCompile(`\b\d\d\d\d-\d\d-\d\d-[0-9a-f]{7,7}\b`) @@ -423,7 +405,7 @@ var gitVersionRx = regexp.MustCompile(`\b\d\d\d\d-\d\d-\d\d-[0-9a-f]{7,7}\b`) // gitVersion returns the git version of the git repo at camRoot as a // string of the form "yyyy-mm-dd-xxxxxxx", with an optional trailing // '+' if there are any local uncomitted modifications to the tree. -func gitVersion(camRoot string) string { +func gitVersion() string { cmd := exec.Command("git", "rev-list", "--max-count=1", "--pretty=format:'%ad-%h'", "--date=short", "HEAD") cmd.Dir = camRoot out, err := cmd.Output() @@ -585,7 +567,7 @@ func deleteUnwantedOldMirrorFiles(dir string, withCamlistored bool) { // have to put it back into place later. return nil } - if !*quiet { + if *verbose { log.Printf("Deleting old file from temp build dir: %s", path) } return os.Remove(path) @@ -594,6 +576,37 @@ func deleteUnwantedOldMirrorFiles(dir string, withCamlistored bool) { }) } +func withSQLite() bool { + cross := runtime.GOOS != *buildOS || runtime.GOARCH != *buildARCH + var sql bool + var err error + if *sqlFlag == "auto" { + sql = !cross && haveSQLite + } else { + sql, err = strconv.ParseBool(*sqlFlag) + if err != nil { + log.Fatalf("Bad boolean --sql flag %q", *sqlFlag) + } + } + + if cross && sql { + log.Fatalf("SQLite isn't available when cross-compiling to another OS. Set --sqlite=false.") + } + if sql && !haveSQLite { + log.Printf("SQLite not found. Either install it, or run make.go with --sqlite=false See https://code.google.com/p/camlistore/wiki/SQLite") + switch runtime.GOOS { + case "darwin": + log.Printf("On OS X, run 'brew install sqlite3 pkg-config'. Get brew from http://mxcl.github.io/homebrew/") + case "linux": + log.Printf("On Linux, run 'sudo apt-get install libsqlite3-dev' or equivalent.") + case "windows": + log.Printf("SQLite is not easy on windows. Please see http://camlistore.org/docs/server-config#windows") + } + os.Exit(2) + } + return sql +} + func checkHaveSQLite() bool { if runtime.GOOS == "windows" { // TODO: Find some other non-pkg-config way to test, like @@ -621,6 +634,25 @@ func checkHaveSQLite() bool { return strings.TrimSpace(string(out)) != "" } +func doEmbed() { + if *verbose { + log.Printf("Embedding resources...") + } + closureEmbed := buildSrcPath("server/camlistored/ui/closure/z_data.go") + closureSrcDir := filepath.Join(camRoot, filepath.FromSlash("third_party/closure/lib")) + err := embedClosure(closureSrcDir, closureEmbed) + if err != nil { + log.Fatal(err) + } + wantDestFile[closureEmbed] = true + if err = buildGenfileembed(); err != nil { + log.Fatal(err) + } + if err = genEmbeds(); err != nil { + log.Fatal(err) + } +} + func embedClosure(closureDir, embedFile string) error { if _, err := os.Stat(closureDir); err != nil { return fmt.Errorf("Could not stat %v: %v", closureDir, err) @@ -737,3 +769,30 @@ func exeName(s string) string { } return s } + +// goPackagePath returns the path to the provided Go package's +// source directory. +// pkg may be a path prefix without any *.go files. +// The error is os.ErrNotExist if GOPATH is unset or the directory +// doesn't exist in any GOPATH component. +func goPackagePath(pkg string) (path string, err error) { + gp := os.Getenv("GOPATH") + if gp == "" { + return path, os.ErrNotExist + } + for _, p := range filepath.SplitList(gp) { + dir := filepath.Join(p, "src", filepath.FromSlash(pkg)) + fi, err := os.Stat(dir) + if os.IsNotExist(err) { + continue + } + if err != nil { + return "", err + } + if !fi.IsDir() { + continue + } + return dir, nil + } + return path, os.ErrNotExist +} diff --git a/pkg/test/world.go b/pkg/test/world.go index e5cac5f47..3373b1269 100644 --- a/pkg/test/world.go +++ b/pkg/test/world.go @@ -94,6 +94,7 @@ func (w *World) Start() error { // Build. { + // TODO(mpl): when running with -v (either with go test or devcam test), append it for make.go as well cmd := exec.Command("go", "run", "make.go") cmd.Dir = w.camRoot log.Print("Running make.go to build camlistore binaries for testing...") @@ -184,7 +185,9 @@ func (w *World) Stop() { if w == nil { return } - w.server.Process.Kill() + if err := w.server.Process.Kill(); err != nil { + log.Fatalf("killed failed: %v", err) + } if d := w.tempDir; d != "" { os.RemoveAll(d)