From ba8f88c21a020f5698940c5e2914fdb711efc3ec Mon Sep 17 00:00:00 2001 From: mpl Date: Wed, 11 Sep 2013 19:51:07 +0200 Subject: [PATCH] devcam test, replaces make presubmit http://camlistore.org/issue/223 Change-Id: Iaaea9926a98d7d3335fd6ee52daa225e51ec4703 --- HACKING | 4 + dev/devcam/appengine.go | 2 +- dev/devcam/camget.go | 2 +- dev/devcam/camput.go | 2 +- dev/devcam/devcam.go | 11 ++- dev/devcam/server.go | 2 +- dev/devcam/test.go | 145 ++++++++++++++++++++++++++++++ make.go | 23 +++-- misc/buildbot/bot.go | 2 +- pkg/schema/schema_test.go | 16 +++- pkg/schema/testdata/test-symlink | 1 - pkg/schema/testdata/test-target | 0 website/content/docs/contributing | 1 + 13 files changed, 193 insertions(+), 18 deletions(-) create mode 100644 dev/devcam/test.go delete mode 120000 pkg/schema/testdata/test-symlink delete mode 100644 pkg/schema/testdata/test-target diff --git a/HACKING b/HACKING index 712f68465..ca97d9428 100644 --- a/HACKING +++ b/HACKING @@ -67,4 +67,8 @@ Once the dev server is running, devcam put permanode - Use the UI: http://localhost:3179/ui/ +Before submitting a patch, you should check that all the tests pass with: + +$ devcam test + Please update this file as appropriate. diff --git a/dev/devcam/appengine.go b/dev/devcam/appengine.go index 757db6ff7..d02a54196 100644 --- a/dev/devcam/appengine.go +++ b/dev/devcam/appengine.go @@ -85,7 +85,7 @@ func (c *gaeCmd) RunCommand(args []string) error { } cmdArgs = append(cmdArgs, args...) cmdArgs = append(cmdArgs, applicationDir) - return runExec(devAppServerBin, cmdArgs) + return runExec(devAppServerBin, cmdArgs, nil) } func (c *gaeCmd) checkFlags(args []string) error { diff --git a/dev/devcam/camget.go b/dev/devcam/camget.go index 4f2f896a8..499de4215 100644 --- a/dev/devcam/camget.go +++ b/dev/devcam/camget.go @@ -90,7 +90,7 @@ func (c *getCmd) RunCommand(args []string) error { cmdArgs = append(cmdArgs, "-server="+blobserver) } cmdArgs = append(cmdArgs, args...) - return runExec(cmdBin, cmdArgs) + return runExec(cmdBin, cmdArgs, nil) } func (c *getCmd) checkFlags(args []string) error { diff --git a/dev/devcam/camput.go b/dev/devcam/camput.go index c299286b5..94d5e6f33 100644 --- a/dev/devcam/camput.go +++ b/dev/devcam/camput.go @@ -91,7 +91,7 @@ func (c *putCmd) RunCommand(args []string) error { "-server=" + blobserver, } cmdArgs = append(cmdArgs, args...) - return runExec(cmdBin, cmdArgs) + return runExec(cmdBin, cmdArgs, nil) } func (c *putCmd) checkFlags(args []string) error { diff --git a/dev/devcam/devcam.go b/dev/devcam/devcam.go index 851723134..cdc0677a0 100644 --- a/dev/devcam/devcam.go +++ b/dev/devcam/devcam.go @@ -39,16 +39,21 @@ var sysExec func(argv0 string, argv []string, envv []string) (err error) // runExec execs bin. If the platform doesn't support exec, it runs it and waits // for it to finish. -func runExec(bin string, args []string) error { +func runExec(bin string, args []string, env []string) error { + envv := os.Environ() + if env != nil { + envv = env + } if sysExec != nil { - sysExec(bin, append([]string{filepath.Base(bin)}, args...), os.Environ()) + sysExec(bin, append([]string{filepath.Base(bin)}, args...), envv) } cmd := exec.Command(bin, args...) + cmd.Env = envv cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr if err := cmd.Start(); err != nil { - return fmt.Errorf("Could not run camput: %v", err) + return fmt.Errorf("Could not run %v: %v", bin, err) } go handleSignals(cmd.Process) return cmd.Wait() diff --git a/dev/devcam/server.go b/dev/devcam/server.go index 2a8eae009..8928abe9a 100644 --- a/dev/devcam/server.go +++ b/dev/devcam/server.go @@ -376,5 +376,5 @@ func (c *serverCmd) RunCommand(args []string) error { "-listen=" + c.listen, "-openbrowser=" + strconv.FormatBool(c.openBrowser), } - return runExec(camliBin, cmdArgs) + return runExec(camliBin, cmdArgs, nil) } diff --git a/dev/devcam/test.go b/dev/devcam/test.go new file mode 100644 index 000000000..548cf92c7 --- /dev/null +++ b/dev/devcam/test.go @@ -0,0 +1,145 @@ +/* +Copyright 2013 The Camlistore Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// This program runs the full test suite of the project. +package main + +import ( + "flag" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + + "camlistore.org/pkg/cmdmain" +) + +type testCmd struct { + // start of flag vars + short bool + // end of flag vars + + // buildGoPath becomes our child "go" processes' GOPATH environment variable + buildGoPath string +} + +func init() { + cmdmain.RegisterCommand("test", func(flags *flag.FlagSet) cmdmain.CommandRunner { + cmd := new(testCmd) + flags.BoolVar(&cmd.short, "short", false, "Use '-short' with go test.") + return cmd + }) +} + +func (c *testCmd) Usage() { + fmt.Fprintf(cmdmain.Stderr, "Usage: devcam test\n") +} + +func (c *testCmd) Describe() string { + return "run the full test suite." +} + +func (c *testCmd) RunCommand(args []string) error { + if len(args) != 0 { + c.Usage() + } + if err := c.syncSrc(); err != nil { + return err + } + buildSrcDir := filepath.Join(c.buildGoPath, "src", "camlistore.org") + if err := os.Chdir(buildSrcDir); err != nil { + return err + } + if err := c.buildSelf(); err != nil { + return err + } + if err := c.runTests(); err != nil { + return err + } + println("PASS") + return nil +} + +func (c *testCmd) syncSrc() error { + args := []string{"run", "make.go", "--onlysync"} + cmd := exec.Command("go", args...) + cmd.Stderr = os.Stderr + out, err := cmd.Output() + if err != nil { + return fmt.Errorf("Error populating tmp src tree: %v", err) + } + c.buildGoPath = strings.TrimSpace(string(out)) + return nil +} + +func (c *testCmd) runTests() error { + args := []string{"test"} + if !strings.HasSuffix(c.buildGoPath, "-nosqlite") { + args = append(args, "--tags=with_sqlite") + } + if c.short { + args = append(args, "-short") + } + args = append(args, []string{ + "./pkg/...", + "./server/camlistored", + "./server/appengine", + "./cmd/...", + }...) + env := append(cleanGoEnv(), + "SKIP_DEP_TESTS=1", + "GOPATH="+c.buildGoPath, + ) + return runExec("go", args, env) +} + +// cleanGoEnv returns a copy of the current environment with GOPATH and +// GOBIN removed. +func cleanGoEnv() (clean []string) { + for _, env := range os.Environ() { + if strings.HasPrefix(env, "GOPATH=") { + continue + } + if strings.HasPrefix(env, "GOBIN=") { + continue + } + clean = append(clean, env) + } + return +} + +func (c *testCmd) buildSelf() error { + args := []string{ + "install", + filepath.FromSlash("./dev/devcam"), + } + cmd := exec.Command("go", args...) + binDir, err := filepath.Abs("bin") + if err != nil { + return fmt.Errorf("Error setting GOBIN: %v", err) + } + cmd.Env = append(cleanGoEnv(), + "GOBIN="+binDir, + "GOPATH="+c.buildGoPath, + ) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + return fmt.Errorf("Error building devcam: %v", err) + } + return nil +} diff --git a/make.go b/make.go index b93bba544..d0e4114ac 100644 --- a/make.go +++ b/make.go @@ -53,6 +53,7 @@ var ( verbose = flag.Bool("v", false, "Verbose mode") targets = flag.String("targets", "", "Optional comma-separated list of targets (i.e go packages) to build and install. Empty means all. Example: camlistore.org/server/camlistored,camlistore.org/cmd/camput") quiet = flag.Bool("quiet", false, "Don't print anything unless there's a failure.") + onlysync = flag.Bool("onlysync", false, "Only populate the temporary source/build tree and output its full path. It is meant to prepare the environment for running the full test suite with 'devcam test'.") // TODO(mpl): looks like ifModsSince is not used anywhere? ifModsSince = flag.Int64("if_mods_since", 0, "If non-zero return immediately without building if there aren't any filesystem modifications past this time (in unix seconds)") buildARCH = flag.String("arch", runtime.GOARCH, "Architecture to build for.") @@ -65,7 +66,7 @@ var ( // Our temporary source tree root and build dir, i.e: buildGoPath + "src/camlistore.org" buildSrcDir string // files mirrored from camRoot to buildSrcDir - rxMirrored = regexp.MustCompile(`^([a-zA-Z0-9\-\_]+\.(?:go|html|js|css|png|jpg|gif|ico))$`) + rxMirrored = regexp.MustCompile(`^([a-zA-Z0-9\-\_]+\.(?:go|html|js|css|png|jpg|gif|ico|gpg|json|err|camli))$`) ) func main() { @@ -127,8 +128,13 @@ func main() { log.Printf("Output binaries: %s", binDir) } + // TODO(mpl): main is getting long. We could probably move all the mirroring + // dance to its own func. // We copy all *.go files from camRoot's goDirs to buildSrcDir. goDirs := []string{"cmd", "pkg", "dev", "server/camlistored", "third_party"} + if *onlysync { + goDirs = append(goDirs, "server/appengine", "config") + } // Copy files we do want in our mirrored GOPATH. This has the side effect of // populating wantDestFile, populated by mirrorFile. var latestSrcMod time.Time @@ -146,6 +152,13 @@ func main() { verifyGoVersion() + if *onlysync { + mirrorFile("make.go", filepath.Join(buildSrcDir, "make.go")) + deleteUnwantedOldMirrorFiles(buildSrcDir, true) + fmt.Println(buildGoPath) + return + } + buildAll := true targs := []string{ "camlistore.org/dev/devcam", @@ -468,13 +481,9 @@ func mirrorDir(src, dst string) (maxMod time.Time, err error) { } base := fi.Name() if fi.IsDir() { - if base == "testdata" { - return filepath.SkipDir - } + return nil } - if strings.HasSuffix(base, "_test.go") || - strings.HasPrefix(base, ".#") || - !rxMirrored.MatchString(base) { + if strings.HasPrefix(base, ".#") || !rxMirrored.MatchString(base) { return nil } suffix, err := filepath.Rel(src, path) diff --git a/misc/buildbot/bot.go b/misc/buildbot/bot.go index f4c45a730..d0efae199 100644 --- a/misc/buildbot/bot.go +++ b/misc/buildbot/bot.go @@ -99,7 +99,7 @@ var NameToCmd = map[string]string{ "buildGoTip1": "./make.bash", "buildCamli1": "go run make.go -v", "buildCamli2": "go build -o devcam ./dev/devcam/", - "buildCamli3": "make presubmit", + "buildCamli3": "devcam test", "runCamli": "./devcam server --wipe --mysql", "hitCamliUi1": "http://localhost:3179/ui/", "camget": "./devcam get ", diff --git a/pkg/schema/schema_test.go b/pkg/schema/schema_test.go index b3f508f32..4e56caddf 100644 --- a/pkg/schema/schema_test.go +++ b/pkg/schema/schema_test.go @@ -19,6 +19,7 @@ package schema import ( "encoding/json" "os" + "path/filepath" "strings" "testing" "time" @@ -49,7 +50,7 @@ func TestJSON(t *testing.T) { func TestRegularFile(t *testing.T) { fileName := "schema_test.go" fi, err := os.Lstat(fileName) - AssertNil(t, err, "test-symlink stat") + AssertNil(t, err, "schema_test.go stat") m := NewCommonFileMap("schema_test.go", fi) json, err := m.JSON() if err != nil { @@ -59,7 +60,18 @@ func TestRegularFile(t *testing.T) { } func TestSymlink(t *testing.T) { - fileName := "testdata/test-symlink" + // We create the symlink now because make.go does not mirror + // symlinks properly, and it is less intrusive to do that here. + defer os.RemoveAll("testdata") + err := os.Mkdir("testdata", 0755) + AssertNil(t, err, "Mkdir") + err = os.Chdir("testdata") + AssertNil(t, err, "Chdir") + err = os.Symlink("test-target", "test-symlink") + AssertNil(t, err, "creating test-symlink") + err = os.Chdir("..") + AssertNil(t, err, "Chdir") + fileName := filepath.Join("testdata", "test-symlink") fi, err := os.Lstat(fileName) AssertNil(t, err, "test-symlink stat") m := NewCommonFileMap(fileName, fi) diff --git a/pkg/schema/testdata/test-symlink b/pkg/schema/testdata/test-symlink deleted file mode 120000 index 5f2fc14e7..000000000 --- a/pkg/schema/testdata/test-symlink +++ /dev/null @@ -1 +0,0 @@ -test-target \ No newline at end of file diff --git a/pkg/schema/testdata/test-target b/pkg/schema/testdata/test-target deleted file mode 100644 index e69de29bb..000000000 diff --git a/website/content/docs/contributing b/website/content/docs/contributing index 4af03e057..bb9e4feff 100644 --- a/website/content/docs/contributing +++ b/website/content/docs/contributing @@ -5,6 +5,7 @@