Merge "devcam test, replaces make presubmit"

HACKING

@@ -67,4 +67,8 @@ Once the dev server is running,
       devcam put permanode
   - Use the UI: http://localhost:3179/ui/
 
+Before submitting a patch, you should check that all the tests pass with:
+
+$ devcam test
+
 Please update this file as appropriate.

dev/devcam/appengine.go

@@ -85,7 +85,7 @@ func (c *gaeCmd) RunCommand(args []string) error {
 	}
 	cmdArgs = append(cmdArgs, args...)
 	cmdArgs = append(cmdArgs, applicationDir)
-	return runExec(devAppServerBin, cmdArgs)
+	return runExec(devAppServerBin, cmdArgs, nil)
 }
 
 func (c *gaeCmd) checkFlags(args []string) error {

dev/devcam/camget.go

@@ -90,7 +90,7 @@ func (c *getCmd) RunCommand(args []string) error {
 		cmdArgs = append(cmdArgs, "-server="+blobserver)
 	}
 	cmdArgs = append(cmdArgs, args...)
-	return runExec(cmdBin, cmdArgs)
+	return runExec(cmdBin, cmdArgs, nil)
 }
 
 func (c *getCmd) checkFlags(args []string) error {

dev/devcam/camput.go

@@ -91,7 +91,7 @@ func (c *putCmd) RunCommand(args []string) error {
 		"-server=" + blobserver,
 	}
 	cmdArgs = append(cmdArgs, args...)
-	return runExec(cmdBin, cmdArgs)
+	return runExec(cmdBin, cmdArgs, nil)
 }
 
 func (c *putCmd) checkFlags(args []string) error {

dev/devcam/devcam.go

@@ -39,16 +39,21 @@ var sysExec func(argv0 string, argv []string, envv []string) (err error)
 
 // runExec execs bin. If the platform doesn't support exec, it runs it and waits
 // for it to finish.
-func runExec(bin string, args []string) error {
+func runExec(bin string, args []string, env []string) error {
+	envv := os.Environ()
+	if env != nil {
+		envv = env
+	}
 	if sysExec != nil {
-		sysExec(bin, append([]string{filepath.Base(bin)}, args...), os.Environ())
+		sysExec(bin, append([]string{filepath.Base(bin)}, args...), envv)
 	}
 
 	cmd := exec.Command(bin, args...)
+	cmd.Env = envv
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
 	if err := cmd.Start(); err != nil {
-		return fmt.Errorf("Could not run camput: %v", err)
+		return fmt.Errorf("Could not run %v: %v", bin, err)
 	}
 	go handleSignals(cmd.Process)
 	return cmd.Wait()

dev/devcam/server.go

@@ -376,5 +376,5 @@ func (c *serverCmd) RunCommand(args []string) error {
 		"-listen=" + c.listen,
 		"-openbrowser=" + strconv.FormatBool(c.openBrowser),
 	}
-	return runExec(camliBin, cmdArgs)
+	return runExec(camliBin, cmdArgs, nil)
 }

dev/devcam/test.go

@@ -0,0 +1,145 @@
+/*
+Copyright 2013 The Camlistore Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// This program runs the full test suite of the project.
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+
+	"camlistore.org/pkg/cmdmain"
+)
+
+type testCmd struct {
+	// start of flag vars
+	short bool
+	// end of flag vars
+
+	// buildGoPath becomes our child "go" processes' GOPATH environment variable
+	buildGoPath string
+}
+
+func init() {
+	cmdmain.RegisterCommand("test", func(flags *flag.FlagSet) cmdmain.CommandRunner {
+		cmd := new(testCmd)
+		flags.BoolVar(&cmd.short, "short", false, "Use '-short' with go test.")
+		return cmd
+	})
+}
+
+func (c *testCmd) Usage() {
+	fmt.Fprintf(cmdmain.Stderr, "Usage: devcam test\n")
+}
+
+func (c *testCmd) Describe() string {
+	return "run the full test suite."
+}
+
+func (c *testCmd) RunCommand(args []string) error {
+	if len(args) != 0 {
+		c.Usage()
+	}
+	if err := c.syncSrc(); err != nil {
+		return err
+	}
+	buildSrcDir := filepath.Join(c.buildGoPath, "src", "camlistore.org")
+	if err := os.Chdir(buildSrcDir); err != nil {
+		return err
+	}
+	if err := c.buildSelf(); err != nil {
+		return err
+	}
+	if err := c.runTests(); err != nil {
+		return err
+	}
+	println("PASS")
+	return nil
+}
+
+func (c *testCmd) syncSrc() error {
+	args := []string{"run", "make.go", "--onlysync"}
+	cmd := exec.Command("go", args...)
+	cmd.Stderr = os.Stderr
+	out, err := cmd.Output()
+	if err != nil {
+		return fmt.Errorf("Error populating tmp src tree: %v", err)
+	}
+	c.buildGoPath = strings.TrimSpace(string(out))
+	return nil
+}
+
+func (c *testCmd) runTests() error {
+	args := []string{"test"}
+	if !strings.HasSuffix(c.buildGoPath, "-nosqlite") {
+		args = append(args, "--tags=with_sqlite")
+	}
+	if c.short {
+		args = append(args, "-short")
+	}
+	args = append(args, []string{
+		"./pkg/...",
+		"./server/camlistored",
+		"./server/appengine",
+		"./cmd/...",
+	}...)
+	env := append(cleanGoEnv(),
+		"SKIP_DEP_TESTS=1",
+		"GOPATH="+c.buildGoPath,
+	)
+	return runExec("go", args, env)
+}
+
+// cleanGoEnv returns a copy of the current environment with GOPATH and
+// GOBIN removed.
+func cleanGoEnv() (clean []string) {
+	for _, env := range os.Environ() {
+		if strings.HasPrefix(env, "GOPATH=") {
+			continue
+		}
+		if strings.HasPrefix(env, "GOBIN=") {
+			continue
+		}
+		clean = append(clean, env)
+	}
+	return
+}
+
+func (c *testCmd) buildSelf() error {
+	args := []string{
+		"install",
+		filepath.FromSlash("./dev/devcam"),
+	}
+	cmd := exec.Command("go", args...)
+	binDir, err := filepath.Abs("bin")
+	if err != nil {
+		return fmt.Errorf("Error setting GOBIN: %v", err)
+	}
+	cmd.Env = append(cleanGoEnv(),
+		"GOBIN="+binDir,
+		"GOPATH="+c.buildGoPath,
+	)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	if err := cmd.Run(); err != nil {
+		return fmt.Errorf("Error building devcam: %v", err)
+	}
+	return nil
+}

make.go

@@ -53,6 +53,7 @@ var (
 	verbose        = flag.Bool("v", false, "Verbose mode")
 	targets        = flag.String("targets", "", "Optional comma-separated list of targets (i.e go packages) to build and install. Empty means all. Example: camlistore.org/server/camlistored,camlistore.org/cmd/camput")
 	quiet          = flag.Bool("quiet", false, "Don't print anything unless there's a failure.")
+	onlysync       = flag.Bool("onlysync", false, "Only populate the temporary source/build tree and output its full path. It is meant to prepare the environment for running the full test suite with 'devcam test'.")
 	// TODO(mpl): looks like ifModsSince is not used anywhere?
 	ifModsSince = flag.Int64("if_mods_since", 0, "If non-zero return immediately without building if there aren't any filesystem modifications past this time (in unix seconds)")
 	buildARCH   = flag.String("arch", runtime.GOARCH, "Architecture to build for.")
@@ -65,7 +66,7 @@ var (
 	// Our temporary source tree root and build dir, i.e: buildGoPath + "src/camlistore.org"
 	buildSrcDir string
 	// files mirrored from camRoot to buildSrcDir
-	rxMirrored = regexp.MustCompile(`^([a-zA-Z0-9\-\_]+\.(?:go|html|js|css|png|jpg|gif|ico))$`)
+	rxMirrored = regexp.MustCompile(`^([a-zA-Z0-9\-\_]+\.(?:go|html|js|css|png|jpg|gif|ico|gpg|json|err|camli))$`)
 )
 
 func main() {
@@ -127,8 +128,13 @@ func main() {
 		log.Printf("Output binaries: %s", binDir)
 	}
 
+	// TODO(mpl): main is getting long. We could probably move all the mirroring
+	// dance to its own func.
 	// We copy all *.go files from camRoot's goDirs to buildSrcDir.
 	goDirs := []string{"cmd", "pkg", "dev", "server/camlistored", "third_party"}
+	if *onlysync {
+		goDirs = append(goDirs, "server/appengine", "config")
+	}
 	// Copy files we do want in our mirrored GOPATH.  This has the side effect of
 	// populating wantDestFile, populated by mirrorFile.
 	var latestSrcMod time.Time
@@ -146,6 +152,13 @@ func main() {
 
 	verifyGoVersion()
 
+	if *onlysync {
+		mirrorFile("make.go", filepath.Join(buildSrcDir, "make.go"))
+		deleteUnwantedOldMirrorFiles(buildSrcDir, true)
+		fmt.Println(buildGoPath)
+		return
+	}
+
 	buildAll := true
 	targs := []string{
 		"camlistore.org/dev/devcam",
@@ -468,13 +481,9 @@ func mirrorDir(src, dst string) (maxMod time.Time, err error) {
 		}
 		base := fi.Name()
 		if fi.IsDir() {
-			if base == "testdata" {
-				return filepath.SkipDir
+			return nil
 		}
-		}
-		if strings.HasSuffix(base, "_test.go") ||
-			strings.HasPrefix(base, ".#") ||
-			!rxMirrored.MatchString(base) {
+		if strings.HasPrefix(base, ".#") || !rxMirrored.MatchString(base) {
 			return nil
 		}
 		suffix, err := filepath.Rel(src, path)

misc/buildbot/bot.go

@@ -99,7 +99,7 @@ var NameToCmd = map[string]string{
 	"buildGoTip1": "./make.bash",
 	"buildCamli1": "go run make.go -v",
 	"buildCamli2": "go build -o devcam ./dev/devcam/",
-	"buildCamli3": "make presubmit",
+	"buildCamli3": "devcam test",
 	"runCamli":    "./devcam server --wipe --mysql",
 	"hitCamliUi1": "http://localhost:3179/ui/",
 	"camget":      "./devcam get ",

pkg/schema/schema_test.go

@@ -19,6 +19,7 @@ package schema
 import (
 	"encoding/json"
 	"os"
+	"path/filepath"
 	"strings"
 	"testing"
 	"time"
@@ -49,7 +50,7 @@ func TestJSON(t *testing.T) {
 func TestRegularFile(t *testing.T) {
 	fileName := "schema_test.go"
 	fi, err := os.Lstat(fileName)
-	AssertNil(t, err, "test-symlink stat")
+	AssertNil(t, err, "schema_test.go stat")
 	m := NewCommonFileMap("schema_test.go", fi)
 	json, err := m.JSON()
 	if err != nil {
@@ -59,7 +60,18 @@ func TestRegularFile(t *testing.T) {
 }
 
 func TestSymlink(t *testing.T) {
-	fileName := "testdata/test-symlink"
+	// We create the symlink now because make.go does not mirror
+	// symlinks properly, and it is less intrusive to do that here.
+	defer os.RemoveAll("testdata")
+	err := os.Mkdir("testdata", 0755)
+	AssertNil(t, err, "Mkdir")
+	err = os.Chdir("testdata")
+	AssertNil(t, err, "Chdir")
+	err = os.Symlink("test-target", "test-symlink")
+	AssertNil(t, err, "creating test-symlink")
+	err = os.Chdir("..")
+	AssertNil(t, err, "Chdir")
+	fileName := filepath.Join("testdata", "test-symlink")
 	fi, err := os.Lstat(fileName)
 	AssertNil(t, err, "test-symlink stat")
 	m := NewCommonFileMap(fileName, fi)

pkg/schema/testdata/test-symlink

@@ -1 +0,0 @@
-test-target

website/content/docs/contributing

@@ -5,6 +5,7 @@
 <ul class='lispaced'>
    <li>Join the <a href="/lists">mailing list</a>.  Say hi.</li>
    <li>Pick an area to tackle.  Discuss it first, especially if it's large and/or not well designed yet.</li>
+   <li>See the <a href="https://camlistore.googlesource.com/camlistore/+/master/HACKING">HACKING</a> file at the root of the source to get started.</li>
    <li>Put your changes online somewhere for review &amp; git pulling.</li>
    <li>Note that you'll need to agree to the appropriate Contributor License Agreement:
        <ul>