Rooba
/
perkeep
mirror of https://github.com/perkeep/perkeep.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge "devcam test: do not "recurse" temp GOPATH, docs, couple more options."

This commit is contained in:
mpl 2014-08-11 14:19:00 +00:00 committed by Gerrit Code Review
parent 0f0ad341c4 d953dee008
commit 1d67424167
4 changed files with 177 additions and 101 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
dev/devcam/test.go
View File

@ -31,7 +31,9 @@ import (
type testCmd struct {
// start of flag vars
short bool
verbose bool
short bool
run string
// end of flag vars
// buildGoPath becomes our child "go" processes' GOPATH environment variable
@ -42,16 +44,18 @@ func init() {
cmdmain.RegisterCommand("test", func(flags *flag.FlagSet) cmdmain.CommandRunner {
cmd := new(testCmd)
flags.BoolVar(&cmd.short, "short", false, "Use '-short' with go test.")
flags.BoolVar(&cmd.verbose, "v", false, "Use '-v' (for verbose) with go test.")
flags.StringVar(&cmd.run, "run", "", "Use '-run' with go test.")
return cmd
})
}
func (c *testCmd) Usage() {
fmt.Fprintf(cmdmain.Stderr, "Usage: devcam test\n")
fmt.Fprintf(cmdmain.Stderr, "Usage: devcam test [test_opts] [targets]\n")
}
func (c *testCmd) Describe() string {
return "run the full test suite."
return "run the full test suite, or the tests in the specified target packages."
}
func (c *testCmd) RunCommand(args []string) error {
@ -79,6 +83,7 @@ func (c *testCmd) env() *Env {
env := NewCopyEnv()
env.NoGo()
env.Set("GOPATH", c.buildGoPath)
env.Set("CAMLI_MAKE_USEGOPATH", "true")
return env
}
@ -123,6 +128,12 @@ func (c *testCmd) runTests(args []string) error {
if c.short {
targs = append(targs, "-short")
}
if c.verbose {
targs = append(targs, "-v")
}
if c.run != "" {
targs = append(targs, "-run="+c.run)
}
if len(args) > 0 {
targs = append(targs, args...)
} else {

3
doc/environment-vars.txt
View File

@ -98,6 +98,9 @@ CAMLI_KV_VERIFY (bool):
CAMLI_MONGO_WIPE (bool):
Wipe out mongo based index on startup.
CAMLI_MAKE_USEGOPATH (bool):
When running make.go, overrides the -use_gopath flag.
CAMLI_NO_FILE_DUP_SEARCH (bool):
This will cause the search-for-exists-before-upload step to be skipped when
camput is uploading files.

253
make.go
View File

@ -49,20 +49,23 @@ var haveSQLite = checkHaveSQLite()
var (
embedResources = flag.Bool("embed_static", true, "Whether to embed resources needed by the UI such as images, css, and javascript.")
sqlFlag = flag.String("sqlite", "auto", "Whether you want SQLite in your build: yes, no, or auto.")
sqlFlag = flag.String("sqlite", "auto", "Whether you want SQLite in your build: true, false, or auto.")
all = flag.Bool("all", false, "Force rebuild of everything (go install -a)")
race = flag.Bool("race", false, "Build race-detector version of binaries (they will run slowly)")
verbose = flag.Bool("v", false, "Verbose mode")
targets = flag.String("targets", "", "Optional comma-separated list of targets (i.e go packages) to build and install. '*' builds everything. Empty builds defaults for this platform. Example: camlistore.org/server/camlistored,camlistore.org/cmd/camput")
quiet = flag.Bool("quiet", false, "Don't print anything unless there's a failure.")
onlysync = flag.Bool("onlysync", false, "Only populate the temporary source/build tree and output its full path. It is meant to prepare the environment for running the full test suite with 'devcam test'.")
// TODO(mpl): looks like ifModsSince is not used anywhere?
ifModsSince = flag.Int64("if_mods_since", 0, "If non-zero return immediately without building if there aren't any filesystem modifications past this time (in unix seconds)")
useGoPath = flag.Bool("use_gopath", false, "Use GOPATH from the environment and work from there. Do not create a temporary source tree with a new GOPATH in it.")
// TODO(mpl): implement ifModsSince, or remove it from all places where we use make.go, because we shouldn't lie.
ifModsSince = flag.Int64("if_mods_since", 0, "Not implemented yet. If non-zero return immediately without building if there aren't any filesystem modifications past this time (in unix seconds)")
buildARCH = flag.String("arch", runtime.GOARCH, "Architecture to build for.")
buildOS = flag.String("os", runtime.GOOS, "Operating system to build for.")
)
var (
// camRoot is the original Camlistore project root, from where the source files are mirrored.
camRoot string
// buildGoPath becomes our child "go" processes' GOPATH environment variable
buildGoPath string
// Our temporary source tree root and build dir, i.e: buildGoPath + "src/camlistore.org"
@ -75,53 +78,36 @@ func main() {
log.SetFlags(0)
flag.Parse()
camRoot, err := os.Getwd()
if err != nil {
log.Fatalf("Failed to get current directory: %v", err)
}
verifyCamlistoreRoot(camRoot)
verifyGoVersion()
cross := runtime.GOOS != *buildOS || runtime.GOARCH != *buildARCH
var sql bool
if *sqlFlag == "auto" {
sql = !cross && haveSQLite
} else {
sql, err = strconv.ParseBool(*sqlFlag)
sql := withSQLite()
if useEnvGoPath, _ := strconv.ParseBool(os.Getenv("CAMLI_MAKE_USEGOPATH")); useEnvGoPath {
*useGoPath = true
}
if *useGoPath {
buildGoPath = os.Getenv("GOPATH")
var err error
camRoot, err = goPackagePath("camlistore.org")
if err != nil {
log.Fatalf("Bad boolean --sql flag %q", *sqlFlag)
log.Fatalf("Cannot run make.go with --use_gopath: %v (is GOPATH not set?)", err)
}
buildSrcDir = camRoot
} else {
var err error
camRoot, err = os.Getwd()
if err != nil {
log.Fatalf("Failed to get current directory: %v", err)
}
mirror(sql)
if *onlysync {
mirrorFile("make.go", filepath.Join(buildSrcDir, "make.go"))
deleteUnwantedOldMirrorFiles(buildSrcDir, true)
fmt.Println(buildGoPath)
return
}
}
if cross && sql {
log.Fatalf("SQLite isn't available when cross-compiling to another OS. Set --sqlite=false.")
}
if sql && !haveSQLite {
log.Printf("SQLite not found. Either install it, or run make.go with --sqlite=false See https://code.google.com/p/camlistore/wiki/SQLite")
switch runtime.GOOS {
case "darwin":
log.Printf("On OS X, run 'brew install sqlite3 pkg-config'. Get brew from http://mxcl.github.io/homebrew/")
case "linux":
log.Printf("On Linux, run 'sudo apt-get install libsqlite3-dev' or equivalent.")
case "windows":
log.Printf("SQLite is not easy on windows. Please see http://camlistore.org/docs/server-config#windows")
}
os.Exit(2)
}
buildBaseDir := "build-gopath"
if !sql {
buildBaseDir += "-nosqlite"
}
buildGoPath = filepath.Join(camRoot, "tmp", buildBaseDir)
binDir := filepath.Join(camRoot, "bin")
buildSrcDir = filepath.Join(buildGoPath, "src", "camlistore.org")
if err := os.MkdirAll(buildSrcDir, 0755); err != nil {
log.Fatal(err)
}
version := getVersion(camRoot)
version := getVersion()
if *verbose {
log.Printf("Camlistore version = %s", version)
@ -130,37 +116,6 @@ func main() {
log.Printf("Output binaries: %s", binDir)
}
// TODO(mpl): main is getting long. We could probably move all the mirroring
// dance to its own func.
// We copy all *.go files from camRoot's goDirs to buildSrcDir.
goDirs := []string{"app", "cmd", "pkg", "dev", "server/camlistored", "third_party"}
if *onlysync {
goDirs = append(goDirs, "server/appengine", "config")
}
// Copy files we do want in our mirrored GOPATH. This has the side effect of
// populating wantDestFile, populated by mirrorFile.
var latestSrcMod time.Time
for _, dir := range goDirs {
oriPath := filepath.Join(camRoot, filepath.FromSlash(dir))
dstPath := buildSrcPath(dir)
if maxMod, err := mirrorDir(oriPath, dstPath, mirrorOpts{sqlite: sql}); err != nil {
log.Fatalf("Error while mirroring %s to %s: %v", oriPath, dstPath, err)
} else {
if maxMod.After(latestSrcMod) {
latestSrcMod = maxMod
}
}
}
verifyGoVersion()
if *onlysync {
mirrorFile("make.go", filepath.Join(buildSrcDir, "make.go"))
deleteUnwantedOldMirrorFiles(buildSrcDir, true)
fmt.Println(buildGoPath)
return
}
buildAll := false
targs := []string{
"camlistore.org/dev/devcam",
@ -188,25 +143,15 @@ func main() {
withCamlistored := stringListContains(targs, "camlistore.org/server/camlistored")
if *embedResources && withCamlistored {
if *verbose {
log.Printf("Embedding resources...")
}
closureEmbed := buildSrcPath("server/camlistored/ui/closure/z_data.go")
closureSrcDir := filepath.Join(camRoot, filepath.FromSlash("third_party/closure/lib"))
err := embedClosure(closureSrcDir, closureEmbed)
if err != nil {
log.Fatal(err)
}
wantDestFile[closureEmbed] = true
if err = buildGenfileembed(); err != nil {
log.Fatal(err)
}
if err = genEmbeds(); err != nil {
log.Fatal(err)
}
// TODO(mpl): in doEmbed, it looks like we not only always recreate the closure
// z_data.go, but we also always regenerate the zembed.*.go, at least for the
// integration tests. I'll look into it.
doEmbed()
}
deleteUnwantedOldMirrorFiles(buildSrcDir, withCamlistored)
if !*useGoPath {
deleteUnwantedOldMirrorFiles(buildSrcDir, withCamlistored)
}
tags := ""
if sql {
@ -273,6 +218,43 @@ func main() {
}
}
// create the tmp GOPATH, and mirror to it from camRoot.
func mirror(sql bool) {
verifyCamlistoreRoot(camRoot)
buildBaseDir := "build-gopath"
if !sql {
buildBaseDir += "-nosqlite"
}
buildGoPath = filepath.Join(camRoot, "tmp", buildBaseDir)
buildSrcDir = filepath.Join(buildGoPath, "src", "camlistore.org")
if err := os.MkdirAll(buildSrcDir, 0755); err != nil {
log.Fatal(err)
}
// We copy all *.go files from camRoot's goDirs to buildSrcDir.
goDirs := []string{"app", "cmd", "pkg", "dev", "server/camlistored", "third_party"}
if *onlysync {
goDirs = append(goDirs, "server/appengine", "config")
}
// Copy files we do want in our mirrored GOPATH. This has the side effect of
// populating wantDestFile, populated by mirrorFile.
var latestSrcMod time.Time
for _, dir := range goDirs {
srcPath := filepath.Join(camRoot, filepath.FromSlash(dir))
dstPath := buildSrcPath(dir)
if maxMod, err := mirrorDir(srcPath, dstPath, mirrorOpts{sqlite: sql}); err != nil {
log.Fatalf("Error while mirroring %s to %s: %v", srcPath, dstPath, err)
} else {
if maxMod.After(latestSrcMod) {
latestSrcMod = maxMod
}
}
}
}
func actualBinDir(dir string) string {
if *buildARCH == runtime.GOARCH && *buildOS == runtime.GOOS {
return dir
@ -410,12 +392,12 @@ func buildGenfileembed() error {
// getVersion returns the version of Camlistore. Either from a VERSION file at the root,
// or from git.
func getVersion(camRoot string) string {
func getVersion() string {
slurp, err := ioutil.ReadFile(filepath.Join(camRoot, "VERSION"))
if err == nil {
return strings.TrimSpace(string(slurp))
}
return gitVersion(camRoot)
return gitVersion()
}
var gitVersionRx = regexp.MustCompile(`\b\d\d\d\d-\d\d-\d\d-[0-9a-f]{7,7}\b`)
@ -423,7 +405,7 @@ var gitVersionRx = regexp.MustCompile(`\b\d\d\d\d-\d\d-\d\d-[0-9a-f]{7,7}\b`)
// gitVersion returns the git version of the git repo at camRoot as a
// string of the form "yyyy-mm-dd-xxxxxxx", with an optional trailing
// '+' if there are any local uncomitted modifications to the tree.
func gitVersion(camRoot string) string {
func gitVersion() string {
cmd := exec.Command("git", "rev-list", "--max-count=1", "--pretty=format:'%ad-%h'", "--date=short", "HEAD")
cmd.Dir = camRoot
out, err := cmd.Output()
@ -585,7 +567,7 @@ func deleteUnwantedOldMirrorFiles(dir string, withCamlistored bool) {
// have to put it back into place later.
return nil
}
if !*quiet {
if *verbose {
log.Printf("Deleting old file from temp build dir: %s", path)
}
return os.Remove(path)
@ -594,6 +576,37 @@ func deleteUnwantedOldMirrorFiles(dir string, withCamlistored bool) {
})
}
func withSQLite() bool {
cross := runtime.GOOS != *buildOS || runtime.GOARCH != *buildARCH
var sql bool
var err error
if *sqlFlag == "auto" {
sql = !cross && haveSQLite
} else {
sql, err = strconv.ParseBool(*sqlFlag)
if err != nil {
log.Fatalf("Bad boolean --sql flag %q", *sqlFlag)
}
}
if cross && sql {
log.Fatalf("SQLite isn't available when cross-compiling to another OS. Set --sqlite=false.")
}
if sql && !haveSQLite {
log.Printf("SQLite not found. Either install it, or run make.go with --sqlite=false See https://code.google.com/p/camlistore/wiki/SQLite")
switch runtime.GOOS {
case "darwin":
log.Printf("On OS X, run 'brew install sqlite3 pkg-config'. Get brew from http://mxcl.github.io/homebrew/")
case "linux":
log.Printf("On Linux, run 'sudo apt-get install libsqlite3-dev' or equivalent.")
case "windows":
log.Printf("SQLite is not easy on windows. Please see http://camlistore.org/docs/server-config#windows")
}
os.Exit(2)
}
return sql
}
func checkHaveSQLite() bool {
if runtime.GOOS == "windows" {
// TODO: Find some other non-pkg-config way to test, like
@ -621,6 +634,25 @@ func checkHaveSQLite() bool {
return strings.TrimSpace(string(out)) != ""
}
func doEmbed() {
if *verbose {
log.Printf("Embedding resources...")
}
closureEmbed := buildSrcPath("server/camlistored/ui/closure/z_data.go")
closureSrcDir := filepath.Join(camRoot, filepath.FromSlash("third_party/closure/lib"))
err := embedClosure(closureSrcDir, closureEmbed)
if err != nil {
log.Fatal(err)
}
wantDestFile[closureEmbed] = true
if err = buildGenfileembed(); err != nil {
log.Fatal(err)
}
if err = genEmbeds(); err != nil {
log.Fatal(err)
}
}
func embedClosure(closureDir, embedFile string) error {
if _, err := os.Stat(closureDir); err != nil {
return fmt.Errorf("Could not stat %v: %v", closureDir, err)
@ -737,3 +769,30 @@ func exeName(s string) string {
}
return s
}
// goPackagePath returns the path to the provided Go package's
// source directory.
// pkg may be a path prefix without any *.go files.
// The error is os.ErrNotExist if GOPATH is unset or the directory
// doesn't exist in any GOPATH component.
func goPackagePath(pkg string) (path string, err error) {
gp := os.Getenv("GOPATH")
if gp == "" {
return path, os.ErrNotExist
}
for _, p := range filepath.SplitList(gp) {
dir := filepath.Join(p, "src", filepath.FromSlash(pkg))
fi, err := os.Stat(dir)
if os.IsNotExist(err) {
continue
}
if err != nil {
return "", err
}
if !fi.IsDir() {
continue
}
return dir, nil
}
return path, os.ErrNotExist
}

5
pkg/test/world.go
View File

@ -94,6 +94,7 @@ func (w *World) Start() error {
// Build.
{
// TODO(mpl): when running with -v (either with go test or devcam test), append it for make.go as well
cmd := exec.Command("go", "run", "make.go")
cmd.Dir = w.camRoot
log.Print("Running make.go to build camlistore binaries for testing...")
@ -184,7 +185,9 @@ func (w *World) Stop() {
if w == nil {
return
}
w.server.Process.Kill()
if err := w.server.Process.Kill(); err != nil {
log.Fatalf("killed failed: %v", err)
}
if d := w.tempDir; d != "" {
os.RemoveAll(d)