From f45e6f5508dcd256f9165229e4d5257eaf3d0a46 Mon Sep 17 00:00:00 2001 From: mpl Date: Wed, 12 Dec 2012 12:32:32 +0100 Subject: [PATCH] buildbot: now with history and prettier output Change-Id: I863a5899024a74930bfa534f2ce79bcd083326fe --- misc/buildbot/bot.go | 1074 ++++++++++++++++++++++++++++++++++-------- 1 file changed, 889 insertions(+), 185 deletions(-) diff --git a/misc/buildbot/bot.go b/misc/buildbot/bot.go index f8ceba7d6..b01462daa 100644 --- a/misc/buildbot/bot.go +++ b/misc/buildbot/bot.go @@ -1,17 +1,36 @@ +/* +Copyright 2012 The Camlistore Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package main import ( "bytes" "flag" "fmt" + "html/template" "io/ioutil" "log" + "math/rand" "net/http" "os" "os/exec" "os/signal" "path/filepath" "regexp" + "runtime" "strings" "sync" "syscall" @@ -19,58 +38,163 @@ import ( ) const ( - maxInterval = 24 * time.Hour - warmup = 17 * time.Second // duration before we test if dev-server has started properly + interval = 1 * time.Minute // polling frequency + warmup = 60 * time.Second // duration before we test if dev-server has started properly + historySize = 30 + testFile = "AUTHORS" ) var ( - debug = flag.Bool("debug", false, "print debug statements") - gotipdir = flag.String("gotip", "./gotip", "path to the Go tip tree") - help = flag.Bool("h", false, "show this help") - host = flag.String("host", "0.0.0.0:8080", "listening hostname and port") - nobuild = flag.Bool("nobuild", false, "skip building Go tip, mostly for debugging") + fast = flag.Bool("fast", false, "run dummy steps instead of the actual tasks, for debugging") + gotipdir = flag.String("gotip", "./go", "path to the Go tip tree") + help = flag.Bool("h", false, "show this help") + host = flag.String("host", "0.0.0.0:8080", "listening hostname and port") + nocleanup = flag.Bool("nocleanup", false, "do not clean up the tmp gopath when failing") + verbose = flag.Bool("verbose", false, "print what's going on") ) var ( + camliHeadHash string cachedCamliRoot string camliRoot string currentTask task dbg *debugger defaultDir string - gopath string - interval time.Duration // frequency at which the tests are run - lastErr error + defaultPATH string + // doBuildCamli0 is for when we're in the go 1 test suite, and + // doBuildCamli1 for the go tip test suite. They are set when + // the camli source tree has been updated and indicate that + // 'make' should be run. + // doBuildGo indicates the go source tree has changed, hence + // go tip should be rebuilt, and in the go tip test suite case + // camlistore should be cleaned (pkg and bin) and rebuilt. + doBuildGo, doBuildCamli0, doBuildCamli1 bool + goPath string + go1Path string + goTipPath string + goTipDir string + goTipHash string + lastErr error + NameToCmd map[string]string + + historylk sync.Mutex + currentTestSuite *testSuite + history History ) +var TaskNames = []string{ + "prepRepo1", + "prepRepo2", + "prepRepo3", + "prepRepo4", + "prepRepo5", + "prepRepo6", + "prepRepo7", + "prepRepo8", + "buildGoTip1", + "buildCamli1", + "buildCamli2", + "runCamli", + "hitCamliUi1", + "hitCamliUi2", + "camput1", + "camget1", + "camget2", + "camput2", +} + +var Cmds = []string{ + "hg pull", + "hg update -C default", + "hg --config extensions.purge= purge --all", + "hg log -r tip --template {node}", + "git reset --hard HEAD", + "git clean -Xdf", + "git pull", + "git rev-parse HEAD", + "./make.bash", + "make", + "make presubmit", + "./dev-server --wipe --mysql", + "http://localhost:3179/ui/", + "http://localhost:3179/ui/new/home.html", + "./dev-camput file --permanode " + testFile, + "./dev-camget ", + "./dev-camget ", + "./dev-camput file --filenodes pkg", +} + func usage() { fmt.Fprintf(os.Stderr, "\t buildbot \n") flag.PrintDefaults() os.Exit(2) } +type debugger struct { + lg *log.Logger +} + +func (dbg *debugger) Printf(format string, v ...interface{}) { + if *verbose { + dbg.lg.Printf(format, v...) + } +} + +func (dbg *debugger) Println(v ...interface{}) { + if v == nil { + return + } + if *verbose { + dbg.lg.Println(v...) + } +} + func setup() { var err error - if _, err := os.Stat(*gotipdir); err != nil { - log.Fatalf("Problem with Go tip dir: %v", err) - } defaultDir, err = os.Getwd() if err != nil { log.Fatal(err) } - interval = time.Second + defaultPATH = os.Getenv("PATH") + if defaultPATH == "" { + log.Fatal("PATH not set") + } dbg = &debugger{log.New(os.Stderr, "", log.LstdFlags)} + NameToCmd = make(map[string]string, 1) + for k, v := range TaskNames { + NameToCmd[v] = Cmds[k] + } - // setup temp gopath - gopath, err = ioutil.TempDir("", "camlibot-gopath") + // check go tip tree + goTipDir, err = filepath.Abs(*gotipdir) + if err != nil { + log.Fatalf("Problem with Go tip dir: %v", err) + } + if _, err := os.Stat(goTipDir); err != nil { + log.Fatalf("Problem with Go tip dir: %v", err) + } + + // setup temp gopath(s) + goTipPath, err = ioutil.TempDir("", "camlibot-gotippath") if err != nil { log.Fatalf("problem with tempdir: %v", err) } - srcDir := filepath.Join(gopath, "src") + srcDir := filepath.Join(goTipPath, "src") err = os.Mkdir(srcDir, 0755) if err != nil { log.Fatalf("problem with src dir: %v", err) } - err = os.Setenv("GOPATH", gopath) + go1Path, err = ioutil.TempDir("", "camlibot-go1path") + if err != nil { + log.Fatalf("problem with tempdir: %v", err) + } + srcDir = filepath.Join(go1Path, "src") + err = os.Mkdir(srcDir, 0755) + if err != nil { + log.Fatalf("problem with src dir: %v", err) + } + goPath = go1Path + err = os.Setenv("GOPATH", goPath) if err != nil { log.Fatalf("problem setting up GOPATH: %v", err) } @@ -87,55 +211,56 @@ func setup() { log.Fatalf("problem with cache dir: %v", err) } } - setCurrentTask("setup", - "git clone https://camlistore.org/r/p/camlistore "+camliRoot) - _, err = runCmd(getCurrentTask().cmd) + setCurrentTask("git clone https://camlistore.org/r/p/camlistore " + camliRoot) + _, err = runCmd(getCurrentTask().Cmd) if err != nil { log.Fatalf("problem with git clone: %v", err) } } else { - // get cache, purge it, update it. + // get cache err = os.Rename(cachedCamliRoot, camliRoot) if err != nil { log.Fatal(err) } - err := os.Chdir(camliRoot) - if err != nil { - log.Fatal(err) - } - defer func() { - err := os.Chdir(defaultDir) - if err != nil { - panic(err) - } - }() - setCurrentTask("setup", "git reset --hard HEAD") - _, err = runCmd(getCurrentTask().cmd) - if err != nil { - log.Fatalf("problem with git reset: %v", err) - } - setCurrentTask("setup", "git clean -Xdf") - _, err = runCmd(getCurrentTask().cmd) - if err != nil { - log.Fatalf("problem with git clean: %v", err) - } - setCurrentTask("setup", "git pull") - _, err = runCmd(getCurrentTask().cmd) - if err != nil { - log.Fatalf("problem with git pull: %v", err) - } } } func cleanTempGopath() { + if *nocleanup { + return + } err := os.Rename(camliRoot, cachedCamliRoot) if err != nil { panic(err) } - err = os.RemoveAll(gopath) + err = os.RemoveAll(go1Path) if err != nil { panic(err) } + err = os.RemoveAll(goTipPath) + if err != nil { + panic(err) + } +} + +func switchGoPath(isTip bool) { + if isTip { + goPath = goTipPath + } else { + goPath = go1Path + } + newRoot := filepath.Join(goPath, "src", "camlistore.org") + if newRoot != camliRoot { + err := os.Rename(camliRoot, newRoot) + if err != nil { + panic(err) + } + camliRoot = newRoot + err = os.Setenv("GOPATH", goPath) + if err != nil { + log.Fatalf("problem setting up GOPATH: %v", err) + } + } } func handleSignals() { @@ -150,7 +275,7 @@ func handleSignals() { } switch sysSig { case syscall.SIGINT, syscall.SIGTERM: - log.Printf("SIGINT: cleaning up %v before terminating", gopath) + log.Printf("SIGINT: cleaning up %v before terminating", goPath) cleanTempGopath() os.Exit(0) default: @@ -159,61 +284,185 @@ func handleSignals() { } } -// TODO(mpl): history of previous commands and their durations -func reportStatus(w http.ResponseWriter, r *http.Request) { - taskStr := "Current task: " + getCurrentTask().name - errStr := "" - if lastErr != nil { - errStr = fmt.Sprintf("Last error: %v", lastErr) +func prepRepo() { + doBuildGo, doBuildCamli0, doBuildCamli1 = false, false, false + if *fast { + if currentTestSuite == nil { + currentTestSuite = &testSuite{ + Run: make([]*task, 0, 1), + GoHash: goTipHash, + CamliHash: camliHeadHash, + IsTip: false, + } + } + return } - fmt.Fprintf(w, "%s\n%s", taskStr, errStr) -} -func increaseInterval() { - if interval < maxInterval { - interval = interval * 2 + // gotip + err := os.Chdir(goTipDir) + if err != nil { + log.Fatal(err) + } + defer func() { + err := os.Chdir(defaultDir) + if err != nil { + panic(err) + } + }() + tasks := []string{ + NameToCmd["prepRepo1"], + NameToCmd["prepRepo2"], + NameToCmd["prepRepo4"], + } + hash := "" + for _, v := range tasks { + setCurrentTask(v) + out, err := runCmd(v) + if err != nil { + if v == NameToCmd["prepRepo1"] { + dbg.Printf("Go repo could not be updated: %v\n", err) + continue + } + log.Fatal(err) + } + hash = strings.TrimRight(out, "\n") + } + dbg.Println("previous head in go tree: " + goTipHash) + dbg.Println("current head in go tree: " + hash) + if hash != "" && hash != goTipHash { + goTipHash = hash + doBuildGo = true + dbg.Println("Changes in go tree detected, go tip will be rebuilt") + } + + // camli + err = os.Chdir(camliRoot) + if err != nil { + log.Fatal(err) + } + tasks = []string{ + NameToCmd["prepRepo5"], + NameToCmd["prepRepo6"], + NameToCmd["prepRepo7"], + NameToCmd["prepRepo8"], + } + hash = "" + for _, v := range tasks { + setCurrentTask(v) + out, err := runCmd(v) + if err != nil { + if v == NameToCmd["prepRepo7"] { + dbg.Printf("Camli repo could not be updated: %v\n", err) + continue + } + log.Fatal(err) + } + hash = strings.TrimRight(out, "\n") + } + dbg.Println("previous head in camli tree: " + camliHeadHash) + dbg.Println("current head in camli tree: " + hash) + if hash != "" && hash != camliHeadHash { + camliHeadHash = hash + doBuildCamli0, doBuildCamli1 = true, true + dbg.Println("Changes in camli tree detected, camlistore will be rebuilt") } } -type debugger struct { - lg *log.Logger -} - -func (dbg *debugger) Printf(format string, v ...interface{}) { - if *debug { - dbg.lg.Printf(format, v) +func addToPATH(gobin string) { + splitter := ":" + switch runtime.GOOS { + case "windows": + splitter = ";" + case "plan9": + panic("unsupported") + } + p := gobin + splitter + defaultPATH + err := os.Setenv("PATH", p) + if err != nil { + log.Fatalf("Could not set PATH to %v: %v", p, err) } } -func (dbg *debugger) Println(v ...interface{}) { - if *debug { - dbg.lg.Println(v) +func restorePATH() { + err := os.Setenv("PATH", defaultPATH) + if err != nil { + log.Fatalf("Could not set PATH to %v: %v", defaultPATH, err) } } type task struct { - lk sync.Mutex - name string + lk sync.Mutex // actual command that is run for this task. it includes the command and // all the arguments, space separated. shell metacharacters not supported. - cmd string + Cmd string + start time.Time + Duration time.Duration } func getCurrentTask() *task { currentTask.lk.Lock() defer currentTask.lk.Unlock() - return &task{name: currentTask.name, cmd: currentTask.cmd} + return &task{Cmd: currentTask.Cmd, start: currentTask.start} } -func setCurrentTask(name, cmd string) { +func setCurrentTask(cmd string) { currentTask.lk.Lock() defer currentTask.lk.Unlock() - currentTask.name = name - currentTask.cmd = cmd + currentTask.Cmd = cmd + currentTask.start = time.Now() +} + +type testSuite struct { + Run []*task + CamliHash string + GoHash string + failedTask int + Err error + Start time.Time + IsTip bool +} + +type History [][2]*testSuite + +func addRun(tsk *task, tskErr error) { + historylk.Lock() + defer historylk.Unlock() + + duration := time.Now().Sub(tsk.start) + tsk.Duration = duration + if len(currentTestSuite.Run) == 0 { + currentTestSuite.Start = tsk.start + } + if tskErr != nil && currentTestSuite.Err == nil { + currentTestSuite.Err = tskErr + currentTestSuite.failedTask = len(currentTestSuite.Run) + } + currentTestSuite.Run = append(currentTestSuite.Run, tsk) +} + +func addTestSuite() { + historylk.Lock() + defer historylk.Unlock() + if len(currentTestSuite.Run) < 1 { + return + } + if !currentTestSuite.IsTip { + // Go 1 + entry := [2]*testSuite{currentTestSuite, nil} + history = append(history, entry) + } else { + // Go tip + entry := [2]*testSuite{history[len(history)-1][0], currentTestSuite} + if len(history) > historySize { + history = append(history[1:historySize], entry) + } else { + history[len(history)-1] = entry + } + } } func runCmd(tsk string) (string, error) { - dbg.Println(getCurrentTask().cmd) + dbg.Println(getCurrentTask().Cmd) fields := strings.Fields(tsk) var args []string if len(fields) > 1 { @@ -225,16 +474,30 @@ func runCmd(tsk string) (string, error) { cmd.Stderr = &stderr err := cmd.Run() if err != nil { - return "", fmt.Errorf("error with %v: %v\n", getCurrentTask().cmd, stderr.String()) + return "", fmt.Errorf("error with %v: %v\n", getCurrentTask().Cmd, stderr.String()) } return stdout.String(), nil } +func gofast(n int) { + for i := 0; i < n; i++ { + tsk := &task{start: time.Now()} + fail := rand.Intn(10) + if fail < 1 { + dbg.Println("random fail") + addRun(tsk, fmt.Errorf("random fail")) + continue + } + addRun(tsk, nil) + } +} + func buildGoTip() error { - if *nobuild { + if *fast { + gofast(3) return nil } - err := os.Chdir(*gotipdir) + err := os.Chdir(goTipDir) if err != nil { log.Fatal(err) } @@ -246,36 +509,44 @@ func buildGoTip() error { }() tasks := []string{ - "hg update -C default", - "hg --config extensions.purge= purge --all", - "./make.bash", + NameToCmd["prepRepo3"], } - for i := 0; i < 2; i++ { - setCurrentTask("building go tip", tasks[i]) - _, err = runCmd(getCurrentTask().cmd) + for _, v := range tasks { + setCurrentTask(v) + tsk := getCurrentTask() + _, err := runCmd(tsk.Cmd) + addRun(tsk, err) if err != nil { return err } } - err = os.Chdir("src") + err = os.Chdir(filepath.Join(goTipDir, "src")) if err != nil { log.Fatal(err) } - setCurrentTask("building go tip", tasks[2]) - _, err = runCmd(getCurrentTask().cmd) + defer func() { + err := os.Chdir(defaultDir) + if err != nil { + panic(err) + } + }() + + setCurrentTask(NameToCmd["buildGoTip1"]) + tsk := getCurrentTask() + _, err = runCmd(tsk.Cmd) + addRun(tsk, err) if err != nil { return err } return nil } -func buildCamli() error { - if e := os.Getenv("GOARCH"); e == "" { - log.Fatal("GOARCH not set") - } - if e := os.Getenv("GOOS"); e == "" { - log.Fatal("GOOS not set") +func buildCamli(isTip bool) error { + if *fast { + gofast(5) + return nil } + switchGoPath(isTip) err := os.Chdir(camliRoot) if err != nil { log.Fatal(err) @@ -287,21 +558,65 @@ func buildCamli() error { } }() - tasks := []string{ - "make", - "make presubmit", + if doBuildGo && isTip { + if *verbose { + dbg.Println("cleaning up gopath/pkg and gopath/bin for full rebuild") + } + // erase and rebuild everything + pkgdir := filepath.Join("..", "..", "pkg") + err := os.RemoveAll(pkgdir) + if err != nil { + log.Fatalf("failed to remove %v: %v", pkgdir, err) + } + bindir := filepath.Join("..", "..", "bin") + err = os.RemoveAll(bindir) + if err != nil { + log.Fatalf("failed to remove %v: %v", bindir, err) + } } + + if *verbose { + setCurrentTask("go version") + out, err := runCmd(getCurrentTask().Cmd) + if err != nil { + log.Fatalf("failed to run 'go version': %v", err) + } + out = strings.TrimRight(out, "\n") + dbg.Printf("Building camlistore in %v with: %v\n", goPath, out) + } + + tasks := []string{} + if doBuildCamli0 || doBuildCamli1 || (doBuildGo && isTip) { + tasks = append(tasks, NameToCmd["buildCamli1"]) + } + tasks = append(tasks, NameToCmd["buildCamli2"]) for _, v := range tasks { - setCurrentTask("building camlistore", v) - _, err := runCmd(v) + setCurrentTask(v) + tsk := getCurrentTask() + _, err := runCmd(tsk.Cmd) + addRun(tsk, err) if err != nil { return err } } + if doBuildGo && isTip { + doBuildGo = false + doBuildCamli1 = false + } + if isTip { + doBuildCamli1 = false + } else { + doBuildCamli0 = false + } + return nil } -func runCamli() (*os.Process, error) { +func runCamli(cerr chan error) (*os.Process, error) { + if *fast { + gofast(1) + return nil, nil + } err := os.Chdir(camliRoot) if err != nil { log.Fatal(err) @@ -313,32 +628,36 @@ func runCamli() (*os.Process, error) { } }() - setCurrentTask("starting camlistore", "./dev-server --wipe --mysql") - dbg.Println(getCurrentTask().cmd) - fields := strings.Fields(getCurrentTask().cmd) + setCurrentTask(NameToCmd["runCamli"]) + dbg.Println(getCurrentTask().Cmd) + fields := strings.Fields(getCurrentTask().Cmd) args := fields[1:] cmd := exec.Command(fields[0], args...) var stderr bytes.Buffer cmd.Stderr = &stderr - cerr := make(chan error) go func() { cerr <- cmd.Run() }() time.Sleep(warmup) select { - case <-cerr: + case err := <-cerr: dbg.Println("dev server DEAD") - return nil, fmt.Errorf("%v: %v\n", getCurrentTask().cmd, "camlistored terminated prematurely") + tsk := getCurrentTask() + addRun(tsk, err) + return nil, fmt.Errorf("%v: %v\n", tsk.Cmd, "camlistored terminated prematurely") default: dbg.Println("dev server OK") + addRun(getCurrentTask(), nil) break } return cmd.Process, nil } -// TODO(mpl): maybe killall or such to be sure func killCamli(proc *os.Process) { + if *fast { + return + } dbg.Println("killing dev server") err := proc.Kill() if err != nil { @@ -348,35 +667,44 @@ func killCamli(proc *os.Process) { } func hitURL(url string) error { - setCurrentTask("Hitting camli", fmt.Sprintf("http.Get(\"%s\")", url)) - dbg.Println(getCurrentTask().cmd) + setCurrentTask(fmt.Sprintf("http.Get(\"%s\")", url)) + dbg.Println(getCurrentTask().Cmd) resp, err := http.Get(url) if err != nil { - return fmt.Errorf("%v: %v\n", getCurrentTask().cmd, err) + return fmt.Errorf("%v: %v\n", getCurrentTask().Cmd, err) } defer resp.Body.Close() if resp.StatusCode != 200 { - return fmt.Errorf("%v, got StatusCode: %d\n", getCurrentTask().cmd, resp.StatusCode) + return fmt.Errorf("%v, got StatusCode: %d\n", getCurrentTask().Cmd, resp.StatusCode) } return nil } -func hitCamli() error { - urls := []string{ - "http://localhost:3179/ui/", - "http://localhost:3179/ui/new/home.html", +func hitCamliUi() error { + if *fast { + gofast(2) + return nil } + urls := []string{ + NameToCmd["hitCamliUi1"], + NameToCmd["hitCamliUi2"], + } + var err error for _, v := range urls { - err := hitURL(v) - if err != nil { - return err + lerr := hitURL(v) + addRun(getCurrentTask(), lerr) + if lerr != nil { + err = lerr } } - return nil + return err } func camliClients() error { - fileName := "AUTHORS" + if *fast { + gofast(3) + return nil + } err := os.Chdir(camliRoot) if err != nil { log.Fatal(err) @@ -389,8 +717,10 @@ func camliClients() error { }() // push the file to camli - setCurrentTask("camput", "./dev-camput file --permanode "+fileName) - out, err := runCmd(getCurrentTask().cmd) + setCurrentTask(NameToCmd["camput1"]) + tsk := getCurrentTask() + out, err := runCmd(tsk.Cmd) + addRun(tsk, err) if err != nil { return err } @@ -398,42 +728,50 @@ func camliClients() error { firstSHA1 := regexp.MustCompile(`.*(sha1-[a-zA-Z0-9]+)\nsha1-[a-zA-Z0-9]+\nsha1-[a-zA-Z0-9]+\n.*`) m := firstSHA1.FindStringSubmatch(out) if m == nil { - return fmt.Errorf("%v: unexpected camput output\n", getCurrentTask().cmd) + return fmt.Errorf("%v: unexpected camput output\n", getCurrentTask().Cmd) } blobref := m[1] // get the file's json to find out the file's blobref - setCurrentTask("camget", "./dev-camget "+blobref) - out, err = runCmd(getCurrentTask().cmd) + setCurrentTask(NameToCmd["camget1"] + blobref) + tsk = getCurrentTask() + out, err = runCmd(tsk.Cmd) + addRun(tsk, err) if err != nil { return err } blobrefPattern := regexp.MustCompile(`"blobRef": "(sha1-[a-zA-Z0-9]+)",\n.*`) m = blobrefPattern.FindStringSubmatch(out) if m == nil { - return fmt.Errorf("%v: unexpected camget output\n", getCurrentTask().cmd) + return fmt.Errorf("%v: unexpected camget output\n", getCurrentTask().Cmd) } blobref = m[1] // finally, get the file back - setCurrentTask("camget", "./dev-camget "+blobref) - out, err = runCmd(getCurrentTask().cmd) + setCurrentTask(NameToCmd["camget2"] + blobref) + tsk = getCurrentTask() + out, err = runCmd(tsk.Cmd) + addRun(tsk, err) if err != nil { return err } // and compare it with the original - fileContents, err := ioutil.ReadFile(fileName) + fileContents, err := ioutil.ReadFile(testFile) if err != nil { - log.Fatalf("Could not read %v: %v", fileName, err) + log.Fatalf("Could not read %v: %v", testFile, err) } if string(fileContents) != out { - return fmt.Errorf("%v: contents fetched with camget differ from %v contents", getCurrentTask().cmd, fileName) + return fmt.Errorf("%v: contents fetched with camget differ from %v contents", getCurrentTask().Cmd, testFile) } return nil } func hitMySQL() error { + if *fast { + gofast(1) + return nil + } err := os.Chdir(camliRoot) if err != nil { log.Fatal(err) @@ -445,14 +783,25 @@ func hitMySQL() error { } }() // upload the full camli pkg tree - setCurrentTask("mysql", "./dev-camput file --filenodes "+filepath.Join(camliRoot, "pkg")) - _, err = runCmd(getCurrentTask().cmd) + setCurrentTask(NameToCmd["camput2"]) + tsk := getCurrentTask() + _, err = runCmd(tsk.Cmd) + addRun(tsk, err) if err != nil { return err } return nil } +func handleErr(err error, proc *os.Process) { + lastErr = err + dbg.Printf("%v", err) + if proc != nil { + killCamli(proc) + } + addTestSuite() +} + func main() { flag.Usage = usage flag.Parse() @@ -464,60 +813,415 @@ func main() { defer cleanTempGopath() go handleSignals() - http.HandleFunc("/", reportStatus) + http.HandleFunc(okPrefix, okHandler) + http.HandleFunc(failPrefix, failHandler) + http.HandleFunc(currentPrefix, progressHandler) + http.HandleFunc("/", statusHandler) go http.ListenAndServe(*host, nil) + tryCount := 0 for { - setCurrentTask("sleeping", fmt.Sprintf("time.Sleep(%d)", interval)) - dbg.Println(getCurrentTask().cmd) - time.Sleep(interval) - increaseInterval() - err := buildGoTip() - if err != nil { - lastErr = err - dbg.Printf("%v", lastErr) - continue - } - err = buildCamli() - if err != nil { - lastErr = err - dbg.Printf("%v", lastErr) - continue - } - proc, err := runCamli() - if err != nil { - lastErr = err - dbg.Printf("%v", lastErr) - continue - } - err = hitCamli() - if err != nil { - lastErr = err - dbg.Printf("%v", lastErr) - killCamli(proc) - continue - } - err = camliClients() - if err != nil { - lastErr = err - dbg.Printf("%v", lastErr) - killCamli(proc) - continue - } - err = hitMySQL() - if err != nil { - lastErr = err - dbg.Printf("%v", lastErr) - killCamli(proc) - continue + if lastErr == nil || tryCount > 1 { + tryCount = 0 + lastErr = nil + prepRepo() } + if doBuildGo || doBuildCamli0 || doBuildCamli1 || lastErr != nil { + for _, isTip := range [2]bool{false, true} { + restorePATH() + currentTestSuite = &testSuite{ + Run: make([]*task, 0, 1), + GoHash: goTipHash, + CamliHash: camliHeadHash, + IsTip: isTip, + } + if isTip { + addToPATH(filepath.Join(goTipDir, "bin")) + if doBuildGo { + err := buildGoTip() + if err != nil { + handleErr(err, nil) + continue + } + } + } - dbg.Println("All good.") - killCamli(proc) - lastErr = nil - interval = time.Second - setCurrentTask("sleeping", fmt.Sprintf("time.Sleep(%d)", maxInterval)) - dbg.Println(getCurrentTask().cmd) - time.Sleep(maxInterval) + err := buildCamli(isTip) + if err != nil { + handleErr(err, nil) + continue + } + cerr := make(chan error) + // TODO(mpl): how/when to close this chan without getting a panic? + // this is important because it is probably a mem leak right now. + proc, err := runCamli(cerr) + if err != nil { + handleErr(err, nil) + continue + } + err = hitCamliUi() + if err != nil { + handleErr(err, proc) + continue + } + err = camliClients() + if err != nil { + handleErr(err, proc) + continue + } + err = hitMySQL() + if err != nil { + handleErr(err, proc) + continue + } + + dbg.Println("All good.") + killCamli(proc) + addTestSuite() + } + tryCount++ + } + setCurrentTask(fmt.Sprintf("time.Sleep(%d)", interval)) + dbg.Println(getCurrentTask().Cmd) + time.Sleep(interval) } } + +var ( + okPrefix = "/ok/" + failPrefix = "/fail/" + currentPrefix = "/current" + statusTpl = template.Must(template.New("status").Funcs(tmplFuncs).Parse(reportHTML)) + taskTpl = template.Must(template.New("task").Parse(taskHTML)) + testSuiteTpl = template.Must(template.New("ok").Parse(testSuiteHTML)) +) + +var tmplFuncs = template.FuncMap{ + "camliRepoURL": camliRepoURL, + "goRepoURL": goRepoURL, + "shortHash": shortHash, +} + +// unlocked; history needs to be protected from the caller. +func getPastTestSuite(key string) (*testSuite, error) { + idx := 0 + date := "" + if strings.HasPrefix(key, "gotip-") { + date = strings.Replace(key, "gotip-", "", -1) + idx = 1 + } else { + date = strings.Replace(key, "go1-", "", -1) + } + for _, v := range history { + if v[idx].Start.String() == date { + return v[idx], nil + } + } + return nil, fmt.Errorf("%v not found in history", date) +} + +type progressData struct { + Ts *testSuite + Current string +} + +func okHandler(w http.ResponseWriter, r *http.Request) { + t := strings.Replace(r.URL.Path, okPrefix, "", -1) + historylk.Lock() + defer historylk.Unlock() + ts, err := getPastTestSuite(t) + if err != nil { + http.NotFound(w, r) + return + } + dat := &progressData{ + Ts: ts, + } + err = testSuiteTpl.Execute(w, dat) + if err != nil { + log.Printf("ok template: %v\n", err) + } +} + +func progressHandler(w http.ResponseWriter, r *http.Request) { + historylk.Lock() + defer historylk.Unlock() + dat := &progressData{ + Ts: currentTestSuite, + Current: getCurrentTask().Cmd, + } + err := testSuiteTpl.Execute(w, dat) + if err != nil { + log.Printf("progress template: %v\n", err) + } +} + +type TaskReport struct { + Cmd string + Err error +} + +func failHandler(w http.ResponseWriter, r *http.Request) { + t := strings.Replace(r.URL.Path, failPrefix, "", -1) + historylk.Lock() + defer historylk.Unlock() + ts, err := getPastTestSuite(t) + if err != nil { + http.NotFound(w, r) + return + } + taskReport := &TaskReport{Cmd: ts.Run[ts.failedTask].Cmd, Err: ts.Err} + err = taskTpl.Execute(w, taskReport) + if err != nil { + log.Printf("fail template: %v\n", err) + } +} + +type status struct { + Hs History + Ts *testSuite +} + +func statusHandler(w http.ResponseWriter, r *http.Request) { + historylk.Lock() + defer historylk.Unlock() + stat := &status{ + Hs: history, + Ts: currentTestSuite, + } + err := statusTpl.Execute(w, stat) + if err != nil { + log.Printf("status template: %v\n", err) + } +} + +// shortHash returns a short version of a hash. +func shortHash(hash string) string { + if len(hash) > 12 { + hash = hash[:12] + } + return hash +} + +func goRepoURL(hash string) string { + return "https://code.google.com/p/go/source/detail?r=" + hash +} + +func camliRepoURL(hash string) string { + return "http://camlistore.org/code/?p=camlistore.git;a=commit;h=" + hash +} + +// style inspired from $GOROOT/misc/dashboard/app/build/ui.html +var styleHTML = ` + +` + +var reportHTML = ` + + + + Camlistore tests Dashboard` + + styleHTML + ` + + + +

Camlibot status

+ + + + + + + + + + + + + + + + + + {{if .Hs}} + {{range $tss := .Hs}} + + {{range $k, $ts := $tss}} + {{if $k | not}} + + + + + {{else}} + + {{end}} + {{end}} + + {{end}} + {{end}} + {{if .Ts}} + {{if .Ts.IsTip | not}} + + + + + + + {{end}} + {{end}} +
 Go tip hashCamli HEAD hashGo1Gotip
{{$ts.Start}} + {{shortHash $ts.GoHash}} + + {{shortHash $ts.CamliHash}} + + {{if $ts.Err}} + fail + {{else}} + ok + {{end}} + + {{if $ts}} + {{if $ts.Err}} + fail + {{else}} + ok + {{end}} + {{else}} + In progress + {{end}} +
{{.Ts.Start}} + {{shortHash .Ts.GoHash}} + + {{shortHash .Ts.CamliHash}} + + In progress +
+ + + +` + +var testSuiteHTML = ` + + + + Camlistore tests Dashboard` + + styleHTML + ` + + + {{if .Ts}} +

Testsuite for {{if .Ts.IsTip}}Go tip{{else}}Go 1{{end}} at {{.Ts.Start}}

+ + + + + + + + + + + {{range $k, $v := .Ts.Run}} + + + + + {{end}} + {{if .Current}} + + + + + {{end}} +
StepDuration
{{$v.Cmd}}{{$v.Duration}}
{{.Current}}(running...)
+ {{end}} + + +` + +var taskHTML = ` + + + + Camlistore tests Dashboard + + +{{if .Cmd}} +

Command:

+

{{.Cmd}}

+{{end}} +{{if .Err}} +

Error:

+

{{.Err}}

+{{end}} + + +`