From 74bb79e63b9aca5fdb11ebb6e204ea33eb301a63 Mon Sep 17 00:00:00 2001
From: mpl <mathieu.lonjaret@gmail.com>
Date: Fri, 18 Oct 2013 19:13:59 +0200
Subject: [PATCH] buildbot: broke into a master and a builder bot

This change is the first major step towards more improvements to the
buildbot. It cleans up various details, but most notably:
-it now uses a task as the basic data structure, to improve readability
and maintainability
-it breaks the bot into a master bot and a builder bot.
-it allows a master to receive reports from remote bots, and the
status page now aggregates all those reports.
-it allows a builder bot to send its reports to several master bots.
-the master bot uses whatever go version is installed to build the
builder bot, or the downloaded go tip if none is found.

The master bot is in charge of
1) monitoring the changes to the Camlistore and Go repos,
2) recompiling builder bot, which allows to add some new tests to the
test suite without having to interrupt the master bot.
3) starting a builder bot instance
4) waiting for the builder bot report and/or polling the builder bot
for its progress
5) answering the requests to display the various status pages.

http://camlistore.org/issue/185

Change-Id: I46a0b8fabbebf76b0c3ed04fd2ee73362d565cf7
---
 misc/buildbot/bot.go                   | 1385 ------------------------
 misc/buildbot/builder/builder.go       |  964 +++++++++++++++++
 misc/buildbot/{ => master}/bot_test.go |    0
 misc/buildbot/master/master.go         | 1109 +++++++++++++++++++
 4 files changed, 2073 insertions(+), 1385 deletions(-)
 delete mode 100644 misc/buildbot/bot.go
 create mode 100644 misc/buildbot/builder/builder.go
 rename misc/buildbot/{ => master}/bot_test.go (100%)
 create mode 100644 misc/buildbot/master/master.go

diff --git a/misc/buildbot/bot.go b/misc/buildbot/bot.go
deleted file mode 100644
index a9879569c..000000000
--- a/misc/buildbot/bot.go
+++ /dev/null
@@ -1,1385 +0,0 @@
-/*
-Copyright 2012 The Camlistore Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-     http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-// The buildbot binary is Camlistore's continuous builder.
-package main
-
-import (
-	"bytes"
-	"flag"
-	"fmt"
-	"html/template"
-	"io"
-	"io/ioutil"
-	"log"
-	"math/rand"
-	"net/http"
-	"os"
-	"os/exec"
-	"os/signal"
-	"path/filepath"
-	"regexp"
-	"runtime"
-	"strings"
-	"sync"
-	"syscall"
-	"time"
-)
-
-const (
-	interval      = 60 * time.Second // polling frequency
-	warmup        = 60 * time.Second // duration before we test if devcam server has started properly
-	historySize   = 30
-	maxStderrSize = 1 << 20 // Keep last 1 MB of logging.
-)
-
-var (
-	fast      = flag.Bool("fast", false, "run dummy steps instead of the actual tasks, for debugging")
-	gotipdir  = flag.String("gotip", "./go", "path to the Go tip tree")
-	help      = flag.Bool("h", false, "show this help")
-	host      = flag.String("host", "0.0.0.0:8080", "listening hostname and port")
-	nocleanup = flag.Bool("nocleanup", false, "do not clean up the tmp gopath when failing")
-	verbose   = flag.Bool("verbose", false, "print what's going on")
-)
-
-var (
-	testFile        = []string{"AUTHORS", "CONTRIBUTORS"}
-	camliHeadHash   string
-	cachedCamliRoot string
-	camliRoot       string
-	camputCacheDir  string
-	currentTask     task
-	dbg             *debugger
-	defaultDir      string
-	defaultPATH     string
-	// doBuildCamli0 is for when we're in the go 1 test suite, and
-	// doBuildCamli1 for the go tip test suite. They are set when
-	// the camli source tree has been updated and indicate that
-	// 'make' should be run.
-	// doBuildGo indicates the go source tree has changed, hence
-	// go tip should be rebuilt, and in the go tip test suite case
-	// camlistore should be cleaned (pkg and bin) and rebuilt.
-	doBuildGo, doBuildCamli0, doBuildCamli1 bool
-	goPath                                  string
-	go1Path                                 string
-	goTipPath                               string
-	goTipDir                                string
-	goTipHash                               string
-	lastErr                                 error
-
-	historylk        sync.Mutex
-	currentTestSuite *testSuite
-	history          History
-
-	// For "If-Modified-Since" requests on the status page.
-	// Updated every time a new test suite starts or ends.
-	lastModified time.Time
-
-	// Override the os.Stderr used by the default logger so we can provide
-	// more debug info on status page.
-	logStderr = newLockedBuffer()
-)
-
-// lockedBuffer protects all Write calls with a mutex.  Users of lockedBuffer
-// must wrap any calls to Bytes, and use of the resulting slice with calls to
-// Lock/Unlock.
-type lockedBuffer struct {
-	sync.Mutex // guards ringBuffer
-	*ringBuffer
-}
-
-func newLockedBuffer() *lockedBuffer {
-	return &lockedBuffer{ringBuffer: newRingBuffer(maxStderrSize)}
-}
-
-func (lb *lockedBuffer) Write(b []byte) (int, error) {
-	lb.Lock()
-	defer lb.Unlock()
-	return lb.ringBuffer.Write(b)
-}
-
-type ringBuffer struct {
-	buf []byte
-	off int // End of ring buffer.
-	l   int // Length of ring buffer filled.
-}
-
-func newRingBuffer(maxSize int) *ringBuffer {
-	return &ringBuffer{
-		buf: make([]byte, maxSize),
-	}
-}
-
-func (rb *ringBuffer) Bytes() []byte {
-	if (rb.off - rb.l) >= 0 {
-		// Partially full buffer with no wrap.
-		return rb.buf[rb.off-rb.l : rb.off]
-	}
-
-	// Buffer has been wrapped, copy second half then first half.
-	start := rb.off - rb.l
-	if start < 0 {
-		start = rb.off
-	}
-	b := make([]byte, 0, cap(rb.buf))
-	b = append(b, rb.buf[start:]...)
-	b = append(b, rb.buf[:start]...)
-	return b
-}
-
-func (rb *ringBuffer) Write(buf []byte) (int, error) {
-	ringLen := cap(rb.buf)
-	for i, b := range buf {
-		rb.buf[(rb.off+i)%ringLen] = b
-	}
-	rb.off = (rb.off + len(buf)) % ringLen
-	rb.l = rb.l + len(buf)
-	if rb.l > ringLen {
-		rb.l = ringLen
-	}
-	return len(buf), nil
-}
-
-var NameToCmd = map[string]string{
-	"prepRepo1":   "hg pull",
-	"prepRepo2":   "hg update -C default",
-	"prepRepo3":   "hg --config extensions.purge= purge --all",
-	"prepRepo4":   "hg log -r tip --template {node}",
-	"prepRepo5":   "git reset --hard HEAD",
-	"prepRepo6":   "git clean -Xdf",
-	"prepRepo7":   "git pull",
-	"prepRepo8":   "git rev-parse HEAD",
-	"buildGoTip1": "./make.bash",
-	"buildCamli1": "go run make.go -v",
-	"buildCamli2": "go build -o devcam ./dev/devcam/",
-	"buildCamli3": "./devcam test",
-	"runCamli":    "./devcam server --wipe --mysql",
-	"hitCamliUi1": "http://localhost:3179/ui/",
-	"camget":      "./devcam get ",
-	"camput1":     "./devcam put file --permanode " + testFile[0],
-	"camput2":     "./devcam put file --vivify " + testFile[1],
-	"camput3":     "./devcam put file --filenodes pkg",
-}
-
-func usage() {
-	fmt.Fprintf(os.Stderr, "\t buildbot \n")
-	flag.PrintDefaults()
-	os.Exit(2)
-}
-
-type debugger struct {
-	lg *log.Logger
-}
-
-func (dbg *debugger) Printf(format string, v ...interface{}) {
-	if *verbose {
-		dbg.lg.Printf(format, v...)
-	}
-}
-
-func (dbg *debugger) Println(v ...interface{}) {
-	if v == nil {
-		return
-	}
-	if *verbose {
-		dbg.lg.Println(v...)
-	}
-}
-
-func setup() {
-	// Install custom stderr for display in status webpage.
-	w := io.MultiWriter(logStderr, os.Stderr)
-	log.SetOutput(w)
-
-	var err error
-	defaultDir, err = os.Getwd()
-	if err != nil {
-		log.Fatal(err)
-	}
-	defaultPATH = os.Getenv("PATH")
-	if defaultPATH == "" {
-		log.Fatal("PATH not set")
-	}
-	dbg = &debugger{log.New(w, "", log.LstdFlags)}
-
-	// check go tip tree
-	goTipDir, err = filepath.Abs(*gotipdir)
-	if err != nil {
-		log.Fatalf("Problem with Go tip dir: %v", err)
-	}
-	if _, err := os.Stat(goTipDir); err != nil {
-		log.Fatalf("Problem with Go tip dir: %v", err)
-	}
-
-	// setup temp gopath(s)
-	goTipPath, err = ioutil.TempDir("", "camlibot-gotippath")
-	if err != nil {
-		log.Fatalf("problem with tempdir: %v", err)
-	}
-	srcDir := filepath.Join(goTipPath, "src")
-	err = os.Mkdir(srcDir, 0755)
-	if err != nil {
-		log.Fatalf("problem with src dir: %v", err)
-	}
-	go1Path, err = ioutil.TempDir("", "camlibot-go1path")
-	if err != nil {
-		log.Fatalf("problem with tempdir: %v", err)
-	}
-	srcDir = filepath.Join(go1Path, "src")
-	err = os.Mkdir(srcDir, 0755)
-	if err != nil {
-		log.Fatalf("problem with src dir: %v", err)
-	}
-	goPath = go1Path
-	err = os.Setenv("GOPATH", goPath)
-	if err != nil {
-		log.Fatalf("problem setting up GOPATH: %v", err)
-	}
-
-	// set up the camlistore tree
-	camliRoot = filepath.Join(srcDir, "camlistore.org")
-	cacheDir := filepath.Join(os.TempDir(), "camlibot-cache")
-	cachedCamliRoot = filepath.Join(cacheDir, "camlistore.org")
-	if _, err := os.Stat(cachedCamliRoot); err != nil {
-		// git clone
-		if _, err := os.Stat(cacheDir); err != nil {
-			err = os.Mkdir(cacheDir, 0755)
-			if err != nil {
-				log.Fatalf("problem with cache dir: %v", err)
-			}
-		}
-		setCurrentTask("git clone https://camlistore.googlesource.com/camlistore " + camliRoot)
-		_, err = runCmd(getCurrentTask().Cmd)
-		if err != nil {
-			log.Fatalf("problem with git clone: %v", err)
-		}
-	} else {
-		// get cache
-		err = os.Rename(cachedCamliRoot, camliRoot)
-		if err != nil {
-			log.Fatal(err)
-		}
-	}
-
-	// recording camput cache dir, so we can clean it up fast everytime
-	homeDir := os.Getenv("HOME")
-	if homeDir == "" {
-		log.Fatal("HOME not set")
-	}
-	camputCacheDir = filepath.Join(homeDir, ".cache", "camlistore")
-}
-
-func cleanTempGopath() {
-	if *nocleanup {
-		return
-	}
-	err := os.Rename(camliRoot, cachedCamliRoot)
-	if err != nil {
-		panic(err)
-	}
-	err = os.RemoveAll(go1Path)
-	if err != nil {
-		panic(err)
-	}
-	err = os.RemoveAll(goTipPath)
-	if err != nil {
-		panic(err)
-	}
-}
-
-func switchGoPath(isTip bool) {
-	if isTip {
-		goPath = goTipPath
-	} else {
-		goPath = go1Path
-	}
-	newRoot := filepath.Join(goPath, "src", "camlistore.org")
-	if newRoot != camliRoot {
-		err := os.Rename(camliRoot, newRoot)
-		if err != nil {
-			panic(err)
-		}
-		camliRoot = newRoot
-		err = os.Setenv("GOPATH", goPath)
-		if err != nil {
-			log.Fatalf("problem setting up GOPATH: %v", err)
-		}
-	}
-}
-
-func handleSignals() {
-	c := make(chan os.Signal)
-	sigs := []os.Signal{syscall.SIGINT, syscall.SIGTERM}
-	signal.Notify(c, sigs...)
-	for {
-		sig := <-c
-		sysSig, ok := sig.(syscall.Signal)
-		if !ok {
-			log.Fatal("Not a unix signal")
-		}
-		switch sysSig {
-		case syscall.SIGINT, syscall.SIGTERM:
-			log.Printf("SIGINT: cleaning up %v before terminating", goPath)
-			cleanTempGopath()
-			os.Exit(0)
-		default:
-			panic("should not get other signals here")
-		}
-	}
-}
-
-func prepRepo() {
-	doBuildGo, doBuildCamli0, doBuildCamli1 = false, false, false
-	if *fast {
-		if currentTestSuite == nil {
-			currentTestSuite = &testSuite{
-				Run:       make([]*task, 0, 1),
-				GoHash:    goTipHash,
-				CamliHash: camliHeadHash,
-				IsTip:     false,
-			}
-		}
-		return
-	}
-
-	// gotip
-	err := os.Chdir(goTipDir)
-	if err != nil {
-		log.Fatal(err)
-	}
-	defer func() {
-		err := os.Chdir(defaultDir)
-		if err != nil {
-			panic(err)
-		}
-	}()
-	tasks := []string{
-		NameToCmd["prepRepo1"],
-		NameToCmd["prepRepo2"],
-		NameToCmd["prepRepo4"],
-	}
-	hash := ""
-	for _, v := range tasks {
-		setCurrentTask(v)
-		out, err := runCmd(v)
-		if err != nil {
-			if v == NameToCmd["prepRepo1"] {
-				dbg.Printf("Go repo could not be updated: %v\n", err)
-				continue
-			}
-			log.Fatal(err)
-		}
-		hash = strings.TrimRight(out, "\n")
-	}
-	dbg.Println("previous head in go tree: " + goTipHash)
-	dbg.Println("current head in go tree: " + hash)
-	if hash != "" && hash != goTipHash {
-		goTipHash = hash
-		doBuildGo = true
-		dbg.Println("Changes in go tree detected, go tip will be rebuilt")
-	}
-
-	// camli
-	err = os.Chdir(camliRoot)
-	if err != nil {
-		log.Fatal(err)
-	}
-	tasks = []string{
-		NameToCmd["prepRepo5"],
-		NameToCmd["prepRepo6"],
-		NameToCmd["prepRepo7"],
-		NameToCmd["prepRepo8"],
-	}
-	hash = ""
-	for _, v := range tasks {
-		setCurrentTask(v)
-		out, err := runCmd(v)
-		if err != nil {
-			if v == NameToCmd["prepRepo7"] {
-				dbg.Printf("Camli repo could not be updated: %v\n", err)
-				continue
-			}
-			log.Fatal(err)
-		}
-		hash = strings.TrimRight(out, "\n")
-	}
-	dbg.Println("previous head in camli tree: " + camliHeadHash)
-	dbg.Println("current head in camli tree: " + hash)
-	if hash != "" && hash != camliHeadHash {
-		camliHeadHash = hash
-		doBuildCamli0, doBuildCamli1 = true, true
-		dbg.Println("Changes in camli tree detected, camlistore will be rebuilt")
-	}
-}
-
-func addToPATH(gobin string) {
-	splitter := ":"
-	switch runtime.GOOS {
-	case "windows":
-		splitter = ";"
-	case "plan9":
-		panic("unsupported")
-	}
-	p := gobin + splitter + defaultPATH
-	err := os.Setenv("PATH", p)
-	if err != nil {
-		log.Fatalf("Could not set PATH to %v: %v", p, err)
-	}
-}
-
-func restorePATH() {
-	err := os.Setenv("PATH", defaultPATH)
-	if err != nil {
-		log.Fatalf("Could not set PATH to %v: %v", defaultPATH, err)
-	}
-}
-
-type task struct {
-	lk sync.Mutex
-	// actual command that is run for this task. it includes the command and
-	// all the arguments, space separated. shell metacharacters not supported.
-	Cmd      string
-	start    time.Time
-	Duration time.Duration
-}
-
-func getCurrentTask() *task {
-	currentTask.lk.Lock()
-	defer currentTask.lk.Unlock()
-	return &task{Cmd: currentTask.Cmd, start: currentTask.start}
-}
-
-func setCurrentTask(cmd string) {
-	currentTask.lk.Lock()
-	defer currentTask.lk.Unlock()
-	currentTask.Cmd = cmd
-	currentTask.start = time.Now()
-}
-
-type testSuite struct {
-	Run        []*task
-	CamliHash  string
-	GoHash     string
-	failedTask int
-	Err        error
-	Start      time.Time
-	IsTip      bool
-}
-
-type History [][2]*testSuite
-
-func addRun(tsk *task, tskErr error) {
-	historylk.Lock()
-	defer historylk.Unlock()
-
-	duration := time.Now().Sub(tsk.start)
-	tsk.Duration = duration
-	if len(currentTestSuite.Run) == 0 {
-		currentTestSuite.Start = tsk.start
-	}
-	if tskErr != nil && currentTestSuite.Err == nil {
-		currentTestSuite.Err = tskErr
-		currentTestSuite.failedTask = len(currentTestSuite.Run)
-	} else {
-		currentTestSuite.failedTask = -1
-	}
-	currentTestSuite.Run = append(currentTestSuite.Run, tsk)
-}
-
-func addTestSuite() {
-	historylk.Lock()
-	defer historylk.Unlock()
-	if len(currentTestSuite.Run) < 1 {
-		return
-	}
-	if !currentTestSuite.IsTip {
-		// Go 1
-		entry := [2]*testSuite{currentTestSuite, nil}
-		history = append(history, entry)
-	} else {
-		// Go tip
-		entry := [2]*testSuite{history[len(history)-1][0], currentTestSuite}
-		if len(history) > historySize {
-			history = append(history[1:historySize], entry)
-		} else {
-			history[len(history)-1] = entry
-		}
-	}
-}
-
-func runCmd(tsk string) (string, error) {
-	dbg.Println(getCurrentTask().Cmd)
-	fields := strings.Fields(tsk)
-	var args []string
-	if len(fields) > 1 {
-		args = fields[1:]
-	}
-	cmd := exec.Command(fields[0], args...)
-	var stdout, stderr bytes.Buffer
-	cmd.Stdout = &stdout
-	cmd.Stderr = &stderr
-	err := cmd.Run()
-	if err != nil {
-		var sout, serr string
-		if sout = stdout.String(); sout == "" {
-			sout = "(empty)"
-		}
-		if serr = stderr.String(); serr == "" {
-			serr = "(empty)"
-		}
-		return "", fmt.Errorf("%v\n\nStdout:\n%s\n\nStderr:\n%s\n", err, sout, serr)
-	}
-	return stdout.String(), nil
-}
-
-func gofast(n int) {
-	for i := 0; i < n; i++ {
-		tsk := &task{start: time.Now()}
-		fail := rand.Intn(10)
-		if fail < 1 {
-			dbg.Println("random fail")
-			addRun(tsk, fmt.Errorf("random fail"))
-			continue
-		}
-		addRun(tsk, nil)
-	}
-}
-
-func buildGoTip() error {
-	if *fast {
-		gofast(3)
-		return nil
-	}
-	err := os.Chdir(goTipDir)
-	if err != nil {
-		log.Fatal(err)
-	}
-	defer func() {
-		err := os.Chdir(defaultDir)
-		if err != nil {
-			panic(err)
-		}
-	}()
-
-	tasks := []string{
-		NameToCmd["prepRepo3"],
-	}
-	for _, v := range tasks {
-		setCurrentTask(v)
-		tsk := getCurrentTask()
-		_, err := runCmd(tsk.Cmd)
-		addRun(tsk, err)
-		if err != nil {
-			return err
-		}
-	}
-	err = os.Chdir(filepath.Join(goTipDir, "src"))
-	if err != nil {
-		log.Fatal(err)
-	}
-	defer func() {
-		err := os.Chdir(defaultDir)
-		if err != nil {
-			panic(err)
-		}
-	}()
-
-	setCurrentTask(NameToCmd["buildGoTip1"])
-	tsk := getCurrentTask()
-	_, err = runCmd(tsk.Cmd)
-	addRun(tsk, err)
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
-func buildCamli(isTip bool) error {
-	if *fast {
-		gofast(5)
-		return nil
-	}
-	switchGoPath(isTip)
-	err := os.Chdir(camliRoot)
-	if err != nil {
-		log.Fatal(err)
-	}
-	defer func() {
-		err := os.Chdir(defaultDir)
-		if err != nil {
-			panic(err)
-		}
-	}()
-
-	if doBuildGo && isTip {
-		if *verbose {
-			dbg.Println("cleaning up gopath/pkg and gopath/bin for full rebuild")
-		}
-		// erase and rebuild everything
-		pkgdir := filepath.Join("..", "..", "pkg")
-		err := os.RemoveAll(pkgdir)
-		if err != nil {
-			log.Fatalf("failed to remove %v: %v", pkgdir, err)
-		}
-		bindir := filepath.Join("..", "..", "bin")
-		err = os.RemoveAll(bindir)
-		if err != nil {
-			log.Fatalf("failed to remove %v: %v", bindir, err)
-		}
-	}
-
-	if *verbose {
-		setCurrentTask("go version")
-		out, err := runCmd(getCurrentTask().Cmd)
-		if err != nil {
-			log.Fatalf("failed to run 'go version': %v", err)
-		}
-		out = strings.TrimRight(out, "\n")
-		dbg.Printf("Building camlistore in %v with: %v\n", goPath, out)
-	}
-
-	tasks := []string{}
-	if doBuildCamli0 || doBuildCamli1 || (doBuildGo && isTip) {
-		tasks = append(tasks, NameToCmd["buildCamli1"], NameToCmd["buildCamli2"])
-	}
-	tasks = append(tasks, NameToCmd["buildCamli3"])
-	for _, v := range tasks {
-		setCurrentTask(v)
-		tsk := getCurrentTask()
-		_, err := runCmd(tsk.Cmd)
-		addRun(tsk, err)
-		if err != nil {
-			return err
-		}
-	}
-	if doBuildGo && isTip {
-		doBuildGo = false
-		doBuildCamli1 = false
-	}
-	if isTip {
-		doBuildCamli1 = false
-	} else {
-		doBuildCamli0 = false
-	}
-
-	return nil
-}
-
-func runCamli() (*os.Process, error) {
-	if *fast {
-		gofast(1)
-		return nil, nil
-	}
-	err := os.Chdir(camliRoot)
-	if err != nil {
-		log.Fatal(err)
-	}
-	defer func() {
-		err := os.Chdir(defaultDir)
-		if err != nil {
-			panic(err)
-		}
-	}()
-
-	setCurrentTask(NameToCmd["runCamli"])
-	dbg.Println(getCurrentTask().Cmd)
-	fields := strings.Fields(getCurrentTask().Cmd)
-	args := fields[1:]
-	cmd := exec.Command(fields[0], args...)
-
-	var output []byte
-	errc := make(chan error, 1)
-	go func() {
-		output, err = cmd.CombinedOutput()
-		errc <- err
-	}()
-	select {
-	case err := <-errc:
-		dbg.Printf("dev server DEAD:\n%s\n", output)
-		tsk := getCurrentTask()
-		addRun(tsk, err)
-		return nil, fmt.Errorf("%v: server failed to start\n", tsk.Cmd)
-	case <-time.After(warmup):
-		dbg.Println("devcam server OK")
-		addRun(getCurrentTask(), nil)
-	}
-	return cmd.Process, nil
-}
-
-func killCamli(proc *os.Process) {
-	if *fast {
-		return
-	}
-	dbg.Println("killing dev server")
-	err := proc.Signal(os.Interrupt)
-	if err != nil {
-		log.Fatalf("Could not kill server: %v", err)
-	}
-	dbg.Println("")
-}
-
-func hitURL(url string) error {
-	setCurrentTask(fmt.Sprintf("http.Get(\"%s\")", url))
-	dbg.Println(getCurrentTask().Cmd)
-	resp, err := http.Get(url)
-	if err != nil {
-		return fmt.Errorf("%v: %v\n", getCurrentTask().Cmd, err)
-	}
-	defer resp.Body.Close()
-	if resp.StatusCode != 200 {
-		return fmt.Errorf("%v, got StatusCode: %d\n", getCurrentTask().Cmd, resp.StatusCode)
-	}
-	return nil
-}
-
-func hitCamliUi() error {
-	if *fast {
-		gofast(2)
-		return nil
-	}
-	urls := []string{
-		NameToCmd["hitCamliUi1"],
-	}
-	var err error
-	for _, v := range urls {
-		lerr := hitURL(v)
-		addRun(getCurrentTask(), lerr)
-		if lerr != nil {
-			err = lerr
-		}
-	}
-	return err
-}
-
-func camputOne(vivify bool) error {
-	if *fast {
-		gofast(3)
-		return nil
-	}
-	err := os.Chdir(camliRoot)
-	if err != nil {
-		log.Fatal(err)
-	}
-	defer func() {
-		err := os.Chdir(defaultDir)
-		if err != nil {
-			panic(err)
-		}
-	}()
-	// clean up camput caches
-	err = os.RemoveAll(camputCacheDir)
-	if err != nil {
-		log.Fatalf("Problem cleaning up camputCacheDir %v: %v", camputCacheDir, err)
-	}
-
-	// push the file to camli
-	tskString := NameToCmd["camput1"]
-	if vivify {
-		tskString = NameToCmd["camput2"]
-	}
-	setCurrentTask(tskString)
-	tsk := getCurrentTask()
-	out, err := runCmd(tsk.Cmd)
-	addRun(tsk, err)
-	if err != nil {
-		return err
-	}
-	// TODO(mpl): parsing camput output is kinda weak.
-	firstSHA1 := regexp.MustCompile(`.*(sha1-[a-zA-Z0-9]+)\nsha1-[a-zA-Z0-9]+\nsha1-[a-zA-Z0-9]+\n.*`)
-	if vivify {
-		firstSHA1 = regexp.MustCompile(`.*(sha1-[a-zA-Z0-9]+)\n.*`)
-	}
-	m := firstSHA1.FindStringSubmatch(out)
-	if m == nil {
-		return fmt.Errorf("%v: unexpected camput output\n", getCurrentTask().Cmd)
-	}
-	blobref := m[1]
-
-	// get the file's json to find out the file's blobref
-	setCurrentTask(NameToCmd["camget"] + blobref)
-	tsk = getCurrentTask()
-	out, err = runCmd(tsk.Cmd)
-	addRun(tsk, err)
-	if err != nil {
-		return err
-	}
-	blobrefPattern := regexp.MustCompile(`"blobRef": "(sha1-[a-zA-Z0-9]+)",\n.*`)
-	m = blobrefPattern.FindStringSubmatch(out)
-	if m == nil {
-		return fmt.Errorf("%v: unexpected camget output\n", getCurrentTask().Cmd)
-	}
-	blobref = m[1]
-
-	// finally, get the file back
-	setCurrentTask(NameToCmd["camget"] + blobref)
-	tsk = getCurrentTask()
-	out, err = runCmd(tsk.Cmd)
-	addRun(tsk, err)
-	if err != nil {
-		return err
-	}
-
-	// and compare it with the original
-	wantFile := testFile[0]
-	if vivify {
-		wantFile = testFile[1]
-	}
-	fileContents, err := ioutil.ReadFile(wantFile)
-	if err != nil {
-		log.Fatalf("Could not read %v: %v", wantFile, err)
-	}
-	if string(fileContents) != out {
-		return fmt.Errorf("%v: contents fetched with camget differ from %v contents", getCurrentTask().Cmd, wantFile)
-	}
-	return nil
-}
-
-func camputMany() error {
-	if *fast {
-		gofast(1)
-		return nil
-	}
-	err := os.Chdir(camliRoot)
-	if err != nil {
-		log.Fatal(err)
-	}
-	defer func() {
-		err := os.Chdir(defaultDir)
-		if err != nil {
-			panic(err)
-		}
-	}()
-	// upload the full camli pkg tree
-	setCurrentTask(NameToCmd["camput3"])
-	tsk := getCurrentTask()
-	_, err = runCmd(tsk.Cmd)
-	addRun(tsk, err)
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
-func handleErr(err error, proc *os.Process) {
-	lastModified = time.Now()
-	lastErr = err
-	dbg.Printf("%v", err)
-	if proc != nil {
-		killCamli(proc)
-	}
-	addTestSuite()
-}
-
-func main() {
-	flag.Usage = usage
-	flag.Parse()
-	if *help {
-		usage()
-	}
-
-	setup()
-	defer cleanTempGopath()
-	go handleSignals()
-
-	http.HandleFunc(okPrefix, okHandler)
-	http.HandleFunc(failPrefix, failHandler)
-	http.HandleFunc(currentPrefix, progressHandler)
-	http.HandleFunc(stderrPrefix, logHandler)
-	http.HandleFunc("/", statusHandler)
-	go http.ListenAndServe(*host, nil)
-
-	tryCount := 0
-	for {
-		if lastErr == nil || tryCount > 1 {
-			tryCount = 0
-			lastErr = nil
-			prepRepo()
-		}
-		if doBuildGo || doBuildCamli0 || doBuildCamli1 || lastErr != nil {
-			for _, isTip := range [2]bool{false, true} {
-				lastModified = time.Now()
-				restorePATH()
-				currentTestSuite = &testSuite{
-					Run:       make([]*task, 0, 1),
-					GoHash:    goTipHash,
-					CamliHash: camliHeadHash,
-					IsTip:     isTip,
-				}
-				if isTip {
-					addToPATH(filepath.Join(goTipDir, "bin"))
-					if doBuildGo {
-						err := buildGoTip()
-						if err != nil {
-							handleErr(err, nil)
-							continue
-						}
-					}
-				}
-
-				err := buildCamli(isTip)
-				if err != nil {
-					handleErr(err, nil)
-					continue
-				}
-				proc, err := runCamli()
-				if err != nil {
-					handleErr(err, nil)
-					continue
-				}
-				err = hitCamliUi()
-				if err != nil {
-					handleErr(err, proc)
-					continue
-				}
-				doVivify := false
-				err = camputOne(doVivify)
-				if err != nil {
-					handleErr(err, proc)
-					continue
-				}
-				doVivify = true
-				err = camputOne(doVivify)
-				if err != nil {
-					handleErr(err, proc)
-					continue
-				}
-				err = camputMany()
-				if err != nil {
-					handleErr(err, proc)
-					continue
-				}
-
-				dbg.Println("All good.")
-				killCamli(proc)
-				addTestSuite()
-				lastModified = time.Now()
-			}
-			tryCount++
-		}
-		setCurrentTask(fmt.Sprintf("time.Sleep(%d)", interval))
-		dbg.Println(getCurrentTask().Cmd)
-		time.Sleep(interval)
-	}
-}
-
-var (
-	okPrefix      = "/ok/"
-	failPrefix    = "/fail/"
-	currentPrefix = "/current"
-	stderrPrefix  = "/stderr"
-	statusTpl     = template.Must(template.New("status").Funcs(tmplFuncs).Parse(reportHTML))
-	taskTpl       = template.Must(template.New("task").Parse(taskHTML))
-	testSuiteTpl  = template.Must(template.New("ok").Parse(testSuiteHTML))
-)
-
-var tmplFuncs = template.FuncMap{
-	"camliRepoURL": camliRepoURL,
-	"goRepoURL":    goRepoURL,
-	"shortHash":    shortHash,
-}
-
-// unlocked; history needs to be protected from the caller.
-func getPastTestSuite(key string) (*testSuite, error) {
-	idx := 0
-	date := ""
-	if strings.HasPrefix(key, "gotip-") {
-		date = strings.Replace(key, "gotip-", "", -1)
-		idx = 1
-	} else {
-		date = strings.Replace(key, "go1-", "", -1)
-	}
-	for _, v := range history {
-		if v[idx].Start.String() == date {
-			return v[idx], nil
-		}
-	}
-	return nil, fmt.Errorf("%v not found in history", date)
-}
-
-type progressData struct {
-	Ts      *testSuite
-	Current string
-}
-
-// modtime is the modification time of the resource to be served, or IsZero().
-// return value is whether this request is now complete.
-func checkLastModified(w http.ResponseWriter, r *http.Request, modtime time.Time) bool {
-	if modtime.IsZero() {
-		return false
-	}
-
-	// The Date-Modified header truncates sub-second precision, so
-	// use mtime < t+1s instead of mtime <= t to check for unmodified.
-	if t, err := time.Parse(http.TimeFormat, r.Header.Get("If-Modified-Since")); err == nil && modtime.Before(t.Add(1*time.Second)) {
-		h := w.Header()
-		delete(h, "Content-Type")
-		delete(h, "Content-Length")
-		w.WriteHeader(http.StatusNotModified)
-		return true
-	}
-	w.Header().Set("Last-Modified", modtime.UTC().Format(http.TimeFormat))
-	return false
-}
-
-func logHandler(w http.ResponseWriter, r *http.Request) {
-	fmt.Fprintln(w, `<!doctype html>
-<html><meta http-equiv="refresh" content="5">
-<body><pre>`)
-	switch r.URL.Path {
-	case stderrPrefix:
-		logStderr.Lock()
-		_, err := w.Write(logStderr.Bytes())
-		logStderr.Unlock()
-		if err != nil {
-			log.Println("Error serving logStderr:", err)
-		}
-	default:
-		fmt.Fprintln(w, "Unknown log file path passed to logHandler:", r.URL.Path)
-		log.Println("Unknown log file path passed to logHandler:", r.URL.Path)
-	}
-}
-
-func okHandler(w http.ResponseWriter, r *http.Request) {
-	t := strings.Replace(r.URL.Path, okPrefix, "", -1)
-	historylk.Lock()
-	defer historylk.Unlock()
-	ts, err := getPastTestSuite(t)
-	if err != nil || len(ts.Run) == 0 {
-		http.NotFound(w, r)
-		return
-	}
-	lastTask := ts.Run[len(ts.Run)-1]
-	lastModTime := lastTask.start.Add(lastTask.Duration)
-	if checkLastModified(w, r, lastModTime) {
-		return
-	}
-	dat := &progressData{
-		Ts: ts,
-	}
-	err = testSuiteTpl.Execute(w, dat)
-	if err != nil {
-		log.Printf("ok template: %v\n", err)
-	}
-}
-
-func progressHandler(w http.ResponseWriter, r *http.Request) {
-	historylk.Lock()
-	defer historylk.Unlock()
-	currentTask := getCurrentTask()
-	if checkLastModified(w, r, currentTask.start) {
-		return
-	}
-	dat := &progressData{
-		Ts:      currentTestSuite,
-		Current: currentTask.Cmd,
-	}
-	err := testSuiteTpl.Execute(w, dat)
-	if err != nil {
-		log.Printf("progress template: %v\n", err)
-	}
-}
-
-type TaskReport struct {
-	Cmd string
-	Err error
-}
-
-func failHandler(w http.ResponseWriter, r *http.Request) {
-	t := strings.Replace(r.URL.Path, failPrefix, "", -1)
-	historylk.Lock()
-	defer historylk.Unlock()
-	ts, err := getPastTestSuite(t)
-	if err != nil || len(ts.Run) == 0 || ts.failedTask == -1 {
-		http.NotFound(w, r)
-		return
-	}
-	failedTask := ts.Run[ts.failedTask]
-	lastModTime := failedTask.start.Add(failedTask.Duration)
-	if checkLastModified(w, r, lastModTime) {
-		return
-	}
-	taskReport := &TaskReport{Cmd: failedTask.Cmd, Err: ts.Err}
-	err = taskTpl.Execute(w, taskReport)
-	if err != nil {
-		log.Printf("fail template: %v\n", err)
-	}
-}
-
-type status struct {
-	Hs History
-	Ts *testSuite
-}
-
-func invertedHistory() (inverted History) {
-	inverted = make(History, len(history))
-	endpos := len(history) - 1
-	for k, v := range history {
-		inverted[endpos-k] = v
-	}
-	return inverted
-}
-
-func statusHandler(w http.ResponseWriter, r *http.Request) {
-	historylk.Lock()
-	defer historylk.Unlock()
-	stat := &status{
-		Hs: invertedHistory(),
-		Ts: currentTestSuite,
-	}
-	if checkLastModified(w, r, lastModified) {
-		return
-	}
-	err := statusTpl.Execute(w, stat)
-	if err != nil {
-		log.Printf("status template: %v\n", err)
-	}
-}
-
-// shortHash returns a short version of a hash.
-func shortHash(hash string) string {
-	if len(hash) > 12 {
-		hash = hash[:12]
-	}
-	return hash
-}
-
-func goRepoURL(hash string) string {
-	return "https://code.google.com/p/go/source/detail?r=" + hash
-}
-
-func camliRepoURL(hash string) string {
-	return "https://camlistore.googlesource.com/camlistore/+/" + hash
-}
-
-// style inspired from $GOROOT/misc/dashboard/app/build/ui.html
-var styleHTML = `
-<style>
-	body {
-		font-family: sans-serif;
-		padding: 0; margin: 0;
-	}
-	h1, h2 {
-		margin: 0;
-		padding: 5px;
-	}
-	h1 {
-		background: #eee;
-	}
-	h2 {
-		margin-top: 20px;
-	}
-	.build, .packages {
-		margin: 5px;
-		border-collapse: collapse;
-	}
-	.build td, .build th, .packages td, .packages th {
-		vertical-align: top;
-		padding: 2px 4px;
-		font-size: 10pt;
-	}
-	.build tr.commit:nth-child(2n) {
-		background-color: #f0f0f0;
-	}
-	.build .hash {
-		font-family: monospace;
-		font-size: 9pt;
-	}
-	.build .result {
-		text-align: center;
-		width: 2em;
-	}
-	.col-hash, .col-result {
-		border-right: solid 1px #ccc;
-	}
-	.build .arch {
-		font-size: 66%;
-		font-weight: normal;
-	}
-	.build .time {
-		color: #666;
-	}
-	.build .ok {
-		font-size: 83%;
-	}
-	a.ok {
-		text-decoration:none;
-	}
-	.build .desc, .build .time, .build .user {
-		white-space: nowrap;
-	}
-	.paginate {
-		padding: 0.5em;
-	}
-	.paginate a {
-		padding: 0.5em;
-		background: #eee;
-		color: blue;
-	}
-	.paginate a.inactive {
-		color: #999;
-	}
-	.pull-right {
-		float: right;
-	}
-	.fail {
-		color: #C00;
-	}
-</style>
-`
-
-var reportHTML = `
-<!DOCTYPE HTML>
-<html>
-	<head>
-		<title>Camlistore tests Dashboard</title>` +
-	styleHTML + `
-	</head>
-	<body>
-
-	<h1>Camlibot status<span class="pull-right"><a href="` + stderrPrefix + `">stderr</a></span></h1>
-
-	<table class="build">
-	<colgroup class="col-hash" span="1"></colgroup>
-	<colgroup class="build" span="1"></colgroup>
-	<colgroup class="build" span="1"></colgroup>
-	<colgroup class="user" span="1"></colgroup>
-	<colgroup class="user" span="1"></colgroup>
-	<tr>
-	<!-- extra row to make alternating colors use dark for first result -->
-	</tr>
-	<tr>
-	<th>&nbsp;</th>
-	<th colspan="1">Go tip hash</th>
-	<th colspan="1">Camli HEAD hash</th>
-	<th colspan="1">Go1</th>
-	<th colspan="1">Gotip</th>
-	</tr>
-	{{if .Ts}}
-		{{if .Ts.IsTip | not}}
-			<tr class="commit">
-				<td class="hash">{{.Ts.Start}}</td>
-				<td class="hash">
-					<a href="{{goRepoURL .Ts.GoHash}}">{{shortHash .Ts.GoHash}}</a>
-				</td>
-				<td class="hash">
-					<a href="{{camliRepoURL .Ts.CamliHash}}">{{shortHash .Ts.CamliHash}}</a>
-				</td>
-				<td class="result">
-					<a href="` + currentPrefix + `" class="ok">In progress</a>
-				</td>
-			</tr>
-		{{end}}
-	{{end}}
-	{{if .Hs}}
-		{{range $tss := .Hs}}
-			<tr class="commit">
-			{{range $k, $ts := $tss}}
-			{{if $k | not}}
-				<td class="hash">{{$ts.Start}}</td>
-				<td class="hash">
-					<a href="{{goRepoURL $ts.GoHash}}">{{shortHash $ts.GoHash}}</a>
-				</td>
-				<td class="hash">
-					<a href="{{camliRepoURL $ts.CamliHash}}">{{shortHash $ts.CamliHash}}</a>
-				</td>
-				<td class="result">
-				{{if $ts.Err}}
-					<a href="` + failPrefix + `go1-{{$ts.Start}}" class="fail">fail</a>
-				{{else}}
-					<a href="` + okPrefix + `go1-{{$ts.Start}}" class="ok">ok</a>
-				{{end}}
-				</td>
-			{{else}}
-				<td class="result">
-				{{if $ts}}
-					{{if $ts.Err}}
-						<a href="` + failPrefix + `gotip-{{$ts.Start}}" class="fail">fail</a>
-					{{else}}
-						<a href="` + okPrefix + `gotip-{{$ts.Start}}" class="ok">ok</a>
-					{{end}}
-				{{else}}
-					<a href="` + currentPrefix + `" class="ok">In progress</a>
-				{{end}}
-				</td>
-			{{end}}
-			{{end}}
-			</tr>
-		{{end}}
-	{{end}}
-	</table>
-
-	</body>
-</html>
-`
-
-var testSuiteHTML = `
-<!DOCTYPE HTML>
-<html>
-	<head>
-		<title>Camlistore tests Dashboard</title>` +
-	styleHTML + `
-	</head>
-	<body>
-	{{if .Ts}}
-		<h2> Testsuite for {{if .Ts.IsTip}}Go tip{{else}}Go 1{{end}} at {{.Ts.Start}} </h2>
-		<table class="build">
-		<colgroup class="col-result" span="1"></colgroup>
-		<colgroup class="col-result" span="1"></colgroup>
-		<tr>
-			<!-- extra row to make alternating colors use dark for first result -->
-		</tr>
-		<tr>
-			<th colspan="1">Step</th>
-			<th colspan="1">Duration</th>
-		</tr>
-		{{range $k, $v := .Ts.Run}}
-		<tr>
-			<td>{{$v.Cmd}}</td>
-			<td>{{$v.Duration}}</td>
-		</tr>
-		{{end}}
-		{{if .Current}}
-		<tr>
-			<td>{{.Current}}</td>
-			<td>(running...)</td>
-		</tr>
-		{{end}}
-		</table>
-	{{end}}
-	</body>
-</html>
-`
-
-var taskHTML = `
-<!DOCTYPE HTML>
-<html>
-	<head>
-		<title>Camlistore tests Dashboard</title>
-	</head>
-	<body>
-{{if .Cmd}}
-	<h2>Command:</h2>
-	<p>{{.Cmd}}</p>
-{{end}}
-{{if .Err}}
-	<h2>Error:</h2>
-	<pre>
-	{{.Err}}
-	</pre>
-{{end}}
-	</body>
-</html>
-`
diff --git a/misc/buildbot/builder/builder.go b/misc/buildbot/builder/builder.go
new file mode 100644
index 000000000..bd7b2560f
--- /dev/null
+++ b/misc/buildbot/builder/builder.go
@@ -0,0 +1,964 @@
+/*
+Copyright 2012 The Camlistore Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// The buildbot is Camlistore's continuous builder.
+// This builder program is started by the master. It then rebuilds
+// Go 1, GoTip, Camlistore, and runs a battery of tests for Camlistore.
+// It then sends a report to the master and terminates.
+// It can also respond to progress requests from the master.
+// If run with -ephemeral=false, it could be used as a remote long lived
+// builder bot.
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os"
+	"os/exec"
+	"os/signal"
+	"path/filepath"
+	"regexp"
+	"runtime"
+	"strings"
+	"sync"
+	"syscall"
+	"time"
+)
+
+const (
+	interval = 60 * time.Second // polling frequency
+	warmup   = 30 * time.Second // duration before we test if devcam server has started properly
+)
+
+var (
+	// TODO(mpl): use that one, same as in master.
+	altCamliRevURL = flag.String("camlirevurl", "", "alternative URL to query about the latest camlistore revision hash (e.g camlistore.org/latesthash), to alleviate hitting too often the Camlistore git repo.")
+	arch           = flag.String("arch", "", "The arch we report the master(s). Defaults to runtime.GOARCH.")
+	// TODO(mpl): drop that option?
+	ephemeral    = flag.Bool("ephemeral", true, "Die after we have run the testSuites for Go1 and Go tip once.") // This will be false for remote bots which run on other archs and send us regular reports.
+	fakeTests    = flag.Bool("faketests", false, "Run fast fake tests instead of the real ones, for faster debugging.")
+	help         = flag.Bool("h", false, "show this help")
+	host         = flag.String("host", "0.0.0.0:8081", "listening hostname and port")
+	masterHosts  = flag.String("masterhosts", "localhost:8080", "listening hostname and port of the master bots, i.e where to send the test suite reports. Comma separated list.")
+	ourOS        = flag.String("os", "", "The OS we report the master(s). Defaults to runtime.GOOS.")
+	skipGo1Build = flag.Bool("skipgo1build", false, "skip initial go1 build, for debugging and quickly going to the next steps.")
+	verbose      = flag.Bool("verbose", false, "print what's going on")
+)
+
+var (
+	testFile                = []string{"AUTHORS", "CONTRIBUTORS"}
+	cacheDir                string
+	camliHeadHash           string
+	camliRoot               string
+	camputCacheDir          string
+	dbg                     *debugger
+	defaultPATH             string
+	doBuildGo, doBuildCamli bool
+	go1Dir                  string
+	goTipDir                string
+	goTipHash               string
+
+	biSuitelk        sync.Mutex
+	currentTestSuite *testSuite
+	currentBiSuite   *biTestSuite
+
+	// Process of the camlistore server, so we can kill it when
+	// we get killed ourselves.
+	camliProc *os.Process
+
+	// For "If-Modified-Since" requests asking for progress.
+	// Updated every time a new test task/run is added to the test suite.
+	lastModified time.Time
+)
+
+var devcamBin = filepath.Join("bin", "devcam")
+var (
+	hgCloneGo1Cmd      = newTask("hg", "clone", "-u", "release", "https://code.google.com/p/go")
+	hgCloneGoTipCmd    = newTask("hg", "clone", "-u", "tip", "https://code.google.com/p/go")
+	hgPullCmd          = newTask("hg", "pull")
+	hgUpdateCmd        = newTask("hg", "update", "-C", "default")
+	hgLogCmd           = newTask("hg", "log", "-r", "tip", "--template", "{node}")
+	hgConfigCmd        = newTask("hg", "--config", "extensions.purge=", "purge", "--all")
+	gitCloneCmd        = newTask("git", "clone", "https://camlistore.googlesource.com/camlistore")
+	gitResetCmd        = newTask("git", "reset", "--hard")
+	gitCleanCmd        = newTask("git", "clean", "-Xdf")
+	gitPullCmd         = newTask("git", "pull")
+	gitRevCmd          = newTask("git", "rev-parse", "HEAD")
+	buildGoCmd         = newTask("./make.bash")
+	buildCamliCmd      = newTask("go", "run", "make.go", "-v")
+	runTestsCmd        = newTask(devcamBin, "test")
+	runCamliCmd        = newTask(devcamBin, "server", "--wipe", "--mysql")
+	camgetCmd          = newTask(devcamBin, "get")
+	camputCmd          = newTask(devcamBin, "put", "file", "--permanode", testFile[0])
+	camputVivifyCmd    = newTask(devcamBin, "put", "file", "--vivify", testFile[1])
+	camputFilenodesCmd = newTask(devcamBin, "put", "file", "--filenodes", "pkg")
+)
+
+func usage() {
+	fmt.Fprintf(os.Stderr, "\t builderBot \n")
+	flag.PrintDefaults()
+	os.Exit(2)
+}
+
+type debugger struct {
+	lg *log.Logger
+}
+
+func (dbg *debugger) Printf(format string, v ...interface{}) {
+	if dbg != nil && *verbose {
+		dbg.lg.Printf(format, v...)
+	}
+}
+
+func (dbg *debugger) Println(v ...interface{}) {
+	if v == nil {
+		return
+	}
+	if dbg != nil && *verbose {
+		dbg.lg.Println(v...)
+	}
+}
+
+type task struct {
+	Program  string
+	Args     []string
+	Start    time.Time
+	Duration time.Duration
+	Err      string
+	hidden   bool
+}
+
+func newTask(program string, args ...string) *task {
+	return &task{Program: program, Args: args}
+}
+
+// because sometimes we do not want to modify the tsk template
+// so we make a copy of it
+func newTaskFrom(tsk *task) *task {
+	return newTask(tsk.Program, tsk.Args...)
+}
+
+func (t *task) String() string {
+	return fmt.Sprintf("%v %v", t.Program, t.Args)
+}
+
+func (t *task) run() (string, error) {
+	var err error
+	defer func() {
+		t.Duration = time.Now().Sub(t.Start)
+		if !t.hidden {
+			biSuitelk.Lock()
+			currentTestSuite.addRun(t)
+			biSuitelk.Unlock()
+		}
+	}()
+	dbg.Println(t.String())
+	cmd := exec.Command(t.Program, t.Args...)
+	var stdout, stderr bytes.Buffer
+	cmd.Stdout = &stdout
+	cmd.Stderr = &stderr
+	t.Start = time.Now()
+	err = cmd.Run()
+	if err != nil {
+		var sout, serr string
+		if sout = stdout.String(); sout == "" {
+			sout = "(empty)"
+		}
+		if serr = stderr.String(); serr == "" {
+			serr = "(empty)"
+		}
+		t.Err = fmt.Sprintf("%v\n\nStdout:\n%s\n\nStderr:\n%s\n", err, sout, serr)
+		return "", errors.New(t.Err)
+	}
+	return stdout.String(), nil
+}
+
+type testSuite struct {
+	Run       []*task
+	CamliHash string
+	GoHash    string
+	Err       string
+	Start     time.Time
+	IsTip     bool
+}
+
+func (ts *testSuite) addRun(tsk *task) {
+	if ts == nil {
+		panic("Tried adding a run to a nil testSuite")
+	}
+	if ts.Start.IsZero() && len(ts.Run) == 0 {
+		ts.Start = tsk.Start
+	}
+	if tsk.Err != "" && ts.Err == "" {
+		ts.Err = tsk.Err
+	}
+	ts.Run = append(ts.Run, tsk)
+	lastModified = time.Now()
+}
+
+type biTestSuite struct {
+	Local bool
+	Go1   testSuite
+	GoTip testSuite
+}
+
+func main() {
+	flag.Usage = usage
+	flag.Parse()
+	if *help {
+		usage()
+	}
+
+	go handleSignals()
+	http.HandleFunc("/progress", progressHandler)
+	go func() {
+		log.Printf("Now starting to listen on %v", *host)
+		if err := http.ListenAndServe(*host, nil); err != nil {
+			log.Fatalf("Could not start listening on %v: %v", *host, err)
+		}
+	}()
+	setup()
+
+	for {
+		biSuitelk.Lock()
+		currentBiSuite = &biTestSuite{}
+		biSuitelk.Unlock()
+		for _, isTip := range [2]bool{false, true} {
+			currentTestSuite = &testSuite{
+				Run:   make([]*task, 0, 1),
+				IsTip: isTip,
+				Start: time.Now(),
+			}
+			// We prepare the Go tip tree as soon as in the Go 1 run, so
+			// we can set GoTipHash in the test suite.
+			if err := prepGoTipTree(isTip); err != nil {
+				endOfSuite(err)
+				continue
+			}
+
+			biSuitelk.Lock()
+			currentTestSuite.GoHash = goTipHash
+			biSuitelk.Unlock()
+			if isTip && doBuildGo && !*fakeTests {
+				if err := buildGoTip(); err != nil {
+					endOfSuite(err)
+					continue
+				}
+			}
+			if err := prepCamliTree(isTip); err != nil {
+				endOfSuite(err)
+				continue
+			}
+			biSuitelk.Lock()
+			currentTestSuite.CamliHash = camliHeadHash
+			biSuitelk.Unlock()
+			if !(doBuildGo || doBuildCamli) {
+				endOfSuite(nil)
+			}
+			restorePATH()
+			goDir := go1Dir
+			if isTip {
+				goDir = goTipDir
+			}
+			addToPATH(filepath.Join(goDir, "bin"))
+			if *fakeTests {
+				if err := fakeRun(); err != nil {
+					endOfSuite(err)
+					continue
+				}
+				endOfSuite(nil)
+				if isTip {
+					break
+				} else {
+					continue
+				}
+			}
+			if err := buildCamli(); err != nil {
+				endOfSuite(err)
+				continue
+			}
+			if err := runTests(); err != nil {
+				endOfSuite(err)
+				continue
+			}
+			if err := runCamli(); err != nil {
+				endOfSuite(err)
+				continue
+			}
+			if err := hitCamliUi(); err != nil {
+				endOfSuite(err)
+				continue
+			}
+			doVivify := false
+			if err := camputOne(doVivify); err != nil {
+				endOfSuite(err)
+				continue
+			}
+			doVivify = true
+			if err := camputOne(doVivify); err != nil {
+				endOfSuite(err)
+				continue
+			}
+			if err := camputMany(); err != nil {
+				endOfSuite(err)
+				continue
+			}
+			endOfSuite(nil)
+		}
+		sanitizeRevs()
+		sendReport()
+		if *ephemeral {
+			break
+		}
+		tsk := newTask("time.Sleep", interval.String())
+		dbg.Println(tsk.String())
+		time.Sleep(interval)
+	}
+}
+
+func sanitizeRevs() {
+	if currentBiSuite == nil {
+		return
+	}
+	if currentBiSuite.GoTip.Start.IsZero() {
+		return
+	}
+	if currentBiSuite.GoTip.CamliHash == "" && currentBiSuite.Go1.CamliHash == "" {
+		dbg.Printf("CamliHash not set in both Go1 and GoTip test suites")
+		return
+	}
+	if currentBiSuite.GoTip.CamliHash == "" && currentBiSuite.Go1.CamliHash == "" {
+		dbg.Printf("GoHash not set in both Go1 and GoTip test suites")
+		return
+	}
+	if currentBiSuite.GoTip.CamliHash != "" && currentBiSuite.Go1.CamliHash != "" &&
+		currentBiSuite.GoTip.CamliHash != currentBiSuite.Go1.CamliHash {
+		panic("CamliHash in GoTip suite and in Go1 suite differ; should not happen.")
+	}
+	if currentBiSuite.GoTip.GoHash != "" && currentBiSuite.Go1.GoHash != "" &&
+		currentBiSuite.GoTip.GoHash != currentBiSuite.Go1.GoHash {
+		panic("GoHash in GoTip suite and in Go1 suite differ; should not happen.")
+	}
+	if currentBiSuite.GoTip.GoHash == "" {
+		currentBiSuite.GoTip.GoHash = currentBiSuite.Go1.GoHash
+	}
+	if currentBiSuite.Go1.GoHash == "" {
+		currentBiSuite.Go1.GoHash = currentBiSuite.GoTip.GoHash
+	}
+	if currentBiSuite.GoTip.CamliHash == "" {
+		currentBiSuite.GoTip.CamliHash = currentBiSuite.Go1.CamliHash
+	}
+	if currentBiSuite.Go1.CamliHash == "" {
+		currentBiSuite.Go1.CamliHash = currentBiSuite.GoTip.CamliHash
+	}
+}
+
+func endOfSuite(err error) {
+	biSuitelk.Lock()
+	defer biSuitelk.Unlock()
+	if currentTestSuite.IsTip {
+		currentBiSuite.GoTip = *currentTestSuite
+	} else {
+		currentBiSuite.Go1 = *currentTestSuite
+	}
+	killCamli()
+	if err != nil {
+		log.Printf("%v", err)
+	} else {
+		dbg.Println("All good.")
+	}
+}
+
+func setup() {
+	var err error
+	defaultPATH = os.Getenv("PATH")
+	if defaultPATH == "" {
+		log.Fatal("PATH not set")
+	}
+	log.SetPrefix("BUILDER: ")
+	dbg = &debugger{log.New(os.Stderr, "BUILDER: ", log.LstdFlags)}
+
+	// the OS we run on
+	if *ourOS == "" {
+		*ourOS = runtime.GOOS
+		if *ourOS == "" {
+			// Can this happen? I don't think so, but just in case...
+			panic("runtime.GOOS was not set")
+		}
+	}
+	// the arch we run on
+	if *arch == "" {
+		*arch = runtime.GOARCH
+		if *arch == "" {
+			panic("runtime.GOARCH was not set")
+		}
+	}
+
+	// cacheDir
+	cacheDir = filepath.Join(os.TempDir(), "camlibot-cache")
+	if err := os.MkdirAll(cacheDir, 0755); err != nil {
+		log.Fatalf("Could not create cache dir %v: %v", cacheDir, err)
+	}
+
+	// get go1 and gotip source
+	if err := os.Chdir(cacheDir); err != nil {
+		log.Fatalf("Could not cd to %v: %v", cacheDir, err)
+	}
+	go1Dir, err = filepath.Abs("go1")
+	if err != nil {
+		log.Fatalf("Problem with Go 1 dir: %v", err)
+	}
+	goTipDir, err = filepath.Abs("gotip")
+	if err != nil {
+		log.Fatalf("Problem with Go tip dir: %v", err)
+	}
+	for _, goDir := range []string{go1Dir, goTipDir} {
+		// if go dirs exist, just reuse them
+		if _, err := os.Stat(goDir); err != nil {
+			if !os.IsNotExist(err) {
+				log.Fatalf("Could not stat %v: %v", goDir, err)
+			}
+			// go1/gotip dir not here, let's clone it.
+			hgCloneCmd := hgCloneGo1Cmd
+			if goDir == goTipDir {
+				hgCloneCmd = hgCloneGoTipCmd
+			}
+			tsk := newTask(hgCloneCmd.Program, hgCloneCmd.Args...)
+			tsk.hidden = true
+			if _, err := tsk.run(); err != nil {
+				log.Fatalf("Could not hg clone %v: %v", goDir, err)
+			}
+			if err := os.Rename("go", goDir); err != nil {
+				log.Fatalf("Could not rename go dir into %v: %v", goDir, err)
+			}
+		}
+	}
+
+	if !*skipGo1Build {
+		// build Go1
+		if err := buildGo1(); err != nil {
+			log.Fatal(err)
+		}
+	}
+
+	// get camlistore source
+	if err := os.Chdir(cacheDir); err != nil {
+		log.Fatal("Could not cd to %v: %v", cacheDir, err)
+	}
+	camliRoot, err = filepath.Abs("camlistore.org")
+	if err != nil {
+		log.Fatal(err)
+	}
+	// if camlistore dir already exists, reuse it
+	if _, err := os.Stat(camliRoot); err != nil {
+		if !os.IsNotExist(err) {
+			log.Fatalf("Could not stat %v: %v", camliRoot, err)
+		}
+		cloneCmd := newTask(gitCloneCmd.Program, append(gitCloneCmd.Args, camliRoot)...)
+		cloneCmd.hidden = true
+		if _, err := cloneCmd.run(); err != nil {
+			log.Fatalf("Could not git clone into %v: %v", camliRoot, err)
+		}
+	}
+
+	// recording camput cache dir, so we can clean it up fast everytime
+	homeDir := os.Getenv("HOME")
+	if homeDir == "" {
+		log.Fatal("HOME not set")
+	}
+	camputCacheDir = filepath.Join(homeDir, ".cache", "camlistore")
+}
+
+func buildGo1() error {
+	if err := os.Chdir(filepath.Join(go1Dir, "src")); err != nil {
+		log.Fatalf("Could not cd to %v: %v", go1Dir, err)
+	}
+	tsk := newTask(buildGoCmd.Program, buildGoCmd.Args...)
+	tsk.hidden = true
+	if _, err := tsk.run(); err != nil {
+		return err
+	}
+	return nil
+}
+
+func handleSignals() {
+	c := make(chan os.Signal)
+	sigs := []os.Signal{syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT}
+	signal.Notify(c, sigs...)
+	for {
+		sig := <-c
+		sysSig, ok := sig.(syscall.Signal)
+		if !ok {
+			log.Fatal("Not a unix signal")
+		}
+		switch sysSig {
+		case syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT:
+			log.Printf("Received %v signal, terminating.", sig)
+			killCamli()
+			os.Exit(0)
+		default:
+			panic("should not get other signals here")
+		}
+	}
+}
+
+func prepGoTipTree(isTip bool) error {
+	if isTip && goTipHash != "" {
+		// go tip tree was already prepared properly in the Go 1 run
+		return nil
+	}
+	doBuildGo = false
+	if err := os.Chdir(goTipDir); err != nil {
+		log.Fatalf("Could not cd to %v: %v", goTipDir, err)
+	}
+	tasks := []*task{
+		newTaskFrom(hgPullCmd),
+		newTaskFrom(hgUpdateCmd),
+		newTaskFrom(hgLogCmd),
+		newTaskFrom(hgConfigCmd),
+	}
+	hash := ""
+	for _, t := range tasks {
+		out, err := t.run()
+		if err != nil {
+			log.Printf("Could not prepare the Go tip tree with %v: %v", t.String(), err)
+			return err
+		}
+		if t.String() == hgLogCmd.String() {
+			hash = strings.TrimRight(out, "\n")
+		}
+	}
+	dbg.Println("previous head in go tree: " + goTipHash)
+	dbg.Println("current head in go tree: " + hash)
+	if hash != "" && hash != goTipHash {
+		goTipHash = hash
+		doBuildGo = true
+		dbg.Println("Changes in go tree detected; Go tip will be rebuilt.")
+	}
+	return nil
+}
+
+func buildGoTip() error {
+	srcDir := filepath.Join(goTipDir, "src")
+	if err := os.Chdir(srcDir); err != nil {
+		log.Fatalf("Could not cd to %v: %v", srcDir, err)
+	}
+	if _, err := newTaskFrom(buildGoCmd).run(); err != nil {
+		return err
+	}
+	return nil
+}
+
+func prepCamliTree(isTip bool) error {
+	doBuildCamli = false
+	// camli
+	if err := os.Chdir(camliRoot); err != nil {
+		log.Fatalf("Could not cd to %v: %v", camliRoot, err)
+	}
+	rev := "HEAD"
+	if isTip {
+		if camliHeadHash == "" {
+			// the previous run with Go 1 somehow failed to set camliHeadHash
+			// so we pretend we're not on tip to retry all the work
+			isTip = !isTip
+		} else {
+			// we reset to the rev that was noted at the previous run with Go 1
+			rev = camliHeadHash
+		}
+	}
+	resetCmd := newTask(gitResetCmd.Program, append(gitResetCmd.Args, rev)...)
+	tasks := []*task{
+		resetCmd,
+		newTaskFrom(gitCleanCmd),
+	}
+	if !isTip {
+		// we only pull at the first run, with Go 1
+		tasks = append(tasks, newTaskFrom(gitPullCmd), newTaskFrom(gitRevCmd))
+	}
+	hash := ""
+	for _, t := range tasks {
+		out, err := t.run()
+		if err != nil {
+			log.Printf("Could not prepare the Camli tree with %v: %v\n", t.String(), err)
+			return err
+		}
+		hash = strings.TrimRight(out, "\n")
+	}
+	if isTip {
+		doBuildCamli = true
+		return nil
+	}
+	dbg.Println("previous head in camli tree: " + camliHeadHash)
+	dbg.Println("current head in camli tree: " + hash)
+	if hash != "" && hash != camliHeadHash {
+		camliHeadHash = hash
+		doBuildCamli = true
+		dbg.Println("Changes in camli tree detected, Camlistore will be rebuilt")
+	}
+	return nil
+}
+
+func restorePATH() {
+	err := os.Setenv("PATH", defaultPATH)
+	if err != nil {
+		log.Fatalf("Could not set PATH to %v: %v", defaultPATH, err)
+	}
+}
+
+func addToPATH(gobin string) {
+	splitter := ":"
+	switch runtime.GOOS {
+	case "windows":
+		splitter = ";"
+	case "plan9":
+		panic("unsupported")
+	}
+	p := gobin + splitter + defaultPATH
+	err := os.Setenv("PATH", p)
+	if err != nil {
+		log.Fatalf("Could not set PATH to %v: %v", p, err)
+	}
+}
+
+func cleanBuildGopaths() {
+	tmpDir := filepath.Join(camliRoot, "tmp")
+	if _, err := os.Stat(tmpDir); err != nil {
+		if !os.IsNotExist(err) {
+			log.Fatalf("Could not stat %v: %v", tmpDir, err)
+		}
+		// Does not exist, we only have to recreate it
+		// TODO(mpl): hmm maybe it should be an error that
+		// it does not exist, since it also contains the
+		// closure stuff?
+		if err := os.MkdirAll(tmpDir, 0755); err != nil {
+			log.Fatalf("Could not mkdir %v: %v", tmpDir, err)
+		}
+		return
+	}
+	f, err := os.Open(tmpDir)
+	if err != nil {
+		log.Fatalf("Could not open %v: %v", tmpDir, err)
+	}
+	defer f.Close()
+	names, err := f.Readdirnames(-1)
+	if err != nil {
+		log.Fatal("Could not read %v: %v", tmpDir, err)
+	}
+	for _, v := range names {
+		if strings.HasPrefix(v, "build-gopath") {
+			if err := os.RemoveAll(filepath.Join(tmpDir, v)); err != nil {
+				log.Fatalf("Could not remove %v: %v", v, err)
+			}
+		}
+	}
+}
+
+func fakeRun() error {
+	if _, err := newTask("sleep", "1").run(); err != nil {
+		return err
+	}
+	return nil
+}
+
+func buildCamli() error {
+	if err := os.Chdir(camliRoot); err != nil {
+		log.Fatalf("Could not cd to %v: %v", camliRoot, err)
+	}
+	// Clean up Camlistore's hermetic gopaths
+	cleanBuildGopaths()
+
+	if *verbose {
+		tsk := newTask("go", "version")
+		out, err := tsk.run()
+		tsk.hidden = true
+		if err != nil {
+			return fmt.Errorf("failed to run 'go version': %v", err)
+		}
+		out = strings.TrimRight(out, "\n")
+		dbg.Printf("Building Camlistore with: %v\n", out)
+	}
+	if _, err := newTaskFrom(buildCamliCmd).run(); err != nil {
+		return err
+	}
+	return nil
+}
+
+func runCamli() error {
+	if err := os.Chdir(camliRoot); err != nil {
+		log.Fatal(err)
+	}
+
+	t := newTaskFrom(runCamliCmd)
+	dbg.Println(t.String())
+	cmd := exec.Command(t.Program, t.Args...)
+	var output []byte
+	errc := make(chan error, 1)
+	t.Start = time.Now()
+	go func() {
+		var err error
+		output, err = cmd.CombinedOutput()
+		if err != nil {
+			err = fmt.Errorf("%v: %v", err, string(output))
+		}
+		errc <- err
+	}()
+	select {
+	case err := <-errc:
+		t.Err = fmt.Sprintf("%v terminated early:\n%v\n", t.String(), err)
+		biSuitelk.Lock()
+		currentTestSuite.addRun(t)
+		biSuitelk.Unlock()
+		log.Println(t.Err)
+		return errors.New(t.Err)
+	case <-time.After(warmup):
+		biSuitelk.Lock()
+		currentTestSuite.addRun(t)
+		camliProc = cmd.Process
+		biSuitelk.Unlock()
+		dbg.Printf("%v running OK so far\n", t.String())
+	}
+	return nil
+}
+
+func killCamli() {
+	if camliProc == nil {
+		return
+	}
+	dbg.Println("killing Camlistore server")
+	if err := camliProc.Kill(); err != nil {
+		log.Fatalf("Could not kill server with pid %v: %v", camliProc.Pid, err)
+	}
+	camliProc = nil
+	dbg.Println("")
+}
+
+func hitCamliUi() error {
+	return hitURL("http://localhost:3179/ui/")
+}
+
+func hitURL(url string) (err error) {
+	tsk := newTask("http.Get", url)
+	defer func() {
+		if err != nil {
+			tsk.Err = fmt.Sprintf("%v", err)
+		}
+		biSuitelk.Lock()
+		currentTestSuite.addRun(tsk)
+		biSuitelk.Unlock()
+	}()
+	dbg.Println(tsk.String())
+	tsk.Start = time.Now()
+	var resp *http.Response
+	resp, err = http.Get(url)
+	if err != nil {
+		return fmt.Errorf("%v: %v\n", tsk.String(), err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != 200 {
+		return fmt.Errorf("%v, got StatusCode: %d\n", tsk.String(), resp.StatusCode)
+	}
+	return nil
+}
+
+func camputOne(vivify bool) error {
+	if err := os.Chdir(camliRoot); err != nil {
+		log.Fatalf("Could not cd to %v: %v", camliRoot, err)
+	}
+
+	// clean up camput caches
+	if err := os.RemoveAll(camputCacheDir); err != nil {
+		log.Fatalf("Problem cleaning up camputCacheDir %v: %v", camputCacheDir, err)
+	}
+
+	// push the file to camli
+	tsk := newTaskFrom(camputCmd)
+	if vivify {
+		tsk = newTaskFrom(camputVivifyCmd)
+	}
+	out, err := tsk.run()
+	if err != nil {
+		return err
+	}
+	// TODO(mpl): parsing camput output is kinda weak.
+	firstSHA1 := regexp.MustCompile(`.*(sha1-[a-zA-Z0-9]+)\nsha1-[a-zA-Z0-9]+\nsha1-[a-zA-Z0-9]+\n.*`)
+	if vivify {
+		firstSHA1 = regexp.MustCompile(`.*(sha1-[a-zA-Z0-9]+)\n.*`)
+	}
+	m := firstSHA1.FindStringSubmatch(out)
+	if m == nil {
+		return fmt.Errorf("%v: unexpected camput output\n", tsk.String())
+	}
+	blobref := m[1]
+
+	// get the file's json to find out the file's blobref
+	tsk = newTask(camgetCmd.Program, append(camgetCmd.Args, blobref)...)
+	out, err = tsk.run()
+	if err != nil {
+		return err
+	}
+	blobrefPattern := regexp.MustCompile(`"blobRef": "(sha1-[a-zA-Z0-9]+)",\n.*`)
+	m = blobrefPattern.FindStringSubmatch(out)
+	if m == nil {
+		return fmt.Errorf("%v: unexpected camget output\n", tsk.String())
+	}
+	blobref = m[1]
+
+	// finally, get the file back
+	tsk = newTask(camgetCmd.Program, append(camgetCmd.Args, blobref)...)
+	out, err = tsk.run()
+	if err != nil {
+		return err
+	}
+
+	// and compare it with the original
+	wantFile := testFile[0]
+	if vivify {
+		wantFile = testFile[1]
+	}
+	fileContents, err := ioutil.ReadFile(wantFile)
+	if err != nil {
+		log.Fatalf("Could not read %v: %v", wantFile, err)
+	}
+	if string(fileContents) != out {
+		return fmt.Errorf("%v: contents fetched with camget differ from %v contents", tsk.String(), wantFile)
+	}
+	return nil
+}
+
+func camputMany() error {
+	err := os.Chdir(camliRoot)
+	if err != nil {
+		log.Fatalf("Could not cd to %v: %v", camliRoot, err)
+	}
+
+	// upload the full camli pkg tree
+	if _, err := newTaskFrom(camputFilenodesCmd).run(); err != nil {
+		return err
+	}
+	return nil
+}
+
+func runTests() error {
+	if err := os.Chdir(camliRoot); err != nil {
+		log.Fatal(err)
+	}
+	if _, err := newTaskFrom(runTestsCmd).run(); err != nil {
+		return err
+	}
+	return nil
+}
+
+const reportPrefix = "/report"
+
+func sendReport() {
+	biSuitelk.Lock()
+	// we make a copy so we can release the lock quickly enough
+	currentBiSuiteCpy := &biTestSuite{
+		Go1:   currentBiSuite.Go1,
+		GoTip: currentBiSuite.GoTip,
+	}
+	biSuitelk.Unlock()
+	masters := strings.Split(*masterHosts, ",")
+	OSArch := *ourOS + "_" + *arch
+	toReport := struct {
+		OSArch string
+		Ts     *biTestSuite
+	}{
+		OSArch: OSArch,
+		Ts:     currentBiSuiteCpy,
+	}
+	for _, v := range masters {
+		reportURL := "http://" + v + reportPrefix
+		// TODO(mpl): ipv6 too I suppose. just make a IsLocalhost func or whatever.
+		// probably can borrow something from camli code for that.
+		if strings.HasPrefix(v, "localhost") || strings.HasPrefix(v, "127.0.0.1") {
+			toReport.Ts.Local = true
+		} else {
+			toReport.Ts.Local = false
+		}
+		report, err := json.MarshalIndent(toReport, "", "  ")
+		if err != nil {
+			log.Printf("JSON serialization error: %v", err)
+			continue
+		}
+		r := bytes.NewReader(report)
+		resp, err := http.Post(reportURL, "text/javascript", r)
+		if err != nil {
+			log.Printf("Could not send report: %v", err)
+			continue
+		}
+		resp.Body.Close()
+	}
+}
+
+func progressHandler(w http.ResponseWriter, r *http.Request) {
+	if r.Method != "GET" {
+		log.Printf("Invalid method in progress handler: %v, want GET", r.Method)
+		http.Error(w, "Invalid method", http.StatusMethodNotAllowed)
+		return
+	}
+	if checkLastModified(w, r, lastModified) {
+		return
+	}
+	biSuitelk.Lock()
+	if currentBiSuite != nil {
+		if currentTestSuite.IsTip {
+			currentBiSuite.GoTip = *currentTestSuite
+		} else {
+			currentBiSuite.Go1 = *currentTestSuite
+		}
+	}
+	sanitizeRevs()
+	report, err := json.MarshalIndent(currentBiSuite, "", "  ")
+	if err != nil {
+		biSuitelk.Unlock()
+		log.Printf("JSON serialization error: %v", err)
+		http.Error(w, "internal error", http.StatusInternalServerError)
+		return
+	}
+	biSuitelk.Unlock()
+	_, err = io.Copy(w, bytes.NewReader(report))
+	if err != nil {
+		log.Printf("Could not send progress report: %v", err)
+	}
+}
+
+// modtime is the modification time of the resource to be served, or IsZero().
+// return value is whether this request is now complete.
+func checkLastModified(w http.ResponseWriter, r *http.Request, modtime time.Time) bool {
+	if modtime.IsZero() {
+		return false
+	}
+
+	// The Date-Modified header truncates sub-second precision, so
+	// use mtime < t+1s instead of mtime <= t to check for unmodified.
+	if t, err := time.Parse(http.TimeFormat, r.Header.Get("If-Modified-Since")); err == nil && modtime.Before(t.Add(1*time.Second)) {
+		h := w.Header()
+		delete(h, "Content-Type")
+		delete(h, "Content-Length")
+		w.WriteHeader(http.StatusNotModified)
+		return true
+	}
+	w.Header().Set("Last-Modified", modtime.UTC().Format(http.TimeFormat))
+	return false
+}
diff --git a/misc/buildbot/bot_test.go b/misc/buildbot/master/bot_test.go
similarity index 100%
rename from misc/buildbot/bot_test.go
rename to misc/buildbot/master/bot_test.go
diff --git a/misc/buildbot/master/master.go b/misc/buildbot/master/master.go
new file mode 100644
index 000000000..b7b66de44
--- /dev/null
+++ b/misc/buildbot/master/master.go
@@ -0,0 +1,1109 @@
+/*
+Copyright 2013 The Camlistore Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// The buildbot is Camlistore's continuous builder.
+// This master program monitors changes to the Go and Camlistore trees,
+// then rebuilds and restarts a builder when a change dictates as much.
+// It receives a report from a builder when it has finished running
+// a test suite, but it can also poll a builder before completion
+// to get a progress report.
+// It also serves the web requests.
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"html/template"
+	"io"
+	"io/ioutil"
+	"log"
+	"net"
+	"net/http"
+	"os"
+	"os/exec"
+	"os/signal"
+	"path/filepath"
+	"regexp"
+	"runtime"
+	"strings"
+	"sync"
+	"syscall"
+	"time"
+)
+
+const (
+	interval      = 60 * time.Second // polling frequency
+	historySize   = 30
+	maxStderrSize = 1 << 20 // Keep last 1 MB of logging.
+)
+
+var (
+	altCamliRevURL = flag.String("camlirevurl", "", "alternative URL to query about the latest camlistore revision hash (e.g camlistore.org/latesthash), to alleviate hitting too often the Camlistore git repo.")
+	builderOpts    = flag.String("builderopts", "", "list of comma separated options that will be passed to the builders (ex: '-verbose=true,-faketests=true,-skipgo1build=true'). Mainly for debugging.")
+	builderPort    = flag.String("builderport", "8081", "listening port for the builder bot")
+	builderSrc     = flag.String("buildersrc", "", "Go source file for the builder bot. For testing changes to the builder bot that haven't been committed yet.")
+	getGo          = flag.Bool("getgo", false, "Do not use the system's Go to build the builder, use the downloaded gotip instead.")
+	help           = flag.Bool("h", false, "show this help")
+	host           = flag.String("host", "0.0.0.0:8080", "listening hostname and port")
+	peers          = flag.String("peers", "", "comma separated list of host:port masters (besides this one) our builders will report to.")
+	verbose        = flag.Bool("verbose", false, "print what's going on")
+)
+
+var (
+	camliHeadHash           string
+	camliRoot               string
+	dbg                     *debugger
+	doBuildGo, doBuildCamli bool
+	goTipDir                string
+	goTipHash               string
+
+	historylk sync.Mutex
+	history   = make(map[string][]*biTestSuite) // key is the OS_Arch on which the tests were run
+
+	inProgresslk sync.Mutex
+	inProgress   *testSuite
+	// Process of the local builder bot, so we can kill it
+	// when we get killed.
+	builderProc *os.Process
+
+	// For "If-Modified-Since" requests on the status page.
+	// Updated every time a new test suite starts or ends.
+	lastModified time.Time
+
+	// Override the os.Stderr used by the default logger so we can provide
+	// more debug info on status page.
+	logStderr   = newLockedBuffer()
+	multiWriter io.Writer
+)
+
+// lockedBuffer protects all Write calls with a mutex.  Users of lockedBuffer
+// must wrap any calls to Bytes, and use of the resulting slice with calls to
+// Lock/Unlock.
+type lockedBuffer struct {
+	sync.Mutex // guards ringBuffer
+	*ringBuffer
+}
+
+func newLockedBuffer() *lockedBuffer {
+	return &lockedBuffer{ringBuffer: newRingBuffer(maxStderrSize)}
+}
+
+func (lb *lockedBuffer) Write(b []byte) (int, error) {
+	lb.Lock()
+	defer lb.Unlock()
+	return lb.ringBuffer.Write(b)
+}
+
+type ringBuffer struct {
+	buf []byte
+	off int // End of ring buffer.
+	l   int // Length of ring buffer filled.
+}
+
+func newRingBuffer(maxSize int) *ringBuffer {
+	return &ringBuffer{
+		buf: make([]byte, maxSize),
+	}
+}
+
+func (rb *ringBuffer) Bytes() []byte {
+	if (rb.off - rb.l) >= 0 {
+		// Partially full buffer with no wrap.
+		return rb.buf[rb.off-rb.l : rb.off]
+	}
+
+	// Buffer has been wrapped, copy second half then first half.
+	start := rb.off - rb.l
+	if start < 0 {
+		start = rb.off
+	}
+	b := make([]byte, 0, cap(rb.buf))
+	b = append(b, rb.buf[start:]...)
+	b = append(b, rb.buf[:start]...)
+	return b
+}
+
+func (rb *ringBuffer) Write(buf []byte) (int, error) {
+	ringLen := cap(rb.buf)
+	for i, b := range buf {
+		rb.buf[(rb.off+i)%ringLen] = b
+	}
+	rb.off = (rb.off + len(buf)) % ringLen
+	rb.l = rb.l + len(buf)
+	if rb.l > ringLen {
+		rb.l = ringLen
+	}
+	return len(buf), nil
+}
+
+var devcamBin = filepath.Join("bin", "devcam")
+var (
+	hgCloneGoTipCmd = newTask("hg", "clone", "-u", "tip", "https://code.google.com/p/go")
+	hgPullCmd       = newTask("hg", "pull")
+	hgLogCmd        = newTask("hg", "log", "-r", "tip", "--template", "{node}")
+	gitCloneCmd     = newTask("git", "clone", "https://camlistore.googlesource.com/camlistore")
+	gitPullCmd      = newTask("git", "pull")
+	gitRevCmd       = newTask("git", "rev-parse", "HEAD")
+	buildGoCmd      = newTask("./make.bash")
+)
+
+func usage() {
+	fmt.Fprintf(os.Stderr, "\t masterBot \n")
+	flag.PrintDefaults()
+	os.Exit(2)
+}
+
+type debugger struct {
+	lg *log.Logger
+}
+
+func (dbg *debugger) Printf(format string, v ...interface{}) {
+	if dbg != nil && *verbose {
+		dbg.lg.Printf(format, v...)
+	}
+}
+
+func (dbg *debugger) Println(v ...interface{}) {
+	if v == nil {
+		return
+	}
+	if dbg != nil && *verbose {
+		dbg.lg.Println(v...)
+	}
+}
+
+type task struct {
+	Program  string
+	Args     []string
+	Start    time.Time
+	Duration time.Duration
+	Err      string
+}
+
+func newTask(program string, args ...string) *task {
+	return &task{Program: program, Args: args}
+}
+
+func (t *task) String() string {
+	return fmt.Sprintf("%v %v", t.Program, t.Args)
+}
+
+func (t *task) run() (string, error) {
+	var err error
+	dbg.Println(t.String())
+	var stdout, stderr bytes.Buffer
+	cmd := exec.Command(t.Program, t.Args...)
+	cmd.Stdout = &stdout
+	cmd.Stderr = &stderr
+	if err = cmd.Run(); err != nil {
+		return "", fmt.Errorf("%v: %v", err, stderr.String())
+	}
+	return stdout.String(), nil
+}
+
+type testSuite struct {
+	Run       []*task
+	CamliHash string
+	GoHash    string
+	Err       string
+	Start     time.Time
+	IsTip     bool
+}
+
+type biTestSuite struct {
+	Local bool
+	Go1   *testSuite
+	GoTip *testSuite
+}
+
+func addTestSuites(OSArch string, ts *biTestSuite) {
+	if ts == nil {
+		return
+	}
+	historylk.Lock()
+	if ts.Local {
+		inProgresslk.Lock()
+		defer inProgresslk.Unlock()
+	}
+	defer historylk.Unlock()
+	historyOSArch := history[OSArch]
+	if len(historyOSArch) > historySize {
+		historyOSArch = append(historyOSArch[1:historySize], ts)
+	} else {
+		historyOSArch = append(historyOSArch, ts)
+	}
+	history[OSArch] = historyOSArch
+	if ts.Local {
+		inProgress = nil
+	}
+	lastModified = time.Now()
+}
+
+func main() {
+	flag.Usage = usage
+	flag.Parse()
+	if *help {
+		usage()
+	}
+
+	go handleSignals()
+	http.HandleFunc(okPrefix, okHandler)
+	http.HandleFunc(failPrefix, failHandler)
+	http.HandleFunc(progressPrefix, progressHandler)
+	http.HandleFunc(stderrPrefix, logHandler)
+	http.HandleFunc("/", statusHandler)
+	http.HandleFunc(reportPrefix, reportHandler)
+	go func() {
+		log.Printf("Now starting to listen on %v", *host)
+		if err := http.ListenAndServe(*host, nil); err != nil {
+			log.Fatalf("Could not start listening on %v: %v", *host, err)
+		}
+	}()
+	setup()
+
+	for {
+		goHash, err := pollGoChange()
+		if err != nil {
+			log.Fatal(err)
+		}
+		camliHash, err := pollCamliChange()
+		if err != nil {
+			log.Fatal(err)
+		}
+		if doBuildGo || doBuildCamli {
+			if err := buildBuilder(); err != nil {
+				log.Printf("Could not build builder bot: %v", err)
+				goto Sleep
+			}
+			cmd, err := startBuilder(goHash, camliHash)
+			if err != nil {
+				log.Printf("Could not start builder bot: %v", err)
+				goto Sleep
+			}
+			dbg.Println("Waiting for builder to finish")
+			if err := cmd.Wait(); err != nil {
+				log.Printf("builder finished with error: %v", err)
+			}
+			resetBuilderState()
+		}
+	Sleep:
+		tsk := newTask("time.Sleep", interval.String())
+		dbg.Println(tsk.String())
+		time.Sleep(interval)
+	}
+}
+
+func resetBuilderState() {
+	inProgresslk.Lock()
+	defer inProgresslk.Unlock()
+	builderProc = nil
+	inProgress = nil
+}
+
+func setup() {
+	// Install custom stderr for display in status webpage.
+	multiWriter = io.MultiWriter(logStderr, os.Stderr)
+	log.SetOutput(multiWriter)
+
+	var err error
+	cwd, err := os.Getwd()
+	if err != nil {
+		log.Fatal(err)
+	}
+	dbg = &debugger{log.New(multiWriter, "", log.LstdFlags)}
+
+	goTipDir, err = filepath.Abs("gotip")
+	if err != nil {
+		log.Fatal(err)
+	}
+	// if gotip dir exist, just reuse it
+	if _, err := os.Stat(goTipDir); err != nil {
+		if !os.IsNotExist(err) {
+			log.Fatalf("Could not stat %v: %v", goTipDir, err)
+		}
+		if _, err := hgCloneGoTipCmd.run(); err != nil {
+			log.Fatalf("Could not hg clone %v: %v", goTipDir, err)
+		}
+		if err := os.Rename("go", goTipDir); err != nil {
+			log.Fatalf("Could not rename go dir into %v: %v", goTipDir, err)
+		}
+	}
+
+	if _, err := exec.LookPath("go"); err != nil {
+		// Go was not found on this machine, but we've already
+		// downloaded gotip anyway, so let's install it and
+		// use it to build the builder bot.
+		*getGo = true
+	}
+
+	if *getGo {
+		// set PATH
+		splitter := ":"
+		switch runtime.GOOS {
+		case "windows":
+			splitter = ";"
+		case "plan9":
+			panic("unsupported OS")
+		}
+		p := os.Getenv("PATH")
+		if p == "" {
+			log.Fatal("PATH not set")
+		}
+		p = filepath.Join(goTipDir, "bin") + splitter + p
+		if err := os.Setenv("PATH", p); err != nil {
+			log.Fatalf("Could not set PATH to %v: %v", p, err)
+		}
+		// and check if we already have a gotip binary
+		if _, err := exec.LookPath("go"); err != nil {
+			// if not, build gotip
+			if err := buildGo(); err != nil {
+				log.Fatal(err)
+			}
+		}
+	}
+
+	// get camlistore source
+	if err := os.Chdir(cwd); err != nil {
+		log.Fatal("Could not cd to %v: %v", cwd, err)
+	}
+	camliRoot, err = filepath.Abs("camlistore.org")
+	if err != nil {
+		log.Fatal(err)
+	}
+	// if camlistore dir already exists, reuse it
+	if _, err := os.Stat(camliRoot); err != nil {
+		if !os.IsNotExist(err) {
+			log.Fatalf("Could not stat %v: %v", camliRoot, err)
+		}
+		cloneCmd := newTask(gitCloneCmd.Program, append(gitCloneCmd.Args, camliRoot)...)
+		if _, err := cloneCmd.run(); err != nil {
+			log.Fatalf("Could not git clone into %v: %v", camliRoot, err)
+		}
+	}
+}
+
+func buildGo() error {
+	if err := os.Chdir(filepath.Join(goTipDir, "src")); err != nil {
+		log.Fatalf("Could not cd to %v: %v", goTipDir, err)
+	}
+	if _, err := buildGoCmd.run(); err != nil {
+		return err
+	}
+	return nil
+}
+
+func handleSignals() {
+	c := make(chan os.Signal)
+	sigs := []os.Signal{syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT}
+	signal.Notify(c, sigs...)
+	for {
+		sig := <-c
+		sysSig, ok := sig.(syscall.Signal)
+		if !ok {
+			log.Fatal("Not a unix signal")
+		}
+		switch sysSig {
+		case syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT:
+			log.Printf("Received %v signal; cleaning up and terminating.", sig)
+			if builderProc != nil {
+				if err := builderProc.Kill(); err != nil {
+					log.Fatalf("Failed to kill our builder bot with pid %v: %v.", builderProc.Pid, err)
+				}
+			}
+			os.Exit(0)
+		default:
+			panic("should not get other signals here")
+		}
+	}
+}
+
+func pollGoChange() (string, error) {
+	doBuildGo = false
+	if err := os.Chdir(goTipDir); err != nil {
+		log.Fatalf("Could not cd to %v: %v", goTipDir, err)
+	}
+	tasks := []*task{
+		hgPullCmd,
+		hgLogCmd,
+	}
+	hash := ""
+	for _, t := range tasks {
+		out, err := t.run()
+		if err != nil {
+			if t.String() == hgPullCmd.String() {
+				log.Printf("Could not pull the Go tree with %v: %v", t.String(), err)
+				continue
+			}
+			log.Printf("Could not prepare the Go tree with %v: %v", t.String(), err)
+			return "", err
+		}
+		hash = strings.TrimRight(out, "\n")
+	}
+	dbg.Println("previous head in go tree: " + goTipHash)
+	dbg.Println("current head in go tree: " + hash)
+	if hash != "" && hash != goTipHash {
+		goTipHash = hash
+		doBuildGo = true
+		dbg.Println("Changes in go tree detected; a builder will be started.")
+	}
+	return hash, nil
+}
+
+func altCamliPolling() (string, error) {
+	resp, err := http.Get(*altCamliRevURL)
+	if err != nil {
+		return "", fmt.Errorf("Could not get camliHash through %v: %v", altCamliRevURL, err)
+	}
+	defer resp.Body.Close()
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return "", fmt.Errorf("Could not read camliHash from %v's response: %v", altCamliRevURL, err)
+	}
+	hash := strings.TrimSpace(string(body))
+	if hash == "" {
+		return "", fmt.Errorf("Got empty hash from %v's response", altCamliRevURL)
+	}
+	return hash, nil
+}
+
+func pollCamliChange() (string, error) {
+	doBuildCamli = false
+	altDone := false
+	var err error
+	rev := ""
+	if *altCamliRevURL != "" {
+		rev, err = altCamliPolling()
+		if err != nil {
+			log.Print(err)
+		} else {
+			altDone = true
+		}
+	}
+	if !altDone {
+		if err := os.Chdir(camliRoot); err != nil {
+			log.Fatalf("Could not cd to %v: %v", camliRoot, err)
+		}
+		tasks := []*task{
+			gitPullCmd,
+			gitRevCmd,
+		}
+		for _, t := range tasks {
+			out, err := t.run()
+			if err != nil {
+				if t.String() == gitPullCmd.String() {
+					log.Printf("Could not pull the Camli repo with %v: %v\n", t.String(), err)
+					continue
+				}
+				log.Printf("Could not prepare the Camli tree with %v: %v\n", t.String(), err)
+				return "", err
+			}
+			rev = strings.TrimRight(out, "\n")
+		}
+	}
+	dbg.Println("previous head in camli tree: " + camliHeadHash)
+	dbg.Println("current head in camli tree: " + rev)
+	if rev != "" && rev != camliHeadHash {
+		camliHeadHash = rev
+		doBuildCamli = true
+		dbg.Println("Changes in camli tree detected; a builder will be started.")
+	}
+	return rev, nil
+}
+
+const builderBotBin = "builderBot"
+
+func buildBuilder() error {
+	// TODO(Bill, mpl): import common auth module for both the master and builder. Or the multi-files
+	// approach. Whatever's cleaner.
+	source := *builderSrc
+	if source == "" {
+		source = filepath.Join(camliRoot, filepath.FromSlash("misc/buildbot/builder/builder.go"))
+	}
+	tsk := newTask(
+		"go",
+		"build",
+		"-o",
+		builderBotBin,
+		source,
+	)
+	if _, err := tsk.run(); err != nil {
+		return err
+	}
+	return nil
+
+}
+
+func startBuilder(goHash, camliHash string) (*exec.Cmd, error) {
+	dbg.Println("Starting builder bot")
+	builderHost := "localhost:" + *builderPort
+	ourHost, ourPort, err := net.SplitHostPort(*host)
+	if err != nil {
+		return nil, fmt.Errorf("Could not find out our host/port: %v", err)
+	}
+	if ourHost == "0.0.0.0" {
+		ourHost = "localhost"
+	}
+	masterHosts := ourHost + ":" + ourPort
+	if *peers != "" {
+		masterHosts += "," + *peers
+	}
+	args := []string{
+		"-host",
+		builderHost,
+		"-masterhosts",
+		masterHosts,
+	}
+	if *builderOpts != "" {
+		moreOpts := strings.Split(*builderOpts, ",")
+		args = append(args, moreOpts...)
+	}
+	cmd := exec.Command("./"+builderBotBin, args...)
+	cmd.Stdout = multiWriter
+	cmd.Stderr = multiWriter
+	if err := cmd.Start(); err != nil {
+		return nil, err
+	}
+	inProgresslk.Lock()
+	defer inProgresslk.Unlock()
+	builderProc = cmd.Process
+	inProgress = &testSuite{
+		Start:     time.Now(),
+		GoHash:    goHash,
+		CamliHash: camliHash,
+	}
+	return cmd, nil
+}
+
+var (
+	okPrefix       = "/ok/"
+	failPrefix     = "/fail/"
+	progressPrefix = "/progress"
+	currentPrefix  = "/current"
+	stderrPrefix   = "/stderr"
+	reportPrefix   = "/report"
+
+	statusTpl    = template.Must(template.New("status").Funcs(tmplFuncs).Parse(statusHTML))
+	taskTpl      = template.Must(template.New("task").Parse(taskHTML))
+	testSuiteTpl = template.Must(template.New("ok").Parse(testSuiteHTML))
+)
+
+var tmplFuncs = template.FuncMap{
+	"camliRepoURL": camliRepoURL,
+	"goRepoURL":    goRepoURL,
+	"shortHash":    shortHash,
+}
+
+var OSArchVersionTime = regexp.MustCompile(`(.*_.*)/(gotip|go1)/(.*)`)
+
+// unlocked; history needs to be protected from the caller.
+func getPastTestSuite(key string) (*testSuite, error) {
+	parts := OSArchVersionTime.FindStringSubmatch(key)
+	if parts == nil || len(parts) != 4 {
+		return nil, fmt.Errorf("bogus osArch/goversion/time url path: %v", key)
+	}
+	isGoTip := false
+	switch parts[2] {
+	case "gotip":
+		isGoTip = true
+	case "go1":
+	default:
+		return nil, fmt.Errorf("bogus go version in url path: %v", parts[2])
+	}
+	historyOSArch, ok := history[parts[1]]
+	if !ok {
+		return nil, fmt.Errorf("os %v not found in history", parts[1])
+	}
+	for _, v := range historyOSArch {
+		ts := v.Go1
+		if isGoTip {
+			ts = v.GoTip
+		}
+		if ts.Start.String() == parts[3] {
+			return ts, nil
+		}
+	}
+	return nil, fmt.Errorf("date %v not found in history for osArch %v", parts[3], parts[1])
+}
+
+// modtime is the modification time of the resource to be served, or IsZero().
+// return value is whether this request is now complete.
+func checkLastModified(w http.ResponseWriter, r *http.Request, modtime time.Time) bool {
+	if modtime.IsZero() {
+		return false
+	}
+
+	// The Date-Modified header truncates sub-second precision, so
+	// use mtime < t+1s instead of mtime <= t to check for unmodified.
+	if t, err := time.Parse(http.TimeFormat, r.Header.Get("If-Modified-Since")); err == nil && modtime.Before(t.Add(1*time.Second)) {
+		h := w.Header()
+		delete(h, "Content-Type")
+		delete(h, "Content-Length")
+		w.WriteHeader(http.StatusNotModified)
+		return true
+	}
+	w.Header().Set("Last-Modified", modtime.UTC().Format(http.TimeFormat))
+	return false
+}
+
+func reportHandler(w http.ResponseWriter, r *http.Request) {
+	if r.Method != "POST" {
+		log.Println("Invalid method for report handler")
+		http.Error(w, "Invalid method", http.StatusMethodNotAllowed)
+		return
+	}
+	body, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		log.Println("Invalid request for report handler")
+		http.Error(w, "Invalid method", http.StatusBadRequest)
+		return
+	}
+	defer r.Body.Close()
+	var report struct {
+		OSArch string
+		Ts     *biTestSuite
+	}
+	err = json.Unmarshal(body, &report)
+	if err != nil {
+		log.Printf("Could not decode builder report: %v", err)
+		http.Error(w, "internal error", http.StatusInternalServerError)
+		return
+	}
+	addTestSuites(report.OSArch, report.Ts)
+	fmt.Fprintf(w, "Report ok")
+}
+
+func logHandler(w http.ResponseWriter, r *http.Request) {
+	fmt.Fprintln(w, `<!doctype html>
+<html><meta http-equiv="refresh" content="5">
+<body><pre>`)
+	switch r.URL.Path {
+	case stderrPrefix:
+		logStderr.Lock()
+		_, err := w.Write(logStderr.Bytes())
+		logStderr.Unlock()
+		if err != nil {
+			log.Println("Error serving logStderr:", err)
+		}
+	default:
+		fmt.Fprintln(w, "Unknown log file path passed to logHandler:", r.URL.Path)
+		log.Println("Unknown log file path passed to logHandler:", r.URL.Path)
+	}
+}
+
+func okHandler(w http.ResponseWriter, r *http.Request) {
+	t := strings.Replace(r.URL.Path, okPrefix, "", -1)
+	historylk.Lock()
+	defer historylk.Unlock()
+	ts, err := getPastTestSuite(t)
+	if err != nil || len(ts.Run) == 0 {
+		http.NotFound(w, r)
+		return
+	}
+	lastTask := ts.Run[len(ts.Run)-1]
+	lastModTime := lastTask.Start.Add(lastTask.Duration)
+	if checkLastModified(w, r, lastModTime) {
+		return
+	}
+	var dat struct {
+		BiTs [2]*testSuite
+	}
+	if ts.IsTip {
+		dat.BiTs[1] = ts
+	} else {
+		dat.BiTs[0] = ts
+	}
+	err = testSuiteTpl.Execute(w, &dat)
+	if err != nil {
+		log.Printf("ok template: %v\n", err)
+	}
+}
+
+func progressHandler(w http.ResponseWriter, r *http.Request) {
+	if inProgress == nil {
+		http.Redirect(w, r, "/", http.StatusTemporaryRedirect)
+		return
+	}
+	// We only display the progress link and ask for progress for
+	// our local builder. The remote ones simply send their full report
+	// when they're done.
+	resp, err := http.Get("http://localhost:" + *builderPort + "/progress")
+	if err != nil {
+		log.Printf("Could not get a progress response from builder: %v", err)
+		http.Error(w, "internal error", http.StatusInternalServerError)
+		return
+	}
+	defer resp.Body.Close()
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		log.Printf("Could not read progress response from builder: %v", err)
+		http.Error(w, "internal error", http.StatusInternalServerError)
+		return
+	}
+	var ts biTestSuite
+	err = json.Unmarshal(body, &ts)
+	if err != nil {
+		log.Printf("Could not decode builder progress report: %v", err)
+		http.Error(w, "internal error", http.StatusInternalServerError)
+		return
+	}
+	lastModified = time.Now()
+	var dat struct {
+		BiTs [2]*testSuite
+	}
+	dat.BiTs[0] = ts.Go1
+	if ts.GoTip != nil && !ts.GoTip.Start.IsZero() {
+		dat.BiTs[1] = ts.GoTip
+	}
+	err = testSuiteTpl.Execute(w, &dat)
+	if err != nil {
+		log.Printf("progress template: %v\n", err)
+	}
+}
+
+func failHandler(w http.ResponseWriter, r *http.Request) {
+	t := strings.Replace(r.URL.Path, failPrefix, "", -1)
+	historylk.Lock()
+	defer historylk.Unlock()
+	ts, err := getPastTestSuite(t)
+	if err != nil || len(ts.Run) == 0 {
+		http.NotFound(w, r)
+		return
+	}
+	var failedTask *task
+	for _, v := range ts.Run {
+		if v.Err != "" {
+			failedTask = v
+			break
+		}
+	}
+	if failedTask == nil {
+		http.NotFound(w, r)
+		return
+	}
+	lastModTime := failedTask.Start.Add(failedTask.Duration)
+	if checkLastModified(w, r, lastModTime) {
+		return
+	}
+	failReport := struct {
+		TaskErr string
+		TsErr   string
+	}{
+		TaskErr: failedTask.String() + "\n" + failedTask.Err,
+		TsErr:   ts.Err,
+	}
+	err = taskTpl.Execute(w, &failReport)
+	if err != nil {
+		log.Printf("fail template: %v\n", err)
+	}
+}
+
+// unprotected read to history, caller needs to lock.
+func invertedHistory(OSArch string) (inverted []*biTestSuite) {
+	historyOSArch, ok := history[OSArch]
+	if !ok {
+		return nil
+	}
+	inverted = make([]*biTestSuite, len(historyOSArch))
+	endpos := len(historyOSArch) - 1
+	for k, v := range historyOSArch {
+		inverted[endpos-k] = v
+	}
+	return inverted
+}
+
+type statusReport struct {
+	OSArch   string
+	Hs       []*biTestSuite
+	Progress *testSuite
+}
+
+func statusHandler(w http.ResponseWriter, r *http.Request) {
+	historylk.Lock()
+	inProgresslk.Lock()
+	defer inProgresslk.Unlock()
+	defer historylk.Unlock()
+	var localOne *statusReport
+	if inProgress != nil {
+		localOne = &statusReport{
+			Progress: &testSuite{
+				Start:     inProgress.Start,
+				CamliHash: inProgress.CamliHash,
+				GoHash:    inProgress.GoHash,
+			},
+		}
+	}
+	var reports []*statusReport
+	for OSArch, historyOSArch := range history {
+		if len(historyOSArch) == 0 {
+			continue
+		}
+		hs := invertedHistory(OSArch)
+		if historyOSArch[0].Local {
+			if localOne == nil {
+				localOne = &statusReport{}
+			}
+			localOne.OSArch = OSArch
+			localOne.Hs = hs
+			continue
+		}
+		reports = append(reports, &statusReport{
+			OSArch: OSArch,
+			Hs:     hs,
+		})
+	}
+	if localOne != nil {
+		reports = append([]*statusReport{localOne}, reports...)
+	}
+	if checkLastModified(w, r, lastModified) {
+		return
+	}
+	err := statusTpl.Execute(w, reports)
+	if err != nil {
+		log.Printf("status template: %v\n", err)
+	}
+}
+
+// shortHash returns a short version of a hash.
+func shortHash(hash string) string {
+	if len(hash) > 12 {
+		hash = hash[:12]
+	}
+	return hash
+}
+
+func goRepoURL(hash string) string {
+	return "https://code.google.com/p/go/source/detail?r=" + hash
+}
+
+func camliRepoURL(hash string) string {
+	return "https://camlistore.googlesource.com/camlistore/+/" + hash
+}
+
+// style inspired from $GOROOT/misc/dashboard/app/build/ui.html
+var styleHTML = `
+<style>
+	body {
+		font-family: sans-serif;
+		padding: 0; margin: 0;
+	}
+	h1, h2 {
+		margin: 0;
+		padding: 5px;
+	}
+	h1 {
+		background: #eee;
+	}
+	h2 {
+		margin-top: 20px;
+	}
+	.build, .packages {
+		margin: 5px;
+		border-collapse: collapse;
+	}
+	.build td, .build th, .packages td, .packages th {
+		vertical-align: top;
+		padding: 2px 4px;
+		font-size: 10pt;
+	}
+	.build tr.commit:nth-child(2n) {
+		background-color: #f0f0f0;
+	}
+	.build .hash {
+		font-family: monospace;
+		font-size: 9pt;
+	}
+	.build .result {
+		text-align: center;
+		width: 2em;
+	}
+	.col-hash, .col-result {
+		border-right: solid 1px #ccc;
+	}
+	.build .arch {
+		font-size: 66%;
+		font-weight: normal;
+	}
+	.build .time {
+		color: #666;
+	}
+	.build .ok {
+		font-size: 83%;
+	}
+	a.ok {
+		text-decoration:none;
+	}
+	.build .desc, .build .time, .build .user {
+		white-space: nowrap;
+	}
+	.paginate {
+		padding: 0.5em;
+	}
+	.paginate a {
+		padding: 0.5em;
+		background: #eee;
+		color: blue;
+	}
+	.paginate a.inactive {
+		color: #999;
+	}
+	.pull-right {
+		float: right;
+	}
+	.fail {
+		color: #C00;
+	}
+</style>
+`
+
+var statusHTML = `
+<!DOCTYPE HTML>
+<html>
+	<head>
+		<title>Camlistore tests Dashboard</title>` +
+	styleHTML + `
+	</head>
+	<body>
+
+	<h1>Camlibot status<span class="pull-right"><a href="` + stderrPrefix + `">stderr</a></span></h1>
+
+	<table class="build">
+	<colgroup class="col-hash" span="1"></colgroup>
+	<colgroup class="build" span="1"></colgroup>
+	<colgroup class="build" span="1"></colgroup>
+	<colgroup class="user" span="1"></colgroup>
+	<colgroup class="user" span="1"></colgroup>
+	<tr>
+	<!-- extra row to make alternating colors use dark for first result -->
+	</tr>
+	{{range $report := .}}
+	<tr>
+	<th>{{$report.OSArch}}</th>
+	<th colspan="1">Go tip hash</th>
+	<th colspan="1">Camli HEAD hash</th>
+	<th colspan="1">Go1</th>
+	<th colspan="1">Gotip</th>
+	</tr>
+	{{if $report.Progress}}
+		<tr class="commit">
+			<td class="hash">{{$report.Progress.Start}}</td>
+			<td class="hash">
+				<a href="{{goRepoURL $report.Progress.GoHash}}">{{shortHash $report.Progress.GoHash}}</a>
+			</td>
+			<td class="hash">
+				<a href="{{camliRepoURL $report.Progress.CamliHash}}">{{shortHash $report.Progress.CamliHash}}</a>
+			</td>
+			<td class="result" colspan="2">
+				<a href="` + progressPrefix + `" class="ok">In progress</a>
+			</td>
+		</tr>
+	{{end}}
+	{{if $report.Hs}}
+		{{range $bits := $report.Hs}}
+			<tr class="commit">
+				<td class="hash">{{$bits.Go1.Start}}</td>
+				<td class="hash">
+					<a href="{{goRepoURL $bits.Go1.GoHash}}">{{shortHash $bits.Go1.GoHash}}</a>
+				</td>
+				<td class="hash">
+					<a href="{{camliRepoURL $bits.Go1.CamliHash}}">{{shortHash $bits.Go1.CamliHash}}</a>
+				</td>
+				<td class="result">
+				{{if $bits.Go1.Err}}
+					<a href="` + failPrefix + `{{$report.OSArch}}/go1/{{$bits.Go1.Start}}" class="fail">fail</a>
+				{{else}}
+					<a href="` + okPrefix + `{{$report.OSArch}}/go1/{{$bits.Go1.Start}}" class="ok">ok</a>
+				{{end}}
+				</td>
+				<td class="result">
+				{{if $bits.GoTip}}
+					{{if $bits.GoTip.Err}}
+						<a href="` + failPrefix + `{{$report.OSArch}}/gotip/{{$bits.GoTip.Start}}" class="fail">fail</a>
+					{{else}}
+						<a href="` + okPrefix + `{{$report.OSArch}}/gotip/{{$bits.GoTip.Start}}" class="ok">ok</a>
+					{{end}}
+				{{else}}
+					<a href="` + currentPrefix + `" class="ok">In progress</a>
+				{{end}}
+				</td>
+			</tr>
+		{{end}}
+	{{end}}
+	<tr>
+	<td colspan="5">&nbsp;</td>
+	</tr>
+	{{end}}
+	</table>
+
+	</body>
+</html>
+`
+
+var testSuiteHTML = `
+<!DOCTYPE HTML>
+<html>
+	<head>
+		<title>Camlistore tests Dashboard</title>` +
+	styleHTML + `
+	</head>
+	<body>
+	{{range $ts := .BiTs}}
+		{{if $ts}}
+		<h2> Testsuite for {{if $ts.IsTip}}Go tip{{else}}Go 1{{end}} at {{$ts.Start}} </h2>
+		<table class="build">
+		<colgroup class="col-result" span="1"></colgroup>
+		<colgroup class="col-result" span="1"></colgroup>
+		<tr>
+			<!-- extra row to make alternating colors use dark for first result -->
+		</tr>
+		<tr>
+			<th colspan="1">Step</th>
+			<th colspan="1">Duration</th>
+		</tr>
+		{{range $k, $tsk := $ts.Run}}
+		<tr>
+			<td>{{printf "%v" $tsk}}</td>
+			<td>{{$tsk.Duration}}</td>
+		</tr>
+		{{end}}
+		</table>
+		{{end}}
+	{{end}}
+	</body>
+</html>
+`
+
+var taskHTML = `
+<!DOCTYPE HTML>
+<html>
+	<head>
+		<title>Camlistore tests Dashboard</title>
+	</head>
+	<body>
+{{if .TaskErr}}
+	<h2>Task:</h2>
+	<p>{{.TaskErr}}</p>
+{{end}}
+{{if .TsErr}}
+	<h2>Error:</h2>
+	<pre>
+	{{.TsErr}}
+	</pre>
+{{end}}
+	</body>
+</html>
+`