builder: add client side Basic Auth support.

Adds support for specifying additional masters to send results to.
The config file, named "builderbot-config", must be located in
osutil.CamliConfigDir(); on Unix, it will be under
$XDG_CONFIG_HOME/camlistore/, if XDG_CONFIG_HOME set, or
~/.config/camlistore/.  On Windows it will be under %APPDATA%\Camlistore\.
The expected format is one host per line, comments are not allowed and used
only for illustration below.  Some examples:

    # Post to default path on host1, no user auth.
    http://host1
    # Post to / on host2, no user auth.
    http://host2/
    # Post to /other/path on host3, no user auth.
    http://host3/other/path
    # Post to default path on host4, user 'user', password 'pass'.
    http://user:pass@host4
    # Post to / on host4 port 7070, user 'user', password 'pass'.
    http://user:pass@host4:7070/

This change also explicitly sets GOPATH to the copy of camlistore.org checked
out by the master before building the builder, and starting the build run.
This ensures we're not building against a random checkout of camlistore
pointed to by the user's environment.

The directory of the checked out source is moved from ${PWD}/camlistore.org
-> ${PWD}/src/camlistore.org so GOPATH can be set to $PWD and the requirement
of packages being under src/ is met.

Change-Id: I6e121c0aae9dae0c1832f782fa32619434ce9d2c

misc/buildbot/builder/builder.go

@@ -24,6 +24,7 @@ limitations under the License.
 package main
 
 import (
+	"bufio"
 	"bytes"
 	"encoding/json"
 	"errors"
@@ -33,6 +34,7 @@ import (
 	"io/ioutil"
 	"log"
 	"net/http"
+	"net/url"
 	"os"
 	"os/exec"
 	"os/signal"
@@ -43,6 +45,8 @@ import (
 	"sync"
 	"syscall"
 	"time"
+
+	"camlistore.org/pkg/osutil"
 )
 
 const (
@@ -389,6 +393,47 @@ func endOfSuite(err error) {
 	}
 }
 
+func masterHostsReader(r io.Reader) ([]string, error) {
+	hosts := []string{}
+	scanner := bufio.NewScanner(r)
+	for scanner.Scan() {
+		l := scanner.Text()
+		u, err := url.Parse(l)
+		if err != nil {
+			return nil, err
+		}
+		if u.Host == "" {
+			return nil, fmt.Errorf("URL missing Host: %q", l)
+		}
+		hosts = append(hosts, u.String())
+	}
+	if err := scanner.Err(); err != nil {
+		return nil, err
+	}
+	return hosts, nil
+}
+
+var masterHostsFile = filepath.Join(osutil.CamliConfigDir(), "builderbot-config")
+
+func loadMasterHosts() error {
+	r, err := os.Open(masterHostsFile)
+	if err != nil {
+		return err
+	}
+	defer r.Close()
+
+	hosts, err := masterHostsReader(r)
+	if err != nil {
+		return err
+	}
+	if *masterHosts != "" {
+		*masterHosts += ","
+	}
+	log.Println("Additional host(s) to send our build reports:", hosts)
+	*masterHosts += strings.Join(hosts, ",")
+	return nil
+}
+
 func setup() {
 	var err error
 	defaultPATH = os.Getenv("PATH")
@@ -398,6 +443,16 @@ func setup() {
 	log.SetPrefix("BUILDER: ")
 	dbg = &debugger{log.New(os.Stderr, "BUILDER: ", log.LstdFlags)}
 
+	err = loadMasterHosts()
+	if err != nil {
+		if os.IsNotExist(err) {
+			log.Printf("%q missing. No additional remote master(s) will receive build report.", masterHostsFile)
+		} else {
+			log.Println("Error parsing master hosts file %q: %v",
+				masterHostsFile, err)
+		}
+	}
+
 	// the OS we run on
 	if *ourOS == "" {
 		*ourOS = runtime.GOOS
@@ -870,6 +925,43 @@ func runTests() error {
 
 const reportPrefix = "/report"
 
+func postToURL(u string, r io.Reader) (*http.Response, error) {
+	// Default to plain HTTP.
+	if !(strings.HasPrefix(u, "http://") || strings.HasPrefix(u, "https://")) {
+		u = "http://" + u
+	}
+	uri, err := url.Parse(u)
+	if err != nil {
+		return nil, err
+	}
+
+	// If the URL explicitly specifies "/" or something else, we'll POST to
+	// that, otherwise default to build-time default.
+	if uri.Path == "" {
+		uri.Path = reportPrefix
+	}
+
+	// Save user/pass if specified in the URL.
+	user := uri.User
+	// But don't send user/pass in URL to server.
+	uri.User = nil
+
+	req, err := http.NewRequest("POST", uri.String(), r)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("Content-Type", "text/javascript")
+	// If user/pass set on original URL, set the auth header for the request.
+	if user != nil {
+		pass, ok := user.Password()
+		if !ok {
+			log.Println("Password not set for", user.Username(), "in", u)
+		}
+		req.SetBasicAuth(user.Username(), pass)
+	}
+	return http.DefaultClient.Do(req)
+}
+
 func sendReport() {
 	biSuitelk.Lock()
 	// we make a copy so we can release the lock quickly enough
@@ -888,7 +980,6 @@ func sendReport() {
 		Ts:     currentBiSuiteCpy,
 	}
 	for _, v := range masters {
-		reportURL := "http://" + v + reportPrefix
 		// TODO(mpl): ipv6 too I suppose. just make a IsLocalhost func or whatever.
 		// probably can borrow something from camli code for that.
 		if strings.HasPrefix(v, "localhost") || strings.HasPrefix(v, "127.0.0.1") {
@@ -899,10 +990,10 @@ func sendReport() {
 		report, err := json.MarshalIndent(toReport, "", "  ")
 		if err != nil {
 			log.Printf("JSON serialization error: %v", err)
-			continue
+			return
 		}
 		r := bytes.NewReader(report)
-		resp, err := http.Post(reportURL, "text/javascript", r)
+		resp, err := postToURL(v, r)
 		if err != nil {
 			log.Printf("Could not send report: %v", err)
 			continue

misc/buildbot/builder/builder_test.go

@@ -0,0 +1,128 @@
+package main
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"strings"
+	"testing"
+)
+
+func TestSendReport(t *testing.T) {
+	wants := []struct {
+		host       string
+		authHeader bool
+		path       string
+	}{
+		// TODO(wathiede): add https tests if needed.
+		{
+			host:       "http://HOST",
+			authHeader: false,
+			path:       reportPrefix,
+		},
+		{
+			host:       "http://user:pass@HOST",
+			authHeader: true,
+			path:       reportPrefix,
+		},
+		{
+			host:       "http://user:pass@HOST/",
+			authHeader: true,
+			path:       "/",
+		},
+		{
+			host:       "http://user:pass@HOST/other",
+			authHeader: true,
+			path:       "/other",
+		},
+	}
+
+	reqNum := 0
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		defer func() { reqNum++ }()
+		if reqNum > len(wants) {
+			t.Fatal("Only expected", len(wants), "requests, got", reqNum)
+		}
+		want := wants[reqNum]
+
+		gotAuthHeader := r.Header.Get("Authorization") != ""
+		if want.authHeader != gotAuthHeader {
+			if gotAuthHeader {
+				t.Error("Got unexpected Authorization header")
+			} else {
+				t.Error("Authorization header missing")
+			}
+		}
+
+		if r.URL.Path != want.path {
+			t.Error("Got path", r.URL.Path, "want", want.path)
+		}
+	}))
+	defer ts.Close()
+
+	testU, err := url.Parse(ts.URL)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var hosts []string
+	for _, want := range wants {
+		u, err := url.Parse(want.host)
+		if err != nil {
+			t.Fatal(err)
+		}
+		u.Host = testU.Host
+		hosts = append(hosts, u.String())
+	}
+	// override --masterhosts.
+	*masterHosts = strings.Join(hosts, ",")
+	currentBiSuite = &biTestSuite{}
+
+	sendReport()
+
+	if reqNum != len(wants) {
+		t.Error("Expected", len(wants), "requests, only got", reqNum)
+	}
+}
+
+func TestMasterHostsReader(t *testing.T) {
+	datum := []struct {
+		body string
+		good bool
+		num  int
+	}{
+		{
+			body: "http://host1",
+			good: true,
+			num:  1,
+		},
+		{
+			body: "http://host1\n",
+			good: true,
+			num:  1,
+		},
+		{
+			body: "# Hello\nhttp://host1\n",
+			good: false,
+			num:  0,
+		},
+		{
+			body: "http://host1\nhttp://host2\n",
+			good: true,
+			num:  2,
+		},
+	}
+
+	for i, d := range datum {
+		hosts, err := masterHostsReader(strings.NewReader(d.body))
+		if d.good && err != nil {
+			t.Error(i, "Unexpected parse failure:", err)
+		}
+		if !d.good && err == nil {
+			t.Error(i, "Expected parse failure, but succeeded")
+		}
+
+		if len(hosts) != d.num {
+			t.Error(i, "Expected", d.num, "hosts, got", len(hosts), hosts)
+		}
+	}
+}

misc/buildbot/master/master.go

@@ -382,7 +382,7 @@ func setup() {
 	if err := os.Chdir(defaultDir); err != nil {
 		log.Fatalf("Could not cd to %v: %v", defaultDir, err)
 	}
-	camliRoot, err = filepath.Abs("camlistore.org")
+	camliRoot, err = filepath.Abs("src/camlistore.org")
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -396,6 +396,10 @@ func setup() {
 			log.Fatalf("Could not git clone into %v: %v", camliRoot, err)
 		}
 	}
+	// override GOPATH to only point to our freshly updated camlistore source.
+	if err := os.Setenv("GOPATH", defaultDir); err != nil {
+		log.Fatalf("Could not set GOPATH to %v: %v", defaultDir, err)
+	}
 }
 
 func buildGo() error {