Merge "builder: add client side Basic Auth support."

misc/buildbot/builder/builder.go

@@ -24,6 +24,7 @@ limitations under the License.
 package main
 
 import (
+	"bufio"
 	"bytes"
 	"encoding/json"
 	"errors"
@@ -33,6 +34,7 @@ import (
 	"io/ioutil"
 	"log"
 	"net/http"
+	"net/url"
 	"os"
 	"os/exec"
 	"os/signal"
@@ -43,6 +45,8 @@ import (
 	"sync"
 	"syscall"
 	"time"
+
+	"camlistore.org/pkg/osutil"
 )
 
 const (
@@ -389,6 +393,47 @@ func endOfSuite(err error) {
 	}
 }
 
+func masterHostsReader(r io.Reader) ([]string, error) {
+	hosts := []string{}
+	scanner := bufio.NewScanner(r)
+	for scanner.Scan() {
+		l := scanner.Text()
+		u, err := url.Parse(l)
+		if err != nil {
+			return nil, err
+		}
+		if u.Host == "" {
+			return nil, fmt.Errorf("URL missing Host: %q", l)
+		}
+		hosts = append(hosts, u.String())
+	}
+	if err := scanner.Err(); err != nil {
+		return nil, err
+	}
+	return hosts, nil
+}
+
+var masterHostsFile = filepath.Join(osutil.CamliConfigDir(), "builderbot-config")
+
+func loadMasterHosts() error {
+	r, err := os.Open(masterHostsFile)
+	if err != nil {
+		return err
+	}
+	defer r.Close()
+
+	hosts, err := masterHostsReader(r)
+	if err != nil {
+		return err
+	}
+	if *masterHosts != "" {
+		*masterHosts += ","
+	}
+	log.Println("Additional host(s) to send our build reports:", hosts)
+	*masterHosts += strings.Join(hosts, ",")
+	return nil
+}
+
 func setup() {
 	var err error
 	defaultPATH = os.Getenv("PATH")
@@ -398,6 +443,16 @@ func setup() {
 	log.SetPrefix("BUILDER: ")
 	dbg = &debugger{log.New(os.Stderr, "BUILDER: ", log.LstdFlags)}
 
+	err = loadMasterHosts()
+	if err != nil {
+		if os.IsNotExist(err) {
+			log.Printf("%q missing. No additional remote master(s) will receive build report.", masterHostsFile)
+		} else {
+			log.Println("Error parsing master hosts file %q: %v",
+				masterHostsFile, err)
+		}
+	}
+
 	// the OS we run on
 	if *ourOS == "" {
 		*ourOS = runtime.GOOS
@@ -870,6 +925,43 @@ func runTests() error {
 
 const reportPrefix = "/report"
 
+func postToURL(u string, r io.Reader) (*http.Response, error) {
+	// Default to plain HTTP.
+	if !(strings.HasPrefix(u, "http://") || strings.HasPrefix(u, "https://")) {
+		u = "http://" + u
+	}
+	uri, err := url.Parse(u)
+	if err != nil {
+		return nil, err
+	}
+
+	// If the URL explicitly specifies "/" or something else, we'll POST to
+	// that, otherwise default to build-time default.
+	if uri.Path == "" {
+		uri.Path = reportPrefix
+	}
+
+	// Save user/pass if specified in the URL.
+	user := uri.User
+	// But don't send user/pass in URL to server.
+	uri.User = nil
+
+	req, err := http.NewRequest("POST", uri.String(), r)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("Content-Type", "text/javascript")
+	// If user/pass set on original URL, set the auth header for the request.
+	if user != nil {
+		pass, ok := user.Password()
+		if !ok {
+			log.Println("Password not set for", user.Username(), "in", u)
+		}
+		req.SetBasicAuth(user.Username(), pass)
+	}
+	return http.DefaultClient.Do(req)
+}
+
 func sendReport() {
 	biSuitelk.Lock()
 	// we make a copy so we can release the lock quickly enough
@@ -888,7 +980,6 @@ func sendReport() {
 		Ts:     currentBiSuiteCpy,
 	}
 	for _, v := range masters {
-		reportURL := "http://" + v + reportPrefix
 		// TODO(mpl): ipv6 too I suppose. just make a IsLocalhost func or whatever.
 		// probably can borrow something from camli code for that.
 		if strings.HasPrefix(v, "localhost") || strings.HasPrefix(v, "127.0.0.1") {
@@ -899,10 +990,10 @@ func sendReport() {
 		report, err := json.MarshalIndent(toReport, "", "  ")
 		if err != nil {
 			log.Printf("JSON serialization error: %v", err)
-			continue
+			return
 		}
 		r := bytes.NewReader(report)
-		resp, err := http.Post(reportURL, "text/javascript", r)
+		resp, err := postToURL(v, r)
 		if err != nil {
 			log.Printf("Could not send report: %v", err)
 			continue

misc/buildbot/builder/builder_test.go

@@ -0,0 +1,128 @@
+package main
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"strings"
+	"testing"
+)
+
+func TestSendReport(t *testing.T) {
+	wants := []struct {
+		host       string
+		authHeader bool
+		path       string
+	}{
+		// TODO(wathiede): add https tests if needed.
+		{
+			host:       "http://HOST",
+			authHeader: false,
+			path:       reportPrefix,
+		},
+		{
+			host:       "http://user:pass@HOST",
+			authHeader: true,
+			path:       reportPrefix,
+		},
+		{
+			host:       "http://user:pass@HOST/",
+			authHeader: true,
+			path:       "/",
+		},
+		{
+			host:       "http://user:pass@HOST/other",
+			authHeader: true,
+			path:       "/other",
+		},
+	}
+
+	reqNum := 0
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		defer func() { reqNum++ }()
+		if reqNum > len(wants) {
+			t.Fatal("Only expected", len(wants), "requests, got", reqNum)
+		}
+		want := wants[reqNum]
+
+		gotAuthHeader := r.Header.Get("Authorization") != ""
+		if want.authHeader != gotAuthHeader {
+			if gotAuthHeader {
+				t.Error("Got unexpected Authorization header")
+			} else {
+				t.Error("Authorization header missing")
+			}
+		}
+
+		if r.URL.Path != want.path {
+			t.Error("Got path", r.URL.Path, "want", want.path)
+		}
+	}))
+	defer ts.Close()
+
+	testU, err := url.Parse(ts.URL)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var hosts []string
+	for _, want := range wants {
+		u, err := url.Parse(want.host)
+		if err != nil {
+			t.Fatal(err)
+		}
+		u.Host = testU.Host
+		hosts = append(hosts, u.String())
+	}
+	// override --masterhosts.
+	*masterHosts = strings.Join(hosts, ",")
+	currentBiSuite = &biTestSuite{}
+
+	sendReport()
+
+	if reqNum != len(wants) {
+		t.Error("Expected", len(wants), "requests, only got", reqNum)
+	}
+}
+
+func TestMasterHostsReader(t *testing.T) {
+	datum := []struct {
+		body string
+		good bool
+		num  int
+	}{
+		{
+			body: "http://host1",
+			good: true,
+			num:  1,
+		},
+		{
+			body: "http://host1\n",
+			good: true,
+			num:  1,
+		},
+		{
+			body: "# Hello\nhttp://host1\n",
+			good: false,
+			num:  0,
+		},
+		{
+			body: "http://host1\nhttp://host2\n",
+			good: true,
+			num:  2,
+		},
+	}
+
+	for i, d := range datum {
+		hosts, err := masterHostsReader(strings.NewReader(d.body))
+		if d.good && err != nil {
+			t.Error(i, "Unexpected parse failure:", err)
+		}
+		if !d.good && err == nil {
+			t.Error(i, "Expected parse failure, but succeeded")
+		}
+
+		if len(hosts) != d.num {
+			t.Error(i, "Expected", d.num, "hosts, got", len(hosts), hosts)
+		}
+	}
+}

misc/buildbot/master/master.go

@@ -382,7 +382,7 @@ func setup() {
 	if err := os.Chdir(defaultDir); err != nil {
 		log.Fatalf("Could not cd to %v: %v", defaultDir, err)
 	}
-	camliRoot, err = filepath.Abs("camlistore.org")
+	camliRoot, err = filepath.Abs("src/camlistore.org")
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -396,6 +396,10 @@ func setup() {
 			log.Fatalf("Could not git clone into %v: %v", camliRoot, err)
 		}
 	}
+	// override GOPATH to only point to our freshly updated camlistore source.
+	if err := os.Setenv("GOPATH", defaultDir); err != nil {
+		log.Fatalf("Could not set GOPATH to %v: %v", defaultDir, err)
+	}
 }
 
 func buildGo() error {