diff --git a/misc/buildbot/builder/builder.go b/misc/buildbot/builder/builder.go index bd7b2560f..9f2f5503a 100644 --- a/misc/buildbot/builder/builder.go +++ b/misc/buildbot/builder/builder.go @@ -24,6 +24,7 @@ limitations under the License. package main import ( + "bufio" "bytes" "encoding/json" "errors" @@ -33,6 +34,7 @@ import ( "io/ioutil" "log" "net/http" + "net/url" "os" "os/exec" "os/signal" @@ -43,6 +45,8 @@ import ( "sync" "syscall" "time" + + "camlistore.org/pkg/osutil" ) const ( @@ -389,6 +393,47 @@ func endOfSuite(err error) { } } +func masterHostsReader(r io.Reader) ([]string, error) { + hosts := []string{} + scanner := bufio.NewScanner(r) + for scanner.Scan() { + l := scanner.Text() + u, err := url.Parse(l) + if err != nil { + return nil, err + } + if u.Host == "" { + return nil, fmt.Errorf("URL missing Host: %q", l) + } + hosts = append(hosts, u.String()) + } + if err := scanner.Err(); err != nil { + return nil, err + } + return hosts, nil +} + +var masterHostsFile = filepath.Join(osutil.CamliConfigDir(), "builderbot-config") + +func loadMasterHosts() error { + r, err := os.Open(masterHostsFile) + if err != nil { + return err + } + defer r.Close() + + hosts, err := masterHostsReader(r) + if err != nil { + return err + } + if *masterHosts != "" { + *masterHosts += "," + } + log.Println("Additional host(s) to send our build reports:", hosts) + *masterHosts += strings.Join(hosts, ",") + return nil +} + func setup() { var err error defaultPATH = os.Getenv("PATH") @@ -398,6 +443,16 @@ func setup() { log.SetPrefix("BUILDER: ") dbg = &debugger{log.New(os.Stderr, "BUILDER: ", log.LstdFlags)} + err = loadMasterHosts() + if err != nil { + if os.IsNotExist(err) { + log.Printf("%q missing. No additional remote master(s) will receive build report.", masterHostsFile) + } else { + log.Println("Error parsing master hosts file %q: %v", + masterHostsFile, err) + } + } + // the OS we run on if *ourOS == "" { *ourOS = runtime.GOOS @@ -870,6 +925,43 @@ func runTests() error { const reportPrefix = "/report" +func postToURL(u string, r io.Reader) (*http.Response, error) { + // Default to plain HTTP. + if !(strings.HasPrefix(u, "http://") || strings.HasPrefix(u, "https://")) { + u = "http://" + u + } + uri, err := url.Parse(u) + if err != nil { + return nil, err + } + + // If the URL explicitly specifies "/" or something else, we'll POST to + // that, otherwise default to build-time default. + if uri.Path == "" { + uri.Path = reportPrefix + } + + // Save user/pass if specified in the URL. + user := uri.User + // But don't send user/pass in URL to server. + uri.User = nil + + req, err := http.NewRequest("POST", uri.String(), r) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "text/javascript") + // If user/pass set on original URL, set the auth header for the request. + if user != nil { + pass, ok := user.Password() + if !ok { + log.Println("Password not set for", user.Username(), "in", u) + } + req.SetBasicAuth(user.Username(), pass) + } + return http.DefaultClient.Do(req) +} + func sendReport() { biSuitelk.Lock() // we make a copy so we can release the lock quickly enough @@ -888,7 +980,6 @@ func sendReport() { Ts: currentBiSuiteCpy, } for _, v := range masters { - reportURL := "http://" + v + reportPrefix // TODO(mpl): ipv6 too I suppose. just make a IsLocalhost func or whatever. // probably can borrow something from camli code for that. if strings.HasPrefix(v, "localhost") || strings.HasPrefix(v, "127.0.0.1") { @@ -899,10 +990,10 @@ func sendReport() { report, err := json.MarshalIndent(toReport, "", " ") if err != nil { log.Printf("JSON serialization error: %v", err) - continue + return } r := bytes.NewReader(report) - resp, err := http.Post(reportURL, "text/javascript", r) + resp, err := postToURL(v, r) if err != nil { log.Printf("Could not send report: %v", err) continue diff --git a/misc/buildbot/builder/builder_test.go b/misc/buildbot/builder/builder_test.go new file mode 100644 index 000000000..36729a02b --- /dev/null +++ b/misc/buildbot/builder/builder_test.go @@ -0,0 +1,128 @@ +package main + +import ( + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" +) + +func TestSendReport(t *testing.T) { + wants := []struct { + host string + authHeader bool + path string + }{ + // TODO(wathiede): add https tests if needed. + { + host: "http://HOST", + authHeader: false, + path: reportPrefix, + }, + { + host: "http://user:pass@HOST", + authHeader: true, + path: reportPrefix, + }, + { + host: "http://user:pass@HOST/", + authHeader: true, + path: "/", + }, + { + host: "http://user:pass@HOST/other", + authHeader: true, + path: "/other", + }, + } + + reqNum := 0 + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + defer func() { reqNum++ }() + if reqNum > len(wants) { + t.Fatal("Only expected", len(wants), "requests, got", reqNum) + } + want := wants[reqNum] + + gotAuthHeader := r.Header.Get("Authorization") != "" + if want.authHeader != gotAuthHeader { + if gotAuthHeader { + t.Error("Got unexpected Authorization header") + } else { + t.Error("Authorization header missing") + } + } + + if r.URL.Path != want.path { + t.Error("Got path", r.URL.Path, "want", want.path) + } + })) + defer ts.Close() + + testU, err := url.Parse(ts.URL) + if err != nil { + t.Fatal(err) + } + var hosts []string + for _, want := range wants { + u, err := url.Parse(want.host) + if err != nil { + t.Fatal(err) + } + u.Host = testU.Host + hosts = append(hosts, u.String()) + } + // override --masterhosts. + *masterHosts = strings.Join(hosts, ",") + currentBiSuite = &biTestSuite{} + + sendReport() + + if reqNum != len(wants) { + t.Error("Expected", len(wants), "requests, only got", reqNum) + } +} + +func TestMasterHostsReader(t *testing.T) { + datum := []struct { + body string + good bool + num int + }{ + { + body: "http://host1", + good: true, + num: 1, + }, + { + body: "http://host1\n", + good: true, + num: 1, + }, + { + body: "# Hello\nhttp://host1\n", + good: false, + num: 0, + }, + { + body: "http://host1\nhttp://host2\n", + good: true, + num: 2, + }, + } + + for i, d := range datum { + hosts, err := masterHostsReader(strings.NewReader(d.body)) + if d.good && err != nil { + t.Error(i, "Unexpected parse failure:", err) + } + if !d.good && err == nil { + t.Error(i, "Expected parse failure, but succeeded") + } + + if len(hosts) != d.num { + t.Error(i, "Expected", d.num, "hosts, got", len(hosts), hosts) + } + } +} diff --git a/misc/buildbot/master/master.go b/misc/buildbot/master/master.go index 1f9121a9a..a0c03b6e3 100644 --- a/misc/buildbot/master/master.go +++ b/misc/buildbot/master/master.go @@ -382,7 +382,7 @@ func setup() { if err := os.Chdir(defaultDir); err != nil { log.Fatalf("Could not cd to %v: %v", defaultDir, err) } - camliRoot, err = filepath.Abs("camlistore.org") + camliRoot, err = filepath.Abs("src/camlistore.org") if err != nil { log.Fatal(err) } @@ -396,6 +396,10 @@ func setup() { log.Fatalf("Could not git clone into %v: %v", camliRoot, err) } } + // override GOPATH to only point to our freshly updated camlistore source. + if err := os.Setenv("GOPATH", defaultDir); err != nil { + log.Fatalf("Could not set GOPATH to %v: %v", defaultDir, err) + } } func buildGo() error {