Merge "buildbot/master: add Basic Auth support."

This commit is contained in:
Mathieu Lonjaret 2013-11-22 15:29:06 +00:00 committed by Gerrit Code Review
commit f8348c5875
7 changed files with 443 additions and 206 deletions

View File

@ -26,6 +26,7 @@ package main
import (
"bufio"
"bytes"
"crypto/tls"
"encoding/json"
"errors"
"flag"
@ -67,6 +68,7 @@ var (
ourOS = flag.String("os", "", "The OS we report the master(s). Defaults to runtime.GOOS.")
skipGo1Build = flag.Bool("skipgo1build", false, "skip initial go1 build, for debugging and quickly going to the next steps.")
verbose = flag.Bool("verbose", false, "print what's going on")
skipTLSCheck = flag.Bool("skiptlscheck", false, "accept any certificate presented by server when uploading results.")
)
var (
@ -75,6 +77,7 @@ var (
camliHeadHash string
camliRoot string
camputCacheDir string
client = http.DefaultClient
dbg *debugger
defaultPATH string
doBuildGo, doBuildCamli bool
@ -233,6 +236,13 @@ func main() {
usage()
}
if *skipTLSCheck {
tr := &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
}
client = &http.Client{Transport: tr}
}
go handleSignals()
http.HandleFunc("/progress", progressHandler)
go func() {
@ -959,7 +969,7 @@ func postToURL(u string, r io.Reader) (*http.Response, error) {
}
req.SetBasicAuth(user.Username(), pass)
}
return http.DefaultClient.Do(req)
return client.Do(req)
}
func sendReport() {

View File

@ -25,6 +25,8 @@ package main
import (
"bytes"
"crypto/sha1"
"encoding/hex"
"encoding/json"
"flag"
"fmt"
@ -44,6 +46,9 @@ import (
"sync"
"syscall"
"time"
"camlistore.org/pkg/httputil"
"camlistore.org/pkg/osutil"
)
const (
@ -62,6 +67,8 @@ var (
host = flag.String("host", "0.0.0.0:8080", "listening hostname and port")
peers = flag.String("peers", "", "comma separated list of host:port masters (besides this one) our builders will report to.")
verbose = flag.Bool("verbose", false, "print what's going on")
certFile = flag.String("tlsCertFile", "", "TLS public key in PEM format. Must be used with -tlsKeyFile")
keyFile = flag.String("tlsKeyFile", "", "TLS private key in PEM format. Must be used with -tlsCertFile")
)
var (
@ -90,6 +97,9 @@ var (
// more debug info on status page.
logStderr = newLockedBuffer()
multiWriter io.Writer
// Set after flag parsing based on certFile & keyFile.
useTLS bool
)
// lockedBuffer protects all Write calls with a mutex. Users of lockedBuffer
@ -152,6 +162,115 @@ func (rb *ringBuffer) Write(buf []byte) (int, error) {
return len(buf), nil
}
var userAuthFile = filepath.Join(osutil.CamliConfigDir(), "masterbot-config.json")
type userAuth struct {
sync.Mutex // guards userPass map.
userPass map[string]string
configFile string
pollInterval time.Duration
lastModTime time.Time
}
func newUserAuth(configFile string) (*userAuth, error) {
ua := &userAuth{
configFile: configFile,
pollInterval: time.Minute,
}
if _, err := os.Stat(configFile); err != nil {
if !os.IsNotExist(err) {
return nil, err
}
// It is okay to have no remote users configured.
log.Printf("no user config file found %q, remote reporting disabled",
configFile)
}
go ua.pollUsers()
return ua, nil
}
func (ua *userAuth) resetMissing(err error) error {
if os.IsNotExist(err) {
ua.Lock()
if ua.userPass != nil {
log.Printf("%q disappeared, remote reporting disabled",
ua.configFile)
}
ua.userPass = nil
ua.Unlock()
return nil
}
return err
}
func (ua *userAuth) loadUsers() error {
s, err := os.Stat(ua.configFile)
if err != nil {
return ua.resetMissing(err)
}
defer func() {
ua.lastModTime = s.ModTime()
}()
if ua.lastModTime.Before(s.ModTime()) {
r, err := os.Open(ua.configFile)
if err != nil {
return ua.resetMissing(err)
}
defer r.Close()
dec := json.NewDecoder(r)
// Use tmp map so failed parsing doesn't accidentally wipe out user
// list.
tmp := make(map[string]string)
err = dec.Decode(&tmp)
if err != nil {
return err
}
ua.Lock()
ua.userPass = tmp
ua.Unlock()
log.Println("Found", len(ua.userPass), "remote users in config",
ua.configFile)
}
return nil
}
func (ua *userAuth) pollUsers() {
for {
if err := ua.loadUsers(); err != nil {
log.Fatalf("Error loading user file %q: %v", ua.configFile, err)
}
time.Sleep(ua.pollInterval)
}
}
func hashPassword(pw string) string {
h := sha1.New()
fmt.Fprint(h, pw)
return hex.EncodeToString(h.Sum(nil))
}
func (ua *userAuth) auth(r *http.Request) bool {
user, pass, err := httputil.BasicAuth(r)
if user == "" || pass == "" || err != nil {
return false
}
ua.Lock()
defer ua.Unlock()
passHash, ok := ua.userPass[user]
if !ok {
return false
}
return passHash == hashPassword(pass)
}
var devcamBin = filepath.Join("bin", "devcam")
var (
hgCloneGoTipCmd = newTask("hg", "clone", "-u", "tip", "https://code.google.com/p/go")
@ -261,18 +380,41 @@ func main() {
if *help {
usage()
}
useTLS = *certFile != "" && *keyFile != ""
go handleSignals()
ua, err := newUserAuth(userAuthFile)
if err != nil {
log.Fatalf("Error creating user auth wrapper: %v", err)
}
authWrapper := func(f http.HandlerFunc) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
if !(httputil.IsLocalhost(r) || ua.auth(r)) {
w.Header().Set("WWW-Authenticate", `Basic realm="buildbot master"`)
http.Error(w, "Unauthorized access", http.StatusUnauthorized)
return
}
f(w, r)
}
}
http.HandleFunc(okPrefix, okHandler)
http.HandleFunc(failPrefix, failHandler)
http.HandleFunc(progressPrefix, progressHandler)
http.HandleFunc(stderrPrefix, logHandler)
http.HandleFunc("/", statusHandler)
http.HandleFunc(reportPrefix, reportHandler)
http.HandleFunc(reportPrefix, authWrapper(reportHandler))
go func() {
log.Printf("Now starting to listen on %v", *host)
if err := http.ListenAndServe(*host, nil); err != nil {
log.Fatalf("Could not start listening on %v: %v", *host, err)
if useTLS {
if err := http.ListenAndServeTLS(*host, *certFile, *keyFile, nil); err != nil {
log.Fatalf("Could not start listening (TLS) on %v: %v", *host, err)
}
} else {
if err := http.ListenAndServe(*host, nil); err != nil {
log.Fatalf("Could not start listening on %v: %v", *host, err)
}
}
}()
setup()
@ -537,8 +679,6 @@ func pollCamliChange() (string, error) {
const builderBotBin = "builderBot"
func buildBuilder() error {
// TODO(Bill, mpl): import common auth module for both the master and builder. Or the multi-files
// approach. Whatever's cleaner.
source := *builderSrc
if source == "" {
if *altCamliRevURL != "" {
@ -593,6 +733,9 @@ func startBuilder(goHash, camliHash string) (*exec.Cmd, error) {
ourHost = "localhost"
}
masterHosts := ourHost + ":" + ourPort
if useTLS {
masterHosts = "https://" + masterHosts
}
if *peers != "" {
masterHosts += "," + *peers
}
@ -694,36 +837,12 @@ func checkLastModified(w http.ResponseWriter, r *http.Request, modtime time.Time
return false
}
func isLocalhost(addr string) bool {
host, _, err := net.SplitHostPort(addr)
if err != nil {
// all of this should not happen since addr should be
// an http.Request.RemoteAddr but never knows...
addrErr, ok := err.(*net.AddrError)
if !ok {
log.Println(err)
return false
}
if addrErr.Err != "missing port in address" {
log.Println(err)
return false
}
host = addr
}
return host == "localhost" || host == "127.0.0.1" || host == "[::1]"
}
func reportHandler(w http.ResponseWriter, r *http.Request) {
if r.Method != "POST" {
log.Println("Invalid method for report handler")
http.Error(w, "Invalid method", http.StatusMethodNotAllowed)
return
}
if !isLocalhost(r.RemoteAddr) {
dbg.Printf("Refusing remote report from %v for now", r.RemoteAddr)
http.Error(w, "No remote bot", http.StatusUnauthorized)
return
}
body, err := ioutil.ReadAll(r.Body)
if err != nil {
log.Println("Invalid request for report handler")

View File

@ -18,17 +18,14 @@ limitations under the License.
package auth
import (
"encoding/base64"
"errors"
"fmt"
"net"
"net/http"
"os"
"regexp"
"runtime"
"strings"
"camlistore.org/pkg/netutil"
"camlistore.org/pkg/httputil"
)
// Operation represents a bitmask of operations. See the OpX constants.
@ -48,8 +45,6 @@ const (
OpAll = OpUpload | OpEnumerate | OpStat | OpRemove | OpGet | OpSign | OpDiscovery
)
var kBasicAuthPattern = regexp.MustCompile(`^Basic ([a-zA-Z0-9\+/=]+)`)
var (
mode AuthMode // the auth logic depending on the choosen auth mechanism
)
@ -165,29 +160,6 @@ func SetMode(m AuthMode) {
mode = m
}
func basicAuth(req *http.Request) (string, string, error) {
auth := req.Header.Get("Authorization")
if auth == "" {
return "", "", fmt.Errorf("Missing \"Authorization\" in header")
}
matches := kBasicAuthPattern.FindStringSubmatch(auth)
if len(matches) != 2 {
return "", "", fmt.Errorf("Bogus Authorization header")
}
encoded := matches[1]
enc := base64.StdEncoding
decBuf := make([]byte, enc.DecodedLen(len(encoded)))
n, err := enc.Decode(decBuf, []byte(encoded))
if err != nil {
return "", "", err
}
pieces := strings.SplitN(string(decBuf[0:n]), ":", 2)
if len(pieces) != 2 {
return "", "", fmt.Errorf("didn't get two pieces")
}
return pieces[0], pieces[1], nil
}
// UserPass is used when the auth string provided in the config
// is of the kind "userpass:username:pass"
// Possible options appended to the config string are
@ -202,7 +174,7 @@ type UserPass struct {
}
func (up *UserPass) AllowedAccess(req *http.Request) Operation {
user, pass, err := basicAuth(req)
user, pass, err := httputil.BasicAuth(req)
if err == nil {
if user == up.Username {
if pass == up.Password {
@ -214,7 +186,7 @@ func (up *UserPass) AllowedAccess(req *http.Request) Operation {
}
}
if up.OrLocalhost && localhostAuthorized(req) {
if up.OrLocalhost && httputil.IsLocalhost(req) {
return OpAll
}
@ -240,7 +212,7 @@ type Localhost struct {
}
func (Localhost) AllowedAccess(req *http.Request) (out Operation) {
if localhostAuthorized(req) {
if httputil.IsLocalhost(req) {
return OpAll
}
return 0
@ -255,7 +227,7 @@ type DevAuth struct {
}
func (da *DevAuth) AllowedAccess(req *http.Request) Operation {
_, pass, err := basicAuth(req)
_, pass, err := httputil.BasicAuth(req)
if err == nil {
if pass == da.Password {
return OpAll
@ -268,7 +240,7 @@ func (da *DevAuth) AllowedAccess(req *http.Request) Operation {
// See if the local TCP port is owned by the same non-root user as this
// server. This check performed last as it may require reading from the
// kernel or exec'ing a program.
if localhostAuthorized(req) {
if httputil.IsLocalhost(req) {
return OpAll
}
@ -279,45 +251,12 @@ func (da *DevAuth) AddAuthHeader(req *http.Request) {
req.SetBasicAuth("", da.Password)
}
func localhostAuthorized(req *http.Request) bool {
uid := os.Getuid()
from, err := netutil.HostPortToIP(req.RemoteAddr, nil)
if err != nil {
return false
}
to, err := netutil.HostPortToIP(req.Host, from)
if err != nil {
return false
}
// If our OS doesn't support uid.
// TODO(bradfitz): netutil on OS X uses "lsof" to figure out
// ownership of tcp connections, but when fuse is mounted and a
// request is outstanding (for instance, a fuse request that's
// making a request to camlistored and landing in this code
// path), lsof then blocks forever waiting on a lock held by the
// VFS, leading to a deadlock. Instead, on darwin, just trust
// any localhost connection here, which is kinda lame, but
// whatever. Macs aren't very multi-user anyway.
if uid == -1 || runtime.GOOS == "darwin" {
return from.IP.IsLoopback() && to.IP.IsLoopback()
}
if uid > 0 {
owner, err := netutil.AddrPairUserid(from, to)
if err == nil && owner == uid {
return true
}
}
return false
}
func isLocalhost(addrPort net.IP) bool {
return addrPort.IsLoopback()
}
func IsLocalhost(req *http.Request) bool {
return localhostAuthorized(req)
return httputil.IsLocalhost(req)
}
// TODO(mpl): if/when we ever need it:

View File

@ -18,10 +18,6 @@ package auth
import (
"fmt"
"io/ioutil"
"net"
"net/http"
"net/http/httptest"
"reflect"
"testing"
)
@ -55,105 +51,3 @@ func TestFromConfig(t *testing.T) {
}
}
}
func testServer(t *testing.T, l net.Listener) *httptest.Server {
ts := &httptest.Server{
Listener: l,
Config: &http.Server{
Handler: http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
if localhostAuthorized(r) {
fmt.Fprintf(rw, "authorized")
return
}
fmt.Fprintf(rw, "unauthorized")
}),
},
}
ts.Start()
return ts
}
func TestLocalhostAuthIPv6(t *testing.T) {
l, err := net.Listen("tcp", "[::1]:0")
if err != nil {
t.Skip("skipping IPv6 test; can't listen on [::1]:0")
}
_, port, err := net.SplitHostPort(l.Addr().String())
if err != nil {
t.Fatal(err)
}
// See if IPv6 works on this machine first. It seems the above
// Listen can pass on Linux but fail here in the dial.
c, err := net.Dial("tcp6", l.Addr().String())
if err != nil {
t.Skipf("skipping IPv6 test; dial back to %s failed with %v", l.Addr(), err)
}
c.Close()
ts := testServer(t, l)
defer ts.Close()
// Use an explicit transport to force IPv6 (http.Get resolves localhost in IPv4 otherwise)
trans := &http.Transport{
Dial: func(network, addr string) (net.Conn, error) {
c, err := net.Dial("tcp6", addr)
return c, err
},
}
testLoginRequest(t, &http.Client{Transport: trans}, "http://[::1]:"+port)
// See if we can get an IPv6 from resolving localhost
localips, err := net.LookupIP("localhost")
if err != nil {
t.Skipf("skipping IPv6 test; resolving localhost failed with %v", err)
}
if hasIPv6(localips) {
testLoginRequest(t, &http.Client{Transport: trans}, "http://localhost:"+port)
} else {
t.Logf("incomplete IPv6 test; resolving localhost didn't return any IPv6 addresses")
}
}
func hasIPv6(ips []net.IP) bool {
for _, ip := range ips {
if ip.To4() == nil {
return true
}
}
return false
}
func TestLocalhostAuthIPv4(t *testing.T) {
l, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Skip("skipping IPv4 test; can't listen on 127.0.0.1:0")
}
_, port, err := net.SplitHostPort(l.Addr().String())
if err != nil {
t.Fatal(err)
}
ts := testServer(t, l)
defer ts.Close()
testLoginRequest(t, &http.Client{}, "http://127.0.0.1:"+port)
testLoginRequest(t, &http.Client{}, "http://localhost:"+port)
}
func testLoginRequest(t *testing.T, client *http.Client, URL string) {
res, err := client.Get(URL)
if err != nil {
t.Fatal(err)
}
body, err := ioutil.ReadAll(res.Body)
if err != nil {
t.Fatal(err)
}
const exp = "authorized"
if string(body) != exp {
t.Errorf("got %q (instead of %v)", string(body), exp)
}
}

94
pkg/httputil/auth.go Normal file
View File

@ -0,0 +1,94 @@
/*
Copyright 2013 The Camlistore Authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package httputil
import (
"encoding/base64"
"fmt"
"net/http"
"os"
"regexp"
"runtime"
"strings"
"camlistore.org/pkg/netutil"
)
var kBasicAuthPattern = regexp.MustCompile(`^Basic ([a-zA-Z0-9\+/=]+)`)
// IsLocalhost reports whether the requesting connection is from this machine
// and has the same owner as this process.
func IsLocalhost(req *http.Request) bool {
uid := os.Getuid()
from, err := netutil.HostPortToIP(req.RemoteAddr, nil)
if err != nil {
return false
}
to, err := netutil.HostPortToIP(req.Host, from)
if err != nil {
return false
}
// If our OS doesn't support uid.
// TODO(bradfitz): netutil on OS X uses "lsof" to figure out
// ownership of tcp connections, but when fuse is mounted and a
// request is outstanding (for instance, a fuse request that's
// making a request to camlistored and landing in this code
// path), lsof then blocks forever waiting on a lock held by the
// VFS, leading to a deadlock. Instead, on darwin, just trust
// any localhost connection here, which is kinda lame, but
// whatever. Macs aren't very multi-user anyway.
if uid == -1 || runtime.GOOS == "darwin" {
return from.IP.IsLoopback() && to.IP.IsLoopback()
}
if uid > 0 {
owner, err := netutil.AddrPairUserid(from, to)
if err == nil && owner == uid {
return true
}
}
return false
}
// BasicAuth parses the Authorization header on req
// If absent or invalid, an error is returned.
func BasicAuth(req *http.Request) (username, password string, err error) {
auth := req.Header.Get("Authorization")
if auth == "" {
err = fmt.Errorf("Missing \"Authorization\" in header")
return
}
matches := kBasicAuthPattern.FindStringSubmatch(auth)
if len(matches) != 2 {
err = fmt.Errorf("Bogus Authorization header")
return
}
encoded := matches[1]
enc := base64.StdEncoding
decBuf := make([]byte, enc.DecodedLen(len(encoded)))
n, err := enc.Decode(decBuf, []byte(encoded))
if err != nil {
return
}
pieces := strings.SplitN(string(decBuf[0:n]), ":", 2)
if len(pieces) != 2 {
err = fmt.Errorf("didn't get two pieces")
return
}
return pieces[0], pieces[1], nil
}

182
pkg/httputil/auth_test.go Normal file
View File

@ -0,0 +1,182 @@
/*
Copyright 2013 The Camlistore Authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package httputil
import (
"fmt"
"io/ioutil"
"net"
"net/http"
"net/http/httptest"
"testing"
)
func testServer(t *testing.T, l net.Listener) *httptest.Server {
ts := &httptest.Server{
Listener: l,
Config: &http.Server{
Handler: http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
if IsLocalhost(r) {
fmt.Fprintf(rw, "authorized")
return
}
fmt.Fprintf(rw, "unauthorized")
}),
},
}
ts.Start()
return ts
}
func TestLocalhostAuthIPv6(t *testing.T) {
l, err := net.Listen("tcp", "[::1]:0")
if err != nil {
t.Skip("skipping IPv6 test; can't listen on [::1]:0")
}
_, port, err := net.SplitHostPort(l.Addr().String())
if err != nil {
t.Fatal(err)
}
// See if IPv6 works on this machine first. It seems the above
// Listen can pass on Linux but fail here in the dial.
c, err := net.Dial("tcp6", l.Addr().String())
if err != nil {
t.Skipf("skipping IPv6 test; dial back to %s failed with %v", l.Addr(), err)
}
c.Close()
ts := testServer(t, l)
defer ts.Close()
// Use an explicit transport to force IPv6 (http.Get resolves localhost in IPv4 otherwise)
trans := &http.Transport{
Dial: func(network, addr string) (net.Conn, error) {
c, err := net.Dial("tcp6", addr)
return c, err
},
}
testLoginRequest(t, &http.Client{Transport: trans}, "http://[::1]:"+port)
// See if we can get an IPv6 from resolving localhost
localips, err := net.LookupIP("localhost")
if err != nil {
t.Skipf("skipping IPv6 test; resolving localhost failed with %v", err)
}
if hasIPv6(localips) {
testLoginRequest(t, &http.Client{Transport: trans}, "http://localhost:"+port)
} else {
t.Logf("incomplete IPv6 test; resolving localhost didn't return any IPv6 addresses")
}
}
func hasIPv6(ips []net.IP) bool {
for _, ip := range ips {
if ip.To4() == nil {
return true
}
}
return false
}
func TestLocalhostAuthIPv4(t *testing.T) {
l, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Skip("skipping IPv4 test; can't listen on 127.0.0.1:0")
}
_, port, err := net.SplitHostPort(l.Addr().String())
if err != nil {
t.Fatal(err)
}
ts := testServer(t, l)
defer ts.Close()
testLoginRequest(t, &http.Client{}, "http://127.0.0.1:"+port)
testLoginRequest(t, &http.Client{}, "http://localhost:"+port)
}
func testLoginRequest(t *testing.T, client *http.Client, URL string) {
res, err := client.Get(URL)
if err != nil {
t.Fatal(err)
}
body, err := ioutil.ReadAll(res.Body)
if err != nil {
t.Fatal(err)
}
const exp = "authorized"
if string(body) != exp {
t.Errorf("got %q (instead of %v)", string(body), exp)
}
}
func TestBasicAuth(t *testing.T) {
for _, d := range []struct {
header string
u, pw string
valid bool
}{
// Empty is invalid.
{},
{
// Missing password.
header: "Basic QWxhZGRpbg==",
},
{
// Malformed base64 encoding.
header: "Basic foo",
},
{
// Malformed header, no 'Basic ' prefix.
header: "QWxhZGRpbg==",
},
{
header: "Basic QWxhZGRpbjpvcGVuIHNlc2FtZQ==",
u: "Aladdin",
pw: "open sesame",
valid: true,
},
} {
req, err := http.NewRequest("GET", "/", nil)
if err != nil {
t.Fatal(err)
}
if d.header != "" {
req.Header.Set("Authorization", d.header)
}
u, pw, err := BasicAuth(req)
t.Log(d.header, err)
if d.valid && err != nil {
t.Error("Want success parse of auth header, got", err)
}
if !d.valid && err == nil {
t.Error("Want error parsing", d.header)
}
if d.u != u {
t.Errorf("Want user %q, got %q", d.u, u)
}
if d.pw != pw {
t.Errorf("Want password %q, got %q", d.pw, pw)
}
}
}

View File

@ -29,7 +29,6 @@ import (
"strconv"
"strings"
"camlistore.org/pkg/auth"
"camlistore.org/pkg/blob"
)
@ -62,7 +61,7 @@ func RequestEntityTooLargeError(conn http.ResponseWriter) {
func ServeError(conn http.ResponseWriter, req *http.Request, err error) {
conn.WriteHeader(http.StatusInternalServerError)
if auth.IsLocalhost(req) {
if IsLocalhost(req) {
fmt.Fprintf(conn, "Server error: %s\n", err)
return
}