get handler cleanup, break apart into smaller bits

lib/go/blobref/fetcher.go

@@ -22,6 +22,9 @@ import (
 )
 
 type Fetcher interface {
+	// Fetch returns a blob.  If the blob is not found then
+	// os.ENOENT should be returned for the error (not a wrapped
+	// error with a ENOENT inside)
 	Fetch(*BlobRef) (file ReadSeekCloser, size int64, err os.Error)
 }
 

lib/go/blobserver/handlers/Makefile

@@ -4,6 +4,7 @@ PREREQ=$(QUOTED_GOROOT)/pkg/$(GOOS)_$(GOARCH)/camli/blobserver.a\
 	$(QUOTED_GOROOT)/pkg/$(GOOS)_$(GOARCH)/camli/blobref.a\
 	$(QUOTED_GOROOT)/pkg/$(GOOS)_$(GOARCH)/camli/httprange.a\
 	$(QUOTED_GOROOT)/pkg/$(GOOS)_$(GOARCH)/camli/httputil.a\
+	$(QUOTED_GOROOT)/pkg/$(GOOS)_$(GOARCH)/camli/auth.a\
 
 TARG=camli/blobserver/handlers
 GOFILES=\

lib/go/blobserver/handlers/get.go

@@ -36,6 +36,7 @@ import (
 	"time"
 )
 
+// TODO: support gets on partitions?  Be less rigid here.
 var kGetPattern *regexp.Regexp = regexp.MustCompile(`^/camli/([a-z0-9]+)-([a-f0-9]+)$`)
 
 func CreateGetHandler(fetcher blobref.Fetcher) func(http.ResponseWriter, *http.Request) {
@@ -53,106 +54,26 @@ func sendUnauthorized(conn http.ResponseWriter) {
 }
 
 func handleGet(conn http.ResponseWriter, req *http.Request, fetcher blobref.Fetcher) {
-	isOwner := auth.IsAuthorized(req)
-
 	blobRef := blobFromUrlPath(req.URL.Path)
 	if blobRef == nil {
 		httputil.BadRequestError(conn, "Malformed GET URL.")
 		return
 	}
 
-	var viaBlobs []*blobref.BlobRef
+	switch {
-	if !isOwner {
+	case auth.IsAuthorized(req):
-		viaPathOkay := false
+		serveBlobRef(conn, req, blobRef, fetcher)
-		startTime := time.Nanoseconds()
+	case auth.TriedAuthorization(req):
-		defer func() {
+		log.Printf("Attempted authorization failed on %s", req.URL)
-			if !viaPathOkay {
+		sendUnauthorized(conn)
-				// Insert a delay, to hide timing attacks probing
+	default:
-				// for the existence of blobs.
+		handleGetViaSharing(conn, req, blobRef, fetcher)
-				sleep := fetchFailureDelayNs - (time.Nanoseconds() - startTime)
-				if sleep > 0 {
-					time.Sleep(sleep)
-				}
-			}
-		}()
-		viaBlobs = make([]*blobref.BlobRef, 0)
-		if via := req.FormValue("via"); via != "" {
-			for _, vs := range strings.Split(via, ",", -1) {
-				if br := blobref.Parse(vs); br == nil {
-					httputil.BadRequestError(conn, "Malformed blobref in via param")
-					return
-				} else {
-					viaBlobs = append(viaBlobs, br)
-				}
-			}
-		}
-
-		fetchChain := make([]*blobref.BlobRef, 0)
-		fetchChain = append(fetchChain, viaBlobs...)
-		fetchChain = append(fetchChain, blobRef)
-		for i, br := range fetchChain {
-			switch i {
-			case 0:
-				file, size, err := fetcher.Fetch(br)
-				if err != nil {
-					log.Printf("Fetch chain 0 of %s failed: %v", br.String(), err)
-					sendUnauthorized(conn)
-					return
-				}
-				defer file.Close()
-				if size > maxJsonSize {
-					log.Printf("Fetch chain 0 of %s too large", br.String())
-					sendUnauthorized(conn)
-					return
-				}
-				jd := json.NewDecoder(file)
-				m := make(map[string]interface{})
-				if err := jd.Decode(&m); err != nil {
-					log.Printf("Fetch chain 0 of %s wasn't JSON: %v", br.String(), err)
-					sendUnauthorized(conn)
-					return
-				}
-				if m["camliType"].(string) != "share" {
-					log.Printf("Fetch chain 0 of %s wasn't a share", br.String())
-					sendUnauthorized(conn)
-					return
-				}
-				if len(fetchChain) > 1 && fetchChain[1].String() != m["target"].(string) {
-					log.Printf("Fetch chain 0->1 (%s -> %q) unauthorized, expected hop to %q",
-						br.String(), fetchChain[1].String(), m["target"])
-					sendUnauthorized(conn)
-					return
-				}
-			case len(fetchChain) - 1:
-				// Last one is fine (as long as its path up to here has been proven, and it's
-				// not the first thing in the chain)
-				continue
-			default:
-				file, _, err := fetcher.Fetch(br)
-				if err != nil {
-					log.Printf("Fetch chain %d of %s failed: %v", i, br.String(), err)
-					sendUnauthorized(conn)
-					return
-				}
-				defer file.Close()
-				lr := io.LimitReader(file, maxJsonSize)
-				slurpBytes, err := ioutil.ReadAll(lr)
-				if err != nil {
-					log.Printf("Fetch chain %d of %s failed in slurp: %v", i, br.String(), err)
-					sendUnauthorized(conn)
-					return
-				}
-				saught := fetchChain[i+1].String()
-				if bytes.IndexAny(slurpBytes, saught) == -1 {
-					log.Printf("Fetch chain %d of %s failed; no reference to %s",
-						i, br.String(), saught)
-					sendUnauthorized(conn)
-					return
-				}
-			}
-		}
-		viaPathOkay = true
 	}
+}
+
+// serveBlobRef sends 'blobref' to 'conn' as directed by the Range header in 'req'
+func serveBlobRef(conn http.ResponseWriter, req *http.Request,
+	blobRef *blobref.BlobRef, fetcher blobref.Fetcher) {
 
 	file, size, err := fetcher.Fetch(blobRef)
 	switch err {
@@ -239,6 +160,7 @@ func handleGet(conn http.ResponseWriter, req *http.Request, fetcher blobref.Fetc
 		killConnection()
 		return
 	}
+
 	if bytesCopied != remainBytes {
 		fmt.Fprintf(os.Stderr, "Error sending file: %v, copied=%d, not %d\n", blobRef,
 			bytesCopied, remainBytes)
@@ -247,6 +169,105 @@ func handleGet(conn http.ResponseWriter, req *http.Request, fetcher blobref.Fetc
 	}
 }
 
+// Unauthenticated user.  Be paranoid.
+func handleGetViaSharing(conn http.ResponseWriter, req *http.Request,
+	blobRef *blobref.BlobRef, fetcher blobref.Fetcher) {
+
+	viaPathOkay := false
+	startTime := time.Nanoseconds()
+	defer func() {
+		if !viaPathOkay {
+			// Insert a delay, to hide timing attacks probing
+			// for the existence of blobs.
+			sleep := fetchFailureDelayNs - (time.Nanoseconds() - startTime)
+			if sleep > 0 {
+				time.Sleep(sleep)
+			}
+		}
+	}()
+	viaBlobs := make([]*blobref.BlobRef, 0)
+	if via := req.FormValue("via"); via != "" {
+		for _, vs := range strings.Split(via, ",", -1) {
+			if br := blobref.Parse(vs); br == nil {
+				httputil.BadRequestError(conn, "Malformed blobref in via param")
+				return
+			} else {
+				viaBlobs = append(viaBlobs, br)
+			}
+		}
+	}
+
+	fetchChain := make([]*blobref.BlobRef, 0)
+	fetchChain = append(fetchChain, viaBlobs...)
+	fetchChain = append(fetchChain, blobRef)
+	for i, br := range fetchChain {
+		switch i {
+		case 0:
+			file, size, err := fetcher.Fetch(br)
+			if err != nil {
+				log.Printf("Fetch chain 0 of %s failed: %v", br.String(), err)
+				sendUnauthorized(conn)
+				return
+			}
+			defer file.Close()
+			if size > maxJsonSize {
+				log.Printf("Fetch chain 0 of %s too large", br.String())
+				sendUnauthorized(conn)
+				return
+			}
+			jd := json.NewDecoder(file)
+			m := make(map[string]interface{})
+			if err := jd.Decode(&m); err != nil {
+				log.Printf("Fetch chain 0 of %s wasn't JSON: %v", br.String(), err)
+				sendUnauthorized(conn)
+				return
+			}
+			if m["camliType"].(string) != "share" {
+				log.Printf("Fetch chain 0 of %s wasn't a share", br.String())
+				sendUnauthorized(conn)
+				return
+			}
+			if len(fetchChain) > 1 && fetchChain[1].String() != m["target"].(string) {
+				log.Printf("Fetch chain 0->1 (%s -> %q) unauthorized, expected hop to %q",
+					br.String(), fetchChain[1].String(), m["target"])
+				sendUnauthorized(conn)
+				return
+			}
+		case len(fetchChain) - 1:
+			// Last one is fine (as long as its path up to here has been proven, and it's
+			// not the first thing in the chain)
+			continue
+		default:
+			file, _, err := fetcher.Fetch(br)
+			if err != nil {
+				log.Printf("Fetch chain %d of %s failed: %v", i, br.String(), err)
+				sendUnauthorized(conn)
+				return
+			}
+			defer file.Close()
+			lr := io.LimitReader(file, maxJsonSize)
+			slurpBytes, err := ioutil.ReadAll(lr)
+			if err != nil {
+				log.Printf("Fetch chain %d of %s failed in slurp: %v", i, br.String(), err)
+				sendUnauthorized(conn)
+				return
+			}
+			saught := fetchChain[i+1].String()
+			if bytes.IndexAny(slurpBytes, saught) == -1 {
+				log.Printf("Fetch chain %d of %s failed; no reference to %s",
+					i, br.String(), saught)
+				sendUnauthorized(conn)
+				return
+			}
+		}
+	}
+
+	viaPathOkay = true
+
+	serveBlobRef(conn, req, blobRef, fetcher)
+
+}
+
 // TODO: copied this from lib/go/schema, but this might not be ideal.
 // unify and speed up?
 func isValidUtf8(s string) bool {
@@ -256,7 +277,6 @@ func isValidUtf8(s string) bool {
 		}
 	}
 	return true
-
 }
 
 func blobFromUrlPath(path string) *blobref.BlobRef {

lib/go/blobserver/interface.go

@@ -31,6 +31,8 @@ func (p Partition) IsDefault() bool {
 }
 
 type BlobReceiver interface {
+	// ReceiveBlob accepts a newly uploaded blob and writes it to
+	// disk.
 	ReceiveBlob(blob *blobref.BlobRef, source io.Reader, mirrorPartions []Partition) (*blobref.SizedBlobRef, os.Error)
 }
 

lib/go/blobserver/localdisk/localdisk.go

@@ -69,6 +69,9 @@ func (ds *diskStorage) Fetch(blob *blobref.BlobRef) (blobref.ReadSeekCloser, int
 	}
 	file, err := os.Open(fileName, os.O_RDONLY, 0)
 	if err != nil {
+		if errorIsNoEnt(err) {
+			err = os.ENOENT
+		}
 		return nil, 0, err
 	}
 	return file, stat.Size, nil

server/go/auth/auth.go

@@ -28,6 +28,12 @@ var kBasicAuthPattern *regexp.Regexp = regexp.MustCompile(`^Basic ([a-zA-Z0-9\+/
 
 var AccessPassword string
 
+func TriedAuthorization(req *http.Request) bool {
+	// Currently a simple test just using HTTP basic auth
+	// (presumably over https); may expand.
+	return req.Header.Get("Authorization") != ""
+}
+
 func IsAuthorized(req *http.Request) bool {
 	auth := req.Header.Get("Authorization")
 	if auth == "" {