Cleanup; make blobserver's GET path use blobref.Fether interface

Also expands the Fetcher interface slightly, and cleans up
some other code in the process.

server/go/blobref/blobref.go

@@ -25,8 +25,14 @@ type BlobRef interface {
 	fmt.Stringer
 }
 
+type ReadSeekCloser interface {
+	io.Reader
+	io.Seeker
+	io.Closer
+}
+
 type Fetcher interface {
-	Fetch(BlobRef) (io.ReadCloser, os.Error)
+	Fetch(BlobRef) (file ReadSeekCloser, size int64, err os.Error)
 }
 
 type blobRef struct {

server/go/blobserver/Makefile

@@ -3,7 +3,7 @@ include $(GOROOT)/src/Make.inc
 TARG=camlistored
 GOFILES=\
 	camlistored.go\
-	blobref.go\
+	localdisk.go\
 	enumerate.go\
 	get.go\
 	preupload.go\

server/go/blobserver/camlistored.go

@@ -8,6 +8,7 @@ import (
 	"camli/auth"
 	"camli/httputil"
 	"camli/webserver"
+	"camli/blobref"
 	"flag"
 	"fmt"
 	"http"
@@ -17,6 +18,8 @@ import (
 var flagStorageRoot *string = flag.String("root", "/tmp/camliroot", "Root directory to store files")
 var stealthMode *bool = flag.Bool("stealth", true, "Run in stealth mode.")
 
+var blobFetcher blobref.Fetcher
+
 func handleCamli(conn http.ResponseWriter, req *http.Request) {
 	handler := func (conn http.ResponseWriter, req *http.Request) {
 		httputil.BadRequestError(conn,
@@ -29,7 +32,7 @@ func handleCamli(conn http.ResponseWriter, req *http.Request) {
 		case "/camli/enumerate-blobs":
 			handler = auth.RequireAuth(handleEnumerateBlobs)
 		default:
-			handler = auth.RequireAuth(handleGet)
+			handler = auth.RequireAuth(createGetHandler(blobFetcher))
 		}
 	case "POST":
 		switch req.URL.Path {
@@ -78,6 +81,8 @@ func main() {
 		}
 	}
 
+	blobFetcher = newDiskStorage(*flagStorageRoot)
+
 	ws := webserver.New()
 	ws.HandleFunc("/", handleRoot)
 	ws.HandleFunc("/camli/", handleCamli)

server/go/blobserver/get.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"camli/blobref"
 	"camli/httputil"
 	"fmt"
 	"http"
@@ -8,29 +9,33 @@ import (
 	"io"
 )
 
-func handleGet(conn http.ResponseWriter, req *http.Request) {
+func createGetHandler(fetcher blobref.Fetcher) func (http.ResponseWriter, *http.Request) {
+	return func (conn http.ResponseWriter, req *http.Request) {
+		handleGet(conn, req, fetcher);
+	}
+}
+
+func handleGet(conn http.ResponseWriter, req *http.Request, fetcher blobref.Fetcher) {
 	blobRef := BlobFromUrlPath(req.URL.Path)
 	if blobRef == nil {
 		httputil.BadRequestError(conn, "Malformed GET URL.")
 		return
 	}
-	fileName := BlobFileName(blobRef)
-	stat, err := os.Stat(fileName)
-	if err == os.ENOENT {
+	file, size, err := fetcher.Fetch(blobRef)
+	switch err {
+	case nil:
+		break
+	case os.ENOENT:
 		conn.WriteHeader(http.StatusNotFound)
 		fmt.Fprintf(conn, "Object not found.")
 		return
-	}
-	if err != nil {
-		httputil.ServerError(conn, err)
-		return
-	}
-	file, err := os.Open(fileName, os.O_RDONLY, 0)
-	if err != nil {
+	default:
 		httputil.ServerError(conn, err)
 		return
 	}
 
+	defer file.Close()
+
 	reqRange := getRequestedRange(req)
 	if reqRange.SkipBytes != 0 {
 		_, err = file.Seek(reqRange.SkipBytes, 0)
@@ -45,7 +50,7 @@ func handleGet(conn http.ResponseWriter, req *http.Request) {
 		input = io.LimitReader(file, reqRange.LimitBytes)
 	}
 
-	remainBytes := stat.Size - reqRange.SkipBytes
+	remainBytes := size - reqRange.SkipBytes
 	if reqRange.LimitBytes != -1 &&
 		reqRange.LimitBytes < remainBytes {
 		remainBytes = reqRange.LimitBytes
@@ -56,7 +61,7 @@ func handleGet(conn http.ResponseWriter, req *http.Request) {
 		conn.SetHeader("Content-Range",
 			fmt.Sprintf("bytes %d-%d/%d", reqRange.SkipBytes,
 			reqRange.SkipBytes + remainBytes,
-			stat.Size))
+			size))
 		conn.WriteHeader(http.StatusPartialContent)
 	}
 	bytesCopied, err := io.Copy(conn, input)
@@ -65,21 +70,22 @@ func handleGet(conn http.ResponseWriter, req *http.Request) {
 	// as they've already been receiving bytes.  But they should be smart enough
 	// to verify the digest doesn't match.  But we close the (chunked) response anyway,
 	// to further signal errors.
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "Error sending file: %v, err=%v\n", blobRef, err)
+	killConnection := func() {
 		closer, _, err := conn.Hijack()
 		if err != nil {
 			closer.Close()
 		}
+	}
+
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error sending file: %v, err=%v\n", blobRef, err)
+		killConnection()
 		return
 	}
 	if bytesCopied != remainBytes {
 		fmt.Fprintf(os.Stderr, "Error sending file: %v, copied=%d, not %d\n", blobRef,
 			bytesCopied, remainBytes)
-		closer, _, err := conn.Hijack()
-		if err != nil {
-			closer.Close()
-		}
+		killConnection()
 		return
 	}
 }

server/go/blobserver/localdisk.go

@@ -4,8 +4,30 @@ import (
 	"camli/blobref"
 	"fmt"
 	"regexp"
+	"os"
 )
 
+type diskStorage struct {
+	Root string
+}
+
+func (ds *diskStorage) Fetch(blob blobref.BlobRef) (blobref.ReadSeekCloser, int64, os.Error) {
+	fileName := BlobFileName(blob)
+	stat, err := os.Stat(fileName)
+	if err == os.ENOENT {
+		return nil, 0, err
+	}
+	file, err := os.Open(fileName, os.O_RDONLY, 0)
+	if err != nil {
+		return nil, 0, err
+	}
+	return file, stat.Size, nil
+}
+
+func newDiskStorage(root string) *diskStorage {
+	return &diskStorage{Root: root}
+}
+
 var kGetPutPattern *regexp.Regexp = regexp.MustCompile(`^/camli/([a-z0-9]+)-([a-f0-9]+)$`)
 
 func BlobFileBaseName(b blobref.BlobRef) string {
@@ -25,4 +47,3 @@ func BlobFromUrlPath(path string) blobref.BlobRef {
 	return blobref.FromPattern(kGetPutPattern, path)
 }
 
-

server/go/blobserver/run.sh

@@ -2,4 +2,4 @@
 
 mkdir /tmp/camliroot
 export CAMLI_PASSWORD=foo
-make && ./camlistored "$@"
+make && ./camlistored -listen=:3179 "$@"

server/go/jsonsign/sign.go

@@ -53,7 +53,7 @@ func (sr *SignRequest) Sign() (signedJson string, err os.Error) {
 		return inputfail("json \"camliSigner\" key is malformed or unsupported")
 	}
 
-	pubkeyReader, err := sr.Fetcher.Fetch(signerBlob)
+	pubkeyReader, _, err := sr.Fetcher.Fetch(signerBlob)
 	if err != nil {
 		// TODO: not really either an inputfail or an execfail.. but going
 		// with exec for now.

server/go/jsonsign/verify.go

@@ -111,7 +111,7 @@ func (vr *VerifyRequest) ParsePayloadMap() bool {
 }
 
 func (vr *VerifyRequest) FindAndParsePublicKeyBlob() bool {
-	reader, err := vr.fetcher.Fetch(vr.CamliSigner)
+	reader, _, err := vr.fetcher.Fetch(vr.CamliSigner)
 	if err != nil {
 		return vr.fail(fmt.Sprintf("Error fetching public key blob: %v", err))
 	}

server/go/sigserver/Makefile

@@ -3,7 +3,6 @@ include $(GOROOT)/src/Make.inc
 TARG=camsigd
 GOFILES=\
 	camsigd.go\
-	keys.go\
 	sign.go\
 	verify.go\
 

server/go/sigserver/camsigd.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"camli/auth"
+	"camli/blobref"
 	"camli/httputil"
 	"camli/webserver"
 	"flag"
@@ -12,6 +13,31 @@ import (
 
 var accessPassword string
 
+var flagPubKeyDir *string = flag.String("pubkey-dir", "test/pubkey-blobs",
+	"Temporary development hack; directory to dig-xxxx.camli public keys.")
+
+type pubkeyDirFetcher struct{}
+func (_ *pubkeyDirFetcher) Fetch(b blobref.BlobRef) (file blobref.ReadSeekCloser, size int64, err os.Error) {
+	fileName := fmt.Sprintf("%s/%s.camli", *flagPubKeyDir, b.String())
+	var stat *os.FileInfo
+	stat, err = os.Stat(fileName)
+	if err != nil {
+		return
+	}
+	file, err = os.Open(fileName, os.O_RDONLY, 0)
+	if err != nil {
+		return
+	}
+	size = stat.Size
+	return
+}
+
+// TODO: for now, the only implementation of the blobref.Fetcher
+// interface for fetching public keys is the "local, from disk"
+// implementation used for testing.  In reality we'd want to be able
+// to fetch these from blobservers.
+var pubKeyFetcher = &pubkeyDirFetcher{}
+
 func handleRoot(conn http.ResponseWriter, req *http.Request) {
 	fmt.Fprintf(conn, "camsigd")
 }

server/go/sigserver/keys.go

@@ -1,23 +0,0 @@
-package main
-
-import (
-	"camli/blobref"
-	"flag"
-	"fmt"
-	"io"
-	"os"
-)
-
-var flagPubKeyDir *string = flag.String("pubkey-dir", "test/pubkey-blobs",
-	"Temporary development hack; directory to dig-xxxx.camli public keys.")
-
-type fromLocalDiskBlobFetcher struct{}
-var blobFetcher = &fromLocalDiskBlobFetcher{}
-func (_ *fromLocalDiskBlobFetcher) Fetch(b blobref.BlobRef) (io.ReadCloser, os.Error) {
-	publicKeyFile := fmt.Sprintf("%s/%s.camli", *flagPubKeyDir, b.String())
-	f, err := os.Open(publicKeyFile, os.O_RDONLY, 0)
-	if err != nil {
-		return nil, err
-	}
-	return f, nil
-}

server/go/sigserver/sign.go

@@ -27,7 +27,7 @@ func handleSign(conn http.ResponseWriter, req *http.Request) {
 		return
 	}
 
-	sreq := &jsonsign.SignRequest{UnsignedJson: jsonStr, Fetcher: blobFetcher}
+	sreq := &jsonsign.SignRequest{UnsignedJson: jsonStr, Fetcher: pubKeyFetcher}
 	signedJson, err := sreq.Sign()
 	if err != nil {
 		// TODO: some aren't really a "bad request"

server/go/sigserver/verify.go

@@ -34,7 +34,7 @@ func handleVerify(conn http.ResponseWriter, req *http.Request) {
 
 	m := make(map[string]interface{})
 
-	vreq := jsonsign.NewVerificationRequest(sjson, blobFetcher)
+	vreq := jsonsign.NewVerificationRequest(sjson, pubKeyFetcher)
 	if vreq.Verify() {
 		m["signatureValid"] = 1
 		m["verifiedData"] = vreq.PayloadMap