diff --git a/server/go/blobref/blobref.go b/server/go/blobref/blobref.go index 9d9380cae..39306072b 100644 --- a/server/go/blobref/blobref.go +++ b/server/go/blobref/blobref.go @@ -25,8 +25,14 @@ type BlobRef interface { fmt.Stringer } +type ReadSeekCloser interface { + io.Reader + io.Seeker + io.Closer +} + type Fetcher interface { - Fetch(BlobRef) (io.ReadCloser, os.Error) + Fetch(BlobRef) (file ReadSeekCloser, size int64, err os.Error) } type blobRef struct { diff --git a/server/go/blobserver/Makefile b/server/go/blobserver/Makefile index 9a1c26ee1..045ba69ee 100644 --- a/server/go/blobserver/Makefile +++ b/server/go/blobserver/Makefile @@ -3,7 +3,7 @@ include $(GOROOT)/src/Make.inc TARG=camlistored GOFILES=\ camlistored.go\ - blobref.go\ + localdisk.go\ enumerate.go\ get.go\ preupload.go\ diff --git a/server/go/blobserver/camlistored.go b/server/go/blobserver/camlistored.go index 9d220df3a..bf1841e0a 100644 --- a/server/go/blobserver/camlistored.go +++ b/server/go/blobserver/camlistored.go @@ -8,6 +8,7 @@ import ( "camli/auth" "camli/httputil" "camli/webserver" + "camli/blobref" "flag" "fmt" "http" @@ -17,6 +18,8 @@ import ( var flagStorageRoot *string = flag.String("root", "/tmp/camliroot", "Root directory to store files") var stealthMode *bool = flag.Bool("stealth", true, "Run in stealth mode.") +var blobFetcher blobref.Fetcher + func handleCamli(conn http.ResponseWriter, req *http.Request) { handler := func (conn http.ResponseWriter, req *http.Request) { httputil.BadRequestError(conn, @@ -29,7 +32,7 @@ func handleCamli(conn http.ResponseWriter, req *http.Request) { case "/camli/enumerate-blobs": handler = auth.RequireAuth(handleEnumerateBlobs) default: - handler = auth.RequireAuth(handleGet) + handler = auth.RequireAuth(createGetHandler(blobFetcher)) } case "POST": switch req.URL.Path { @@ -78,6 +81,8 @@ func main() { } } + blobFetcher = newDiskStorage(*flagStorageRoot) + ws := webserver.New() ws.HandleFunc("/", handleRoot) ws.HandleFunc("/camli/", handleCamli) diff --git a/server/go/blobserver/get.go b/server/go/blobserver/get.go index df850fa07..507c70f10 100644 --- a/server/go/blobserver/get.go +++ b/server/go/blobserver/get.go @@ -1,6 +1,7 @@ package main import ( + "camli/blobref" "camli/httputil" "fmt" "http" @@ -8,29 +9,33 @@ import ( "io" ) -func handleGet(conn http.ResponseWriter, req *http.Request) { +func createGetHandler(fetcher blobref.Fetcher) func (http.ResponseWriter, *http.Request) { + return func (conn http.ResponseWriter, req *http.Request) { + handleGet(conn, req, fetcher); + } +} + +func handleGet(conn http.ResponseWriter, req *http.Request, fetcher blobref.Fetcher) { blobRef := BlobFromUrlPath(req.URL.Path) if blobRef == nil { httputil.BadRequestError(conn, "Malformed GET URL.") return } - fileName := BlobFileName(blobRef) - stat, err := os.Stat(fileName) - if err == os.ENOENT { + file, size, err := fetcher.Fetch(blobRef) + switch err { + case nil: + break + case os.ENOENT: conn.WriteHeader(http.StatusNotFound) fmt.Fprintf(conn, "Object not found.") return - } - if err != nil { - httputil.ServerError(conn, err) - return - } - file, err := os.Open(fileName, os.O_RDONLY, 0) - if err != nil { + default: httputil.ServerError(conn, err) return } + defer file.Close() + reqRange := getRequestedRange(req) if reqRange.SkipBytes != 0 { _, err = file.Seek(reqRange.SkipBytes, 0) @@ -45,7 +50,7 @@ func handleGet(conn http.ResponseWriter, req *http.Request) { input = io.LimitReader(file, reqRange.LimitBytes) } - remainBytes := stat.Size - reqRange.SkipBytes + remainBytes := size - reqRange.SkipBytes if reqRange.LimitBytes != -1 && reqRange.LimitBytes < remainBytes { remainBytes = reqRange.LimitBytes @@ -56,7 +61,7 @@ func handleGet(conn http.ResponseWriter, req *http.Request) { conn.SetHeader("Content-Range", fmt.Sprintf("bytes %d-%d/%d", reqRange.SkipBytes, reqRange.SkipBytes + remainBytes, - stat.Size)) + size)) conn.WriteHeader(http.StatusPartialContent) } bytesCopied, err := io.Copy(conn, input) @@ -65,21 +70,22 @@ func handleGet(conn http.ResponseWriter, req *http.Request) { // as they've already been receiving bytes. But they should be smart enough // to verify the digest doesn't match. But we close the (chunked) response anyway, // to further signal errors. - if err != nil { - fmt.Fprintf(os.Stderr, "Error sending file: %v, err=%v\n", blobRef, err) + killConnection := func() { closer, _, err := conn.Hijack() if err != nil { closer.Close() } + } + + if err != nil { + fmt.Fprintf(os.Stderr, "Error sending file: %v, err=%v\n", blobRef, err) + killConnection() return } if bytesCopied != remainBytes { fmt.Fprintf(os.Stderr, "Error sending file: %v, copied=%d, not %d\n", blobRef, bytesCopied, remainBytes) - closer, _, err := conn.Hijack() - if err != nil { - closer.Close() - } + killConnection() return } } diff --git a/server/go/blobserver/blobref.go b/server/go/blobserver/localdisk.go similarity index 58% rename from server/go/blobserver/blobref.go rename to server/go/blobserver/localdisk.go index 60498c9bc..29dd987b5 100644 --- a/server/go/blobserver/blobref.go +++ b/server/go/blobserver/localdisk.go @@ -4,8 +4,30 @@ import ( "camli/blobref" "fmt" "regexp" + "os" ) +type diskStorage struct { + Root string +} + +func (ds *diskStorage) Fetch(blob blobref.BlobRef) (blobref.ReadSeekCloser, int64, os.Error) { + fileName := BlobFileName(blob) + stat, err := os.Stat(fileName) + if err == os.ENOENT { + return nil, 0, err + } + file, err := os.Open(fileName, os.O_RDONLY, 0) + if err != nil { + return nil, 0, err + } + return file, stat.Size, nil +} + +func newDiskStorage(root string) *diskStorage { + return &diskStorage{Root: root} +} + var kGetPutPattern *regexp.Regexp = regexp.MustCompile(`^/camli/([a-z0-9]+)-([a-f0-9]+)$`) func BlobFileBaseName(b blobref.BlobRef) string { @@ -25,4 +47,3 @@ func BlobFromUrlPath(path string) blobref.BlobRef { return blobref.FromPattern(kGetPutPattern, path) } - diff --git a/server/go/blobserver/run.sh b/server/go/blobserver/run.sh index db227cf39..74db39349 100755 --- a/server/go/blobserver/run.sh +++ b/server/go/blobserver/run.sh @@ -2,4 +2,4 @@ mkdir /tmp/camliroot export CAMLI_PASSWORD=foo -make && ./camlistored "$@" +make && ./camlistored -listen=:3179 "$@" diff --git a/server/go/jsonsign/sign.go b/server/go/jsonsign/sign.go index f5d2790e4..7de15d3b3 100644 --- a/server/go/jsonsign/sign.go +++ b/server/go/jsonsign/sign.go @@ -53,7 +53,7 @@ func (sr *SignRequest) Sign() (signedJson string, err os.Error) { return inputfail("json \"camliSigner\" key is malformed or unsupported") } - pubkeyReader, err := sr.Fetcher.Fetch(signerBlob) + pubkeyReader, _, err := sr.Fetcher.Fetch(signerBlob) if err != nil { // TODO: not really either an inputfail or an execfail.. but going // with exec for now. diff --git a/server/go/jsonsign/verify.go b/server/go/jsonsign/verify.go index 5e339d55b..c0fe73347 100644 --- a/server/go/jsonsign/verify.go +++ b/server/go/jsonsign/verify.go @@ -111,7 +111,7 @@ func (vr *VerifyRequest) ParsePayloadMap() bool { } func (vr *VerifyRequest) FindAndParsePublicKeyBlob() bool { - reader, err := vr.fetcher.Fetch(vr.CamliSigner) + reader, _, err := vr.fetcher.Fetch(vr.CamliSigner) if err != nil { return vr.fail(fmt.Sprintf("Error fetching public key blob: %v", err)) } diff --git a/server/go/sigserver/Makefile b/server/go/sigserver/Makefile index 595317434..486b4f59b 100644 --- a/server/go/sigserver/Makefile +++ b/server/go/sigserver/Makefile @@ -3,7 +3,6 @@ include $(GOROOT)/src/Make.inc TARG=camsigd GOFILES=\ camsigd.go\ - keys.go\ sign.go\ verify.go\ diff --git a/server/go/sigserver/camsigd.go b/server/go/sigserver/camsigd.go index e9ac095f1..161f2f150 100644 --- a/server/go/sigserver/camsigd.go +++ b/server/go/sigserver/camsigd.go @@ -2,6 +2,7 @@ package main import ( "camli/auth" + "camli/blobref" "camli/httputil" "camli/webserver" "flag" @@ -12,6 +13,31 @@ import ( var accessPassword string +var flagPubKeyDir *string = flag.String("pubkey-dir", "test/pubkey-blobs", + "Temporary development hack; directory to dig-xxxx.camli public keys.") + +type pubkeyDirFetcher struct{} +func (_ *pubkeyDirFetcher) Fetch(b blobref.BlobRef) (file blobref.ReadSeekCloser, size int64, err os.Error) { + fileName := fmt.Sprintf("%s/%s.camli", *flagPubKeyDir, b.String()) + var stat *os.FileInfo + stat, err = os.Stat(fileName) + if err != nil { + return + } + file, err = os.Open(fileName, os.O_RDONLY, 0) + if err != nil { + return + } + size = stat.Size + return +} + +// TODO: for now, the only implementation of the blobref.Fetcher +// interface for fetching public keys is the "local, from disk" +// implementation used for testing. In reality we'd want to be able +// to fetch these from blobservers. +var pubKeyFetcher = &pubkeyDirFetcher{} + func handleRoot(conn http.ResponseWriter, req *http.Request) { fmt.Fprintf(conn, "camsigd") } diff --git a/server/go/sigserver/keys.go b/server/go/sigserver/keys.go deleted file mode 100644 index 73600d052..000000000 --- a/server/go/sigserver/keys.go +++ /dev/null @@ -1,23 +0,0 @@ -package main - -import ( - "camli/blobref" - "flag" - "fmt" - "io" - "os" -) - -var flagPubKeyDir *string = flag.String("pubkey-dir", "test/pubkey-blobs", - "Temporary development hack; directory to dig-xxxx.camli public keys.") - -type fromLocalDiskBlobFetcher struct{} -var blobFetcher = &fromLocalDiskBlobFetcher{} -func (_ *fromLocalDiskBlobFetcher) Fetch(b blobref.BlobRef) (io.ReadCloser, os.Error) { - publicKeyFile := fmt.Sprintf("%s/%s.camli", *flagPubKeyDir, b.String()) - f, err := os.Open(publicKeyFile, os.O_RDONLY, 0) - if err != nil { - return nil, err - } - return f, nil -} diff --git a/server/go/sigserver/sign.go b/server/go/sigserver/sign.go index fd7925e81..2986f5781 100644 --- a/server/go/sigserver/sign.go +++ b/server/go/sigserver/sign.go @@ -27,7 +27,7 @@ func handleSign(conn http.ResponseWriter, req *http.Request) { return } - sreq := &jsonsign.SignRequest{UnsignedJson: jsonStr, Fetcher: blobFetcher} + sreq := &jsonsign.SignRequest{UnsignedJson: jsonStr, Fetcher: pubKeyFetcher} signedJson, err := sreq.Sign() if err != nil { // TODO: some aren't really a "bad request" diff --git a/server/go/sigserver/verify.go b/server/go/sigserver/verify.go index 7b1be49e6..4f90a7112 100644 --- a/server/go/sigserver/verify.go +++ b/server/go/sigserver/verify.go @@ -34,7 +34,7 @@ func handleVerify(conn http.ResponseWriter, req *http.Request) { m := make(map[string]interface{}) - vreq := jsonsign.NewVerificationRequest(sjson, blobFetcher) + vreq := jsonsign.NewVerificationRequest(sjson, pubKeyFetcher) if vreq.Verify() { m["signatureValid"] = 1 m["verifiedData"] = vreq.PayloadMap