From 02717908efdd344cb216e9599b3bea3d9fdc9b44 Mon Sep 17 00:00:00 2001 From: Brett Slatkin Date: Tue, 25 Jan 2011 21:33:19 -0800 Subject: [PATCH 1/2] adds -via to camget; adds -reqlog to blobserver --- clients/go/camget/camget.go | 34 ++++++++++++++++++++++++----- lib/go/client/client.go | 4 ++++ lib/go/client/get.go | 24 ++++++++++++++++++-- server/go/blobserver/camlistored.go | 5 +++++ 4 files changed, 59 insertions(+), 8 deletions(-) diff --git a/clients/go/camget/camget.go b/clients/go/camget/camget.go index c412cc414..f11e35566 100644 --- a/clients/go/camget/camget.go +++ b/clients/go/camget/camget.go @@ -21,12 +21,14 @@ import ( "io" "log" "os" + "strings" ) var flagVerbose *bool = flag.Bool("verbose", false, "be verbose") var flagCheck *bool = flag.Bool("check", false, "just check for the existence of listed blobs; returning 0 if all our present") var flagOutput *string = flag.String("o", "-", "Output file/directory to create. Use -f to overwrite.") +var flagVia *string = flag.String("via", "", "Fetch the blob via the given comma-separated sharerefs (dev only).") func main() { flag.Parse() @@ -41,20 +43,40 @@ func main() { for n := 0; n < flag.NArg(); n++ { arg := flag.Arg(n) - blobref := blobref.Parse(arg) - if blobref == nil { + br := blobref.Parse(arg) + if br == nil { log.Exitf("Failed to parse argument \"%s\" as a blobref.", arg) } if *flagVerbose { - log.Printf("Need to fetch %s", blobref.String()) + log.Printf("Need to fetch %s", br.String()) + } + var ( + r blobref.ReadSeekCloser + err os.Error + ) + + if len(*flagVia) > 0 { + vs := strings.Split(*flagVia, ",", -1) + abr := make([]*blobref.BlobRef, len(vs)) + for i, sbr := range vs { + abr[i] = blobref.Parse(sbr) + if abr[i] == nil { + log.Exitf("Invalid -via blobref: %q", sbr) + } + if *flagVerbose { + log.Printf("via: %s", sbr) + } + } + r, _, err = client.FetchVia(br, abr) + } else { + r, _, err = client.Fetch(br) } - r, _, err := client.Fetch(blobref) if err != nil { - log.Exitf("Failed to fetch %q: %s", blobref, err) + log.Exitf("Failed to fetch %q: %s", br, err) } _, err = io.Copy(w, r) if err != nil { - log.Exitf("Failed transferring %q: %s", blobref, err) + log.Exitf("Failed transferring %q: %s", br, err) } } diff --git a/lib/go/client/client.go b/lib/go/client/client.go index 98c998813..921dd1ecf 100644 --- a/lib/go/client/client.go +++ b/lib/go/client/client.go @@ -64,6 +64,10 @@ func (c *Client) Stats() Stats { return c.stats // copy } +func (c *Client) HasAuthCredentials() bool { + return c.password != "" +} + func (c *Client) authHeader() string { return "Basic " + encodeBase64("username:" + c.password) } diff --git a/lib/go/client/get.go b/lib/go/client/get.go index da52bc340..1e045aaa9 100644 --- a/lib/go/client/get.go +++ b/lib/go/client/get.go @@ -1,6 +1,7 @@ package client import ( + "bytes" "camli/blobref" "camli/http" "fmt" @@ -10,10 +11,29 @@ import ( ) func (c *Client) Fetch(b *blobref.BlobRef) (blobref.ReadSeekCloser, int64, os.Error) { + return c.FetchVia(b, nil) +} + +func (c *Client) FetchVia(b *blobref.BlobRef, v []*blobref.BlobRef) (blobref.ReadSeekCloser, int64, os.Error) { url := fmt.Sprintf("%s/camli/%s", c.server, b) + if len(v) > 0 { + buf := bytes.NewBufferString(url) + buf.WriteString("?via=") + for i, br := range v { + if i != 0 { + buf.WriteString(",") + } + buf.WriteString(br.String()) + } + url = buf.String() + } + req := http.NewGetRequest(url) - req.Header["Authorization"] = c.authHeader() + + if c.HasAuthCredentials() { + req.Header["Authorization"] = c.authHeader() + } resp, err := req.Send() if err != nil { return nil, 0, err @@ -24,7 +44,7 @@ func (c *Client) Fetch(b *blobref.BlobRef) (blobref.ReadSeekCloser, int64, os.Er size, _ = strconv.Atoi64(s) } - return nopSeeker{resp.Body}, size, nil + return nopSeeker{resp.Body}, size, nil } type nopSeeker struct { diff --git a/server/go/blobserver/camlistored.go b/server/go/blobserver/camlistored.go index bf1841e0a..63565fd57 100644 --- a/server/go/blobserver/camlistored.go +++ b/server/go/blobserver/camlistored.go @@ -12,10 +12,12 @@ import ( "flag" "fmt" "http" + "log" "os" ) var flagStorageRoot *string = flag.String("root", "/tmp/camliroot", "Root directory to store files") +var flagRequestLog *bool = flag.Bool("reqlog", false, "Log incoming requests") var stealthMode *bool = flag.Bool("stealth", true, "Run in stealth mode.") var blobFetcher blobref.Fetcher @@ -26,6 +28,9 @@ func handleCamli(conn http.ResponseWriter, req *http.Request) { fmt.Sprintf("Unsupported path (%s) or method (%s).", req.URL.Path, req.Method)) } + if *flagRequestLog { + log.Printf("%s %s", req.Method, req.RawURL) + } switch req.Method { case "GET": switch req.URL.Path { From f5758f6c68ad8ff3b86c6db27c9f46cb65e0ff0a Mon Sep 17 00:00:00 2001 From: Brett Slatkin Date: Tue, 25 Jan 2011 22:44:03 -0800 Subject: [PATCH 2/2] adds "share" camli object schema; support for camput --share --- clients/go/camput/camput.go | 28 ++++++++++++++++++++++++++-- lib/go/schema/schema.go | 12 ++++++++++++ 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/clients/go/camput/camput.go b/clients/go/camput/camput.go index 254cb42e1..f97c73963 100644 --- a/clients/go/camput/camput.go +++ b/clients/go/camput/camput.go @@ -22,7 +22,9 @@ import ( var flagBlob = flag.Bool("blob", false, "upload a file's bytes as a single blob") var flagFile = flag.Bool("file", false, "upload a file's bytes as a blob, as well as its JSON file record") var flagPermanode = flag.Bool("permanode", false, "create a new permanode") -var flagInit = flag.Bool("init", false, "First-time configuration.") +var flagInit = flag.Bool("init", false, "first-time configuration.") +var flagShare = flag.Bool("share", false, "create a camli share by haveref with the given blobrefs") +var flagTransitive = flag.Bool("transitive", true, "share the transitive closure of the given blobrefs") var flagVerbose = flag.Bool("verbose", false, "be verbose") @@ -171,6 +173,17 @@ func (up *Uploader) UploadNewPermanode() (*client.PutResult, os.Error) { return up.Upload(client.NewUploadHandleFromString(signed)) } +func (up *Uploader) UploadShare(target *blobref.BlobRef, transitive bool) (*client.PutResult, os.Error) { + unsigned := schema.NewShareRef(schema.ShareHaveRef, target, transitive) + + signed, err := up.SignMap(unsigned) + if err != nil { + return nil, err + } + + return up.Upload(client.NewUploadHandleFromString(signed)) +} + func sumSet(flags ...*bool) (count int) { for _, f := range flags { if *f { @@ -190,6 +203,7 @@ Usage: camput camput --init # first time configuration camput --blob camput --file + camput --share [--transitive] `) flag.PrintDefaults() os.Exit(1) @@ -211,7 +225,7 @@ func handleResult(what string, pr *client.PutResult, err os.Error) { func main() { flag.Parse() - if sumSet(flagFile, flagBlob, flagPermanode, flagInit) != 1 { + if sumSet(flagFile, flagBlob, flagPermanode, flagInit, flagShare) != 1 { // TODO: say which ones are conflicting usage("Conflicting mode options.") } @@ -242,6 +256,16 @@ func main() { handleResult("file", pr, err) } } + case *flagShare: + if flag.NArg() != 1 { + log.Exitf("--share only supports one blobref") + } + br := blobref.Parse(flag.Arg(0)) + if br == nil { + log.Exitf("BlobRef is invalid: %q", flag.Arg(0)) + } + pr, err := uploader.UploadShare(br, *flagTransitive) + handleResult("share", pr, err) } if *flagVerbose { diff --git a/lib/go/schema/schema.go b/lib/go/schema/schema.go index 970108b09..6440ea9a8 100644 --- a/lib/go/schema/schema.go +++ b/lib/go/schema/schema.go @@ -203,6 +203,18 @@ func PopulateDirectoryMap(m map[string]interface{}, staticSetRef *blobref.BlobRe m["entries"] = staticSetRef.String() } +func NewShareRef(authType string, target *blobref.BlobRef, transitive bool) map[string]interface{} { + m := newCamliMap(1, "" /* no type yet */) + m["camliType"] = "share" + m["authType"] = authType + m["target"] = target.String() + m["transitive"] = transitive + return m +} + +// Types of ShareRefs +const ShareHaveRef = "haveref" + func rfc3339FromNanos(epochnanos int64) string { nanos := epochnanos % 1e9 esec := epochnanos / 1e9