From 6e4f9a17d76c27013d3954b68cad3523d264ff43 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Tue, 18 Jan 2011 10:29:38 -0800 Subject: [PATCH] Actually upload permanode; clean up config, uploading, signing, logging --- clients/go/camput/camput.go | 22 ++++++++++------------ clients/go/camput/init.go | 26 +++++++++++++++++++++++--- lib/go/client/client.go | 20 +++++++++++++++++++- lib/go/client/config.go | 2 +- lib/go/client/upload.go | 22 ++++++++++++++++------ lib/go/jsonsign/sign.go | 32 ++++++++++++++++++++++++-------- 6 files changed, 93 insertions(+), 31 deletions(-) diff --git a/clients/go/camput/camput.go b/clients/go/camput/camput.go index 4d15c1e93..254cb42e1 100644 --- a/clients/go/camput/camput.go +++ b/clients/go/camput/camput.go @@ -5,7 +5,6 @@ package main import ( - "bytes" "camli/blobref" "camli/client" "camli/schema" @@ -138,12 +137,7 @@ func (up *Uploader) UploadMap(m map[string]interface{}) (*client.PutResult, os.E if *flagVerbose { fmt.Printf("json: %s\n", json) } - s1 := sha1.New() - s1.Write([]byte(json)) - bref := blobref.FromHash("sha1", s1) - buf := bytes.NewBufferString(json) - h := &client.UploadHandle{BlobRef: bref, Size: int64(len(json)), Contents: buf} - return up.Upload(h) + return up.Upload(client.NewUploadHandleFromString(json)) } func (up *Uploader) SignMap(m map[string]interface{}) (string, os.Error) { @@ -159,8 +153,9 @@ func (up *Uploader) SignMap(m map[string]interface{}) (string, os.Error) { return "", err } sr := &jsonsign.SignRequest{ - UnsignedJson: unsigned, - Fetcher: up.Client.GetBlobFetcher(), + UnsignedJson: unsigned, + Fetcher: up.Client.GetBlobFetcher(), + UseAgent: true, } return sr.Sign() } @@ -173,8 +168,7 @@ func (up *Uploader) UploadNewPermanode() (*client.PutResult, os.Error) { return nil, err } - log.Printf("Got signed permanode: %q", signed) - return nil, nil + return up.Upload(client.NewUploadHandleFromString(signed)) } func sumSet(flags ...*bool) (count int) { @@ -222,7 +216,11 @@ func main() { usage("Conflicting mode options.") } - uploader := &Uploader{client.NewOrFail()} + client := client.NewOrFail() + if !*flagVerbose { + client.SetLogger(nil) + } + uploader := &Uploader{client} switch { case *flagInit: diff --git a/clients/go/camput/init.go b/clients/go/camput/init.go index 39a66afbb..f319732d2 100644 --- a/clients/go/camput/init.go +++ b/clients/go/camput/init.go @@ -17,7 +17,7 @@ import ( var flagGpgKey = flag.String("gpgkey", "", "(init option only) GPG key to use for signing.") func doInit() { - blobDir := path.Join(client.ConfigDir(), "blobs") + blobDir := path.Join(client.ConfigDir(), "keyblobs") os.Mkdir(client.ConfigDir(), 0700) os.Mkdir(blobDir, 0700) @@ -79,9 +79,29 @@ func doInit() { if f, err := os.Open(client.ConfigFilePath(), os.O_CREAT|os.O_WRONLY, 0600); err == nil { defer f.Close() m := make(map[string]interface{}) - m["blobServer"] = "http://localhost:3179/" - m["blobServerPassword"] = "test" m["publicKeyBlobref"] = bref.String() + + blobPut := make([]map[string]string, 1) + blobPut[0] = map[string]string{ + "alias": "local", + "host": "http://localhost:3179/", + "password": "test", + } + m["blobPut"] = blobPut + + blobGet := make([]map[string]string, 2) + blobGet[0] = map[string]string{ + "alias": "keyblobs", + "path": "$HOME/.camli/keyblobs", + } + blobGet[1] = map[string]string{ + "alias": "local", + "host": "http://localhost:3179/", + "password": "test", + } + m["blobGet"] = blobGet + + jsonBytes, err := json.MarshalIndent(m, "", " ") if err != nil { log.Exitf("JSON serialization error: %v", err) diff --git a/lib/go/client/client.go b/lib/go/client/client.go index f53895aee..98c998813 100644 --- a/lib/go/client/client.go +++ b/lib/go/client/client.go @@ -2,6 +2,8 @@ package client import ( "fmt" + "log" + "os" "sync" ) @@ -25,6 +27,8 @@ type Client struct { statsMutex sync.Mutex stats Stats + + log *log.Logger // not nil } type ByCountAndBytes struct { @@ -37,7 +41,21 @@ func (bb *ByCountAndBytes) String() string { } func NewOrFail() *Client { - return &Client{server: blobServerOrDie(), password: passwordOrDie()} + log := log.New(os.Stderr, "", log.Ldate|log.Ltime) + return &Client{server: blobServerOrDie(), password: passwordOrDie(), log: log} +} + +type devNullWriter struct{} +func (_ *devNullWriter) Write(p []byte) (int, os.Error) { + return len(p), nil +} + +func (c *Client) SetLogger(logger *log.Logger) { + if logger == nil { + c.log = log.New(&devNullWriter{}, "", 0) + } else { + c.log = logger + } } func (c *Client) Stats() Stats { diff --git a/lib/go/client/config.go b/lib/go/client/config.go index 4be2c8d39..295518175 100644 --- a/lib/go/client/config.go +++ b/lib/go/client/config.go @@ -121,5 +121,5 @@ func (c *Client) SignerPublicKeyBlobref() *blobref.BlobRef { func (c *Client) GetBlobFetcher() blobref.Fetcher { // TODO: make a NewSeriesAttemptFetcher(...all configured fetch paths...) - return blobref.NewSimpleDirectoryFetcher(path.Join(os.Getenv("HOME"), ".camli", "blobs")) + return blobref.NewSimpleDirectoryFetcher(path.Join(os.Getenv("HOME"), ".camli", "keyblobs")) } diff --git a/lib/go/client/upload.go b/lib/go/client/upload.go index bbe65ccfc..a66ffb7f1 100644 --- a/lib/go/client/upload.go +++ b/lib/go/client/upload.go @@ -4,6 +4,7 @@ import ( "bytes" "camli/blobref" "camli/http" + "crypto/sha1" "encoding/base64" "fmt" "io" @@ -25,13 +26,22 @@ type PutResult struct { Skipped bool // already present on blobserver } +// Note: must not touch data after calling this. +func NewUploadHandleFromString(data string) *UploadHandle { + s1 := sha1.New() + s1.Write([]byte(data)) + bref := blobref.FromHash("sha1", s1) + buf := bytes.NewBufferString(data) + return &UploadHandle{BlobRef: bref, Size: int64(len(data)), Contents: buf} +} + func encodeBase64(s string) string { buf := make([]byte, base64.StdEncoding.EncodedLen(len(s))) base64.StdEncoding.Encode(buf, []byte(s)) return string(buf) } -func jsonFromResponse(resp *http.Response) (map[string]interface{}, os.Error) { +func (c *Client) jsonFromResponse(resp *http.Response) (map[string]interface{}, os.Error) { if resp.StatusCode != 200 { log.Printf("Failed to JSON from response; status code is %d", resp.StatusCode) io.Copy(os.Stderr, resp.Body) @@ -52,7 +62,7 @@ func (c *Client) Upload(h *UploadHandle) (*PutResult, os.Error) { error := func(msg string, e os.Error) (*PutResult, os.Error) { err := os.NewError(fmt.Sprintf("Error uploading blob %s: %s; err=%s", h.BlobRef, msg, e)) - log.Print(err.String()) + c.log.Print(err.String()) return nil, err } @@ -80,7 +90,7 @@ func (c *Client) Upload(h *UploadHandle) (*PutResult, os.Error) { return error("preupload http error", err) } - pur, err := jsonFromResponse(resp) + pur, err := c.jsonFromResponse(resp) if err != nil { return error("preupload json parse error", err) } @@ -115,7 +125,7 @@ func (c *Client) Upload(h *UploadHandle) (*PutResult, os.Error) { h.BlobRef, h.BlobRef) multiPartFooter := "\r\n--"+boundary+"--\r\n" - log.Printf("Uploading to URL: %s", uploadUrl) + c.log.Printf("Uploading to URL: %s", uploadUrl) req = http.NewPostRequest(uploadUrl, "multipart/form-data; boundary="+boundary, io.MultiReader( @@ -153,14 +163,14 @@ func (c *Client) Upload(h *UploadHandle) (*PutResult, os.Error) { } } - ures, err := jsonFromResponse(resp) + ures, err := c.jsonFromResponse(resp) if err != nil { return error("json parse from upload error", err) } errorText, ok := ures["errorText"].(string) if ok { - log.Printf("Blob server reports error: %s", errorText) + c.log.Printf("Blob server reports error: %s", errorText) } received, ok := ures["received"].([]interface{}) diff --git a/lib/go/jsonsign/sign.go b/lib/go/jsonsign/sign.go index 4b1d35644..1fc282623 100644 --- a/lib/go/jsonsign/sign.go +++ b/lib/go/jsonsign/sign.go @@ -25,6 +25,11 @@ var flagSecretRing *string = flag.String("secret-keyring", "./test/test-secring. type SignRequest struct { UnsignedJson string Fetcher blobref.Fetcher + UseAgent bool + + // In server-mode, don't use any default (user) keys + // TODO: formalize what this means? + ServerMode bool } func (sr *SignRequest) Sign() (signedJson string, err os.Error) { @@ -72,16 +77,27 @@ func (sr *SignRequest) Sign() (signedJson string, err os.Error) { } trimmedJson = trimmedJson[0:len(trimmedJson)-1] - cmd, err := exec.Run( - *gpgPath, - []string{ - "--no-default-keyring", - "--keyring", *flagRing, // TODO: needed for signing? - "--secret-keyring", *flagSecretRing, + args := []string{"gpg", "--local-user", pk.KeyIdString(), "--detach-sign", - "--armor", - "-"}, + "--armor"} + + if sr.UseAgent { + args = append(args, "--use-agent") + } + + if sr.ServerMode { + args = append(args, + "--no-default-keyring", + "--keyring", *flagRing, // TODO: needed for signing? + "--secret-keyring", *flagSecretRing) + } + + args = append(args, "-") + + cmd, err := exec.Run( + *gpgPath, + args, os.Environ(), ".", exec.Pipe, // stdin