Rooba
/
perkeep
mirror of https://github.com/perkeep/perkeep.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Actually upload permanode; clean up config, uploading, signing, logging

This commit is contained in:
Brad Fitzpatrick 2011-01-18 10:29:38 -08:00
parent 3eaf825674
commit 6e4f9a17d7
6 changed files with 93 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
clients/go/camput/camput.go
View File

@ -5,7 +5,6 @@
package main
import (
"bytes"
"camli/blobref"
"camli/client"
"camli/schema"
@ -138,12 +137,7 @@ func (up *Uploader) UploadMap(m map[string]interface{}) (*client.PutResult, os.E
if *flagVerbose {
fmt.Printf("json: %s\n", json)
}
s1 := sha1.New()
s1.Write([]byte(json))
bref := blobref.FromHash("sha1", s1)
buf := bytes.NewBufferString(json)
h := &client.UploadHandle{BlobRef: bref, Size: int64(len(json)), Contents: buf}
return up.Upload(h)
return up.Upload(client.NewUploadHandleFromString(json))
}
func (up *Uploader) SignMap(m map[string]interface{}) (string, os.Error) {
@ -159,8 +153,9 @@ func (up *Uploader) SignMap(m map[string]interface{}) (string, os.Error) {
return "", err
}
sr := &jsonsign.SignRequest{
UnsignedJson: unsigned,
Fetcher: up.Client.GetBlobFetcher(),
UnsignedJson: unsigned,
Fetcher: up.Client.GetBlobFetcher(),
UseAgent: true,
}
return sr.Sign()
}
@ -173,8 +168,7 @@ func (up *Uploader) UploadNewPermanode() (*client.PutResult, os.Error) {
return nil, err
}
log.Printf("Got signed permanode: %q", signed)
return nil, nil
return up.Upload(client.NewUploadHandleFromString(signed))
}
func sumSet(flags ...*bool) (count int) {
@ -222,7 +216,11 @@ func main() {
usage("Conflicting mode options.")
}
uploader := &Uploader{client.NewOrFail()}
client := client.NewOrFail()
if !*flagVerbose {
client.SetLogger(nil)
}
uploader := &Uploader{client}
switch {
case *flagInit:

26
clients/go/camput/init.go
View File

@ -17,7 +17,7 @@ import (
var flagGpgKey = flag.String("gpgkey", "", "(init option only) GPG key to use for signing.")
func doInit() {
blobDir := path.Join(client.ConfigDir(), "blobs")
blobDir := path.Join(client.ConfigDir(), "keyblobs")
os.Mkdir(client.ConfigDir(), 0700)
os.Mkdir(blobDir, 0700)
@ -79,9 +79,29 @@ func doInit() {
if f, err := os.Open(client.ConfigFilePath(), os.O_CREAT|os.O_WRONLY, 0600); err == nil {
defer f.Close()
m := make(map[string]interface{})
m["blobServer"] = "http://localhost:3179/"
m["blobServerPassword"] = "test"
m["publicKeyBlobref"] = bref.String()
blobPut := make([]map[string]string, 1)
blobPut[0] = map[string]string{
"alias": "local",
"host": "http://localhost:3179/",
"password": "test",
}
m["blobPut"] = blobPut
blobGet := make([]map[string]string, 2)
blobGet[0] = map[string]string{
"alias": "keyblobs",
"path": "$HOME/.camli/keyblobs",
}
blobGet[1] = map[string]string{
"alias": "local",
"host": "http://localhost:3179/",
"password": "test",
}
m["blobGet"] = blobGet
jsonBytes, err := json.MarshalIndent(m, "", " ")
if err != nil {
log.Exitf("JSON serialization error: %v", err)

20
lib/go/client/client.go
View File

@ -2,6 +2,8 @@ package client
import (
"fmt"
"log"
"os"
"sync"
)
@ -25,6 +27,8 @@ type Client struct {
statsMutex sync.Mutex
stats Stats
log *log.Logger // not nil
}
type ByCountAndBytes struct {
@ -37,7 +41,21 @@ func (bb *ByCountAndBytes) String() string {
}
func NewOrFail() *Client {
return &Client{server: blobServerOrDie(), password: passwordOrDie()}
log := log.New(os.Stderr, "", log.Ldate|log.Ltime)
return &Client{server: blobServerOrDie(), password: passwordOrDie(), log: log}
}
type devNullWriter struct{}
func (_ *devNullWriter) Write(p []byte) (int, os.Error) {
return len(p), nil
}
func (c *Client) SetLogger(logger *log.Logger) {
if logger == nil {
c.log = log.New(&devNullWriter{}, "", 0)
} else {
c.log = logger
}
}
func (c *Client) Stats() Stats {

2
lib/go/client/config.go
View File

@ -121,5 +121,5 @@ func (c *Client) SignerPublicKeyBlobref() *blobref.BlobRef {
func (c *Client) GetBlobFetcher() blobref.Fetcher {
// TODO: make a NewSeriesAttemptFetcher(...all configured fetch paths...)
return blobref.NewSimpleDirectoryFetcher(path.Join(os.Getenv("HOME"), ".camli", "blobs"))
return blobref.NewSimpleDirectoryFetcher(path.Join(os.Getenv("HOME"), ".camli", "keyblobs"))
}

22
lib/go/client/upload.go
View File

@ -4,6 +4,7 @@ import (
"bytes"
"camli/blobref"
"camli/http"
"crypto/sha1"
"encoding/base64"
"fmt"
"io"
@ -25,13 +26,22 @@ type PutResult struct {
Skipped bool // already present on blobserver
}
// Note: must not touch data after calling this.
func NewUploadHandleFromString(data string) *UploadHandle {
s1 := sha1.New()
s1.Write([]byte(data))
bref := blobref.FromHash("sha1", s1)
buf := bytes.NewBufferString(data)
return &UploadHandle{BlobRef: bref, Size: int64(len(data)), Contents: buf}
}
func encodeBase64(s string) string {
buf := make([]byte, base64.StdEncoding.EncodedLen(len(s)))
base64.StdEncoding.Encode(buf, []byte(s))
return string(buf)
}
func jsonFromResponse(resp *http.Response) (map[string]interface{}, os.Error) {
func (c *Client) jsonFromResponse(resp *http.Response) (map[string]interface{}, os.Error) {
if resp.StatusCode != 200 {
log.Printf("Failed to JSON from response; status code is %d", resp.StatusCode)
io.Copy(os.Stderr, resp.Body)
@ -52,7 +62,7 @@ func (c *Client) Upload(h *UploadHandle) (*PutResult, os.Error) {
error := func(msg string, e os.Error) (*PutResult, os.Error) {
err := os.NewError(fmt.Sprintf("Error uploading blob %s: %s; err=%s",
h.BlobRef, msg, e))
log.Print(err.String())
c.log.Print(err.String())
return nil, err
}
@ -80,7 +90,7 @@ func (c *Client) Upload(h *UploadHandle) (*PutResult, os.Error) {
return error("preupload http error", err)
}
pur, err := jsonFromResponse(resp)
pur, err := c.jsonFromResponse(resp)
if err != nil {
return error("preupload json parse error", err)
}
@ -115,7 +125,7 @@ func (c *Client) Upload(h *UploadHandle) (*PutResult, os.Error) {
h.BlobRef, h.BlobRef)
multiPartFooter := "\r\n--"+boundary+"--\r\n"
log.Printf("Uploading to URL: %s", uploadUrl)
c.log.Printf("Uploading to URL: %s", uploadUrl)
req = http.NewPostRequest(uploadUrl,
"multipart/form-data; boundary="+boundary,
io.MultiReader(
@ -153,14 +163,14 @@ func (c *Client) Upload(h *UploadHandle) (*PutResult, os.Error) {
}
}
ures, err := jsonFromResponse(resp)
ures, err := c.jsonFromResponse(resp)
if err != nil {
return error("json parse from upload error", err)
}
errorText, ok := ures["errorText"].(string)
if ok {
log.Printf("Blob server reports error: %s", errorText)
c.log.Printf("Blob server reports error: %s", errorText)
}
received, ok := ures["received"].([]interface{})

32
lib/go/jsonsign/sign.go
View File

@ -25,6 +25,11 @@ var flagSecretRing *string = flag.String("secret-keyring", "./test/test-secring.
type SignRequest struct {
UnsignedJson string
Fetcher blobref.Fetcher
UseAgent bool
// In server-mode, don't use any default (user) keys
// TODO: formalize what this means?
ServerMode bool
}
func (sr *SignRequest) Sign() (signedJson string, err os.Error) {
@ -72,16 +77,27 @@ func (sr *SignRequest) Sign() (signedJson string, err os.Error) {
}
trimmedJson = trimmedJson[0:len(trimmedJson)-1]
cmd, err := exec.Run(
*gpgPath,
[]string{
"--no-default-keyring",
"--keyring", *flagRing, // TODO: needed for signing?
"--secret-keyring", *flagSecretRing,
args := []string{"gpg",
"--local-user", pk.KeyIdString(),
"--detach-sign",
"--armor",
"-"},
"--armor"}
if sr.UseAgent {
args = append(args, "--use-agent")
}
if sr.ServerMode {
args = append(args,
"--no-default-keyring",
"--keyring", *flagRing, // TODO: needed for signing?
"--secret-keyring", *flagSecretRing)
}
args = append(args, "-")
cmd, err := exec.Run(
*gpgPath,
args,
os.Environ(),
".",
exec.Pipe, // stdin