From 12f236da37624bf2e78c50840042b2fb16862401 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Sun, 25 Sep 2011 17:40:01 -0700 Subject: [PATCH] Start of major camput command-line cleanup. So far only "init" and "file" are done: $ grep -r RegisterCommand clients/go/camput clients/go/camput/init.go: RegisterCommand("init", c) clients/go/camput/files.go: RegisterCommand("file", cmd) New command line syntax is: Usage: camput [globalopts] [commandopts] [commandargs] Examples: camput init camput file [opts] camput permanode [opts] (create a new permanode) camput share [opts] camput blob (raw, without any metadata) camput blob - (read from stdin) camput attr Set attribute camput attr --add Adds attribute (e.g. "tag") camput attr --del [] Deletes named attribute [value] For mode-specific help: camput MODE -help Change-Id: I4aeabeff6450beab83c13d94c16bd007addb8b58 --- clients/go/camput/camput.go | 222 ++++++++++++++++----------------- clients/go/camput/files.go | 106 ++++++++++++++++ clients/go/camput/flatcache.go | 1 - clients/go/camput/init.go | 38 +++++- clients/go/camput/permanode.go | 33 +++++ lib/go/camli/client/config.go | 2 +- 6 files changed, 282 insertions(+), 120 deletions(-) create mode 100644 clients/go/camput/files.go create mode 100644 clients/go/camput/permanode.go diff --git a/clients/go/camput/camput.go b/clients/go/camput/camput.go index 971966cd8..2d9952c0e 100644 --- a/clients/go/camput/camput.go +++ b/clients/go/camput/camput.go @@ -25,7 +25,6 @@ import ( "log" "os" "sort" - "strings" "camli/blobref" "camli/client" @@ -36,15 +35,8 @@ import ( const buffered = 16 // arbitrary // Things that can be uploaded. (at most one of these) -var flagBlob = flag.Bool("blob", false, "upload a file's bytes as a single blob") -var flagFile = flag.Bool("file", false, "upload a file's bytes as a blob, as well as its JSON file record") -var flagPermanode = flag.Bool("permanode", false, "create a new permanode") -var flagInit = flag.Bool("init", false, "first-time configuration.") -var flagShare = flag.Bool("share", false, "create a camli share by haveref with the given blobrefs") -var flagTransitive = flag.Bool("transitive", true, "share the transitive closure of the given blobrefs") -var flagRemove = flag.Bool("remove", false, "remove the list of blobrefs") -var flagName = flag.String("name", "", "Optional name attribute to set on permanode when using -permanode and -file") -var flagTag = flag.String("tag", "", "Optional tag attribute to set on permanode when using -permanode and -file. Single value or comma separated ones.") +//var flagTransitive = flag.Bool("transitive", true, "share the transitive closure of the given blobrefs") +//var flagRemove = flag.Bool("remove", false, "remove the list of blobrefs") var ( flagVerbose = flag.Bool("verbose", false, "be verbose") @@ -52,13 +44,31 @@ var ( ) var ( -flagUseStatCache = flag.Bool("statcache", false, "Use the stat cache, assuming unchanged files already uploaded in the past are still there. Fast, but potentially dangerous.") - flagUseHaveCache = flag.Bool("havecache", false, "Use the 'have cache', a cache keeping track of what blobs the remote server should already have from previous uploads.") + flagUseStatCache = flag.Bool("statcache", false, "Use the stat cache, assuming unchanged files already uploaded in the past are still there. Fast, but potentially dangerous.") + flagUseHaveCache = flag.Bool("havecache", false, "Use the 'have cache', a cache keeping track of what blobs the remote server should already have from previous uploads.") ) -var flagSetAttr = flag.Bool("set-attr", false, "set (replace) an attribute") -var flagAddAttr = flag.Bool("add-attr", false, "add an attribute, additional if one already exists") +//var flagSetAttr = flag.Bool("set-attr", false, "set (replace) an attribute") +//var flagAddAttr = flag.Bool("add-attr", false, "add an attribute, additional if one already exists") +var ErrUsage = os.NewError("invalid command usage") + +type CommandRunner interface { + Usage() + RunCommand(up *Uploader, args []string) os.Error +} + +var modeCommands = make(map[string]CommandRunner) + +func RegisterCommand(mode string, cmd CommandRunner) { + if _, dup := modeCommands[mode]; dup { + log.Fatalf("duplicate command %q registered", mode) + } + modeCommands[mode] = cmd +} + +// wereErrors gets set to true if any error was encountered, which +// changes the os.Exit value var wereErrors = false // UploadCache is the "stat cache" for regular files. Given a current @@ -79,6 +89,7 @@ type Uploader struct { *client.Client entityFetcher jsonsign.EntityFetcher + transport *tinkerTransport pwd string statCache UploadCache haveCache HaveCache @@ -341,12 +352,30 @@ func usage(msg string) { fmt.Println("Error:", msg) } fmt.Println(` -Usage: camput +Usage: camput [globalopts] [commandopts] [commandargs] - camput --init # first time configuration - camput --blob - camput --file - camput --share [--transitive] +Examples: + + camput init + + camput file [opts] + + camput permanode [opts] (create a new permanode) + + camput share [opts] + + camput blob (raw, without any metadata) + camput blob - (read from stdin) + + camput attr Set attribute + camput attr --add Adds attribute (e.g. "tag") + camput attr --del [] Deletes named attribute [value] + +For mode-specific help: + + camput MODE -help + +Global options: `) flag.PrintDefaults() os.Exit(1) @@ -361,17 +390,7 @@ func handleResult(what string, pr *client.PutResult, err os.Error) { fmt.Println(pr.BlobRef.String()) } -func main() { - jsonsign.AddFlags() - flag.Parse() - - nOpts := sumSet(flagFile, flagBlob, flagPermanode, flagInit, flagShare, flagRemove, - flagSetAttr, flagAddAttr) - if !(nOpts == 1 || - (nOpts == 2 && *flagFile && *flagPermanode)) { - usage("Conflicting mode options.") - } - +func makeUploader() *Uploader { cc := client.NewOrFail() if !*flagVerbose { cc.SetLogger(nil) @@ -387,9 +406,10 @@ func main() { } up := &Uploader{ - Client: cc, - pwd: pwd, - filecapc: make(chan bool, 10 /* TODO: config option on max files at a time */ ), + Client: cc, + transport: transport, + pwd: pwd, + filecapc: make(chan bool, 10 /* TODO: config option on max files at a time */ ), entityFetcher: &jsonsign.CachingEntityFetcher{ Fetcher: &jsonsign.FileEntityFetcher{File: cc.SecretRingFile()}, }, @@ -405,79 +425,59 @@ func main() { defer cache.Save() up.haveCache = cache } + return up +} - switch { - case *flagInit: - doInit() - return - case *flagFile || *flagBlob: - var ( - permaNode *client.PutResult - lastPut *client.PutResult - err os.Error - ) - if n := flag.NArg(); *flagPermanode { - if n != 1 { - log.Fatalf("Options --permanode and --file can only be used together when there's exactly one argument") - } - permaNode, err = up.UploadNewPermanode() - if err != nil { - log.Fatalf("Error uploading permanode: %v", err) - } - } - for n := 0; n < flag.NArg(); n++ { - if *flagBlob { - lastPut, err = up.UploadFileBlob(flag.Arg(n)) - handleResult("blob", lastPut, err) - } else { - lastPut, err = up.UploadFile(flag.Arg(n)) - handleResult("file", lastPut, err) - } - } - if permaNode != nil { - put, err := up.UploadAndSignMap(schema.NewSetAttributeClaim(permaNode.BlobRef, "camliContent", lastPut.BlobRef.String())) - handleResult("claim-permanode-content", put, err) - if *flagName != "" { - put, err := up.UploadAndSignMap(schema.NewSetAttributeClaim(permaNode.BlobRef, "name", *flagName)) - handleResult("claim-permanode-name", put, err) - } - if *flagTag != "" { - tags := strings.Split(*flagTag, ",") - m := schema.NewSetAttributeClaim(permaNode.BlobRef, "tag", tags[0]) - for _, tag := range tags { - m = schema.NewAddAttributeClaim(permaNode.BlobRef, "tag", tag) - put, err := up.UploadAndSignMap(m) - handleResult("claim-permanode-tag", put, err) - } - } - handleResult("permanode", permaNode, nil) - } - case *flagPermanode: - if flag.NArg() > 0 { - log.Fatalf("--permanode doesn't take any additional arguments") - } - pr, err := up.UploadNewPermanode() - handleResult("permanode", pr, err) - if *flagName != "" { - put, err := up.UploadAndSignMap(schema.NewSetAttributeClaim(pr.BlobRef, "name", *flagName)) - handleResult("permanode-name", put, err) - } - case *flagShare: - if flag.NArg() != 1 { - log.Fatalf("--share only supports one blobref") - } - br := blobref.Parse(flag.Arg(0)) - if br == nil { - log.Fatalf("BlobRef is invalid: %q", flag.Arg(0)) - } - pr, err := up.UploadShare(br, *flagTransitive) - handleResult("share", pr, err) - case *flagRemove: - if flag.NArg() == 0 { - log.Fatalf("--remove takes one or more blobrefs") - } - err := up.RemoveBlobs(blobref.ParseMulti(flag.Args())) - if err != nil { +func main() { + jsonsign.AddFlags() + flag.Parse() + + if flag.NArg() == 0 { + usage("No mode given.") + } + + mode := flag.Arg(0) + cmd, ok := modeCommands[mode] + if !ok { + usage(fmt.Sprintf("Unknown mode %q", mode)) + } + + up := makeUploader() + err := cmd.RunCommand(up, flag.Args()[1:]) + if err == ErrUsage { + cmd.Usage() + os.Exit(1) + } + if *flagVerbose { + stats := up.Stats() + log.Printf("Client stats: %s", stats.String()) + log.Printf(" #HTTP reqs: %d", up.transport.reqs) + } + if err != nil || wereErrors /* TODO: remove this part */ { + log.Printf("Error: %v", err) + os.Exit(2) + } +} + +// TODO(bradfitz): finish converting these to the new style + +/* + case *flagShare: + if flag.NArg() != 1 { + log.Fatalf("--share only supports one blobref") + } + br := blobref.Parse(flag.Arg(0)) + if br == nil { + log.Fatalf("BlobRef is invalid: %q", flag.Arg(0)) + } + pr, err := up.UploadShare(br, *flagTransitive) + handleResult("share", pr, err) + case *flagRemove: + if flag.NArg() == 0 { + log.Fatalf("--remove takes one or more blobrefs") + } + err := up.RemoveBlobs(blobref.ParseMulti(flag.Args())) + if err != nil { log.Printf("Error removing blobs %s: %s", strings.Join(flag.Args(), ","), err) wereErrors = true } @@ -495,14 +495,6 @@ func main() { } put, err := up.UploadAndSignMap(m) handleResult(m["claimType"].(string), put, err) - } - - if *flagVerbose { - stats := up.Stats() - log.Printf("Client stats: %s", stats.String()) - log.Printf(" #HTTP reqs: %d", transport.reqs) - } - if wereErrors { - os.Exit(2) - } } + +*/ diff --git a/clients/go/camput/files.go b/clients/go/camput/files.go new file mode 100644 index 000000000..43e6c542f --- /dev/null +++ b/clients/go/camput/files.go @@ -0,0 +1,106 @@ +/* +Copyright 2011 Google Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "flag" + "fmt" + "os" + "strings" + + "camli/client" + "camli/schema" +) + +type fileCmd struct { + flags *flag.FlagSet + + name string + tag string + + makePermanode bool +} + +func init() { + flags := flag.NewFlagSet("file options", flag.ContinueOnError) + flags.Usage = func() {} + cmd := &fileCmd{flags: flags} + + flags.BoolVar(&cmd.makePermanode, "permanode", false, "Create an associate a new permanode for the uploaded file or directory.") + flags.StringVar(&cmd.name, "name", "", "Optional name attribute to set on permanode when using -permanode and -file") + flags.StringVar(&cmd.tag, "tag", "", "Optional tag attribute to set on permanode when using -permanode and -file. Single value or comma separated ones.") + + RegisterCommand("file", cmd) +} + +func (c *fileCmd) Usage() { + fmt.Fprintf(os.Stderr, "Usage: camput [globalopts] file [fileopts] \n\nFile options:\n") + c.flags.PrintDefaults() +} + +func (c *fileCmd) RunCommand(up *Uploader, args []string) os.Error { + if err := c.flags.Parse(args); err != nil { + return ErrUsage + } + args = c.flags.Args() + if len(args) == 0 { + return os.NewError("No files or directories given.") + } + + var ( + permaNode *client.PutResult + lastPut *client.PutResult + err os.Error + ) + if c.makePermanode { + if len(args) != 1 { + return fmt.Errorf("The --permanode flag can only be used with exactly one file or directory argument") + } + permaNode, err = up.UploadNewPermanode() + if err != nil { + return fmt.Errorf("Uploading permanode: %v", err) + } + } + + for _, arg := range args { + //if *flagBlob { + //lastPut, err = up.UploadFileBlob(flag.Arg(n)) + // handleResult("blob", lastPut, err) + lastPut, err = up.UploadFile(arg) + handleResult("file", lastPut, err) + + if permaNode != nil { + put, err := up.UploadAndSignMap(schema.NewSetAttributeClaim(permaNode.BlobRef, "camliContent", lastPut.BlobRef.String())) + handleResult("claim-permanode-content", put, err) + if c.name != "" { + put, err := up.UploadAndSignMap(schema.NewSetAttributeClaim(permaNode.BlobRef, "name", c.name)) + handleResult("claim-permanode-name", put, err) + } + if c.tag != "" { + tags := strings.Split(c.tag, ",") + m := schema.NewSetAttributeClaim(permaNode.BlobRef, "tag", tags[0]) + for _, tag := range tags { + m = schema.NewAddAttributeClaim(permaNode.BlobRef, "tag", tag) + put, err := up.UploadAndSignMap(m) + handleResult("claim-permanode-tag", put, err) + } + } + handleResult("permanode", permaNode, nil) + } + } + return nil +} diff --git a/clients/go/camput/flatcache.go b/clients/go/camput/flatcache.go index 49f694b84..54ca5e574 100644 --- a/clients/go/camput/flatcache.go +++ b/clients/go/camput/flatcache.go @@ -79,7 +79,6 @@ func cacheKey(pwd, filename string) string { return filepath.Clean(pwd) + "\x00" + filepath.Clean(filename) } - func (c *FlatStatCache) CachedPutResult(pwd, filename string, fi *os.FileInfo) (*client.PutResult, os.Error) { c.mu.Lock() defer c.mu.Unlock() diff --git a/clients/go/camput/init.go b/clients/go/camput/init.go index 19600a361..3ca8d3ecc 100644 --- a/clients/go/camput/init.go +++ b/clients/go/camput/init.go @@ -20,6 +20,7 @@ import ( "crypto/sha1" "exec" "flag" + "fmt" "os" "io/ioutil" "path" @@ -32,14 +33,44 @@ import ( "camli/osutil" ) -var flagGpgKey = flag.String("gpgkey", "", "(init option only) GPG key to use for signing.") +type initCmd struct { + flags *flag.FlagSet + gpgkey string +} + +func init() { + flags := flag.NewFlagSet("init options", flag.ContinueOnError) + flags.Usage = func() {} + c := &initCmd{flags: flags} + + flags.StringVar(&c.gpgkey, "gpgkey", "", "GPG key to use for signing (overrides $GPGKEY environment)") + + RegisterCommand("init", c) +} + +func (c *initCmd) Usage() { + fmt.Fprintf(os.Stderr, `Usage: camput init [opts] + +Initialize the camput configuration file. + +Options: +`) + c.flags.PrintDefaults() +} + +func (c *initCmd) RunCommand(up *Uploader, args []string) os.Error { + if err := c.flags.Parse(args); err != nil { + return ErrUsage + } + if c.flags.NArg() > 0 { + return ErrUsage + } -func doInit() { blobDir := path.Join(osutil.CamliConfigDir(), "keyblobs") os.Mkdir(osutil.CamliConfigDir(), 0700) os.Mkdir(blobDir, 0700) - keyId := *flagGpgKey + keyId := c.gpgkey if keyId == "" { keyId = os.Getenv("GPGKEY") } @@ -115,4 +146,5 @@ func doInit() { } log.Printf("Wrote %q; modify as necessary.", client.ConfigFilePath()) } + return nil } diff --git a/clients/go/camput/permanode.go b/clients/go/camput/permanode.go new file mode 100644 index 000000000..90c0daf7e --- /dev/null +++ b/clients/go/camput/permanode.go @@ -0,0 +1,33 @@ +/* +Copyright 2011 Google Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "os" +) + +type permanodeCmd struct{} + +func (permanodeCmd) RunCommand(up *Uploader, args []string) os.Error { + pr, err := up.UploadNewPermanode() + handleResult("permanode", pr, err) + //if *flagName != "" { + // put, err := up.UploadAndSignMap(schema.NewSetAttributeClaim(pr.BlobRef, "name", *flagName)) + //handleResult("permanode-name", put, err) + //} + return nil +} diff --git a/lib/go/camli/client/config.go b/lib/go/camli/client/config.go index e0c0720a3..9629ad97a 100644 --- a/lib/go/camli/client/config.go +++ b/lib/go/camli/client/config.go @@ -125,7 +125,7 @@ func SignerPublicKeyBlobref() *blobref.BlobRef { key := "keyId" keyId, ok := config[key].(string) if !ok { - log.Printf("No key %q in JSON configuration file %q; have you run \"camput --init\"?", key, ConfigFilePath()) + log.Printf("No key %q in JSON configuration file %q; have you run \"camput init\"?", key, ConfigFilePath()) return nil } keyRing, _ := config["secretRing"].(string)