From 59d0d6cb7ede83b0af96f57f71bf1594d03016ba Mon Sep 17 00:00:00 2001 From: mpl Date: Tue, 19 Feb 2013 00:35:43 +0100 Subject: [PATCH] Create camtool cmd, with sync subcommand. Code in common with camput was factorized in pkg/cmdmain/ Partly addresses http://code.google.com/p/camlistore/issues/detail?id=117 Change-Id: Iabebea7ea78e56bdb8a6eccee9456c52bfe9cceb --- cmd/camput/attr.go | 10 +- cmd/camput/blobs.go | 10 +- cmd/camput/camput.go | 205 ++---------------- cmd/camput/files.go | 22 +- cmd/camput/flatcache.go | 2 +- cmd/camput/init.go | 9 +- cmd/camput/logging.go | 4 +- cmd/camput/permanode.go | 8 +- cmd/camput/rawobj.go | 8 +- cmd/camput/remove.go | 11 +- cmd/camput/share.go | 13 +- cmd/camtool/camtool.go | 28 +++ cmd/{camsync/camsync.go => camtool/sync.go} | 196 +++++++++-------- pkg/cmdmain/cmdmain.go | 224 ++++++++++++++++++++ 14 files changed, 435 insertions(+), 315 deletions(-) create mode 100644 cmd/camtool/camtool.go rename cmd/{camsync/camsync.go => camtool/sync.go} (64%) create mode 100644 pkg/cmdmain/cmdmain.go diff --git a/cmd/camput/attr.go b/cmd/camput/attr.go index 7a93fecdf..b68bfcb96 100644 --- a/cmd/camput/attr.go +++ b/cmd/camput/attr.go @@ -22,16 +22,18 @@ import ( "fmt" "camlistore.org/pkg/blobref" + "camlistore.org/pkg/cmdmain" "camlistore.org/pkg/schema" ) type attrCmd struct { add bool del bool + up *Uploader } func init() { - RegisterCommand("attr", func(flags *flag.FlagSet) CommandRunner { + cmdmain.RegisterCommand("attr", func(flags *flag.FlagSet) cmdmain.CommandRunner { cmd := new(attrCmd) flags.BoolVar(&cmd.add, "add", false, `Adds attribute (e.g. "tag")`) flags.BoolVar(&cmd.del, "del", false, "Deletes named attribute [value]") @@ -40,7 +42,7 @@ func init() { } func (c *attrCmd) Usage() { - errf("Usage: camput [globalopts] attr [attroption] ") + cmdmain.Errf("Usage: camput [globalopts] attr [attroption] ") } func (c *attrCmd) Examples() []string { @@ -51,7 +53,7 @@ func (c *attrCmd) Examples() []string { } } -func (c *attrCmd) RunCommand(up *Uploader, args []string) error { +func (c *attrCmd) RunCommand(args []string) error { if len(args) != 3 { return errors.New("Attr takes 3 args: ") } @@ -75,7 +77,7 @@ func (c *attrCmd) RunCommand(up *Uploader, args []string) error { return errors.New("del not yet implemented") } } - put, err := up.UploadAndSignBlob(bb) + put, err := getUploader().UploadAndSignBlob(bb) handleResult(bb.Type(), put, err) return nil } diff --git a/cmd/camput/blobs.go b/cmd/camput/blobs.go index 0f34c067c..fe355e15a 100644 --- a/cmd/camput/blobs.go +++ b/cmd/camput/blobs.go @@ -27,18 +27,19 @@ import ( "camlistore.org/pkg/blobref" "camlistore.org/pkg/client" + "camlistore.org/pkg/cmdmain" ) type blobCmd struct{} func init() { - RegisterCommand("blob", func(flags *flag.FlagSet) CommandRunner { + cmdmain.RegisterCommand("blob", func(flags *flag.FlagSet) cmdmain.CommandRunner { return new(blobCmd) }) } func (c *blobCmd) Usage() { - fmt.Fprintf(stderr, "Usage: camput [globalopts] blob \n camput [globalopts] blob -\n") + fmt.Fprintf(cmdmain.Stderr, "Usage: camput [globalopts] blob \n camput [globalopts] blob -\n") } func (c *blobCmd) Examples() []string { @@ -48,11 +49,12 @@ func (c *blobCmd) Examples() []string { } } -func (c *blobCmd) RunCommand(up *Uploader, args []string) error { +func (c *blobCmd) RunCommand(args []string) error { if len(args) == 0 { return errors.New("No files given.") } + up := getUploader() for _, arg := range args { var ( handle *client.UploadHandle @@ -75,7 +77,7 @@ func (c *blobCmd) RunCommand(up *Uploader, args []string) error { func stdinBlobHandle() (uh *client.UploadHandle, err error) { var buf bytes.Buffer - size, err := io.Copy(&buf, stdin) + size, err := io.Copy(&buf, cmdmain.Stdin) if err != nil { return } diff --git a/cmd/camput/camput.go b/cmd/camput/camput.go index f6f0975ee..45c36698b 100644 --- a/cmd/camput/camput.go +++ b/cmd/camput/camput.go @@ -19,17 +19,15 @@ package main import ( "flag" "fmt" - "io" "log" "net/http" "net/url" "os" - "sort" "strconv" "strings" - "camlistore.org/pkg/buildinfo" "camlistore.org/pkg/client" + "camlistore.org/pkg/cmdmain" "camlistore.org/pkg/httputil" "camlistore.org/pkg/jsonsign" ) @@ -38,120 +36,38 @@ const buffered = 16 // arbitrary var ( flagProxyLocal = false - flagVersion = flag.Bool("version", false, "show version") - flagHelp = flag.Bool("help", false, "print usage") - flagVerbose = flag.Bool("verbose", false, "extra debug logging") flagHTTP = flag.Bool("verbose_http", false, "show HTTP request summaries") ) -var ErrUsage = UsageError("invalid command usage") - -type UsageError string - -func (ue UsageError) Error() string { - return "Usage error: " + string(ue) -} - -type CommandRunner interface { - Usage() - RunCommand(up *Uploader, args []string) error -} - -type Exampler interface { - Examples() []string -} - -var modeCommand = make(map[string]CommandRunner) -var modeFlags = make(map[string]*flag.FlagSet) +var cachedUploader *Uploader // initialized by getUploader func init() { if debug, _ := strconv.ParseBool(os.Getenv("CAMLI_DEBUG")); debug { flag.BoolVar(&flagProxyLocal, "proxy_local", false, "If true, the HTTP_PROXY environment is also used for localhost requests. This can be helpful during debugging.") } + cmdmain.ExtraFlagRegistration = func() { + jsonsign.AddFlags() + client.AddFlags() + } + cmdmain.PreExit = func() { + up := getUploader() + stats := up.Stats() + log.Printf("Client stats: %s", stats.String()) + log.Printf(" #HTTP reqs: %d", up.transport.Requests()) + } } -func RegisterCommand(mode string, makeCmd func(Flags *flag.FlagSet) CommandRunner) { - if _, dup := modeCommand[mode]; dup { - log.Fatalf("duplicate command %q registered", mode) +func getUploader() *Uploader { + if cachedUploader == nil { + cachedUploader = newUploader() } - flags := flag.NewFlagSet(mode+" options", flag.ContinueOnError) - flags.Usage = func() {} - modeFlags[mode] = flags - modeCommand[mode] = makeCmd(flags) + return cachedUploader } // wereErrors gets set to true if any error was encountered, which // changes the os.Exit value var wereErrors = false -type namedMode struct { - Name string - Command CommandRunner -} - -func allModes(startModes []string) <-chan namedMode { - ch := make(chan namedMode) - go func() { - defer close(ch) - done := map[string]bool{} - for _, name := range startModes { - done[name] = true - cmd := modeCommand[name] - if cmd == nil { - panic("bogus mode: " + name) - } - ch <- namedMode{name, cmd} - } - var rest []string - for name := range modeCommand { - if !done[name] { - rest = append(rest, name) - } - } - sort.Strings(rest) - for _, name := range rest { - ch <- namedMode{name, modeCommand[name]} - } - }() - return ch -} - -func errf(format string, args ...interface{}) { - fmt.Fprintf(stderr, format, args...) -} - -func usage(msg string) { - if msg != "" { - errf("Error: %v\n", msg) - } - errf(` -Usage: camput [globalopts] [commandopts] [commandargs] - -Examples: -`) - order := []string{"init", "file", "permanode", "blob", "attr"} - for mode := range allModes(order) { - errf("\n") - if ex, ok := mode.Command.(Exampler); ok { - for _, example := range ex.Examples() { - errf(" camput %s %s\n", mode.Name, example) - } - } else { - errf(" camput %s ...\n", mode.Name) - } - } - - errf(` -For mode-specific help: - - camput -help - -Global options: -`) - flag.PrintDefaults() - exit(1) -} - func handleResult(what string, pr *client.PutResult, err error) error { if err != nil { log.Printf("Error putting %s: %s", what, err) @@ -192,7 +108,7 @@ func proxyFromEnvironment(req *http.Request) (*url.URL, error) { func newUploader() *Uploader { cc := client.NewOrFail() - if !*flagVerbose { + if !*cmdmain.FlagVerbose { cc.SetLogger(nil) } @@ -233,90 +149,11 @@ func newUploader() *Uploader { } } -func hasFlags(flags *flag.FlagSet) bool { - any := false - flags.VisitAll(func(*flag.Flag) { - any = true - }) - return any -} - func main() { - jsonsign.AddFlags() - client.AddFlags() flag.Parse() - camputMain(flag.Args()...) -} - -func realExit(code int) { - os.Exit(code) -} - -// Indirections for replacement by tests: -var ( - stderr io.Writer = os.Stderr - stdout io.Writer = os.Stdout - stdin io.Reader = os.Stdin - - exit = realExit - - // TODO: abstract out vfs operation. should never call os.Stat, os.Open, os.Create, etc. - // Only use fs.Stat, fs.Open, where vs is an interface type. - - // TODO: switch from using the global flag FlagSet and use our own. right now - // running "go test -v" dumps the flag usage data to the global stderr. -) - -// camputMain is separated from main for testing from camput -func camputMain(args ...string) { - if *flagVersion { - fmt.Fprintf(stderr, "camget version: %s\n", buildinfo.Version()) - return - } - if *flagHelp { - usage("") - } - if len(args) == 0 { - usage("No mode given.") - } - - mode := args[0] - cmd, ok := modeCommand[mode] - if !ok { - usage(fmt.Sprintf("Unknown mode %q", mode)) - } - - var up *Uploader - if mode != "init" { - up = newUploader() - } - - cmdFlags := modeFlags[mode] - err := cmdFlags.Parse(args[1:]) - if err != nil { - err = ErrUsage - } else { - err = cmd.RunCommand(up, cmdFlags.Args()) - } - if ue, isUsage := err.(UsageError); isUsage { - if isUsage { - errf("%s\n", ue) - } - cmd.Usage() - errf("\nGlobal options:\n") - flag.PrintDefaults() - - if hasFlags(cmdFlags) { - errf("\nMode-specific options for mode %q:\n", mode) - cmdFlags.PrintDefaults() - } - exit(1) - } - if *flagVerbose { - stats := up.Stats() - log.Printf("Client stats: %s", stats.String()) - log.Printf(" #HTTP reqs: %d", up.transport.Requests()) - } + err := cmdmain.Main() + // TODO(mpl): see how errors go with other camtool modes + // and move some of this accordingly to cmdmain. previousErrors := wereErrors if err != nil { wereErrors = true @@ -325,6 +162,6 @@ func camputMain(args ...string) { } } if wereErrors { - exit(2) + cmdmain.Exit(2) } } diff --git a/cmd/camput/files.go b/cmd/camput/files.go index 192d6c182..9c46fc977 100644 --- a/cmd/camput/files.go +++ b/cmd/camput/files.go @@ -39,6 +39,7 @@ import ( "camlistore.org/pkg/blobref" "camlistore.org/pkg/blobserver" "camlistore.org/pkg/client" + "camlistore.org/pkg/cmdmain" "camlistore.org/pkg/schema" ) @@ -61,7 +62,7 @@ type fileCmd struct { } func init() { - RegisterCommand("file", func(flags *flag.FlagSet) CommandRunner { + cmdmain.RegisterCommand("file", func(flags *flag.FlagSet) cmdmain.CommandRunner { cmd := new(fileCmd) flags.BoolVar(&cmd.makePermanode, "permanode", false, "Create an associate a new permanode for the uploaded file or directory.") flags.BoolVar(&cmd.filePermanodes, "filenodes", false, "Create (if necessary) content-based permanodes for each uploaded file.") @@ -94,7 +95,7 @@ func init() { } func (c *fileCmd) Usage() { - fmt.Fprintf(stderr, "Usage: camput [globalopts] file [fileopts] \n") + fmt.Fprintf(cmdmain.Stderr, "Usage: camput [globalopts] file [fileopts] \n") } func (c *fileCmd) Examples() []string { @@ -105,21 +106,22 @@ func (c *fileCmd) Examples() []string { } } -func (c *fileCmd) RunCommand(up *Uploader, args []string) error { +func (c *fileCmd) RunCommand(args []string) error { if c.vivify { if c.makePermanode || c.filePermanodes || c.tag != "" || c.name != "" { - return UsageError("--vivify excludes any other option") + return cmdmain.UsageError("--vivify excludes any other option") } } if c.name != "" && !c.makePermanode { - return UsageError("Can't set name without using --permanode") + return cmdmain.UsageError("Can't set name without using --permanode") } if c.tag != "" && !c.makePermanode && !c.filePermanodes { - return UsageError("Can't set tag without using --permanode or --filenodes") + return cmdmain.UsageError("Can't set tag without using --permanode or --filenodes") } if c.histo != "" && !c.memstats { - return UsageError("Can't use histo without memstats") + return cmdmain.UsageError("Can't use histo without memstats") } + up := getUploader() if c.memstats { sr := new(statsStatReceiver) up.altStatReceiver = sr @@ -130,7 +132,7 @@ func (c *fileCmd) RunCommand(up *Uploader, args []string) error { if c.makePermanode || c.filePermanodes { testSigBlobRef := up.Client.SignerPublicKeyBlobref() if testSigBlobRef == nil { - return UsageError("A GPG key is needed to create permanodes; configure one or use vivify mode.") + return cmdmain.UsageError("A GPG key is needed to create permanodes; configure one or use vivify mode.") } } up.fileOpts = &fileOptions{ @@ -198,7 +200,7 @@ func (c *fileCmd) RunCommand(up *Uploader, args []string) error { } if len(args) == 0 { - return UsageError("No files or directories given.") + return cmdmain.UsageError("No files or directories given.") } for _, filename := range args { fi, err := os.Stat(filename) @@ -472,7 +474,7 @@ func (up *Uploader) fileMapFromDuplicate(bs blobserver.StatReceiver, fileMap *sc if dupFileRef == nil { return nil, false } - if *flagVerbose { + if *cmdmain.FlagVerbose { log.Printf("Found dup of contents %s in file schema %s", sum, dupFileRef) } dupMap, err := up.Client.FetchSchemaBlob(dupFileRef) diff --git a/cmd/camput/flatcache.go b/cmd/camput/flatcache.go index ee9eaa4d5..50a8eaf42 100644 --- a/cmd/camput/flatcache.go +++ b/cmd/camput/flatcache.go @@ -79,7 +79,7 @@ func escapeGen(gen string) string { } func NewFlatStatCache(gen string) *FlatStatCache { - filename := filepath.Join(osutil.CacheDir(), "camput.statcache." + escapeGen(gen)) + filename := filepath.Join(osutil.CacheDir(), "camput.statcache."+escapeGen(gen)) fc := &FlatStatCache{ filename: filename, m: make(map[string]fileInfoPutRes), diff --git a/cmd/camput/init.go b/cmd/camput/init.go index 039142813..79bf3abb0 100644 --- a/cmd/camput/init.go +++ b/cmd/camput/init.go @@ -29,6 +29,7 @@ import ( "camlistore.org/pkg/blobref" "camlistore.org/pkg/client" + "camlistore.org/pkg/cmdmain" "camlistore.org/pkg/jsonsign" "camlistore.org/pkg/osutil" ) @@ -38,7 +39,7 @@ type initCmd struct { } func init() { - RegisterCommand("init", func(flags *flag.FlagSet) CommandRunner { + cmdmain.RegisterCommand("init", func(flags *flag.FlagSet) cmdmain.CommandRunner { cmd := new(initCmd) flags.StringVar(&cmd.gpgkey, "gpgkey", "", "GPG key to use for signing (overrides $GPGKEY environment)") return cmd @@ -46,7 +47,7 @@ func init() { } func (c *initCmd) Usage() { - fmt.Fprintf(stderr, `Usage: camput init [opts] + fmt.Fprintf(cmdmain.Stderr, `Usage: camput init [opts] Initialize the camput configuration file. @@ -106,9 +107,9 @@ func (c *initCmd) getPublicKeyArmored(keyId string) (b []byte, err error) { return nil, fmt.Errorf("failed to export armored public key ID %q from locations: %q", keyId, files) } -func (c *initCmd) RunCommand(_ *Uploader, args []string) error { +func (c *initCmd) RunCommand(args []string) error { if len(args) > 0 { - return ErrUsage + return cmdmain.ErrUsage } blobDir := path.Join(osutil.CamliConfigDir(), "keyblobs") diff --git a/cmd/camput/logging.go b/cmd/camput/logging.go index a4456cc16..fa1947369 100644 --- a/cmd/camput/logging.go +++ b/cmd/camput/logging.go @@ -18,6 +18,8 @@ package main import ( "log" + + "camlistore.org/pkg/cmdmain" ) type Logger interface { @@ -30,7 +32,7 @@ type flagLogger struct { var flagCacheLog *bool -var vlog = &flagLogger{&flagVerbose} +var vlog = &flagLogger{&cmdmain.FlagVerbose} var cachelog = &flagLogger{&flagCacheLog} func (fl *flagLogger) Printf(format string, args ...interface{}) { diff --git a/cmd/camput/permanode.go b/cmd/camput/permanode.go index 089b50965..4c4a5a672 100644 --- a/cmd/camput/permanode.go +++ b/cmd/camput/permanode.go @@ -24,6 +24,7 @@ import ( "time" "camlistore.org/pkg/client" + "camlistore.org/pkg/cmdmain" "camlistore.org/pkg/schema" ) @@ -35,7 +36,7 @@ type permanodeCmd struct { } func init() { - RegisterCommand("permanode", func(flags *flag.FlagSet) CommandRunner { + cmdmain.RegisterCommand("permanode", func(flags *flag.FlagSet) cmdmain.CommandRunner { cmd := new(permanodeCmd) flags.StringVar(&cmd.name, "name", "", "Optional name attribute to set on new permanode") flags.StringVar(&cmd.tag, "tag", "", "Optional tag(s) to set on new permanode; comma separated.") @@ -46,7 +47,7 @@ func init() { } func (c *permanodeCmd) Usage() { - errf("Usage: camput [globalopts] permanode [permanodeopts]\n") + cmdmain.Errf("Usage: camput [globalopts] permanode [permanodeopts]\n") } func (c *permanodeCmd) Examples() []string { @@ -56,7 +57,7 @@ func (c *permanodeCmd) Examples() []string { } } -func (c *permanodeCmd) RunCommand(up *Uploader, args []string) error { +func (c *permanodeCmd) RunCommand(args []string) error { if len(args) > 0 { return errors.New("Permanode command doesn't take any additional arguments") } @@ -64,6 +65,7 @@ func (c *permanodeCmd) RunCommand(up *Uploader, args []string) error { var ( permaNode *client.PutResult err error + up = getUploader() ) if (c.key != "") != (c.sigTime != "") { return errors.New("Both --key and --sigtime must be used to produce deterministic permanodes.") diff --git a/cmd/camput/rawobj.go b/cmd/camput/rawobj.go index eafab9b8d..76cdd941a 100644 --- a/cmd/camput/rawobj.go +++ b/cmd/camput/rawobj.go @@ -21,6 +21,7 @@ import ( "flag" "strings" + "camlistore.org/pkg/cmdmain" "camlistore.org/pkg/schema" ) @@ -30,7 +31,7 @@ type rawCmd struct { } func init() { - RegisterCommand("rawobj", func(flags *flag.FlagSet) CommandRunner { + cmdmain.RegisterCommand("rawobj", func(flags *flag.FlagSet) cmdmain.CommandRunner { cmd := new(rawCmd) flags.StringVar(&cmd.vals, "vals", "", "Pipe-separated key=value properties") flags.BoolVar(&cmd.signed, "signed", true, "whether to sign the JSON object") @@ -39,14 +40,14 @@ func init() { } func (c *rawCmd) Usage() { - errf("Usage: camput [globalopts] rawobj [rawopts]\n") + cmdmain.Errf("Usage: camput [globalopts] rawobj [rawopts]\n") } func (c *rawCmd) Examples() []string { return []string{"(debug command)"} } -func (c *rawCmd) RunCommand(up *Uploader, args []string) error { +func (c *rawCmd) RunCommand(args []string) error { if len(args) > 0 { return errors.New("Raw Object command doesn't take any additional arguments") } @@ -61,6 +62,7 @@ func (c *rawCmd) RunCommand(up *Uploader, args []string) error { bb.SetRawStringField(kv[0], kv[1]) } + up := getUploader() if c.signed { put, err := up.UploadAndSignBlob(bb) handleResult("raw-object-signed", put, err) diff --git a/cmd/camput/remove.go b/cmd/camput/remove.go index 7c9ecf7df..8d037aa8a 100644 --- a/cmd/camput/remove.go +++ b/cmd/camput/remove.go @@ -21,27 +21,28 @@ import ( "fmt" "camlistore.org/pkg/blobref" + "camlistore.org/pkg/cmdmain" ) type removeCmd struct{} func init() { - RegisterCommand("remove", func(flags *flag.FlagSet) CommandRunner { + cmdmain.RegisterCommand("remove", func(flags *flag.FlagSet) cmdmain.CommandRunner { cmd := new(removeCmd) return cmd }) } func (c *removeCmd) Usage() { - fmt.Fprintf(stderr, `Usage: camput remove + fmt.Fprintf(cmdmain.Stderr, `Usage: camput remove This command is for debugging only. You're not expected to use it in practice. `) } -func (c *removeCmd) RunCommand(up *Uploader, args []string) error { +func (c *removeCmd) RunCommand(args []string) error { if len(args) == 0 { - return ErrUsage + return cmdmain.ErrUsage } - return up.RemoveBlobs(blobref.ParseMulti(args)) + return getUploader().RemoveBlobs(blobref.ParseMulti(args)) } diff --git a/cmd/camput/share.go b/cmd/camput/share.go index cbb8c5b4b..ae9e35395 100644 --- a/cmd/camput/share.go +++ b/cmd/camput/share.go @@ -22,6 +22,7 @@ import ( "camlistore.org/pkg/blobref" "camlistore.org/pkg/client" + "camlistore.org/pkg/cmdmain" "camlistore.org/pkg/schema" ) @@ -30,7 +31,7 @@ type shareCmd struct { } func init() { - RegisterCommand("share", func(flags *flag.FlagSet) CommandRunner { + cmdmain.RegisterCommand("share", func(flags *flag.FlagSet) cmdmain.CommandRunner { cmd := new(shareCmd) flags.BoolVar(&cmd.transitive, "transitive", false, "share everything reachable from the given blobref") return cmd @@ -38,7 +39,7 @@ func init() { } func (c *shareCmd) Usage() { - fmt.Fprintf(stderr, `Usage: camput share [opts] + fmt.Fprintf(cmdmain.Stderr, `Usage: camput share [opts] `) } @@ -48,15 +49,15 @@ func (c *shareCmd) Examples() []string { } } -func (c *shareCmd) RunCommand(up *Uploader, args []string) error { +func (c *shareCmd) RunCommand(args []string) error { if len(args) != 1 { - return UsageError("share takes exactly one argument, a blobref") + return cmdmain.UsageError("share takes exactly one argument, a blobref") } br := blobref.Parse(args[0]) if br == nil { - return UsageError("invalid blobref") + return cmdmain.UsageError("invalid blobref") } - pr, err := up.UploadShare(br, c.transitive) + pr, err := getUploader().UploadShare(br, c.transitive) handleResult("share", pr, err) return nil } diff --git a/cmd/camtool/camtool.go b/cmd/camtool/camtool.go new file mode 100644 index 000000000..b2a0fd600 --- /dev/null +++ b/cmd/camtool/camtool.go @@ -0,0 +1,28 @@ +/* +Copyright 2013 The Camlistore Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "camlistore.org/pkg/cmdmain" +) + +func main() { + err := cmdmain.Main() + if err != nil { + cmdmain.Exit(2) + } +} diff --git a/cmd/camsync/camsync.go b/cmd/camtool/sync.go similarity index 64% rename from cmd/camsync/camsync.go rename to cmd/camtool/sync.go index 66571c934..8a993443b 100644 --- a/cmd/camsync/camsync.go +++ b/cmd/camtool/sync.go @@ -1,5 +1,5 @@ /* -Copyright 2011 Google Inc. +Copyright 2013 The Camlistore Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -26,21 +26,96 @@ import ( "camlistore.org/pkg/blobref" "camlistore.org/pkg/client" + "camlistore.org/pkg/cmdmain" ) -var ( - flagLoop = flag.Bool("loop", false, "sync in a loop once done; requires --removesrc") - flagVerbose = flag.Bool("verbose", false, "be verbose") - flagAll = flag.Bool("all", false, "Discover all sync destinations configured on the source server and run them.") +type syncCmd struct { + src string + dest string - flagSrc = flag.String("src", "", "Source blobserver is either a URL prefix (with optional path), a host[:port], or blank to use the Camlistore client config's default host.") - flagDest = flag.String("dest", "", "Destination blobserver, or 'stdout' to just enumerate the --src blobs to stdout") + loop bool + verbose bool + all bool + removeSrc bool - flagRemoveSource = flag.Bool("removesrc", false, - "remove each blob from the source after syncing to the destination; for queue processing") -) + logger *log.Logger +} -var logger *log.Logger = nil +func init() { + cmdmain.RegisterCommand("sync", func(flags *flag.FlagSet) cmdmain.CommandRunner { + cmd := new(syncCmd) + flags.StringVar(&cmd.src, "src", "", "Source blobserver is either a URL prefix (with optional path), a host[:port], or blank to use the Camlistore client config's default host.") + flags.StringVar(&cmd.dest, "dest", "", "Destination blobserver, or 'stdout' to just enumerate the --src blobs to stdout.") + + flags.BoolVar(&cmd.loop, "loop", false, "Create an associate a new permanode for the uploaded file or directory.") + flags.BoolVar(&cmd.verbose, "verbose", false, "Be verbose.") + flags.BoolVar(&cmd.all, "all", false, "Discover all sync destinations configured on the source server and run them.") + flags.BoolVar(&cmd.removeSrc, "removesrc", false, "Remove each blob from the source after syncing to the destination; for queue processing.") + + return cmd + }) +} + +func (c *syncCmd) Usage() { + fmt.Fprintf(os.Stderr, "Usage: camtool [globalopts] sync [syncopts] \n") +} + +func (c *syncCmd) Examples() []string { + return []string{ + "--all", + "--src http://localhost:3179/bs/ --dest http://localhost:3179/index-mem/", + } +} + +func (c *syncCmd) RunCommand(args []string) error { + if c.loop && !c.removeSrc { + return cmdmain.UsageError("Can't use --loop without --removesrc") + } + if c.verbose { + c.logger = log.New(os.Stderr, "", 0) // else nil + } + if c.all { + err := c.syncAll() + if err != nil { + return fmt.Errorf("sync all failed: %v", err) + } + return nil + } + if c.dest == "" { + return cmdmain.UsageError("No --dest specified.") + } + + discl := c.discoClient() + discl.SetLogger(c.logger) + src, err := discl.BlobRoot() + if err != nil { + return fmt.Errorf("Failed to get blob source: %v", err) + } + + sc := client.New(src) + sc.SetupAuth() + dc := client.New(c.dest) + dc.SetupAuth() + + sc.SetLogger(c.logger) + dc.SetLogger(c.logger) + + passNum := 0 + for { + passNum++ + stats, err := c.doPass(sc, dc) + if c.verbose { + log.Printf("sync stats - pass: %d, blobs: %d, bytes %d\n", passNum, stats.BlobsCopied, stats.BytesCopied) + } + if err != nil { + return fmt.Errorf("sync failed: %v", err) + } + if !c.loop { + break + } + } + return nil +} type SyncStats struct { BlobsCopied int @@ -48,26 +123,18 @@ type SyncStats struct { ErrorCount int } -func usage(err string) { - if err != "" { - fmt.Fprintf(os.Stderr, "Error: %s\n\nUsage:\n", err) - } - flag.PrintDefaults() - os.Exit(2) -} - -func syncAll() error { - if *flagLoop { - usage("--all can not be used with --loop") +func (c *syncCmd) syncAll() error { + if c.loop { + return cmdmain.UsageError("--all can not be used with --loop") } - dc := discoClient() - dc.SetLogger(logger) + dc := c.discoClient() + dc.SetLogger(c.logger) syncHandlers, err := dc.SyncHandlers() if err != nil { - log.Fatalf("sync handlers discovery failed: %v", err) + return fmt.Errorf("sync handlers discovery failed: %v", err) } - if *flagVerbose { + if c.verbose { log.Printf("To be synced:\n") for _, sh := range syncHandlers { log.Printf("%v -> %v", sh.From, sh.To) @@ -75,16 +142,16 @@ func syncAll() error { } for _, sh := range syncHandlers { from := client.New(sh.From) - from.SetLogger(logger) + from.SetLogger(c.logger) from.SetupAuth() to := client.New(sh.To) - to.SetLogger(logger) + to.SetLogger(c.logger) to.SetupAuth() - if *flagVerbose { + if c.verbose { log.Printf("Now syncing: %v -> %v", sh.From, sh.To) } - stats, err := doPass(from, to) - if *flagVerbose { + stats, err := c.doPass(from, to) + if c.verbose { log.Printf("sync stats, blobs: %d, bytes %d\n", stats.BlobsCopied, stats.BytesCopied) } if err != nil { @@ -98,69 +165,18 @@ func syncAll() error { // based from --src or from the configuration file if --src // is blank. The returned client can then be used to discover // the blobRoot and syncHandlers. -func discoClient() *client.Client { +func (c *syncCmd) discoClient() *client.Client { var cl *client.Client - if *flagSrc == "" { + if c.src == "" { cl = client.NewOrFail() } else { - cl = client.New(*flagSrc) + cl = client.New(c.src) } cl.SetupAuth() return cl } -func main() { - flag.Parse() - - if *flagLoop && !*flagRemoveSource { - usage("Can't use --loop without --removesrc") - } - if *flagVerbose { - logger = log.New(os.Stderr, "", 0) - } - if *flagAll { - err := syncAll() - if err != nil { - log.Fatalf("sync all failed: %v", err) - } - return - } - if *flagDest == "" { - usage("No --dest specified.") - } - - discl := discoClient() - discl.SetLogger(logger) - src, err := discl.BlobRoot() - if err != nil { - log.Fatalf("Failed to get blob source: %v", err) - } - - sc := client.New(src) - sc.SetupAuth() - dc := client.New(*flagDest) - dc.SetupAuth() - - sc.SetLogger(logger) - dc.SetLogger(logger) - - passNum := 0 - for { - passNum++ - stats, err := doPass(sc, dc) - if *flagVerbose { - log.Printf("sync stats - pass: %d, blobs: %d, bytes %d\n", passNum, stats.BlobsCopied, stats.BytesCopied) - } - if err != nil { - log.Fatalf("sync failed: %v", err) - } - if !*flagLoop { - break - } - } -} - -func doPass(sc, dc *client.Client) (stats SyncStats, retErr error) { +func (c *syncCmd) doPass(sc, dc *client.Client) (stats SyncStats, retErr error) { srcBlobs := make(chan blobref.SizedBlobRef, 100) destBlobs := make(chan blobref.SizedBlobRef, 100) srcErr := make(chan error) @@ -175,7 +191,7 @@ func doPass(sc, dc *client.Client) (stats SyncStats, retErr error) { } } - if *flagDest == "stdout" { + if c.dest == "stdout" { for sb := range srcBlobs { fmt.Printf("%s %d\n", sb.BlobRef, sb.Size) } @@ -195,7 +211,7 @@ func doPass(sc, dc *client.Client) (stats SyncStats, retErr error) { destNotHaveBlobs := make(chan blobref.SizedBlobRef) sizeMismatch := make(chan *blobref.BlobRef) readSrcBlobs := srcBlobs - if *flagVerbose { + if c.verbose { readSrcBlobs = loggingBlobRefChannel(srcBlobs) } mismatches := []*blobref.BlobRef{} @@ -237,7 +253,7 @@ For: stats.BlobsCopied++ stats.BytesCopied += pr.Size } - if *flagRemoveSource { + if c.removeSrc { if err = sc.RemoveBlob(sb.BlobRef); err != nil { stats.ErrorCount++ log.Printf("Failed to delete %s from source: %v", sb.BlobRef, err) @@ -249,7 +265,7 @@ For: checkSourceError() checkDestError() if retErr == nil && stats.ErrorCount > 0 { - retErr = errors.New(fmt.Sprintf("%d errors during sync", stats.ErrorCount)) + retErr = fmt.Errorf("%d errors during sync", stats.ErrorCount) } return stats, retErr } diff --git a/pkg/cmdmain/cmdmain.go b/pkg/cmdmain/cmdmain.go new file mode 100644 index 000000000..d72739e9a --- /dev/null +++ b/pkg/cmdmain/cmdmain.go @@ -0,0 +1,224 @@ +/* +Copyright 2013 The Camlistore Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cmdmain + +import ( + "flag" + "fmt" + "io" + "log" + "os" + "sort" + + "camlistore.org/pkg/buildinfo" +) + +var ( + FlagVersion = flag.Bool("version", false, "show version") + FlagHelp = flag.Bool("help", false, "print usage") + FlagVerbose = flag.Bool("verbose", false, "extra debug logging") +) + +var ( + // ExtraFlagRegistration allows to add more flags from + // other packages (with AddFlags) when Main starts. + ExtraFlagRegistration = func() {} + // PreExit is meant to dump additional stats and other + // verbiage before Main terminates. + PreExit = func() {} +) + +var ErrUsage = UsageError("invalid command") + +type UsageError string + +func (ue UsageError) Error() string { + return "Usage error: " + string(ue) +} + +var ( + // mode name to actual subcommand mapping + modeCommand = make(map[string]CommandRunner) + modeFlags = make(map[string]*flag.FlagSet) + + // Indirections for replacement by tests + Stderr io.Writer = os.Stderr + Stdout io.Writer = os.Stdout + Stdin io.Reader = os.Stdin + + Exit = realExit + // TODO: abstract out vfs operation. should never call os.Stat, os.Open, os.Create, etc. + // Only use fs.Stat, fs.Open, where vs is an interface type. + // TODO: switch from using the global flag FlagSet and use our own. right now + // running "go test -v" dumps the flag usage data to the global stderr. +) + +func realExit(code int) { + os.Exit(code) +} + +// CommandRunner is the type that a command mode should implement. +type CommandRunner interface { + Usage() + RunCommand(args []string) error +} + +type exampler interface { + Examples() []string +} + +// RegisterCommand adds a mode to the list of modes for the main command. +// It is meant to be called in init() for each subcommand. +func RegisterCommand(mode string, makeCmd func(Flags *flag.FlagSet) CommandRunner) { + if _, dup := modeCommand[mode]; dup { + log.Fatalf("duplicate command %q registered", mode) + } + flags := flag.NewFlagSet(mode+" options", flag.ContinueOnError) + flags.Usage = func() {} + modeFlags[mode] = flags + modeCommand[mode] = makeCmd(flags) +} + +type namedMode struct { + Name string + Command CommandRunner +} + +// TODO(mpl): do we actually need this? I changed usage +// to simply iterate over all of modeCommand and it seems +// fine. +func allModes(startModes []string) <-chan namedMode { + ch := make(chan namedMode) + go func() { + defer close(ch) + done := map[string]bool{} + for _, name := range startModes { + done[name] = true + cmd := modeCommand[name] + if cmd == nil { + panic("bogus mode: " + name) + } + ch <- namedMode{name, cmd} + } + var rest []string + for name := range modeCommand { + if !done[name] { + rest = append(rest, name) + } + } + sort.Strings(rest) + for _, name := range rest { + ch <- namedMode{name, modeCommand[name]} + } + }() + return ch +} + +func hasFlags(flags *flag.FlagSet) bool { + any := false + flags.VisitAll(func(*flag.Flag) { + any = true + }) + return any +} + +// Errf prints to Stderr +func Errf(format string, args ...interface{}) { + fmt.Fprintf(Stderr, format, args...) +} + +func usage(msg string) { + cmdName := os.Args[0] + if msg != "" { + Errf("Error: %v\n", msg) + } + Errf(` +Usage: ` + cmdName + ` [globalopts] [commandopts] [commandargs] + +Examples: +`) + for mode, cmd := range modeCommand { + Errf("\n") + if ex, ok := cmd.(exampler); ok { + for _, example := range ex.Examples() { + Errf(" %s %s %s\n", cmdName, mode, example) + } + } else { + Errf(" %s %s ...\n", cmdName, mode) + } + } + + Errf(` +For mode-specific help: + + ` + cmdName + ` -help + +Global options: +`) + flag.PrintDefaults() + Exit(1) +} + +// Main is meant to be the core of a command that has +// subcommands (modes), such as camput or camtool. +func Main() error { + ExtraFlagRegistration() + flag.Parse() + args := flag.Args() + if *FlagVersion { + fmt.Fprintf(Stderr, "camput version: %s\n", buildinfo.Version()) + return nil + } + if *FlagHelp { + usage("") + } + if len(args) == 0 { + usage("No mode given.") + } + + mode := args[0] + cmd, ok := modeCommand[mode] + if !ok { + usage(fmt.Sprintf("Unknown mode %q", mode)) + } + + cmdFlags := modeFlags[mode] + err := cmdFlags.Parse(args[1:]) + if err != nil { + err = ErrUsage + } else { + err = cmd.RunCommand(cmdFlags.Args()) + } + if ue, isUsage := err.(UsageError); isUsage { + if isUsage { + Errf("%s\n", ue) + } + cmd.Usage() + Errf("\nGlobal options:\n") + flag.PrintDefaults() + + if hasFlags(cmdFlags) { + Errf("\nMode-specific options for mode %q:\n", mode) + cmdFlags.PrintDefaults() + } + Exit(1) + } + if *FlagVerbose { + PreExit() + } + return err +}