Rooba
/
perkeep
mirror of https://github.com/perkeep/perkeep.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge "Create camtool cmd, with sync subcommand."

This commit is contained in:
mpl 2013-02-20 22:47:33 +00:00 committed by Gerrit Code Review
parent d3aa327ac6 59d0d6cb7e
commit 317c87c189
14 changed files with 435 additions and 315 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
cmd/camput/attr.go
View File

@ -22,16 +22,18 @@ import (
"fmt"
"camlistore.org/pkg/blobref"
"camlistore.org/pkg/cmdmain"
"camlistore.org/pkg/schema"
)
type attrCmd struct {
add bool
del bool
up *Uploader
}
func init() {
RegisterCommand("attr", func(flags *flag.FlagSet) CommandRunner {
cmdmain.RegisterCommand("attr", func(flags *flag.FlagSet) cmdmain.CommandRunner {
cmd := new(attrCmd)
flags.BoolVar(&cmd.add, "add", false, `Adds attribute (e.g. "tag")`)
flags.BoolVar(&cmd.del, "del", false, "Deletes named attribute [value]")
@ -40,7 +42,7 @@ func init() {
}
func (c *attrCmd) Usage() {
errf("Usage: camput [globalopts] attr [attroption] <permanode> <name> <value>")
cmdmain.Errf("Usage: camput [globalopts] attr [attroption] <permanode> <name> <value>")
}
func (c *attrCmd) Examples() []string {
@ -51,7 +53,7 @@ func (c *attrCmd) Examples() []string {
}
}
func (c *attrCmd) RunCommand(up *Uploader, args []string) error {
func (c *attrCmd) RunCommand(args []string) error {
if len(args) != 3 {
return errors.New("Attr takes 3 args: <permanode> <attr> <value>")
}
@ -75,7 +77,7 @@ func (c *attrCmd) RunCommand(up *Uploader, args []string) error {
return errors.New("del not yet implemented")
}
}
put, err := up.UploadAndSignBlob(bb)
put, err := getUploader().UploadAndSignBlob(bb)
handleResult(bb.Type(), put, err)
return nil
}

10
cmd/camput/blobs.go
View File

@ -27,18 +27,19 @@ import (
"camlistore.org/pkg/blobref"
"camlistore.org/pkg/client"
"camlistore.org/pkg/cmdmain"
)
type blobCmd struct{}
func init() {
RegisterCommand("blob", func(flags *flag.FlagSet) CommandRunner {
cmdmain.RegisterCommand("blob", func(flags *flag.FlagSet) cmdmain.CommandRunner {
return new(blobCmd)
})
}
func (c *blobCmd) Usage() {
fmt.Fprintf(stderr, "Usage: camput [globalopts] blob <files>\n camput [globalopts] blob -\n")
fmt.Fprintf(cmdmain.Stderr, "Usage: camput [globalopts] blob <files>\n camput [globalopts] blob -\n")
}
func (c *blobCmd) Examples() []string {
@ -48,11 +49,12 @@ func (c *blobCmd) Examples() []string {
}
}
func (c *blobCmd) RunCommand(up *Uploader, args []string) error {
func (c *blobCmd) RunCommand(args []string) error {
if len(args) == 0 {
return errors.New("No files given.")
}
up := getUploader()
for _, arg := range args {
var (
handle *client.UploadHandle
@ -75,7 +77,7 @@ func (c *blobCmd) RunCommand(up *Uploader, args []string) error {
func stdinBlobHandle() (uh *client.UploadHandle, err error) {
var buf bytes.Buffer
size, err := io.Copy(&buf, stdin)
size, err := io.Copy(&buf, cmdmain.Stdin)
if err != nil {
return
}

205
cmd/camput/camput.go
View File

@ -19,17 +19,15 @@ package main
import (
"flag"
"fmt"
"io"
"log"
"net/http"
"net/url"
"os"
"sort"
"strconv"
"strings"
"camlistore.org/pkg/buildinfo"
"camlistore.org/pkg/client"
"camlistore.org/pkg/cmdmain"
"camlistore.org/pkg/httputil"
"camlistore.org/pkg/jsonsign"
)
@ -38,120 +36,38 @@ const buffered = 16 // arbitrary
var (
flagProxyLocal = false
flagVersion = flag.Bool("version", false, "show version")
flagHelp = flag.Bool("help", false, "print usage")
flagVerbose = flag.Bool("verbose", false, "extra debug logging")
flagHTTP = flag.Bool("verbose_http", false, "show HTTP request summaries")
)
var ErrUsage = UsageError("invalid command usage")
type UsageError string
func (ue UsageError) Error() string {
return "Usage error: " + string(ue)
}
type CommandRunner interface {
Usage()
RunCommand(up *Uploader, args []string) error
}
type Exampler interface {
Examples() []string
}
var modeCommand = make(map[string]CommandRunner)
var modeFlags = make(map[string]*flag.FlagSet)
var cachedUploader *Uploader // initialized by getUploader
func init() {
if debug, _ := strconv.ParseBool(os.Getenv("CAMLI_DEBUG")); debug {
flag.BoolVar(&flagProxyLocal, "proxy_local", false, "If true, the HTTP_PROXY environment is also used for localhost requests. This can be helpful during debugging.")
}
cmdmain.ExtraFlagRegistration = func() {
jsonsign.AddFlags()
client.AddFlags()
}
cmdmain.PreExit = func() {
up := getUploader()
stats := up.Stats()
log.Printf("Client stats: %s", stats.String())
log.Printf(" #HTTP reqs: %d", up.transport.Requests())
}
}
func RegisterCommand(mode string, makeCmd func(Flags *flag.FlagSet) CommandRunner) {
if _, dup := modeCommand[mode]; dup {
log.Fatalf("duplicate command %q registered", mode)
func getUploader() *Uploader {
if cachedUploader == nil {
cachedUploader = newUploader()
}
flags := flag.NewFlagSet(mode+" options", flag.ContinueOnError)
flags.Usage = func() {}
modeFlags[mode] = flags
modeCommand[mode] = makeCmd(flags)
return cachedUploader
}
// wereErrors gets set to true if any error was encountered, which
// changes the os.Exit value
var wereErrors = false
type namedMode struct {
Name string
Command CommandRunner
}
func allModes(startModes []string) <-chan namedMode {
ch := make(chan namedMode)
go func() {
defer close(ch)
done := map[string]bool{}
for _, name := range startModes {
done[name] = true
cmd := modeCommand[name]
if cmd == nil {
panic("bogus mode: " + name)
}
ch <- namedMode{name, cmd}
}
var rest []string
for name := range modeCommand {
if !done[name] {
rest = append(rest, name)
}
}
sort.Strings(rest)
for _, name := range rest {
ch <- namedMode{name, modeCommand[name]}
}
}()
return ch
}
func errf(format string, args ...interface{}) {
fmt.Fprintf(stderr, format, args...)
}
func usage(msg string) {
if msg != "" {
errf("Error: %v\n", msg)
}
errf(`
Usage: camput [globalopts] <mode> [commandopts] [commandargs]
Examples:
`)
order := []string{"init", "file", "permanode", "blob", "attr"}
for mode := range allModes(order) {
errf("\n")
if ex, ok := mode.Command.(Exampler); ok {
for _, example := range ex.Examples() {
errf(" camput %s %s\n", mode.Name, example)
}
} else {
errf(" camput %s ...\n", mode.Name)
}
}
errf(`
For mode-specific help:
camput <mode> -help
Global options:
`)
flag.PrintDefaults()
exit(1)
}
func handleResult(what string, pr *client.PutResult, err error) error {
if err != nil {
log.Printf("Error putting %s: %s", what, err)
@ -192,7 +108,7 @@ func proxyFromEnvironment(req *http.Request) (*url.URL, error) {
func newUploader() *Uploader {
cc := client.NewOrFail()
if !*flagVerbose {
if !*cmdmain.FlagVerbose {
cc.SetLogger(nil)
}
@ -233,90 +149,11 @@ func newUploader() *Uploader {
}
}
func hasFlags(flags *flag.FlagSet) bool {
any := false
flags.VisitAll(func(*flag.Flag) {
any = true
})
return any
}
func main() {
jsonsign.AddFlags()
client.AddFlags()
flag.Parse()
camputMain(flag.Args()...)
}
func realExit(code int) {
os.Exit(code)
}
// Indirections for replacement by tests:
var (
stderr io.Writer = os.Stderr
stdout io.Writer = os.Stdout
stdin io.Reader = os.Stdin
exit = realExit
// TODO: abstract out vfs operation. should never call os.Stat, os.Open, os.Create, etc.
// Only use fs.Stat, fs.Open, where vs is an interface type.
// TODO: switch from using the global flag FlagSet and use our own. right now
// running "go test -v" dumps the flag usage data to the global stderr.
)
// camputMain is separated from main for testing from camput
func camputMain(args ...string) {
if *flagVersion {
fmt.Fprintf(stderr, "camget version: %s\n", buildinfo.Version())
return
}
if *flagHelp {
usage("")
}
if len(args) == 0 {
usage("No mode given.")
}
mode := args[0]
cmd, ok := modeCommand[mode]
if !ok {
usage(fmt.Sprintf("Unknown mode %q", mode))
}
var up *Uploader
if mode != "init" {
up = newUploader()
}
cmdFlags := modeFlags[mode]
err := cmdFlags.Parse(args[1:])
if err != nil {
err = ErrUsage
} else {
err = cmd.RunCommand(up, cmdFlags.Args())
}
if ue, isUsage := err.(UsageError); isUsage {
if isUsage {
errf("%s\n", ue)
}
cmd.Usage()
errf("\nGlobal options:\n")
flag.PrintDefaults()
if hasFlags(cmdFlags) {
errf("\nMode-specific options for mode %q:\n", mode)
cmdFlags.PrintDefaults()
}
exit(1)
}
if *flagVerbose {
stats := up.Stats()
log.Printf("Client stats: %s", stats.String())
log.Printf(" #HTTP reqs: %d", up.transport.Requests())
}
err := cmdmain.Main()
// TODO(mpl): see how errors go with other camtool modes
// and move some of this accordingly to cmdmain.
previousErrors := wereErrors
if err != nil {
wereErrors = true
@ -325,6 +162,6 @@ func camputMain(args ...string) {
}
}
if wereErrors {
exit(2)
cmdmain.Exit(2)
}
}

22
cmd/camput/files.go
View File

@ -39,6 +39,7 @@ import (
"camlistore.org/pkg/blobref"
"camlistore.org/pkg/blobserver"
"camlistore.org/pkg/client"
"camlistore.org/pkg/cmdmain"
"camlistore.org/pkg/schema"
)
@ -61,7 +62,7 @@ type fileCmd struct {
}
func init() {
RegisterCommand("file", func(flags *flag.FlagSet) CommandRunner {
cmdmain.RegisterCommand("file", func(flags *flag.FlagSet) cmdmain.CommandRunner {
cmd := new(fileCmd)
flags.BoolVar(&cmd.makePermanode, "permanode", false, "Create an associate a new permanode for the uploaded file or directory.")
flags.BoolVar(&cmd.filePermanodes, "filenodes", false, "Create (if necessary) content-based permanodes for each uploaded file.")
@ -94,7 +95,7 @@ func init() {
}
func (c *fileCmd) Usage() {
fmt.Fprintf(stderr, "Usage: camput [globalopts] file [fileopts] <file/director(ies)>\n")
fmt.Fprintf(cmdmain.Stderr, "Usage: camput [globalopts] file [fileopts] <file/director(ies)>\n")
}
func (c *fileCmd) Examples() []string {
@ -105,21 +106,22 @@ func (c *fileCmd) Examples() []string {
}
}
func (c *fileCmd) RunCommand(up *Uploader, args []string) error {
func (c *fileCmd) RunCommand(args []string) error {
if c.vivify {
if c.makePermanode || c.filePermanodes || c.tag != "" || c.name != "" {
return UsageError("--vivify excludes any other option")
return cmdmain.UsageError("--vivify excludes any other option")
}
}
if c.name != "" && !c.makePermanode {
return UsageError("Can't set name without using --permanode")
return cmdmain.UsageError("Can't set name without using --permanode")
}
if c.tag != "" && !c.makePermanode && !c.filePermanodes {
return UsageError("Can't set tag without using --permanode or --filenodes")
return cmdmain.UsageError("Can't set tag without using --permanode or --filenodes")
}
if c.histo != "" && !c.memstats {
return UsageError("Can't use histo without memstats")
return cmdmain.UsageError("Can't use histo without memstats")
}
up := getUploader()
if c.memstats {
sr := new(statsStatReceiver)
up.altStatReceiver = sr
@ -130,7 +132,7 @@ func (c *fileCmd) RunCommand(up *Uploader, args []string) error {
if c.makePermanode || c.filePermanodes {
testSigBlobRef := up.Client.SignerPublicKeyBlobref()
if testSigBlobRef == nil {
return UsageError("A GPG key is needed to create permanodes; configure one or use vivify mode.")
return cmdmain.UsageError("A GPG key is needed to create permanodes; configure one or use vivify mode.")
}
}
up.fileOpts = &fileOptions{
@ -198,7 +200,7 @@ func (c *fileCmd) RunCommand(up *Uploader, args []string) error {
}
if len(args) == 0 {
return UsageError("No files or directories given.")
return cmdmain.UsageError("No files or directories given.")
}
for _, filename := range args {
fi, err := os.Stat(filename)
@ -472,7 +474,7 @@ func (up *Uploader) fileMapFromDuplicate(bs blobserver.StatReceiver, fileMap *sc
if dupFileRef == nil {
return nil, false
}
if *flagVerbose {
if *cmdmain.FlagVerbose {
log.Printf("Found dup of contents %s in file schema %s", sum, dupFileRef)
}
dupMap, err := up.Client.FetchSchemaBlob(dupFileRef)

2
cmd/camput/flatcache.go
View File

@ -79,7 +79,7 @@ func escapeGen(gen string) string {
}
func NewFlatStatCache(gen string) *FlatStatCache {
filename := filepath.Join(osutil.CacheDir(), "camput.statcache." + escapeGen(gen))
filename := filepath.Join(osutil.CacheDir(), "camput.statcache."+escapeGen(gen))
fc := &FlatStatCache{
filename: filename,
m: make(map[string]fileInfoPutRes),

9
cmd/camput/init.go
View File

@ -29,6 +29,7 @@ import (
"camlistore.org/pkg/blobref"
"camlistore.org/pkg/client"
"camlistore.org/pkg/cmdmain"
"camlistore.org/pkg/jsonsign"
"camlistore.org/pkg/osutil"
)
@ -38,7 +39,7 @@ type initCmd struct {
}
func init() {
RegisterCommand("init", func(flags *flag.FlagSet) CommandRunner {
cmdmain.RegisterCommand("init", func(flags *flag.FlagSet) cmdmain.CommandRunner {
cmd := new(initCmd)
flags.StringVar(&cmd.gpgkey, "gpgkey", "", "GPG key to use for signing (overrides $GPGKEY environment)")
return cmd
@ -46,7 +47,7 @@ func init() {
}
func (c *initCmd) Usage() {
fmt.Fprintf(stderr, `Usage: camput init [opts]
fmt.Fprintf(cmdmain.Stderr, `Usage: camput init [opts]
Initialize the camput configuration file.
@ -106,9 +107,9 @@ func (c *initCmd) getPublicKeyArmored(keyId string) (b []byte, err error) {
return nil, fmt.Errorf("failed to export armored public key ID %q from locations: %q", keyId, files)
}
func (c *initCmd) RunCommand(_ *Uploader, args []string) error {
func (c *initCmd) RunCommand(args []string) error {
if len(args) > 0 {
return ErrUsage
return cmdmain.ErrUsage
}
blobDir := path.Join(osutil.CamliConfigDir(), "keyblobs")

4
cmd/camput/logging.go
View File

@ -18,6 +18,8 @@ package main
import (
"log"
"camlistore.org/pkg/cmdmain"
)
type Logger interface {
@ -30,7 +32,7 @@ type flagLogger struct {
var flagCacheLog *bool
var vlog = &flagLogger{&flagVerbose}
var vlog = &flagLogger{&cmdmain.FlagVerbose}
var cachelog = &flagLogger{&flagCacheLog}
func (fl *flagLogger) Printf(format string, args ...interface{}) {

8
cmd/camput/permanode.go
View File

@ -24,6 +24,7 @@ import (
"time"
"camlistore.org/pkg/client"
"camlistore.org/pkg/cmdmain"
"camlistore.org/pkg/schema"
)
@ -35,7 +36,7 @@ type permanodeCmd struct {
}
func init() {
RegisterCommand("permanode", func(flags *flag.FlagSet) CommandRunner {
cmdmain.RegisterCommand("permanode", func(flags *flag.FlagSet) cmdmain.CommandRunner {
cmd := new(permanodeCmd)
flags.StringVar(&cmd.name, "name", "", "Optional name attribute to set on new permanode")
flags.StringVar(&cmd.tag, "tag", "", "Optional tag(s) to set on new permanode; comma separated.")
@ -46,7 +47,7 @@ func init() {
}
func (c *permanodeCmd) Usage() {
errf("Usage: camput [globalopts] permanode [permanodeopts]\n")
cmdmain.Errf("Usage: camput [globalopts] permanode [permanodeopts]\n")
}
func (c *permanodeCmd) Examples() []string {
@ -56,7 +57,7 @@ func (c *permanodeCmd) Examples() []string {
}
}
func (c *permanodeCmd) RunCommand(up *Uploader, args []string) error {
func (c *permanodeCmd) RunCommand(args []string) error {
if len(args) > 0 {
return errors.New("Permanode command doesn't take any additional arguments")
}
@ -64,6 +65,7 @@ func (c *permanodeCmd) RunCommand(up *Uploader, args []string) error {
var (
permaNode *client.PutResult
err error
up = getUploader()
)
if (c.key != "") != (c.sigTime != "") {
return errors.New("Both --key and --sigtime must be used to produce deterministic permanodes.")

8
cmd/camput/rawobj.go
View File

@ -21,6 +21,7 @@ import (
"flag"
"strings"
"camlistore.org/pkg/cmdmain"
"camlistore.org/pkg/schema"
)
@ -30,7 +31,7 @@ type rawCmd struct {
}
func init() {
RegisterCommand("rawobj", func(flags *flag.FlagSet) CommandRunner {
cmdmain.RegisterCommand("rawobj", func(flags *flag.FlagSet) cmdmain.CommandRunner {
cmd := new(rawCmd)
flags.StringVar(&cmd.vals, "vals", "", "Pipe-separated key=value properties")
flags.BoolVar(&cmd.signed, "signed", true, "whether to sign the JSON object")
@ -39,14 +40,14 @@ func init() {
}
func (c *rawCmd) Usage() {
errf("Usage: camput [globalopts] rawobj [rawopts]\n")
cmdmain.Errf("Usage: camput [globalopts] rawobj [rawopts]\n")
}
func (c *rawCmd) Examples() []string {
return []string{"(debug command)"}
}
func (c *rawCmd) RunCommand(up *Uploader, args []string) error {
func (c *rawCmd) RunCommand(args []string) error {
if len(args) > 0 {
return errors.New("Raw Object command doesn't take any additional arguments")
}
@ -61,6 +62,7 @@ func (c *rawCmd) RunCommand(up *Uploader, args []string) error {
bb.SetRawStringField(kv[0], kv[1])
}
up := getUploader()
if c.signed {
put, err := up.UploadAndSignBlob(bb)
handleResult("raw-object-signed", put, err)

11
cmd/camput/remove.go
View File

@ -21,27 +21,28 @@ import (
"fmt"
"camlistore.org/pkg/blobref"
"camlistore.org/pkg/cmdmain"
)
type removeCmd struct{}
func init() {
RegisterCommand("remove", func(flags *flag.FlagSet) CommandRunner {
cmdmain.RegisterCommand("remove", func(flags *flag.FlagSet) cmdmain.CommandRunner {
cmd := new(removeCmd)
return cmd
})
}
func (c *removeCmd) Usage() {
fmt.Fprintf(stderr, `Usage: camput remove <blobref(s)>
fmt.Fprintf(cmdmain.Stderr, `Usage: camput remove <blobref(s)>
This command is for debugging only. You're not expected to use it in practice.
`)
}
func (c *removeCmd) RunCommand(up *Uploader, args []string) error {
func (c *removeCmd) RunCommand(args []string) error {
if len(args) == 0 {
return ErrUsage
return cmdmain.ErrUsage
}
return up.RemoveBlobs(blobref.ParseMulti(args))
return getUploader().RemoveBlobs(blobref.ParseMulti(args))
}

13
cmd/camput/share.go
View File

@ -22,6 +22,7 @@ import (
"camlistore.org/pkg/blobref"
"camlistore.org/pkg/client"
"camlistore.org/pkg/cmdmain"
"camlistore.org/pkg/schema"
)
@ -30,7 +31,7 @@ type shareCmd struct {
}
func init() {
RegisterCommand("share", func(flags *flag.FlagSet) CommandRunner {
cmdmain.RegisterCommand("share", func(flags *flag.FlagSet) cmdmain.CommandRunner {
cmd := new(shareCmd)
flags.BoolVar(&cmd.transitive, "transitive", false, "share everything reachable from the given blobref")
return cmd
@ -38,7 +39,7 @@ func init() {
}
func (c *shareCmd) Usage() {
fmt.Fprintf(stderr, `Usage: camput share [opts] <blobref>
fmt.Fprintf(cmdmain.Stderr, `Usage: camput share [opts] <blobref>
`)
}
@ -48,15 +49,15 @@ func (c *shareCmd) Examples() []string {
}
}
func (c *shareCmd) RunCommand(up *Uploader, args []string) error {
func (c *shareCmd) RunCommand(args []string) error {
if len(args) != 1 {
return UsageError("share takes exactly one argument, a blobref")
return cmdmain.UsageError("share takes exactly one argument, a blobref")
}
br := blobref.Parse(args[0])
if br == nil {
return UsageError("invalid blobref")
return cmdmain.UsageError("invalid blobref")
}
pr, err := up.UploadShare(br, c.transitive)
pr, err := getUploader().UploadShare(br, c.transitive)
handleResult("share", pr, err)
return nil
}

28
cmd/camtool/camtool.go Normal file
View File

@ -0,0 +1,28 @@
/*
Copyright 2013 The Camlistore Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package main
import (
"camlistore.org/pkg/cmdmain"
)
func main() {
err := cmdmain.Main()
if err != nil {
cmdmain.Exit(2)
}
}

196
cmd/camsync/camsync.go → cmd/camtool/sync.go
View File

@ -1,5 +1,5 @@
/*
Copyright 2011 Google Inc.
Copyright 2013 The Camlistore Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
@ -26,21 +26,96 @@ import (
"camlistore.org/pkg/blobref"
"camlistore.org/pkg/client"
"camlistore.org/pkg/cmdmain"
)
var (
flagLoop = flag.Bool("loop", false, "sync in a loop once done; requires --removesrc")
flagVerbose = flag.Bool("verbose", false, "be verbose")
flagAll = flag.Bool("all", false, "Discover all sync destinations configured on the source server and run them.")
type syncCmd struct {
src string
dest string
flagSrc = flag.String("src", "", "Source blobserver is either a URL prefix (with optional path), a host[:port], or blank to use the Camlistore client config's default host.")
flagDest = flag.String("dest", "", "Destination blobserver, or 'stdout' to just enumerate the --src blobs to stdout")
loop bool
verbose bool
all bool
removeSrc bool
flagRemoveSource = flag.Bool("removesrc", false,
"remove each blob from the source after syncing to the destination; for queue processing")
)
logger *log.Logger
}
var logger *log.Logger = nil
func init() {
cmdmain.RegisterCommand("sync", func(flags *flag.FlagSet) cmdmain.CommandRunner {
cmd := new(syncCmd)
flags.StringVar(&cmd.src, "src", "", "Source blobserver is either a URL prefix (with optional path), a host[:port], or blank to use the Camlistore client config's default host.")
flags.StringVar(&cmd.dest, "dest", "", "Destination blobserver, or 'stdout' to just enumerate the --src blobs to stdout.")
flags.BoolVar(&cmd.loop, "loop", false, "Create an associate a new permanode for the uploaded file or directory.")
flags.BoolVar(&cmd.verbose, "verbose", false, "Be verbose.")
flags.BoolVar(&cmd.all, "all", false, "Discover all sync destinations configured on the source server and run them.")
flags.BoolVar(&cmd.removeSrc, "removesrc", false, "Remove each blob from the source after syncing to the destination; for queue processing.")
return cmd
})
}
func (c *syncCmd) Usage() {
fmt.Fprintf(os.Stderr, "Usage: camtool [globalopts] sync [syncopts] \n")
}
func (c *syncCmd) Examples() []string {
return []string{
"--all",
"--src http://localhost:3179/bs/ --dest http://localhost:3179/index-mem/",
}
}
func (c *syncCmd) RunCommand(args []string) error {
if c.loop && !c.removeSrc {
return cmdmain.UsageError("Can't use --loop without --removesrc")
}
if c.verbose {
c.logger = log.New(os.Stderr, "", 0) // else nil
}
if c.all {
err := c.syncAll()
if err != nil {
return fmt.Errorf("sync all failed: %v", err)
}
return nil
}
if c.dest == "" {
return cmdmain.UsageError("No --dest specified.")
}
discl := c.discoClient()
discl.SetLogger(c.logger)
src, err := discl.BlobRoot()
if err != nil {
return fmt.Errorf("Failed to get blob source: %v", err)
}
sc := client.New(src)
sc.SetupAuth()
dc := client.New(c.dest)
dc.SetupAuth()
sc.SetLogger(c.logger)
dc.SetLogger(c.logger)
passNum := 0
for {
passNum++
stats, err := c.doPass(sc, dc)
if c.verbose {
log.Printf("sync stats - pass: %d, blobs: %d, bytes %d\n", passNum, stats.BlobsCopied, stats.BytesCopied)
}
if err != nil {
return fmt.Errorf("sync failed: %v", err)
}
if !c.loop {
break
}
}
return nil
}
type SyncStats struct {
BlobsCopied int
@ -48,26 +123,18 @@ type SyncStats struct {
ErrorCount int
}
func usage(err string) {
if err != "" {
fmt.Fprintf(os.Stderr, "Error: %s\n\nUsage:\n", err)
}
flag.PrintDefaults()
os.Exit(2)
}
func syncAll() error {
if *flagLoop {
usage("--all can not be used with --loop")
func (c *syncCmd) syncAll() error {
if c.loop {
return cmdmain.UsageError("--all can not be used with --loop")
}
dc := discoClient()
dc.SetLogger(logger)
dc := c.discoClient()
dc.SetLogger(c.logger)
syncHandlers, err := dc.SyncHandlers()
if err != nil {
log.Fatalf("sync handlers discovery failed: %v", err)
return fmt.Errorf("sync handlers discovery failed: %v", err)
}
if *flagVerbose {
if c.verbose {
log.Printf("To be synced:\n")
for _, sh := range syncHandlers {
log.Printf("%v -> %v", sh.From, sh.To)
@ -75,16 +142,16 @@ func syncAll() error {
}
for _, sh := range syncHandlers {
from := client.New(sh.From)
from.SetLogger(logger)
from.SetLogger(c.logger)
from.SetupAuth()
to := client.New(sh.To)
to.SetLogger(logger)
to.SetLogger(c.logger)
to.SetupAuth()
if *flagVerbose {
if c.verbose {
log.Printf("Now syncing: %v -> %v", sh.From, sh.To)
}
stats, err := doPass(from, to)
if *flagVerbose {
stats, err := c.doPass(from, to)
if c.verbose {
log.Printf("sync stats, blobs: %d, bytes %d\n", stats.BlobsCopied, stats.BytesCopied)
}
if err != nil {
@ -98,69 +165,18 @@ func syncAll() error {
// based from --src or from the configuration file if --src
// is blank. The returned client can then be used to discover
// the blobRoot and syncHandlers.
func discoClient() *client.Client {
func (c *syncCmd) discoClient() *client.Client {
var cl *client.Client
if *flagSrc == "" {
if c.src == "" {
cl = client.NewOrFail()
} else {
cl = client.New(*flagSrc)
cl = client.New(c.src)
}
cl.SetupAuth()
return cl
}
func main() {
flag.Parse()
if *flagLoop && !*flagRemoveSource {
usage("Can't use --loop without --removesrc")
}
if *flagVerbose {
logger = log.New(os.Stderr, "", 0)
}
if *flagAll {
err := syncAll()
if err != nil {
log.Fatalf("sync all failed: %v", err)
}
return
}
if *flagDest == "" {
usage("No --dest specified.")
}
discl := discoClient()
discl.SetLogger(logger)
src, err := discl.BlobRoot()
if err != nil {
log.Fatalf("Failed to get blob source: %v", err)
}
sc := client.New(src)
sc.SetupAuth()
dc := client.New(*flagDest)
dc.SetupAuth()
sc.SetLogger(logger)
dc.SetLogger(logger)
passNum := 0
for {
passNum++
stats, err := doPass(sc, dc)
if *flagVerbose {
log.Printf("sync stats - pass: %d, blobs: %d, bytes %d\n", passNum, stats.BlobsCopied, stats.BytesCopied)
}
if err != nil {
log.Fatalf("sync failed: %v", err)
}
if !*flagLoop {
break
}
}
}
func doPass(sc, dc *client.Client) (stats SyncStats, retErr error) {
func (c *syncCmd) doPass(sc, dc *client.Client) (stats SyncStats, retErr error) {
srcBlobs := make(chan blobref.SizedBlobRef, 100)
destBlobs := make(chan blobref.SizedBlobRef, 100)
srcErr := make(chan error)
@ -175,7 +191,7 @@ func doPass(sc, dc *client.Client) (stats SyncStats, retErr error) {
}
}
if *flagDest == "stdout" {
if c.dest == "stdout" {
for sb := range srcBlobs {
fmt.Printf("%s %d\n", sb.BlobRef, sb.Size)
}
@ -195,7 +211,7 @@ func doPass(sc, dc *client.Client) (stats SyncStats, retErr error) {
destNotHaveBlobs := make(chan blobref.SizedBlobRef)
sizeMismatch := make(chan *blobref.BlobRef)
readSrcBlobs := srcBlobs
if *flagVerbose {
if c.verbose {
readSrcBlobs = loggingBlobRefChannel(srcBlobs)
}
mismatches := []*blobref.BlobRef{}
@ -237,7 +253,7 @@ For:
stats.BlobsCopied++
stats.BytesCopied += pr.Size
}
if *flagRemoveSource {
if c.removeSrc {
if err = sc.RemoveBlob(sb.BlobRef); err != nil {
stats.ErrorCount++
log.Printf("Failed to delete %s from source: %v", sb.BlobRef, err)
@ -249,7 +265,7 @@ For:
checkSourceError()
checkDestError()
if retErr == nil && stats.ErrorCount > 0 {
retErr = errors.New(fmt.Sprintf("%d errors during sync", stats.ErrorCount))
retErr = fmt.Errorf("%d errors during sync", stats.ErrorCount)
}
return stats, retErr
}

224
pkg/cmdmain/cmdmain.go Normal file
View File

@ -0,0 +1,224 @@
/*
Copyright 2013 The Camlistore Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cmdmain
import (
"flag"
"fmt"
"io"
"log"
"os"
"sort"
"camlistore.org/pkg/buildinfo"
)
var (
FlagVersion = flag.Bool("version", false, "show version")
FlagHelp = flag.Bool("help", false, "print usage")
FlagVerbose = flag.Bool("verbose", false, "extra debug logging")
)
var (
// ExtraFlagRegistration allows to add more flags from
// other packages (with AddFlags) when Main starts.
ExtraFlagRegistration = func() {}
// PreExit is meant to dump additional stats and other
// verbiage before Main terminates.
PreExit = func() {}
)
var ErrUsage = UsageError("invalid command")
type UsageError string
func (ue UsageError) Error() string {
return "Usage error: " + string(ue)
}
var (
// mode name to actual subcommand mapping
modeCommand = make(map[string]CommandRunner)
modeFlags = make(map[string]*flag.FlagSet)
// Indirections for replacement by tests
Stderr io.Writer = os.Stderr
Stdout io.Writer = os.Stdout
Stdin io.Reader = os.Stdin
Exit = realExit
// TODO: abstract out vfs operation. should never call os.Stat, os.Open, os.Create, etc.
// Only use fs.Stat, fs.Open, where vs is an interface type.
// TODO: switch from using the global flag FlagSet and use our own. right now
// running "go test -v" dumps the flag usage data to the global stderr.
)
func realExit(code int) {
os.Exit(code)
}
// CommandRunner is the type that a command mode should implement.
type CommandRunner interface {
Usage()
RunCommand(args []string) error
}
type exampler interface {
Examples() []string
}
// RegisterCommand adds a mode to the list of modes for the main command.
// It is meant to be called in init() for each subcommand.
func RegisterCommand(mode string, makeCmd func(Flags *flag.FlagSet) CommandRunner) {
if _, dup := modeCommand[mode]; dup {
log.Fatalf("duplicate command %q registered", mode)
}
flags := flag.NewFlagSet(mode+" options", flag.ContinueOnError)
flags.Usage = func() {}
modeFlags[mode] = flags
modeCommand[mode] = makeCmd(flags)
}
type namedMode struct {
Name string
Command CommandRunner
}
// TODO(mpl): do we actually need this? I changed usage
// to simply iterate over all of modeCommand and it seems
// fine.
func allModes(startModes []string) <-chan namedMode {
ch := make(chan namedMode)
go func() {
defer close(ch)
done := map[string]bool{}
for _, name := range startModes {
done[name] = true
cmd := modeCommand[name]
if cmd == nil {
panic("bogus mode: " + name)
}
ch <- namedMode{name, cmd}
}
var rest []string
for name := range modeCommand {
if !done[name] {
rest = append(rest, name)
}
}
sort.Strings(rest)
for _, name := range rest {
ch <- namedMode{name, modeCommand[name]}
}
}()
return ch
}
func hasFlags(flags *flag.FlagSet) bool {
any := false
flags.VisitAll(func(*flag.Flag) {
any = true
})
return any
}
// Errf prints to Stderr
func Errf(format string, args ...interface{}) {
fmt.Fprintf(Stderr, format, args...)
}
func usage(msg string) {
cmdName := os.Args[0]
if msg != "" {
Errf("Error: %v\n", msg)
}
Errf(`
Usage: ` + cmdName + ` [globalopts] <mode> [commandopts] [commandargs]
Examples:
`)
for mode, cmd := range modeCommand {
Errf("\n")
if ex, ok := cmd.(exampler); ok {
for _, example := range ex.Examples() {
Errf(" %s %s %s\n", cmdName, mode, example)
}
} else {
Errf(" %s %s ...\n", cmdName, mode)
}
}
Errf(`
For mode-specific help:
` + cmdName + ` <mode> -help
Global options:
`)
flag.PrintDefaults()
Exit(1)
}
// Main is meant to be the core of a command that has
// subcommands (modes), such as camput or camtool.
func Main() error {
ExtraFlagRegistration()
flag.Parse()
args := flag.Args()
if *FlagVersion {
fmt.Fprintf(Stderr, "camput version: %s\n", buildinfo.Version())
return nil
}
if *FlagHelp {
usage("")
}
if len(args) == 0 {
usage("No mode given.")
}
mode := args[0]
cmd, ok := modeCommand[mode]
if !ok {
usage(fmt.Sprintf("Unknown mode %q", mode))
}
cmdFlags := modeFlags[mode]
err := cmdFlags.Parse(args[1:])
if err != nil {
err = ErrUsage
} else {
err = cmd.RunCommand(cmdFlags.Args())
}
if ue, isUsage := err.(UsageError); isUsage {
if isUsage {
Errf("%s\n", ue)
}
cmd.Usage()
Errf("\nGlobal options:\n")
flag.PrintDefaults()
if hasFlags(cmdFlags) {
Errf("\nMode-specific options for mode %q:\n", mode)
cmdFlags.PrintDefaults()
}
Exit(1)
}
if *FlagVerbose {
PreExit()
}
return err
}