From 83a0f2b47c8dead2d6a40a648a72ec2b73454d1f Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Fri, 16 Sep 2011 18:03:13 -0700 Subject: [PATCH] some camput work, adding start of a stat cache Change-Id: I2d4090362a639eed2af6ce02dbaae81f249dfd57 --- .last_go_version | 2 +- clients/go/camput/camput.go | 44 +++++++++++++++++++++++++++++++++---- 2 files changed, 41 insertions(+), 5 deletions(-) diff --git a/.last_go_version b/.last_go_version index d6c25788b..d952dff08 100644 --- a/.last_go_version +++ b/.last_go_version @@ -1 +1 @@ -6g version weekly.2011-09-07 9772+ +6g version weekly.2011-09-16 9779 diff --git a/clients/go/camput/camput.go b/clients/go/camput/camput.go index 3353b16f7..0d6cf83dd 100644 --- a/clients/go/camput/camput.go +++ b/clients/go/camput/camput.go @@ -51,10 +51,18 @@ var flagSplits = flag.Bool("debug-splits", false, "show splits") var wereErrors = false +type UploadCache interface { + CachedPutResult(pwd, filename string, fi *os.FileInfo) (*client.PutResult, os.Error) + AddCachedPutResult(pwd, filename string, fi *os.FileInfo, pr *client.PutResult) +} + type Uploader struct { *client.Client entityFetcher jsonsign.EntityFetcher + pwd string + cache UploadCache + filecapc chan bool } @@ -68,10 +76,14 @@ func blobDetails(contents io.ReadSeeker) (bref *blobref.BlobRef, size int64, err return } -func (up *Uploader) UploadFileBlob(filename string) (*client.PutResult, os.Error) { +func vprintf(format string, args ...interface{}) { if *flagVerbose { - log.Printf("Uploading filename: %s", filename) + log.Printf(format, args...) } +} + +func (up *Uploader) UploadFileBlob(filename string) (*client.PutResult, os.Error) { + vprintf("Uploading filename: %s", filename) fi, err := os.Stat(filename) if err != nil { return nil, err @@ -101,7 +113,7 @@ func (up *Uploader) endFileUpload() { <-up.filecapc } -func (up *Uploader) UploadFile(filename string) (*client.PutResult, os.Error) { +func (up *Uploader) UploadFile(filename string) (respr *client.PutResult, outerr os.Error) { up.beginFileUpload() defer up.endFileUpload() @@ -110,6 +122,20 @@ func (up *Uploader) UploadFile(filename string) (*client.PutResult, os.Error) { return nil, err } + if up.cache != nil { + cachedRes, err := up.cache.CachedPutResult(up.pwd, filename, fi) + if err == nil { + return cachedRes, nil + } + if fi.IsRegular() { + defer func() { + if respr != nil && outerr == nil { + up.cache.AddCachedPutResult(up.pwd, filename, fi, respr) + } + }() + } + } + m := schema.NewCommonFileMap(filename, fi) switch { @@ -171,6 +197,7 @@ func (up *Uploader) UploadFile(filename string) (*client.PutResult, os.Error) { } mappr, err := up.UploadMap(m) + vprintf("Uploaded map: %v => %v, %v", m, mappr, err) return mappr, err } @@ -281,9 +308,18 @@ func main() { if !*flagVerbose { cc.SetLogger(nil) } + + pwd, err := os.Getwd() + if err != nil { + log.Fatalf("os.Getwd: %v", err) + } + cache := NewFlatCache() + defer cache.Save() + up := &Uploader{ Client: cc, - + pwd: pwd, + cache: cache, filecapc: make(chan bool, 10 /* TODO: config option on max files at a time */ ), entityFetcher: &jsonsign.CachingEntityFetcher{ Fetcher: &jsonsign.FileEntityFetcher{File: cc.SecretRingFile()},