From fd8af444315f4f6ea47fb0453eb6366e3ce977d9 Mon Sep 17 00:00:00 2001 From: mpl Date: Wed, 26 Jun 2013 18:50:35 +0200 Subject: [PATCH] camput: stat cache: use a different key in -filenodes mode http://camlistore.org/issue/157 Change-Id: If58116be6073e0bd160d57bb8f778e498a8763bb --- cmd/camput/cache.go | 15 ++++++++++++--- cmd/camput/files.go | 19 ++++++++++++++----- cmd/camput/flatcache.go | 24 ++++++++++++++++-------- cmd/camput/sqlitecache.go | 26 ++++++++++++++++++++++---- 4 files changed, 64 insertions(+), 20 deletions(-) diff --git a/cmd/camput/cache.go b/cmd/camput/cache.go index b687fe8fa..484d367d8 100644 --- a/cmd/camput/cache.go +++ b/cmd/camput/cache.go @@ -30,10 +30,19 @@ type HaveCache interface { } // UploadCache is the "stat cache" for regular files. Given a current -// working directory, possibly relative filename, and stat info, +// working directory, possibly relative filename, stat info, and +// whether that file was uploaded with a permanode (-filenodes), // returns what the ultimate put result (the top-level "file" schema // blob) for that regular file was. type UploadCache interface { - CachedPutResult(pwd, filename string, fi os.FileInfo) (*client.PutResult, error) - AddCachedPutResult(pwd, filename string, fi os.FileInfo, pr *client.PutResult) + // CachedPutResult looks in the cache for the put result for the file + // that was uploaded. If withPermanode, it is only a hit if a planned + // permanode for the file was created and uploaded too, and vice-versa. + // The returned PutResult is always for the "file" schema blob. + CachedPutResult(pwd, filename string, fi os.FileInfo, withPermanode bool) (*client.PutResult, error) + // AddCachedPutResult stores in the cache the put result for the file that + // was uploaded. If withPermanode, it means a planned permanode was created + // for this file when it was uploaded (with -filenodes), and the cache entry + // will reflect that. + AddCachedPutResult(pwd, filename string, fi os.FileInfo, pr *client.PutResult, withPermanode bool) } diff --git a/cmd/camput/files.go b/cmd/camput/files.go index 98727da4b..8720cfd22 100644 --- a/cmd/camput/files.go +++ b/cmd/camput/files.go @@ -604,7 +604,10 @@ func (up *Uploader) uploadNodeRegularFile(n *node) (*client.PutResult, error) { } } - // TODO(mpl): test that none of these claims get uploaded if they've already been done + // The work for those planned permanodes (and the claims) is redone + // everytime we get here (i.e past the stat cache). However, they're + // caught by the have cache, so they won't be reuploaded for nothing + // at least. if up.fileOpts.wantFilePermanode() { if td, ok := fileContents.(*trackDigestReader); ok { sum = td.Sum() @@ -713,18 +716,21 @@ func (up *Uploader) UploadFile(filename string) (*client.PutResult, error) { fi: fi, } + withPermanode := up.fileOpts.wantFilePermanode() if up.statCache != nil && !up.fileOpts.wantVivify() { // Note: ignoring cache hits if wantVivify, otherwise // a non-vivify put followed by a vivify one wouldn't // end up doing the vivify. - if cachedRes, err := up.statCache.CachedPutResult(up.pwd, n.fullPath, n.fi); err == nil { + if cachedRes, err := up.statCache.CachedPutResult( + up.pwd, n.fullPath, n.fi, withPermanode); err == nil { return cachedRes, nil } } pr, err := up.uploadNode(n) if err == nil && up.statCache != nil { - up.statCache.AddCachedPutResult(up.pwd, n.fullPath, n.fi, pr) + up.statCache.AddCachedPutResult( + up.pwd, n.fullPath, n.fi, pr, withPermanode) } return pr, err @@ -949,6 +955,7 @@ func (t *TreeUpload) run() { uploadsdonec := make(chan bool) var upload chan<- *node + withPermanode := t.up.fileOpts.wantFilePermanode() if t.DiskUsageMode { upload = NewNodeWorker(1, func(n *node, ok bool) { if !ok { @@ -972,7 +979,8 @@ func (t *TreeUpload) run() { } n.SetPutResult(put, nil) if c := t.up.statCache; c != nil && !n.fi.IsDir() { - c.AddCachedPutResult(t.up.pwd, n.fullPath, n.fi, put) + c.AddCachedPutResult( + t.up.pwd, n.fullPath, n.fi, put, withPermanode) } uploadedc <- n }) @@ -991,7 +999,8 @@ func (t *TreeUpload) run() { return } if !n.fi.IsDir() { - cachedRes, err := t.up.statCache.CachedPutResult(t.up.pwd, n.fullPath, n.fi) + cachedRes, err := t.up.statCache.CachedPutResult( + t.up.pwd, n.fullPath, n.fi, withPermanode) if err == nil { n.SetPutResult(cachedRes, nil) cachelog.Printf("Cache HIT on %q -> %v", n.fullPath, cachedRes) diff --git a/cmd/camput/flatcache.go b/cmd/camput/flatcache.go index 50a8eaf42..ab76dcce8 100644 --- a/cmd/camput/flatcache.go +++ b/cmd/camput/flatcache.go @@ -136,21 +136,29 @@ var _ UploadCache = (*FlatStatCache)(nil) var errCacheMiss = errors.New("not in cache") -// cacheKey returns the cleaned absolute path of joining pwd and filename. -func cacheKey(pwd, filename string) string { +// flatCacheKey returns the key used for a stat entry in the flat cache. +// It is the cleaned absolute path of joining pwd and filename, to which +// "|Perm" is appended if -filenodes is being used. +func flatCacheKey(pwd, filename string, withPermanode bool) string { + var fullPath string if filepath.IsAbs(filename) { - return filepath.Clean(filename) + fullPath = filepath.Clean(filename) + } else { + fullPath = filepath.Join(pwd, filename) } - return filepath.Join(pwd, filename) + if withPermanode { + return fmt.Sprintf("%v|Perm", fullPath) + } + return fullPath } -func (c *FlatStatCache) CachedPutResult(pwd, filename string, fi os.FileInfo) (*client.PutResult, error) { +func (c *FlatStatCache) CachedPutResult(pwd, filename string, fi os.FileInfo, withPermanode bool) (*client.PutResult, error) { c.mu.RLock() defer c.mu.RUnlock() fp := fileInfoToFingerprint(fi) - key := cacheKey(pwd, filename) + key := flatCacheKey(pwd, filename, withPermanode) val, ok := c.m[key] if !ok { cachelog.Printf("cache MISS on %q: not in cache", key) @@ -164,10 +172,10 @@ func (c *FlatStatCache) CachedPutResult(pwd, filename string, fi os.FileInfo) (* return &pr, nil } -func (c *FlatStatCache) AddCachedPutResult(pwd, filename string, fi os.FileInfo, pr *client.PutResult) { +func (c *FlatStatCache) AddCachedPutResult(pwd, filename string, fi os.FileInfo, pr *client.PutResult, withPermanode bool) { c.mu.Lock() defer c.mu.Unlock() - key := cacheKey(pwd, filename) + key := flatCacheKey(pwd, filename, withPermanode) val := fileInfoPutRes{fileInfoToFingerprint(fi), *pr} cachelog.Printf("Adding to stat cache %q: %v", key, val) diff --git a/cmd/camput/sqlitecache.go b/cmd/camput/sqlitecache.go index d889d7429..2c0678737 100644 --- a/cmd/camput/sqlitecache.go +++ b/cmd/camput/sqlitecache.go @@ -177,8 +177,26 @@ func (c *SQLiteStatCache) startSQLiteChild() error { return nil } -func (c *SQLiteStatCache) CachedPutResult(pwd, filename string, fi os.FileInfo) (*client.PutResult, error) { - key := cacheKey(pwd, filename) + "|" + string(fileInfoToFingerprint(fi)) +// sqliteCacheKey returns the key used for a stat entry in the sqlite cache. +// It is the cleaned absolute path of joining pwd and filename, +// concatenated with a fingerprint based on the file's info. If +// -filenodes is being used, the suffix "|Perm" is also appended. +func sqliteCacheKey(pwd, filename string, fi os.FileInfo, withPermanode bool) string { + var fullPath string + if filepath.IsAbs(filename) { + fullPath = filepath.Clean(filename) + } else { + fullPath = filepath.Join(pwd, filename) + } + key := fmt.Sprintf("%v|%v", fullPath, string(fileInfoToFingerprint(fi))) + if withPermanode { + return fmt.Sprintf("%v|Perm", key) + } + return key +} + +func (c *SQLiteStatCache) CachedPutResult(pwd, filename string, fi os.FileInfo, withPermanode bool) (*client.PutResult, error) { + key := sqliteCacheKey(pwd, filename, fi, withPermanode) query := fmt.Sprintf("%v'%v';\n", statKeyQuery, key) c.mu.Lock() err := c.startSQLiteChild() @@ -221,8 +239,8 @@ func (c *SQLiteStatCache) CachedPutResult(pwd, filename string, fi os.FileInfo) }, nil } -func (c *SQLiteStatCache) AddCachedPutResult(pwd, filename string, fi os.FileInfo, pr *client.PutResult) { - key := cacheKey(pwd, filename) + "|" + string(fileInfoToFingerprint(fi)) +func (c *SQLiteStatCache) AddCachedPutResult(pwd, filename string, fi os.FileInfo, pr *client.PutResult, withPermanode bool) { + key := sqliteCacheKey(pwd, filename, fi, withPermanode) val := pr.BlobRef.String() + "|" + strconv.FormatInt(pr.Size, 10) repl := strings.NewReplacer("?1", key, "?2", val) query := repl.Replace(noteStatStmt)