Merge "camput: stat cache: use a different key in -filenodes mode"

cmd/camput/cache.go

@@ -30,10 +30,19 @@ type HaveCache interface {
 }
 
 // UploadCache is the "stat cache" for regular files.  Given a current
-// working directory, possibly relative filename, and stat info,
+// working directory, possibly relative filename, stat info, and
+// whether that file was uploaded with a permanode (-filenodes),
 // returns what the ultimate put result (the top-level "file" schema
 // blob) for that regular file was.
 type UploadCache interface {
-	CachedPutResult(pwd, filename string, fi os.FileInfo) (*client.PutResult, error)
+	// CachedPutResult looks in the cache for the put result for the file
-	AddCachedPutResult(pwd, filename string, fi os.FileInfo, pr *client.PutResult)
+	// that was uploaded. If withPermanode, it is only a hit if a planned
+	// permanode for the file was created and uploaded too, and vice-versa.
+	// The returned PutResult is always for the "file" schema blob.
+	CachedPutResult(pwd, filename string, fi os.FileInfo, withPermanode bool) (*client.PutResult, error)
+	// AddCachedPutResult stores in the cache the put result for the file that
+	// was uploaded. If withPermanode, it means a planned permanode was created
+	// for this file when it was uploaded (with -filenodes), and the cache entry
+	// will reflect that.
+	AddCachedPutResult(pwd, filename string, fi os.FileInfo, pr *client.PutResult, withPermanode bool)
 }

cmd/camput/files.go

@@ -604,7 +604,10 @@ func (up *Uploader) uploadNodeRegularFile(n *node) (*client.PutResult, error) {
 		}
 	}
 
-	// TODO(mpl): test that none of these claims get uploaded if they've already been done
+	// The work for those planned permanodes (and the claims) is redone
+	// everytime we get here (i.e past the stat cache). However, they're
+	// caught by the have cache, so they won't be reuploaded for nothing
+	// at least.
 	if up.fileOpts.wantFilePermanode() {
 		if td, ok := fileContents.(*trackDigestReader); ok {
 			sum = td.Sum()
@@ -713,18 +716,21 @@ func (up *Uploader) UploadFile(filename string) (*client.PutResult, error) {
 		fi:       fi,
 	}
 
+	withPermanode := up.fileOpts.wantFilePermanode()
 	if up.statCache != nil && !up.fileOpts.wantVivify() {
 		// Note: ignoring cache hits if wantVivify, otherwise
 		// a non-vivify put followed by a vivify one wouldn't
 		// end up doing the vivify.
-		if cachedRes, err := up.statCache.CachedPutResult(up.pwd, n.fullPath, n.fi); err == nil {
+		if cachedRes, err := up.statCache.CachedPutResult(
+			up.pwd, n.fullPath, n.fi, withPermanode); err == nil {
 			return cachedRes, nil
 		}
 	}
 
 	pr, err := up.uploadNode(n)
 	if err == nil && up.statCache != nil {
-		up.statCache.AddCachedPutResult(up.pwd, n.fullPath, n.fi, pr)
+		up.statCache.AddCachedPutResult(
+			up.pwd, n.fullPath, n.fi, pr, withPermanode)
 	}
 
 	return pr, err
@@ -949,6 +955,7 @@ func (t *TreeUpload) run() {
 
 	uploadsdonec := make(chan bool)
 	var upload chan<- *node
+	withPermanode := t.up.fileOpts.wantFilePermanode()
 	if t.DiskUsageMode {
 		upload = NewNodeWorker(1, func(n *node, ok bool) {
 			if !ok {
@@ -972,7 +979,8 @@ func (t *TreeUpload) run() {
 			}
 			n.SetPutResult(put, nil)
 			if c := t.up.statCache; c != nil && !n.fi.IsDir() {
-				c.AddCachedPutResult(t.up.pwd, n.fullPath, n.fi, put)
+				c.AddCachedPutResult(
+					t.up.pwd, n.fullPath, n.fi, put, withPermanode)
 			}
 			uploadedc <- n
 		})
@@ -991,7 +999,8 @@ func (t *TreeUpload) run() {
 			return
 		}
 		if !n.fi.IsDir() {
-			cachedRes, err := t.up.statCache.CachedPutResult(t.up.pwd, n.fullPath, n.fi)
+			cachedRes, err := t.up.statCache.CachedPutResult(
+				t.up.pwd, n.fullPath, n.fi, withPermanode)
 			if err == nil {
 				n.SetPutResult(cachedRes, nil)
 				cachelog.Printf("Cache HIT on %q -> %v", n.fullPath, cachedRes)

cmd/camput/flatcache.go

@@ -136,21 +136,29 @@ var _ UploadCache = (*FlatStatCache)(nil)
 
 var errCacheMiss = errors.New("not in cache")
 
-// cacheKey returns the cleaned absolute path of joining pwd and filename.
+// flatCacheKey returns the key used for a stat entry in the flat cache.
-func cacheKey(pwd, filename string) string {
+// It is the cleaned absolute path of joining pwd and filename, to which
+// "|Perm" is appended if -filenodes is being used.
+func flatCacheKey(pwd, filename string, withPermanode bool) string {
+	var fullPath string
 	if filepath.IsAbs(filename) {
-		return filepath.Clean(filename)
+		fullPath = filepath.Clean(filename)
+	} else {
+		fullPath = filepath.Join(pwd, filename)
 	}
-	return filepath.Join(pwd, filename)
+	if withPermanode {
+		return fmt.Sprintf("%v|Perm", fullPath)
+	}
+	return fullPath
 }
 
-func (c *FlatStatCache) CachedPutResult(pwd, filename string, fi os.FileInfo) (*client.PutResult, error) {
+func (c *FlatStatCache) CachedPutResult(pwd, filename string, fi os.FileInfo, withPermanode bool) (*client.PutResult, error) {
 	c.mu.RLock()
 	defer c.mu.RUnlock()
 
 	fp := fileInfoToFingerprint(fi)
 
-	key := cacheKey(pwd, filename)
+	key := flatCacheKey(pwd, filename, withPermanode)
 	val, ok := c.m[key]
 	if !ok {
 		cachelog.Printf("cache MISS on %q: not in cache", key)
@@ -164,10 +172,10 @@ func (c *FlatStatCache) CachedPutResult(pwd, filename string, fi os.FileInfo) (*
 	return &pr, nil
 }
 
-func (c *FlatStatCache) AddCachedPutResult(pwd, filename string, fi os.FileInfo, pr *client.PutResult) {
+func (c *FlatStatCache) AddCachedPutResult(pwd, filename string, fi os.FileInfo, pr *client.PutResult, withPermanode bool) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
-	key := cacheKey(pwd, filename)
+	key := flatCacheKey(pwd, filename, withPermanode)
 	val := fileInfoPutRes{fileInfoToFingerprint(fi), *pr}
 
 	cachelog.Printf("Adding to stat cache %q: %v", key, val)

cmd/camput/sqlitecache.go

@@ -177,8 +177,26 @@ func (c *SQLiteStatCache) startSQLiteChild() error {
 	return nil
 }
 
-func (c *SQLiteStatCache) CachedPutResult(pwd, filename string, fi os.FileInfo) (*client.PutResult, error) {
+// sqliteCacheKey returns the key used for a stat entry in the sqlite cache.
-	key := cacheKey(pwd, filename) + "|" + string(fileInfoToFingerprint(fi))
+// It is the cleaned absolute path of joining pwd and filename,
+// concatenated with a fingerprint based on the file's info. If
+// -filenodes is being used, the suffix "|Perm" is also appended.
+func sqliteCacheKey(pwd, filename string, fi os.FileInfo, withPermanode bool) string {
+	var fullPath string
+	if filepath.IsAbs(filename) {
+		fullPath = filepath.Clean(filename)
+	} else {
+		fullPath = filepath.Join(pwd, filename)
+	}
+	key := fmt.Sprintf("%v|%v", fullPath, string(fileInfoToFingerprint(fi)))
+	if withPermanode {
+		return fmt.Sprintf("%v|Perm", key)
+	}
+	return key
+}
+
+func (c *SQLiteStatCache) CachedPutResult(pwd, filename string, fi os.FileInfo, withPermanode bool) (*client.PutResult, error) {
+	key := sqliteCacheKey(pwd, filename, fi, withPermanode)
 	query := fmt.Sprintf("%v'%v';\n", statKeyQuery, key)
 	c.mu.Lock()
 	err := c.startSQLiteChild()
@@ -221,8 +239,8 @@ func (c *SQLiteStatCache) CachedPutResult(pwd, filename string, fi os.FileInfo)
 	}, nil
 }
 
-func (c *SQLiteStatCache) AddCachedPutResult(pwd, filename string, fi os.FileInfo, pr *client.PutResult) {
+func (c *SQLiteStatCache) AddCachedPutResult(pwd, filename string, fi os.FileInfo, pr *client.PutResult, withPermanode bool) {
-	key := cacheKey(pwd, filename) + "|" + string(fileInfoToFingerprint(fi))
+	key := sqliteCacheKey(pwd, filename, fi, withPermanode)
 	val := pr.BlobRef.String() + "|" + strconv.FormatInt(pr.Size, 10)
 	repl := strings.NewReplacer("?1", key, "?2", val)
 	query := repl.Replace(noteStatStmt)