search, index: add WholeRef to pkg camtypes' FileInfo struct

So when you describe a file, you also gets its wholeref.

TODO: we'll need to migrate old indexes to this new format on
start-up.

Change-Id: I4a3fb000d68bde46474275c2070ef285a6d6ecfc

pkg/index/index.go

@@ -1021,6 +1021,7 @@ func (x *Index) GetFileInfo(fileRef blob.Ref) (camtypes.FileInfo, error) {
 	}
 	ikey := "fileinfo|" + fileRef.String()
 	tkey := "filetimes|" + fileRef.String()
+	// TODO: switch this to use syncutil.Group
 	wg := new(sync.WaitGroup)
 	wg.Add(2)
 	var iv, tv string // info value, time value
@@ -1040,6 +1041,10 @@ func (x *Index) GetFileInfo(fileRef blob.Ref) (camtypes.FileInfo, error) {
 		log.Printf("index: bogus key %q = %q", ikey, iv)
 		return camtypes.FileInfo{}, os.ErrNotExist
 	}
+	var wholeRef blob.Ref
+	if len(valPart) >= 4 {
+		wholeRef, _ = blob.Parse(valPart[3])
+	}
 	size, err := strconv.ParseInt(valPart[0], 10, 64)
 	if err != nil {
 		log.Printf("index: bogus integer at position 0 in key %q = %q", ikey, iv)
@@ -1050,6 +1055,7 @@ func (x *Index) GetFileInfo(fileRef blob.Ref) (camtypes.FileInfo, error) {
 		Size:     size,
 		FileName: fileName,
 		MIMEType: urld(valPart[2]),
+		WholeRef: wholeRef,
 	}
 
 	if tv != "" {

pkg/index/indextest/tests.go

@@ -875,7 +875,7 @@ func Files(t *testing.T, initIdx func() *index.Index) {
 	// FileInfo
 	{
 		key := fmt.Sprintf("fileinfo|%s", fileRef)
-		if g, e := id.Get(key), "31|foo.html|text%2Fhtml"; g != e {
+		if g, e := id.Get(key), "31|foo.html|text%2Fhtml|sha1-153cb1b63a8f120a0e3e14ff34c64f169df9430f"; g != e {
 			t.Fatalf("%q = %q, want %q", key, g, e)
 		}
 
@@ -883,17 +883,20 @@ func Files(t *testing.T, initIdx func() *index.Index) {
 		if err != nil {
 			t.Fatalf("GetFileInfo = %v", err)
 		}
-		if g, e := fi.Size, int64(31); g != e {
-			t.Errorf("Size = %d, want %d", g, e)
+		if got, want := fi.Size, int64(31); got != want {
+			t.Errorf("Size = %d, want %d", got, want)
 		}
-		if g, e := fi.FileName, "foo.html"; g != e {
-			t.Errorf("FileName = %q, want %q", g, e)
+		if got, want := fi.FileName, "foo.html"; got != want {
+			t.Errorf("FileName = %q, want %q", got, want)
 		}
-		if g, e := fi.MIMEType, "text/html"; g != e {
-			t.Errorf("MIMEType = %q, want %q", g, e)
+		if got, want := fi.MIMEType, "text/html"; got != want {
+			t.Errorf("MIMEType = %q, want %q", got, want)
 		}
-		if g, e := fi.Time, fileTime; !g.Time().Equal(e) {
-			t.Errorf("Time = %v; want %v", g, e)
+		if got, want := fi.Time, fileTime; !got.Time().Equal(want) {
+			t.Errorf("Time = %v; want %v", got, want)
+		}
+		if got, want := fi.WholeRef, blob.MustParse("sha1-153cb1b63a8f120a0e3e14ff34c64f169df9430f"); got != want {
+			t.Errorf("WholeRef = %v; want %v", got, want)
 		}
 	}
 }

pkg/index/keys.go

@@ -20,6 +20,8 @@ import (
 	"bytes"
 	"fmt"
 	"strings"
+
+	"camlistore.org/pkg/blob"
 )
 
 // requiredSchemaVersion is incremented every time
@@ -107,6 +109,14 @@ func (k *keyType) build(isPrefix, isKey bool, parts []part, args ...interface{})
 				panic("doesn't look like a time: " + s)
 			}
 			buf.WriteString(reverseTimeString(s))
+		case typeBlobRef:
+			if br, ok := arg.(blob.Ref); ok {
+				if br.Valid() {
+					buf.WriteString(br.String())
+				}
+				break
+			}
+			fallthrough
 		default:
 			if s, ok := arg.(string); ok {
 				buf.WriteString(s)
@@ -242,6 +252,7 @@ var (
 			{"size", typeIntStr},
 			{"filename", typeStr},
 			{"mimetype", typeStr},
+			{"whole", typeBlobRef},
 		},
 	}
 

pkg/index/receive.go

@@ -442,7 +442,7 @@ func (ix *Index) populateFile(fetcher blob.Fetcher, b *schema.Blob, mm *mutation
 	}
 
 	mm.Set(keyWholeToFileRef.Key(wholeRef, blobRef), "1")
-	mm.Set(keyFileInfo.Key(blobRef), keyFileInfo.Val(size, b.FileName(), mime))
+	mm.Set(keyFileInfo.Key(blobRef), keyFileInfo.Val(size, b.FileName(), mime, wholeRef))
 	mm.Set(keyFileTimes.Key(blobRef), keyFileTimes.Val(time3339s))
 
 	if strings.HasPrefix(mime, "audio/") {
@@ -657,7 +657,7 @@ func (ix *Index) populateDir(fetcher blob.Fetcher, b *schema.Blob, mm *mutationM
 		return nil
 	}
 
-	mm.Set(keyFileInfo.Key(blobRef), keyFileInfo.Val(len(sts), b.FileName(), ""))
+	mm.Set(keyFileInfo.Key(blobRef), keyFileInfo.Val(len(sts), b.FileName(), "", blob.Ref{}))
 	for _, br := range sts {
 		mm.Set(keyStaticDirChild.Key(blobRef, br.String()), "1")
 	}

pkg/types/camtypes/search.go

@@ -86,12 +86,13 @@ func (cl ClaimsByDate) String() string {
 
 // FileInfo describes a file or directory.
 type FileInfo struct {
+	// FileName is the base name of the file or directory.
 	FileName string `json:"fileName"`
 
 	// TODO(mpl): I've noticed that Size is actually set to the
 	// number of entries in the dir. fix the doc or the behaviour?
 
-	// Size is the size of files. It is not set for directories.
+	// Size is the size of file. It is not set for directories.
 	Size int64 `json:"size"`
 
 	// MIMEType may be set for files, but never for directories.
@@ -106,6 +107,11 @@ type FileInfo struct {
 	// original/modification times found. If ModTime doesn't differ
 	// from Time, ModTime is omitted (zero).
 	ModTime *types.Time3339 `json:"modTime,omitempty"`
+
+	// WholeRef is the digest of the entire file contents.
+	// This will be zero for non-regular files, and may also be zero
+	// for files above a certain size threshold.
+	WholeRef blob.Ref `json:"wholeRef,omitempty"`
 }
 
 func (fi *FileInfo) IsImage() bool {