index,search: fix/finish wholeRef for fileInfo indexing

Bump index requiredSchemaVersion to 5.

Issue #581
Issue #577

Change-Id: I18e30aef280f97781132ef0841a189c7ca7e64be

pkg/index/corpus.go

@@ -514,18 +514,27 @@ func (c *Corpus) mergeFileInfoRow(k, v []byte) error {
 
 	// TODO: could at least use strutil.ParseUintBytes to not stringify and retain
 	// the length bytes of v.
-	c.ss = strutil.AppendSplitN(c.ss[:0], string(v), "|", 3)
-	if len(c.ss) != 3 {
-		return fmt.Errorf("unexpected fileinfo value %q", k)
+	c.ss = strutil.AppendSplitN(c.ss[:0], string(v), "|", 4)
+	if len(c.ss) != 3 && len(c.ss) != 4 {
+		return fmt.Errorf("unexpected fileinfo value %q", v)
 	}
 	size, err := strconv.ParseInt(c.ss[0], 10, 64)
 	if err != nil {
-		return fmt.Errorf("unexpected fileinfo value %q", k)
+		return fmt.Errorf("unexpected fileinfo value %q", v)
+	}
+	var wholeRef blob.Ref
+	if len(c.ss) == 4 && c.ss[3] != "" { // checking for "" because of special files such as symlinks.
+		var ok bool
+		wholeRef, ok = blob.Parse(urld(c.ss[3]))
+		if !ok {
+			return fmt.Errorf("invalid wholeRef blobref in value %q for fileinfo key %q", v, k)
+		}
 	}
 	c.mutateFileInfo(br, func(fi *camtypes.FileInfo) {
 		fi.Size = size
 		fi.FileName = c.str(urld(c.ss[1]))
 		fi.MIMEType = c.str(urld(c.ss[2]))
+		fi.WholeRef = wholeRef
 	})
 	return nil
 }

pkg/index/index.go

@@ -156,6 +156,10 @@ func New(s sorted.KeyValue) (*Index, error) {
 			// the user with a more useful tip:
 			tip = `(For the dev server, run "devcam server --wipe" to wipe both your blobs and index)`
 		} else {
+			if is4To5SchemaBump(schemaVersion) {
+				log.Print("Your index does not have WholeRef in FileInfo entries. It may not be fatal but some things might not work and you should consider reindexing soon.")
+				break
+			}
 			tip = "Run 'camlistored --reindex' (it might take awhile, but shows status). Alternative: 'camtool dbinit' (or just delete the file for a file based index), and then 'camtool sync --all'"
 		}
 		return nil, fmt.Errorf("index schema version is %d; required one is %d. You need to reindex. %s",
@@ -170,6 +174,10 @@ func New(s sorted.KeyValue) (*Index, error) {
 	return idx, nil
 }
 
+func is4To5SchemaBump(schemaVersion int) bool {
+	return schemaVersion == 4 && requiredSchemaVersion == 5
+}
+
 func newFromConfig(ld blobserver.Loader, config jsonconfig.Obj) (blobserver.Storage, error) {
 	blobPrefix := config.RequiredString("blobSource")
 	kvConfig := config.RequiredObject("storage")

pkg/index/index_test.go

@@ -161,21 +161,34 @@ func skipFromList(name string) bool {
 	return false
 }
 
-func TestMergeFileInfoRow(t *testing.T) {
+func testMergeFileInfoRow(t *testing.T, wholeRef string) {
 	c := index.ExpNewCorpus()
-	c.Exp_mergeFileInfoRow("fileinfo|sha1-579f7f246bd420d486ddeb0dadbb256cfaf8bf6b",
-		"100|something%2egif|image%2Fgif")
-	fi := c.Exp_files(blob.MustParse("sha1-579f7f246bd420d486ddeb0dadbb256cfaf8bf6b"))
+	value := "100|something%2egif|image%2Fgif"
 	want := camtypes.FileInfo{
 		Size:     100,
 		MIMEType: "image/gif",
 		FileName: "something.gif",
 	}
+	if wholeRef != "" {
+		value += "|" + wholeRef
+		want.WholeRef = blob.MustParse(wholeRef)
+	}
+	c.Exp_mergeFileInfoRow("fileinfo|sha1-579f7f246bd420d486ddeb0dadbb256cfaf8bf6b", value)
+	fi := c.Exp_files(blob.MustParse("sha1-579f7f246bd420d486ddeb0dadbb256cfaf8bf6b"))
 	if !reflect.DeepEqual(want, fi) {
 		t.Errorf("Got %+v; want %+v", fi, want)
 	}
 }
 
+// When requiredSchemaVersion was at 4, i.e. wholeRef hadn't been introduced into fileInfo
+func TestMergeFileInfoRow4(t *testing.T) {
+	testMergeFileInfoRow(t, "")
+}
+
+func TestMergeFileInfoRow(t *testing.T) {
+	testMergeFileInfoRow(t, "sha1-142b504945338158e0149d4ed25a41a522a28e88")
+}
+
 var (
 	chunk1 = &test.Blob{Contents: "foo"}
 	chunk2 = &test.Blob{Contents: "bar"}

pkg/index/keys.go

@@ -27,7 +27,8 @@ import (
 // requiredSchemaVersion is incremented every time
 // an index key type is added, changed, or removed.
 // Version 4: EXIF tags + GPS
-const requiredSchemaVersion = 4
+// Version 5: wholeRef added to keyFileInfo
+const requiredSchemaVersion = 5
 
 // type of key returns the identifier in k before the first ":" or "|".
 // (Originally we packed keys by hand and there are a mix of styles)

pkg/search/handler_test.go

@@ -480,7 +480,8 @@ var handlerTests = []handlerTest{
 			  "file": {
 				"fileName": "dude.jpg",
 				"size": 1932,
-				"mimeType": "image/jpeg"
+				"mimeType": "image/jpeg",
+				"wholeRef": "sha1-142b504945338158e0149d4ed25a41a522a28e88"
 			  },
 			  "image": {
 				"width": 50,
@@ -572,7 +573,8 @@ var handlerTests = []handlerTest{
 			  "file": {
 				"fileName": "dude.jpg",
 				"size": 1932,
-				"mimeType": "image/jpeg"
+				"mimeType": "image/jpeg",
+				"wholeRef": "sha1-142b504945338158e0149d4ed25a41a522a28e88"
 			  },
 			  "image": {
 				"width": 50,

pkg/search/query_test.go

@@ -631,7 +631,7 @@ func TestQueryPermanodeModtime(t *testing.T) {
 func TestDecodeFileInfo(t *testing.T) {
 	testQuery(t, func(qt *queryTest) {
 		id := qt.id
-		fileRef, _ := id.UploadFile("file.gif", "GIF87afoo", time.Unix(456, 0))
+		fileRef, wholeRef := id.UploadFile("file.gif", "GIF87afoo", time.Unix(456, 0))
 		res, err := qt.Handler().Describe(&DescribeRequest{
 			BlobRef: fileRef,
 		})
@@ -652,6 +652,10 @@ func TestDecodeFileInfo(t *testing.T) {
 			qt.t.Errorf("DescribedBlob.File = %+v; mime type is not image/gif", db.File)
 			return
 		}
+		if db.File.WholeRef != wholeRef {
+			qt.t.Errorf("DescribedBlob.WholeRef: got %v, wanted %v", wholeRef, db.File.WholeRef)
+			return
+		}
 	})
 }