From 017d57d9d2ac3ba96c011e8ad9afc067b90f2e06 Mon Sep 17 00:00:00 2001 From: mpl Date: Fri, 20 Feb 2015 17:03:22 +0100 Subject: [PATCH] index,search: fix/finish wholeRef for fileInfo indexing Bump index requiredSchemaVersion to 5. Issue #581 Issue #577 Change-Id: I18e30aef280f97781132ef0841a189c7ca7e64be --- pkg/index/corpus.go | 17 +++++++++++++---- pkg/index/index.go | 8 ++++++++ pkg/index/index_test.go | 21 +++++++++++++++++---- pkg/index/keys.go | 3 ++- pkg/search/handler_test.go | 6 ++++-- pkg/search/query_test.go | 6 +++++- 6 files changed, 49 insertions(+), 12 deletions(-) diff --git a/pkg/index/corpus.go b/pkg/index/corpus.go index 7cb2a6ac3..51acafa20 100644 --- a/pkg/index/corpus.go +++ b/pkg/index/corpus.go @@ -514,18 +514,27 @@ func (c *Corpus) mergeFileInfoRow(k, v []byte) error { // TODO: could at least use strutil.ParseUintBytes to not stringify and retain // the length bytes of v. - c.ss = strutil.AppendSplitN(c.ss[:0], string(v), "|", 3) - if len(c.ss) != 3 { - return fmt.Errorf("unexpected fileinfo value %q", k) + c.ss = strutil.AppendSplitN(c.ss[:0], string(v), "|", 4) + if len(c.ss) != 3 && len(c.ss) != 4 { + return fmt.Errorf("unexpected fileinfo value %q", v) } size, err := strconv.ParseInt(c.ss[0], 10, 64) if err != nil { - return fmt.Errorf("unexpected fileinfo value %q", k) + return fmt.Errorf("unexpected fileinfo value %q", v) + } + var wholeRef blob.Ref + if len(c.ss) == 4 && c.ss[3] != "" { // checking for "" because of special files such as symlinks. + var ok bool + wholeRef, ok = blob.Parse(urld(c.ss[3])) + if !ok { + return fmt.Errorf("invalid wholeRef blobref in value %q for fileinfo key %q", v, k) + } } c.mutateFileInfo(br, func(fi *camtypes.FileInfo) { fi.Size = size fi.FileName = c.str(urld(c.ss[1])) fi.MIMEType = c.str(urld(c.ss[2])) + fi.WholeRef = wholeRef }) return nil } diff --git a/pkg/index/index.go b/pkg/index/index.go index 32cad54be..3266c5400 100644 --- a/pkg/index/index.go +++ b/pkg/index/index.go @@ -156,6 +156,10 @@ func New(s sorted.KeyValue) (*Index, error) { // the user with a more useful tip: tip = `(For the dev server, run "devcam server --wipe" to wipe both your blobs and index)` } else { + if is4To5SchemaBump(schemaVersion) { + log.Print("Your index does not have WholeRef in FileInfo entries. It may not be fatal but some things might not work and you should consider reindexing soon.") + break + } tip = "Run 'camlistored --reindex' (it might take awhile, but shows status). Alternative: 'camtool dbinit' (or just delete the file for a file based index), and then 'camtool sync --all'" } return nil, fmt.Errorf("index schema version is %d; required one is %d. You need to reindex. %s", @@ -170,6 +174,10 @@ func New(s sorted.KeyValue) (*Index, error) { return idx, nil } +func is4To5SchemaBump(schemaVersion int) bool { + return schemaVersion == 4 && requiredSchemaVersion == 5 +} + func newFromConfig(ld blobserver.Loader, config jsonconfig.Obj) (blobserver.Storage, error) { blobPrefix := config.RequiredString("blobSource") kvConfig := config.RequiredObject("storage") diff --git a/pkg/index/index_test.go b/pkg/index/index_test.go index cfaff6d41..d3990eacb 100644 --- a/pkg/index/index_test.go +++ b/pkg/index/index_test.go @@ -161,21 +161,34 @@ func skipFromList(name string) bool { return false } -func TestMergeFileInfoRow(t *testing.T) { +func testMergeFileInfoRow(t *testing.T, wholeRef string) { c := index.ExpNewCorpus() - c.Exp_mergeFileInfoRow("fileinfo|sha1-579f7f246bd420d486ddeb0dadbb256cfaf8bf6b", - "100|something%2egif|image%2Fgif") - fi := c.Exp_files(blob.MustParse("sha1-579f7f246bd420d486ddeb0dadbb256cfaf8bf6b")) + value := "100|something%2egif|image%2Fgif" want := camtypes.FileInfo{ Size: 100, MIMEType: "image/gif", FileName: "something.gif", } + if wholeRef != "" { + value += "|" + wholeRef + want.WholeRef = blob.MustParse(wholeRef) + } + c.Exp_mergeFileInfoRow("fileinfo|sha1-579f7f246bd420d486ddeb0dadbb256cfaf8bf6b", value) + fi := c.Exp_files(blob.MustParse("sha1-579f7f246bd420d486ddeb0dadbb256cfaf8bf6b")) if !reflect.DeepEqual(want, fi) { t.Errorf("Got %+v; want %+v", fi, want) } } +// When requiredSchemaVersion was at 4, i.e. wholeRef hadn't been introduced into fileInfo +func TestMergeFileInfoRow4(t *testing.T) { + testMergeFileInfoRow(t, "") +} + +func TestMergeFileInfoRow(t *testing.T) { + testMergeFileInfoRow(t, "sha1-142b504945338158e0149d4ed25a41a522a28e88") +} + var ( chunk1 = &test.Blob{Contents: "foo"} chunk2 = &test.Blob{Contents: "bar"} diff --git a/pkg/index/keys.go b/pkg/index/keys.go index c69d0fd9c..6476586e1 100644 --- a/pkg/index/keys.go +++ b/pkg/index/keys.go @@ -27,7 +27,8 @@ import ( // requiredSchemaVersion is incremented every time // an index key type is added, changed, or removed. // Version 4: EXIF tags + GPS -const requiredSchemaVersion = 4 +// Version 5: wholeRef added to keyFileInfo +const requiredSchemaVersion = 5 // type of key returns the identifier in k before the first ":" or "|". // (Originally we packed keys by hand and there are a mix of styles) diff --git a/pkg/search/handler_test.go b/pkg/search/handler_test.go index 336cb83bd..de44d0991 100644 --- a/pkg/search/handler_test.go +++ b/pkg/search/handler_test.go @@ -480,7 +480,8 @@ var handlerTests = []handlerTest{ "file": { "fileName": "dude.jpg", "size": 1932, - "mimeType": "image/jpeg" + "mimeType": "image/jpeg", + "wholeRef": "sha1-142b504945338158e0149d4ed25a41a522a28e88" }, "image": { "width": 50, @@ -572,7 +573,8 @@ var handlerTests = []handlerTest{ "file": { "fileName": "dude.jpg", "size": 1932, - "mimeType": "image/jpeg" + "mimeType": "image/jpeg", + "wholeRef": "sha1-142b504945338158e0149d4ed25a41a522a28e88" }, "image": { "width": 50, diff --git a/pkg/search/query_test.go b/pkg/search/query_test.go index f146ee670..406b2a39f 100644 --- a/pkg/search/query_test.go +++ b/pkg/search/query_test.go @@ -631,7 +631,7 @@ func TestQueryPermanodeModtime(t *testing.T) { func TestDecodeFileInfo(t *testing.T) { testQuery(t, func(qt *queryTest) { id := qt.id - fileRef, _ := id.UploadFile("file.gif", "GIF87afoo", time.Unix(456, 0)) + fileRef, wholeRef := id.UploadFile("file.gif", "GIF87afoo", time.Unix(456, 0)) res, err := qt.Handler().Describe(&DescribeRequest{ BlobRef: fileRef, }) @@ -652,6 +652,10 @@ func TestDecodeFileInfo(t *testing.T) { qt.t.Errorf("DescribedBlob.File = %+v; mime type is not image/gif", db.File) return } + if db.File.WholeRef != wholeRef { + qt.t.Errorf("DescribedBlob.WholeRef: got %v, wanted %v", wholeRef, db.File.WholeRef) + return + } }) }