From f9d0ad487e0b0d8f4e033d75a652275effd9f218 Mon Sep 17 00:00:00 2001 From: mpl Date: Sun, 26 Feb 2012 13:49:03 +0100 Subject: [PATCH] index packages reorganization Change-Id: Ie91c4bb5ea02a49e59ad093ac972d84b17a046d4 --- pkg/index/export_test.go | 29 ++ pkg/index/index.go | 30 +- pkg/index/index_test.go | 475 +--------------------------- pkg/index/indextest/tests.go | 413 ++++++++++++++++++++++++ pkg/index/memindex.go | 8 +- pkg/index/mongo/export_test.go | 25 ++ pkg/index/{ => mongo}/mongoindex.go | 40 ++- pkg/index/mongo/mongoindex_test.go | 80 +++++ pkg/index/mongo/testdata | 1 + pkg/index/testdata | 1 + 10 files changed, 622 insertions(+), 480 deletions(-) create mode 100644 pkg/index/export_test.go create mode 100644 pkg/index/indextest/tests.go create mode 100644 pkg/index/mongo/export_test.go rename pkg/index/{ => mongo}/mongoindex.go (86%) create mode 100644 pkg/index/mongo/mongoindex_test.go create mode 120000 pkg/index/mongo/testdata create mode 120000 pkg/index/testdata diff --git a/pkg/index/export_test.go b/pkg/index/export_test.go new file mode 100644 index 000000000..6f1a719ef --- /dev/null +++ b/pkg/index/export_test.go @@ -0,0 +1,29 @@ +/* +Copyright 2012 Google Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package index + +func ExpReverseTimeString(s string) string { + return reverseTimeString(s) +} + +func ExpUnreverseTimeString(s string) string { + return unreverseTimeString(s) +} + +func ExpNewMemoryIndex() *Index { + return newMemoryIndex() +} diff --git a/pkg/index/index.go b/pkg/index/index.go index c1a8f810f..586a9e43e 100644 --- a/pkg/index/index.go +++ b/pkg/index/index.go @@ -87,14 +87,40 @@ type BatchMutation interface { Delete(key string) } +type Mutation interface { + Key() string + Value() string + IsDelete() bool +} + type mutation struct { key string value string // used if !delete delete bool // if to be deleted } +func (m mutation) Key() string { + return m.key +} + +func (m mutation) Value() string { + return m.value +} + +func (m mutation) IsDelete() bool { + return m.delete +} + +func NewBatchMutation() BatchMutation { + return &batch{} +} + type batch struct { - m []mutation + m []Mutation +} + +func (b *batch) Mutations() []Mutation { + return b.m } func (b *batch) Delete(key string) { @@ -498,3 +524,5 @@ func (x *Index) GetFileInfo(fileRef *blobref.BlobRef) (*search.FileInfo, error) } return fi, nil } + +func (x *Index) Storage() IndexStorage { return x.s } diff --git a/pkg/index/index_test.go b/pkg/index/index_test.go index 51c9a13cf..226e1c436 100644 --- a/pkg/index/index_test.go +++ b/pkg/index/index_test.go @@ -14,477 +14,36 @@ See the License for the specific language governing permissions and limitations under the License. */ -package index +package index_test import ( - "errors" - "fmt" - "log" - "os" - "reflect" - "sync" "testing" - "time" - "camlistore.org/pkg/blobref" - "camlistore.org/pkg/jsonsign" - "camlistore.org/pkg/schema" - "camlistore.org/pkg/search" - "camlistore.org/pkg/test" + "camlistore.org/pkg/index" + "camlistore.org/pkg/index/indextest" ) -var ( - once sync.Once - mongoNotAvailable bool -) - -type IndexDeps struct { - Index *Index - - BlobSource *test.Fetcher - - // Following three needed for signing: - PublicKeyFetcher *test.Fetcher - EntityFetcher jsonsign.EntityFetcher // fetching decrypted openpgp entities - SignerBlobRef *blobref.BlobRef - - now time.Time // fake clock, nanos since epoch -} - -func (id *IndexDeps) Get(key string) string { - v, _ := id.Index.s.Get(key) - return v -} - -func (id *IndexDeps) dumpIndex(t *testing.T) { - t.Logf("Begin index dump:") - it := id.Index.s.Find("") - for it.Next() { - t.Logf(" %q = %q", it.Key(), it.Value()) - } - if err := it.Close(); err != nil { - t.Fatalf("iterator close = %v", err) - } - t.Logf("End index dump.") -} - -func (id *IndexDeps) uploadAndSignMap(m map[string]interface{}) *blobref.BlobRef { - m["camliSigner"] = id.SignerBlobRef - unsigned, err := schema.MapToCamliJson(m) - if err != nil { - panic("uploadAndSignMap: " + err.Error()) - } - sr := &jsonsign.SignRequest{ - UnsignedJson: unsigned, - Fetcher: id.PublicKeyFetcher, - EntityFetcher: id.EntityFetcher, - } - signed, err := sr.Sign() - if err != nil { - panic("problem signing: " + err.Error()) - } - tb := &test.Blob{Contents: signed} - _, err = id.Index.ReceiveBlob(tb.BlobRef(), tb.Reader()) - if err != nil { - panic(fmt.Sprintf("problem indexing blob: %v\nblob was:\n%s", err, signed)) - } - return tb.BlobRef() -} - -// NewPermanode creates (& signs) a new permanode and adds it -// to the index, returning its blobref. -func (id *IndexDeps) NewPermanode() *blobref.BlobRef { - unsigned := schema.NewUnsignedPermanode() - return id.uploadAndSignMap(unsigned) -} - -func (id *IndexDeps) advanceTime() string { - id.now = id.now.Add(1 * time.Second) - return schema.RFC3339FromTime(id.now) -} - -func (id *IndexDeps) lastTime() time.Time { - return id.now -} - -func (id *IndexDeps) SetAttribute(permaNode *blobref.BlobRef, attr, value string) *blobref.BlobRef { - m := schema.NewSetAttributeClaim(permaNode, attr, value) - m["claimDate"] = id.advanceTime() - return id.uploadAndSignMap(m) -} - -func (id *IndexDeps) UploadFile(fileName string, contents string) (fileRef, wholeRef *blobref.BlobRef) { - cb := &test.Blob{Contents: contents} - id.BlobSource.AddBlob(cb) - wholeRef = cb.BlobRef() - _, err := id.Index.ReceiveBlob(wholeRef, cb.Reader()) - if err != nil { - panic(err) - } - - m := schema.NewFileMap(fileName) - schema.PopulateParts(m, int64(len(contents)), []schema.BytesPart{ - schema.BytesPart{ - Size: uint64(len(contents)), - BlobRef: wholeRef, - }}) - fjson, err := schema.MapToCamliJson(m) - if err != nil { - panic(err) - } - fb := &test.Blob{Contents: fjson} - log.Printf("Blob is: %s", fjson) - id.BlobSource.AddBlob(fb) - fileRef = fb.BlobRef() - _, err = id.Index.ReceiveBlob(fileRef, fb.Reader()) - if err != nil { - panic(err) - } - return -} - -func NewIndexDeps(index *Index) *IndexDeps { - secretRingFile := "../jsonsign/testdata/test-secring.gpg" - pubKey := &test.Blob{Contents: `-----BEGIN PGP PUBLIC KEY BLOCK----- - -xsBNBEzgoVsBCAC/56aEJ9BNIGV9FVP+WzenTAkg12k86YqlwJVAB/VwdMlyXxvi -bCT1RVRfnYxscs14LLfcMWF3zMucw16mLlJCBSLvbZ0jn4h+/8vK5WuAdjw2YzLs -WtBcjWn3lV6tb4RJz5gtD/o1w8VWxwAnAVIWZntKAWmkcChCRgdUeWso76+plxE5 -aRYBJqdT1mctGqNEISd/WYPMgwnWXQsVi3x4z1dYu2tD9uO1dkAff12z1kyZQIBQ -rexKYRRRh9IKAayD4kgS0wdlULjBU98aeEaMz1ckuB46DX3lAYqmmTEL/Rl9cOI0 -Enpn/oOOfYFa5h0AFndZd1blMvruXfdAobjVABEBAAE= -=28/7 ------END PGP PUBLIC KEY BLOCK-----`} - - id := &IndexDeps{ - Index: index, - BlobSource: new(test.Fetcher), - PublicKeyFetcher: new(test.Fetcher), - EntityFetcher: &jsonsign.CachingEntityFetcher{ - Fetcher: &jsonsign.FileEntityFetcher{File: secretRingFile}, - }, - SignerBlobRef: pubKey.BlobRef(), - now: time.Unix(1322443956, 123456), - } - // Add dev-camput's test key public key, keyid 26F5ABDA, - // blobref sha1-ad87ca5c78bd0ce1195c46f7c98e6025abbaf007 - if id.SignerBlobRef.String() != "sha1-ad87ca5c78bd0ce1195c46f7c98e6025abbaf007" { - panic("unexpected signer blobref") - } - id.PublicKeyFetcher.AddBlob(pubKey) - id.Index.KeyFetcher = id.PublicKeyFetcher - id.Index.BlobSource = id.BlobSource - return id -} - -func checkMongoUp() { - mgw := &MongoWrapper{ - Servers: "localhost", - } - mongoNotAvailable = !mgw.TestConnection(500 * time.Millisecond) -} - -func initMongoIndex() *Index { - mgw := &MongoWrapper{ - Servers: "localhost", - Database: "camlitest", - Collection: "keys", - } - idx, err := newMongoIndex(mgw) - if err != nil { - panic(err) - } - err = idx.s.Delete("") - if err != nil { - panic(err) - } - return idx -} - -type mongoTester struct{} - -func (mongoTester) test(t *testing.T, tfn func(*testing.T, func() *Index)) { - once.Do(checkMongoUp) - if mongoNotAvailable { - err := errors.New("Not running; start a mongoDB daemon on the standard port (27017). The \"keys\" collection in the \"camlitest\" database will be used.") - t.Logf("Mongo not available locally for testing: %v", err) - return - } - tfn(t, initMongoIndex) -} - -func TestIndex_Memory(t *testing.T) { - testIndex(t, newMemoryIndex) -} - -func TestIndex_Mongo(t *testing.T) { - mongoTester{}.test(t, testIndex) -} - -func testIndex(t *testing.T, initIdx func() *Index) { - id := NewIndexDeps(initIdx()) - pn := id.NewPermanode() - t.Logf("uploaded permanode %q", pn) - br1 := id.SetAttribute(pn, "foo", "foo1") - br1Time := id.lastTime() - t.Logf("set attribute %q", br1) - br2 := id.SetAttribute(pn, "foo", "foo2") - br2Time := id.lastTime() - t.Logf("set attribute %q", br2) - rootClaim := id.SetAttribute(pn, "camliRoot", "rootval") - rootClaimTime := id.lastTime() - t.Logf("set attribute %q", rootClaim) - - id.dumpIndex(t) - - key := "signerkeyid:sha1-ad87ca5c78bd0ce1195c46f7c98e6025abbaf007" - if g, e := id.Get(key), "2931A67C26F5ABDA"; g != e { - t.Fatalf("%q = %q, want %q", key, g, e) - } - - key = "have:" + pn.String() - pnSizeStr := id.Get(key) - if pnSizeStr == "" { - t.Fatalf("missing key %q", key) - } - - key = "meta:" + pn.String() - if g, e := id.Get(key), pnSizeStr+"|application/json; camliType=permanode"; g != e { - t.Errorf("key %q = %q, want %q", key, g, e) - } - - key = "recpn|2931A67C26F5ABDA|rt7988-88-71T98:67:62.999876543Z|" + br1.String() - if g, e := id.Get(key), pn.String(); g != e { - t.Fatalf("%q = %q, want %q (permanode)", key, g, e) - } - - key = "recpn|2931A67C26F5ABDA|rt7988-88-71T98:67:61.999876543Z|" + br2.String() - if g, e := id.Get(key), pn.String(); g != e { - t.Fatalf("%q = %q, want %q (permanode)", key, g, e) - } - - // PermanodeOfSignerAttrValue - { - gotPN, err := id.Index.PermanodeOfSignerAttrValue(id.SignerBlobRef, "camliRoot", "rootval") - if err != nil { - t.Fatalf("id.Index.PermanodeOfSignerAttrValue = %v", err) - } - if gotPN.String() != pn.String() { - t.Errorf("id.Index.PermanodeOfSignerAttrValue = %q, want %q", gotPN, pn) - } - _, err = id.Index.PermanodeOfSignerAttrValue(id.SignerBlobRef, "camliRoot", "MISSING") - if err == nil { - t.Errorf("expected an error from PermanodeOfSignerAttrValue on missing value") - } - } - - // GetRecentPermanodes - { - ch := make(chan *search.Result, 10) // only expect 1 result, but 3 if buggy. - err := id.Index.GetRecentPermanodes(ch, id.SignerBlobRef, 50) - if err != nil { - t.Fatalf("GetRecentPermanodes = %v", err) - } - got := []*search.Result{} - for r := range ch { - got = append(got, r) - } - want := []*search.Result{ - &search.Result{ - BlobRef: pn, - Signer: id.SignerBlobRef, - LastModTime: 1322443959, - }, - } - if !reflect.DeepEqual(got, want) { - t.Errorf("GetRecentPermanode results differ.\n got: %v\nwant: %v", - search.Results(got), search.Results(want)) - } - } - - // GetBlobMimeType - { - mime, size, err := id.Index.GetBlobMimeType(pn) - if err != nil { - t.Errorf("GetBlobMimeType(%q) = %v", pn, err) - } else { - if e := "application/json; camliType=permanode"; mime != e { - t.Errorf("GetBlobMimeType(%q) mime = %q, want %q", pn, mime, e) - } - if size == 0 { - t.Errorf("GetBlobMimeType(%q) size is zero", pn) - } - } - _, _, err = id.Index.GetBlobMimeType(blobref.Parse("abc-123")) - if err != os.ErrNotExist { - t.Errorf("GetBlobMimeType(dummy blobref) = %v; want os.ENOENT", err) - } - } - - // GetOwnerClaims - { - claims, err := id.Index.GetOwnerClaims(pn, id.SignerBlobRef) - if err != nil { - t.Errorf("GetOwnerClaims = %v", err) - } else { - want := search.ClaimList([]*search.Claim{ - &search.Claim{ - BlobRef: br1, - Permanode: pn, - Signer: id.SignerBlobRef, - Date: br1Time.UTC(), - Type: "set-attribute", - Attr: "foo", - Value: "foo1", - }, - &search.Claim{ - BlobRef: br2, - Permanode: pn, - Signer: id.SignerBlobRef, - Date: br2Time.UTC(), - Type: "set-attribute", - Attr: "foo", - Value: "foo2", - }, - &search.Claim{ - BlobRef: rootClaim, - Permanode: pn, - Signer: id.SignerBlobRef, - Date: rootClaimTime.UTC(), - Type: "set-attribute", - Attr: "camliRoot", - Value: "rootval", - }, - }) - if !reflect.DeepEqual(claims, want) { - t.Errorf("GetOwnerClaims results differ.\n got: %v\nwant: %v", - claims, want) - } - } - } -} - -func TestPathsOfSignerTarget_Memory(t *testing.T) { - testPathsOfSignerTarget(t, newMemoryIndex) -} - -func TestPathsOfSignerTarget_Mongo(t *testing.T) { - mongoTester{}.test(t, testPathsOfSignerTarget) -} - -func testPathsOfSignerTarget(t *testing.T, initIdx func() *Index) { - id := NewIndexDeps(initIdx()) - pn := id.NewPermanode() - t.Logf("uploaded permanode %q", pn) - - claim1 := id.SetAttribute(pn, "camliPath:somedir", "targ-123") - claim2 := id.SetAttribute(pn, "camliPath:with|pipe", "targ-124") - t.Logf("made path claims %q and %q", claim1, claim2) - - id.dumpIndex(t) - - type test struct { - blobref string - want int - } - tests := []test{ - {"targ-123", 1}, - {"targ-124", 1}, - {"targ-125", 0}, - } - for _, tt := range tests { - signer := id.SignerBlobRef - paths, err := id.Index.PathsOfSignerTarget(signer, blobref.Parse(tt.blobref)) - if err != nil { - t.Fatalf("PathsOfSignerTarget(%q): %v", tt.blobref, err) - } - if len(paths) != tt.want { - t.Fatalf("PathsOfSignerTarget(%q) got %d results; want %d", - tt.blobref, len(paths), tt.want) - } - if tt.blobref == "targ-123" { - p := paths[0] - want := fmt.Sprintf( - "Path{Claim: %s, 2011-11-28T01:32:37.000123456Z; Base: %s + Suffix \"somedir\" => Target targ-123}", - claim1, pn) - if g := p.String(); g != want { - t.Errorf("claim wrong.\n got: %s\nwant: %s", g, want) - } - } - } - - path, err := id.Index.PathLookup(id.SignerBlobRef, pn, "with|pipe", time.Now()) - if err != nil { - t.Fatalf("PathLookup = %v", err) - } - if g, e := path.Target.String(), "targ-124"; g != e { - t.Errorf("PathLookup = %q; want %q", g, e) - } -} - -func TestFiles_Memory(t *testing.T) { - testFiles(t, newMemoryIndex) -} - -func TestFiles_Mongo(t *testing.T) { - mongoTester{}.test(t, testFiles) -} - -func testFiles(t *testing.T, initIdx func() *Index) { - id := NewIndexDeps(initIdx()) - fileRef, wholeRef := id.UploadFile("foo.html", "I am an html file.") - t.Logf("uploaded fileref %q, wholeRef %q", fileRef, wholeRef) - id.dumpIndex(t) - - // ExistingFileSchemas - { - key := fmt.Sprintf("wholetofile|%s|%s", wholeRef, fileRef) - if g, e := id.Get(key), "1"; g != e { - t.Fatalf("%q = %q, want %q", key, g, e) - } - - refs, err := id.Index.ExistingFileSchemas(wholeRef) - if err != nil { - t.Fatalf("ExistingFileSchemas = %v", err) - } - want := []*blobref.BlobRef{fileRef} - if !reflect.DeepEqual(refs, want) { - t.Errorf("ExistingFileSchemas got = %#v, want %#v", refs, want) - } - } - - // FileInfo - { - key := fmt.Sprintf("fileinfo|%s", fileRef) - if g, e := id.Get(key), "31|foo.html|text%2Fhtml"; g != e { - t.Fatalf("%q = %q, want %q", key, g, e) - } - - fi, err := id.Index.GetFileInfo(fileRef) - if err != nil { - t.Fatalf("GetFileInfo = %v", err) - } - if g, e := fi.Size, int64(31); g != e { - t.Errorf("Size = %d, want %d", g, e) - } - if g, e := fi.FileName, "foo.html"; g != e { - t.Errorf("FileName = %q, want %q", g, e) - } - if g, e := fi.MimeType, "text/html"; g != e { - t.Errorf("MimeType = %q, want %q", g, e) - } - } -} - func TestReverseTimeString(t *testing.T) { in := "2011-11-27T01:23:45Z" - got := reverseTimeString(in) + got := index.ExpReverseTimeString(in) want := "rt7988-88-72T98:76:54Z" if got != want { t.Fatalf("reverseTimeString = %q, want %q", got, want) } - back := unreverseTimeString(got) + back := index.ExpUnreverseTimeString(got) if back != in { t.Fatalf("unreverseTimeString = %q, want %q", back, in) } } + +func TestIndex_Memory(t *testing.T) { + indextest.Index(t, index.ExpNewMemoryIndex) +} + +func TestPathsOfSignerTarget_Memory(t *testing.T) { + indextest.PathsOfSignerTarget(t, index.ExpNewMemoryIndex) +} + +func TestFiles_Memory(t *testing.T) { + indextest.Files(t, index.ExpNewMemoryIndex) +} diff --git a/pkg/index/indextest/tests.go b/pkg/index/indextest/tests.go new file mode 100644 index 000000000..a8b672bdb --- /dev/null +++ b/pkg/index/indextest/tests.go @@ -0,0 +1,413 @@ +/* +Copyright 2011 Google Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package indextest + +import ( + "fmt" + "log" + "os" + "reflect" + "testing" + "time" + + "camlistore.org/pkg/blobref" + "camlistore.org/pkg/index" + "camlistore.org/pkg/jsonsign" + "camlistore.org/pkg/schema" + "camlistore.org/pkg/search" + "camlistore.org/pkg/test" +) + +type IndexDeps struct { + Index *index.Index + + BlobSource *test.Fetcher + + // Following three needed for signing: + PublicKeyFetcher *test.Fetcher + EntityFetcher jsonsign.EntityFetcher // fetching decrypted openpgp entities + SignerBlobRef *blobref.BlobRef + + now time.Time // fake clock, nanos since epoch +} + +func (id *IndexDeps) Get(key string) string { + v, _ := id.Index.Storage().Get(key) + return v +} + +func (id *IndexDeps) dumpIndex(t *testing.T) { + t.Logf("Begin index dump:") + it := id.Index.Storage().Find("") + for it.Next() { + t.Logf(" %q = %q", it.Key(), it.Value()) + } + if err := it.Close(); err != nil { + t.Fatalf("iterator close = %v", err) + } + t.Logf("End index dump.") +} + +func (id *IndexDeps) uploadAndSignMap(m map[string]interface{}) *blobref.BlobRef { + m["camliSigner"] = id.SignerBlobRef + unsigned, err := schema.MapToCamliJson(m) + if err != nil { + panic("uploadAndSignMap: " + err.Error()) + } + sr := &jsonsign.SignRequest{ + UnsignedJson: unsigned, + Fetcher: id.PublicKeyFetcher, + EntityFetcher: id.EntityFetcher, + } + signed, err := sr.Sign() + if err != nil { + panic("problem signing: " + err.Error()) + } + tb := &test.Blob{Contents: signed} + _, err = id.Index.ReceiveBlob(tb.BlobRef(), tb.Reader()) + if err != nil { + panic(fmt.Sprintf("problem indexing blob: %v\nblob was:\n%s", err, signed)) + } + return tb.BlobRef() +} + +// NewPermanode creates (& signs) a new permanode and adds it +// to the index, returning its blobref. +func (id *IndexDeps) NewPermanode() *blobref.BlobRef { + unsigned := schema.NewUnsignedPermanode() + return id.uploadAndSignMap(unsigned) +} + +func (id *IndexDeps) advanceTime() string { + id.now = id.now.Add(1 * time.Second) + return schema.RFC3339FromTime(id.now) +} + +func (id *IndexDeps) lastTime() time.Time { + return id.now +} + +func (id *IndexDeps) SetAttribute(permaNode *blobref.BlobRef, attr, value string) *blobref.BlobRef { + m := schema.NewSetAttributeClaim(permaNode, attr, value) + m["claimDate"] = id.advanceTime() + return id.uploadAndSignMap(m) +} + +func (id *IndexDeps) UploadFile(fileName string, contents string) (fileRef, wholeRef *blobref.BlobRef) { + cb := &test.Blob{Contents: contents} + id.BlobSource.AddBlob(cb) + wholeRef = cb.BlobRef() + _, err := id.Index.ReceiveBlob(wholeRef, cb.Reader()) + if err != nil { + panic(err) + } + + m := schema.NewFileMap(fileName) + schema.PopulateParts(m, int64(len(contents)), []schema.BytesPart{ + schema.BytesPart{ + Size: uint64(len(contents)), + BlobRef: wholeRef, + }}) + fjson, err := schema.MapToCamliJson(m) + if err != nil { + panic(err) + } + fb := &test.Blob{Contents: fjson} + log.Printf("Blob is: %s", fjson) + id.BlobSource.AddBlob(fb) + fileRef = fb.BlobRef() + _, err = id.Index.ReceiveBlob(fileRef, fb.Reader()) + if err != nil { + panic(err) + } + return +} + +func NewIndexDeps(index *index.Index) *IndexDeps { + // TODO(mpl): do better than the quick hack with the testdata symlink when things + // have settled regarding the organization of the packages. + secretRingFile := "testdata/test-secring.gpg" + pubKey := &test.Blob{Contents: `-----BEGIN PGP PUBLIC KEY BLOCK----- + +xsBNBEzgoVsBCAC/56aEJ9BNIGV9FVP+WzenTAkg12k86YqlwJVAB/VwdMlyXxvi +bCT1RVRfnYxscs14LLfcMWF3zMucw16mLlJCBSLvbZ0jn4h+/8vK5WuAdjw2YzLs +WtBcjWn3lV6tb4RJz5gtD/o1w8VWxwAnAVIWZntKAWmkcChCRgdUeWso76+plxE5 +aRYBJqdT1mctGqNEISd/WYPMgwnWXQsVi3x4z1dYu2tD9uO1dkAff12z1kyZQIBQ +rexKYRRRh9IKAayD4kgS0wdlULjBU98aeEaMz1ckuB46DX3lAYqmmTEL/Rl9cOI0 +Enpn/oOOfYFa5h0AFndZd1blMvruXfdAobjVABEBAAE= +=28/7 +-----END PGP PUBLIC KEY BLOCK-----`} + + id := &IndexDeps{ + Index: index, + BlobSource: new(test.Fetcher), + PublicKeyFetcher: new(test.Fetcher), + EntityFetcher: &jsonsign.CachingEntityFetcher{ + Fetcher: &jsonsign.FileEntityFetcher{File: secretRingFile}, + }, + SignerBlobRef: pubKey.BlobRef(), + now: time.Unix(1322443956, 123456), + } + // Add dev-camput's test key public key, keyid 26F5ABDA, + // blobref sha1-ad87ca5c78bd0ce1195c46f7c98e6025abbaf007 + if id.SignerBlobRef.String() != "sha1-ad87ca5c78bd0ce1195c46f7c98e6025abbaf007" { + panic("unexpected signer blobref") + } + id.PublicKeyFetcher.AddBlob(pubKey) + id.Index.KeyFetcher = id.PublicKeyFetcher + id.Index.BlobSource = id.BlobSource + return id +} + +func Index(t *testing.T, initIdx func() *index.Index) { + id := NewIndexDeps(initIdx()) + pn := id.NewPermanode() + t.Logf("uploaded permanode %q", pn) + br1 := id.SetAttribute(pn, "foo", "foo1") + br1Time := id.lastTime() + t.Logf("set attribute %q", br1) + br2 := id.SetAttribute(pn, "foo", "foo2") + br2Time := id.lastTime() + t.Logf("set attribute %q", br2) + rootClaim := id.SetAttribute(pn, "camliRoot", "rootval") + rootClaimTime := id.lastTime() + t.Logf("set attribute %q", rootClaim) + + id.dumpIndex(t) + + key := "signerkeyid:sha1-ad87ca5c78bd0ce1195c46f7c98e6025abbaf007" + if g, e := id.Get(key), "2931A67C26F5ABDA"; g != e { + t.Fatalf("%q = %q, want %q", key, g, e) + } + + key = "have:" + pn.String() + pnSizeStr := id.Get(key) + if pnSizeStr == "" { + t.Fatalf("missing key %q", key) + } + + key = "meta:" + pn.String() + if g, e := id.Get(key), pnSizeStr+"|application/json; camliType=permanode"; g != e { + t.Errorf("key %q = %q, want %q", key, g, e) + } + + key = "recpn|2931A67C26F5ABDA|rt7988-88-71T98:67:62.999876543Z|" + br1.String() + if g, e := id.Get(key), pn.String(); g != e { + t.Fatalf("%q = %q, want %q (permanode)", key, g, e) + } + + key = "recpn|2931A67C26F5ABDA|rt7988-88-71T98:67:61.999876543Z|" + br2.String() + if g, e := id.Get(key), pn.String(); g != e { + t.Fatalf("%q = %q, want %q (permanode)", key, g, e) + } + + // PermanodeOfSignerAttrValue + { + gotPN, err := id.Index.PermanodeOfSignerAttrValue(id.SignerBlobRef, "camliRoot", "rootval") + if err != nil { + t.Fatalf("id.Index.PermanodeOfSignerAttrValue = %v", err) + } + if gotPN.String() != pn.String() { + t.Errorf("id.Index.PermanodeOfSignerAttrValue = %q, want %q", gotPN, pn) + } + _, err = id.Index.PermanodeOfSignerAttrValue(id.SignerBlobRef, "camliRoot", "MISSING") + if err == nil { + t.Errorf("expected an error from PermanodeOfSignerAttrValue on missing value") + } + } + + // GetRecentPermanodes + { + ch := make(chan *search.Result, 10) // only expect 1 result, but 3 if buggy. + err := id.Index.GetRecentPermanodes(ch, id.SignerBlobRef, 50) + if err != nil { + t.Fatalf("GetRecentPermanodes = %v", err) + } + got := []*search.Result{} + for r := range ch { + got = append(got, r) + } + want := []*search.Result{ + &search.Result{ + BlobRef: pn, + Signer: id.SignerBlobRef, + LastModTime: 1322443959, + }, + } + if !reflect.DeepEqual(got, want) { + t.Errorf("GetRecentPermanode results differ.\n got: %v\nwant: %v", + search.Results(got), search.Results(want)) + } + } + + // GetBlobMimeType + { + mime, size, err := id.Index.GetBlobMimeType(pn) + if err != nil { + t.Errorf("GetBlobMimeType(%q) = %v", pn, err) + } else { + if e := "application/json; camliType=permanode"; mime != e { + t.Errorf("GetBlobMimeType(%q) mime = %q, want %q", pn, mime, e) + } + if size == 0 { + t.Errorf("GetBlobMimeType(%q) size is zero", pn) + } + } + _, _, err = id.Index.GetBlobMimeType(blobref.Parse("abc-123")) + if err != os.ErrNotExist { + t.Errorf("GetBlobMimeType(dummy blobref) = %v; want os.ENOENT", err) + } + } + + // GetOwnerClaims + { + claims, err := id.Index.GetOwnerClaims(pn, id.SignerBlobRef) + if err != nil { + t.Errorf("GetOwnerClaims = %v", err) + } else { + want := search.ClaimList([]*search.Claim{ + &search.Claim{ + BlobRef: br1, + Permanode: pn, + Signer: id.SignerBlobRef, + Date: br1Time.UTC(), + Type: "set-attribute", + Attr: "foo", + Value: "foo1", + }, + &search.Claim{ + BlobRef: br2, + Permanode: pn, + Signer: id.SignerBlobRef, + Date: br2Time.UTC(), + Type: "set-attribute", + Attr: "foo", + Value: "foo2", + }, + &search.Claim{ + BlobRef: rootClaim, + Permanode: pn, + Signer: id.SignerBlobRef, + Date: rootClaimTime.UTC(), + Type: "set-attribute", + Attr: "camliRoot", + Value: "rootval", + }, + }) + if !reflect.DeepEqual(claims, want) { + t.Errorf("GetOwnerClaims results differ.\n got: %v\nwant: %v", + claims, want) + } + } + } +} + +func PathsOfSignerTarget(t *testing.T, initIdx func() *index.Index) { + id := NewIndexDeps(initIdx()) + pn := id.NewPermanode() + t.Logf("uploaded permanode %q", pn) + + claim1 := id.SetAttribute(pn, "camliPath:somedir", "targ-123") + claim2 := id.SetAttribute(pn, "camliPath:with|pipe", "targ-124") + t.Logf("made path claims %q and %q", claim1, claim2) + + id.dumpIndex(t) + + type test struct { + blobref string + want int + } + tests := []test{ + {"targ-123", 1}, + {"targ-124", 1}, + {"targ-125", 0}, + } + for _, tt := range tests { + signer := id.SignerBlobRef + paths, err := id.Index.PathsOfSignerTarget(signer, blobref.Parse(tt.blobref)) + if err != nil { + t.Fatalf("PathsOfSignerTarget(%q): %v", tt.blobref, err) + } + if len(paths) != tt.want { + t.Fatalf("PathsOfSignerTarget(%q) got %d results; want %d", + tt.blobref, len(paths), tt.want) + } + if tt.blobref == "targ-123" { + p := paths[0] + want := fmt.Sprintf( + "Path{Claim: %s, 2011-11-28T01:32:37.000123456Z; Base: %s + Suffix \"somedir\" => Target targ-123}", + claim1, pn) + if g := p.String(); g != want { + t.Errorf("claim wrong.\n got: %s\nwant: %s", g, want) + } + } + } + + path, err := id.Index.PathLookup(id.SignerBlobRef, pn, "with|pipe", time.Now()) + if err != nil { + t.Fatalf("PathLookup = %v", err) + } + if g, e := path.Target.String(), "targ-124"; g != e { + t.Errorf("PathLookup = %q; want %q", g, e) + } +} + +func Files(t *testing.T, initIdx func() *index.Index) { + id := NewIndexDeps(initIdx()) + fileRef, wholeRef := id.UploadFile("foo.html", "I am an html file.") + t.Logf("uploaded fileref %q, wholeRef %q", fileRef, wholeRef) + id.dumpIndex(t) + + // ExistingFileSchemas + { + key := fmt.Sprintf("wholetofile|%s|%s", wholeRef, fileRef) + if g, e := id.Get(key), "1"; g != e { + t.Fatalf("%q = %q, want %q", key, g, e) + } + + refs, err := id.Index.ExistingFileSchemas(wholeRef) + if err != nil { + t.Fatalf("ExistingFileSchemas = %v", err) + } + want := []*blobref.BlobRef{fileRef} + if !reflect.DeepEqual(refs, want) { + t.Errorf("ExistingFileSchemas got = %#v, want %#v", refs, want) + } + } + + // FileInfo + { + key := fmt.Sprintf("fileinfo|%s", fileRef) + if g, e := id.Get(key), "31|foo.html|text%2Fhtml"; g != e { + t.Fatalf("%q = %q, want %q", key, g, e) + } + + fi, err := id.Index.GetFileInfo(fileRef) + if err != nil { + t.Fatalf("GetFileInfo = %v", err) + } + if g, e := fi.Size, int64(31); g != e { + t.Errorf("Size = %d, want %d", g, e) + } + if g, e := fi.FileName, "foo.html"; g != e { + t.Errorf("FileName = %q, want %q", g, e) + } + if g, e := fi.MimeType, "text/html"; g != e { + t.Errorf("MimeType = %q, want %q", g, e) + } + } +} diff --git a/pkg/index/memindex.go b/pkg/index/memindex.go index 4efe6a2b1..ab7c5ecc2 100644 --- a/pkg/index/memindex.go +++ b/pkg/index/memindex.go @@ -119,13 +119,13 @@ func (mk *memKeys) CommitBatch(bm BatchMutation) error { } mk.mu.Lock() defer mk.mu.Unlock() - for _, m := range b.m { - if m.delete { - if err := mk.db.Delete([]byte(m.key), nil); err != nil { + for _, m := range b.Mutations() { + if m.IsDelete() { + if err := mk.db.Delete([]byte(m.Key()), nil); err != nil { return err } } else { - if err := mk.db.Set([]byte(m.key), []byte(m.value), nil); err != nil { + if err := mk.db.Set([]byte(m.Key()), []byte(m.Value()), nil); err != nil { return err } } diff --git a/pkg/index/mongo/export_test.go b/pkg/index/mongo/export_test.go new file mode 100644 index 000000000..b293ef522 --- /dev/null +++ b/pkg/index/mongo/export_test.go @@ -0,0 +1,25 @@ +/* +Copyright 2012 Google Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mongo + +import ( + "camlistore.org/pkg/index" +) + +func NewMongoIndex(mgw *MongoWrapper) (*index.Index, error) { + return newMongoIndex(mgw) +} diff --git a/pkg/index/mongoindex.go b/pkg/index/mongo/mongoindex.go similarity index 86% rename from pkg/index/mongoindex.go rename to pkg/index/mongo/mongoindex.go index 630162250..ff7050757 100644 --- a/pkg/index/mongoindex.go +++ b/pkg/index/mongo/mongoindex.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package index +package mongo import ( "errors" @@ -25,6 +25,7 @@ import ( "time" "camlistore.org/pkg/blobserver" + "camlistore.org/pkg/index" "camlistore.org/pkg/jsonconfig" "camlistore.org/third_party/launchpad.net/mgo" @@ -75,10 +76,10 @@ func (mgw *MongoWrapper) getConnection() (*mgo.Session, error) { return session, nil } -// TODO(mpl): I'm only calling getCollection at the beginning, and +// TODO(mpl): I'm only calling getCollection at the beginning, and // keeping the collection around and reusing it everywhere, instead // of calling getCollection everytime, because that's the easiest. -// But I can easily change that. Gustavo says it does not make +// But I can easily change that. Gustavo says it does not make // much difference either way. // Brad, what do you think? func (mgw *MongoWrapper) getCollection() (*mgo.Collection, error) { @@ -97,13 +98,13 @@ func init() { blobserver.StorageConstructor(newMongoIndexFromConfig)) } -func newMongoIndex(mgw *MongoWrapper) (*Index, error) { +func newMongoIndex(mgw *MongoWrapper) (*index.Index, error) { db, err := mgw.getCollection() if err != nil { return nil, err } mongoStorage := &mongoKeys{db: db} - return New(mongoStorage), nil + return index.New(mongoStorage), nil } func newMongoIndexFromConfig(ld blobserver.Loader, config jsonconfig.Obj) (blobserver.Storage, error) { @@ -136,7 +137,7 @@ func newMongoIndexFromConfig(ld blobserver.Loader, config jsonconfig.Obj) (blobs return nil, err } if dowipe { - err = ix.s.Delete("") + err = ix.Storage().Delete("") if err != nil { return nil, err } @@ -191,7 +192,7 @@ func (mk *mongoKeys) Get(key string) (string, error) { err := q.One(&res) if err != nil { if err == mgo.NotFound { - return "", ErrNotFound + return "", index.ErrNotFound } else { return "", err } @@ -199,7 +200,7 @@ func (mk *mongoKeys) Get(key string) (string, error) { return res[mgoValue].(string), err } -func (mk *mongoKeys) Find(key string) Iterator { +func (mk *mongoKeys) Find(key string) index.Iterator { mk.mu.Lock() defer mk.mu.Unlock() // TODO(mpl): escape other special chars, or maybe replace $regex with something @@ -227,24 +228,29 @@ func (mk *mongoKeys) Delete(key string) error { return mk.db.Remove(&bson.M{mgoKey: key}) } -func (mk *mongoKeys) BeginBatch() BatchMutation { - return &batch{} +func (mk *mongoKeys) BeginBatch() index.BatchMutation { + return index.NewBatchMutation() } -func (mk *mongoKeys) CommitBatch(bm BatchMutation) error { - b, ok := bm.(*batch) +type batch interface { + Mutations() []index.Mutation +} + +func (mk *mongoKeys) CommitBatch(bm index.BatchMutation) error { + b, ok := bm.(batch) if !ok { - return errors.New("invalid batch type; not an instance returned by BeginBatch") + return errors.New("invalid batch type") } + mk.mu.Lock() defer mk.mu.Unlock() - for _, m := range b.m { - if m.delete { - if err := mk.db.Remove(bson.M{mgoKey: m.key}); err != nil { + for _, m := range b.Mutations() { + if m.IsDelete() { + if err := mk.db.Remove(bson.M{mgoKey: m.Key()}); err != nil { return err } } else { - if _, err := mk.db.Upsert(&bson.M{mgoKey: m.key}, &bson.M{mgoKey: m.key, mgoValue: m.value}); err != nil { + if _, err := mk.db.Upsert(&bson.M{mgoKey: m.Key()}, &bson.M{mgoKey: m.Key(), mgoValue: m.Value()}); err != nil { return err } } diff --git a/pkg/index/mongo/mongoindex_test.go b/pkg/index/mongo/mongoindex_test.go new file mode 100644 index 000000000..cbb02d69d --- /dev/null +++ b/pkg/index/mongo/mongoindex_test.go @@ -0,0 +1,80 @@ +/* +Copyright 2011 Google Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mongo_test + +import ( + "errors" + "sync" + "testing" + "time" + + "camlistore.org/pkg/index" + "camlistore.org/pkg/index/indextest" + "camlistore.org/pkg/index/mongo" +) + +var ( + once sync.Once + mongoNotAvailable bool +) + +func checkMongoUp() { + mgw := &mongo.MongoWrapper{ + Servers: "localhost", + } + mongoNotAvailable = !mgw.TestConnection(500 * time.Millisecond) +} + +func initMongoIndex() *index.Index { + mgw := &mongo.MongoWrapper{ + Servers: "localhost", + Database: "camlitest", + Collection: "keys", + } + idx, err := mongo.NewMongoIndex(mgw) + if err != nil { + panic(err) + } + err = idx.Storage().Delete("") + if err != nil { + panic(err) + } + return idx +} + +type mongoTester struct{} + +func (mongoTester) test(t *testing.T, tfn func(*testing.T, func() *index.Index)) { + once.Do(checkMongoUp) + if mongoNotAvailable { + err := errors.New("Not running; start a mongoDB daemon on the standard port (27017). The \"keys\" collection in the \"camlitest\" database will be used.") + t.Fatalf("Mongo not available locally for testing: %v", err) + } + tfn(t, initMongoIndex) +} + +func TestIndex_Mongo(t *testing.T) { + mongoTester{}.test(t, indextest.Index) +} + +func TestPathsOfSignerTarget_Mongo(t *testing.T) { + mongoTester{}.test(t, indextest.PathsOfSignerTarget) +} + +func TestFiles_Mongo(t *testing.T) { + mongoTester{}.test(t, indextest.Files) +} diff --git a/pkg/index/mongo/testdata b/pkg/index/mongo/testdata new file mode 120000 index 000000000..4f7c11684 --- /dev/null +++ b/pkg/index/mongo/testdata @@ -0,0 +1 @@ +../testdata \ No newline at end of file diff --git a/pkg/index/testdata b/pkg/index/testdata new file mode 120000 index 000000000..4684d1d03 --- /dev/null +++ b/pkg/index/testdata @@ -0,0 +1 @@ +../jsonsign/testdata \ No newline at end of file