Rooba
/
perkeep
mirror of https://github.com/perkeep/perkeep.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

index packages reorganization 

Change-Id: Ie91c4bb5ea02a49e59ad093ac972d84b17a046d4
This commit is contained in:
mpl 2012-02-26 13:49:03 +01:00
parent d1e1f2bcb1
commit f9d0ad487e
10 changed files with 622 additions and 480 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
pkg/index/export_test.go Normal file
View File

@ -0,0 +1,29 @@
/*
Copyright 2012 Google Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package index
func ExpReverseTimeString(s string) string {
return reverseTimeString(s)
}
func ExpUnreverseTimeString(s string) string {
return unreverseTimeString(s)
}
func ExpNewMemoryIndex() *Index {
return newMemoryIndex()
}

30
pkg/index/index.go
View File

@ -87,14 +87,40 @@ type BatchMutation interface {
Delete(key string)
}
type Mutation interface {
Key() string
Value() string
IsDelete() bool
}
type mutation struct {
key string
value string // used if !delete
delete bool // if to be deleted
}
func (m mutation) Key() string {
return m.key
}
func (m mutation) Value() string {
return m.value
}
func (m mutation) IsDelete() bool {
return m.delete
}
func NewBatchMutation() BatchMutation {
return &batch{}
}
type batch struct {
m []mutation
m []Mutation
}
func (b *batch) Mutations() []Mutation {
return b.m
}
func (b *batch) Delete(key string) {
@ -498,3 +524,5 @@ func (x *Index) GetFileInfo(fileRef *blobref.BlobRef) (*search.FileInfo, error)
}
return fi, nil
}
func (x *Index) Storage() IndexStorage { return x.s }

475
pkg/index/index_test.go
View File

@ -14,477 +14,36 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
package index
package index_test
import (
"errors"
"fmt"
"log"
"os"
"reflect"
"sync"
"testing"
"time"
"camlistore.org/pkg/blobref"
"camlistore.org/pkg/jsonsign"
"camlistore.org/pkg/schema"
"camlistore.org/pkg/search"
"camlistore.org/pkg/test"
"camlistore.org/pkg/index"
"camlistore.org/pkg/index/indextest"
)
var (
once sync.Once
mongoNotAvailable bool
)
type IndexDeps struct {
Index *Index
BlobSource *test.Fetcher
// Following three needed for signing:
PublicKeyFetcher *test.Fetcher
EntityFetcher jsonsign.EntityFetcher // fetching decrypted openpgp entities
SignerBlobRef *blobref.BlobRef
now time.Time // fake clock, nanos since epoch
}
func (id *IndexDeps) Get(key string) string {
v, _ := id.Index.s.Get(key)
return v
}
func (id *IndexDeps) dumpIndex(t *testing.T) {
t.Logf("Begin index dump:")
it := id.Index.s.Find("")
for it.Next() {
t.Logf(" %q = %q", it.Key(), it.Value())
}
if err := it.Close(); err != nil {
t.Fatalf("iterator close = %v", err)
}
t.Logf("End index dump.")
}
func (id *IndexDeps) uploadAndSignMap(m map[string]interface{}) *blobref.BlobRef {
m["camliSigner"] = id.SignerBlobRef
unsigned, err := schema.MapToCamliJson(m)
if err != nil {
panic("uploadAndSignMap: " + err.Error())
}
sr := &jsonsign.SignRequest{
UnsignedJson: unsigned,
Fetcher: id.PublicKeyFetcher,
EntityFetcher: id.EntityFetcher,
}
signed, err := sr.Sign()
if err != nil {
panic("problem signing: " + err.Error())
}
tb := &test.Blob{Contents: signed}
_, err = id.Index.ReceiveBlob(tb.BlobRef(), tb.Reader())
if err != nil {
panic(fmt.Sprintf("problem indexing blob: %v\nblob was:\n%s", err, signed))
}
return tb.BlobRef()
}
// NewPermanode creates (& signs) a new permanode and adds it
// to the index, returning its blobref.
func (id *IndexDeps) NewPermanode() *blobref.BlobRef {
unsigned := schema.NewUnsignedPermanode()
return id.uploadAndSignMap(unsigned)
}
func (id *IndexDeps) advanceTime() string {
id.now = id.now.Add(1 * time.Second)
return schema.RFC3339FromTime(id.now)
}
func (id *IndexDeps) lastTime() time.Time {
return id.now
}
func (id *IndexDeps) SetAttribute(permaNode *blobref.BlobRef, attr, value string) *blobref.BlobRef {
m := schema.NewSetAttributeClaim(permaNode, attr, value)
m["claimDate"] = id.advanceTime()
return id.uploadAndSignMap(m)
}
func (id *IndexDeps) UploadFile(fileName string, contents string) (fileRef, wholeRef *blobref.BlobRef) {
cb := &test.Blob{Contents: contents}
id.BlobSource.AddBlob(cb)
wholeRef = cb.BlobRef()
_, err := id.Index.ReceiveBlob(wholeRef, cb.Reader())
if err != nil {
panic(err)
}
m := schema.NewFileMap(fileName)
schema.PopulateParts(m, int64(len(contents)), []schema.BytesPart{
schema.BytesPart{
Size: uint64(len(contents)),
BlobRef: wholeRef,
}})
fjson, err := schema.MapToCamliJson(m)
if err != nil {
panic(err)
}
fb := &test.Blob{Contents: fjson}
log.Printf("Blob is: %s", fjson)
id.BlobSource.AddBlob(fb)
fileRef = fb.BlobRef()
_, err = id.Index.ReceiveBlob(fileRef, fb.Reader())
if err != nil {
panic(err)
}
return
}
func NewIndexDeps(index *Index) *IndexDeps {
secretRingFile := "../jsonsign/testdata/test-secring.gpg"
pubKey := &test.Blob{Contents: `-----BEGIN PGP PUBLIC KEY BLOCK-----
xsBNBEzgoVsBCAC/56aEJ9BNIGV9FVP+WzenTAkg12k86YqlwJVAB/VwdMlyXxvi
bCT1RVRfnYxscs14LLfcMWF3zMucw16mLlJCBSLvbZ0jn4h+/8vK5WuAdjw2YzLs
WtBcjWn3lV6tb4RJz5gtD/o1w8VWxwAnAVIWZntKAWmkcChCRgdUeWso76+plxE5
aRYBJqdT1mctGqNEISd/WYPMgwnWXQsVi3x4z1dYu2tD9uO1dkAff12z1kyZQIBQ
rexKYRRRh9IKAayD4kgS0wdlULjBU98aeEaMz1ckuB46DX3lAYqmmTEL/Rl9cOI0
Enpn/oOOfYFa5h0AFndZd1blMvruXfdAobjVABEBAAE=
=28/7
-----END PGP PUBLIC KEY BLOCK-----`}
id := &IndexDeps{
Index: index,
BlobSource: new(test.Fetcher),
PublicKeyFetcher: new(test.Fetcher),
EntityFetcher: &jsonsign.CachingEntityFetcher{
Fetcher: &jsonsign.FileEntityFetcher{File: secretRingFile},
},
SignerBlobRef: pubKey.BlobRef(),
now: time.Unix(1322443956, 123456),
}
// Add dev-camput's test key public key, keyid 26F5ABDA,
// blobref sha1-ad87ca5c78bd0ce1195c46f7c98e6025abbaf007
if id.SignerBlobRef.String() != "sha1-ad87ca5c78bd0ce1195c46f7c98e6025abbaf007" {
panic("unexpected signer blobref")
}
id.PublicKeyFetcher.AddBlob(pubKey)
id.Index.KeyFetcher = id.PublicKeyFetcher
id.Index.BlobSource = id.BlobSource
return id
}
func checkMongoUp() {
mgw := &MongoWrapper{
Servers: "localhost",
}
mongoNotAvailable = !mgw.TestConnection(500 * time.Millisecond)
}
func initMongoIndex() *Index {
mgw := &MongoWrapper{
Servers: "localhost",
Database: "camlitest",
Collection: "keys",
}
idx, err := newMongoIndex(mgw)
if err != nil {
panic(err)
}
err = idx.s.Delete("")
if err != nil {
panic(err)
}
return idx
}
type mongoTester struct{}
func (mongoTester) test(t *testing.T, tfn func(*testing.T, func() *Index)) {
once.Do(checkMongoUp)
if mongoNotAvailable {
err := errors.New("Not running; start a mongoDB daemon on the standard port (27017). The \"keys\" collection in the \"camlitest\" database will be used.")
t.Logf("Mongo not available locally for testing: %v", err)
return
}
tfn(t, initMongoIndex)
}
func TestIndex_Memory(t *testing.T) {
testIndex(t, newMemoryIndex)
}
func TestIndex_Mongo(t *testing.T) {
mongoTester{}.test(t, testIndex)
}
func testIndex(t *testing.T, initIdx func() *Index) {
id := NewIndexDeps(initIdx())
pn := id.NewPermanode()
t.Logf("uploaded permanode %q", pn)
br1 := id.SetAttribute(pn, "foo", "foo1")
br1Time := id.lastTime()
t.Logf("set attribute %q", br1)
br2 := id.SetAttribute(pn, "foo", "foo2")
br2Time := id.lastTime()
t.Logf("set attribute %q", br2)
rootClaim := id.SetAttribute(pn, "camliRoot", "rootval")
rootClaimTime := id.lastTime()
t.Logf("set attribute %q", rootClaim)
id.dumpIndex(t)
key := "signerkeyid:sha1-ad87ca5c78bd0ce1195c46f7c98e6025abbaf007"
if g, e := id.Get(key), "2931A67C26F5ABDA"; g != e {
t.Fatalf("%q = %q, want %q", key, g, e)
}
key = "have:" + pn.String()
pnSizeStr := id.Get(key)
if pnSizeStr == "" {
t.Fatalf("missing key %q", key)
}
key = "meta:" + pn.String()
if g, e := id.Get(key), pnSizeStr+"|application/json; camliType=permanode"; g != e {
t.Errorf("key %q = %q, want %q", key, g, e)
}
key = "recpn|2931A67C26F5ABDA|rt7988-88-71T98:67:62.999876543Z|" + br1.String()
if g, e := id.Get(key), pn.String(); g != e {
t.Fatalf("%q = %q, want %q (permanode)", key, g, e)
}
key = "recpn|2931A67C26F5ABDA|rt7988-88-71T98:67:61.999876543Z|" + br2.String()
if g, e := id.Get(key), pn.String(); g != e {
t.Fatalf("%q = %q, want %q (permanode)", key, g, e)
}
// PermanodeOfSignerAttrValue
{
gotPN, err := id.Index.PermanodeOfSignerAttrValue(id.SignerBlobRef, "camliRoot", "rootval")
if err != nil {
t.Fatalf("id.Index.PermanodeOfSignerAttrValue = %v", err)
}
if gotPN.String() != pn.String() {
t.Errorf("id.Index.PermanodeOfSignerAttrValue = %q, want %q", gotPN, pn)
}
_, err = id.Index.PermanodeOfSignerAttrValue(id.SignerBlobRef, "camliRoot", "MISSING")
if err == nil {
t.Errorf("expected an error from PermanodeOfSignerAttrValue on missing value")
}
}
// GetRecentPermanodes
{
ch := make(chan *search.Result, 10) // only expect 1 result, but 3 if buggy.
err := id.Index.GetRecentPermanodes(ch, id.SignerBlobRef, 50)
if err != nil {
t.Fatalf("GetRecentPermanodes = %v", err)
}
got := []*search.Result{}
for r := range ch {
got = append(got, r)
}
want := []*search.Result{
&search.Result{
BlobRef: pn,
Signer: id.SignerBlobRef,
LastModTime: 1322443959,
},
}
if !reflect.DeepEqual(got, want) {
t.Errorf("GetRecentPermanode results differ.\n got: %v\nwant: %v",
search.Results(got), search.Results(want))
}
}
// GetBlobMimeType
{
mime, size, err := id.Index.GetBlobMimeType(pn)
if err != nil {
t.Errorf("GetBlobMimeType(%q) = %v", pn, err)
} else {
if e := "application/json; camliType=permanode"; mime != e {
t.Errorf("GetBlobMimeType(%q) mime = %q, want %q", pn, mime, e)
}
if size == 0 {
t.Errorf("GetBlobMimeType(%q) size is zero", pn)
}
}
_, _, err = id.Index.GetBlobMimeType(blobref.Parse("abc-123"))
if err != os.ErrNotExist {
t.Errorf("GetBlobMimeType(dummy blobref) = %v; want os.ENOENT", err)
}
}
// GetOwnerClaims
{
claims, err := id.Index.GetOwnerClaims(pn, id.SignerBlobRef)
if err != nil {
t.Errorf("GetOwnerClaims = %v", err)
} else {
want := search.ClaimList([]*search.Claim{
&search.Claim{
BlobRef: br1,
Permanode: pn,
Signer: id.SignerBlobRef,
Date: br1Time.UTC(),
Type: "set-attribute",
Attr: "foo",
Value: "foo1",
},
&search.Claim{
BlobRef: br2,
Permanode: pn,
Signer: id.SignerBlobRef,
Date: br2Time.UTC(),
Type: "set-attribute",
Attr: "foo",
Value: "foo2",
},
&search.Claim{
BlobRef: rootClaim,
Permanode: pn,
Signer: id.SignerBlobRef,
Date: rootClaimTime.UTC(),
Type: "set-attribute",
Attr: "camliRoot",
Value: "rootval",
},
})
if !reflect.DeepEqual(claims, want) {
t.Errorf("GetOwnerClaims results differ.\n got: %v\nwant: %v",
claims, want)
}
}
}
}
func TestPathsOfSignerTarget_Memory(t *testing.T) {
testPathsOfSignerTarget(t, newMemoryIndex)
}
func TestPathsOfSignerTarget_Mongo(t *testing.T) {
mongoTester{}.test(t, testPathsOfSignerTarget)
}
func testPathsOfSignerTarget(t *testing.T, initIdx func() *Index) {
id := NewIndexDeps(initIdx())
pn := id.NewPermanode()
t.Logf("uploaded permanode %q", pn)
claim1 := id.SetAttribute(pn, "camliPath:somedir", "targ-123")
claim2 := id.SetAttribute(pn, "camliPath:with|pipe", "targ-124")
t.Logf("made path claims %q and %q", claim1, claim2)
id.dumpIndex(t)
type test struct {
blobref string
want int
}
tests := []test{
{"targ-123", 1},
{"targ-124", 1},
{"targ-125", 0},
}
for _, tt := range tests {
signer := id.SignerBlobRef
paths, err := id.Index.PathsOfSignerTarget(signer, blobref.Parse(tt.blobref))
if err != nil {
t.Fatalf("PathsOfSignerTarget(%q): %v", tt.blobref, err)
}
if len(paths) != tt.want {
t.Fatalf("PathsOfSignerTarget(%q) got %d results; want %d",
tt.blobref, len(paths), tt.want)
}
if tt.blobref == "targ-123" {
p := paths[0]
want := fmt.Sprintf(
"Path{Claim: %s, 2011-11-28T01:32:37.000123456Z; Base: %s + Suffix \"somedir\" => Target targ-123}",
claim1, pn)
if g := p.String(); g != want {
t.Errorf("claim wrong.\n got: %s\nwant: %s", g, want)
}
}
}
path, err := id.Index.PathLookup(id.SignerBlobRef, pn, "with|pipe", time.Now())
if err != nil {
t.Fatalf("PathLookup = %v", err)
}
if g, e := path.Target.String(), "targ-124"; g != e {
t.Errorf("PathLookup = %q; want %q", g, e)
}
}
func TestFiles_Memory(t *testing.T) {
testFiles(t, newMemoryIndex)
}
func TestFiles_Mongo(t *testing.T) {
mongoTester{}.test(t, testFiles)
}
func testFiles(t *testing.T, initIdx func() *Index) {
id := NewIndexDeps(initIdx())
fileRef, wholeRef := id.UploadFile("foo.html", "<html>I am an html file.</html>")
t.Logf("uploaded fileref %q, wholeRef %q", fileRef, wholeRef)
id.dumpIndex(t)
// ExistingFileSchemas
{
key := fmt.Sprintf("wholetofile|%s|%s", wholeRef, fileRef)
if g, e := id.Get(key), "1"; g != e {
t.Fatalf("%q = %q, want %q", key, g, e)
}
refs, err := id.Index.ExistingFileSchemas(wholeRef)
if err != nil {
t.Fatalf("ExistingFileSchemas = %v", err)
}
want := []*blobref.BlobRef{fileRef}
if !reflect.DeepEqual(refs, want) {
t.Errorf("ExistingFileSchemas got = %#v, want %#v", refs, want)
}
}
// FileInfo
{
key := fmt.Sprintf("fileinfo|%s", fileRef)
if g, e := id.Get(key), "31|foo.html|text%2Fhtml"; g != e {
t.Fatalf("%q = %q, want %q", key, g, e)
}
fi, err := id.Index.GetFileInfo(fileRef)
if err != nil {
t.Fatalf("GetFileInfo = %v", err)
}
if g, e := fi.Size, int64(31); g != e {
t.Errorf("Size = %d, want %d", g, e)
}
if g, e := fi.FileName, "foo.html"; g != e {
t.Errorf("FileName = %q, want %q", g, e)
}
if g, e := fi.MimeType, "text/html"; g != e {
t.Errorf("MimeType = %q, want %q", g, e)
}
}
}
func TestReverseTimeString(t *testing.T) {
in := "2011-11-27T01:23:45Z"
got := reverseTimeString(in)
got := index.ExpReverseTimeString(in)
want := "rt7988-88-72T98:76:54Z"
if got != want {
t.Fatalf("reverseTimeString = %q, want %q", got, want)
}
back := unreverseTimeString(got)
back := index.ExpUnreverseTimeString(got)
if back != in {
t.Fatalf("unreverseTimeString = %q, want %q", back, in)
}
}
func TestIndex_Memory(t *testing.T) {
indextest.Index(t, index.ExpNewMemoryIndex)
}
func TestPathsOfSignerTarget_Memory(t *testing.T) {
indextest.PathsOfSignerTarget(t, index.ExpNewMemoryIndex)
}
func TestFiles_Memory(t *testing.T) {
indextest.Files(t, index.ExpNewMemoryIndex)
}

413
pkg/index/indextest/tests.go Normal file
View File

@ -0,0 +1,413 @@
/*
Copyright 2011 Google Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package indextest
import (
"fmt"
"log"
"os"
"reflect"
"testing"
"time"
"camlistore.org/pkg/blobref"
"camlistore.org/pkg/index"
"camlistore.org/pkg/jsonsign"
"camlistore.org/pkg/schema"
"camlistore.org/pkg/search"
"camlistore.org/pkg/test"
)
type IndexDeps struct {
Index *index.Index
BlobSource *test.Fetcher
// Following three needed for signing:
PublicKeyFetcher *test.Fetcher
EntityFetcher jsonsign.EntityFetcher // fetching decrypted openpgp entities
SignerBlobRef *blobref.BlobRef
now time.Time // fake clock, nanos since epoch
}
func (id *IndexDeps) Get(key string) string {
v, _ := id.Index.Storage().Get(key)
return v
}
func (id *IndexDeps) dumpIndex(t *testing.T) {
t.Logf("Begin index dump:")
it := id.Index.Storage().Find("")
for it.Next() {
t.Logf(" %q = %q", it.Key(), it.Value())
}
if err := it.Close(); err != nil {
t.Fatalf("iterator close = %v", err)
}
t.Logf("End index dump.")
}
func (id *IndexDeps) uploadAndSignMap(m map[string]interface{}) *blobref.BlobRef {
m["camliSigner"] = id.SignerBlobRef
unsigned, err := schema.MapToCamliJson(m)
if err != nil {
panic("uploadAndSignMap: " + err.Error())
}
sr := &jsonsign.SignRequest{
UnsignedJson: unsigned,
Fetcher: id.PublicKeyFetcher,
EntityFetcher: id.EntityFetcher,
}
signed, err := sr.Sign()
if err != nil {
panic("problem signing: " + err.Error())
}
tb := &test.Blob{Contents: signed}
_, err = id.Index.ReceiveBlob(tb.BlobRef(), tb.Reader())
if err != nil {
panic(fmt.Sprintf("problem indexing blob: %v\nblob was:\n%s", err, signed))
}
return tb.BlobRef()
}
// NewPermanode creates (& signs) a new permanode and adds it
// to the index, returning its blobref.
func (id *IndexDeps) NewPermanode() *blobref.BlobRef {
unsigned := schema.NewUnsignedPermanode()
return id.uploadAndSignMap(unsigned)
}
func (id *IndexDeps) advanceTime() string {
id.now = id.now.Add(1 * time.Second)
return schema.RFC3339FromTime(id.now)
}
func (id *IndexDeps) lastTime() time.Time {
return id.now
}
func (id *IndexDeps) SetAttribute(permaNode *blobref.BlobRef, attr, value string) *blobref.BlobRef {
m := schema.NewSetAttributeClaim(permaNode, attr, value)
m["claimDate"] = id.advanceTime()
return id.uploadAndSignMap(m)
}
func (id *IndexDeps) UploadFile(fileName string, contents string) (fileRef, wholeRef *blobref.BlobRef) {
cb := &test.Blob{Contents: contents}
id.BlobSource.AddBlob(cb)
wholeRef = cb.BlobRef()
_, err := id.Index.ReceiveBlob(wholeRef, cb.Reader())
if err != nil {
panic(err)
}
m := schema.NewFileMap(fileName)
schema.PopulateParts(m, int64(len(contents)), []schema.BytesPart{
schema.BytesPart{
Size: uint64(len(contents)),
BlobRef: wholeRef,
}})
fjson, err := schema.MapToCamliJson(m)
if err != nil {
panic(err)
}
fb := &test.Blob{Contents: fjson}
log.Printf("Blob is: %s", fjson)
id.BlobSource.AddBlob(fb)
fileRef = fb.BlobRef()
_, err = id.Index.ReceiveBlob(fileRef, fb.Reader())
if err != nil {
panic(err)
}
return
}
func NewIndexDeps(index *index.Index) *IndexDeps {
// TODO(mpl): do better than the quick hack with the testdata symlink when things
// have settled regarding the organization of the packages.
secretRingFile := "testdata/test-secring.gpg"
pubKey := &test.Blob{Contents: `-----BEGIN PGP PUBLIC KEY BLOCK-----
xsBNBEzgoVsBCAC/56aEJ9BNIGV9FVP+WzenTAkg12k86YqlwJVAB/VwdMlyXxvi
bCT1RVRfnYxscs14LLfcMWF3zMucw16mLlJCBSLvbZ0jn4h+/8vK5WuAdjw2YzLs
WtBcjWn3lV6tb4RJz5gtD/o1w8VWxwAnAVIWZntKAWmkcChCRgdUeWso76+plxE5
aRYBJqdT1mctGqNEISd/WYPMgwnWXQsVi3x4z1dYu2tD9uO1dkAff12z1kyZQIBQ
rexKYRRRh9IKAayD4kgS0wdlULjBU98aeEaMz1ckuB46DX3lAYqmmTEL/Rl9cOI0
Enpn/oOOfYFa5h0AFndZd1blMvruXfdAobjVABEBAAE=
=28/7
-----END PGP PUBLIC KEY BLOCK-----`}
id := &IndexDeps{
Index: index,
BlobSource: new(test.Fetcher),
PublicKeyFetcher: new(test.Fetcher),
EntityFetcher: &jsonsign.CachingEntityFetcher{
Fetcher: &jsonsign.FileEntityFetcher{File: secretRingFile},
},
SignerBlobRef: pubKey.BlobRef(),
now: time.Unix(1322443956, 123456),
}
// Add dev-camput's test key public key, keyid 26F5ABDA,
// blobref sha1-ad87ca5c78bd0ce1195c46f7c98e6025abbaf007
if id.SignerBlobRef.String() != "sha1-ad87ca5c78bd0ce1195c46f7c98e6025abbaf007" {
panic("unexpected signer blobref")
}
id.PublicKeyFetcher.AddBlob(pubKey)
id.Index.KeyFetcher = id.PublicKeyFetcher
id.Index.BlobSource = id.BlobSource
return id
}
func Index(t *testing.T, initIdx func() *index.Index) {
id := NewIndexDeps(initIdx())
pn := id.NewPermanode()
t.Logf("uploaded permanode %q", pn)
br1 := id.SetAttribute(pn, "foo", "foo1")
br1Time := id.lastTime()
t.Logf("set attribute %q", br1)
br2 := id.SetAttribute(pn, "foo", "foo2")
br2Time := id.lastTime()
t.Logf("set attribute %q", br2)
rootClaim := id.SetAttribute(pn, "camliRoot", "rootval")
rootClaimTime := id.lastTime()
t.Logf("set attribute %q", rootClaim)
id.dumpIndex(t)
key := "signerkeyid:sha1-ad87ca5c78bd0ce1195c46f7c98e6025abbaf007"
if g, e := id.Get(key), "2931A67C26F5ABDA"; g != e {
t.Fatalf("%q = %q, want %q", key, g, e)
}
key = "have:" + pn.String()
pnSizeStr := id.Get(key)
if pnSizeStr == "" {
t.Fatalf("missing key %q", key)
}
key = "meta:" + pn.String()
if g, e := id.Get(key), pnSizeStr+"|application/json; camliType=permanode"; g != e {
t.Errorf("key %q = %q, want %q", key, g, e)
}
key = "recpn|2931A67C26F5ABDA|rt7988-88-71T98:67:62.999876543Z|" + br1.String()
if g, e := id.Get(key), pn.String(); g != e {
t.Fatalf("%q = %q, want %q (permanode)", key, g, e)
}
key = "recpn|2931A67C26F5ABDA|rt7988-88-71T98:67:61.999876543Z|" + br2.String()
if g, e := id.Get(key), pn.String(); g != e {
t.Fatalf("%q = %q, want %q (permanode)", key, g, e)
}
// PermanodeOfSignerAttrValue
{
gotPN, err := id.Index.PermanodeOfSignerAttrValue(id.SignerBlobRef, "camliRoot", "rootval")
if err != nil {
t.Fatalf("id.Index.PermanodeOfSignerAttrValue = %v", err)
}
if gotPN.String() != pn.String() {
t.Errorf("id.Index.PermanodeOfSignerAttrValue = %q, want %q", gotPN, pn)
}
_, err = id.Index.PermanodeOfSignerAttrValue(id.SignerBlobRef, "camliRoot", "MISSING")
if err == nil {
t.Errorf("expected an error from PermanodeOfSignerAttrValue on missing value")
}
}
// GetRecentPermanodes
{
ch := make(chan *search.Result, 10) // only expect 1 result, but 3 if buggy.
err := id.Index.GetRecentPermanodes(ch, id.SignerBlobRef, 50)
if err != nil {
t.Fatalf("GetRecentPermanodes = %v", err)
}
got := []*search.Result{}
for r := range ch {
got = append(got, r)
}
want := []*search.Result{
&search.Result{
BlobRef: pn,
Signer: id.SignerBlobRef,
LastModTime: 1322443959,
},
}
if !reflect.DeepEqual(got, want) {
t.Errorf("GetRecentPermanode results differ.\n got: %v\nwant: %v",
search.Results(got), search.Results(want))
}
}
// GetBlobMimeType
{
mime, size, err := id.Index.GetBlobMimeType(pn)
if err != nil {
t.Errorf("GetBlobMimeType(%q) = %v", pn, err)
} else {
if e := "application/json; camliType=permanode"; mime != e {
t.Errorf("GetBlobMimeType(%q) mime = %q, want %q", pn, mime, e)
}
if size == 0 {
t.Errorf("GetBlobMimeType(%q) size is zero", pn)
}
}
_, _, err = id.Index.GetBlobMimeType(blobref.Parse("abc-123"))
if err != os.ErrNotExist {
t.Errorf("GetBlobMimeType(dummy blobref) = %v; want os.ENOENT", err)
}
}
// GetOwnerClaims
{
claims, err := id.Index.GetOwnerClaims(pn, id.SignerBlobRef)
if err != nil {
t.Errorf("GetOwnerClaims = %v", err)
} else {
want := search.ClaimList([]*search.Claim{
&search.Claim{
BlobRef: br1,
Permanode: pn,
Signer: id.SignerBlobRef,
Date: br1Time.UTC(),
Type: "set-attribute",
Attr: "foo",
Value: "foo1",
},
&search.Claim{
BlobRef: br2,
Permanode: pn,
Signer: id.SignerBlobRef,
Date: br2Time.UTC(),
Type: "set-attribute",
Attr: "foo",
Value: "foo2",
},
&search.Claim{
BlobRef: rootClaim,
Permanode: pn,
Signer: id.SignerBlobRef,
Date: rootClaimTime.UTC(),
Type: "set-attribute",
Attr: "camliRoot",
Value: "rootval",
},
})
if !reflect.DeepEqual(claims, want) {
t.Errorf("GetOwnerClaims results differ.\n got: %v\nwant: %v",
claims, want)
}
}
}
}
func PathsOfSignerTarget(t *testing.T, initIdx func() *index.Index) {
id := NewIndexDeps(initIdx())
pn := id.NewPermanode()
t.Logf("uploaded permanode %q", pn)
claim1 := id.SetAttribute(pn, "camliPath:somedir", "targ-123")
claim2 := id.SetAttribute(pn, "camliPath:with|pipe", "targ-124")
t.Logf("made path claims %q and %q", claim1, claim2)
id.dumpIndex(t)
type test struct {
blobref string
want int
}
tests := []test{
{"targ-123", 1},
{"targ-124", 1},
{"targ-125", 0},
}
for _, tt := range tests {
signer := id.SignerBlobRef
paths, err := id.Index.PathsOfSignerTarget(signer, blobref.Parse(tt.blobref))
if err != nil {
t.Fatalf("PathsOfSignerTarget(%q): %v", tt.blobref, err)
}
if len(paths) != tt.want {
t.Fatalf("PathsOfSignerTarget(%q) got %d results; want %d",
tt.blobref, len(paths), tt.want)
}
if tt.blobref == "targ-123" {
p := paths[0]
want := fmt.Sprintf(
"Path{Claim: %s, 2011-11-28T01:32:37.000123456Z; Base: %s + Suffix \"somedir\" => Target targ-123}",
claim1, pn)
if g := p.String(); g != want {
t.Errorf("claim wrong.\n got: %s\nwant: %s", g, want)
}
}
}
path, err := id.Index.PathLookup(id.SignerBlobRef, pn, "with|pipe", time.Now())
if err != nil {
t.Fatalf("PathLookup = %v", err)
}
if g, e := path.Target.String(), "targ-124"; g != e {
t.Errorf("PathLookup = %q; want %q", g, e)
}
}
func Files(t *testing.T, initIdx func() *index.Index) {
id := NewIndexDeps(initIdx())
fileRef, wholeRef := id.UploadFile("foo.html", "<html>I am an html file.</html>")
t.Logf("uploaded fileref %q, wholeRef %q", fileRef, wholeRef)
id.dumpIndex(t)
// ExistingFileSchemas
{
key := fmt.Sprintf("wholetofile|%s|%s", wholeRef, fileRef)
if g, e := id.Get(key), "1"; g != e {
t.Fatalf("%q = %q, want %q", key, g, e)
}
refs, err := id.Index.ExistingFileSchemas(wholeRef)
if err != nil {
t.Fatalf("ExistingFileSchemas = %v", err)
}
want := []*blobref.BlobRef{fileRef}
if !reflect.DeepEqual(refs, want) {
t.Errorf("ExistingFileSchemas got = %#v, want %#v", refs, want)
}
}
// FileInfo
{
key := fmt.Sprintf("fileinfo|%s", fileRef)
if g, e := id.Get(key), "31|foo.html|text%2Fhtml"; g != e {
t.Fatalf("%q = %q, want %q", key, g, e)
}
fi, err := id.Index.GetFileInfo(fileRef)
if err != nil {
t.Fatalf("GetFileInfo = %v", err)
}
if g, e := fi.Size, int64(31); g != e {
t.Errorf("Size = %d, want %d", g, e)
}
if g, e := fi.FileName, "foo.html"; g != e {
t.Errorf("FileName = %q, want %q", g, e)
}
if g, e := fi.MimeType, "text/html"; g != e {
t.Errorf("MimeType = %q, want %q", g, e)
}
}
}

8
pkg/index/memindex.go
View File

@ -119,13 +119,13 @@ func (mk *memKeys) CommitBatch(bm BatchMutation) error {
}
mk.mu.Lock()
defer mk.mu.Unlock()
for _, m := range b.m {
if m.delete {
if err := mk.db.Delete([]byte(m.key), nil); err != nil {
for _, m := range b.Mutations() {
if m.IsDelete() {
if err := mk.db.Delete([]byte(m.Key()), nil); err != nil {
return err
}
} else {
if err := mk.db.Set([]byte(m.key), []byte(m.value), nil); err != nil {
if err := mk.db.Set([]byte(m.Key()), []byte(m.Value()), nil); err != nil {
return err
}
}

25
pkg/index/mongo/export_test.go Normal file
View File

@ -0,0 +1,25 @@
/*
Copyright 2012 Google Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package mongo
import (
"camlistore.org/pkg/index"
)
func NewMongoIndex(mgw *MongoWrapper) (*index.Index, error) {
return newMongoIndex(mgw)
}

40
pkg/index/mongoindex.go → pkg/index/mongo/mongoindex.go
View File

@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
package index
package mongo
import (
"errors"
@ -25,6 +25,7 @@ import (
"time"
"camlistore.org/pkg/blobserver"
"camlistore.org/pkg/index"
"camlistore.org/pkg/jsonconfig"
"camlistore.org/third_party/launchpad.net/mgo"
@ -75,10 +76,10 @@ func (mgw *MongoWrapper) getConnection() (*mgo.Session, error) {
return session, nil
}
// TODO(mpl): I'm only calling getCollection at the beginning, and
// TODO(mpl): I'm only calling getCollection at the beginning, and
// keeping the collection around and reusing it everywhere, instead
// of calling getCollection everytime, because that's the easiest.
// But I can easily change that. Gustavo says it does not make
// But I can easily change that. Gustavo says it does not make
// much difference either way.
// Brad, what do you think?
func (mgw *MongoWrapper) getCollection() (*mgo.Collection, error) {
@ -97,13 +98,13 @@ func init() {
blobserver.StorageConstructor(newMongoIndexFromConfig))
}
func newMongoIndex(mgw *MongoWrapper) (*Index, error) {
func newMongoIndex(mgw *MongoWrapper) (*index.Index, error) {
db, err := mgw.getCollection()
if err != nil {
return nil, err
}
mongoStorage := &mongoKeys{db: db}
return New(mongoStorage), nil
return index.New(mongoStorage), nil
}
func newMongoIndexFromConfig(ld blobserver.Loader, config jsonconfig.Obj) (blobserver.Storage, error) {
@ -136,7 +137,7 @@ func newMongoIndexFromConfig(ld blobserver.Loader, config jsonconfig.Obj) (blobs
return nil, err
}
if dowipe {
err = ix.s.Delete("")
err = ix.Storage().Delete("")
if err != nil {
return nil, err
}
@ -191,7 +192,7 @@ func (mk *mongoKeys) Get(key string) (string, error) {
err := q.One(&res)
if err != nil {
if err == mgo.NotFound {
return "", ErrNotFound
return "", index.ErrNotFound
} else {
return "", err
}
@ -199,7 +200,7 @@ func (mk *mongoKeys) Get(key string) (string, error) {
return res[mgoValue].(string), err
}
func (mk *mongoKeys) Find(key string) Iterator {
func (mk *mongoKeys) Find(key string) index.Iterator {
mk.mu.Lock()
defer mk.mu.Unlock()
// TODO(mpl): escape other special chars, or maybe replace $regex with something
@ -227,24 +228,29 @@ func (mk *mongoKeys) Delete(key string) error {
return mk.db.Remove(&bson.M{mgoKey: key})
}
func (mk *mongoKeys) BeginBatch() BatchMutation {
return &batch{}
func (mk *mongoKeys) BeginBatch() index.BatchMutation {
return index.NewBatchMutation()
}
func (mk *mongoKeys) CommitBatch(bm BatchMutation) error {
b, ok := bm.(*batch)
type batch interface {
Mutations() []index.Mutation
}
func (mk *mongoKeys) CommitBatch(bm index.BatchMutation) error {
b, ok := bm.(batch)
if !ok {
return errors.New("invalid batch type; not an instance returned by BeginBatch")
return errors.New("invalid batch type")
}
mk.mu.Lock()
defer mk.mu.Unlock()
for _, m := range b.m {
if m.delete {
if err := mk.db.Remove(bson.M{mgoKey: m.key}); err != nil {
for _, m := range b.Mutations() {
if m.IsDelete() {
if err := mk.db.Remove(bson.M{mgoKey: m.Key()}); err != nil {
return err
}
} else {
if _, err := mk.db.Upsert(&bson.M{mgoKey: m.key}, &bson.M{mgoKey: m.key, mgoValue: m.value}); err != nil {
if _, err := mk.db.Upsert(&bson.M{mgoKey: m.Key()}, &bson.M{mgoKey: m.Key(), mgoValue: m.Value()}); err != nil {
return err
}
}

80
pkg/index/mongo/mongoindex_test.go Normal file
View File

@ -0,0 +1,80 @@
/*
Copyright 2011 Google Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package mongo_test
import (
"errors"
"sync"
"testing"
"time"
"camlistore.org/pkg/index"
"camlistore.org/pkg/index/indextest"
"camlistore.org/pkg/index/mongo"
)
var (
once sync.Once
mongoNotAvailable bool
)
func checkMongoUp() {
mgw := &mongo.MongoWrapper{
Servers: "localhost",
}
mongoNotAvailable = !mgw.TestConnection(500 * time.Millisecond)
}
func initMongoIndex() *index.Index {
mgw := &mongo.MongoWrapper{
Servers: "localhost",
Database: "camlitest",
Collection: "keys",
}
idx, err := mongo.NewMongoIndex(mgw)
if err != nil {
panic(err)
}
err = idx.Storage().Delete("")
if err != nil {
panic(err)
}
return idx
}
type mongoTester struct{}
func (mongoTester) test(t *testing.T, tfn func(*testing.T, func() *index.Index)) {
once.Do(checkMongoUp)
if mongoNotAvailable {
err := errors.New("Not running; start a mongoDB daemon on the standard port (27017). The \"keys\" collection in the \"camlitest\" database will be used.")
t.Fatalf("Mongo not available locally for testing: %v", err)
}
tfn(t, initMongoIndex)
}
func TestIndex_Mongo(t *testing.T) {
mongoTester{}.test(t, indextest.Index)
}
func TestPathsOfSignerTarget_Mongo(t *testing.T) {
mongoTester{}.test(t, indextest.PathsOfSignerTarget)
}
func TestFiles_Mongo(t *testing.T) {
mongoTester{}.test(t, indextest.Files)
}

1
pkg/index/mongo/testdata Symbolic link
View File

@ -0,0 +1 @@
../testdata

1
pkg/index/testdata Symbolic link
View File

@ -0,0 +1 @@
../jsonsign/testdata