From eec23f4a21c8c981b9fcb5df81757f70be87fd4b Mon Sep 17 00:00:00 2001 From: mpl Date: Tue, 4 Aug 2015 19:09:52 +0200 Subject: [PATCH] serverinit: use user-defined kv type everywhere, otherwise leveldb. In a few places, cznic/kv use was hardcoded, instead of picking the user set kv type, if any. This change makes sure we use the user-defined one, if set, syndtr/leveldb otherwise. Relatedly, the low-level config generated for diskpacked now uses that same principle for its internal index. Also, if a diskpacked is configured without an index, it now defaults to syndtr/leveldb. Please note that this patch DOES NOT change the default kv type in the automatically generated high-level config. Related: #632 Change-Id: I55dbd385230878e92776e6b83d34c36fd1fa9382 --- pkg/blobserver/diskpacked/diskpacked.go | 2 +- pkg/blobserver/diskpacked/reindex.go | 2 + pkg/serverinit/genconfig.go | 64 ++++++++++++++------ pkg/serverinit/testdata/diskpacked-want.json | 10 ++- pkg/serverinit/testdata/leveldb-want.json | 8 +-- pkg/serverinit/testdata/memindex-want.json | 8 +-- pkg/serverinit/testdata/mongo-want.json | 8 +-- pkg/serverinit/testdata/sqlite-want.json | 4 +- pkg/sorted/kv.go | 2 + 9 files changed, 75 insertions(+), 33 deletions(-) diff --git a/pkg/blobserver/diskpacked/diskpacked.go b/pkg/blobserver/diskpacked/diskpacked.go index c953ee638..7bc8eb502 100644 --- a/pkg/blobserver/diskpacked/diskpacked.go +++ b/pkg/blobserver/diskpacked/diskpacked.go @@ -106,7 +106,7 @@ var ( writeTotVar = expvar.NewMap("diskpacked-total-write-bytes") ) -const defaultIndexType = "kv" +const defaultIndexType = sorted.DefaultKVFileType const defaultIndexFile = "index." + defaultIndexType // IsDir reports whether dir is a diskpacked directory. diff --git a/pkg/blobserver/diskpacked/reindex.go b/pkg/blobserver/diskpacked/reindex.go index bd6a0276c..51e974c8e 100644 --- a/pkg/blobserver/diskpacked/reindex.go +++ b/pkg/blobserver/diskpacked/reindex.go @@ -33,6 +33,8 @@ import ( // possible index formats _ "camlistore.org/pkg/sorted/kvfile" + _ "camlistore.org/pkg/sorted/leveldb" + _ "camlistore.org/pkg/sorted/sqlite" ) var camliDebug, _ = strconv.ParseBool(os.Getenv("CAMLI_DEBUG")) diff --git a/pkg/serverinit/genconfig.go b/pkg/serverinit/genconfig.go index 960519998..87473b5a3 100644 --- a/pkg/serverinit/genconfig.go +++ b/pkg/serverinit/genconfig.go @@ -32,6 +32,7 @@ import ( "camlistore.org/pkg/jsonconfig" "camlistore.org/pkg/jsonsign" "camlistore.org/pkg/osutil" + "camlistore.org/pkg/sorted" "camlistore.org/pkg/types/serverconfig" "camlistore.org/pkg/wkfs" ) @@ -171,6 +172,21 @@ func (b *lowBuilder) addPublishedConfig(tlsO *tlsOpts) error { return nil } +// kvFileType returns the file based sorted type defined for index storage, if +// any. It defaults to "leveldb" otherwise. +func (b *lowBuilder) kvFileType() string { + switch { + case b.high.SQLite != "": + return "sqlite" + case b.high.KVFile != "": + return "kv" + case b.high.LevelDB != "": + return "leveldb" + default: + return sorted.DefaultKVFileType + } +} + func (b *lowBuilder) addUIConfig() { args := map[string]interface{}{ "cache": "/cache/", @@ -181,8 +197,8 @@ func (b *lowBuilder) addUIConfig() { var thumbCache map[string]interface{} if b.high.BlobPath != "" { thumbCache = map[string]interface{}{ - "type": "kv", - "file": filepath.Join(b.high.BlobPath, "thumbmeta.kv"), + "type": b.kvFileType(), + "file": filepath.Join(b.high.BlobPath, "thumbmeta."+b.kvFileType()), } } if thumbCache == nil { @@ -371,8 +387,8 @@ func (b *lowBuilder) addS3Config(s3 string) error { "to": s3Prefix, "queue": b.thatQueueUnlessMemory( map[string]interface{}{ - "type": "kv", - "file": filepath.Join(b.high.BlobPath, "sync-to-s3-queue.kv"), + "type": b.kvFileType(), + "file": filepath.Join(b.high.BlobPath, "sync-to-s3-queue."+b.kvFileType()), }), }) } @@ -415,8 +431,8 @@ func (b *lowBuilder) addGoogleDriveConfig(v string) error { "to": prefix, "queue": b.thatQueueUnlessMemory( map[string]interface{}{ - "type": "kv", - "file": filepath.Join(b.high.BlobPath, "sync-to-googledrive-queue.kv"), + "type": b.kvFileType(), + "file": filepath.Join(b.high.BlobPath, "sync-to-googledrive-queue."+b.kvFileType()), }), }) } @@ -459,8 +475,8 @@ func (b *lowBuilder) addGoogleCloudStorageConfig(v string) error { "to": gsPrefix, "queue": b.thatQueueUnlessMemory( map[string]interface{}{ - "type": "kv", - "file": filepath.Join(b.high.BlobPath, "sync-to-googlecloud-queue.kv"), + "type": b.kvFileType(), + "file": filepath.Join(b.high.BlobPath, "sync-to-googlecloud-queue."+b.kvFileType()), }), }) return nil @@ -555,14 +571,10 @@ func (b *lowBuilder) syncToIndexArgs() (map[string]interface{}, error) { if dir == "" { dir = b.indexFileDir() } - typ := "kv" - if b.high.SQLite != "" { - typ = "sqlite" - } a["queue"] = b.thatQueueUnlessMemory( map[string]interface{}{ - "type": typ, - "file": filepath.Join(dir, "sync-to-index-queue."+typ), + "type": b.kvFileType(), + "file": filepath.Join(dir, "sync-to-index-queue."+b.kvFileType()), }) return a, nil @@ -642,14 +654,32 @@ func (b *lowBuilder) genLowLevelPrefixes() error { "largeBlobs": "/bs-packed/", "metaIndex": blobPackedIndex, }) + } else if b.high.PackBlobs { + b.addPrefix("/bs/", "storage-"+storageType, args{ + "path": b.high.BlobPath, + "metaIndex": map[string]interface{}{ + "type": b.kvFileType(), + "file": filepath.Join(b.high.BlobPath, "index."+b.kvFileType()), + }, + }) } else { b.addPrefix("/bs/", "storage-"+storageType, args{ "path": b.high.BlobPath, }) } - b.addPrefix("/cache/", "storage-"+storageType, args{ - "path": filepath.Join(b.high.BlobPath, "/cache"), - }) + if b.high.PackBlobs { + b.addPrefix("/cache/", "storage-"+storageType, args{ + "path": filepath.Join(b.high.BlobPath, "/cache"), + "metaIndex": map[string]interface{}{ + "type": b.kvFileType(), + "file": filepath.Join(b.high.BlobPath, "cache", "index."+b.kvFileType()), + }, + }) + } else { + b.addPrefix("/cache/", "storage-"+storageType, args{ + "path": filepath.Join(b.high.BlobPath, "/cache"), + }) + } } else if b.high.MemoryStorage { b.addPrefix("/bs/", "storage-memory", nil) b.addPrefix("/cache/", "storage-memory", nil) diff --git a/pkg/serverinit/testdata/diskpacked-want.json b/pkg/serverinit/testdata/diskpacked-want.json index fc6f8852b..03aadf280 100644 --- a/pkg/serverinit/testdata/diskpacked-want.json +++ b/pkg/serverinit/testdata/diskpacked-want.json @@ -38,12 +38,20 @@ "/bs/": { "handler": "storage-diskpacked", "handlerArgs": { - "path": "/tmp/blobs" + "path": "/tmp/blobs", + "metaIndex": { + "file": "/tmp/blobs/index.kv", + "type": "kv" + } } }, "/cache/": { "handler": "storage-diskpacked", "handlerArgs": { + "metaIndex": { + "file": "/tmp/blobs/cache/index.kv", + "type": "kv" + }, "path": "/tmp/blobs/cache" } }, diff --git a/pkg/serverinit/testdata/leveldb-want.json b/pkg/serverinit/testdata/leveldb-want.json index c1f2c54b6..2c4ec4358 100644 --- a/pkg/serverinit/testdata/leveldb-want.json +++ b/pkg/serverinit/testdata/leveldb-want.json @@ -97,8 +97,8 @@ "handlerArgs": { "from": "/bs/", "queue": { - "file": "/tmp/blobs/sync-to-index-queue.kv", - "type": "kv" + "file": "/tmp/blobs/sync-to-index-queue.leveldb", + "type": "leveldb" }, "to": "/index/" } @@ -108,8 +108,8 @@ "handlerArgs": { "cache": "/cache/", "scaledImage": { - "file": "/tmp/blobs/thumbmeta.kv", - "type": "kv" + "file": "/tmp/blobs/thumbmeta.leveldb", + "type": "leveldb" } } } diff --git a/pkg/serverinit/testdata/memindex-want.json b/pkg/serverinit/testdata/memindex-want.json index c124eac59..9d22fe215 100644 --- a/pkg/serverinit/testdata/memindex-want.json +++ b/pkg/serverinit/testdata/memindex-want.json @@ -96,8 +96,8 @@ "handlerArgs": { "from": "/bs/", "queue": { - "file": "/tmp/blobs/sync-to-index-queue.kv", - "type": "kv" + "file": "/tmp/blobs/sync-to-index-queue.leveldb", + "type": "leveldb" }, "to": "/index/" } @@ -107,8 +107,8 @@ "handlerArgs": { "cache": "/cache/", "scaledImage": { - "file": "/tmp/blobs/thumbmeta.kv", - "type": "kv" + "file": "/tmp/blobs/thumbmeta.leveldb", + "type": "leveldb" } } } diff --git a/pkg/serverinit/testdata/mongo-want.json b/pkg/serverinit/testdata/mongo-want.json index 2de3f8c06..4f14079db 100644 --- a/pkg/serverinit/testdata/mongo-want.json +++ b/pkg/serverinit/testdata/mongo-want.json @@ -100,8 +100,8 @@ "handlerArgs": { "from": "/bs/", "queue": { - "file": "/tmp/blobs/sync-to-index-queue.kv", - "type": "kv" + "file": "/tmp/blobs/sync-to-index-queue.leveldb", + "type": "leveldb" }, "to": "/index/" } @@ -111,8 +111,8 @@ "handlerArgs": { "cache": "/cache/", "scaledImage": { - "file": "/tmp/blobs/thumbmeta.kv", - "type": "kv" + "file": "/tmp/blobs/thumbmeta.leveldb", + "type": "leveldb" } } } diff --git a/pkg/serverinit/testdata/sqlite-want.json b/pkg/serverinit/testdata/sqlite-want.json index 14b63fd71..688a07f10 100644 --- a/pkg/serverinit/testdata/sqlite-want.json +++ b/pkg/serverinit/testdata/sqlite-want.json @@ -107,8 +107,8 @@ "handlerArgs": { "cache": "/cache/", "scaledImage": { - "file": "/tmp/blobs/thumbmeta.kv", - "type": "kv" + "file": "/tmp/blobs/thumbmeta.sqlite", + "type": "sqlite" } } } diff --git a/pkg/sorted/kv.go b/pkg/sorted/kv.go index 83b0ee3d5..2daaa8398 100644 --- a/pkg/sorted/kv.go +++ b/pkg/sorted/kv.go @@ -29,6 +29,8 @@ const ( MaxValueSize = 63000 // Maximum size, in bytes, for a value in any store implementing KeyValue. MaxKeySize and MaxValueSize values originate from InnoDB and MySQL limitations. ) +const DefaultKVFileType = "leveldb" + var ( ErrNotFound = errors.New("sorted: key not found") ErrKeyTooLarge = fmt.Errorf("sorted: key size is over %v", MaxKeySize)