Rooba
/
perkeep
mirror of https://github.com/perkeep/perkeep.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

serverinit: use user-defined kv type everywhere, otherwise leveldb. 

In a few places, cznic/kv use was hardcoded, instead of picking the user
set kv type, if any. This change makes sure we use the user-defined one,
if set, syndtr/leveldb otherwise.

Relatedly, the low-level config generated for diskpacked now uses that
same principle for its internal index. Also, if a diskpacked is
configured without an index, it now defaults to syndtr/leveldb.

Please note that this patch DOES NOT change the default kv type in the
automatically generated high-level config.

Related: #632

Change-Id: I55dbd385230878e92776e6b83d34c36fd1fa9382
This commit is contained in:
mpl 2015-08-04 19:09:52 +02:00
parent 9a24acca6c
commit eec23f4a21
9 changed files with 75 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
pkg/blobserver/diskpacked/diskpacked.go
View File

@ -106,7 +106,7 @@ var (
writeTotVar = expvar.NewMap("diskpacked-total-write-bytes")
)
const defaultIndexType = "kv"
const defaultIndexType = sorted.DefaultKVFileType
const defaultIndexFile = "index." + defaultIndexType
// IsDir reports whether dir is a diskpacked directory.

2
pkg/blobserver/diskpacked/reindex.go
View File

@ -33,6 +33,8 @@ import (
// possible index formats
_ "camlistore.org/pkg/sorted/kvfile"
_ "camlistore.org/pkg/sorted/leveldb"
_ "camlistore.org/pkg/sorted/sqlite"
)
var camliDebug, _ = strconv.ParseBool(os.Getenv("CAMLI_DEBUG"))

64
pkg/serverinit/genconfig.go
View File

@ -32,6 +32,7 @@ import (
"camlistore.org/pkg/jsonconfig"
"camlistore.org/pkg/jsonsign"
"camlistore.org/pkg/osutil"
"camlistore.org/pkg/sorted"
"camlistore.org/pkg/types/serverconfig"
"camlistore.org/pkg/wkfs"
)
@ -171,6 +172,21 @@ func (b *lowBuilder) addPublishedConfig(tlsO *tlsOpts) error {
return nil
}
// kvFileType returns the file based sorted type defined for index storage, if
// any. It defaults to "leveldb" otherwise.
func (b *lowBuilder) kvFileType() string {
switch {
case b.high.SQLite != "":
return "sqlite"
case b.high.KVFile != "":
return "kv"
case b.high.LevelDB != "":
return "leveldb"
default:
return sorted.DefaultKVFileType
}
}
func (b *lowBuilder) addUIConfig() {
args := map[string]interface{}{
"cache": "/cache/",
@ -181,8 +197,8 @@ func (b *lowBuilder) addUIConfig() {
var thumbCache map[string]interface{}
if b.high.BlobPath != "" {
thumbCache = map[string]interface{}{
"type": "kv",
"file": filepath.Join(b.high.BlobPath, "thumbmeta.kv"),
"type": b.kvFileType(),
"file": filepath.Join(b.high.BlobPath, "thumbmeta."+b.kvFileType()),
}
}
if thumbCache == nil {
@ -371,8 +387,8 @@ func (b *lowBuilder) addS3Config(s3 string) error {
"to": s3Prefix,
"queue": b.thatQueueUnlessMemory(
map[string]interface{}{
"type": "kv",
"file": filepath.Join(b.high.BlobPath, "sync-to-s3-queue.kv"),
"type": b.kvFileType(),
"file": filepath.Join(b.high.BlobPath, "sync-to-s3-queue."+b.kvFileType()),
}),
})
}
@ -415,8 +431,8 @@ func (b *lowBuilder) addGoogleDriveConfig(v string) error {
"to": prefix,
"queue": b.thatQueueUnlessMemory(
map[string]interface{}{
"type": "kv",
"file": filepath.Join(b.high.BlobPath, "sync-to-googledrive-queue.kv"),
"type": b.kvFileType(),
"file": filepath.Join(b.high.BlobPath, "sync-to-googledrive-queue."+b.kvFileType()),
}),
})
}
@ -459,8 +475,8 @@ func (b *lowBuilder) addGoogleCloudStorageConfig(v string) error {
"to": gsPrefix,
"queue": b.thatQueueUnlessMemory(
map[string]interface{}{
"type": "kv",
"file": filepath.Join(b.high.BlobPath, "sync-to-googlecloud-queue.kv"),
"type": b.kvFileType(),
"file": filepath.Join(b.high.BlobPath, "sync-to-googlecloud-queue."+b.kvFileType()),
}),
})
return nil
@ -555,14 +571,10 @@ func (b *lowBuilder) syncToIndexArgs() (map[string]interface{}, error) {
if dir == "" {
dir = b.indexFileDir()
}
typ := "kv"
if b.high.SQLite != "" {
typ = "sqlite"
}
a["queue"] = b.thatQueueUnlessMemory(
map[string]interface{}{
"type": typ,
"file": filepath.Join(dir, "sync-to-index-queue."+typ),
"type": b.kvFileType(),
"file": filepath.Join(dir, "sync-to-index-queue."+b.kvFileType()),
})
return a, nil
@ -642,14 +654,32 @@ func (b *lowBuilder) genLowLevelPrefixes() error {
"largeBlobs": "/bs-packed/",
"metaIndex": blobPackedIndex,
})
} else if b.high.PackBlobs {
b.addPrefix("/bs/", "storage-"+storageType, args{
"path": b.high.BlobPath,
"metaIndex": map[string]interface{}{
"type": b.kvFileType(),
"file": filepath.Join(b.high.BlobPath, "index."+b.kvFileType()),
},
})
} else {
b.addPrefix("/bs/", "storage-"+storageType, args{
"path": b.high.BlobPath,
})
}
b.addPrefix("/cache/", "storage-"+storageType, args{
"path": filepath.Join(b.high.BlobPath, "/cache"),
})
if b.high.PackBlobs {
b.addPrefix("/cache/", "storage-"+storageType, args{
"path": filepath.Join(b.high.BlobPath, "/cache"),
"metaIndex": map[string]interface{}{
"type": b.kvFileType(),
"file": filepath.Join(b.high.BlobPath, "cache", "index."+b.kvFileType()),
},
})
} else {
b.addPrefix("/cache/", "storage-"+storageType, args{
"path": filepath.Join(b.high.BlobPath, "/cache"),
})
}
} else if b.high.MemoryStorage {
b.addPrefix("/bs/", "storage-memory", nil)
b.addPrefix("/cache/", "storage-memory", nil)

10
pkg/serverinit/testdata/diskpacked-want.json vendored
View File

@ -38,12 +38,20 @@
"/bs/": {
"handler": "storage-diskpacked",
"handlerArgs": {
"path": "/tmp/blobs"
"path": "/tmp/blobs",
"metaIndex": {
"file": "/tmp/blobs/index.kv",
"type": "kv"
}
}
},
"/cache/": {
"handler": "storage-diskpacked",
"handlerArgs": {
"metaIndex": {
"file": "/tmp/blobs/cache/index.kv",
"type": "kv"
},
"path": "/tmp/blobs/cache"
}
},

8
pkg/serverinit/testdata/leveldb-want.json vendored
View File

@ -97,8 +97,8 @@
"handlerArgs": {
"from": "/bs/",
"queue": {
"file": "/tmp/blobs/sync-to-index-queue.kv",
"type": "kv"
"file": "/tmp/blobs/sync-to-index-queue.leveldb",
"type": "leveldb"
},
"to": "/index/"
}
@ -108,8 +108,8 @@
"handlerArgs": {
"cache": "/cache/",
"scaledImage": {
"file": "/tmp/blobs/thumbmeta.kv",
"type": "kv"
"file": "/tmp/blobs/thumbmeta.leveldb",
"type": "leveldb"
}
}
}

8
pkg/serverinit/testdata/memindex-want.json vendored
View File

@ -96,8 +96,8 @@
"handlerArgs": {
"from": "/bs/",
"queue": {
"file": "/tmp/blobs/sync-to-index-queue.kv",
"type": "kv"
"file": "/tmp/blobs/sync-to-index-queue.leveldb",
"type": "leveldb"
},
"to": "/index/"
}
@ -107,8 +107,8 @@
"handlerArgs": {
"cache": "/cache/",
"scaledImage": {
"file": "/tmp/blobs/thumbmeta.kv",
"type": "kv"
"file": "/tmp/blobs/thumbmeta.leveldb",
"type": "leveldb"
}
}
}

8
pkg/serverinit/testdata/mongo-want.json vendored
View File

@ -100,8 +100,8 @@
"handlerArgs": {
"from": "/bs/",
"queue": {
"file": "/tmp/blobs/sync-to-index-queue.kv",
"type": "kv"
"file": "/tmp/blobs/sync-to-index-queue.leveldb",
"type": "leveldb"
},
"to": "/index/"
}
@ -111,8 +111,8 @@
"handlerArgs": {
"cache": "/cache/",
"scaledImage": {
"file": "/tmp/blobs/thumbmeta.kv",
"type": "kv"
"file": "/tmp/blobs/thumbmeta.leveldb",
"type": "leveldb"
}
}
}

4
pkg/serverinit/testdata/sqlite-want.json vendored
View File

@ -107,8 +107,8 @@
"handlerArgs": {
"cache": "/cache/",
"scaledImage": {
"file": "/tmp/blobs/thumbmeta.kv",
"type": "kv"
"file": "/tmp/blobs/thumbmeta.sqlite",
"type": "sqlite"
}
}
}

2
pkg/sorted/kv.go
View File

@ -29,6 +29,8 @@ const (
MaxValueSize = 63000 // Maximum size, in bytes, for a value in any store implementing KeyValue. MaxKeySize and MaxValueSize values originate from InnoDB and MySQL limitations.
)
const DefaultKVFileType = "leveldb"
var (
ErrNotFound = errors.New("sorted: key not found")
ErrKeyTooLarge = fmt.Errorf("sorted: key size is over %v", MaxKeySize)