Rooba
/
perkeep
mirror of https://github.com/perkeep/perkeep.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

diskpacked: optional support for alternate index sorted.KeyValue store 

By default it will use cznic/kv. But if you want to use something else
(like MySQL, or leveldb, or Postgres, or Mongo), you can.

Change-Id: I8ce3571a701717ffde3b80856c72a9e3223ab439
This commit is contained in:
Brad Fitzpatrick 2014-03-10 18:54:47 -07:00
parent a30364a243
commit bf6bde214a
2 changed files with 30 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
pkg/blobserver/diskpacked/diskpacked.go
View File

@ -121,12 +121,12 @@ func New(dir string) (blobserver.Storage, error) {
// TODO: detect existing max size from size of files, if obvious,
// and set maxSize to that?
}
return newStorage(dir, maxSize)
return newStorage(dir, maxSize, nil)
}
// newStorage returns a new storage in path root with the given maxFileSize,
// or defaultMaxFileSize (512MB) if <= 0
func newStorage(root string, maxFileSize int64) (s *storage, err error) {
func newStorage(root string, maxFileSize int64, indexConf jsonconfig.Obj) (s *storage, err error) {
fi, err := os.Stat(root)
if os.IsNotExist(err) {
return nil, fmt.Errorf("storage root %q doesn't exist", root)
@ -137,7 +137,12 @@ func newStorage(root string, maxFileSize int64) (s *storage, err error) {
if !fi.IsDir() {
return nil, fmt.Errorf("storage root %q exists but is not a directory.", root)
}
index, err := kvfile.NewStorage(filepath.Join(root, indexKV))
var index sorted.KeyValue
if indexConf != nil {
index, err = sorted.NewKeyValue(indexConf)
} else {
index, err = kvfile.NewStorage(filepath.Join(root, indexKV))
}
if err != nil {
return nil, err
}
@ -171,12 +176,15 @@ func newStorage(root string, maxFileSize int64) (s *storage, err error) {
}
func newFromConfig(_ blobserver.Loader, config jsonconfig.Obj) (storage blobserver.Storage, err error) {
path := config.RequiredString("path")
maxFileSize := config.OptionalInt("maxFileSize", 0)
var (
path = config.RequiredString("path")
maxFileSize = config.OptionalInt("maxFileSize", 0)
indexConf = config.OptionalObject("metaIndex")
)
if err := config.Validate(); err != nil {
return nil, err
}
return newStorage(path, int64(maxFileSize))
return newStorage(path, int64(maxFileSize), indexConf)
}
func init() {

17
pkg/blobserver/diskpacked/diskpacked_test.go
View File

@ -24,16 +24,27 @@ import (
"camlistore.org/pkg/blobserver"
"camlistore.org/pkg/blobserver/storagetest"
"camlistore.org/pkg/jsonconfig"
"camlistore.org/pkg/test"
)
func newTempDiskpacked(t *testing.T) (sto blobserver.Storage, cleanup func()) {
return newTempDiskpackedWithIndex(t, nil)
}
func newTempDiskpackedMemory(t *testing.T) (sto blobserver.Storage, cleanup func()) {
return newTempDiskpackedWithIndex(t, jsonconfig.Obj{
"type": "memory",
})
}
func newTempDiskpackedWithIndex(t *testing.T, indexConf jsonconfig.Obj) (sto blobserver.Storage, cleanup func()) {
dir, err := ioutil.TempDir("", "diskpacked-test")
if err != nil {
t.Fatal(err)
}
t.Logf("diskpacked test dir is %q", dir)
s, err := newStorage(dir, 1<<20)
s, err := newStorage(dir, 1<<20, indexConf)
if err != nil {
t.Fatalf("newStorage: %v", err)
}
@ -47,6 +58,10 @@ func TestDiskpacked(t *testing.T) {
storagetest.Test(t, newTempDiskpacked)
}
func TestDiskpackedAltIndex(t *testing.T) {
storagetest.Test(t, newTempDiskpackedMemory)
}
func TestDoubleReceive(t *testing.T) {
sto, cleanup := newTempDiskpacked(t)
defer cleanup()