Rooba
/
perkeep
mirror of https://github.com/perkeep/perkeep.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

serverinit: high-level config support for using blobpacked 

Users need to set:

     "packRelated": true,

... in their server config. This will probably become the default
value in the future.

This is currently mutually exclusive with diskpacked, which isn't good
at deleting things, and blobpacked loves to delete things (from the
loose blobs).

Updates #532

Change-Id: I8f4ea9406859b2705f26e9d1103d3acf9d1a8411
This commit is contained in:
Brad Fitzpatrick 2015-01-18 18:08:18 -08:00
parent 098ec2d70c
commit 4155ac6a3c
5 changed files with 223 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

106
pkg/serverinit/genconfig.go
View File

@ -266,6 +266,14 @@ func (b *lowBuilder) dbIndexStorage(rdbms string, confStr string, sortedType str
}
func (b *lowBuilder) sortedStorage(sortedType string) (map[string]interface{}, error) {
return b.sortedStorageAt(sortedType, "")
}
// filePrefix gives a file path of where to put the database. It can be omitted by
// some sorted implementations, but is required by others.
// The filePrefix should be to a file, not a directory, and should not end in a ".ext" extension.
// An extension like ".kv" or ".sqlite" will be added.
func (b *lowBuilder) sortedStorageAt(sortedType, filePrefix string) (map[string]interface{}, error) {
if b.high.MySQL != "" {
return b.dbIndexStorage("mysql", b.high.MySQL, sortedType)
}
@ -275,33 +283,40 @@ func (b *lowBuilder) sortedStorage(sortedType string) (map[string]interface{}, e
if b.high.Mongo != "" {
return b.mongoIndexStorage(b.high.Mongo, sortedType)
}
if sortedType != "index" {
return nil, fmt.Errorf("TODO: finish SQLite & KVFile for non-index queues")
}
if b.high.SQLite != "" {
return map[string]interface{}{
"type": "sqlite",
"file": b.high.SQLite,
}, nil
}
if b.high.KVFile != "" {
return map[string]interface{}{
"type": "kv",
"file": b.high.KVFile,
}, nil
}
if b.high.LevelDB != "" {
return map[string]interface{}{
"type": "leveldb",
"file": b.high.LevelDB,
}, nil
}
if b.high.MemoryIndex {
return map[string]interface{}{
"type": "memory",
}, nil
}
panic("indexArgs called when not in index mode")
if sortedType != "index" && filePrefix == "" {
return nil, fmt.Errorf("internal error: use of sortedStorageAt with a non-index type and no file location for non-database sorted implementation")
}
// dbFile returns path directly if sortedType == "index", else it returns filePrefix+"."+ext.
dbFile := func(path, ext string) string {
if sortedType == "index" {
return path
}
return filePrefix + "." + ext
}
if b.high.SQLite != "" {
return map[string]interface{}{
"type": "sqlite",
"file": dbFile(b.high.SQLite, "sqlite"),
}, nil
}
if b.high.KVFile != "" {
return map[string]interface{}{
"type": "kv",
"file": dbFile(b.high.KVFile, "kv"),
}, nil
}
if b.high.LevelDB != "" {
return map[string]interface{}{
"type": "leveldb",
"file": dbFile(b.high.LevelDB, "leveldb"),
}, nil
}
panic("internal error: sortedStorageAt didn't find a sorted implementation")
}
func (b *lowBuilder) thatQueueUnlessMemory(thatQueue map[string]interface{}) (queue map[string]interface{}) {
@ -422,6 +437,11 @@ func (b *lowBuilder) addGoogleCloudStorageConfig(v string) error {
clientID = "auto"
}
if b.high.PackRelated {
// TODO(mpl): implement
return errors.New("TODO: finish genconfig support for GCS+blobpacked")
}
isPrimary := !b.hasPrefix("/bs/")
gsPrefix := ""
if isPrimary {
@ -571,9 +591,27 @@ func (b *lowBuilder) genLowLevelPrefixes() error {
storageType = "diskpacked"
}
if b.high.BlobPath != "" {
b.addPrefix("/bs/", "storage-"+storageType, args{
"path": b.high.BlobPath,
})
if b.high.PackRelated {
b.addPrefix("/bs-loose/", "storage-filesystem", args{
"path": b.high.BlobPath,
})
b.addPrefix("/bs-packed/", "storage-filesystem", args{
"path": filepath.Join(b.high.BlobPath, "packed"),
})
blobPackedIndex, err := b.sortedStorageAt("blobpacked_index", filepath.Join(b.high.BlobPath, "packed", "packindex"))
if err != nil {
return err
}
b.addPrefix("/bs/", "storage-blobpacked", args{
"smallBlobs": "/bs-loose/",
"largeBlobs": "/bs-packed/",
"metaIndex": blobPackedIndex,
})
} else {
b.addPrefix("/bs/", "storage-"+storageType, args{
"path": b.high.BlobPath,
})
}
b.addPrefix("/cache/", "storage-"+storageType, args{
"path": filepath.Join(b.high.BlobPath, "/cache"),
})
@ -648,6 +686,12 @@ func (b *lowBuilder) build() (*Config, error) {
if conf.Listen != "" {
low["listen"] = conf.Listen
}
if conf.PackBlobs && conf.PackRelated {
return nil, errors.New("can't use both packBlobs (for 'diskpacked') and packRelated (for 'blobpacked')")
}
if conf.PackRelated && numSet(conf.S3, conf.GoogleDrive, conf.MemoryStorage) != 0 {
return nil, errors.New("Unsupported packRelated usage. packRelated (blobpacked) only works with localdisk and Google Cloud Storage for now.")
}
low["https"] = conf.HTTPS
low["auth"] = conf.Auth
@ -666,16 +710,18 @@ func (b *lowBuilder) build() (*Config, error) {
return nil, errors.New("no 'identity' in server config")
}
nolocaldisk := conf.BlobPath == ""
if nolocaldisk {
if conf.MemoryStorage && conf.BlobPath != "" {
return nil, errors.New("memoryStorage and blobPath are mutually exclusive.")
}
noLocalDisk := conf.BlobPath == ""
if noLocalDisk {
if !conf.MemoryStorage && conf.S3 == "" && conf.GoogleCloudStorage == "" {
return nil, errors.New("Unless memoryStorage is set, you must specify at least one storage option for your blobserver (blobPath (for localdisk), s3, googlecloudstorage).")
}
if !conf.MemoryStorage && conf.S3 != "" && conf.GoogleCloudStorage != "" {
return nil, errors.New("Using S3 as a primary storage and Google Cloud Storage as a mirror is not supported for now.")
}
} else if conf.MemoryStorage {
return nil, errors.New("memoryStorage and blobPath are mutually exclusive.")
}
if conf.ShareHandler && conf.ShareHandlerPath == "" {
@ -690,7 +736,7 @@ func (b *lowBuilder) build() (*Config, error) {
if conf.MemoryStorage {
noMkdir = true
}
if nolocaldisk {
if noLocalDisk {
// Whether camlistored is run from EC2 or not, we use
// a temp dir as the cache when primary storage is S3.
// TODO(mpl): s3CacheBucket

4
pkg/serverinit/serverinit_test.go
View File

@ -99,6 +99,10 @@ func TestConfigs(t *testing.T) {
t.Fatal(err)
}
for _, name := range names {
if strings.HasPrefix(name, ".#") {
// Emacs noise.
continue
}
if *flagOnly != "" && !strings.Contains(name, *flagOnly) {
continue
}

130
pkg/serverinit/testdata/blobpacked_localdisk-want.json vendored Normal file
View File

@ -0,0 +1,130 @@
{
"auth": "userpass:camlistore:pass3179",
"https": false,
"listen": "localhost:3179",
"prefixes": {
"/": {
"handler": "root",
"handlerArgs": {
"blobRoot": "/bs-and-maybe-also-index/",
"ownerName": "Alice",
"searchRoot": "/my-search/",
"statusRoot": "/status/",
"stealth": false
}
},
"/bs-and-index/": {
"handler": "storage-replica",
"handlerArgs": {
"backends": [
"/bs/",
"/index/"
]
}
},
"/bs-and-maybe-also-index/": {
"handler": "storage-cond",
"handlerArgs": {
"read": "/bs/",
"write": {
"else": "/bs/",
"if": "isSchema",
"then": "/bs-and-index/"
}
}
},
"/bs/": {
"handler": "storage-blobpacked",
"handlerArgs": {
"smallBlobs": "/bs-loose/",
"largeBlobs": "/bs-packed/",
"metaIndex": {
"file": "/tmp/blobs/packed/packindex.kv",
"type": "kv"
}
}
},
"/bs-loose/": {
"handler": "storage-filesystem",
"handlerArgs": {
"path": "/tmp/blobs"
}
},
"/bs-packed/": {
"handler": "storage-filesystem",
"handlerArgs": {
"path": "/tmp/blobs/packed"
}
},
"/cache/": {
"handler": "storage-filesystem",
"handlerArgs": {
"path": "/tmp/blobs/cache"
}
},
"/importer/": {
"handler": "importer",
"handlerArgs": {}
},
"/index/": {
"handler": "storage-index",
"handlerArgs": {
"blobSource": "/bs/",
"storage": {
"file": "/path/to/indexkv.db",
"type": "kv"
}
}
},
"/my-search/": {
"handler": "search",
"handlerArgs": {
"index": "/index/",
"owner": "sha1-f2b0b7da718b97ce8c31591d8ed4645c777f3ef4",
"slurpToMemory": true
}
},
"/setup/": {
"handler": "setup"
},
"/share/": {
"handler": "share",
"handlerArgs": {
"blobRoot": "/bs/"
}
},
"/sighelper/": {
"handler": "jsonsign",
"handlerArgs": {
"keyId": "26F5ABDA",
"publicKeyDest": "/bs-and-index/",
"secretRing": "/path/to/secring"
}
},
"/status/": {
"handler": "status"
},
"/sync/": {
"handler": "sync",
"handlerArgs": {
"from": "/bs/",
"queue": {
"file": "/tmp/blobs/sync-to-index-queue.kv",
"type": "kv"
},
"to": "/index/"
}
},
"/ui/": {
"handler": "ui",
"handlerArgs": {
"cache": "/cache/",
"jsonSignRoot": "/sighelper/",
"scaledImage": {
"file": "/tmp/blobs/thumbmeta.kv",
"type": "kv"
}
}
}
}
}

11
pkg/serverinit/testdata/blobpacked_localdisk.json vendored Normal file
View File

@ -0,0 +1,11 @@
{
"listen": "localhost:3179",
"auth": "userpass:camlistore:pass3179",
"blobPath": "/tmp/blobs",
"packRelated": true,
"kvIndexFile": "/path/to/indexkv.db",
"identity": "26F5ABDA",
"identitySecretRing": "/path/to/secring",
"ownerName": "Alice",
"shareHandlerPath": "/share/"
}

3
pkg/types/serverconfig/config.go
View File

@ -44,7 +44,8 @@ type Config struct {
// Blob storage.
MemoryStorage bool `json:"memoryStorage,omitempty"` // do not store anything (blobs or queues) on localdisk, use memory instead.
BlobPath string `json:"blobPath,omitempty"` // path to the directory containing the blobs.
PackBlobs bool `json:"packBlobs,omitempty"` // use diskpacked instead of the default filestorage.
PackBlobs bool `json:"packBlobs,omitempty"` // use "diskpacked" instead of the default filestorage. (exclusive with PackRelated)
PackRelated bool `json:"packRelated,omitempty"` // use "blobpacked" instead of the default storage (exclusive with PackBlobs)
S3 string `json:"s3,omitempty"` // Amazon S3 credentials: access_key_id:secret_access_key:bucket[:hostname].
GoogleCloudStorage string `json:"googlecloudstorage,omitempty"` // Google Cloud credentials: clientId:clientSecret:refreshToken:bucket[/optional/dir] or ":bucket[/optional/dir/]" for auto on GCE
GoogleDrive string `json:"googledrive,omitempty"` // Google Drive credentials: clientId:clientSecret:refreshToken:parentId.