From 4155ac6a3c018374d350db6a178079298a7fdd5c Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Sun, 18 Jan 2015 18:08:18 -0800 Subject: [PATCH] serverinit: high-level config support for using blobpacked Users need to set: "packRelated": true, ... in their server config. This will probably become the default value in the future. This is currently mutually exclusive with diskpacked, which isn't good at deleting things, and blobpacked loves to delete things (from the loose blobs). Updates #532 Change-Id: I8f4ea9406859b2705f26e9d1103d3acf9d1a8411 --- pkg/serverinit/genconfig.go | 106 ++++++++++---- pkg/serverinit/serverinit_test.go | 4 + .../testdata/blobpacked_localdisk-want.json | 130 ++++++++++++++++++ .../testdata/blobpacked_localdisk.json | 11 ++ pkg/types/serverconfig/config.go | 3 +- 5 files changed, 223 insertions(+), 31 deletions(-) create mode 100644 pkg/serverinit/testdata/blobpacked_localdisk-want.json create mode 100644 pkg/serverinit/testdata/blobpacked_localdisk.json diff --git a/pkg/serverinit/genconfig.go b/pkg/serverinit/genconfig.go index 088348e17..f468872cf 100644 --- a/pkg/serverinit/genconfig.go +++ b/pkg/serverinit/genconfig.go @@ -266,6 +266,14 @@ func (b *lowBuilder) dbIndexStorage(rdbms string, confStr string, sortedType str } func (b *lowBuilder) sortedStorage(sortedType string) (map[string]interface{}, error) { + return b.sortedStorageAt(sortedType, "") +} + +// filePrefix gives a file path of where to put the database. It can be omitted by +// some sorted implementations, but is required by others. +// The filePrefix should be to a file, not a directory, and should not end in a ".ext" extension. +// An extension like ".kv" or ".sqlite" will be added. +func (b *lowBuilder) sortedStorageAt(sortedType, filePrefix string) (map[string]interface{}, error) { if b.high.MySQL != "" { return b.dbIndexStorage("mysql", b.high.MySQL, sortedType) } @@ -275,33 +283,40 @@ func (b *lowBuilder) sortedStorage(sortedType string) (map[string]interface{}, e if b.high.Mongo != "" { return b.mongoIndexStorage(b.high.Mongo, sortedType) } - if sortedType != "index" { - return nil, fmt.Errorf("TODO: finish SQLite & KVFile for non-index queues") - } - if b.high.SQLite != "" { - return map[string]interface{}{ - "type": "sqlite", - "file": b.high.SQLite, - }, nil - } - if b.high.KVFile != "" { - return map[string]interface{}{ - "type": "kv", - "file": b.high.KVFile, - }, nil - } - if b.high.LevelDB != "" { - return map[string]interface{}{ - "type": "leveldb", - "file": b.high.LevelDB, - }, nil - } if b.high.MemoryIndex { return map[string]interface{}{ "type": "memory", }, nil } - panic("indexArgs called when not in index mode") + if sortedType != "index" && filePrefix == "" { + return nil, fmt.Errorf("internal error: use of sortedStorageAt with a non-index type and no file location for non-database sorted implementation") + } + // dbFile returns path directly if sortedType == "index", else it returns filePrefix+"."+ext. + dbFile := func(path, ext string) string { + if sortedType == "index" { + return path + } + return filePrefix + "." + ext + } + if b.high.SQLite != "" { + return map[string]interface{}{ + "type": "sqlite", + "file": dbFile(b.high.SQLite, "sqlite"), + }, nil + } + if b.high.KVFile != "" { + return map[string]interface{}{ + "type": "kv", + "file": dbFile(b.high.KVFile, "kv"), + }, nil + } + if b.high.LevelDB != "" { + return map[string]interface{}{ + "type": "leveldb", + "file": dbFile(b.high.LevelDB, "leveldb"), + }, nil + } + panic("internal error: sortedStorageAt didn't find a sorted implementation") } func (b *lowBuilder) thatQueueUnlessMemory(thatQueue map[string]interface{}) (queue map[string]interface{}) { @@ -422,6 +437,11 @@ func (b *lowBuilder) addGoogleCloudStorageConfig(v string) error { clientID = "auto" } + if b.high.PackRelated { + // TODO(mpl): implement + return errors.New("TODO: finish genconfig support for GCS+blobpacked") + } + isPrimary := !b.hasPrefix("/bs/") gsPrefix := "" if isPrimary { @@ -571,9 +591,27 @@ func (b *lowBuilder) genLowLevelPrefixes() error { storageType = "diskpacked" } if b.high.BlobPath != "" { - b.addPrefix("/bs/", "storage-"+storageType, args{ - "path": b.high.BlobPath, - }) + if b.high.PackRelated { + b.addPrefix("/bs-loose/", "storage-filesystem", args{ + "path": b.high.BlobPath, + }) + b.addPrefix("/bs-packed/", "storage-filesystem", args{ + "path": filepath.Join(b.high.BlobPath, "packed"), + }) + blobPackedIndex, err := b.sortedStorageAt("blobpacked_index", filepath.Join(b.high.BlobPath, "packed", "packindex")) + if err != nil { + return err + } + b.addPrefix("/bs/", "storage-blobpacked", args{ + "smallBlobs": "/bs-loose/", + "largeBlobs": "/bs-packed/", + "metaIndex": blobPackedIndex, + }) + } else { + b.addPrefix("/bs/", "storage-"+storageType, args{ + "path": b.high.BlobPath, + }) + } b.addPrefix("/cache/", "storage-"+storageType, args{ "path": filepath.Join(b.high.BlobPath, "/cache"), }) @@ -648,6 +686,12 @@ func (b *lowBuilder) build() (*Config, error) { if conf.Listen != "" { low["listen"] = conf.Listen } + if conf.PackBlobs && conf.PackRelated { + return nil, errors.New("can't use both packBlobs (for 'diskpacked') and packRelated (for 'blobpacked')") + } + if conf.PackRelated && numSet(conf.S3, conf.GoogleDrive, conf.MemoryStorage) != 0 { + return nil, errors.New("Unsupported packRelated usage. packRelated (blobpacked) only works with localdisk and Google Cloud Storage for now.") + } low["https"] = conf.HTTPS low["auth"] = conf.Auth @@ -666,16 +710,18 @@ func (b *lowBuilder) build() (*Config, error) { return nil, errors.New("no 'identity' in server config") } - nolocaldisk := conf.BlobPath == "" - if nolocaldisk { + if conf.MemoryStorage && conf.BlobPath != "" { + return nil, errors.New("memoryStorage and blobPath are mutually exclusive.") + } + + noLocalDisk := conf.BlobPath == "" + if noLocalDisk { if !conf.MemoryStorage && conf.S3 == "" && conf.GoogleCloudStorage == "" { return nil, errors.New("Unless memoryStorage is set, you must specify at least one storage option for your blobserver (blobPath (for localdisk), s3, googlecloudstorage).") } if !conf.MemoryStorage && conf.S3 != "" && conf.GoogleCloudStorage != "" { return nil, errors.New("Using S3 as a primary storage and Google Cloud Storage as a mirror is not supported for now.") } - } else if conf.MemoryStorage { - return nil, errors.New("memoryStorage and blobPath are mutually exclusive.") } if conf.ShareHandler && conf.ShareHandlerPath == "" { @@ -690,7 +736,7 @@ func (b *lowBuilder) build() (*Config, error) { if conf.MemoryStorage { noMkdir = true } - if nolocaldisk { + if noLocalDisk { // Whether camlistored is run from EC2 or not, we use // a temp dir as the cache when primary storage is S3. // TODO(mpl): s3CacheBucket diff --git a/pkg/serverinit/serverinit_test.go b/pkg/serverinit/serverinit_test.go index f742b5584..a955d1461 100644 --- a/pkg/serverinit/serverinit_test.go +++ b/pkg/serverinit/serverinit_test.go @@ -99,6 +99,10 @@ func TestConfigs(t *testing.T) { t.Fatal(err) } for _, name := range names { + if strings.HasPrefix(name, ".#") { + // Emacs noise. + continue + } if *flagOnly != "" && !strings.Contains(name, *flagOnly) { continue } diff --git a/pkg/serverinit/testdata/blobpacked_localdisk-want.json b/pkg/serverinit/testdata/blobpacked_localdisk-want.json new file mode 100644 index 000000000..7612c2aba --- /dev/null +++ b/pkg/serverinit/testdata/blobpacked_localdisk-want.json @@ -0,0 +1,130 @@ +{ + "auth": "userpass:camlistore:pass3179", + "https": false, + "listen": "localhost:3179", + "prefixes": { + "/": { + "handler": "root", + "handlerArgs": { + "blobRoot": "/bs-and-maybe-also-index/", + "ownerName": "Alice", + "searchRoot": "/my-search/", + "statusRoot": "/status/", + "stealth": false + } + }, + "/bs-and-index/": { + "handler": "storage-replica", + "handlerArgs": { + "backends": [ + "/bs/", + "/index/" + ] + } + }, + "/bs-and-maybe-also-index/": { + "handler": "storage-cond", + "handlerArgs": { + "read": "/bs/", + "write": { + "else": "/bs/", + "if": "isSchema", + "then": "/bs-and-index/" + } + } + }, + "/bs/": { + "handler": "storage-blobpacked", + "handlerArgs": { + "smallBlobs": "/bs-loose/", + "largeBlobs": "/bs-packed/", + "metaIndex": { + "file": "/tmp/blobs/packed/packindex.kv", + "type": "kv" + } + } + }, + "/bs-loose/": { + "handler": "storage-filesystem", + "handlerArgs": { + "path": "/tmp/blobs" + } + }, + "/bs-packed/": { + "handler": "storage-filesystem", + "handlerArgs": { + "path": "/tmp/blobs/packed" + } + }, + "/cache/": { + "handler": "storage-filesystem", + "handlerArgs": { + "path": "/tmp/blobs/cache" + } + }, + "/importer/": { + "handler": "importer", + "handlerArgs": {} + }, + "/index/": { + "handler": "storage-index", + "handlerArgs": { + "blobSource": "/bs/", + "storage": { + "file": "/path/to/indexkv.db", + "type": "kv" + } + } + }, + "/my-search/": { + "handler": "search", + "handlerArgs": { + "index": "/index/", + "owner": "sha1-f2b0b7da718b97ce8c31591d8ed4645c777f3ef4", + "slurpToMemory": true + } + }, + "/setup/": { + "handler": "setup" + }, + "/share/": { + "handler": "share", + "handlerArgs": { + "blobRoot": "/bs/" + } + }, + "/sighelper/": { + "handler": "jsonsign", + "handlerArgs": { + "keyId": "26F5ABDA", + "publicKeyDest": "/bs-and-index/", + "secretRing": "/path/to/secring" + } + }, + "/status/": { + "handler": "status" + }, + "/sync/": { + "handler": "sync", + "handlerArgs": { + "from": "/bs/", + "queue": { + "file": "/tmp/blobs/sync-to-index-queue.kv", + "type": "kv" + }, + "to": "/index/" + } + }, + "/ui/": { + "handler": "ui", + "handlerArgs": { + "cache": "/cache/", + "jsonSignRoot": "/sighelper/", + "scaledImage": { + "file": "/tmp/blobs/thumbmeta.kv", + "type": "kv" + } + } + } + } +} diff --git a/pkg/serverinit/testdata/blobpacked_localdisk.json b/pkg/serverinit/testdata/blobpacked_localdisk.json new file mode 100644 index 000000000..7abf7cc86 --- /dev/null +++ b/pkg/serverinit/testdata/blobpacked_localdisk.json @@ -0,0 +1,11 @@ +{ + "listen": "localhost:3179", + "auth": "userpass:camlistore:pass3179", + "blobPath": "/tmp/blobs", + "packRelated": true, + "kvIndexFile": "/path/to/indexkv.db", + "identity": "26F5ABDA", + "identitySecretRing": "/path/to/secring", + "ownerName": "Alice", + "shareHandlerPath": "/share/" +} diff --git a/pkg/types/serverconfig/config.go b/pkg/types/serverconfig/config.go index 961052b4d..9f39c2576 100644 --- a/pkg/types/serverconfig/config.go +++ b/pkg/types/serverconfig/config.go @@ -44,7 +44,8 @@ type Config struct { // Blob storage. MemoryStorage bool `json:"memoryStorage,omitempty"` // do not store anything (blobs or queues) on localdisk, use memory instead. BlobPath string `json:"blobPath,omitempty"` // path to the directory containing the blobs. - PackBlobs bool `json:"packBlobs,omitempty"` // use diskpacked instead of the default filestorage. + PackBlobs bool `json:"packBlobs,omitempty"` // use "diskpacked" instead of the default filestorage. (exclusive with PackRelated) + PackRelated bool `json:"packRelated,omitempty"` // use "blobpacked" instead of the default storage (exclusive with PackBlobs) S3 string `json:"s3,omitempty"` // Amazon S3 credentials: access_key_id:secret_access_key:bucket[:hostname]. GoogleCloudStorage string `json:"googlecloudstorage,omitempty"` // Google Cloud credentials: clientId:clientSecret:refreshToken:bucket[/optional/dir] or ":bucket[/optional/dir/]" for auto on GCE GoogleDrive string `json:"googledrive,omitempty"` // Google Drive credentials: clientId:clientSecret:refreshToken:parentId.