From 28bc00767602e9f6ddf1043fb7842d81d2c7819f Mon Sep 17 00:00:00 2001 From: mpl Date: Thu, 26 Oct 2017 16:03:27 +0200 Subject: [PATCH] pkg/serverinit: add low-level config generation for Backblaze B2 Fixes #971 Change-Id: Iba944e3597009b18a380007b72fba5127e9a1698 --- doc/server-config.md | 1 + pkg/serverinit/genconfig.go | 87 ++++++++++++- .../testdata/b2_nolocaldisk-want.json | 118 ++++++++++++++++++ pkg/serverinit/testdata/b2_nolocaldisk.json | 12 ++ pkg/types/serverconfig/config.go | 1 + 5 files changed, 217 insertions(+), 2 deletions(-) create mode 100644 pkg/serverinit/testdata/b2_nolocaldisk-want.json create mode 100644 pkg/serverinit/testdata/b2_nolocaldisk.json diff --git a/doc/server-config.md b/doc/server-config.md index c4875d335..1bb9c0531 100644 --- a/doc/server-config.md +++ b/doc/server-config.md @@ -94,6 +94,7 @@ At least one of these must be set: * `blobPath`: local disk path to store blobs. (valid for diskpacked too). * `s3`: "`key:secret:bucket[/optional/dir]`" or "`key:secret:bucket[/optional/dir]:hostname`" (with colons, but no quotes). +* `b2`: "`account_id:application_key:bucket[/optional/dir]`". * `googlecloudstorage`: "`clientId:clientSecret:refreshToken:bucketName[/optional/dir]`" Additionally, there are two mutually exclusive options which only apply if `blobPath` is set: diff --git a/pkg/serverinit/genconfig.go b/pkg/serverinit/genconfig.go index 330ca8c4a..6755878ed 100644 --- a/pkg/serverinit/genconfig.go +++ b/pkg/serverinit/genconfig.go @@ -519,6 +519,81 @@ func (b *lowBuilder) addS3Config(s3 string) error { return nil } +func (b *lowBuilder) addB2Config(b2 string) error { + f := strings.SplitN(b2, ":", 3) + if len(f) < 3 { + return errors.New(`genconfig: expected "b2" field to be of form "account_id:application_key:bucket[/optional/dir]"`) + } + account, key, bucket := f[0], f[1], f[2] + isReplica := b.hasPrefix("/bs/") + b2Prefix := "" + b2Auth := map[string]interface{}{ + "account_id": account, + "application_key": key, + } + b2Args := args{ + "auth": b2Auth, + "bucket": bucket, + } + if isReplica { + b2Prefix = "/sto-b2/" + b.addPrefix(b2Prefix, "storage-b2", b2Args) + if b.high.BlobPath == "" && !b.high.MemoryStorage { + panic("unexpected empty blobpath with sync-to-b2") + } + b.addPrefix("/sync-to-b2/", "sync", args{ + "from": "/bs/", + "to": b2Prefix, + "queue": b.thatQueueUnlessMemory( + map[string]interface{}{ + "type": b.kvFileType(), + "file": filepath.Join(b.high.BlobPath, "sync-to-b2-queue."+b.kvFileType()), + }), + }) + return nil + } + + b.addPrefix("/cache/", "storage-filesystem", args{ + "path": filepath.Join(tempDir(), "camli-cache"), + }) + + b2Prefix = "/bs/" + if !b.high.PackRelated { + b.addPrefix(b2Prefix, "storage-b2", b2Args) + return nil + } + packedB2Args := func(bucket string) args { + a := args{ + "bucket": bucket, + "auth": map[string]interface{}{ + "account_id": account, + "application_key": key, + }, + } + return a + } + + b.addPrefix("/bs-loose/", "storage-b2", packedB2Args(path.Join(bucket, "loose"))) + b.addPrefix("/bs-packed/", "storage-b2", packedB2Args(path.Join(bucket, "packed"))) + + // If index is DBMS, then blobPackedIndex is in DBMS too, with + // whatever dbname is defined for "blobpacked_index", or defaulting + // to "blobpacked_index". Otherwise blobPackedIndex is same + // file-based DB as the index, in same dir, but named + // packindex.dbtype. + blobPackedIndex, err := b.sortedStorageAt("blobpacked_index", filepath.Join(b.indexFileDir(), "packindex")) + if err != nil { + return err + } + b.addPrefix(b2Prefix, "storage-blobpacked", args{ + "smallBlobs": "/bs-loose/", + "largeBlobs": "/bs-packed/", + "metaIndex": blobPackedIndex, + }) + + return nil +} + func (b *lowBuilder) addGoogleDriveConfig(v string) error { f := strings.SplitN(v, ":", 4) if len(f) != 4 { @@ -911,12 +986,15 @@ func (b *lowBuilder) build() (*Config, error) { noLocalDisk := conf.BlobPath == "" if noLocalDisk { - if !conf.MemoryStorage && conf.S3 == "" && conf.GoogleCloudStorage == "" { - return nil, errors.New("Unless memoryStorage is set, you must specify at least one storage option for your blobserver (blobPath (for localdisk), s3, googlecloudstorage).") + if !conf.MemoryStorage && conf.S3 == "" && conf.B2 == "" && conf.GoogleCloudStorage == "" { + return nil, errors.New("Unless memoryStorage is set, you must specify at least one storage option for your blobserver (blobPath (for localdisk), s3, b2, googlecloudstorage).") } if !conf.MemoryStorage && conf.S3 != "" && conf.GoogleCloudStorage != "" { return nil, errors.New("Using S3 as a primary storage and Google Cloud Storage as a mirror is not supported for now.") } + if !conf.MemoryStorage && conf.B2 != "" && conf.GoogleCloudStorage != "" { + return nil, errors.New("Using B2 as a primary storage and Google Cloud Storage as a mirror is not supported for now.") + } } if conf.ShareHandler && conf.ShareHandlerPath == "" { conf.ShareHandlerPath = "/share/" @@ -1008,6 +1086,11 @@ func (b *lowBuilder) build() (*Config, error) { return nil, err } } + if conf.B2 != "" { + if err := b.addB2Config(conf.B2); err != nil { + return nil, err + } + } if conf.GoogleDrive != "" { if err := b.addGoogleDriveConfig(conf.GoogleDrive); err != nil { return nil, err diff --git a/pkg/serverinit/testdata/b2_nolocaldisk-want.json b/pkg/serverinit/testdata/b2_nolocaldisk-want.json new file mode 100644 index 000000000..be5739528 --- /dev/null +++ b/pkg/serverinit/testdata/b2_nolocaldisk-want.json @@ -0,0 +1,118 @@ +{ + "auth": "userpass:camlistore:pass3179", + "https": false, + "listen": "localhost:3179", + "prefixes": { + "/": { + "handler": "root", + "handlerArgs": { + "blobRoot": "/bs-and-maybe-also-index/", + "helpRoot": "/help/", + "jsonSignRoot": "/sighelper/", + "searchRoot": "/my-search/", + "shareRoot": "/share/", + "statusRoot": "/status/", + "stealth": false + } + }, + "/bs-and-index/": { + "handler": "storage-replica", + "handlerArgs": { + "backends": [ + "/bs/", + "/index/" + ] + } + }, + "/bs-and-maybe-also-index/": { + "handler": "storage-cond", + "handlerArgs": { + "read": "/bs/", + "write": { + "else": "/bs/", + "if": "isSchema", + "then": "/bs-and-index/" + } + } + }, + "/bs/": { + "handler": "storage-b2", + "handlerArgs": { + "auth": { + "account_id": "account", + "application_key": "key" + }, + "bucket": "bucket" + } + }, + "/cache/": { + "handler": "storage-filesystem", + "handlerArgs": { + "path": "/tmp/camli-cache" + } + }, + "/help/": { + "handler": "help" + }, + "/importer/": { + "handler": "importer", + "handlerArgs": {} + }, + "/index/": { + "handler": "storage-index", + "handlerArgs": { + "blobSource": "/bs/", + "storage": { + "file": "/path/to/indexkv.db", + "type": "kv" + } + } + }, + "/my-search/": { + "handler": "search", + "handlerArgs": { + "index": "/index/", + "owner": "sha1-f2b0b7da718b97ce8c31591d8ed4645c777f3ef4", + "slurpToMemory": true + } + }, + "/setup/": { + "handler": "setup" + }, + "/share/": { + "handler": "share", + "handlerArgs": { + "blobRoot": "/bs/", + "index": "/index/" + } + }, + "/sighelper/": { + "handler": "jsonsign", + "handlerArgs": { + "keyId": "26F5ABDA", + "publicKeyDest": "/bs-and-index/", + "secretRing": "/path/to/secring" + } + }, + "/status/": { + "handler": "status" + }, + "/sync/": { + "handler": "sync", + "handlerArgs": { + "from": "/bs/", + "queue": { + "file": "/path/to/sync-to-index-queue.kv", + "type": "kv" + }, + "to": "/index/" + } + }, + "/ui/": { + "handler": "ui", + "handlerArgs": { + "cache": "/cache/" + } + } + } +} diff --git a/pkg/serverinit/testdata/b2_nolocaldisk.json b/pkg/serverinit/testdata/b2_nolocaldisk.json new file mode 100644 index 000000000..518e3704f --- /dev/null +++ b/pkg/serverinit/testdata/b2_nolocaldisk.json @@ -0,0 +1,12 @@ +{ + "listen": "localhost:3179", + "https": false, + "auth": "userpass:camlistore:pass3179", + "identity": "26F5ABDA", + "identitySecretRing": "/path/to/secring", + "kvIndexFile": "/path/to/indexkv.db", + "b2": "account:key:bucket", + "replicateTo": [], + "publish": {}, + "shareHandlerPath": "/share/" +} diff --git a/pkg/types/serverconfig/config.go b/pkg/types/serverconfig/config.go index 2d798ab06..1e6864014 100644 --- a/pkg/types/serverconfig/config.go +++ b/pkg/types/serverconfig/config.go @@ -57,6 +57,7 @@ type Config struct { PackBlobs bool `json:"packBlobs,omitempty"` // use "diskpacked" instead of the default filestorage. (exclusive with PackRelated) PackRelated bool `json:"packRelated,omitempty"` // use "blobpacked" instead of the default storage (exclusive with PackBlobs) S3 string `json:"s3,omitempty"` // Amazon S3 credentials: access_key_id:secret_access_key:bucket[/optional/dir][:hostname]. + B2 string `json:"b2,omitempty"` // Backblaze B2 credentials: account_id:application_key:bucket[/optional/dir]. GoogleCloudStorage string `json:"googlecloudstorage,omitempty"` // Google Cloud credentials: clientId:clientSecret:refreshToken:bucket[/optional/dir] or ":bucket[/optional/dir/]" for auto on GCE GoogleDrive string `json:"googledrive,omitempty"` // Google Drive credentials: clientId:clientSecret:refreshToken:parentId. ShareHandler bool `json:"shareHandler,omitempty"` // enable the share handler. If true, and shareHandlerPath is empty then shareHandlerPath will default to "/share/" when generating the low-level config.