From e2a6f3bc44f2231ab70bb09711f9025002a6bb69 Mon Sep 17 00:00:00 2001 From: aviau Date: Sat, 2 Jan 2021 22:16:00 -0500 Subject: [PATCH] genconfig: support blobpacked in cloud replicas Cloud-backed storage currently does not use blobpacked if they are a replica. This causes issues if they become the main storage. Example use case where this would cause issues: - Create a configuration with s3, blobPath, and packRelated. - Add objects - Remove blobPath - `pk list` - All blobs are gone because the s3-only config uses blobpacked but the s3 replica was not. --- pkg/serverinit/genconfig.go | 252 +++++++++-------- .../testdata/packrelated_replicated-want.json | 262 ++++++++++++++++++ .../testdata/packrelated_replicated.json | 14 + 3 files changed, 401 insertions(+), 127 deletions(-) create mode 100644 pkg/serverinit/testdata/packrelated_replicated-want.json create mode 100644 pkg/serverinit/testdata/packrelated_replicated.json diff --git a/pkg/serverinit/genconfig.go b/pkg/serverinit/genconfig.go index 6abfb4153..e70acd0e7 100644 --- a/pkg/serverinit/genconfig.go +++ b/pkg/serverinit/genconfig.go @@ -546,18 +546,51 @@ func (b *lowBuilder) addS3Config(s3 string) error { hostname = f[3] } isReplica := b.hasPrefix("/bs/") - s3Prefix := "" - s3Args := args{ - "aws_access_key": accessKey, - "aws_secret_access_key": secret, - "bucket": bucket, - } - if hostname != "" { - s3Args["hostname"] = hostname - } + s3Prefix := "/bs/" if isReplica { s3Prefix = "/sto-s3/" - b.addPrefix(s3Prefix, "storage-s3", s3Args) + } + + s3Args := func(bucket string) args { + a := args{ + "bucket": bucket, + "aws_access_key": accessKey, + "aws_secret_access_key": secret, + } + if hostname != "" { + a["hostname"] = hostname + } + return a + } + + if !b.high.PackRelated { + b.addPrefix(s3Prefix, "storage-s3", s3Args(bucket)) + } else { + bsLoose := "/bs-loose/" + bsPacked := "/bs-packed/" + if isReplica { + bsLoose = "/sto-s3-bs-loose/" + bsPacked = "/sto-s3-bs-packed/" + } + + b.addPrefix(bsLoose, "storage-s3", s3Args(path.Join(bucket, "loose"))) + b.addPrefix(bsPacked, "storage-s3", s3Args(path.Join(bucket, "packed"))) + + // If index is DBMS, then blobPackedIndex is in DBMS too. + // Otherwise blobPackedIndex is same file-based DB as the index, + // in same dir, but named packindex.dbtype. + blobPackedIndex, err := b.sortedStorageAt(dbBlobpackedIndex, filepath.Join(b.indexFileDir(), "packindex")) + if err != nil { + return err + } + b.addPrefix(s3Prefix, "storage-blobpacked", args{ + "smallBlobs": bsLoose, + "largeBlobs": bsPacked, + "metaIndex": blobPackedIndex, + }) + } + + if isReplica { if b.high.BlobPath == "" && !b.high.MemoryStorage { panic("unexpected empty blobpath with sync-to-s3") } @@ -579,39 +612,6 @@ func (b *lowBuilder) addS3Config(s3 string) error { "path": filepath.Join(tempDir(), "camli-cache"), }) - s3Prefix = "/bs/" - if !b.high.PackRelated { - b.addPrefix(s3Prefix, "storage-s3", s3Args) - return nil - } - packedS3Args := func(bucket string) args { - a := args{ - "bucket": bucket, - "aws_access_key": accessKey, - "aws_secret_access_key": secret, - } - if hostname != "" { - a["hostname"] = hostname - } - return a - } - - b.addPrefix("/bs-loose/", "storage-s3", packedS3Args(path.Join(bucket, "loose"))) - b.addPrefix("/bs-packed/", "storage-s3", packedS3Args(path.Join(bucket, "packed"))) - - // If index is DBMS, then blobPackedIndex is in DBMS too. - // Otherwise blobPackedIndex is same file-based DB as the index, - // in same dir, but named packindex.dbtype. - blobPackedIndex, err := b.sortedStorageAt(dbBlobpackedIndex, filepath.Join(b.indexFileDir(), "packindex")) - if err != nil { - return err - } - b.addPrefix(s3Prefix, "storage-blobpacked", args{ - "smallBlobs": "/bs-loose/", - "largeBlobs": "/bs-packed/", - "metaIndex": blobPackedIndex, - }) - return nil } @@ -622,18 +622,50 @@ func (b *lowBuilder) addB2Config(b2 string) error { } account, key, bucket := f[0], f[1], f[2] isReplica := b.hasPrefix("/bs/") - b2Prefix := "" - b2Auth := map[string]interface{}{ - "account_id": account, - "application_key": key, - } - b2Args := args{ - "auth": b2Auth, - "bucket": bucket, - } + b2Prefix := "/bs/" if isReplica { b2Prefix = "/sto-b2/" - b.addPrefix(b2Prefix, "storage-b2", b2Args) + } + + b2Args := func(bucket string) args { + a := args{ + "bucket": bucket, + "auth": map[string]interface{}{ + "account_id": account, + "application_key": key, + }, + } + return a + } + + if !b.high.PackRelated { + b.addPrefix(b2Prefix, "storage-b2", b2Args(bucket)) + } else { + bsLoose := "/bs-loose/" + bsPacked := "/bs-packed/" + if isReplica { + bsLoose = "/sto-b2-bs-loose/" + bsPacked = "/sto-b2-bs-packed/" + } + + b.addPrefix(bsLoose, "storage-b2", b2Args(path.Join(bucket, "loose"))) + b.addPrefix(bsPacked, "storage-b2", b2Args(path.Join(bucket, "packed"))) + + // If index is DBMS, then blobPackedIndex is in DBMS too. + // Otherwise blobPackedIndex is same file-based DB as the index, + // in same dir, but named packindex.dbtype. + blobPackedIndex, err := b.sortedStorageAt(dbBlobpackedIndex, filepath.Join(b.indexFileDir(), "packindex")) + if err != nil { + return err + } + b.addPrefix(b2Prefix, "storage-blobpacked", args{ + "smallBlobs": "/bs-loose/", + "largeBlobs": "/bs-packed/", + "metaIndex": blobPackedIndex, + }) + } + + if isReplica { if b.high.BlobPath == "" && !b.high.MemoryStorage { panic("unexpected empty blobpath with sync-to-b2") } @@ -653,38 +685,6 @@ func (b *lowBuilder) addB2Config(b2 string) error { "path": filepath.Join(tempDir(), "camli-cache"), }) - b2Prefix = "/bs/" - if !b.high.PackRelated { - b.addPrefix(b2Prefix, "storage-b2", b2Args) - return nil - } - packedB2Args := func(bucket string) args { - a := args{ - "bucket": bucket, - "auth": map[string]interface{}{ - "account_id": account, - "application_key": key, - }, - } - return a - } - - b.addPrefix("/bs-loose/", "storage-b2", packedB2Args(path.Join(bucket, "loose"))) - b.addPrefix("/bs-packed/", "storage-b2", packedB2Args(path.Join(bucket, "packed"))) - - // If index is DBMS, then blobPackedIndex is in DBMS too. - // Otherwise blobPackedIndex is same file-based DB as the index, - // in same dir, but named packindex.dbtype. - blobPackedIndex, err := b.sortedStorageAt(dbBlobpackedIndex, filepath.Join(b.indexFileDir(), "packindex")) - if err != nil { - return err - } - b.addPrefix(b2Prefix, "storage-blobpacked", args{ - "smallBlobs": "/bs-loose/", - "largeBlobs": "/bs-packed/", - "metaIndex": blobPackedIndex, - }) - return nil } @@ -752,17 +752,54 @@ func (b *lowBuilder) addGoogleCloudStorageConfig(v string) error { } isReplica := b.hasPrefix("/bs/") + gsPrefix := "/bs/" if isReplica { - gsPrefix := "/sto-googlecloudstorage/" - b.addPrefix(gsPrefix, "storage-googlecloudstorage", args{ + gsPrefix = "/sto-googlecloudstorage/" + } + + gsArgs := func(bucket string) args { + a := args{ "bucket": bucket, "auth": map[string]interface{}{ "client_id": clientID, "client_secret": secret, "refresh_token": refreshToken, }, - }) + } + return a + } + if !b.high.PackRelated { + b.addPrefix(gsPrefix, "storage-googlecloudstorage", gsArgs(bucket)) + } else { + bsLoose := "/bs-loose/" + bsPacked := "/bs-packed/" + if isReplica { + bsLoose = "/sto-googlecloudstorage-bs-loose/" + bsPacked = "/sto-googlecloudstorage-bs-packed/" + } + + b.addPrefix(bsLoose, "storage-googlecloudstorage", gsArgs(path.Join(bucket, "loose"))) + b.addPrefix(bsPacked, "storage-googlecloudstorage", gsArgs(path.Join(bucket, "packed"))) + + // If index is DBMS, then blobPackedIndex is in DBMS too. + // Otherwise blobPackedIndex is same file-based DB as the index, + // in same dir, but named packindex.dbtype. + blobPackedIndex, err := b.sortedStorageAt(dbBlobpackedIndex, filepath.Join(b.indexFileDir(), "packindex")) + if err != nil { + return err + } + b.addPrefix(gsPrefix, "storage-blobpacked", args{ + "smallBlobs": bsLoose, + "largeBlobs": bsPacked, + "metaIndex": blobPackedIndex, + }) + } + + if isReplica { + if b.high.BlobPath == "" && !b.high.MemoryStorage { + panic("unexpected empty blobpath with sync-to-googlecloudstorage") + } b.addPrefix("/sync-to-googlecloudstorage/", "sync", args{ "from": "/bs/", "to": gsPrefix, @@ -779,45 +816,6 @@ func (b *lowBuilder) addGoogleCloudStorageConfig(v string) error { b.addPrefix("/cache/", "storage-filesystem", args{ "path": filepath.Join(tempDir(), "camli-cache"), }) - if b.high.PackRelated { - b.addPrefix("/bs-loose/", "storage-googlecloudstorage", args{ - "bucket": bucket + "/loose", - "auth": map[string]interface{}{ - "client_id": clientID, - "client_secret": secret, - "refresh_token": refreshToken, - }, - }) - b.addPrefix("/bs-packed/", "storage-googlecloudstorage", args{ - "bucket": bucket + "/packed", - "auth": map[string]interface{}{ - "client_id": clientID, - "client_secret": secret, - "refresh_token": refreshToken, - }, - }) - // If index is DBMS, then blobPackedIndex is in DBMS too. - // Otherwise blobPackedIndex is same file-based DB as the index, - // in same dir, but named packindex.dbtype. - blobPackedIndex, err := b.sortedStorageAt(dbBlobpackedIndex, filepath.Join(b.indexFileDir(), "packindex")) - if err != nil { - return err - } - b.addPrefix("/bs/", "storage-blobpacked", args{ - "smallBlobs": "/bs-loose/", - "largeBlobs": "/bs-packed/", - "metaIndex": blobPackedIndex, - }) - return nil - } - b.addPrefix("/bs/", "storage-googlecloudstorage", args{ - "bucket": bucket, - "auth": map[string]interface{}{ - "client_id": clientID, - "client_secret": secret, - "refresh_token": refreshToken, - }, - }) return nil } diff --git a/pkg/serverinit/testdata/packrelated_replicated-want.json b/pkg/serverinit/testdata/packrelated_replicated-want.json new file mode 100644 index 000000000..574f49d26 --- /dev/null +++ b/pkg/serverinit/testdata/packrelated_replicated-want.json @@ -0,0 +1,262 @@ +{ + "auth": "userpass:camlistore:pass3179", + "https": false, + "listen": "localhost:3179", + "prefixes": { + "/": { + "handler": "root", + "handlerArgs": { + "blobRoot": "/bs-and-maybe-also-index/", + "helpRoot": "/help/", + "jsonSignRoot": "/sighelper/", + "searchRoot": "/my-search/", + "shareRoot": "/share/", + "statusRoot": "/status/", + "stealth": false + } + }, + "/bs-and-index/": { + "handler": "storage-replica", + "handlerArgs": { + "backends": [ + "/bs/", + "/index/" + ] + } + }, + "/bs-and-maybe-also-index/": { + "handler": "storage-cond", + "handlerArgs": { + "read": "/bs/", + "write": { + "else": "/bs/", + "if": "isSchema", + "then": "/bs-and-index/" + } + } + }, + "/bs-loose/": { + "handler": "storage-filesystem", + "handlerArgs": { + "path": "/path/to/blobs/" + } + }, + "/bs-packed/": { + "handler": "storage-filesystem", + "handlerArgs": { + "path": "/path/to/blobs/packed" + } + }, + "/bs/": { + "handler": "storage-blobpacked", + "handlerArgs": { + "largeBlobs": "/bs-packed/", + "metaIndex": { + "file": "/path/to/blobs/packed/packindex.kv", + "type": "kv" + }, + "smallBlobs": "/bs-loose/" + } + }, + "/cache/": { + "handler": "storage-filesystem", + "handlerArgs": { + "path": "/path/to/blobs/cache" + } + }, + "/help/": { + "handler": "help" + }, + "/importer/": { + "handler": "importer", + "handlerArgs": {} + }, + "/index/": { + "handler": "storage-index", + "handlerArgs": { + "blobSource": "/bs/", + "storage": { + "file": "/path/to/indexkv.db", + "type": "kv" + } + } + }, + "/my-search/": { + "handler": "search", + "handlerArgs": { + "index": "/index/", + "owner": { + "identity": "2931A67C26F5ABDA", + "secringFile": "/path/to/secring" + }, + "slurpToMemory": true + } + }, + "/setup/": { + "handler": "setup" + }, + "/share/": { + "handler": "share", + "handlerArgs": { + "blobRoot": "/bs/", + "index": "/index/" + } + }, + "/sighelper/": { + "handler": "jsonsign", + "handlerArgs": { + "keyId": "2931A67C26F5ABDA", + "publicKeyDest": "/bs-and-index/", + "secretRing": "/path/to/secring" + } + }, + "/status/": { + "handler": "status" + }, + "/sto-b2-bs-loose/": { + "handler": "storage-b2", + "handlerArgs": { + "auth": { + "account_id": "b2account", + "application_key": "b2key" + }, + "bucket": "b2bucket/loose" + } + }, + "/sto-b2-bs-packed/": { + "handler": "storage-b2", + "handlerArgs": { + "auth": { + "account_id": "b2account", + "application_key": "b2key" + }, + "bucket": "b2bucket/packed" + } + }, + "/sto-b2/": { + "handler": "storage-blobpacked", + "handlerArgs": { + "largeBlobs": "/bs-packed/", + "metaIndex": { + "file": "/path/to/packindex.kv", + "type": "kv" + }, + "smallBlobs": "/bs-loose/" + } + }, + "/sto-googlecloudstorage-bs-loose/": { + "handler": "storage-googlecloudstorage", + "handlerArgs": { + "auth": { + "client_id": "gcsClientId", + "client_secret": "gcsClientSecret", + "refresh_token": "gcsRefreshToken" + }, + "bucket": "gcsBucketName/blobs/loose" + } + }, + "/sto-googlecloudstorage-bs-packed/": { + "handler": "storage-googlecloudstorage", + "handlerArgs": { + "auth": { + "client_id": "gcsClientId", + "client_secret": "gcsClientSecret", + "refresh_token": "gcsRefreshToken" + }, + "bucket": "gcsBucketName/blobs/packed" + } + }, + "/sto-s3-bs-loose/": { + "handler": "storage-s3", + "handlerArgs": { + "aws_access_key": "s3key", + "aws_secret_access_key": "s3secret", + "bucket": "s3bucket/loose" + } + }, + "/sto-s3-bs-packed/": { + "handler": "storage-s3", + "handlerArgs": { + "aws_access_key": "s3key", + "aws_secret_access_key": "s3secret", + "bucket": "s3bucket/packed" + } + }, + "/sto-s3/": { + "handler": "storage-blobpacked", + "handlerArgs": { + "largeBlobs": "/sto-s3-bs-packed/", + "metaIndex": { + "file": "/path/to/packindex.kv", + "type": "kv" + }, + "smallBlobs": "/sto-s3-bs-loose/" + } + }, + "/sto-googlecloudstorage/": { + "handler": "storage-blobpacked", + "handlerArgs": { + "largeBlobs": "/sto-googlecloudstorage-bs-packed/", + "metaIndex": { + "file": "/path/to/packindex.kv", + "type": "kv" + }, + "smallBlobs": "/sto-googlecloudstorage-bs-loose/" + } + }, + "/sync-to-b2/": { + "handler": "sync", + "handlerArgs": { + "from": "/bs/", + "queue": { + "file": "/path/to/blobs/sync-to-b2-queue.kv", + "type": "kv" + }, + "to": "/sto-b2/" + } + }, + "/sync-to-googlecloudstorage/": { + "handler": "sync", + "handlerArgs": { + "from": "/bs/", + "queue": { + "file": "/path/to/blobs/sync-to-googlecloud-queue.kv", + "type": "kv" + }, + "to": "/sto-googlecloudstorage/" + } + }, + "/sync-to-s3/": { + "handler": "sync", + "handlerArgs": { + "from": "/bs/", + "queue": { + "file": "/path/to/blobs/sync-to-s3-queue.kv", + "type": "kv" + }, + "to": "/sto-s3/" + } + }, + "/sync/": { + "handler": "sync", + "handlerArgs": { + "from": "/bs/", + "queue": { + "file": "/path/to/blobs/sync-to-index-queue.kv", + "type": "kv" + }, + "to": "/index/" + } + }, + "/ui/": { + "handler": "ui", + "handlerArgs": { + "cache": "/cache/", + "scaledImage": { + "file": "/path/to/blobs/thumbmeta.kv", + "type": "kv" + } + } + } + } +} diff --git a/pkg/serverinit/testdata/packrelated_replicated.json b/pkg/serverinit/testdata/packrelated_replicated.json new file mode 100644 index 000000000..6b3abdc70 --- /dev/null +++ b/pkg/serverinit/testdata/packrelated_replicated.json @@ -0,0 +1,14 @@ +{ + "listen": "localhost:3179", + "https": false, + "auth": "userpass:camlistore:pass3179", + "identity": "26F5ABDA", + "identitySecretRing": "/path/to/secring", + "kvIndexFile": "/path/to/indexkv.db", + "s3": "s3key:s3secret:s3bucket", + "b2": "b2account:b2key:b2bucket", + "googlecloudstorage": "gcsClientId:gcsClientSecret:gcsRefreshToken:gcsBucketName/blobs", + "blobPath": "/path/to/blobs/", + "packRelated": true, + "shareHandlerPath": "/share/" +}