mirror of https://github.com/perkeep/perkeep.git
genconfig: support blobpacked in cloud replicas
Cloud-backed storage currently does not use blobpacked if they are a replica. This causes issues if they become the main storage. Example use case where this would cause issues: - Create a configuration with s3, blobPath, and packRelated. - Add objects - Remove blobPath - `pk list` - All blobs are gone because the s3-only config uses blobpacked but the s3 replica was not.
This commit is contained in:
aviau
Alexandre Viau
parent
30af144717
commit
e2a6f3bc44
|
|
@ -546,18 +546,51 @@ func (b *lowBuilder) addS3Config(s3 string) error {
|
|||
hostname = f[3]
|
||||
}
|
||||
isReplica := b.hasPrefix("/bs/")
|
||||
s3Prefix := ""
|
||||
s3Args := args{
|
||||
"aws_access_key": accessKey,
|
||||
"aws_secret_access_key": secret,
|
||||
"bucket": bucket,
|
||||
}
|
||||
if hostname != "" {
|
||||
s3Args["hostname"] = hostname
|
||||
}
|
||||
s3Prefix := "/bs/"
|
||||
if isReplica {
|
||||
s3Prefix = "/sto-s3/"
|
||||
b.addPrefix(s3Prefix, "storage-s3", s3Args)
|
||||
}
|
||||
|
||||
s3Args := func(bucket string) args {
|
||||
a := args{
|
||||
"bucket": bucket,
|
||||
"aws_access_key": accessKey,
|
||||
"aws_secret_access_key": secret,
|
||||
}
|
||||
if hostname != "" {
|
||||
a["hostname"] = hostname
|
||||
}
|
||||
return a
|
||||
}
|
||||
|
||||
if !b.high.PackRelated {
|
||||
b.addPrefix(s3Prefix, "storage-s3", s3Args(bucket))
|
||||
} else {
|
||||
bsLoose := "/bs-loose/"
|
||||
bsPacked := "/bs-packed/"
|
||||
if isReplica {
|
||||
bsLoose = "/sto-s3-bs-loose/"
|
||||
bsPacked = "/sto-s3-bs-packed/"
|
||||
}
|
||||
|
||||
b.addPrefix(bsLoose, "storage-s3", s3Args(path.Join(bucket, "loose")))
|
||||
b.addPrefix(bsPacked, "storage-s3", s3Args(path.Join(bucket, "packed")))
|
||||
|
||||
// If index is DBMS, then blobPackedIndex is in DBMS too.
|
||||
// Otherwise blobPackedIndex is same file-based DB as the index,
|
||||
// in same dir, but named packindex.dbtype.
|
||||
blobPackedIndex, err := b.sortedStorageAt(dbBlobpackedIndex, filepath.Join(b.indexFileDir(), "packindex"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
b.addPrefix(s3Prefix, "storage-blobpacked", args{
|
||||
"smallBlobs": bsLoose,
|
||||
"largeBlobs": bsPacked,
|
||||
"metaIndex": blobPackedIndex,
|
||||
})
|
||||
}
|
||||
|
||||
if isReplica {
|
||||
if b.high.BlobPath == "" && !b.high.MemoryStorage {
|
||||
panic("unexpected empty blobpath with sync-to-s3")
|
||||
}
|
||||
|
|
@ -579,39 +612,6 @@ func (b *lowBuilder) addS3Config(s3 string) error {
|
|||
"path": filepath.Join(tempDir(), "camli-cache"),
|
||||
})
|
||||
|
||||
s3Prefix = "/bs/"
|
||||
if !b.high.PackRelated {
|
||||
b.addPrefix(s3Prefix, "storage-s3", s3Args)
|
||||
return nil
|
||||
}
|
||||
packedS3Args := func(bucket string) args {
|
||||
a := args{
|
||||
"bucket": bucket,
|
||||
"aws_access_key": accessKey,
|
||||
"aws_secret_access_key": secret,
|
||||
}
|
||||
if hostname != "" {
|
||||
a["hostname"] = hostname
|
||||
}
|
||||
return a
|
||||
}
|
||||
|
||||
b.addPrefix("/bs-loose/", "storage-s3", packedS3Args(path.Join(bucket, "loose")))
|
||||
b.addPrefix("/bs-packed/", "storage-s3", packedS3Args(path.Join(bucket, "packed")))
|
||||
|
||||
// If index is DBMS, then blobPackedIndex is in DBMS too.
|
||||
// Otherwise blobPackedIndex is same file-based DB as the index,
|
||||
// in same dir, but named packindex.dbtype.
|
||||
blobPackedIndex, err := b.sortedStorageAt(dbBlobpackedIndex, filepath.Join(b.indexFileDir(), "packindex"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
b.addPrefix(s3Prefix, "storage-blobpacked", args{
|
||||
"smallBlobs": "/bs-loose/",
|
||||
"largeBlobs": "/bs-packed/",
|
||||
"metaIndex": blobPackedIndex,
|
||||
})
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
@ -622,18 +622,50 @@ func (b *lowBuilder) addB2Config(b2 string) error {
|
|||
}
|
||||
account, key, bucket := f[0], f[1], f[2]
|
||||
isReplica := b.hasPrefix("/bs/")
|
||||
b2Prefix := ""
|
||||
b2Auth := map[string]interface{}{
|
||||
"account_id": account,
|
||||
"application_key": key,
|
||||
}
|
||||
b2Args := args{
|
||||
"auth": b2Auth,
|
||||
"bucket": bucket,
|
||||
}
|
||||
b2Prefix := "/bs/"
|
||||
if isReplica {
|
||||
b2Prefix = "/sto-b2/"
|
||||
b.addPrefix(b2Prefix, "storage-b2", b2Args)
|
||||
}
|
||||
|
||||
b2Args := func(bucket string) args {
|
||||
a := args{
|
||||
"bucket": bucket,
|
||||
"auth": map[string]interface{}{
|
||||
"account_id": account,
|
||||
"application_key": key,
|
||||
},
|
||||
}
|
||||
return a
|
||||
}
|
||||
|
||||
if !b.high.PackRelated {
|
||||
b.addPrefix(b2Prefix, "storage-b2", b2Args(bucket))
|
||||
} else {
|
||||
bsLoose := "/bs-loose/"
|
||||
bsPacked := "/bs-packed/"
|
||||
if isReplica {
|
||||
bsLoose = "/sto-b2-bs-loose/"
|
||||
bsPacked = "/sto-b2-bs-packed/"
|
||||
}
|
||||
|
||||
b.addPrefix(bsLoose, "storage-b2", b2Args(path.Join(bucket, "loose")))
|
||||
b.addPrefix(bsPacked, "storage-b2", b2Args(path.Join(bucket, "packed")))
|
||||
|
||||
// If index is DBMS, then blobPackedIndex is in DBMS too.
|
||||
// Otherwise blobPackedIndex is same file-based DB as the index,
|
||||
// in same dir, but named packindex.dbtype.
|
||||
blobPackedIndex, err := b.sortedStorageAt(dbBlobpackedIndex, filepath.Join(b.indexFileDir(), "packindex"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
b.addPrefix(b2Prefix, "storage-blobpacked", args{
|
||||
"smallBlobs": "/bs-loose/",
|
||||
"largeBlobs": "/bs-packed/",
|
||||
"metaIndex": blobPackedIndex,
|
||||
})
|
||||
}
|
||||
|
||||
if isReplica {
|
||||
if b.high.BlobPath == "" && !b.high.MemoryStorage {
|
||||
panic("unexpected empty blobpath with sync-to-b2")
|
||||
}
|
||||
|
|
@ -653,38 +685,6 @@ func (b *lowBuilder) addB2Config(b2 string) error {
|
|||
"path": filepath.Join(tempDir(), "camli-cache"),
|
||||
})
|
||||
|
||||
b2Prefix = "/bs/"
|
||||
if !b.high.PackRelated {
|
||||
b.addPrefix(b2Prefix, "storage-b2", b2Args)
|
||||
return nil
|
||||
}
|
||||
packedB2Args := func(bucket string) args {
|
||||
a := args{
|
||||
"bucket": bucket,
|
||||
"auth": map[string]interface{}{
|
||||
"account_id": account,
|
||||
"application_key": key,
|
||||
},
|
||||
}
|
||||
return a
|
||||
}
|
||||
|
||||
b.addPrefix("/bs-loose/", "storage-b2", packedB2Args(path.Join(bucket, "loose")))
|
||||
b.addPrefix("/bs-packed/", "storage-b2", packedB2Args(path.Join(bucket, "packed")))
|
||||
|
||||
// If index is DBMS, then blobPackedIndex is in DBMS too.
|
||||
// Otherwise blobPackedIndex is same file-based DB as the index,
|
||||
// in same dir, but named packindex.dbtype.
|
||||
blobPackedIndex, err := b.sortedStorageAt(dbBlobpackedIndex, filepath.Join(b.indexFileDir(), "packindex"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
b.addPrefix(b2Prefix, "storage-blobpacked", args{
|
||||
"smallBlobs": "/bs-loose/",
|
||||
"largeBlobs": "/bs-packed/",
|
||||
"metaIndex": blobPackedIndex,
|
||||
})
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
@ -752,17 +752,54 @@ func (b *lowBuilder) addGoogleCloudStorageConfig(v string) error {
|
|||
}
|
||||
|
||||
isReplica := b.hasPrefix("/bs/")
|
||||
gsPrefix := "/bs/"
|
||||
if isReplica {
|
||||
gsPrefix := "/sto-googlecloudstorage/"
|
||||
b.addPrefix(gsPrefix, "storage-googlecloudstorage", args{
|
||||
gsPrefix = "/sto-googlecloudstorage/"
|
||||
}
|
||||
|
||||
gsArgs := func(bucket string) args {
|
||||
a := args{
|
||||
"bucket": bucket,
|
||||
"auth": map[string]interface{}{
|
||||
"client_id": clientID,
|
||||
"client_secret": secret,
|
||||
"refresh_token": refreshToken,
|
||||
},
|
||||
})
|
||||
}
|
||||
return a
|
||||
}
|
||||
|
||||
if !b.high.PackRelated {
|
||||
b.addPrefix(gsPrefix, "storage-googlecloudstorage", gsArgs(bucket))
|
||||
} else {
|
||||
bsLoose := "/bs-loose/"
|
||||
bsPacked := "/bs-packed/"
|
||||
if isReplica {
|
||||
bsLoose = "/sto-googlecloudstorage-bs-loose/"
|
||||
bsPacked = "/sto-googlecloudstorage-bs-packed/"
|
||||
}
|
||||
|
||||
b.addPrefix(bsLoose, "storage-googlecloudstorage", gsArgs(path.Join(bucket, "loose")))
|
||||
b.addPrefix(bsPacked, "storage-googlecloudstorage", gsArgs(path.Join(bucket, "packed")))
|
||||
|
||||
// If index is DBMS, then blobPackedIndex is in DBMS too.
|
||||
// Otherwise blobPackedIndex is same file-based DB as the index,
|
||||
// in same dir, but named packindex.dbtype.
|
||||
blobPackedIndex, err := b.sortedStorageAt(dbBlobpackedIndex, filepath.Join(b.indexFileDir(), "packindex"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
b.addPrefix(gsPrefix, "storage-blobpacked", args{
|
||||
"smallBlobs": bsLoose,
|
||||
"largeBlobs": bsPacked,
|
||||
"metaIndex": blobPackedIndex,
|
||||
})
|
||||
}
|
||||
|
||||
if isReplica {
|
||||
if b.high.BlobPath == "" && !b.high.MemoryStorage {
|
||||
panic("unexpected empty blobpath with sync-to-googlecloudstorage")
|
||||
}
|
||||
b.addPrefix("/sync-to-googlecloudstorage/", "sync", args{
|
||||
"from": "/bs/",
|
||||
"to": gsPrefix,
|
||||
|
|
@ -779,45 +816,6 @@ func (b *lowBuilder) addGoogleCloudStorageConfig(v string) error {
|
|||
b.addPrefix("/cache/", "storage-filesystem", args{
|
||||
"path": filepath.Join(tempDir(), "camli-cache"),
|
||||
})
|
||||
if b.high.PackRelated {
|
||||
b.addPrefix("/bs-loose/", "storage-googlecloudstorage", args{
|
||||
"bucket": bucket + "/loose",
|
||||
"auth": map[string]interface{}{
|
||||
"client_id": clientID,
|
||||
"client_secret": secret,
|
||||
"refresh_token": refreshToken,
|
||||
},
|
||||
})
|
||||
b.addPrefix("/bs-packed/", "storage-googlecloudstorage", args{
|
||||
"bucket": bucket + "/packed",
|
||||
"auth": map[string]interface{}{
|
||||
"client_id": clientID,
|
||||
"client_secret": secret,
|
||||
"refresh_token": refreshToken,
|
||||
},
|
||||
})
|
||||
// If index is DBMS, then blobPackedIndex is in DBMS too.
|
||||
// Otherwise blobPackedIndex is same file-based DB as the index,
|
||||
// in same dir, but named packindex.dbtype.
|
||||
blobPackedIndex, err := b.sortedStorageAt(dbBlobpackedIndex, filepath.Join(b.indexFileDir(), "packindex"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
b.addPrefix("/bs/", "storage-blobpacked", args{
|
||||
"smallBlobs": "/bs-loose/",
|
||||
"largeBlobs": "/bs-packed/",
|
||||
"metaIndex": blobPackedIndex,
|
||||
})
|
||||
return nil
|
||||
}
|
||||
b.addPrefix("/bs/", "storage-googlecloudstorage", args{
|
||||
"bucket": bucket,
|
||||
"auth": map[string]interface{}{
|
||||
"client_id": clientID,
|
||||
"client_secret": secret,
|
||||
"refresh_token": refreshToken,
|
||||
},
|
||||
})
|
||||
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,262 @@
|
|||
{
|
||||
"auth": "userpass:camlistore:pass3179",
|
||||
"https": false,
|
||||
"listen": "localhost:3179",
|
||||
"prefixes": {
|
||||
"/": {
|
||||
"handler": "root",
|
||||
"handlerArgs": {
|
||||
"blobRoot": "/bs-and-maybe-also-index/",
|
||||
"helpRoot": "/help/",
|
||||
"jsonSignRoot": "/sighelper/",
|
||||
"searchRoot": "/my-search/",
|
||||
"shareRoot": "/share/",
|
||||
"statusRoot": "/status/",
|
||||
"stealth": false
|
||||
}
|
||||
},
|
||||
"/bs-and-index/": {
|
||||
"handler": "storage-replica",
|
||||
"handlerArgs": {
|
||||
"backends": [
|
||||
"/bs/",
|
||||
"/index/"
|
||||
]
|
||||
}
|
||||
},
|
||||
"/bs-and-maybe-also-index/": {
|
||||
"handler": "storage-cond",
|
||||
"handlerArgs": {
|
||||
"read": "/bs/",
|
||||
"write": {
|
||||
"else": "/bs/",
|
||||
"if": "isSchema",
|
||||
"then": "/bs-and-index/"
|
||||
}
|
||||
}
|
||||
},
|
||||
"/bs-loose/": {
|
||||
"handler": "storage-filesystem",
|
||||
"handlerArgs": {
|
||||
"path": "/path/to/blobs/"
|
||||
}
|
||||
},
|
||||
"/bs-packed/": {
|
||||
"handler": "storage-filesystem",
|
||||
"handlerArgs": {
|
||||
"path": "/path/to/blobs/packed"
|
||||
}
|
||||
},
|
||||
"/bs/": {
|
||||
"handler": "storage-blobpacked",
|
||||
"handlerArgs": {
|
||||
"largeBlobs": "/bs-packed/",
|
||||
"metaIndex": {
|
||||
"file": "/path/to/blobs/packed/packindex.kv",
|
||||
"type": "kv"
|
||||
},
|
||||
"smallBlobs": "/bs-loose/"
|
||||
}
|
||||
},
|
||||
"/cache/": {
|
||||
"handler": "storage-filesystem",
|
||||
"handlerArgs": {
|
||||
"path": "/path/to/blobs/cache"
|
||||
}
|
||||
},
|
||||
"/help/": {
|
||||
"handler": "help"
|
||||
},
|
||||
"/importer/": {
|
||||
"handler": "importer",
|
||||
"handlerArgs": {}
|
||||
},
|
||||
"/index/": {
|
||||
"handler": "storage-index",
|
||||
"handlerArgs": {
|
||||
"blobSource": "/bs/",
|
||||
"storage": {
|
||||
"file": "/path/to/indexkv.db",
|
||||
"type": "kv"
|
||||
}
|
||||
}
|
||||
},
|
||||
"/my-search/": {
|
||||
"handler": "search",
|
||||
"handlerArgs": {
|
||||
"index": "/index/",
|
||||
"owner": {
|
||||
"identity": "2931A67C26F5ABDA",
|
||||
"secringFile": "/path/to/secring"
|
||||
},
|
||||
"slurpToMemory": true
|
||||
}
|
||||
},
|
||||
"/setup/": {
|
||||
"handler": "setup"
|
||||
},
|
||||
"/share/": {
|
||||
"handler": "share",
|
||||
"handlerArgs": {
|
||||
"blobRoot": "/bs/",
|
||||
"index": "/index/"
|
||||
}
|
||||
},
|
||||
"/sighelper/": {
|
||||
"handler": "jsonsign",
|
||||
"handlerArgs": {
|
||||
"keyId": "2931A67C26F5ABDA",
|
||||
"publicKeyDest": "/bs-and-index/",
|
||||
"secretRing": "/path/to/secring"
|
||||
}
|
||||
},
|
||||
"/status/": {
|
||||
"handler": "status"
|
||||
},
|
||||
"/sto-b2-bs-loose/": {
|
||||
"handler": "storage-b2",
|
||||
"handlerArgs": {
|
||||
"auth": {
|
||||
"account_id": "b2account",
|
||||
"application_key": "b2key"
|
||||
},
|
||||
"bucket": "b2bucket/loose"
|
||||
}
|
||||
},
|
||||
"/sto-b2-bs-packed/": {
|
||||
"handler": "storage-b2",
|
||||
"handlerArgs": {
|
||||
"auth": {
|
||||
"account_id": "b2account",
|
||||
"application_key": "b2key"
|
||||
},
|
||||
"bucket": "b2bucket/packed"
|
||||
}
|
||||
},
|
||||
"/sto-b2/": {
|
||||
"handler": "storage-blobpacked",
|
||||
"handlerArgs": {
|
||||
"largeBlobs": "/bs-packed/",
|
||||
"metaIndex": {
|
||||
"file": "/path/to/packindex.kv",
|
||||
"type": "kv"
|
||||
},
|
||||
"smallBlobs": "/bs-loose/"
|
||||
}
|
||||
},
|
||||
"/sto-googlecloudstorage-bs-loose/": {
|
||||
"handler": "storage-googlecloudstorage",
|
||||
"handlerArgs": {
|
||||
"auth": {
|
||||
"client_id": "gcsClientId",
|
||||
"client_secret": "gcsClientSecret",
|
||||
"refresh_token": "gcsRefreshToken"
|
||||
},
|
||||
"bucket": "gcsBucketName/blobs/loose"
|
||||
}
|
||||
},
|
||||
"/sto-googlecloudstorage-bs-packed/": {
|
||||
"handler": "storage-googlecloudstorage",
|
||||
"handlerArgs": {
|
||||
"auth": {
|
||||
"client_id": "gcsClientId",
|
||||
"client_secret": "gcsClientSecret",
|
||||
"refresh_token": "gcsRefreshToken"
|
||||
},
|
||||
"bucket": "gcsBucketName/blobs/packed"
|
||||
}
|
||||
},
|
||||
"/sto-s3-bs-loose/": {
|
||||
"handler": "storage-s3",
|
||||
"handlerArgs": {
|
||||
"aws_access_key": "s3key",
|
||||
"aws_secret_access_key": "s3secret",
|
||||
"bucket": "s3bucket/loose"
|
||||
}
|
||||
},
|
||||
"/sto-s3-bs-packed/": {
|
||||
"handler": "storage-s3",
|
||||
"handlerArgs": {
|
||||
"aws_access_key": "s3key",
|
||||
"aws_secret_access_key": "s3secret",
|
||||
"bucket": "s3bucket/packed"
|
||||
}
|
||||
},
|
||||
"/sto-s3/": {
|
||||
"handler": "storage-blobpacked",
|
||||
"handlerArgs": {
|
||||
"largeBlobs": "/sto-s3-bs-packed/",
|
||||
"metaIndex": {
|
||||
"file": "/path/to/packindex.kv",
|
||||
"type": "kv"
|
||||
},
|
||||
"smallBlobs": "/sto-s3-bs-loose/"
|
||||
}
|
||||
},
|
||||
"/sto-googlecloudstorage/": {
|
||||
"handler": "storage-blobpacked",
|
||||
"handlerArgs": {
|
||||
"largeBlobs": "/sto-googlecloudstorage-bs-packed/",
|
||||
"metaIndex": {
|
||||
"file": "/path/to/packindex.kv",
|
||||
"type": "kv"
|
||||
},
|
||||
"smallBlobs": "/sto-googlecloudstorage-bs-loose/"
|
||||
}
|
||||
},
|
||||
"/sync-to-b2/": {
|
||||
"handler": "sync",
|
||||
"handlerArgs": {
|
||||
"from": "/bs/",
|
||||
"queue": {
|
||||
"file": "/path/to/blobs/sync-to-b2-queue.kv",
|
||||
"type": "kv"
|
||||
},
|
||||
"to": "/sto-b2/"
|
||||
}
|
||||
},
|
||||
"/sync-to-googlecloudstorage/": {
|
||||
"handler": "sync",
|
||||
"handlerArgs": {
|
||||
"from": "/bs/",
|
||||
"queue": {
|
||||
"file": "/path/to/blobs/sync-to-googlecloud-queue.kv",
|
||||
"type": "kv"
|
||||
},
|
||||
"to": "/sto-googlecloudstorage/"
|
||||
}
|
||||
},
|
||||
"/sync-to-s3/": {
|
||||
"handler": "sync",
|
||||
"handlerArgs": {
|
||||
"from": "/bs/",
|
||||
"queue": {
|
||||
"file": "/path/to/blobs/sync-to-s3-queue.kv",
|
||||
"type": "kv"
|
||||
},
|
||||
"to": "/sto-s3/"
|
||||
}
|
||||
},
|
||||
"/sync/": {
|
||||
"handler": "sync",
|
||||
"handlerArgs": {
|
||||
"from": "/bs/",
|
||||
"queue": {
|
||||
"file": "/path/to/blobs/sync-to-index-queue.kv",
|
||||
"type": "kv"
|
||||
},
|
||||
"to": "/index/"
|
||||
}
|
||||
},
|
||||
"/ui/": {
|
||||
"handler": "ui",
|
||||
"handlerArgs": {
|
||||
"cache": "/cache/",
|
||||
"scaledImage": {
|
||||
"file": "/path/to/blobs/thumbmeta.kv",
|
||||
"type": "kv"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
{
|
||||
"listen": "localhost:3179",
|
||||
"https": false,
|
||||
"auth": "userpass:camlistore:pass3179",
|
||||
"identity": "26F5ABDA",
|
||||
"identitySecretRing": "/path/to/secring",
|
||||
"kvIndexFile": "/path/to/indexkv.db",
|
||||
"s3": "s3key:s3secret:s3bucket",
|
||||
"b2": "b2account:b2key:b2bucket",
|
||||
"googlecloudstorage": "gcsClientId:gcsClientSecret:gcsRefreshToken:gcsBucketName/blobs",
|
||||
"blobPath": "/path/to/blobs/",
|
||||
"packRelated": true,
|
||||
"shareHandlerPath": "/share/"
|
||||
}
|
||||
Loading…
Reference in New Issue