Fixing tests:

* Adding 'vendor' argument to generate configuration.
* Update json testcases
This commit is contained in:
kalidor 2022-04-07 09:08:51 +02:00 committed by Brad Fitzpatrick
parent da7ae20271
commit 81682f009a
5 changed files with 31 additions and 31 deletions

View File

@ -538,10 +538,11 @@ func (b *lowBuilder) thatQueueUnlessMemory(thatQueue map[string]interface{}) (qu
return thatQueue
}
func (b *lowBuilder) addS3Config(s3 string) error {
func (b *lowBuilder) addS3Config(s3 string, vendor string) error {
f := strings.SplitN(s3, ":", 4)
if len(f) < 3 {
return errors.New(`genconfig: expected "s3" field to be of form "access_key_id:secret_access_key:bucket[/optional/dir][:hostname]"`)
m := fmt.Sprintf(`genconfig: expected "%s" field to be of form "access_key_id:secret_access_key:bucket[/optional/dir][:hostname]"`, vendor)
return errors.New(m)
}
accessKey, secret, bucket := f[0], f[1], f[2]
var hostname string
@ -551,7 +552,7 @@ func (b *lowBuilder) addS3Config(s3 string) error {
isReplica := b.hasPrefix("/bs/")
s3Prefix := "/bs/"
if isReplica {
s3Prefix = "/sto-s3/"
s3Prefix = fmt.Sprintf("/sto-%s/", vendor)
}
s3Args := func(bucket string) args {
@ -572,8 +573,8 @@ func (b *lowBuilder) addS3Config(s3 string) error {
bsLoose := "/bs-loose/"
bsPacked := "/bs-packed/"
if isReplica {
bsLoose = "/sto-s3-bs-loose/"
bsPacked = "/sto-s3-bs-packed/"
bsLoose = fmt.Sprintf("/sto-%s-bs-loose/", vendor)
bsPacked = fmt.Sprintf("/sto-%s-bs-packed/", vendor)
}
b.addPrefix(bsLoose, "storage-s3", s3Args(path.Join(bucket, "loose")))
@ -597,13 +598,15 @@ func (b *lowBuilder) addS3Config(s3 string) error {
if b.high.BlobPath == "" && !b.high.MemoryStorage {
panic("unexpected empty blobpath with sync-to-s3")
}
b.addPrefix("/sync-to-s3/", "sync", args{
p := fmt.Sprintf("/sync-to-%s/", vendor)
queue := fmt.Sprintf("sync-to-%s-queue.", vendor)
b.addPrefix(p, "sync", args{
"from": "/bs/",
"to": s3Prefix,
"queue": b.thatQueueUnlessMemory(
map[string]interface{}{
"type": b.kvFileType(),
"file": filepath.Join(b.high.BlobPath, "sync-to-s3-queue."+b.kvFileType()),
"file": filepath.Join(b.high.BlobPath, queue+b.kvFileType()),
}),
})
return nil
@ -619,7 +622,7 @@ func (b *lowBuilder) addS3Config(s3 string) error {
}
func (b *lowBuilder) addB2Config(b2 string) error {
return b.addS3Config(b2)
return b.addS3Config(b2, "b2")
}
func (b *lowBuilder) addGoogleDriveConfig(v string) error {
@ -1125,7 +1128,7 @@ func (b *lowBuilder) build() (*Config, error) {
}
if conf.S3 != "" {
if err := b.addS3Config(conf.S3); err != nil {
if err := b.addS3Config(conf.S3, "s3"); err != nil {
return nil, err
}
}

View File

@ -36,13 +36,12 @@
}
},
"/bs/": {
"handler": "storage-b2",
"handler": "storage-s3",
"handlerArgs": {
"auth": {
"account_id": "account",
"application_key": "key"
},
"bucket": "bucket"
"aws_access_key": "account",
"aws_secret_access_key": "key",
"bucket": "bucket",
"hostname": "endpoint"
}
},
"/cache/": {

View File

@ -5,7 +5,7 @@
"identity": "26F5ABDA",
"identitySecretRing": "/path/to/secring",
"kvIndexFile": "/path/to/indexkv.db",
"b2": "account:key:bucket",
"b2": "account:key:bucket:endpoint",
"replicateTo": [],
"publish": {},
"shareHandlerPath": "/share/"

View File

@ -114,34 +114,32 @@
"handler": "status"
},
"/sto-b2-bs-loose/": {
"handler": "storage-b2",
"handler": "storage-s3",
"handlerArgs": {
"auth": {
"account_id": "b2account",
"application_key": "b2key"
},
"bucket": "b2bucket/loose"
"aws_access_key": "b2account",
"aws_secret_access_key": "b2key",
"bucket": "b2bucket/loose",
"hostname": "b2endpoint"
}
},
"/sto-b2-bs-packed/": {
"handler": "storage-b2",
"handler": "storage-s3",
"handlerArgs": {
"auth": {
"account_id": "b2account",
"application_key": "b2key"
},
"bucket": "b2bucket/packed"
"aws_access_key": "b2account",
"aws_secret_access_key": "b2key",
"bucket": "b2bucket/packed",
"hostname": "b2endpoint"
}
},
"/sto-b2/": {
"handler": "storage-blobpacked",
"handlerArgs": {
"largeBlobs": "/bs-packed/",
"largeBlobs": "/sto-b2-bs-packed/",
"metaIndex": {
"file": "/path/to/packindex.kv",
"type": "kv"
},
"smallBlobs": "/bs-loose/"
"smallBlobs": "/sto-b2-bs-loose/"
}
},
"/sto-googlecloudstorage-bs-loose/": {

View File

@ -6,7 +6,7 @@
"identitySecretRing": "/path/to/secring",
"kvIndexFile": "/path/to/indexkv.db",
"s3": "s3key:s3secret:s3bucket",
"b2": "b2account:b2key:b2bucket",
"b2": "b2account:b2key:b2bucket:b2endpoint",
"googlecloudstorage": "gcsClientId:gcsClientSecret:gcsRefreshToken:gcsBucketName/blobs",
"blobPath": "/path/to/blobs/",
"packRelated": true,