From b78ce17e13445068c7960fa0d392fcdcf2e708db Mon Sep 17 00:00:00 2001 From: mpl Date: Mon, 2 Feb 2015 15:48:20 +0100 Subject: [PATCH] serverinit: add genconfig for blobpacked on Google Cloud Storage. And enable it (blobpacked) on GCE. Change-Id: I7b51c186325e80987e48d32392d88bb4c30ee3d5 --- misc/docker/camlistored/Makefile | 9 +- pkg/serverinit/env.go | 2 + pkg/serverinit/genconfig.go | 83 ++++++---- .../testdata/blobpacked_googlecloud-want.json | 152 ++++++++++++++++++ .../testdata/blobpacked_googlecloud.json | 17 ++ pkg/types/serverconfig/config.go | 2 +- 6 files changed, 233 insertions(+), 32 deletions(-) create mode 100644 pkg/serverinit/testdata/blobpacked_googlecloud-want.json create mode 100644 pkg/serverinit/testdata/blobpacked_googlecloud.json diff --git a/misc/docker/camlistored/Makefile b/misc/docker/camlistored/Makefile index fc7e3dc79..7e1954b74 100644 --- a/misc/docker/camlistored/Makefile +++ b/misc/docker/camlistored/Makefile @@ -1,10 +1,13 @@ docker: Dockerfile camlistored - docker build -t camlistore/camlistored . + docker build -t camlistored . .PHONY: camlistored camlistored: (cd ../../../; go run make.go --docker_camlistored) -push: docker - docker push camlistore/camlistored +camlistored.tar.gz: docker + docker save camlistored | gzip -c > $@ + +push: camlistored.tar.gz + gsutil cp $< gs://camlistore-release/docker/ diff --git a/pkg/serverinit/env.go b/pkg/serverinit/env.go index 08fe531d1..3d0f49a6c 100644 --- a/pkg/serverinit/env.go +++ b/pkg/serverinit/env.go @@ -71,6 +71,7 @@ func DefaultEnvConfig() (*Config, error) { IdentitySecretRing: secRing, GoogleCloudStorage: ":" + strings.TrimPrefix(blobBucket, "gs://"), DBNames: map[string]string{}, + PackRelated: true, } // Detect a linked Docker MySQL container. It must have alias "mysqldb". @@ -79,6 +80,7 @@ func DefaultEnvConfig() (*Config, error) { highConf.MySQL = "root@" + hostPort + ":" // no password highConf.DBNames["queue-sync-to-index"] = "sync_index_queue" highConf.DBNames["ui_thumbcache"] = "ui_thumbmeta_cache" + highConf.DBNames["blobpacked_index"] = "blobpacked_index" } else { // TODO: also detect Cloud SQL. highConf.KVFile = "/index.kv" diff --git a/pkg/serverinit/genconfig.go b/pkg/serverinit/genconfig.go index eb845058f..1afc85dc7 100644 --- a/pkg/serverinit/genconfig.go +++ b/pkg/serverinit/genconfig.go @@ -87,7 +87,7 @@ func (b *lowBuilder) runIndex() bool { return b.high.RunIndex.Get() } func (b *lowBuilder) copyIndexToMemory() bool { return b.high.CopyIndexToMemory.Get() } // dbName returns which database to use for the provided user ("of"). -// The user key be a key as describe in pkg/types/serverconfig/config.go's +// The user should be a key as described in pkg/types/serverconfig/config.go's // description of DBNames: "index", "queue-sync-to-index", etc. func (b *lowBuilder) dbName(of string) string { if v, ok := b.high.DBNames[of]; ok && v != "" { @@ -443,34 +443,18 @@ func (b *lowBuilder) addGoogleCloudStorageConfig(v string) error { clientID = "auto" } - if b.high.PackRelated { - // TODO(mpl): implement - return errors.New("TODO: finish genconfig support for GCS+blobpacked") - } - - isPrimary := !b.hasPrefix("/bs/") - gsPrefix := "" - if isPrimary { - gsPrefix = "/bs/" - } else { - gsPrefix = "/sto-googlecloudstorage/" - } - - b.addPrefix(gsPrefix, "storage-googlecloudstorage", args{ - "bucket": bucket, - "auth": map[string]interface{}{ - "client_id": clientID, - "client_secret": secret, - "refresh_token": refreshToken, - }, - }) - - if isPrimary { - // TODO: cacheBucket like s3CacheBucket? - b.addPrefix("/cache/", "storage-filesystem", args{ - "path": filepath.Join(tempDir(), "camli-cache"), + isReplica := b.hasPrefix("/bs/") + if isReplica { + gsPrefix := "/sto-googlecloudstorage/" + b.addPrefix(gsPrefix, "storage-googlecloudstorage", args{ + "bucket": bucket, + "auth": map[string]interface{}{ + "client_id": clientID, + "client_secret": secret, + "refresh_token": refreshToken, + }, }) - } else { + b.addPrefix("/sync-to-googlecloudstorage/", "sync", args{ "from": "/bs/", "to": gsPrefix, @@ -480,7 +464,50 @@ func (b *lowBuilder) addGoogleCloudStorageConfig(v string) error { "file": filepath.Join(b.high.BlobPath, "sync-to-googlecloud-queue.kv"), }), }) + return nil } + + // TODO: cacheBucket like s3CacheBucket? + b.addPrefix("/cache/", "storage-filesystem", args{ + "path": filepath.Join(tempDir(), "camli-cache"), + }) + if b.high.PackRelated { + b.addPrefix("/bs-loose/", "storage-googlecloudstorage", args{ + "bucket": bucket + "/loose", + "auth": map[string]interface{}{ + "client_id": clientID, + "client_secret": secret, + "refresh_token": refreshToken, + }, + }) + b.addPrefix("/bs-packed/", "storage-googlecloudstorage", args{ + "bucket": bucket + "/packed", + "auth": map[string]interface{}{ + "client_id": clientID, + "client_secret": secret, + "refresh_token": refreshToken, + }, + }) + blobPackedIndex, err := b.sortedStorageAt("blobpacked_index", "") + if err != nil { + return err + } + b.addPrefix("/bs/", "storage-blobpacked", args{ + "smallBlobs": "/bs-loose/", + "largeBlobs": "/bs-packed/", + "metaIndex": blobPackedIndex, + }) + return nil + } + b.addPrefix("/bs/", "storage-googlecloudstorage", args{ + "bucket": bucket, + "auth": map[string]interface{}{ + "client_id": clientID, + "client_secret": secret, + "refresh_token": refreshToken, + }, + }) + return nil } diff --git a/pkg/serverinit/testdata/blobpacked_googlecloud-want.json b/pkg/serverinit/testdata/blobpacked_googlecloud-want.json new file mode 100644 index 000000000..4b9b1ab49 --- /dev/null +++ b/pkg/serverinit/testdata/blobpacked_googlecloud-want.json @@ -0,0 +1,152 @@ +{ + "auth": "userpass:camlistore:pass3179", + "https": false, + "listen": "localhost:3179", + "prefixes": { + "/": { + "handler": "root", + "handlerArgs": { + "blobRoot": "/bs-and-maybe-also-index/", + "ownerName": "Alice", + "searchRoot": "/my-search/", + "statusRoot": "/status/", + "stealth": false + } + }, + "/bs-and-index/": { + "handler": "storage-replica", + "handlerArgs": { + "backends": [ + "/bs/", + "/index/" + ] + } + }, + "/bs-and-maybe-also-index/": { + "handler": "storage-cond", + "handlerArgs": { + "read": "/bs/", + "write": { + "else": "/bs/", + "if": "isSchema", + "then": "/bs-and-index/" + } + } + }, + "/bs/": { + "handler": "storage-blobpacked", + "handlerArgs": { + "smallBlobs": "/bs-loose/", + "largeBlobs": "/bs-packed/", + "metaIndex": { + "database": "blobpacked_index", + "host": "localhost", + "password": "root", + "type": "mysql", + "user": "root" + } + } + }, + "/bs-loose/": { + "handler": "storage-googlecloudstorage", + "handlerArgs": { + "auth": { + "client_id": "clientId", + "client_secret": "clientSecret", + "refresh_token": "refreshToken" + }, + "bucket": "bucketName/blobs/loose" + } + }, + "/bs-packed/": { + "handler": "storage-googlecloudstorage", + "handlerArgs": { + "auth": { + "client_id": "clientId", + "client_secret": "clientSecret", + "refresh_token": "refreshToken" + }, + "bucket": "bucketName/blobs/packed" + } + }, + "/cache/": { + "handler": "storage-filesystem", + "handlerArgs": { + "path": "/tmp/camli-cache" + } + }, + "/importer/": { + "handler": "importer", + "handlerArgs": {} + }, + "/index/": { + "handler": "storage-index", + "handlerArgs": { + "blobSource": "/bs/", + "storage": { + "database": "camlitest", + "host": "localhost", + "password": "root", + "type": "mysql", + "user": "root" + } + } + }, + "/my-search/": { + "handler": "search", + "handlerArgs": { + "index": "/index/", + "owner": "sha1-f2b0b7da718b97ce8c31591d8ed4645c777f3ef4", + "slurpToMemory": true + } + }, + "/setup/": { + "handler": "setup" + }, + "/share/": { + "handler": "share", + "handlerArgs": { + "blobRoot": "/bs/" + } + }, + "/sighelper/": { + "handler": "jsonsign", + "handlerArgs": { + "keyId": "26F5ABDA", + "publicKeyDest": "/bs-and-index/", + "secretRing": "/path/to/secring" + } + }, + "/status/": { + "handler": "status" + }, + "/sync/": { + "handler": "sync", + "handlerArgs": { + "from": "/bs/", + "queue": { + "database": "sync_index_queue", + "type": "mysql", + "host": "localhost", + "user": "root", + "password": "root" + }, + "to": "/index/" + } + }, + "/ui/": { + "handler": "ui", + "handlerArgs": { + "cache": "/cache/", + "jsonSignRoot": "/sighelper/", + "scaledImage": { + "database": "ui_thumbmeta_cache", + "host": "localhost", + "type": "mysql", + "user": "root", + "password": "root" + } + } + } + } +} diff --git a/pkg/serverinit/testdata/blobpacked_googlecloud.json b/pkg/serverinit/testdata/blobpacked_googlecloud.json new file mode 100644 index 000000000..f32df4796 --- /dev/null +++ b/pkg/serverinit/testdata/blobpacked_googlecloud.json @@ -0,0 +1,17 @@ +{ + "listen": "localhost:3179", + "auth": "userpass:camlistore:pass3179", + "googlecloudstorage": "clientId:clientSecret:refreshToken:bucketName/blobs", + "packRelated": true, + "dbNames": { + "index": "camlitest", + "queue-sync-to-index": "sync_index_queue", + "blobpacked_index": "blobpacked_index", + "ui_thumbcache": "ui_thumbmeta_cache" + }, + "mysql": "root@localhost:root", + "identity": "26F5ABDA", + "identitySecretRing": "/path/to/secring", + "ownerName": "Alice", + "shareHandlerPath": "/share/" +} diff --git a/pkg/types/serverconfig/config.go b/pkg/types/serverconfig/config.go index 4171aafa4..a6d290624 100644 --- a/pkg/types/serverconfig/config.go +++ b/pkg/types/serverconfig/config.go @@ -62,7 +62,7 @@ type Config struct { CopyIndexToMemory types.InvertedBool `json:"copyIndexToMemory,omitempty"` // copy disk-based index to memory on start-up. MemoryIndex bool `json:"memoryIndex,omitempty"` // use memory-only indexer. DBName string `json:"dbname,omitempty"` // name of the database for mysql, postgres, mongo. - LevelDB string `json:"levelDB,omitempty"` // path to the levelDB file, for indexing with github.com/syndtr/goleveldb. + LevelDB string `json:"levelDB,omitempty"` // path to the levelDB directory, for indexing with github.com/syndtr/goleveldb. KVFile string `json:"kvIndexFile,omitempty"` // path to the kv file, for indexing with github.com/cznic/kv. MySQL string `json:"mysql,omitempty"` // MySQL credentials (username@host:password), for indexing with MySQL. Mongo string `json:"mongo,omitempty"` // MongoDB credentials ([username:password@]host), for indexing with MongoDB.