mirror of https://github.com/perkeep/perkeep.git
serverinit: add genconfig for blobpacked on Google Cloud Storage.
And enable it (blobpacked) on GCE. Change-Id: I7b51c186325e80987e48d32392d88bb4c30ee3d5
This commit is contained in:
mpl
parent
ec6bfeb586
commit
b78ce17e13
|
|
@ -1,10 +1,13 @@
|
|||
docker: Dockerfile camlistored
|
||||
docker build -t camlistore/camlistored .
|
||||
docker build -t camlistored .
|
||||
|
||||
.PHONY: camlistored
|
||||
|
||||
camlistored:
|
||||
(cd ../../../; go run make.go --docker_camlistored)
|
||||
|
||||
push: docker
|
||||
docker push camlistore/camlistored
|
||||
camlistored.tar.gz: docker
|
||||
docker save camlistored | gzip -c > $@
|
||||
|
||||
push: camlistored.tar.gz
|
||||
gsutil cp $< gs://camlistore-release/docker/
|
||||
|
|
|
|||
|
|
@ -71,6 +71,7 @@ func DefaultEnvConfig() (*Config, error) {
|
|||
IdentitySecretRing: secRing,
|
||||
GoogleCloudStorage: ":" + strings.TrimPrefix(blobBucket, "gs://"),
|
||||
DBNames: map[string]string{},
|
||||
PackRelated: true,
|
||||
}
|
||||
|
||||
// Detect a linked Docker MySQL container. It must have alias "mysqldb".
|
||||
|
|
@ -79,6 +80,7 @@ func DefaultEnvConfig() (*Config, error) {
|
|||
highConf.MySQL = "root@" + hostPort + ":" // no password
|
||||
highConf.DBNames["queue-sync-to-index"] = "sync_index_queue"
|
||||
highConf.DBNames["ui_thumbcache"] = "ui_thumbmeta_cache"
|
||||
highConf.DBNames["blobpacked_index"] = "blobpacked_index"
|
||||
} else {
|
||||
// TODO: also detect Cloud SQL.
|
||||
highConf.KVFile = "/index.kv"
|
||||
|
|
|
|||
|
|
@ -87,7 +87,7 @@ func (b *lowBuilder) runIndex() bool { return b.high.RunIndex.Get() }
|
|||
func (b *lowBuilder) copyIndexToMemory() bool { return b.high.CopyIndexToMemory.Get() }
|
||||
|
||||
// dbName returns which database to use for the provided user ("of").
|
||||
// The user key be a key as describe in pkg/types/serverconfig/config.go's
|
||||
// The user should be a key as described in pkg/types/serverconfig/config.go's
|
||||
// description of DBNames: "index", "queue-sync-to-index", etc.
|
||||
func (b *lowBuilder) dbName(of string) string {
|
||||
if v, ok := b.high.DBNames[of]; ok && v != "" {
|
||||
|
|
@ -443,19 +443,9 @@ func (b *lowBuilder) addGoogleCloudStorageConfig(v string) error {
|
|||
clientID = "auto"
|
||||
}
|
||||
|
||||
if b.high.PackRelated {
|
||||
// TODO(mpl): implement
|
||||
return errors.New("TODO: finish genconfig support for GCS+blobpacked")
|
||||
}
|
||||
|
||||
isPrimary := !b.hasPrefix("/bs/")
|
||||
gsPrefix := ""
|
||||
if isPrimary {
|
||||
gsPrefix = "/bs/"
|
||||
} else {
|
||||
gsPrefix = "/sto-googlecloudstorage/"
|
||||
}
|
||||
|
||||
isReplica := b.hasPrefix("/bs/")
|
||||
if isReplica {
|
||||
gsPrefix := "/sto-googlecloudstorage/"
|
||||
b.addPrefix(gsPrefix, "storage-googlecloudstorage", args{
|
||||
"bucket": bucket,
|
||||
"auth": map[string]interface{}{
|
||||
|
|
@ -465,12 +455,6 @@ func (b *lowBuilder) addGoogleCloudStorageConfig(v string) error {
|
|||
},
|
||||
})
|
||||
|
||||
if isPrimary {
|
||||
// TODO: cacheBucket like s3CacheBucket?
|
||||
b.addPrefix("/cache/", "storage-filesystem", args{
|
||||
"path": filepath.Join(tempDir(), "camli-cache"),
|
||||
})
|
||||
} else {
|
||||
b.addPrefix("/sync-to-googlecloudstorage/", "sync", args{
|
||||
"from": "/bs/",
|
||||
"to": gsPrefix,
|
||||
|
|
@ -480,7 +464,50 @@ func (b *lowBuilder) addGoogleCloudStorageConfig(v string) error {
|
|||
"file": filepath.Join(b.high.BlobPath, "sync-to-googlecloud-queue.kv"),
|
||||
}),
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
// TODO: cacheBucket like s3CacheBucket?
|
||||
b.addPrefix("/cache/", "storage-filesystem", args{
|
||||
"path": filepath.Join(tempDir(), "camli-cache"),
|
||||
})
|
||||
if b.high.PackRelated {
|
||||
b.addPrefix("/bs-loose/", "storage-googlecloudstorage", args{
|
||||
"bucket": bucket + "/loose",
|
||||
"auth": map[string]interface{}{
|
||||
"client_id": clientID,
|
||||
"client_secret": secret,
|
||||
"refresh_token": refreshToken,
|
||||
},
|
||||
})
|
||||
b.addPrefix("/bs-packed/", "storage-googlecloudstorage", args{
|
||||
"bucket": bucket + "/packed",
|
||||
"auth": map[string]interface{}{
|
||||
"client_id": clientID,
|
||||
"client_secret": secret,
|
||||
"refresh_token": refreshToken,
|
||||
},
|
||||
})
|
||||
blobPackedIndex, err := b.sortedStorageAt("blobpacked_index", "")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
b.addPrefix("/bs/", "storage-blobpacked", args{
|
||||
"smallBlobs": "/bs-loose/",
|
||||
"largeBlobs": "/bs-packed/",
|
||||
"metaIndex": blobPackedIndex,
|
||||
})
|
||||
return nil
|
||||
}
|
||||
b.addPrefix("/bs/", "storage-googlecloudstorage", args{
|
||||
"bucket": bucket,
|
||||
"auth": map[string]interface{}{
|
||||
"client_id": clientID,
|
||||
"client_secret": secret,
|
||||
"refresh_token": refreshToken,
|
||||
},
|
||||
})
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,152 @@
|
|||
{
|
||||
"auth": "userpass:camlistore:pass3179",
|
||||
"https": false,
|
||||
"listen": "localhost:3179",
|
||||
"prefixes": {
|
||||
"/": {
|
||||
"handler": "root",
|
||||
"handlerArgs": {
|
||||
"blobRoot": "/bs-and-maybe-also-index/",
|
||||
"ownerName": "Alice",
|
||||
"searchRoot": "/my-search/",
|
||||
"statusRoot": "/status/",
|
||||
"stealth": false
|
||||
}
|
||||
},
|
||||
"/bs-and-index/": {
|
||||
"handler": "storage-replica",
|
||||
"handlerArgs": {
|
||||
"backends": [
|
||||
"/bs/",
|
||||
"/index/"
|
||||
]
|
||||
}
|
||||
},
|
||||
"/bs-and-maybe-also-index/": {
|
||||
"handler": "storage-cond",
|
||||
"handlerArgs": {
|
||||
"read": "/bs/",
|
||||
"write": {
|
||||
"else": "/bs/",
|
||||
"if": "isSchema",
|
||||
"then": "/bs-and-index/"
|
||||
}
|
||||
}
|
||||
},
|
||||
"/bs/": {
|
||||
"handler": "storage-blobpacked",
|
||||
"handlerArgs": {
|
||||
"smallBlobs": "/bs-loose/",
|
||||
"largeBlobs": "/bs-packed/",
|
||||
"metaIndex": {
|
||||
"database": "blobpacked_index",
|
||||
"host": "localhost",
|
||||
"password": "root",
|
||||
"type": "mysql",
|
||||
"user": "root"
|
||||
}
|
||||
}
|
||||
},
|
||||
"/bs-loose/": {
|
||||
"handler": "storage-googlecloudstorage",
|
||||
"handlerArgs": {
|
||||
"auth": {
|
||||
"client_id": "clientId",
|
||||
"client_secret": "clientSecret",
|
||||
"refresh_token": "refreshToken"
|
||||
},
|
||||
"bucket": "bucketName/blobs/loose"
|
||||
}
|
||||
},
|
||||
"/bs-packed/": {
|
||||
"handler": "storage-googlecloudstorage",
|
||||
"handlerArgs": {
|
||||
"auth": {
|
||||
"client_id": "clientId",
|
||||
"client_secret": "clientSecret",
|
||||
"refresh_token": "refreshToken"
|
||||
},
|
||||
"bucket": "bucketName/blobs/packed"
|
||||
}
|
||||
},
|
||||
"/cache/": {
|
||||
"handler": "storage-filesystem",
|
||||
"handlerArgs": {
|
||||
"path": "/tmp/camli-cache"
|
||||
}
|
||||
},
|
||||
"/importer/": {
|
||||
"handler": "importer",
|
||||
"handlerArgs": {}
|
||||
},
|
||||
"/index/": {
|
||||
"handler": "storage-index",
|
||||
"handlerArgs": {
|
||||
"blobSource": "/bs/",
|
||||
"storage": {
|
||||
"database": "camlitest",
|
||||
"host": "localhost",
|
||||
"password": "root",
|
||||
"type": "mysql",
|
||||
"user": "root"
|
||||
}
|
||||
}
|
||||
},
|
||||
"/my-search/": {
|
||||
"handler": "search",
|
||||
"handlerArgs": {
|
||||
"index": "/index/",
|
||||
"owner": "sha1-f2b0b7da718b97ce8c31591d8ed4645c777f3ef4",
|
||||
"slurpToMemory": true
|
||||
}
|
||||
},
|
||||
"/setup/": {
|
||||
"handler": "setup"
|
||||
},
|
||||
"/share/": {
|
||||
"handler": "share",
|
||||
"handlerArgs": {
|
||||
"blobRoot": "/bs/"
|
||||
}
|
||||
},
|
||||
"/sighelper/": {
|
||||
"handler": "jsonsign",
|
||||
"handlerArgs": {
|
||||
"keyId": "26F5ABDA",
|
||||
"publicKeyDest": "/bs-and-index/",
|
||||
"secretRing": "/path/to/secring"
|
||||
}
|
||||
},
|
||||
"/status/": {
|
||||
"handler": "status"
|
||||
},
|
||||
"/sync/": {
|
||||
"handler": "sync",
|
||||
"handlerArgs": {
|
||||
"from": "/bs/",
|
||||
"queue": {
|
||||
"database": "sync_index_queue",
|
||||
"type": "mysql",
|
||||
"host": "localhost",
|
||||
"user": "root",
|
||||
"password": "root"
|
||||
},
|
||||
"to": "/index/"
|
||||
}
|
||||
},
|
||||
"/ui/": {
|
||||
"handler": "ui",
|
||||
"handlerArgs": {
|
||||
"cache": "/cache/",
|
||||
"jsonSignRoot": "/sighelper/",
|
||||
"scaledImage": {
|
||||
"database": "ui_thumbmeta_cache",
|
||||
"host": "localhost",
|
||||
"type": "mysql",
|
||||
"user": "root",
|
||||
"password": "root"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
{
|
||||
"listen": "localhost:3179",
|
||||
"auth": "userpass:camlistore:pass3179",
|
||||
"googlecloudstorage": "clientId:clientSecret:refreshToken:bucketName/blobs",
|
||||
"packRelated": true,
|
||||
"dbNames": {
|
||||
"index": "camlitest",
|
||||
"queue-sync-to-index": "sync_index_queue",
|
||||
"blobpacked_index": "blobpacked_index",
|
||||
"ui_thumbcache": "ui_thumbmeta_cache"
|
||||
},
|
||||
"mysql": "root@localhost:root",
|
||||
"identity": "26F5ABDA",
|
||||
"identitySecretRing": "/path/to/secring",
|
||||
"ownerName": "Alice",
|
||||
"shareHandlerPath": "/share/"
|
||||
}
|
||||
|
|
@ -62,7 +62,7 @@ type Config struct {
|
|||
CopyIndexToMemory types.InvertedBool `json:"copyIndexToMemory,omitempty"` // copy disk-based index to memory on start-up.
|
||||
MemoryIndex bool `json:"memoryIndex,omitempty"` // use memory-only indexer.
|
||||
DBName string `json:"dbname,omitempty"` // name of the database for mysql, postgres, mongo.
|
||||
LevelDB string `json:"levelDB,omitempty"` // path to the levelDB file, for indexing with github.com/syndtr/goleveldb.
|
||||
LevelDB string `json:"levelDB,omitempty"` // path to the levelDB directory, for indexing with github.com/syndtr/goleveldb.
|
||||
KVFile string `json:"kvIndexFile,omitempty"` // path to the kv file, for indexing with github.com/cznic/kv.
|
||||
MySQL string `json:"mysql,omitempty"` // MySQL credentials (username@host:password), for indexing with MySQL.
|
||||
Mongo string `json:"mongo,omitempty"` // MongoDB credentials ([username:password@]host), for indexing with MongoDB.
|
||||
|
|
|
|||
Loading…
Reference in New Issue