serverinit: add genconfig for blobpacked on Google Cloud Storage.

And enable it (blobpacked) on GCE.

Change-Id: I7b51c186325e80987e48d32392d88bb4c30ee3d5

misc/docker/camlistored/Makefile

@@ -1,10 +1,13 @@
 docker: Dockerfile camlistored
-	docker build -t camlistore/camlistored .
+	docker build -t camlistored .
 
 .PHONY: camlistored
 
 camlistored:
 	(cd ../../../; go run make.go --docker_camlistored)
 
-push: docker
+camlistored.tar.gz: docker
-	docker push camlistore/camlistored
+	docker save camlistored | gzip -c > $@
+
+push: camlistored.tar.gz
+	gsutil cp $< gs://camlistore-release/docker/

pkg/serverinit/env.go

@@ -71,6 +71,7 @@ func DefaultEnvConfig() (*Config, error) {
 		IdentitySecretRing: secRing,
 		GoogleCloudStorage: ":" + strings.TrimPrefix(blobBucket, "gs://"),
 		DBNames:            map[string]string{},
+		PackRelated:        true,
 	}
 
 	// Detect a linked Docker MySQL container. It must have alias "mysqldb".
@@ -79,6 +80,7 @@ func DefaultEnvConfig() (*Config, error) {
 		highConf.MySQL = "root@" + hostPort + ":" // no password
 		highConf.DBNames["queue-sync-to-index"] = "sync_index_queue"
 		highConf.DBNames["ui_thumbcache"] = "ui_thumbmeta_cache"
+		highConf.DBNames["blobpacked_index"] = "blobpacked_index"
 	} else {
 		// TODO: also detect Cloud SQL.
 		highConf.KVFile = "/index.kv"

pkg/serverinit/genconfig.go

@@ -87,7 +87,7 @@ func (b *lowBuilder) runIndex() bool          { return b.high.RunIndex.Get() }
 func (b *lowBuilder) copyIndexToMemory() bool { return b.high.CopyIndexToMemory.Get() }
 
 // dbName returns which database to use for the provided user ("of").
-// The user key be a key as describe in pkg/types/serverconfig/config.go's
+// The user should be a key as described in pkg/types/serverconfig/config.go's
 // description of DBNames: "index", "queue-sync-to-index", etc.
 func (b *lowBuilder) dbName(of string) string {
 	if v, ok := b.high.DBNames[of]; ok && v != "" {
@@ -443,34 +443,18 @@ func (b *lowBuilder) addGoogleCloudStorageConfig(v string) error {
 		clientID = "auto"
 	}
 
-	if b.high.PackRelated {
+	isReplica := b.hasPrefix("/bs/")
-		// TODO(mpl): implement
+	if isReplica {
-		return errors.New("TODO: finish genconfig support for GCS+blobpacked")
+		gsPrefix := "/sto-googlecloudstorage/"
-	}
+		b.addPrefix(gsPrefix, "storage-googlecloudstorage", args{
-
+			"bucket": bucket,
-	isPrimary := !b.hasPrefix("/bs/")
+			"auth": map[string]interface{}{
-	gsPrefix := ""
+				"client_id":     clientID,
-	if isPrimary {
+				"client_secret": secret,
-		gsPrefix = "/bs/"
+				"refresh_token": refreshToken,
-	} else {
+			},
-		gsPrefix = "/sto-googlecloudstorage/"
-	}
-
-	b.addPrefix(gsPrefix, "storage-googlecloudstorage", args{
-		"bucket": bucket,
-		"auth": map[string]interface{}{
-			"client_id":     clientID,
-			"client_secret": secret,
-			"refresh_token": refreshToken,
-		},
-	})
-
-	if isPrimary {
-		// TODO: cacheBucket like s3CacheBucket?
-		b.addPrefix("/cache/", "storage-filesystem", args{
-			"path": filepath.Join(tempDir(), "camli-cache"),
 		})
-	} else {
+
 		b.addPrefix("/sync-to-googlecloudstorage/", "sync", args{
 			"from": "/bs/",
 			"to":   gsPrefix,
@@ -480,7 +464,50 @@ func (b *lowBuilder) addGoogleCloudStorageConfig(v string) error {
 					"file": filepath.Join(b.high.BlobPath, "sync-to-googlecloud-queue.kv"),
 				}),
 		})
+		return nil
 	}
+
+	// TODO: cacheBucket like s3CacheBucket?
+	b.addPrefix("/cache/", "storage-filesystem", args{
+		"path": filepath.Join(tempDir(), "camli-cache"),
+	})
+	if b.high.PackRelated {
+		b.addPrefix("/bs-loose/", "storage-googlecloudstorage", args{
+			"bucket": bucket + "/loose",
+			"auth": map[string]interface{}{
+				"client_id":     clientID,
+				"client_secret": secret,
+				"refresh_token": refreshToken,
+			},
+		})
+		b.addPrefix("/bs-packed/", "storage-googlecloudstorage", args{
+			"bucket": bucket + "/packed",
+			"auth": map[string]interface{}{
+				"client_id":     clientID,
+				"client_secret": secret,
+				"refresh_token": refreshToken,
+			},
+		})
+		blobPackedIndex, err := b.sortedStorageAt("blobpacked_index", "")
+		if err != nil {
+			return err
+		}
+		b.addPrefix("/bs/", "storage-blobpacked", args{
+			"smallBlobs": "/bs-loose/",
+			"largeBlobs": "/bs-packed/",
+			"metaIndex":  blobPackedIndex,
+		})
+		return nil
+	}
+	b.addPrefix("/bs/", "storage-googlecloudstorage", args{
+		"bucket": bucket,
+		"auth": map[string]interface{}{
+			"client_id":     clientID,
+			"client_secret": secret,
+			"refresh_token": refreshToken,
+		},
+	})
+
 	return nil
 }
 

pkg/serverinit/testdata/blobpacked_googlecloud-want.json

@@ -0,0 +1,152 @@
+{
+	"auth": "userpass:camlistore:pass3179",
+	"https": false,
+	"listen": "localhost:3179",
+	"prefixes": {
+		"/": {
+			"handler": "root",
+			"handlerArgs": {
+				"blobRoot": "/bs-and-maybe-also-index/",
+				"ownerName": "Alice",
+				"searchRoot": "/my-search/",
+				"statusRoot": "/status/",
+				"stealth": false
+			}
+		},
+		"/bs-and-index/": {
+			"handler": "storage-replica",
+			"handlerArgs": {
+				"backends": [
+					"/bs/",
+					"/index/"
+				]
+			}
+		},
+		"/bs-and-maybe-also-index/": {
+			"handler": "storage-cond",
+			"handlerArgs": {
+				"read": "/bs/",
+				"write": {
+					"else": "/bs/",
+					"if": "isSchema",
+					"then": "/bs-and-index/"
+				}
+			}
+		},
+		"/bs/": {
+			"handler": "storage-blobpacked",
+			"handlerArgs": {
+				"smallBlobs": "/bs-loose/",
+				"largeBlobs": "/bs-packed/",
+				"metaIndex": {
+					"database": "blobpacked_index",
+					"host": "localhost",
+					"password": "root",
+					"type": "mysql",
+					"user": "root"
+				}
+			}
+		},
+		"/bs-loose/": {
+			"handler": "storage-googlecloudstorage",
+			"handlerArgs": {
+				"auth": {
+					"client_id": "clientId",
+					"client_secret": "clientSecret",
+					"refresh_token": "refreshToken"
+				},
+				"bucket": "bucketName/blobs/loose"
+			}
+		},
+		"/bs-packed/": {
+			"handler": "storage-googlecloudstorage",
+			"handlerArgs": {
+				"auth": {
+					"client_id": "clientId",
+					"client_secret": "clientSecret",
+					"refresh_token": "refreshToken"
+				},
+				"bucket": "bucketName/blobs/packed"
+			}
+		},
+		"/cache/": {
+			"handler": "storage-filesystem",
+			"handlerArgs": {
+				"path": "/tmp/camli-cache"
+			}
+		},
+		"/importer/": {
+			"handler": "importer",
+			"handlerArgs": {}
+		},
+		"/index/": {
+			"handler": "storage-index",
+			"handlerArgs": {
+		        "blobSource": "/bs/",
+		        "storage": {
+		          "database": "camlitest",
+		          "host": "localhost",
+		          "password": "root",
+		          "type": "mysql",
+		          "user": "root"
+		        }
+		      }
+		},
+		"/my-search/": {
+			"handler": "search",
+			"handlerArgs": {
+				"index": "/index/",
+				"owner": "sha1-f2b0b7da718b97ce8c31591d8ed4645c777f3ef4",
+				"slurpToMemory": true
+			}
+		},
+		"/setup/": {
+			"handler": "setup"
+		},
+		"/share/": {
+			"handler": "share",
+			"handlerArgs": {
+				"blobRoot": "/bs/"
+			}
+		},
+		"/sighelper/": {
+			"handler": "jsonsign",
+			"handlerArgs": {
+				"keyId": "26F5ABDA",
+				"publicKeyDest": "/bs-and-index/",
+				"secretRing": "/path/to/secring"
+			}
+		},
+		"/status/": {
+			"handler": "status"
+		},
+		"/sync/": {
+			"handler": "sync",
+			"handlerArgs": {
+				"from": "/bs/",
+				"queue": {
+					"database": "sync_index_queue",
+					"type": "mysql",
+					"host": "localhost",
+					"user": "root",
+					"password": "root"
+				},
+				"to": "/index/"
+			}
+		},
+		"/ui/": {
+			"handler": "ui",
+			"handlerArgs": {
+				"cache": "/cache/",
+				"jsonSignRoot": "/sighelper/",
+				"scaledImage": {
+					"database": "ui_thumbmeta_cache",
+					"host": "localhost",
+					"type": "mysql",
+					"user": "root",
+					"password": "root"
+				}
+			}
+		}
+	}
+}

pkg/serverinit/testdata/blobpacked_googlecloud.json

@@ -0,0 +1,17 @@
+{
+	"listen": "localhost:3179",
+	"auth": "userpass:camlistore:pass3179",
+	"googlecloudstorage": "clientId:clientSecret:refreshToken:bucketName/blobs",
+	"packRelated": true,
+	"dbNames": {
+		"index": "camlitest",
+		"queue-sync-to-index": "sync_index_queue",
+		"blobpacked_index": "blobpacked_index",
+		"ui_thumbcache": "ui_thumbmeta_cache"
+	},
+	"mysql": "root@localhost:root",
+	"identity": "26F5ABDA",
+	"identitySecretRing": "/path/to/secring",
+	"ownerName": "Alice",
+	"shareHandlerPath": "/share/"
+}

pkg/types/serverconfig/config.go

@@ -62,7 +62,7 @@ type Config struct {
 	CopyIndexToMemory types.InvertedBool `json:"copyIndexToMemory,omitempty"` // copy disk-based index to memory on start-up.
 	MemoryIndex       bool               `json:"memoryIndex,omitempty"`       // use memory-only indexer.
 	DBName            string             `json:"dbname,omitempty"`            // name of the database for mysql, postgres, mongo.
-	LevelDB           string             `json:"levelDB,omitempty"`           // path to the levelDB file, for indexing with github.com/syndtr/goleveldb.
+	LevelDB           string             `json:"levelDB,omitempty"`           // path to the levelDB directory, for indexing with github.com/syndtr/goleveldb.
 	KVFile            string             `json:"kvIndexFile,omitempty"`       // path to the kv file, for indexing with github.com/cznic/kv.
 	MySQL             string             `json:"mysql,omitempty"`             // MySQL credentials (username@host:password), for indexing with MySQL.
 	Mongo             string             `json:"mongo,omitempty"`             // MongoDB credentials ([username:password@]host), for indexing with MongoDB.