Merge "Add diskPack as a high level local storage destination"

pkg/serverconfig/genconfig.go

@@ -42,6 +42,7 @@ type configPrefixesParams struct {
 	keyId            string
 	indexerPath      string
 	blobPath         string
+	packBlobs        bool
 	searchOwner      blob.Ref
 	shareHandlerPath string
 	flickr           string
@@ -433,9 +434,13 @@ func genLowLevelPrefixes(params *configPrefixesParams, ownerName string) (m json
 		},
 	}
 
+	storageType := "filesystem"
+	if params.packBlobs {
+		storageType = "diskpacked"
+	}
 	if params.blobPath != "" {
 		m["/bs/"] = map[string]interface{}{
-			"handler": "storage-filesystem",
+			"handler": "storage-" + storageType,
 			"handlerArgs": map[string]interface{}{
 				"path": params.blobPath,
 			},
@@ -527,6 +532,7 @@ func genLowLevelConfig(conf *Config) (lowLevelConf *Config, err error) {
 
 		// Blob storage options
 		blobPath           = conf.OptionalString("blobPath", "")
+		packBlobs          = conf.OptionalBool("packBlobs", false)      // use diskpacked instead of the default filestorage
 		s3                 = conf.OptionalString("s3", "")                 // "access_key_id:secret_access_key:bucket[:hostname]"
 		googlecloudstorage = conf.OptionalString("googlecloudstorage", "") // "clientId:clientSecret:refreshToken:bucket"
 		googledrive        = conf.OptionalString("googledrive", "")        // "clientId:clientSecret:refreshToken:parentId"
@@ -655,6 +661,7 @@ func genLowLevelConfig(conf *Config) (lowLevelConf *Config, err error) {
 		keyId:            keyId,
 		indexerPath:      indexerPath,
 		blobPath:         blobPath,
+		packBlobs:        packBlobs,
 		searchOwner:      blob.SHA1FromString(armoredPublicKey),
 		shareHandlerPath: shareHandlerPath,
 		flickr:           flickr,

pkg/serverconfig/testdata/diskpacked-want.json

@@ -0,0 +1,102 @@
+{
+	"auth": "userpass:camlistore:pass3179",
+	"https": false,
+	"listen": "localhost:3179",
+	"prefixes": {
+		"/": {
+			"handler": "root",
+			"handlerArgs": {
+				"blobRoot": "/bs-and-maybe-also-index/",
+				"ownerName": "Alice",
+				"searchRoot": "/my-search/",
+				"statusRoot": "/status/",
+				"stealth": false
+			}
+		},
+		"/bs-and-index/": {
+			"handler": "storage-replica",
+			"handlerArgs": {
+				"backends": [
+					"/bs/",
+					"/index-kv/"
+				]
+			}
+		},
+		"/bs-and-maybe-also-index/": {
+			"handler": "storage-cond",
+			"handlerArgs": {
+				"read": "/bs/",
+				"write": {
+					"else": "/bs/",
+					"if": "isSchema",
+					"then": "/bs-and-index/"
+				}
+			}
+		},
+		"/bs/": {
+			"handler": "storage-diskpacked",
+			"handlerArgs": {
+				"path": "/tmp/blobs"
+			}
+		},
+		"/cache/": {
+			"handler": "storage-filesystem",
+			"handlerArgs": {
+				"path": "/tmp/blobs/cache"
+			}
+		},
+		"/index-kv/": {
+			"handler": "storage-kvfileindexer",
+			"handlerArgs": {
+				"blobSource": "/bs/",
+				"file": "/path/to/indexkv.db"
+			}
+		},
+		"/my-search/": {
+			"handler": "search",
+			"handlerArgs": {
+				"index": "/index-kv/",
+				"owner": "sha1-f2b0b7da718b97ce8c31591d8ed4645c777f3ef4"
+			}
+		},
+		"/setup/": {
+			"handler": "setup"
+		},
+		"/share/": {
+			"handler": "share",
+			"handlerArgs": {
+				"blobRoot": "/bs/"
+			}
+		},
+		"/sighelper/": {
+			"handler": "jsonsign",
+			"handlerArgs": {
+				"keyId": "26F5ABDA",
+				"publicKeyDest": "/bs-and-index/",
+				"secretRing": "/home/gthomas/src/camlistore.org/pkg/jsonsign/testdata/test-secring.gpg"
+			}
+		},
+		"/status/": {
+			"handler": "status"
+		},
+		"/sync/": {
+			"handler": "sync",
+			"handlerArgs": {
+				"from": "/bs/",
+				"queue": {
+					"file": "/tmp/blobs/sync-to-index-queue.kv",
+					"type": "kv"
+				},
+				"to": "/index-kv/"
+			}
+		},
+		"/ui/": {
+			"handler": "ui",
+			"handlerArgs": {
+				"cache": "/cache/",
+				"jsonSignRoot": "/sighelper/",
+				"scaledImage": "lrucache"
+			}
+		}
+	}
+}

pkg/serverconfig/testdata/diskpacked.json

@@ -0,0 +1,11 @@
+{
+	"listen": "localhost:3179",
+	"auth": "userpass:camlistore:pass3179",
+	"blobPath": "/tmp/blobs",
+    "packBlobs": true,
+        "kvIndexFile": "/path/to/indexkv.db",
+	"identity": "26F5ABDA",
+	"identitySecretRing": "/path/to/secring",
+	"ownerName": "Alice",
+	"shareHandlerPath": "/share/"
+}