serverinit: enable memory storage in config

http://camlistore.org/issue/416

Change-Id: Idde32273ed651a5876581ad0ea06010970b92a9b

pkg/serverinit/genconfig.go

@@ -298,6 +298,15 @@ func (b *lowBuilder) sortedStorage(sortedType string) (map[string]interface{}, e
 	panic("indexArgs called when not in index mode")
 }
 
+func (b *lowBuilder) thatQueueUnlessMemory(thatQueue map[string]interface{}) (queue map[string]interface{}) {
+	if b.high.MemoryStorage {
+		return map[string]interface{}{
+			"type": "memory",
+		}
+	}
+	return thatQueue
+}
+
 func (b *lowBuilder) addS3Config(s3 string) error {
 	f := strings.SplitN(s3, ":", 4)
 	if len(f) < 3 {
@@ -331,16 +340,17 @@ func (b *lowBuilder) addS3Config(s3 string) error {
 			"path": filepath.Join(tempDir(), "camli-cache"),
 		})
 	} else {
-		if b.high.BlobPath == "" {
+		if b.high.BlobPath == "" && !b.high.MemoryStorage {
 			panic("unexpected empty blobpath with sync-to-s3")
 		}
 		b.addPrefix("/sync-to-s3/", "sync", args{
 			"from": "/bs/",
 			"to":   s3Prefix,
-			"queue": map[string]interface{}{
+			"queue": b.thatQueueUnlessMemory(
-				"type": "kv",
+				map[string]interface{}{
-				"file": filepath.Join(b.high.BlobPath, "sync-to-s3-queue.kv"),
+					"type": "kv",
-			},
+					"file": filepath.Join(b.high.BlobPath, "sync-to-s3-queue.kv"),
+				}),
 		})
 	}
 	return nil
@@ -377,11 +387,11 @@ func (b *lowBuilder) addGoogleDriveConfig(v string) error {
 		b.addPrefix("/sync-to-googledrive/", "sync", args{
 			"from": "/bs/",
 			"to":   prefix,
-			"queue": map[string]interface{}{
+			"queue": b.thatQueueUnlessMemory(
-				"type": "kv",
+				map[string]interface{}{
-				"file": filepath.Join(b.high.BlobPath,
+					"type": "kv",
-					"sync-to-googledrive-queue.kv"),
+					"file": filepath.Join(b.high.BlobPath, "sync-to-googledrive-queue.kv"),
-			},
+				}),
 		})
 	}
 
@@ -432,11 +442,11 @@ func (b *lowBuilder) addGoogleCloudStorageConfig(v string) error {
 		b.addPrefix("/sync-to-googlecloudstorage/", "sync", args{
 			"from": "/bs/",
 			"to":   gsPrefix,
-			"queue": map[string]interface{}{
+			"queue": b.thatQueueUnlessMemory(
-				"type": "kv",
+				map[string]interface{}{
-				"file": filepath.Join(b.high.BlobPath,
+					"type": "kv",
-					"sync-to-googlecloud-queue.kv"),
+					"file": filepath.Join(b.high.BlobPath, "sync-to-googlecloud-queue.kv"),
-			},
+				}),
 		})
 	}
 	return nil
@@ -473,7 +483,7 @@ func (b *lowBuilder) syncToIndexArgs() (map[string]interface{}, error) {
 	// TODO: currently when using s3, the index must be
 	// sqlite or kvfile, since only through one of those
 	// can we get a directory.
-	if b.high.BlobPath == "" && b.indexFileDir() == "" {
+	if !b.high.MemoryStorage && b.high.BlobPath == "" && b.indexFileDir() == "" {
 		// We don't actually have a working sync handler, but we keep a stub registered
 		// so it can be referred to from other places.
 		// See http://camlistore.org/issue/201
@@ -489,10 +499,11 @@ func (b *lowBuilder) syncToIndexArgs() (map[string]interface{}, error) {
 	if b.high.SQLite != "" {
 		typ = "sqlite"
 	}
-	a["queue"] = map[string]interface{}{
+	a["queue"] = b.thatQueueUnlessMemory(
-		"type": typ,
+		map[string]interface{}{
-		"file": filepath.Join(dir, "sync-to-index-queue."+typ),
+			"type": typ,
-	}
+			"file": filepath.Join(dir, "sync-to-index-queue."+typ),
+		})
 
 	return a, nil
 }
@@ -558,6 +569,9 @@ func (b *lowBuilder) genLowLevelPrefixes() error {
 		b.addPrefix("/cache/", "storage-"+storageType, args{
 			"path": filepath.Join(b.high.BlobPath, "/cache"),
 		})
+	} else if b.high.MemoryStorage {
+		b.addPrefix("/bs/", "storage-memory", nil)
+		b.addPrefix("/cache/", "storage-memory", nil)
 	}
 
 	if b.runIndex() {
@@ -646,12 +660,14 @@ func (b *lowBuilder) build() (*Config, error) {
 
 	nolocaldisk := conf.BlobPath == ""
 	if nolocaldisk {
-		if conf.S3 == "" && conf.GoogleCloudStorage == "" {
+		if !conf.MemoryStorage && conf.S3 == "" && conf.GoogleCloudStorage == "" {
-			return nil, errors.New("You need at least one of blobPath (for localdisk) or s3 or googlecloudstorage configured for a blobserver.")
+			return nil, errors.New("Unless memoryStorage is set, you must specify at least one storage option for your blobserver (blobPath (for localdisk), s3, googlecloudstorage).")
 		}
-		if conf.S3 != "" && conf.GoogleCloudStorage != "" {
+		if !conf.MemoryStorage && conf.S3 != "" && conf.GoogleCloudStorage != "" {
 			return nil, errors.New("Using S3 as a primary storage and Google Cloud Storage as a mirror is not supported for now.")
 		}
+	} else if conf.MemoryStorage {
+		return nil, errors.New("memoryStorage and blobPath are mutually exclusive.")
 	}
 
 	if conf.ShareHandler && conf.ShareHandlerPath == "" {
@@ -663,6 +679,9 @@ func (b *lowBuilder) build() (*Config, error) {
 	}
 
 	var cacheDir string
+	if conf.MemoryStorage {
+		noMkdir = true
+	}
 	if nolocaldisk {
 		// Whether camlistored is run from EC2 or not, we use
 		// a temp dir as the cache when primary storage is S3.

pkg/serverinit/testdata/memory_storage-want.json

@@ -0,0 +1,102 @@
+{
+	"auth": "userpass:camlistore:pass3179",
+	"https": false,
+	"listen": "localhost:3179",
+	"prefixes": {
+		"/": {
+			"handler": "root",
+			"handlerArgs": {
+				"blobRoot": "/bs-and-maybe-also-index/",
+				"ownerName": "Alice",
+				"searchRoot": "/my-search/",
+				"statusRoot": "/status/",
+				"stealth": false
+			}
+		},
+		"/bs-and-index/": {
+			"handler": "storage-replica",
+			"handlerArgs": {
+				"backends": [
+					"/bs/",
+					"/index/"
+				]
+			}
+		},
+		"/bs-and-maybe-also-index/": {
+			"handler": "storage-cond",
+			"handlerArgs": {
+				"read": "/bs/",
+				"write": {
+					"else": "/bs/",
+					"if": "isSchema",
+					"then": "/bs-and-index/"
+				}
+			}
+		},
+		"/bs/": {
+			"handler": "storage-memory"
+		},
+		"/cache/": {
+			"handler": "storage-memory"
+		},
+		"/importer/": {
+			"handler": "importer",
+			"handlerArgs": {}
+		},
+		"/index/": {
+			"handler": "storage-index",
+			"handlerArgs": {
+				"blobSource": "/bs/",
+				"storage": {
+					"file": "/path/to/indexkv.db",
+					"type": "kv"
+				}
+			}
+		},
+		"/my-search/": {
+			"handler": "search",
+			"handlerArgs": {
+				"index": "/index/",
+				"owner": "sha1-f2b0b7da718b97ce8c31591d8ed4645c777f3ef4",
+				"slurpToMemory": true
+			}
+		},
+		"/setup/": {
+			"handler": "setup"
+		},
+		"/share/": {
+			"handler": "share",
+			"handlerArgs": {
+				"blobRoot": "/bs/"
+			}
+		},
+		"/sighelper/": {
+			"handler": "jsonsign",
+			"handlerArgs": {
+				"keyId": "26F5ABDA",
+				"publicKeyDest": "/bs-and-index/",
+				"secretRing": "/path/to/secring"
+			}
+		},
+		"/status/": {
+			"handler": "status"
+		},
+		"/sync/": {
+			"handler": "sync",
+			"handlerArgs": {
+				"from": "/bs/",
+				"queue": {
+					"type": "memory"
+				},
+				"to": "/index/"
+			}
+		},
+		"/ui/": {
+			"handler": "ui",
+			"handlerArgs": {
+				"cache": "/cache/",
+				"jsonSignRoot": "/sighelper/"
+			}
+		}
+	}
+}

pkg/serverinit/testdata/memory_storage.json

@@ -0,0 +1,10 @@
+{
+	"listen": "localhost:3179",
+	"auth": "userpass:camlistore:pass3179",
+	"memoryStorage": true,
+	"kvIndexFile": "/path/to/indexkv.db",
+	"identity": "26F5ABDA",
+	"identitySecretRing": "/path/to/secring",
+	"ownerName": "Alice",
+	"shareHandlerPath": "/share/"
+}

pkg/types/serverconfig/config.go

@@ -42,6 +42,7 @@ type Config struct {
 	OwnerName  string `json:"ownerName,omitempty"`
 
 	// Blob storage.
+	MemoryStorage      bool   `json:"memoryStorage,omitempty"`      // do not store anything (blobs or queues) on localdisk, use memory instead.
 	BlobPath           string `json:"blobPath,omitempty"`           // path to the directory containing the blobs.
 	PackBlobs          bool   `json:"packBlobs,omitempty"`          // use diskpacked instead of the default filestorage.
 	S3                 string `json:"s3,omitempty"`                 // Amazon S3 credentials: access_key_id:secret_access_key:bucket[:hostname].

website/content/docs/server-config

@@ -44,6 +44,7 @@ web browser and restart the server.</p>
 <h2 id="storage">Storage options</h2>
 <p>At least one of these must be set:</p>
 <ul>
+<li><b><code>memoryStorage</code></b>: if true, blobs will be stored in memory only. This is generally only useful for debugging & development.</li>
 <li><b><code>blobPath</code></b>: local disk path to store blobs. (valid for diskpacked too).</li>
 <li><b><code>s3</code></b>: "<code>key:secret:bucket</code>" or "<code>key:secret:bucket:hostname</code>" (with colons, but no quotes).</li>
 <li><b><code>googlecloudstorage</code></b>: "<code>clientId:clientSecret:refreshToken:bucketName</code>"</li>