From bf54591e8d2cd7e6d5a9533879d146c4afed8b57 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Wed, 11 Dec 2013 12:20:22 +0400 Subject: [PATCH] Remove the memIndex option from genconfig and devcam; default memoryIndex to on. We had the legacy "memIndex" option to put the Camlistore index in memory (the leveldb memdb) as an option in genconfig called "memIndex", and we also had an option called "memoryIndex" for whether to slurp the on-disk index to memory on start-up. Too confusing! Instead, delete "memIndex" (since it's not the default anyway, now that we have kv). Then, also (the original point of this change): default the memoryIndex option to true, so search works for people by default. This option might go away in the future if it becomes the only required way. Also, document this. Change-Id: Iddffa6e19adbf09c5aacd063aa44de362d90633b --- config/dev-server-config.json | 12 +- dev/devcam/server.go | 17 +-- pkg/server/wizard.go | 1 - pkg/serverconfig/genconfig.go | 35 ++---- pkg/serverconfig/testdata/baseurl-want.json | 5 +- pkg/serverconfig/testdata/baseurlbad.json | 2 +- pkg/serverconfig/testdata/default-want.json | 5 +- .../testdata/diskpacked-want.json | 3 +- .../testdata/google_nolocaldisk-want.json | 114 ++++++++---------- .../testdata/google_nolocaldisk.json | 2 +- .../testdata/listenbase-want.json | 18 +-- pkg/serverconfig/testdata/listenbase.json | 2 +- pkg/serverconfig/testdata/mem-want.json | 18 +-- pkg/serverconfig/testdata/mem.json | 2 +- pkg/serverconfig/testdata/mongo-want.json | 5 +- pkg/serverconfig/testdata/noindex.err | 2 +- .../testdata/s3_alt_host-want.json | 113 ++++++++--------- pkg/serverconfig/testdata/s3_alt_host.json | 2 +- .../testdata/s3_google_nolocaldisk.json | 2 +- .../testdata/s3_nolocaldisk-want.json | 113 ++++++++--------- pkg/serverconfig/testdata/s3_nolocaldisk.json | 2 +- .../testdata/s3_nolocaldisk_mysql-want.json | 104 ++++++++-------- pkg/serverconfig/testdata/sqlite-want.json | 5 +- pkg/serverconfig/testdata/tls-want.json | 18 +-- pkg/serverconfig/testdata/tls.json | 2 +- pkg/serverconfig/testdata/with_blog-want.json | 18 +-- pkg/serverconfig/testdata/with_blog.json | 2 +- .../testdata/with_gallery-want.json | 18 +-- pkg/serverconfig/testdata/with_gallery.json | 2 +- .../testdata/with_sourceroot-want.json | 18 +-- .../testdata/with_sourceroot.json | 2 +- website/content/docs/server-config | 1 + 32 files changed, 308 insertions(+), 357 deletions(-) diff --git a/config/dev-server-config.json b/config/dev-server-config.json index 61aa6805a..ade1e1773 100644 --- a/config/dev-server-config.json +++ b/config/dev-server-config.json @@ -72,8 +72,8 @@ "from": "/bs/", "to": ["_env", "${CAMLI_INDEXER_PATH}"], "queue": { "type": "memory" }, - "fullSyncOnStart": ["_env", "${CAMLI_MEMINDEX_ENABLED}"], - "blockingFullSyncOnStart": ["_env", "${CAMLI_MEMINDEX_ENABLED}"] + "fullSyncOnStart": ["_env", "${CAMLI_FULL_INDEX_SYNC_ON_START}"], + "blockingFullSyncOnStart": ["_env", "${CAMLI_FULL_INDEX_SYNC_ON_START}"] } }, @@ -205,14 +205,6 @@ } }, - "/index-mem/": { - "enabled": ["_env", "${CAMLI_MEMINDEX_ENABLED}"], - "handler": "storage-memory-only-dev-indexer", - "handlerArgs": { - "blobSource": "/bs/" - } - }, - "/index-kv/": { "enabled": ["_env", "${CAMLI_KVINDEX_ENABLED}"], "handler": "storage-kvfileindexer", diff --git a/dev/devcam/server.go b/dev/devcam/server.go index 866be17cd..e705a7b67 100644 --- a/dev/devcam/server.go +++ b/dev/devcam/server.go @@ -53,7 +53,6 @@ type serverCmd struct { mysql bool postgres bool sqlite bool - memindex bool // memory index; default is kvfile slow bool throttle int @@ -82,11 +81,10 @@ func init() { flags.BoolVar(&cmd.wipe, "wipe", false, "Wipe the blobs on disk and the indexer.") flags.BoolVar(&cmd.debug, "debug", false, "Enable http debugging.") - flags.BoolVar(&cmd.mongo, "mongo", false, "Use mongodb as the indexer. Excludes -mysql, -postgres, -sqlite, -memindex.") - flags.BoolVar(&cmd.mysql, "mysql", false, "Use mysql as the indexer. Excludes -mongo, -postgres, -sqlite, -memindex.") - flags.BoolVar(&cmd.postgres, "postgres", false, "Use postgres as the indexer. Excludes -mongo, -mysql, -sqlite, -memindex.") - flags.BoolVar(&cmd.sqlite, "sqlite", false, "Use sqlite as the indexer. Excludes -mongo, -mysql, -postgres, -memindex.") - flags.BoolVar(&cmd.memindex, "memindex", false, "Use memory as the indexer. Excludes -mongo, -mysql, -sqlite, -postgres.") + flags.BoolVar(&cmd.mongo, "mongo", false, "Use mongodb as the indexer. Excludes -mysql, -postgres, -sqlite.") + flags.BoolVar(&cmd.mysql, "mysql", false, "Use mysql as the indexer. Excludes -mongo, -postgres, -sqlite.") + flags.BoolVar(&cmd.postgres, "postgres", false, "Use postgres as the indexer. Excludes -mongo, -mysql, -sqlite.") + flags.BoolVar(&cmd.sqlite, "sqlite", false, "Use sqlite as the indexer. Excludes -mongo, -mysql, -postgres.") flags.BoolVar(&cmd.slow, "slow", false, "Add artificial latency.") flags.IntVar(&cmd.throttle, "throttle", 150, "If -slow, this is the rate in kBps, to which we should throttle.") @@ -119,7 +117,7 @@ func (c *serverCmd) checkFlags(args []string) error { c.Usage() } nindex := 0 - for _, v := range []bool{c.mongo, c.mysql, c.postgres, c.sqlite, c.memindex} { + for _, v := range []bool{c.mongo, c.mysql, c.postgres, c.sqlite} { if v { nindex++ } @@ -171,13 +169,13 @@ func (c *serverCmd) setEnvVars() error { if user == "" { return errors.New("Could not get username from environment") } + setenv("CAMLI_FULL_INDEX_SYNC_ON_START", "false") // TODO: option to make this true setenv("CAMLI_DBNAME", "devcamli"+user) setenv("CAMLI_MYSQL_ENABLED", "false") setenv("CAMLI_MONGO_ENABLED", "false") setenv("CAMLI_POSTGRES_ENABLED", "false") setenv("CAMLI_SQLITE_ENABLED", "false") setenv("CAMLI_KVINDEX_ENABLED", "false") - setenv("CAMLI_MEMINDEX_ENABLED", "false") switch { case c.mongo: setenv("CAMLI_MONGO_ENABLED", "true") @@ -195,9 +193,6 @@ func (c *serverCmd) setEnvVars() error { panic("no camliRoot set") } setenv("CAMLI_DBNAME", filepath.Join(c.camliRoot, "sqliteindex.db")) - case c.memindex: - setenv("CAMLI_MEMINDEX_ENABLED", "true") - setenv("CAMLI_INDEXER_PATH", "/index-mem/") default: setenv("CAMLI_KVINDEX_ENABLED", "true") setenv("CAMLI_INDEXER_PATH", "/index-kv/") diff --git a/pkg/server/wizard.go b/pkg/server/wizard.go index a50f6bbe1..825aa2a65 100644 --- a/pkg/server/wizard.go +++ b/pkg/server/wizard.go @@ -40,7 +40,6 @@ import ( var ignoredFields = map[string]bool{ "gallery": true, "blog": true, - "memIndex": true, "replicateTo": true, } diff --git a/pkg/serverconfig/genconfig.go b/pkg/serverconfig/genconfig.go index f00007fdd..15b3e4e8e 100644 --- a/pkg/serverconfig/genconfig.go +++ b/pkg/serverconfig/genconfig.go @@ -188,15 +188,6 @@ func addMySQLConfig(prefixes jsonconfig.Obj, dbname string, dbinfo string) { addSQLConfig("mysql", prefixes, dbname, dbinfo) } -func addMemindexConfig(prefixes jsonconfig.Obj) { - ob := map[string]interface{}{} - ob["handler"] = "storage-memory-only-dev-indexer" - ob["handlerArgs"] = map[string]interface{}{ - "blobSource": "/bs/", - } - prefixes["/index-mem/"] = ob -} - func addSQLiteConfig(prefixes jsonconfig.Obj, file string) { ob := map[string]interface{}{} ob["handler"] = "storage-sqliteindexer" @@ -508,14 +499,14 @@ func genLowLevelPrefixes(params *configPrefixesParams, ownerName string) (m json } searchArgs := map[string]interface{}{ - "index": params.indexerPath, - "owner": params.searchOwner.String(), + "index": params.indexerPath, + "owner": params.searchOwner.String(), } if params.memoryIndex { searchArgs["slurpToMemory"] = true } m["/my-search/"] = map[string]interface{}{ - "handler": "search", + "handler": "search", "handlerArgs": searchArgs, } } @@ -548,12 +539,11 @@ func genLowLevelConfig(conf *Config) (lowLevelConf *Config, err error) { shareHandlerPath = conf.OptionalString("shareHandlerPath", "") // Index options - memoryIndex = conf.OptionalBool("memoryIndex", false) - runIndex = conf.OptionalBool("runIndex", true) // if false: no search, no UI, etc. - dbname = conf.OptionalString("dbname", "") // for mysql, postgres, mongo + memoryIndex = conf.OptionalBool("memoryIndex", true) // copy disk-based index to memory on start-up + runIndex = conf.OptionalBool("runIndex", true) // if false: no search, no UI, etc. + dbname = conf.OptionalString("dbname", "") // for mysql, postgres, mongo mysql = conf.OptionalString("mysql", "") postgres = conf.OptionalString("postgres", "") - memIndex = conf.OptionalBool("memIndex", false) mongo = conf.OptionalString("mongo", "") sqliteFile = conf.OptionalString("sqlite", "") kvFile = conf.OptionalString("kvIndexFile", "") @@ -617,14 +607,14 @@ func genLowLevelConfig(conf *Config) (lowLevelConf *Config, err error) { } var indexerPath string - numIndexers := numSet(mongo, mysql, postgres, sqliteFile, memIndex, kvFile) + numIndexers := numSet(mongo, mysql, postgres, sqliteFile, kvFile) switch { case runIndex && numIndexers == 0: - return nil, fmt.Errorf("Unless runIndex is set to false, you must specify an index option (kvIndexFile, mongo, mysql, postgres, sqlite, memIndex).") + return nil, fmt.Errorf("Unless runIndex is set to false, you must specify an index option (kvIndexFile, mongo, mysql, postgres, sqlite).") case runIndex && numIndexers != 1: - return nil, fmt.Errorf("With runIndex set true, you can only pick exactly one indexer (mongo, mysql, postgres, sqlite, memIndex).") + return nil, fmt.Errorf("With runIndex set true, you can only pick exactly one indexer (mongo, mysql, postgres, sqlite).") case !runIndex && numIndexers != 0: - return nil, fmt.Errorf("With runIndex disabled, you can't specify any of mongo, mysql, postgres, sqlite, memIndex.") + return nil, fmt.Errorf("With runIndex disabled, you can't specify any of mongo, mysql, postgres, sqlite.") case mysql != "": indexerPath = "/index-mysql/" case postgres != "": @@ -635,8 +625,6 @@ func genLowLevelConfig(conf *Config) (lowLevelConf *Config, err error) { indexerPath = "/index-sqlite/" case kvFile != "": indexerPath = "/index-kv/" - case memIndex: - indexerPath = "/index-mem/" } entity, err := jsonsign.EntityFromSecring(keyId, secretRing) @@ -736,9 +724,6 @@ func genLowLevelConfig(conf *Config) (lowLevelConf *Config, err error) { return nil, err } } - if indexerPath == "/index-mem/" { - addMemindexConfig(prefixes) - } obj["prefixes"] = (map[string]interface{})(prefixes) diff --git a/pkg/serverconfig/testdata/baseurl-want.json b/pkg/serverconfig/testdata/baseurl-want.json index 9a9a7d404..da012da9b 100644 --- a/pkg/serverconfig/testdata/baseurl-want.json +++ b/pkg/serverconfig/testdata/baseurl-want.json @@ -57,7 +57,8 @@ "handler": "search", "handlerArgs": { "index": "/index-kv/", - "owner": "sha1-f2b0b7da718b97ce8c31591d8ed4645c777f3ef4" + "owner": "sha1-f2b0b7da718b97ce8c31591d8ed4645c777f3ef4", + "slurpToMemory": true } }, "/setup/": { @@ -100,4 +101,4 @@ } } } -} \ No newline at end of file +} diff --git a/pkg/serverconfig/testdata/baseurlbad.json b/pkg/serverconfig/testdata/baseurlbad.json index 17af9d1b1..0f6e419ad 100644 --- a/pkg/serverconfig/testdata/baseurlbad.json +++ b/pkg/serverconfig/testdata/baseurlbad.json @@ -6,5 +6,5 @@ "blobPath": "/tmp/blobs", "identity": "26F5ABDA", "identitySecretRing": "/path/to/secring", - "memIndex": true + "kvIndexFile": "/path/to/indexkv.db" } diff --git a/pkg/serverconfig/testdata/default-want.json b/pkg/serverconfig/testdata/default-want.json index fa0e99272..b36211b87 100644 --- a/pkg/serverconfig/testdata/default-want.json +++ b/pkg/serverconfig/testdata/default-want.json @@ -56,7 +56,8 @@ "handler": "search", "handlerArgs": { "index": "/index-kv/", - "owner": "sha1-f2b0b7da718b97ce8c31591d8ed4645c777f3ef4" + "owner": "sha1-f2b0b7da718b97ce8c31591d8ed4645c777f3ef4", + "slurpToMemory": true } }, "/setup/": { @@ -99,4 +100,4 @@ } } } -} \ No newline at end of file +} diff --git a/pkg/serverconfig/testdata/diskpacked-want.json b/pkg/serverconfig/testdata/diskpacked-want.json index d643b47d7..845437582 100644 --- a/pkg/serverconfig/testdata/diskpacked-want.json +++ b/pkg/serverconfig/testdata/diskpacked-want.json @@ -56,7 +56,8 @@ "handler": "search", "handlerArgs": { "index": "/index-kv/", - "owner": "sha1-f2b0b7da718b97ce8c31591d8ed4645c777f3ef4" + "owner": "sha1-f2b0b7da718b97ce8c31591d8ed4645c777f3ef4", + "slurpToMemory": true } }, "/setup/": { diff --git a/pkg/serverconfig/testdata/google_nolocaldisk-want.json b/pkg/serverconfig/testdata/google_nolocaldisk-want.json index bf372bb14..568d2bd62 100644 --- a/pkg/serverconfig/testdata/google_nolocaldisk-want.json +++ b/pkg/serverconfig/testdata/google_nolocaldisk-want.json @@ -1,7 +1,7 @@ { - "listen": "localhost:3179", "auth": "userpass:camlistore:pass3179", "https": false, + "listen": "localhost:3179", "prefixes": { "/": { "handler": "root", @@ -12,68 +12,26 @@ "stealth": false } }, - - "/ui/": { - "handler": "ui", - "handlerArgs": { - "jsonSignRoot": "/sighelper/", - "cache": "/cache/", - "scaledImage": "lrucache" - } - }, - - "/setup/": { - "handler": "setup" - }, - - "/status/": { - "handler": "status" - }, - - "/share/": { - "handler": "share", - "handlerArgs": { - "blobRoot": "/bs/" - } - }, - - "/sync/": { - "handler": "sync", - "handlerArgs": { - "from": "/bs/", - "to": "/index-mem/", - "idle": true - } - }, - - "/sighelper/": { - "handler": "jsonsign", - "handlerArgs": { - "secretRing": "/path/to/secring", - "keyId": "26F5ABDA", - "publicKeyDest": "/bs-and-index/" - } - }, - "/bs-and-index/": { "handler": "storage-replica", "handlerArgs": { - "backends": ["/bs/", "/index-mem/"] + "backends": [ + "/bs/", + "/index-kv/" + ] } }, - "/bs-and-maybe-also-index/": { "handler": "storage-cond", "handlerArgs": { + "read": "/bs/", "write": { + "else": "/bs/", "if": "isSchema", - "then": "/bs-and-index/", - "else": "/bs/" - }, - "read": "/bs/" + "then": "/bs-and-index/" + } } }, - "/bs/": { "handler": "storage-googlecloudstorage", "handlerArgs": { @@ -85,30 +43,62 @@ "bucket": "bucketName" } }, - "/cache/": { "handler": "storage-filesystem", "handlerArgs": { "path": "/tmp/camli-cache" } }, - - "/index-mem/": { - "handler": "storage-memory-only-dev-indexer", + "/index-kv/": { + "handler": "storage-kvfileindexer", "handlerArgs": { - "blobSource": "/bs/" + "blobSource": "/bs/", + "file": "/path/to/indexkv.db" } }, - "/my-search/": { "handler": "search", "handlerArgs": { - "index": "/index-mem/", - "owner": "sha1-f2b0b7da718b97ce8c31591d8ed4645c777f3ef4" + "index": "/index-kv/", + "owner": "sha1-f2b0b7da718b97ce8c31591d8ed4645c777f3ef4", + "slurpToMemory": true + } + }, + "/setup/": { + "handler": "setup" + }, + "/share/": { + "handler": "share", + "handlerArgs": { + "blobRoot": "/bs/" + } + }, + "/sighelper/": { + "handler": "jsonsign", + "handlerArgs": { + "keyId": "26F5ABDA", + "publicKeyDest": "/bs-and-index/", + "secretRing": "/path/to/secring" + } + }, + "/status/": { + "handler": "status" + }, + "/sync/": { + "handler": "sync", + "handlerArgs": { + "from": "/bs/", + "idle": true, + "to": "/index-kv/" + } + }, + "/ui/": { + "handler": "ui", + "handlerArgs": { + "cache": "/cache/", + "jsonSignRoot": "/sighelper/", + "scaledImage": "lrucache" } } - } - } - diff --git a/pkg/serverconfig/testdata/google_nolocaldisk.json b/pkg/serverconfig/testdata/google_nolocaldisk.json index 8d0a816bf..9f8a20943 100644 --- a/pkg/serverconfig/testdata/google_nolocaldisk.json +++ b/pkg/serverconfig/testdata/google_nolocaldisk.json @@ -4,7 +4,7 @@ "auth": "userpass:camlistore:pass3179", "identity": "26F5ABDA", "identitySecretRing": "/path/to/secring", - "memIndex": true, + "kvIndexFile": "/path/to/indexkv.db", "googlecloudstorage": "clientId:clientSecret:refreshToken:bucketName", "replicateTo": [], "publish": {}, diff --git a/pkg/serverconfig/testdata/listenbase-want.json b/pkg/serverconfig/testdata/listenbase-want.json index 6fa1f708b..e3472b703 100644 --- a/pkg/serverconfig/testdata/listenbase-want.json +++ b/pkg/serverconfig/testdata/listenbase-want.json @@ -18,7 +18,7 @@ "handlerArgs": { "backends": [ "/bs/", - "/index-mem/" + "/index-kv/" ] } }, @@ -45,17 +45,19 @@ "path": "/tmp/blobs/cache" } }, - "/index-mem/": { - "handler": "storage-memory-only-dev-indexer", + "/index-kv/": { + "handler": "storage-kvfileindexer", "handlerArgs": { - "blobSource": "/bs/" + "blobSource": "/bs/", + "file": "/path/to/indexkv.db" } }, "/my-search/": { "handler": "search", "handlerArgs": { - "index": "/index-mem/", - "owner": "sha1-f2b0b7da718b97ce8c31591d8ed4645c777f3ef4" + "index": "/index-kv/", + "owner": "sha1-f2b0b7da718b97ce8c31591d8ed4645c777f3ef4", + "slurpToMemory": true } }, "/setup/": { @@ -86,7 +88,7 @@ "file": "/tmp/blobs/sync-to-index-queue.kv", "type": "kv" }, - "to": "/index-mem/" + "to": "/index-kv/" } }, "/ui/": { @@ -98,4 +100,4 @@ } } } -} \ No newline at end of file +} diff --git a/pkg/serverconfig/testdata/listenbase.json b/pkg/serverconfig/testdata/listenbase.json index 2dd5e91de..7246c44f4 100644 --- a/pkg/serverconfig/testdata/listenbase.json +++ b/pkg/serverconfig/testdata/listenbase.json @@ -3,7 +3,7 @@ "baseURL": "http://foo.com/", "auth": "userpass:camlistore:pass3179", "blobPath": "/tmp/blobs", - "memIndex": true, + "kvIndexFile": "/path/to/indexkv.db", "identity": "26F5ABDA", "identitySecretRing": "/path/to/secring", "shareHandler": true diff --git a/pkg/serverconfig/testdata/mem-want.json b/pkg/serverconfig/testdata/mem-want.json index 4b566875b..2b6c63898 100644 --- a/pkg/serverconfig/testdata/mem-want.json +++ b/pkg/serverconfig/testdata/mem-want.json @@ -18,7 +18,7 @@ "handlerArgs": { "backends": [ "/bs/", - "/index-mem/" + "/index-kv/" ] } }, @@ -45,17 +45,19 @@ "path": "/tmp/blobs/cache" } }, - "/index-mem/": { - "handler": "storage-memory-only-dev-indexer", + "/index-kv/": { + "handler": "storage-kvfileindexer", "handlerArgs": { - "blobSource": "/bs/" + "blobSource": "/bs/", + "file": "/path/to/indexkv.db" } }, "/my-search/": { "handler": "search", "handlerArgs": { - "index": "/index-mem/", - "owner": "sha1-f2b0b7da718b97ce8c31591d8ed4645c777f3ef4" + "index": "/index-kv/", + "owner": "sha1-f2b0b7da718b97ce8c31591d8ed4645c777f3ef4", + "slurpToMemory": true } }, "/setup/": { @@ -141,7 +143,7 @@ "file": "/tmp/blobs/sync-to-index-queue.kv", "type": "kv" }, - "to": "/index-mem/" + "to": "/index-kv/" } }, "/ui/": { @@ -153,4 +155,4 @@ } } } -} \ No newline at end of file +} diff --git a/pkg/serverconfig/testdata/mem.json b/pkg/serverconfig/testdata/mem.json index 609f79b55..57f7017f0 100644 --- a/pkg/serverconfig/testdata/mem.json +++ b/pkg/serverconfig/testdata/mem.json @@ -5,7 +5,7 @@ "blobPath": "/tmp/blobs", "identity": "26F5ABDA", "identitySecretRing": "/path/to/secring", - "memIndex": true, + "kvIndexFile": "/path/to/indexkv.db", "s3": "key:secret:bucket", "googlecloudstorage": "clientId:clientSecret:refreshToken:bucketName", "googledrive": "clientId:clientSecret:refreshToken:parentDirId", diff --git a/pkg/serverconfig/testdata/mongo-want.json b/pkg/serverconfig/testdata/mongo-want.json index 93b3a4fc5..367240664 100644 --- a/pkg/serverconfig/testdata/mongo-want.json +++ b/pkg/serverconfig/testdata/mongo-want.json @@ -60,7 +60,8 @@ "handler": "search", "handlerArgs": { "index": "/index-mongo/", - "owner": "sha1-f2b0b7da718b97ce8c31591d8ed4645c777f3ef4" + "owner": "sha1-f2b0b7da718b97ce8c31591d8ed4645c777f3ef4", + "slurpToMemory": true } }, "/setup/": { @@ -103,4 +104,4 @@ } } } -} \ No newline at end of file +} diff --git a/pkg/serverconfig/testdata/noindex.err b/pkg/serverconfig/testdata/noindex.err index 4618929a0..ed18b1f7b 100644 --- a/pkg/serverconfig/testdata/noindex.err +++ b/pkg/serverconfig/testdata/noindex.err @@ -1 +1 @@ -Unless runIndex is set to false, you must specify an index option (kvIndexFile, mongo, mysql, postgres, sqlite, memIndex). +Unless runIndex is set to false, you must specify an index option (kvIndexFile, mongo, mysql, postgres, sqlite). diff --git a/pkg/serverconfig/testdata/s3_alt_host-want.json b/pkg/serverconfig/testdata/s3_alt_host-want.json index 8dc3f9c17..a2426fccf 100644 --- a/pkg/serverconfig/testdata/s3_alt_host-want.json +++ b/pkg/serverconfig/testdata/s3_alt_host-want.json @@ -1,7 +1,7 @@ { - "listen": "localhost:3179", "auth": "userpass:camlistore:pass3179", "https": false, + "listen": "localhost:3179", "prefixes": { "/": { "handler": "root", @@ -12,68 +12,26 @@ "stealth": false } }, - - "/ui/": { - "handler": "ui", - "handlerArgs": { - "jsonSignRoot": "/sighelper/", - "cache": "/cache/", - "scaledImage": "lrucache" - } - }, - - "/setup/": { - "handler": "setup" - }, - - "/status/": { - "handler": "status" - }, - - "/share/": { - "handler": "share", - "handlerArgs": { - "blobRoot": "/bs/" - } - }, - - "/sync/": { - "handler": "sync", - "handlerArgs": { - "from": "/bs/", - "to": "/index-mem/", - "idle": true - } - }, - - "/sighelper/": { - "handler": "jsonsign", - "handlerArgs": { - "secretRing": "/path/to/secring", - "keyId": "26F5ABDA", - "publicKeyDest": "/bs-and-index/" - } - }, - "/bs-and-index/": { "handler": "storage-replica", "handlerArgs": { - "backends": ["/bs/", "/index-mem/"] + "backends": [ + "/bs/", + "/index-kv/" + ] } }, - "/bs-and-maybe-also-index/": { "handler": "storage-cond", "handlerArgs": { + "read": "/bs/", "write": { + "else": "/bs/", "if": "isSchema", - "then": "/bs-and-index/", - "else": "/bs/" - }, - "read": "/bs/" + "then": "/bs-and-index/" + } } }, - "/bs/": { "handler": "storage-s3", "handlerArgs": { @@ -83,29 +41,62 @@ "hostname": "foo.com" } }, - "/cache/": { "handler": "storage-filesystem", "handlerArgs": { "path": "/tmp/camli-cache" } }, - - "/index-mem/": { - "handler": "storage-memory-only-dev-indexer", + "/index-kv/": { + "handler": "storage-kvfileindexer", "handlerArgs": { - "blobSource": "/bs/" + "blobSource": "/bs/", + "file": "/path/to/indexkv.db" } }, - "/my-search/": { "handler": "search", "handlerArgs": { - "index": "/index-mem/", - "owner": "sha1-f2b0b7da718b97ce8c31591d8ed4645c777f3ef4" + "index": "/index-kv/", + "owner": "sha1-f2b0b7da718b97ce8c31591d8ed4645c777f3ef4", + "slurpToMemory": true + } + }, + "/setup/": { + "handler": "setup" + }, + "/share/": { + "handler": "share", + "handlerArgs": { + "blobRoot": "/bs/" + } + }, + "/sighelper/": { + "handler": "jsonsign", + "handlerArgs": { + "keyId": "26F5ABDA", + "publicKeyDest": "/bs-and-index/", + "secretRing": "/path/to/secring" + } + }, + "/status/": { + "handler": "status" + }, + "/sync/": { + "handler": "sync", + "handlerArgs": { + "from": "/bs/", + "idle": true, + "to": "/index-kv/" + } + }, + "/ui/": { + "handler": "ui", + "handlerArgs": { + "cache": "/cache/", + "jsonSignRoot": "/sighelper/", + "scaledImage": "lrucache" } } - } - } diff --git a/pkg/serverconfig/testdata/s3_alt_host.json b/pkg/serverconfig/testdata/s3_alt_host.json index acd6dffb1..77fd4653f 100644 --- a/pkg/serverconfig/testdata/s3_alt_host.json +++ b/pkg/serverconfig/testdata/s3_alt_host.json @@ -4,7 +4,7 @@ "auth": "userpass:camlistore:pass3179", "identity": "26F5ABDA", "identitySecretRing": "/path/to/secring", - "memIndex": true, + "kvIndexFile": "/path/to/indexkv.db", "s3": "key:secret:bucket:foo.com", "replicateTo": [], "publish": {}, diff --git a/pkg/serverconfig/testdata/s3_google_nolocaldisk.json b/pkg/serverconfig/testdata/s3_google_nolocaldisk.json index 0fe116850..31be495e3 100644 --- a/pkg/serverconfig/testdata/s3_google_nolocaldisk.json +++ b/pkg/serverconfig/testdata/s3_google_nolocaldisk.json @@ -4,7 +4,7 @@ "auth": "userpass:camlistore:pass3179", "identity": "26F5ABDA", "identitySecretRing": "/path/to/secring", - "memIndex": true, + "kvIndexFile": "/path/to/indexkv.db", "s3": "key:secret:bucket", "googlecloudstorage": "clientId:clientSecret:refreshToken:bucketName", "replicateTo": [], diff --git a/pkg/serverconfig/testdata/s3_nolocaldisk-want.json b/pkg/serverconfig/testdata/s3_nolocaldisk-want.json index e411d399a..63d5794cd 100644 --- a/pkg/serverconfig/testdata/s3_nolocaldisk-want.json +++ b/pkg/serverconfig/testdata/s3_nolocaldisk-want.json @@ -1,7 +1,7 @@ { - "listen": "localhost:3179", "auth": "userpass:camlistore:pass3179", "https": false, + "listen": "localhost:3179", "prefixes": { "/": { "handler": "root", @@ -12,68 +12,26 @@ "stealth": false } }, - - "/ui/": { - "handler": "ui", - "handlerArgs": { - "jsonSignRoot": "/sighelper/", - "cache": "/cache/", - "scaledImage": "lrucache" - } - }, - - "/setup/": { - "handler": "setup" - }, - - "/status/": { - "handler": "status" - }, - - "/share/": { - "handler": "share", - "handlerArgs": { - "blobRoot": "/bs/" - } - }, - - "/sync/": { - "handler": "sync", - "handlerArgs": { - "from": "/bs/", - "to": "/index-mem/", - "idle": true - } - }, - - "/sighelper/": { - "handler": "jsonsign", - "handlerArgs": { - "secretRing": "/path/to/secring", - "keyId": "26F5ABDA", - "publicKeyDest": "/bs-and-index/" - } - }, - "/bs-and-index/": { "handler": "storage-replica", "handlerArgs": { - "backends": ["/bs/", "/index-mem/"] + "backends": [ + "/bs/", + "/index-kv/" + ] } }, - "/bs-and-maybe-also-index/": { "handler": "storage-cond", "handlerArgs": { + "read": "/bs/", "write": { + "else": "/bs/", "if": "isSchema", - "then": "/bs-and-index/", - "else": "/bs/" - }, - "read": "/bs/" + "then": "/bs-and-index/" + } } }, - "/bs/": { "handler": "storage-s3", "handlerArgs": { @@ -82,29 +40,62 @@ "bucket": "bucket" } }, - "/cache/": { "handler": "storage-filesystem", "handlerArgs": { "path": "/tmp/camli-cache" } }, - - "/index-mem/": { - "handler": "storage-memory-only-dev-indexer", + "/index-kv/": { + "handler": "storage-kvfileindexer", "handlerArgs": { - "blobSource": "/bs/" + "blobSource": "/bs/", + "file": "/path/to/indexkv.db" } }, - "/my-search/": { "handler": "search", "handlerArgs": { - "index": "/index-mem/", - "owner": "sha1-f2b0b7da718b97ce8c31591d8ed4645c777f3ef4" + "index": "/index-kv/", + "owner": "sha1-f2b0b7da718b97ce8c31591d8ed4645c777f3ef4", + "slurpToMemory": true + } + }, + "/setup/": { + "handler": "setup" + }, + "/share/": { + "handler": "share", + "handlerArgs": { + "blobRoot": "/bs/" + } + }, + "/sighelper/": { + "handler": "jsonsign", + "handlerArgs": { + "keyId": "26F5ABDA", + "publicKeyDest": "/bs-and-index/", + "secretRing": "/path/to/secring" + } + }, + "/status/": { + "handler": "status" + }, + "/sync/": { + "handler": "sync", + "handlerArgs": { + "from": "/bs/", + "idle": true, + "to": "/index-kv/" + } + }, + "/ui/": { + "handler": "ui", + "handlerArgs": { + "cache": "/cache/", + "jsonSignRoot": "/sighelper/", + "scaledImage": "lrucache" } } - } - } diff --git a/pkg/serverconfig/testdata/s3_nolocaldisk.json b/pkg/serverconfig/testdata/s3_nolocaldisk.json index bb7f0b534..5d741bb95 100644 --- a/pkg/serverconfig/testdata/s3_nolocaldisk.json +++ b/pkg/serverconfig/testdata/s3_nolocaldisk.json @@ -4,7 +4,7 @@ "auth": "userpass:camlistore:pass3179", "identity": "26F5ABDA", "identitySecretRing": "/path/to/secring", - "memIndex": true, + "kvIndexFile": "/path/to/indexkv.db", "s3": "key:secret:bucket", "replicateTo": [], "publish": {}, diff --git a/pkg/serverconfig/testdata/s3_nolocaldisk_mysql-want.json b/pkg/serverconfig/testdata/s3_nolocaldisk_mysql-want.json index 3dd09dd2e..c9b784832 100644 --- a/pkg/serverconfig/testdata/s3_nolocaldisk_mysql-want.json +++ b/pkg/serverconfig/testdata/s3_nolocaldisk_mysql-want.json @@ -1,7 +1,7 @@ { - "listen": "localhost:3179", "auth": "userpass:camlistore:pass3179", "https": false, + "listen": "localhost:3179", "prefixes": { "/": { "handler": "root", @@ -12,68 +12,26 @@ "stealth": false } }, - - "/ui/": { - "handler": "ui", - "handlerArgs": { - "jsonSignRoot": "/sighelper/", - "cache": "/cache/", - "scaledImage": "lrucache" - } - }, - - "/setup/": { - "handler": "setup" - }, - - "/status/": { - "handler": "status" - }, - - "/share/": { - "handler": "share", - "handlerArgs": { - "blobRoot": "/bs/" - } - }, - - "/sync/": { - "handler": "sync", - "handlerArgs": { - "from": "/bs/", - "to": "/index-mysql/", - "idle": true - } - }, - - "/sighelper/": { - "handler": "jsonsign", - "handlerArgs": { - "secretRing": "/path/to/secring", - "keyId": "26F5ABDA", - "publicKeyDest": "/bs-and-index/" - } - }, - "/bs-and-index/": { "handler": "storage-replica", "handlerArgs": { - "backends": ["/bs/", "/index-mysql/"] + "backends": [ + "/bs/", + "/index-mysql/" + ] } }, - "/bs-and-maybe-also-index/": { "handler": "storage-cond", "handlerArgs": { + "read": "/bs/", "write": { + "else": "/bs/", "if": "isSchema", - "then": "/bs-and-index/", - "else": "/bs/" - }, - "read": "/bs/" + "then": "/bs-and-index/" + } } }, - "/bs/": { "handler": "storage-s3", "handlerArgs": { @@ -82,14 +40,12 @@ "bucket": "bucket" } }, - "/cache/": { "handler": "storage-filesystem", "handlerArgs": { "path": "/tmp/camli-cache" } }, - "/index-mysql/": { "enabled": true, "handler": "storage-mysqlindexer", @@ -101,15 +57,49 @@ "user": "user" } }, - "/my-search/": { "handler": "search", "handlerArgs": { "index": "/index-mysql/", - "owner": "sha1-f2b0b7da718b97ce8c31591d8ed4645c777f3ef4" + "owner": "sha1-f2b0b7da718b97ce8c31591d8ed4645c777f3ef4", + "slurpToMemory": true + } + }, + "/setup/": { + "handler": "setup" + }, + "/share/": { + "handler": "share", + "handlerArgs": { + "blobRoot": "/bs/" + } + }, + "/sighelper/": { + "handler": "jsonsign", + "handlerArgs": { + "keyId": "26F5ABDA", + "publicKeyDest": "/bs-and-index/", + "secretRing": "/path/to/secring" + } + }, + "/status/": { + "handler": "status" + }, + "/sync/": { + "handler": "sync", + "handlerArgs": { + "from": "/bs/", + "idle": true, + "to": "/index-mysql/" + } + }, + "/ui/": { + "handler": "ui", + "handlerArgs": { + "cache": "/cache/", + "jsonSignRoot": "/sighelper/", + "scaledImage": "lrucache" } } - } - } diff --git a/pkg/serverconfig/testdata/sqlite-want.json b/pkg/serverconfig/testdata/sqlite-want.json index cc92349f5..3106bffa3 100644 --- a/pkg/serverconfig/testdata/sqlite-want.json +++ b/pkg/serverconfig/testdata/sqlite-want.json @@ -55,7 +55,8 @@ "handler": "search", "handlerArgs": { "index": "/index-sqlite/", - "owner": "sha1-f2b0b7da718b97ce8c31591d8ed4645c777f3ef4" + "owner": "sha1-f2b0b7da718b97ce8c31591d8ed4645c777f3ef4", + "slurpToMemory": true } }, "/setup/": { @@ -98,4 +99,4 @@ } } } -} \ No newline at end of file +} diff --git a/pkg/serverconfig/testdata/tls-want.json b/pkg/serverconfig/testdata/tls-want.json index 2c29c6974..a43663bee 100644 --- a/pkg/serverconfig/testdata/tls-want.json +++ b/pkg/serverconfig/testdata/tls-want.json @@ -19,7 +19,7 @@ "handlerArgs": { "backends": [ "/bs/", - "/index-mem/" + "/index-kv/" ] } }, @@ -46,17 +46,19 @@ "path": "/tmp/blobs/cache" } }, - "/index-mem/": { - "handler": "storage-memory-only-dev-indexer", + "/index-kv/": { + "handler": "storage-kvfileindexer", "handlerArgs": { - "blobSource": "/bs/" + "blobSource": "/bs/", + "file": "/path/to/indexkv.db" } }, "/my-search/": { "handler": "search", "handlerArgs": { - "index": "/index-mem/", - "owner": "sha1-f2b0b7da718b97ce8c31591d8ed4645c777f3ef4" + "index": "/index-kv/", + "owner": "sha1-f2b0b7da718b97ce8c31591d8ed4645c777f3ef4", + "slurpToMemory": true } }, "/setup/": { @@ -87,7 +89,7 @@ "file": "/tmp/blobs/sync-to-index-queue.kv", "type": "kv" }, - "to": "/index-mem/" + "to": "/index-kv/" } }, "/ui/": { @@ -99,4 +101,4 @@ } } } -} \ No newline at end of file +} diff --git a/pkg/serverconfig/testdata/tls.json b/pkg/serverconfig/testdata/tls.json index a7a46be70..8f840927a 100644 --- a/pkg/serverconfig/testdata/tls.json +++ b/pkg/serverconfig/testdata/tls.json @@ -7,7 +7,7 @@ "blobPath": "/tmp/blobs", "identity": "26F5ABDA", "identitySecretRing": "/path/to/secring", - "memIndex": true, + "kvIndexFile": "/path/to/indexkv.db", "s3": "", "replicateTo": [], "publish": {}, diff --git a/pkg/serverconfig/testdata/with_blog-want.json b/pkg/serverconfig/testdata/with_blog-want.json index ac748a486..b777e94a2 100644 --- a/pkg/serverconfig/testdata/with_blog-want.json +++ b/pkg/serverconfig/testdata/with_blog-want.json @@ -35,7 +35,7 @@ "handlerArgs": { "backends": [ "/bs/", - "/index-mem/" + "/index-kv/" ] } }, @@ -62,17 +62,19 @@ "path": "/tmp/blobs/cache" } }, - "/index-mem/": { - "handler": "storage-memory-only-dev-indexer", + "/index-kv/": { + "handler": "storage-kvfileindexer", "handlerArgs": { - "blobSource": "/bs/" + "blobSource": "/bs/", + "file": "/path/to/indexkv.db" } }, "/my-search/": { "handler": "search", "handlerArgs": { - "index": "/index-mem/", - "owner": "sha1-f2b0b7da718b97ce8c31591d8ed4645c777f3ef4" + "index": "/index-kv/", + "owner": "sha1-f2b0b7da718b97ce8c31591d8ed4645c777f3ef4", + "slurpToMemory": true } }, "/setup/": { @@ -103,7 +105,7 @@ "file": "/tmp/blobs/sync-to-index-queue.kv", "type": "kv" }, - "to": "/index-mem/" + "to": "/index-kv/" } }, "/ui/": { @@ -118,4 +120,4 @@ } } } -} \ No newline at end of file +} diff --git a/pkg/serverconfig/testdata/with_blog.json b/pkg/serverconfig/testdata/with_blog.json index 274037678..cfd98200d 100644 --- a/pkg/serverconfig/testdata/with_blog.json +++ b/pkg/serverconfig/testdata/with_blog.json @@ -5,7 +5,7 @@ "blobPath": "/tmp/blobs", "identity": "26F5ABDA", "identitySecretRing": "/path/to/secring", - "memIndex": true, + "kvIndexFile": "/path/to/indexkv.db", "s3": "", "publish": { "/blog/": { diff --git a/pkg/serverconfig/testdata/with_gallery-want.json b/pkg/serverconfig/testdata/with_gallery-want.json index 65d13630d..bd14c6862 100644 --- a/pkg/serverconfig/testdata/with_gallery-want.json +++ b/pkg/serverconfig/testdata/with_gallery-want.json @@ -17,7 +17,7 @@ "handlerArgs": { "backends": [ "/bs/", - "/index-mem/" + "/index-kv/" ] } }, @@ -44,17 +44,19 @@ "path": "/tmp/blobs/cache" } }, - "/index-mem/": { - "handler": "storage-memory-only-dev-indexer", + "/index-kv/": { + "handler": "storage-kvfileindexer", "handlerArgs": { - "blobSource": "/bs/" + "blobSource": "/bs/", + "file": "/path/to/indexkv.db" } }, "/my-search/": { "handler": "search", "handlerArgs": { - "index": "/index-mem/", - "owner": "sha1-f2b0b7da718b97ce8c31591d8ed4645c777f3ef4" + "index": "/index-kv/", + "owner": "sha1-f2b0b7da718b97ce8c31591d8ed4645c777f3ef4", + "slurpToMemory": true } }, "/pics/": { @@ -106,7 +108,7 @@ "file": "/tmp/blobs/sync-to-index-queue.kv", "type": "kv" }, - "to": "/index-mem/" + "to": "/index-kv/" } }, "/ui/": { @@ -121,4 +123,4 @@ } } } -} \ No newline at end of file +} diff --git a/pkg/serverconfig/testdata/with_gallery.json b/pkg/serverconfig/testdata/with_gallery.json index bd775e3d6..2a9747f08 100644 --- a/pkg/serverconfig/testdata/with_gallery.json +++ b/pkg/serverconfig/testdata/with_gallery.json @@ -5,7 +5,7 @@ "blobPath": "/tmp/blobs", "identity": "26F5ABDA", "identitySecretRing": "/path/to/secring", - "memIndex": true, + "kvIndexFile": "/path/to/indexkv.db", "s3": "", "publish": { "/pics/": { diff --git a/pkg/serverconfig/testdata/with_sourceroot-want.json b/pkg/serverconfig/testdata/with_sourceroot-want.json index fa1b83251..bfedaad5f 100644 --- a/pkg/serverconfig/testdata/with_sourceroot-want.json +++ b/pkg/serverconfig/testdata/with_sourceroot-want.json @@ -17,7 +17,7 @@ "handlerArgs": { "backends": [ "/bs/", - "/index-mem/" + "/index-kv/" ] } }, @@ -44,17 +44,19 @@ "path": "/tmp/blobs/cache" } }, - "/index-mem/": { - "handler": "storage-memory-only-dev-indexer", + "/index-kv/": { + "handler": "storage-kvfileindexer", "handlerArgs": { - "blobSource": "/bs/" + "blobSource": "/bs/", + "file": "/path/to/indexkv.db" } }, "/my-search/": { "handler": "search", "handlerArgs": { - "index": "/index-mem/", - "owner": "sha1-f2b0b7da718b97ce8c31591d8ed4645c777f3ef4" + "index": "/index-kv/", + "owner": "sha1-f2b0b7da718b97ce8c31591d8ed4645c777f3ef4", + "slurpToMemory": true } }, "/setup/": { @@ -104,7 +106,7 @@ "file": "/tmp/blobs/sync-to-index-queue.kv", "type": "kv" }, - "to": "/index-mem/" + "to": "/index-kv/" } }, "/ui/": { @@ -117,4 +119,4 @@ } } } -} \ No newline at end of file +} diff --git a/pkg/serverconfig/testdata/with_sourceroot.json b/pkg/serverconfig/testdata/with_sourceroot.json index 8de70120a..d3f96b60c 100644 --- a/pkg/serverconfig/testdata/with_sourceroot.json +++ b/pkg/serverconfig/testdata/with_sourceroot.json @@ -5,7 +5,7 @@ "blobPath": "/tmp/blobs", "identity": "26F5ABDA", "identitySecretRing": "/path/to/secring", - "memIndex": true, + "kvIndexFile": "/path/to/indexkv.db", "s3": "key:secret:bucket", "replicateTo": [], "publish": {}, diff --git a/website/content/docs/server-config b/website/content/docs/server-config index 92eae150b..f05f5a460 100644 --- a/website/content/docs/server-config +++ b/website/content/docs/server-config @@ -38,6 +38,7 @@ web browser and restart the server.

  • shareHandler: if true, the server's sharing functionality is enabled, letting your friends have access to any content you've specifically shared. Its URL prefix path defaults to "/share/".
  • shareHandlerPath: Optional. If non-empty, it specifies the URL prefix path to the share handler, and the shareHandler value is ignored (i.e the share handler is enabled). Example: "/public/".
  • runIndex: defaults to true. If "false", no search, no UI, no indexing. (These can be controlled at a more granular level by writing a low-level config file)
    • +
  • memoryIndex/code>: defaults to true. If "false", don't slurp the whole index into memory on start-up. Specifying false will result in certain queries being slow, unavailable, or unsorted (work in progress). This option may be unsupported in the future. Keeping this set to "true" is recommended.
  • sourceRoot: Optional. If non-empty, it specifies the path to an alternative Camlistore source tree, in order to override the embedded UI and/or Closure resources. The UI files will be expected in <sourceRoot>/server/camlistored/ui and the Closure library in <sourceRoot>/third_party/closure/lib.