diff --git a/config/dev-server-config.json b/config/dev-server-config.json index 6adca1e41..6204c28dd 100644 --- a/config/dev-server-config.json +++ b/config/dev-server-config.json @@ -1,304 +1,319 @@ { "_for-emacs": "-*- mode: js2;-*-", - "handlerConfig": true, - "baseURL": ["_env", "${CAMLI_BASEURL}"], - "auth": ["_env", "${CAMLI_AUTH}"], - "https": ["_env", "${CAMLI_TLS}", false], - "httpsCert": "config/selfgen_pem.crt", - "httpsKey": "config/selfgen_pem.key", - "prefixes": { - "/": { - "handler": "root", - "handlerArgs": { - "ownerName": ["_env", "${USER}-dev"], - "blobRoot": "/bs-recv/", - "statusRoot": "/status/", - "searchRoot": "/my-search/", - "stealth": false - } - }, + "handlerConfig": true, + "baseURL": ["_env", "${CAMLI_BASEURL}"], + "auth": ["_env", "${CAMLI_AUTH}"], + "https": ["_env", "${CAMLI_TLS}", false], + "httpsCert": "config/selfgen_pem.crt", + "httpsKey": "config/selfgen_pem.key", + "prefixes": { + "/": { + "handler": "root", + "handlerArgs": { + "ownerName": ["_env", "${USER}-dev"], + "blobRoot": "/bs-recv/", + "statusRoot": "/status/", + "searchRoot": "/my-search/", + "stealth": false + } + }, - "/blog/": { - "enabled": ["_env", "${CAMLI_PUBLISH_ENABLED}"], - "handler": "publish", - "handlerArgs": { - "rootName": "dev-blog-root", - "blobRoot": "/bs/", - "searchRoot": "/my-search/", - "cache": "/cache/", - "goTemplate": "blog.html", - "devBootstrapPermanodeUsing": "/sighelper/" - } - }, + "/blog/": { + "enabled": ["_env", "${CAMLI_PUBLISH_ENABLED}"], + "handler": "publish", + "handlerArgs": { + "rootName": "dev-blog-root", + "blobRoot": "/bs/", + "searchRoot": "/my-search/", + "cache": "/cache/", + "goTemplate": "blog.html", + "devBootstrapPermanodeUsing": "/sighelper/" + } + }, - "/pics/": { - "enabled": ["_env", "${CAMLI_PUBLISH_ENABLED}"], - "handler": "publish", - "handlerArgs": { - "rootName": "dev-pics-root", - "blobRoot": "/bs/", - "searchRoot": "/my-search/", - "cache": "/cache/", - "css": ["pics.css"], - "js": ["pics.js"], - "goTemplate": "gallery.html", - "devBootstrapPermanodeUsing": "/sighelper/" - } - }, + "/pics/": { + "enabled": ["_env", "${CAMLI_PUBLISH_ENABLED}"], + "handler": "publish", + "handlerArgs": { + "rootName": "dev-pics-root", + "blobRoot": "/bs/", + "searchRoot": "/my-search/", + "cache": "/cache/", + "css": ["pics.css"], + "js": ["pics.js"], + "goTemplate": "gallery.html", + "devBootstrapPermanodeUsing": "/sighelper/" + } + }, - "/stub-test-disable/": { - "handler": "publish", - "enabled": false, - "handlerArgs": { - } - }, + "/stub-test-disable/": { + "handler": "publish", + "enabled": false, + "handlerArgs": { + } + }, - "/ui/": { - "handler": "ui", - "handlerArgs": { - "sourceRoot": ["_env", "${CAMLI_DEV_CAMLI_ROOT}", ""], - "jsonSignRoot": "/sighelper/", - "cache": "/cache/", - "scaledImage": { - "type": "kv", - "file": ["_env", "${CAMLI_ROOT_CACHE}/thumbnails.kv", ""] - }, - "publishRoots": ["/blog/", "/pics/"] - } - }, + "/ui/": { + "handler": "ui", + "handlerArgs": { + "sourceRoot": ["_env", "${CAMLI_DEV_CAMLI_ROOT}", ""], + "jsonSignRoot": "/sighelper/", + "cache": "/cache/", + "scaledImage": { + "type": "kv", + "file": ["_env", "${CAMLI_ROOT_CACHE}/thumbnails.kv", ""] + }, + "publishRoots": ["/blog/", "/pics/"] + } + }, - "/status/": { - "handler": "status" - }, + "/status/": { + "handler": "status" + }, - "/sync-index/": { - "handler": "sync", - "handlerArgs": { - "from": "/bs/", - "to": ["_env", "${CAMLI_INDEXER_PATH}"], - "queue": { "type": "memory" }, - "fullSyncOnStart": ["_env", "${CAMLI_FULL_INDEX_SYNC_ON_START}"], - "blockingFullSyncOnStart": ["_env", "${CAMLI_FULL_INDEX_SYNC_ON_START}"] - } - }, + "/sync-index/": { + "handler": "sync", + "handlerArgs": { + "from": "/bs/", + "to": ["_env", "${CAMLI_INDEXER_PATH}"], + "queue": { "type": "memory" }, + "fullSyncOnStart": ["_env", "${CAMLI_FULL_INDEX_SYNC_ON_START}"], + "blockingFullSyncOnStart": ["_env", "${CAMLI_FULL_INDEX_SYNC_ON_START}"] + } + }, - "/sync-r1/": { - "handler": "sync", - "handlerArgs": { - "from": "/bs/", - "to": "/r1/", - "queue": { "type": "memory" } - } - }, + "/sync-r1/": { + "handler": "sync", + "handlerArgs": { + "from": "/bs/", + "to": "/r1/", + "queue": { "type": "memory" } + } + }, - "/sighelper/": { - "handler": "jsonsign", - "handlerArgs": { - "secretRing": ["_env", "${CAMLI_SECRET_RING}"], - "keyId": ["_env", "${CAMLI_KEYID}"], - "publicKeyDest": "/bs/" - } - }, + "/sighelper/": { + "handler": "jsonsign", + "handlerArgs": { + "secretRing": ["_env", "${CAMLI_SECRET_RING}"], + "keyId": ["_env", "${CAMLI_KEYID}"], + "publicKeyDest": "/bs/" + } + }, - "/bs-recv/": { - "handler": "storage-replica", - "handlerArgs": { - "minWritesForSuccess": 2, - "backends": ["/bs/", ["_env", "${CAMLI_INDEXER_PATH}"]], - "readBackends": ["/bs/"] - } - }, + "/bs-recv/": { + "handler": "storage-replica", + "handlerArgs": { + "minWritesForSuccess": 2, + "backends": ["/bs/", ["_env", "${CAMLI_INDEXER_PATH}"]], + "readBackends": ["/bs/"] + } + }, - "/cond-unused/": { - "handler": "storage-cond", - "handlerArgs": { - "write": { - "if": "isSchema", - "then": "/bs-recv/", - "else": "/bs/" - }, - "read": "/bs/" - } - }, + "/cond-unused/": { + "handler": "storage-cond", + "handlerArgs": { + "write": { + "if": "isSchema", + "then": "/bs-recv/", + "else": "/bs/" + }, + "read": "/bs/" + } + }, - "/bs/": { - "handler": "storage-filesystem", - "handlerArgs": { - "path": ["_env", "${CAMLI_ROOT}"] - } - }, + "/bs/": { + "handler": "storage-filesystem", + "handlerArgs": { + "path": ["_env", "${CAMLI_ROOT}"] + } + }, - "/cache/": { - "handler": "storage-filesystem", - "handlerArgs": { - "path": ["_env", "${CAMLI_ROOT_CACHE}"] - } - }, + "/cache/": { + "handler": "storage-filesystem", + "handlerArgs": { + "path": ["_env", "${CAMLI_ROOT_CACHE}"] + } + }, - "/sharder/": { - "handler": "storage-shard", - "handlerArgs": { - "backends": ["/s1/", "/s2/"] - } - }, + "/sharder/": { + "handler": "storage-shard", + "handlerArgs": { + "backends": ["/s1/", "/s2/"] + } + }, - "/s1/": { - "handler": "storage-filesystem", - "handlerArgs": { - "path": ["_env", "${CAMLI_ROOT_SHARD1}"] - } - }, + "/s1/": { + "handler": "storage-filesystem", + "handlerArgs": { + "path": ["_env", "${CAMLI_ROOT_SHARD1}"] + } + }, - "/s2/": { - "handler": "storage-filesystem", - "handlerArgs": { - "path": ["_env", "${CAMLI_ROOT_SHARD2}"] - } - }, + "/s2/": { + "handler": "storage-filesystem", + "handlerArgs": { + "path": ["_env", "${CAMLI_ROOT_SHARD2}"] + } + }, - "/repl/": { - "handler": "storage-replica", - "handlerArgs": { - "backends": ["/r1/", "/r2/", "/r3/"], - "minWritesForSuccess": 2 - } - }, + "/repl/": { + "handler": "storage-replica", + "handlerArgs": { + "backends": ["/r1/", "/r2/", "/r3/"], + "minWritesForSuccess": 2 + } + }, - "/r1/": { - "handler": "storage-diskpacked", - "handlerArgs": { - "path": ["_env", "${CAMLI_ROOT_REPLICA1}"] - } - }, + "/r1/": { + "handler": "storage-diskpacked", + "handlerArgs": { + "path": ["_env", "${CAMLI_ROOT_REPLICA1}"] + } + }, - "/r2/": { - "handler": "storage-filesystem", - "handlerArgs": { - "path": ["_env", "${CAMLI_ROOT_REPLICA2}"] - } - }, + "/r2/": { + "handler": "storage-filesystem", + "handlerArgs": { + "path": ["_env", "${CAMLI_ROOT_REPLICA2}"] + } + }, - "/r3/": { - "handler": "storage-filesystem", - "handlerArgs": { - "path": ["_env", "${CAMLI_ROOT_REPLICA3}"] - } - }, + "/r3/": { + "handler": "storage-filesystem", + "handlerArgs": { + "path": ["_env", "${CAMLI_ROOT_REPLICA3}"] + } + }, - "/enc/": { - "handler": "storage-encrypt", - "handlerArgs": { - "I_AGREE": "that encryption support hasn't been peer-reviewed, isn't finished, and its format might change.", - "meta": "/encmeta/", - "blobs": "/encblob/", - "metaIndex": { "type": "memory" }, - "key": "000102030405060708090a0b0c0d0e0f" - } - }, + "/enc/": { + "handler": "storage-encrypt", + "handlerArgs": { + "I_AGREE": "that encryption support hasn't been peer-reviewed, isn't finished, and its format might change.", + "meta": "/encmeta/", + "blobs": "/encblob/", + "metaIndex": { "type": "memory" }, + "key": "000102030405060708090a0b0c0d0e0f" + } + }, - "/encmeta/": { - "handler": "storage-filesystem", - "handlerArgs": { - "path": ["_env", "${CAMLI_ROOT_ENCMETA}"] - } - }, + "/encmeta/": { + "handler": "storage-filesystem", + "handlerArgs": { + "path": ["_env", "${CAMLI_ROOT_ENCMETA}"] + } + }, - "/encblob/": { - "handler": "storage-filesystem", - "handlerArgs": { - "path": ["_env", "${CAMLI_ROOT_ENCBLOB}"] - } - }, + "/encblob/": { + "handler": "storage-filesystem", + "handlerArgs": { + "path": ["_env", "${CAMLI_ROOT_ENCBLOB}"] + } + }, - "/index-kv/": { - "enabled": ["_env", "${CAMLI_KVINDEX_ENABLED}"], - "handler": "storage-kvfileindexer", - "handlerArgs": { - "blobSource": "/bs/", - "file": ["_env", "${CAMLI_DBNAME}", ""] - } - }, + "/index-kv/": { + "enabled": ["_env", "${CAMLI_KVINDEX_ENABLED}"], + "handler": "storage-index", + "handlerArgs": { + "blobSource": "/bs/", + "storage": { + "type": "kv", + "file": ["_env", "${CAMLI_DBNAME}", ""] + } + } + }, - "/index-mongo/": { - "enabled": ["_env", "${CAMLI_MONGO_ENABLED}", true], - "handler": "storage-mongodbindexer", - "handlerArgs": { - "host": "localhost", - "database": ["_env", "${CAMLI_DBNAME}"], - "blobSource": "/bs/" - } - }, + "/index-mongo/": { + "enabled": ["_env", "${CAMLI_MONGO_ENABLED}", true], + "handler": "storage-index", + "handlerArgs": { + "blobSource": "/bs/", + "storage": { + "type": "mongo", + "host": "localhost", + "database": ["_env", "${CAMLI_DBNAME}"] + } + } + }, - "/index-mysql/": { - "enabled": ["_env", "${CAMLI_MYSQL_ENABLED}", true], - "handler": "storage-mysqlindexer", - "handlerArgs": { - "database": ["_env", "${CAMLI_DBNAME}"], - "user": "root", - "password": "root", - "host": "127.0.0.1", - "blobSource": "/bs/" - } - }, + "/index-mysql/": { + "enabled": ["_env", "${CAMLI_MYSQL_ENABLED}", true], + "handler": "storage-index", + "handlerArgs": { + "blobSource": "/bs/", + "storage": { + "type": "mysql", + "database": ["_env", "${CAMLI_DBNAME}"], + "user": "root", + "password": "root", + "host": "127.0.0.1" + } + } + }, - "/index-postgres/": { - "enabled": ["_env", "${CAMLI_POSTGRES_ENABLED}", true], - "handler": "storage-postgresindexer", - "handlerArgs": { - "database": ["_env", "${CAMLI_DBNAME}"], - "user": "postgres", - "password": "postgres", - "host": "127.0.0.1", - "blobSource": "/bs/" - } - }, + "/index-postgres/": { + "enabled": ["_env", "${CAMLI_POSTGRES_ENABLED}", true], + "handler": "storage-index", + "handlerArgs": { + "blobSource": "/bs/", + "storage": { + "type": "postgres", + "database": ["_env", "${CAMLI_DBNAME}"], + "user": "postgres", + "password": "postgres", + "host": "127.0.0.1" + } + } + }, - "/index-sqlite/": { - "enabled": ["_env", "${CAMLI_SQLITE_ENABLED}", true], - "handler": "storage-sqliteindexer", - "handlerArgs": { - "file": ["_env", "${CAMLI_DBNAME}"], - "blobSource": "/bs/" - } - }, + "/index-sqlite/": { + "enabled": ["_env", "${CAMLI_SQLITE_ENABLED}", true], + "handler": "storage-index", + "handlerArgs": { + "blobSource": "/bs/", + "storage": { + "type": "sqlite", + "file": ["_env", "${CAMLI_DBNAME}"] + } + } + }, - "/my-search/": { - "handler": "search", - "handlerArgs": { - "index": ["_env", "${CAMLI_INDEXER_PATH}"], - "owner": ["_env", "${CAMLI_PUBKEY_BLOBREF}"], - "slurpToMemory": true, - "devBlockStartupOn": "/sync-index/" - } - }, + "/my-search/": { + "handler": "search", + "handlerArgs": { + "index": ["_env", "${CAMLI_INDEXER_PATH}"], + "owner": ["_env", "${CAMLI_PUBKEY_BLOBREF}"], + "slurpToMemory": true, + "devBlockStartupOn": "/sync-index/" + } + }, - "/importer/": { - "handler": "importer", - "handlerArgs": { - "dummy": { - "clientID": "dummyID", - "clientSecret": "foobar" - }, - "flickr": { - "clientSecret": ["_env", "${CAMLI_FLICKR_API_KEY}", ""] - }, - "foursquare": { - "clientSecret": ["_env", "${CAMLI_FOURSQUARE_API_KEY}", ""] - }, - "picasa": { - "clientSecret": ["_env", "${CAMLI_PICASA_API_KEY}", ""] - }, - "twitter": { - "clientSecret": ["_env", "${CAMLI_TWITTER_API_KEY}", ""] - } - } - }, + "/importer/": { + "handler": "importer", + "handlerArgs": { + "dummy": { + "clientID": "dummyID", + "clientSecret": "foobar" + }, + "flickr": { + "clientSecret": ["_env", "${CAMLI_FLICKR_API_KEY}", ""] + }, + "foursquare": { + "clientSecret": ["_env", "${CAMLI_FOURSQUARE_API_KEY}", ""] + }, + "picasa": { + "clientSecret": ["_env", "${CAMLI_PICASA_API_KEY}", ""] + }, + "twitter": { + "clientSecret": ["_env", "${CAMLI_TWITTER_API_KEY}", ""] + } + } + }, - "/share/": { - "handler": "share", - "handlerArgs": { - "blobRoot": "/bs/" - } - } - } + "/share/": { + "handler": "share", + "handlerArgs": { + "blobRoot": "/bs/" + } + } + } } diff --git a/pkg/index/index.go b/pkg/index/index.go index d8d299fe7..3ae8c990d 100644 --- a/pkg/index/index.go +++ b/pkg/index/index.go @@ -32,6 +32,7 @@ import ( "camlistore.org/pkg/blob" "camlistore.org/pkg/blobserver" "camlistore.org/pkg/context" + "camlistore.org/pkg/jsonconfig" "camlistore.org/pkg/schema" "camlistore.org/pkg/sorted" "camlistore.org/pkg/strutil" @@ -39,6 +40,10 @@ import ( "camlistore.org/pkg/types/camtypes" ) +func init() { + blobserver.RegisterStorageConstructor("index", newFromConfig) +} + type Index struct { *blobserver.NoImplStorage @@ -141,6 +146,35 @@ func New(s sorted.KeyValue) (*Index, error) { return idx, nil } +func newFromConfig(ld blobserver.Loader, config jsonconfig.Obj) (blobserver.Storage, error) { + blobPrefix := config.RequiredString("blobSource") + kvConfig := config.RequiredObject("storage") + if err := config.Validate(); err != nil { + return nil, err + } + kv, err := sorted.NewKeyValue(kvConfig) + if err != nil { + return nil, err + } + + ix, err := New(kv) + if err != nil { + return nil, err + } + + sto, err := ld.GetStorage(blobPrefix) + if err != nil { + ix.Close() + return nil, err + } + ix.BlobSource = sto + + // Good enough, for now: + ix.KeyFetcher = ix.BlobSource + + return ix, err +} + func (x *Index) String() string { return fmt.Sprintf("Camlistore index, using key/value implementation %T", x.s) } diff --git a/pkg/index/index_test.go b/pkg/index/index_test.go index 287193ab0..1f914ae47 100644 --- a/pkg/index/index_test.go +++ b/pkg/index/index_test.go @@ -69,9 +69,15 @@ func TestDelete_Memory(t *testing.T) { } var ( - // those dirs are not packages implementing indexers, + // those test files are not specific to an indexer implementation // hence we do not want to check them. - excludedDirs = []string{"indextest", "testdata", "sqlindex"} + notAnIndexer = []string{ + "corpus_bench_test.go", + "corpus_test.go", + "export_test.go", + "index_test.go", + "keys_test.go", + } // A map is used in hasAllRequiredTests to note which required // tests have been found in a package, by setting the corresponding // booleans to true. Those are the keys for this map. @@ -81,54 +87,43 @@ var ( // This function checks that all the functions using the tests // defined in indextest, namely: // TestIndex_, TestPathOfSignerTarget_, TestFiles_ -// do exist in the provided package. -func hasAllRequiredTests(path string, t *testing.T) error { +// do exist in the provided test file. +func hasAllRequiredTests(name string, t *testing.T) error { tests := make(map[string]bool) for _, v := range requiredTests { tests[v] = false } - dir, err := os.Open(path) - if err != nil { - t.Fatal(err) - } - names, err := dir.Readdirnames(-1) - if err != nil { - t.Fatal(err) - } - defer dir.Close() - for _, name := range names { - if strings.HasPrefix(name, ".") || !strings.HasSuffix(name, "_test.go") { - continue - } - fset := token.NewFileSet() - f, err := parser.ParseFile(fset, filepath.Join(path, name), nil, 0) - if err != nil { - t.Fatalf("%v: %v", filepath.Join(path, name), err) - } - ast.Inspect(f, func(n ast.Node) bool { - switch x := n.(type) { - case *ast.FuncDecl: - name := x.Name.Name - for k, _ := range tests { - if strings.HasPrefix(name, k) { - tests[k] = true - } + if !strings.HasSuffix(name, "_test.go") || skipFromList(name) { + return nil + } + fset := token.NewFileSet() + f, err := parser.ParseFile(fset, name, nil, 0) + if err != nil { + t.Fatalf("%v: %v", name, err) + } + ast.Inspect(f, func(n ast.Node) bool { + switch x := n.(type) { + case *ast.FuncDecl: + name := x.Name.Name + for k, _ := range tests { + if strings.HasPrefix(name, k) { + tests[k] = true } } - return true - }) - } + } + return true + }) for k, v := range tests { if !v { - return fmt.Errorf("%v not implemented in %v", k, path) + return fmt.Errorf("%v not implemented in %v", k, name) } } return nil } -// For each package implementing an indexer, this checks that +// For each test file dedicated to an indexer implementation, this checks that // all the required tests are present in its test suite. func TestIndexerTestsCompleteness(t *testing.T) { cwd, err := os.Open(".") @@ -143,21 +138,22 @@ func TestIndexerTestsCompleteness(t *testing.T) { for _, file := range files { name := file.Name() - if !file.IsDir() || skipDir(name) { + if file.IsDir() || strings.HasPrefix(name, ".") { continue } if err := hasAllRequiredTests(name, t); err != nil { t.Error(err) } } + // special case for sqlite as it is the only one left in its own package + if err := hasAllRequiredTests(filepath.FromSlash("sqlite/sqlite_test.go"), t); err != nil { + t.Error(err) + } } -func skipDir(name string) bool { - if strings.HasPrefix(name, "_") { - return true - } - for _, v := range excludedDirs { - if v == name { +func skipFromList(name string) bool { + for _, v := range notAnIndexer { + if name == v { return true } } diff --git a/pkg/index/kvfile/kvfile.go b/pkg/index/kvfile/kvfile.go deleted file mode 100644 index a1c5660e5..000000000 --- a/pkg/index/kvfile/kvfile.go +++ /dev/null @@ -1,57 +0,0 @@ -/* -Copyright 2013 The Camlistore Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package kvfile implements the Camlistore index storage abstraction -// on top of a single mutable database file on disk using -// github.com/cznic/kv. -package kvfile - -import ( - "camlistore.org/pkg/blobserver" - "camlistore.org/pkg/index" - "camlistore.org/pkg/jsonconfig" - "camlistore.org/pkg/sorted/kvfile" -) - -func init() { - blobserver.RegisterStorageConstructor("kvfileindexer", - blobserver.StorageConstructor(newFromConfig)) -} - -func newFromConfig(ld blobserver.Loader, config jsonconfig.Obj) (blobserver.Storage, error) { - blobPrefix := config.RequiredString("blobSource") - kv, err := kvfile.NewKeyValue(config) - if err != nil { - return nil, err - } - - ix, err := index.New(kv) - if err != nil { - return nil, err - } - - sto, err := ld.GetStorage(blobPrefix) - if err != nil { - ix.Close() - return nil, err - } - ix.BlobSource = sto - - // Good enough, for now: - ix.KeyFetcher = ix.BlobSource - - return ix, err -} diff --git a/pkg/index/kvfile/kvfile_test.go b/pkg/index/kvfile_test.go similarity index 56% rename from pkg/index/kvfile/kvfile_test.go rename to pkg/index/kvfile_test.go index f49283669..60a7cd46e 100644 --- a/pkg/index/kvfile/kvfile_test.go +++ b/pkg/index/kvfile_test.go @@ -14,12 +14,13 @@ See the License for the specific language governing permissions and limitations under the License. */ -package kvfile_test +package index_test import ( "io/ioutil" "os" "path/filepath" + "sync" "testing" "camlistore.org/pkg/index" @@ -30,7 +31,7 @@ import ( "camlistore.org/pkg/test" ) -func newSorted(t *testing.T) (kv sorted.KeyValue, cleanup func()) { +func newKvfileSorted(t *testing.T) (kv sorted.KeyValue, cleanup func()) { td, err := ioutil.TempDir("", "kvfile-test") if err != nil { t.Fatal(err) @@ -46,48 +47,50 @@ func newSorted(t *testing.T) (kv sorted.KeyValue, cleanup func()) { } } -func TestSortedKV(t *testing.T) { - kv, cleanup := newSorted(t) +func TestSorted_Kvfile(t *testing.T) { + kv, cleanup := newKvfileSorted(t) defer cleanup() kvtest.TestSorted(t, kv) } -type tester struct{} - -func (tester) test(t *testing.T, tfn func(*testing.T, func() *index.Index)) { +func indexTest(t *testing.T, + sortedGenfn func(t *testing.T) (sorted.KeyValue, func()), + tfn func(*testing.T, func() *index.Index)) { defer test.TLog(t)() + var mu sync.Mutex // guards cleanups var cleanups []func() defer func() { + mu.Lock() // never unlocked for _, fn := range cleanups { fn() } }() - - initIndex := func() *index.Index { - kv, cleanup := newSorted(t) + makeIndex := func() *index.Index { + s, cleanup := sortedGenfn(t) + mu.Lock() cleanups = append(cleanups, cleanup) - return index.MustNew(t, kv) + mu.Unlock() + return index.MustNew(t, s) } - - tfn(t, initIndex) + tfn(t, makeIndex) } -func TestIndex_KV(t *testing.T) { - tester{}.test(t, indextest.Index) +func TestIndex_Kvfile(t *testing.T) { + indexTest(t, newKvfileSorted, indextest.Index) } -func TestPathsOfSignerTarget_KV(t *testing.T) { - tester{}.test(t, indextest.PathsOfSignerTarget) +func TestPathsOfSignerTarget_Kvfile(t *testing.T) { + indexTest(t, newKvfileSorted, indextest.PathsOfSignerTarget) } -func TestFiles_KV(t *testing.T) { - tester{}.test(t, indextest.Files) +func TestFiles_Kvfile(t *testing.T) { + indexTest(t, newKvfileSorted, indextest.Files) } -func TestEdgesTo_KV(t *testing.T) { - tester{}.test(t, indextest.EdgesTo) +func TestEdgesTo_Kvfile(t *testing.T) { + indexTest(t, newKvfileSorted, indextest.EdgesTo) } -func TestDelete_KV(t *testing.T) { - tester{}.test(t, indextest.Delete) +func TestDelete_Kvfile(t *testing.T) { + indexTest(t, newKvfileSorted, indextest.Delete) } diff --git a/pkg/index/mongo/mongoindex.go b/pkg/index/mongo/mongoindex.go deleted file mode 100644 index 87bf45c2a..000000000 --- a/pkg/index/mongo/mongoindex.go +++ /dev/null @@ -1,58 +0,0 @@ -/* -Copyright 2011 The Camlistore Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package mongo implements the Camlistore index storage abstraction -// on top of MongoDB. -package mongo - -import ( - "camlistore.org/pkg/blobserver" - "camlistore.org/pkg/index" - "camlistore.org/pkg/jsonconfig" - "camlistore.org/pkg/sorted/mongo" -) - -func init() { - blobserver.RegisterStorageConstructor("mongodbindexer", newFromConfig) -} - -func newFromConfig(ld blobserver.Loader, config jsonconfig.Obj) (blobserver.Storage, error) { - blobPrefix := config.RequiredString("blobSource") - mongoConf, err := mongo.ConfigFromJSON(config) - if err != nil { - return nil, err - } - kv, err := mongo.NewKeyValue(mongoConf) - if err != nil { - return nil, err - } - - ix, err := index.New(kv) - if err != nil { - return nil, err - } - - sto, err := ld.GetStorage(blobPrefix) - if err != nil { - return nil, err - } - ix.BlobSource = sto - - // Good enough, for now: - ix.KeyFetcher = ix.BlobSource - - return ix, err -} diff --git a/pkg/index/mongo/mongoindex_test.go b/pkg/index/mongo/mongoindex_test.go deleted file mode 100644 index c00bd9d4a..000000000 --- a/pkg/index/mongo/mongoindex_test.go +++ /dev/null @@ -1,123 +0,0 @@ -/* -Copyright 2011 Google Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package mongo_test - -import ( - "errors" - "sync" - "testing" - "time" - - "camlistore.org/pkg/index" - "camlistore.org/pkg/index/indextest" - "camlistore.org/pkg/sorted" - "camlistore.org/pkg/sorted/kvtest" - "camlistore.org/pkg/sorted/mongo" - "camlistore.org/pkg/test" -) - -var ( - once sync.Once - mongoNotAvailable bool -) - -func checkMongoUp() { - mongoNotAvailable = !mongo.Ping("localhost", 500*time.Millisecond) -} - -func skipOrFailIfNoMongo(t *testing.T) { - once.Do(checkMongoUp) - if mongoNotAvailable { - err := errors.New("Not running; start a mongoDB daemon on the standard port (27017). The \"keys\" collection in the \"camlitest\" database will be used.") - test.DependencyErrorOrSkip(t) - t.Fatalf("Mongo not available locally for testing: %v", err) - } -} - -func newSorted(t *testing.T) (kv sorted.KeyValue, cleanup func()) { - skipOrFailIfNoMongo(t) - - // connect without credentials and wipe the database - cfg := mongo.Config{ - Server: "localhost", - Database: "camlitest", - } - var err error - kv, err = mongo.NewKeyValue(cfg) - if err != nil { - t.Fatal(err) - } - wiper, ok := kv.(sorted.Wiper) - if !ok { - panic("mongo KeyValue not a Wiper") - } - err = wiper.Wipe() - if err != nil { - t.Fatal(err) - } - return kv, func() { - kv.Close() - } -} - -func TestSortedKV(t *testing.T) { - kv, cleanup := newSorted(t) - defer cleanup() - kvtest.TestSorted(t, kv) -} - -type mongoTester struct{} - -func (mongoTester) test(t *testing.T, tfn func(*testing.T, func() *index.Index)) { - defer test.TLog(t)() - var mu sync.Mutex // guards cleanups - var cleanups []func() - defer func() { - mu.Lock() // never unlocked - for _, fn := range cleanups { - fn() - } - }() - initIndex := func() *index.Index { - kv, cleanup := newSorted(t) - mu.Lock() - cleanups = append(cleanups, cleanup) - mu.Unlock() - return index.MustNew(t, kv) - } - tfn(t, initIndex) -} - -func TestIndex_Mongo(t *testing.T) { - mongoTester{}.test(t, indextest.Index) -} - -func TestPathsOfSignerTarget_Mongo(t *testing.T) { - mongoTester{}.test(t, indextest.PathsOfSignerTarget) -} - -func TestFiles_Mongo(t *testing.T) { - mongoTester{}.test(t, indextest.Files) -} - -func TestEdgesTo_Mongo(t *testing.T) { - mongoTester{}.test(t, indextest.EdgesTo) -} - -func TestDelete_Mongo(t *testing.T) { - mongoTester{}.test(t, indextest.Delete) -} diff --git a/pkg/index/mongo_test.go b/pkg/index/mongo_test.go new file mode 100644 index 000000000..d56d6dd51 --- /dev/null +++ b/pkg/index/mongo_test.go @@ -0,0 +1,72 @@ +/* +Copyright 2011 Google Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package index_test + +import ( + "testing" + + "camlistore.org/pkg/index/indextest" + "camlistore.org/pkg/osutil" + "camlistore.org/pkg/sorted" + "camlistore.org/pkg/sorted/kvtest" + "camlistore.org/pkg/sorted/mongo" + "camlistore.org/pkg/test/dockertest" +) + +func newMongoSorted(t *testing.T) (kv sorted.KeyValue, cleanup func()) { + dbname := "camlitest_" + osutil.Username() + containerID, ip := dockertest.SetupMongoContainer(t) + + kv, err := mongo.NewKeyValue(mongo.Config{ + Server: ip, + Database: dbname, + }) + if err != nil { + containerID.Kill() + t.Fatal(err) + } + return kv, func() { + kv.Close() + containerID.Kill() + } +} + +func TestSorted_Mongo(t *testing.T) { + kv, cleanup := newMongoSorted(t) + defer cleanup() + kvtest.TestSorted(t, kv) +} + +func TestIndex_Mongo(t *testing.T) { + indexTest(t, newMongoSorted, indextest.Index) +} + +func TestPathsOfSignerTarget_Mongo(t *testing.T) { + indexTest(t, newMongoSorted, indextest.PathsOfSignerTarget) +} + +func TestFiles_Mongo(t *testing.T) { + indexTest(t, newMongoSorted, indextest.Files) +} + +func TestEdgesTo_Mongo(t *testing.T) { + indexTest(t, newMongoSorted, indextest.EdgesTo) +} + +func TestDelete_Mongo(t *testing.T) { + indexTest(t, newMongoSorted, indextest.Delete) +} diff --git a/pkg/index/mysql/mysql_test.go b/pkg/index/mysql/mysql_test.go deleted file mode 100644 index dc4c41f0a..000000000 --- a/pkg/index/mysql/mysql_test.go +++ /dev/null @@ -1,142 +0,0 @@ -/* -Copyright 2012 The Camlistore Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package mysql_test - -import ( - "database/sql" - "errors" - "log" - "sync" - "testing" - - "camlistore.org/pkg/index" - "camlistore.org/pkg/index/indextest" - "camlistore.org/pkg/osutil" - "camlistore.org/pkg/sorted" - "camlistore.org/pkg/sorted/kvtest" - "camlistore.org/pkg/sorted/mysql" - "camlistore.org/pkg/test" - - _ "camlistore.org/third_party/github.com/ziutek/mymysql/godrv" -) - -var ( - once sync.Once - dbAvailable bool - rootdb *sql.DB -) - -func checkDB() { - var err error - rootdb, err = sql.Open("mymysql", "mysql/root/root") - if err != nil { - log.Printf("Could not open rootdb: %v", err) - return - } - var n int - err = rootdb.QueryRow("SELECT COUNT(*) FROM user").Scan(&n) - if err == nil { - dbAvailable = true - } -} - -func skipOrFailIfNoMySQL(t *testing.T) { - once.Do(checkDB) - if !dbAvailable { - // TODO(bradfitz): accept somehow other passwords than - // 'root', and/or try localhost unix socket - // connections rather than using TCP localhost? - err := errors.New("Not running; start a MySQL daemon on the standard port (3306) with root password 'root'") - test.DependencyErrorOrSkip(t) - t.Fatalf("MySQL not available locally for testing: %v", err) - } -} - -func do(db *sql.DB, sql string) { - _, err := db.Exec(sql) - if err == nil { - return - } - log.Fatalf("Error %v running SQL: %s", err, sql) -} - -func newSorted(t *testing.T) (kv sorted.KeyValue, clean func()) { - skipOrFailIfNoMySQL(t) - dbname := "camlitest_" + osutil.Username() - do(rootdb, "DROP DATABASE IF EXISTS "+dbname) - do(rootdb, "CREATE DATABASE "+dbname) - - kv, err := mysql.NewKeyValue(mysql.Config{ - Host: "localhost:3306", - Database: dbname, - User: "root", - Password: "root", - }) - if err != nil { - t.Fatal(err) - } - return kv, func() { - kv.Close() - } -} - -func TestSortedKV(t *testing.T) { - kv, clean := newSorted(t) - defer clean() - kvtest.TestSorted(t, kv) -} - -type mysqlTester struct{} - -func (mysqlTester) test(t *testing.T, tfn func(*testing.T, func() *index.Index)) { - var mu sync.Mutex // guards cleanups - var cleanups []func() - defer func() { - mu.Lock() // never unlocked - for _, fn := range cleanups { - fn() - } - }() - makeIndex := func() *index.Index { - s, cleanup := newSorted(t) - mu.Lock() - cleanups = append(cleanups, cleanup) - mu.Unlock() - return index.MustNew(t, s) - } - tfn(t, makeIndex) -} - -func TestIndex_MySQL(t *testing.T) { - mysqlTester{}.test(t, indextest.Index) -} - -func TestPathsOfSignerTarget_MySQL(t *testing.T) { - mysqlTester{}.test(t, indextest.PathsOfSignerTarget) -} - -func TestFiles_MySQL(t *testing.T) { - mysqlTester{}.test(t, indextest.Files) -} - -func TestEdgesTo_MySQL(t *testing.T) { - mysqlTester{}.test(t, indextest.EdgesTo) -} - -func TestDelete_MySQL(t *testing.T) { - mysqlTester{}.test(t, indextest.Delete) -} diff --git a/pkg/index/mysql/mysqlindexer.go b/pkg/index/mysql/mysqlindexer.go deleted file mode 100644 index 2536f57fd..000000000 --- a/pkg/index/mysql/mysqlindexer.go +++ /dev/null @@ -1,60 +0,0 @@ -/* -Copyright 2011 The Camlistore Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package mysql implements the Camlistore index storage abstraction -// on top of MySQL. -package mysql - -import ( - "camlistore.org/pkg/blobserver" - "camlistore.org/pkg/index" - "camlistore.org/pkg/jsonconfig" - "camlistore.org/pkg/sorted/mysql" - - _ "camlistore.org/third_party/github.com/ziutek/mymysql/godrv" -) - -func init() { - blobserver.RegisterStorageConstructor("mysqlindexer", newFromConfig) -} - -func newFromConfig(ld blobserver.Loader, config jsonconfig.Obj) (blobserver.Storage, error) { - blobPrefix := config.RequiredString("blobSource") - mysqlConf, err := mysql.ConfigFromJSON(config) - if err != nil { - return nil, err - } - kv, err := mysql.NewKeyValue(mysqlConf) - if err != nil { - return nil, err - } - - ix, err := index.New(kv) - if err != nil { - return nil, err - } - - sto, err := ld.GetStorage(blobPrefix) - if err != nil { - ix.Close() - return nil, err - } - ix.BlobSource = sto - // Good enough, for now: - ix.KeyFetcher = ix.BlobSource - - return ix, nil -} diff --git a/pkg/index/mysql_test.go b/pkg/index/mysql_test.go new file mode 100644 index 000000000..3e7fdb010 --- /dev/null +++ b/pkg/index/mysql_test.go @@ -0,0 +1,74 @@ +/* +Copyright 2012 The Camlistore Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package index_test + +import ( + "testing" + + "camlistore.org/pkg/index/indextest" + "camlistore.org/pkg/osutil" + "camlistore.org/pkg/sorted" + "camlistore.org/pkg/sorted/kvtest" + "camlistore.org/pkg/sorted/mysql" + "camlistore.org/pkg/test/dockertest" +) + +func newMySQLSorted(t *testing.T) (kv sorted.KeyValue, clean func()) { + dbname := "camlitest_" + osutil.Username() + containerID, ip := dockertest.SetupMySQLContainer(t, dbname) + + kv, err := mysql.NewKeyValue(mysql.Config{ + Host: ip + ":3306", + Database: dbname, + User: dockertest.MySQLUsername, + Password: dockertest.MySQLPassword, + }) + if err != nil { + containerID.Kill() + t.Fatal(err) + } + return kv, func() { + kv.Close() + containerID.Kill() + } +} + +func TestSorted_MySQL(t *testing.T) { + kv, clean := newMySQLSorted(t) + defer clean() + kvtest.TestSorted(t, kv) +} + +func TestIndex_MySQL(t *testing.T) { + indexTest(t, newMySQLSorted, indextest.Index) +} + +func TestPathsOfSignerTarget_MySQL(t *testing.T) { + indexTest(t, newMySQLSorted, indextest.PathsOfSignerTarget) +} + +func TestFiles_MySQL(t *testing.T) { + indexTest(t, newMySQLSorted, indextest.Files) +} + +func TestEdgesTo_MySQL(t *testing.T) { + indexTest(t, newMySQLSorted, indextest.EdgesTo) +} + +func TestDelete_MySQL(t *testing.T) { + indexTest(t, newMySQLSorted, indextest.Delete) +} diff --git a/pkg/index/postgres/postgres_test.go b/pkg/index/postgres/postgres_test.go deleted file mode 100644 index 868c10e87..000000000 --- a/pkg/index/postgres/postgres_test.go +++ /dev/null @@ -1,219 +0,0 @@ -/* -Copyright 2012 The Camlistore Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package postgres_test - -import ( - "database/sql" - "errors" - "fmt" - "log" - "strings" - "sync" - "testing" - - "camlistore.org/pkg/index" - "camlistore.org/pkg/index/indextest" - "camlistore.org/pkg/osutil" - "camlistore.org/pkg/sorted" - "camlistore.org/pkg/sorted/kvtest" - "camlistore.org/pkg/sorted/postgres" - "camlistore.org/pkg/test" - - _ "camlistore.org/third_party/github.com/lib/pq" -) - -var ( - once sync.Once - dbAvailable bool - rootdb *sql.DB -) - -func checkDB() { - var err error - rootdb, err = sql.Open("postgres", "user=postgres password=postgres host=localhost dbname=postgres") - if err != nil { - log.Printf("Could not open postgres rootdb: %v", err) - return - } - var n int - err = rootdb.QueryRow("SELECT COUNT(*) FROM user").Scan(&n) - if err == nil { - dbAvailable = true - } -} - -func skipOrFailIfNoPostgreSQL(t *testing.T) { - once.Do(checkDB) - if !dbAvailable { - err := errors.New("Not running; start a postgres daemon on the standard port (5432) with password 'postgres' for postgres user") - test.DependencyErrorOrSkip(t) - t.Fatalf("PostGreSQL not available locally for testing: %v", err) - } -} - -func do(db *sql.DB, sql string) { - _, err := db.Exec(sql) - if err == nil { - return - } - log.Fatalf("Error %v running SQL: %s", err, sql) -} - -func doQuery(db *sql.DB, sql string) { - r, err := db.Query(sql) - if err == nil { - r.Close() - return - } - log.Fatalf("Error %v running SQL query: %s", err, sql) -} - -// closeAllSessions connects to the "postgres" DB on cfg.Host, and closes all connections to cfg.Database. -func closeAllSessions(cfg postgres.Config) error { - conninfo := fmt.Sprintf("user=%s dbname=postgres host=%s password=%s sslmode=%s", - cfg.User, cfg.Host, cfg.Password, cfg.SSLMode) - rootdb, err := sql.Open("postgres", conninfo) - if err != nil { - return fmt.Errorf("Could not open root db: %v", err) - } - defer rootdb.Close() - query := ` -SELECT - pg_terminate_backend(pg_stat_activity.pid) -FROM - pg_stat_activity -WHERE - datname = '` + cfg.Database + - `' AND pid <> pg_backend_pid()` - - // They changed procpid to pid in 9.2 - if version(rootdb) < "9.2" { - query = strings.Replace(query, ".pid", ".procpid", 1) - query = strings.Replace(query, "AND pid", "AND procpid", 1) - } - r, err := rootdb.Query(query) - if err != nil { - return fmt.Errorf("Error running SQL query\n %v\n: %s", query, err) - } - r.Close() - return nil -} - -func version(db *sql.DB) string { - version := "" - if err := db.QueryRow("SELECT version()").Scan(&version); err != nil { - log.Fatalf("Could not get PostgreSQL version: %v", err) - } - fields := strings.Fields(version) - if len(fields) < 2 { - log.Fatalf("Could not get PostgreSQL version because bogus answer: %q", version) - } - return fields[1] -} - -func newSorted(t *testing.T) (kv sorted.KeyValue, clean func()) { - skipOrFailIfNoPostgreSQL(t) - dbname := "camlitest_" + osutil.Username() - if err := closeAllSessions(postgres.Config{ - User: "postgres", - Password: "postgres", - SSLMode: "require", - Database: dbname, - Host: "localhost", - }); err != nil { - t.Fatalf("Could not close all old sessions to %q: %v", dbname, err) - } - do(rootdb, "DROP DATABASE IF EXISTS "+dbname) - do(rootdb, "CREATE DATABASE "+dbname+" LC_COLLATE = 'C' TEMPLATE = template0") - - kv, err := postgres.NewKeyValue(postgres.Config{ - Host: "localhost", - Database: dbname, - User: "postgres", - Password: "postgres", - SSLMode: "require", - }) - if err != nil { - t.Fatal(err) - } - return kv, func() { - kv.Close() - } -} - -func TestSortedKV(t *testing.T) { - kv, clean := newSorted(t) - defer clean() - kvtest.TestSorted(t, kv) -} - -type postgresTester struct{} - -func (postgresTester) test(t *testing.T, tfn func(*testing.T, func() *index.Index)) { - var mu sync.Mutex // guards cleanups - var cleanups []func() - defer func() { - mu.Lock() // never unlocked - for _, fn := range cleanups { - fn() - } - }() - makeIndex := func() *index.Index { - s, cleanup := newSorted(t) - mu.Lock() - cleanups = append(cleanups, cleanup) - mu.Unlock() - return index.MustNew(t, s) - } - tfn(t, makeIndex) -} - -func TestIndex_Postgres(t *testing.T) { - if testing.Short() { - t.Logf("skipping test in short mode") - return - } - postgresTester{}.test(t, indextest.Index) -} - -func TestPathsOfSignerTarget_Postgres(t *testing.T) { - if testing.Short() { - t.Logf("skipping test in short mode") - return - } - postgresTester{}.test(t, indextest.PathsOfSignerTarget) -} - -func TestFiles_Postgres(t *testing.T) { - if testing.Short() { - t.Logf("skipping test in short mode") - return - } - postgresTester{}.test(t, indextest.Files) -} - -func TestEdgesTo_Postgres(t *testing.T) { - if testing.Short() { - t.Logf("skipping test in short mode") - return - } - postgresTester{}.test(t, indextest.EdgesTo) -} - -func TestDelete_Postgres(t *testing.T) { - postgresTester{}.test(t, indextest.Delete) -} diff --git a/pkg/index/postgres/postgresindexer.go b/pkg/index/postgres/postgresindexer.go deleted file mode 100644 index b0df79918..000000000 --- a/pkg/index/postgres/postgresindexer.go +++ /dev/null @@ -1,60 +0,0 @@ -/* -Copyright 2012 The Camlistore Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package postgres implements the Camlistore index storage abstraction -// on top of PostgreSQL. -package postgres - -import ( - "camlistore.org/pkg/blobserver" - "camlistore.org/pkg/index" - "camlistore.org/pkg/jsonconfig" - "camlistore.org/pkg/sorted/postgres" - - _ "camlistore.org/third_party/github.com/lib/pq" -) - -func init() { - blobserver.RegisterStorageConstructor("postgresindexer", newFromConfig) -} - -func newFromConfig(ld blobserver.Loader, config jsonconfig.Obj) (blobserver.Storage, error) { - blobPrefix := config.RequiredString("blobSource") - postgresConf, err := postgres.ConfigFromJSON(config) - if err != nil { - return nil, err - } - kv, err := postgres.NewKeyValue(postgresConf) - if err != nil { - return nil, err - } - - ix, err := index.New(kv) - if err != nil { - return nil, err - } - - sto, err := ld.GetStorage(blobPrefix) - if err != nil { - ix.Close() - return nil, err - } - ix.BlobSource = sto - // Good enough, for now: - ix.KeyFetcher = ix.BlobSource - - return ix, nil -} diff --git a/pkg/index/postgres_test.go b/pkg/index/postgres_test.go new file mode 100644 index 000000000..fc544ab37 --- /dev/null +++ b/pkg/index/postgres_test.go @@ -0,0 +1,75 @@ +/* +Copyright 2012 The Camlistore Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package index_test + +import ( + "testing" + + "camlistore.org/pkg/index/indextest" + "camlistore.org/pkg/osutil" + "camlistore.org/pkg/sorted" + "camlistore.org/pkg/sorted/kvtest" + "camlistore.org/pkg/sorted/postgres" + "camlistore.org/pkg/test/dockertest" +) + +func newPostgresSorted(t *testing.T) (kv sorted.KeyValue, clean func()) { + dbname := "camlitest_" + osutil.Username() + containerID, ip := dockertest.SetupPostgreSQLContainer(t, dbname) + + kv, err := postgres.NewKeyValue(postgres.Config{ + Host: ip, + Database: dbname, + User: dockertest.PostgresUsername, + Password: dockertest.PostgresPassword, + SSLMode: "disable", + }) + if err != nil { + containerID.Kill() + t.Fatal(err) + } + return kv, func() { + kv.Close() + containerID.Kill() + } +} + +func TestSorted_Postgres(t *testing.T) { + kv, clean := newPostgresSorted(t) + defer clean() + kvtest.TestSorted(t, kv) +} + +func TestIndex_Postgres(t *testing.T) { + indexTest(t, newPostgresSorted, indextest.Index) +} + +func TestPathsOfSignerTarget_Postgres(t *testing.T) { + indexTest(t, newPostgresSorted, indextest.PathsOfSignerTarget) +} + +func TestFiles_Postgres(t *testing.T) { + indexTest(t, newPostgresSorted, indextest.Files) +} + +func TestEdgesTo_Postgres(t *testing.T) { + indexTest(t, newPostgresSorted, indextest.EdgesTo) +} + +func TestDelete_Postgres(t *testing.T) { + indexTest(t, newPostgresSorted, indextest.Delete) +} diff --git a/pkg/index/sqlite/sqlite.go b/pkg/index/sqlite/sqlite.go deleted file mode 100644 index 57403c663..000000000 --- a/pkg/index/sqlite/sqlite.go +++ /dev/null @@ -1,58 +0,0 @@ -/* -Copyright 2012 The Camlistore Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package sqlite implements the Camlistore index storage abstraction -// using an SQLite database file. -package sqlite - -import ( - "camlistore.org/pkg/blobserver" - "camlistore.org/pkg/index" - "camlistore.org/pkg/jsonconfig" - "camlistore.org/pkg/sorted/sqlite" -) - -func init() { - blobserver.RegisterStorageConstructor("sqliteindexer", newFromConfig) -} - -func newFromConfig(ld blobserver.Loader, config jsonconfig.Obj) (blobserver.Storage, error) { - blobPrefix := config.RequiredString("blobSource") - file := config.RequiredString("file") - if err := config.Validate(); err != nil { - return nil, err - } - kv, err := sqlite.NewKeyValue(file) - if err != nil { - return nil, err - } - - ix, err := index.New(kv) - if err != nil { - return nil, err - } - - sto, err := ld.GetStorage(blobPrefix) - if err != nil { - ix.Close() - return nil, err - } - ix.BlobSource = sto - // Good enough, for now: - ix.KeyFetcher = ix.BlobSource - - return ix, nil -} diff --git a/pkg/index/sqlite/sqlite_test.go b/pkg/index/sqlite/sqlite_test.go index c62d6b99f..8883a1d31 100644 --- a/pkg/index/sqlite/sqlite_test.go +++ b/pkg/index/sqlite/sqlite_test.go @@ -72,7 +72,7 @@ func newSorted(t *testing.T) (kv sorted.KeyValue, clean func()) { } } -func TestSortedKV(t *testing.T) { +func TestSorted_SQLite(t *testing.T) { kv, clean := newSorted(t) defer clean() kvtest.TestSorted(t, kv) diff --git a/pkg/serverinit/genconfig.go b/pkg/serverinit/genconfig.go index 46f84b2ca..b389f36bf 100644 --- a/pkg/serverinit/genconfig.go +++ b/pkg/serverinit/genconfig.go @@ -17,8 +17,11 @@ limitations under the License. package serverinit import ( + "encoding/json" "errors" "fmt" + "io/ioutil" + "log" "net/url" "os" "path/filepath" @@ -37,7 +40,8 @@ import ( type configPrefixesParams struct { secretRing string keyId string - indexerPath string + haveIndex bool + haveSQLite bool blobPath string packBlobs bool searchOwner blob.Ref @@ -144,15 +148,18 @@ func addMongoConfig(prefixes jsonconfig.Obj, dbname string, dbinfo string) { } ob := map[string]interface{}{} ob["enabled"] = true - ob["handler"] = "storage-mongodbindexer" + ob["handler"] = "storage-index" ob["handlerArgs"] = map[string]interface{}{ - "host": host, - "user": fields[0], - "password": fields[1], - "database": dbname, "blobSource": "/bs/", + "storage": map[string]interface{}{ + "type": "mongo", + "host": host, + "user": fields[0], + "password": fields[1], + "database": dbname, + }, } - prefixes["/index-mongo/"] = ob + prefixes["/index/"] = ob } func addSQLConfig(rdbms string, prefixes jsonconfig.Obj, dbname string, dbinfo string) { @@ -167,15 +174,18 @@ func addSQLConfig(rdbms string, prefixes jsonconfig.Obj, dbname string, dbinfo s } ob := map[string]interface{}{} ob["enabled"] = true - ob["handler"] = "storage-" + rdbms + "indexer" + ob["handler"] = "storage-index" ob["handlerArgs"] = map[string]interface{}{ - "host": fields[0], - "user": user, - "password": fields[1], - "database": dbname, "blobSource": "/bs/", + "storage": map[string]interface{}{ + "type": rdbms, + "host": fields[0], + "user": user, + "password": fields[1], + "database": dbname, + }, } - prefixes["/index-"+rdbms+"/"] = ob + prefixes["/index/"] = ob } func addPostgresConfig(prefixes jsonconfig.Obj, dbname string, dbinfo string) { @@ -188,20 +198,26 @@ func addMySQLConfig(prefixes jsonconfig.Obj, dbname string, dbinfo string) { func addSQLiteConfig(prefixes jsonconfig.Obj, file string) { ob := map[string]interface{}{} - ob["handler"] = "storage-sqliteindexer" + ob["handler"] = "storage-index" ob["handlerArgs"] = map[string]interface{}{ "blobSource": "/bs/", - "file": file, + "storage": map[string]interface{}{ + "type": "sqlite", + "file": file, + }, } - prefixes["/index-sqlite/"] = ob + prefixes["/index/"] = ob } func addKVConfig(prefixes jsonconfig.Obj, file string) { - prefixes["/index-kv/"] = map[string]interface{}{ - "handler": "storage-kvfileindexer", + prefixes["/index/"] = map[string]interface{}{ + "handler": "storage-index", "handlerArgs": map[string]interface{}{ "blobSource": "/bs/", - "file": file, + "storage": map[string]interface{}{ + "type": "kv", + "file": file, + }, }, } } @@ -384,7 +400,7 @@ func addGoogleCloudStorageConfig(params *configPrefixesParams, prefixes jsonconf func genLowLevelPrefixes(params *configPrefixesParams, ownerName string) (m jsonconfig.Obj) { m = make(jsonconfig.Obj) - haveIndex := params.indexerPath != "" + haveIndex := params.haveIndex root := "/bs/" pubKeyDest := root if haveIndex { @@ -474,7 +490,7 @@ func genLowLevelPrefixes(params *configPrefixesParams, ownerName string) (m json if haveIndex { syncArgs := map[string]interface{}{ "from": "/bs/", - "to": params.indexerPath, + "to": "/index/", } // TODO: currently when using s3, the index must be @@ -491,7 +507,7 @@ func genLowLevelPrefixes(params *configPrefixesParams, ownerName string) (m json dir = params.indexFileDir } typ := "kv" - if params.indexerPath == "/index-sqlite/" { + if params.haveSQLite { typ = "sqlite" } syncArgs["queue"] = map[string]interface{}{ @@ -507,7 +523,7 @@ func genLowLevelPrefixes(params *configPrefixesParams, ownerName string) (m json m["/bs-and-index/"] = map[string]interface{}{ "handler": "storage-replica", "handlerArgs": map[string]interface{}{ - "backends": []interface{}{"/bs/", params.indexerPath}, + "backends": []interface{}{"/bs/", "/index/"}, }, } @@ -524,7 +540,7 @@ func genLowLevelPrefixes(params *configPrefixesParams, ownerName string) (m json } searchArgs := map[string]interface{}{ - "index": params.indexerPath, + "index": "/index/", "owner": params.searchOwner.String(), } if params.memoryIndex { @@ -581,10 +597,11 @@ func genLowLevelConfig(conf *serverconfig.Config) (lowLevelConf *Config, err err conf.DBName = "camli" + username } - var indexerPath string // e.g. "/index-kv/" + var haveSQLite bool var indexFileDir string // filesystem directory of sqlite, kv, or similar numIndexers := numSet(conf.Mongo, conf.MySQL, conf.PostgreSQL, conf.SQLite, conf.KVFile) runIndex := conf.RunIndex.Get() + switch { case runIndex && numIndexers == 0: return nil, fmt.Errorf("Unless runIndex is set to false, you must specify an index option (kvIndexFile, mongo, mysql, postgres, sqlite).") @@ -592,17 +609,10 @@ func genLowLevelConfig(conf *serverconfig.Config) (lowLevelConf *Config, err err return nil, fmt.Errorf("With runIndex set true, you can only pick exactly one indexer (mongo, mysql, postgres, sqlite).") case !runIndex && numIndexers != 0: return nil, fmt.Errorf("With runIndex disabled, you can't specify any of mongo, mysql, postgres, sqlite.") - case conf.MySQL != "": - indexerPath = "/index-mysql/" - case conf.PostgreSQL != "": - indexerPath = "/index-postgres/" - case conf.Mongo != "": - indexerPath = "/index-mongo/" case conf.SQLite != "": - indexerPath = "/index-sqlite/" + haveSQLite = true indexFileDir = filepath.Dir(conf.SQLite) case conf.KVFile != "": - indexerPath = "/index-kv/" indexFileDir = filepath.Dir(conf.KVFile) } @@ -632,7 +642,8 @@ func genLowLevelConfig(conf *serverconfig.Config) (lowLevelConf *Config, err err prefixesParams := &configPrefixesParams{ secretRing: conf.IdentitySecretRing, keyId: conf.Identity, - indexerPath: indexerPath, + haveIndex: runIndex, + haveSQLite: haveSQLite, blobPath: conf.BlobPath, packBlobs: conf.PackBlobs, searchOwner: blob.SHA1FromString(armoredPublicKey), @@ -742,3 +753,59 @@ func setMap(m map[string]interface{}, v ...interface{}) { } setMap(m[v[0].(string)].(map[string]interface{}), v[1:]...) } + +// WriteDefaultConfigFile generates a new default high-level server configuration +// file at filePath. If useSQLite, the default indexer will use SQLite, otherwise +// kv. If filePath already exists, it is overwritten. +func WriteDefaultConfigFile(filePath string, useSQLite bool) error { + conf := serverconfig.Config{ + Listen: ":3179", + HTTPS: false, + Auth: "localhost", + ReplicateTo: make([]interface{}, 0), + } + blobDir := osutil.CamliBlobRoot() + if err := os.MkdirAll(blobDir, 0700); err != nil { + return fmt.Errorf("Could not create default blobs directory: %v", err) + } + conf.BlobPath = blobDir + if useSQLite { + conf.SQLite = filepath.Join(osutil.CamliVarDir(), "camli-index.db") + } else { + conf.KVFile = filepath.Join(osutil.CamliVarDir(), "camli-index.kvdb") + } + + var keyId string + secRing := osutil.SecretRingFile() + _, err := os.Stat(secRing) + switch { + case err == nil: + keyId, err = jsonsign.KeyIdFromRing(secRing) + if err != nil { + return fmt.Errorf("Could not find any keyId in file %q: %v", secRing, err) + } + log.Printf("Re-using identity with keyId %q found in file %s", keyId, secRing) + case os.IsNotExist(err): + keyId, err = jsonsign.GenerateNewSecRing(secRing) + if err != nil { + return fmt.Errorf("Could not generate new secRing at file %q: %v", secRing, err) + } + log.Printf("Generated new identity with keyId %q in file %s", keyId, secRing) + } + if err != nil { + return fmt.Errorf("Could not stat secret ring %q: %v", secRing, err) + } + conf.Identity = keyId + conf.IdentitySecretRing = secRing + + confData, err := json.MarshalIndent(conf, "", " ") + if err != nil { + return fmt.Errorf("Could not json encode config file : %v", err) + } + + if err := ioutil.WriteFile(filePath, confData, 0600); err != nil { + return fmt.Errorf("Could not create or write default server config: %v", err) + } + + return nil +} diff --git a/pkg/serverinit/testdata/baseurl-want.json b/pkg/serverinit/testdata/baseurl-want.json index fe58d7a18..1f5c0d47c 100644 --- a/pkg/serverinit/testdata/baseurl-want.json +++ b/pkg/serverinit/testdata/baseurl-want.json @@ -19,7 +19,7 @@ "handlerArgs": { "backends": [ "/bs/", - "/index-kv/" + "/index/" ] } }, @@ -46,17 +46,20 @@ "path": "/tmp/blobs/cache" } }, - "/index-kv/": { - "handler": "storage-kvfileindexer", + "/index/": { + "handler": "storage-index", "handlerArgs": { "blobSource": "/bs/", - "file": "/path/to/indexkv.db" + "storage": { + "type": "kv", + "file": "/path/to/indexkv.db" + } } }, "/my-search/": { "handler": "search", "handlerArgs": { - "index": "/index-kv/", + "index": "/index/", "owner": "sha1-f2b0b7da718b97ce8c31591d8ed4645c777f3ef4", "slurpToMemory": true } @@ -89,7 +92,7 @@ "file": "/tmp/blobs/sync-to-index-queue.kv", "type": "kv" }, - "to": "/index-kv/" + "to": "/index/" } }, "/ui/": { diff --git a/pkg/serverinit/testdata/default-want.json b/pkg/serverinit/testdata/default-want.json index 72ffcc91c..1aa30e5d9 100644 --- a/pkg/serverinit/testdata/default-want.json +++ b/pkg/serverinit/testdata/default-want.json @@ -18,7 +18,7 @@ "handlerArgs": { "backends": [ "/bs/", - "/index-kv/" + "/index/" ] } }, @@ -45,17 +45,20 @@ "path": "/tmp/blobs/cache" } }, - "/index-kv/": { - "handler": "storage-kvfileindexer", + "/index/": { + "handler": "storage-index", "handlerArgs": { "blobSource": "/bs/", - "file": "/path/to/indexkv.db" + "storage": { + "type": "kv", + "file": "/path/to/indexkv.db" + } } }, "/my-search/": { "handler": "search", "handlerArgs": { - "index": "/index-kv/", + "index": "/index/", "owner": "sha1-f2b0b7da718b97ce8c31591d8ed4645c777f3ef4", "slurpToMemory": true } @@ -88,7 +91,7 @@ "file": "/tmp/blobs/sync-to-index-queue.kv", "type": "kv" }, - "to": "/index-kv/" + "to": "/index/" } }, "/ui/": { diff --git a/pkg/serverinit/testdata/diskpacked-want.json b/pkg/serverinit/testdata/diskpacked-want.json index 8f2b2b733..058188e96 100644 --- a/pkg/serverinit/testdata/diskpacked-want.json +++ b/pkg/serverinit/testdata/diskpacked-want.json @@ -18,7 +18,7 @@ "handlerArgs": { "backends": [ "/bs/", - "/index-kv/" + "/index/" ] } }, @@ -45,17 +45,20 @@ "path": "/tmp/blobs/cache" } }, - "/index-kv/": { - "handler": "storage-kvfileindexer", + "/index/": { + "handler": "storage-index", "handlerArgs": { "blobSource": "/bs/", - "file": "/path/to/indexkv.db" + "storage": { + "type": "kv", + "file": "/path/to/indexkv.db" + } } }, "/my-search/": { "handler": "search", "handlerArgs": { - "index": "/index-kv/", + "index": "/index/", "owner": "sha1-f2b0b7da718b97ce8c31591d8ed4645c777f3ef4", "slurpToMemory": true } @@ -88,7 +91,7 @@ "file": "/tmp/blobs/sync-to-index-queue.kv", "type": "kv" }, - "to": "/index-kv/" + "to": "/index/" } }, "/ui/": { diff --git a/pkg/serverinit/testdata/flickr-want.json b/pkg/serverinit/testdata/flickr-want.json index 3f9a94335..5e5d74353 100644 --- a/pkg/serverinit/testdata/flickr-want.json +++ b/pkg/serverinit/testdata/flickr-want.json @@ -18,7 +18,7 @@ "handlerArgs": { "backends": [ "/bs/", - "/index-kv/" + "/index/" ] } }, @@ -51,17 +51,20 @@ "apiKey": "monkey:balls" } }, - "/index-kv/": { - "handler": "storage-kvfileindexer", + "/index/": { + "handler": "storage-index", "handlerArgs": { "blobSource": "/bs/", - "file": "/path/to/indexkv.db" + "storage": { + "type": "kv", + "file": "/path/to/indexkv.db" + } } }, "/my-search/": { "handler": "search", "handlerArgs": { - "index": "/index-kv/", + "index": "/index/", "owner": "sha1-f2b0b7da718b97ce8c31591d8ed4645c777f3ef4", "slurpToMemory": true } @@ -94,7 +97,7 @@ "file": "/tmp/blobs/sync-to-index-queue.kv", "type": "kv" }, - "to": "/index-kv/" + "to": "/index/" } }, "/ui/": { diff --git a/pkg/serverinit/testdata/google_nolocaldisk-want.json b/pkg/serverinit/testdata/google_nolocaldisk-want.json index e9ded04fb..7977f6ff4 100644 --- a/pkg/serverinit/testdata/google_nolocaldisk-want.json +++ b/pkg/serverinit/testdata/google_nolocaldisk-want.json @@ -17,7 +17,7 @@ "handlerArgs": { "backends": [ "/bs/", - "/index-kv/" + "/index/" ] } }, @@ -49,17 +49,20 @@ "path": "/tmp/camli-cache" } }, - "/index-kv/": { - "handler": "storage-kvfileindexer", + "/index/": { + "handler": "storage-index", "handlerArgs": { "blobSource": "/bs/", - "file": "/path/to/indexkv.db" + "storage": { + "type": "kv", + "file": "/path/to/indexkv.db" + } } }, "/my-search/": { "handler": "search", "handlerArgs": { - "index": "/index-kv/", + "index": "/index/", "owner": "sha1-f2b0b7da718b97ce8c31591d8ed4645c777f3ef4", "slurpToMemory": true } @@ -92,7 +95,7 @@ "file": "/path/to/sync-to-index-queue.kv", "type": "kv" }, - "to": "/index-kv/" + "to": "/index/" } }, "/ui/": { diff --git a/pkg/serverinit/testdata/listenbase-want.json b/pkg/serverinit/testdata/listenbase-want.json index e16a6093e..471f83708 100644 --- a/pkg/serverinit/testdata/listenbase-want.json +++ b/pkg/serverinit/testdata/listenbase-want.json @@ -18,7 +18,7 @@ "handlerArgs": { "backends": [ "/bs/", - "/index-kv/" + "/index/" ] } }, @@ -45,17 +45,20 @@ "path": "/tmp/blobs/cache" } }, - "/index-kv/": { - "handler": "storage-kvfileindexer", + "/index/": { + "handler": "storage-index", "handlerArgs": { "blobSource": "/bs/", - "file": "/path/to/indexkv.db" + "storage": { + "type": "kv", + "file": "/path/to/indexkv.db" + } } }, "/my-search/": { "handler": "search", "handlerArgs": { - "index": "/index-kv/", + "index": "/index/", "owner": "sha1-f2b0b7da718b97ce8c31591d8ed4645c777f3ef4", "slurpToMemory": true } @@ -88,7 +91,7 @@ "file": "/tmp/blobs/sync-to-index-queue.kv", "type": "kv" }, - "to": "/index-kv/" + "to": "/index/" } }, "/ui/": { diff --git a/pkg/serverinit/testdata/mem-want.json b/pkg/serverinit/testdata/mem-want.json index e32a1c6e5..cf3fe1704 100644 --- a/pkg/serverinit/testdata/mem-want.json +++ b/pkg/serverinit/testdata/mem-want.json @@ -18,7 +18,7 @@ "handlerArgs": { "backends": [ "/bs/", - "/index-kv/" + "/index/" ] } }, @@ -45,17 +45,20 @@ "path": "/tmp/blobs/cache" } }, - "/index-kv/": { - "handler": "storage-kvfileindexer", + "/index/": { + "handler": "storage-index", "handlerArgs": { "blobSource": "/bs/", - "file": "/path/to/indexkv.db" + "storage": { + "type": "kv", + "file": "/path/to/indexkv.db" + } } }, "/my-search/": { "handler": "search", "handlerArgs": { - "index": "/index-kv/", + "index": "/index/", "owner": "sha1-f2b0b7da718b97ce8c31591d8ed4645c777f3ef4", "slurpToMemory": true } @@ -151,7 +154,7 @@ "file": "/tmp/blobs/sync-to-index-queue.kv", "type": "kv" }, - "to": "/index-kv/" + "to": "/index/" } }, "/ui/": { diff --git a/pkg/serverinit/testdata/memindex-want.json b/pkg/serverinit/testdata/memindex-want.json index 72ffcc91c..1aa30e5d9 100644 --- a/pkg/serverinit/testdata/memindex-want.json +++ b/pkg/serverinit/testdata/memindex-want.json @@ -18,7 +18,7 @@ "handlerArgs": { "backends": [ "/bs/", - "/index-kv/" + "/index/" ] } }, @@ -45,17 +45,20 @@ "path": "/tmp/blobs/cache" } }, - "/index-kv/": { - "handler": "storage-kvfileindexer", + "/index/": { + "handler": "storage-index", "handlerArgs": { "blobSource": "/bs/", - "file": "/path/to/indexkv.db" + "storage": { + "type": "kv", + "file": "/path/to/indexkv.db" + } } }, "/my-search/": { "handler": "search", "handlerArgs": { - "index": "/index-kv/", + "index": "/index/", "owner": "sha1-f2b0b7da718b97ce8c31591d8ed4645c777f3ef4", "slurpToMemory": true } @@ -88,7 +91,7 @@ "file": "/tmp/blobs/sync-to-index-queue.kv", "type": "kv" }, - "to": "/index-kv/" + "to": "/index/" } }, "/ui/": { diff --git a/pkg/serverinit/testdata/mongo-want.json b/pkg/serverinit/testdata/mongo-want.json index 52fa9d2e2..425057b4d 100644 --- a/pkg/serverinit/testdata/mongo-want.json +++ b/pkg/serverinit/testdata/mongo-want.json @@ -18,7 +18,7 @@ "handlerArgs": { "backends": [ "/bs/", - "/index-mongo/" + "/index/" ] } }, @@ -45,21 +45,24 @@ "path": "/tmp/blobs/cache" } }, - "/index-mongo/": { + "/index/": { "enabled": true, - "handler": "storage-mongodbindexer", + "handler": "storage-index", "handlerArgs": { "blobSource": "/bs/", - "database": "camlitest", - "host": "localhost", - "password": "", - "user": "" + "storage": { + "type": "mongo", + "database": "camlitest", + "host": "localhost", + "password": "", + "user": "" + } } }, "/my-search/": { "handler": "search", "handlerArgs": { - "index": "/index-mongo/", + "index": "/index/", "owner": "sha1-f2b0b7da718b97ce8c31591d8ed4645c777f3ef4", "slurpToMemory": true } @@ -92,7 +95,7 @@ "file": "/tmp/blobs/sync-to-index-queue.kv", "type": "kv" }, - "to": "/index-mongo/" + "to": "/index/" } }, "/ui/": { diff --git a/pkg/serverinit/testdata/multipublish-want.json b/pkg/serverinit/testdata/multipublish-want.json index 1c7755fe0..da25f70cc 100644 --- a/pkg/serverinit/testdata/multipublish-want.json +++ b/pkg/serverinit/testdata/multipublish-want.json @@ -18,7 +18,7 @@ "handlerArgs": { "backends": [ "/bs/", - "/index-kv/" + "/index/" ] } }, @@ -45,17 +45,20 @@ "path": "/tmp/blobs/cache" } }, - "/index-kv/": { - "handler": "storage-kvfileindexer", + "/index/": { + "handler": "storage-index", "handlerArgs": { "blobSource": "/bs/", - "file": "/path/to/indexkv.db" + "storage": { + "type": "kv", + "file": "/path/to/indexkv.db" + } } }, "/my-search/": { "handler": "search", "handlerArgs": { - "index": "/index-kv/", + "index": "/index/", "owner": "sha1-f2b0b7da718b97ce8c31591d8ed4645c777f3ef4", "slurpToMemory": true } @@ -137,7 +140,7 @@ "file": "/tmp/blobs/sync-to-index-queue.kv", "type": "kv" }, - "to": "/index-kv/" + "to": "/index/" } }, "/ui/": { diff --git a/pkg/serverinit/testdata/s3_alt_host-want.json b/pkg/serverinit/testdata/s3_alt_host-want.json index 630f76a8e..677dd5665 100644 --- a/pkg/serverinit/testdata/s3_alt_host-want.json +++ b/pkg/serverinit/testdata/s3_alt_host-want.json @@ -17,7 +17,7 @@ "handlerArgs": { "backends": [ "/bs/", - "/index-kv/" + "/index/" ] } }, @@ -47,17 +47,20 @@ "path": "/tmp/camli-cache" } }, - "/index-kv/": { - "handler": "storage-kvfileindexer", + "/index/": { + "handler": "storage-index", "handlerArgs": { "blobSource": "/bs/", - "file": "/path/to/indexkv.db" + "storage": { + "type": "kv", + "file": "/path/to/indexkv.db" + } } }, "/my-search/": { "handler": "search", "handlerArgs": { - "index": "/index-kv/", + "index": "/index/", "owner": "sha1-f2b0b7da718b97ce8c31591d8ed4645c777f3ef4", "slurpToMemory": true } @@ -90,7 +93,7 @@ "file": "/path/to/sync-to-index-queue.kv", "type": "kv" }, - "to": "/index-kv/" + "to": "/index/" } }, "/ui/": { diff --git a/pkg/serverinit/testdata/s3_nolocaldisk-want.json b/pkg/serverinit/testdata/s3_nolocaldisk-want.json index d7c146acd..56e1c55a3 100644 --- a/pkg/serverinit/testdata/s3_nolocaldisk-want.json +++ b/pkg/serverinit/testdata/s3_nolocaldisk-want.json @@ -17,7 +17,7 @@ "handlerArgs": { "backends": [ "/bs/", - "/index-kv/" + "/index/" ] } }, @@ -46,17 +46,20 @@ "path": "/tmp/camli-cache" } }, - "/index-kv/": { - "handler": "storage-kvfileindexer", + "/index/": { + "handler": "storage-index", "handlerArgs": { "blobSource": "/bs/", - "file": "/path/to/indexkv.db" + "storage": { + "type": "kv", + "file": "/path/to/indexkv.db" + } } }, "/my-search/": { "handler": "search", "handlerArgs": { - "index": "/index-kv/", + "index": "/index/", "owner": "sha1-f2b0b7da718b97ce8c31591d8ed4645c777f3ef4", "slurpToMemory": true } @@ -89,7 +92,7 @@ "file": "/path/to/sync-to-index-queue.kv", "type": "kv" }, - "to": "/index-kv/" + "to": "/index/" } }, "/ui/": { diff --git a/pkg/serverinit/testdata/s3_nolocaldisk_mysql-want.json b/pkg/serverinit/testdata/s3_nolocaldisk_mysql-want.json index 8936a9468..9d677b1b8 100644 --- a/pkg/serverinit/testdata/s3_nolocaldisk_mysql-want.json +++ b/pkg/serverinit/testdata/s3_nolocaldisk_mysql-want.json @@ -17,7 +17,7 @@ "handlerArgs": { "backends": [ "/bs/", - "/index-mysql/" + "/index/" ] } }, @@ -46,21 +46,24 @@ "path": "/tmp/camli-cache" } }, - "/index-mysql/": { + "/index/": { "enabled": true, - "handler": "storage-mysqlindexer", + "handler": "storage-index", "handlerArgs": { "blobSource": "/bs/", - "database": "camlitest", - "host": "localhost", - "password": "password", - "user": "user" + "storage": { + "type": "mysql", + "database": "camlitest", + "host": "localhost", + "password": "password", + "user": "user" + } } }, "/my-search/": { "handler": "search", "handlerArgs": { - "index": "/index-mysql/", + "index": "/index/", "owner": "sha1-f2b0b7da718b97ce8c31591d8ed4645c777f3ef4", "slurpToMemory": true } @@ -90,7 +93,7 @@ "handlerArgs": { "from": "/bs/", "idle": true, - "to": "/index-mysql/" + "to": "/index/" } }, "/ui/": { diff --git a/pkg/serverinit/testdata/sqlite-want.json b/pkg/serverinit/testdata/sqlite-want.json index e727bf10b..cf3e16b6e 100644 --- a/pkg/serverinit/testdata/sqlite-want.json +++ b/pkg/serverinit/testdata/sqlite-want.json @@ -17,7 +17,7 @@ "handlerArgs": { "backends": [ "/bs/", - "/index-sqlite/" + "/index/" ] } }, @@ -44,17 +44,20 @@ "path": "/tmp/blobs/cache" } }, - "/index-sqlite/": { - "handler": "storage-sqliteindexer", + "/index/": { + "handler": "storage-index", "handlerArgs": { "blobSource": "/bs/", - "file": "/tmp/camli.db" + "storage": { + "type": "sqlite", + "file": "/tmp/camli.db" + } } }, "/my-search/": { "handler": "search", "handlerArgs": { - "index": "/index-sqlite/", + "index": "/index/", "owner": "sha1-f2b0b7da718b97ce8c31591d8ed4645c777f3ef4", "slurpToMemory": true } @@ -87,7 +90,7 @@ "file": "/tmp/blobs/sync-to-index-queue.sqlite", "type": "sqlite" }, - "to": "/index-sqlite/" + "to": "/index/" } }, "/ui/": { diff --git a/pkg/serverinit/testdata/tls-want.json b/pkg/serverinit/testdata/tls-want.json index dad44d07c..a68a189e1 100644 --- a/pkg/serverinit/testdata/tls-want.json +++ b/pkg/serverinit/testdata/tls-want.json @@ -19,7 +19,7 @@ "handlerArgs": { "backends": [ "/bs/", - "/index-kv/" + "/index/" ] } }, @@ -46,17 +46,20 @@ "path": "/tmp/blobs/cache" } }, - "/index-kv/": { - "handler": "storage-kvfileindexer", + "/index/": { + "handler": "storage-index", "handlerArgs": { "blobSource": "/bs/", - "file": "/path/to/indexkv.db" + "storage": { + "type": "kv", + "file": "/path/to/indexkv.db" + } } }, "/my-search/": { "handler": "search", "handlerArgs": { - "index": "/index-kv/", + "index": "/index/", "owner": "sha1-f2b0b7da718b97ce8c31591d8ed4645c777f3ef4", "slurpToMemory": true } @@ -89,7 +92,7 @@ "file": "/tmp/blobs/sync-to-index-queue.kv", "type": "kv" }, - "to": "/index-kv/" + "to": "/index/" } }, "/ui/": { diff --git a/pkg/serverinit/testdata/with_blog-want.json b/pkg/serverinit/testdata/with_blog-want.json index 4e2969447..77572e452 100644 --- a/pkg/serverinit/testdata/with_blog-want.json +++ b/pkg/serverinit/testdata/with_blog-want.json @@ -38,7 +38,7 @@ "handlerArgs": { "backends": [ "/bs/", - "/index-kv/" + "/index/" ] } }, @@ -65,17 +65,20 @@ "path": "/tmp/blobs/cache" } }, - "/index-kv/": { - "handler": "storage-kvfileindexer", + "/index/": { + "handler": "storage-index", "handlerArgs": { "blobSource": "/bs/", - "file": "/path/to/indexkv.db" + "storage": { + "type": "kv", + "file": "/path/to/indexkv.db" + } } }, "/my-search/": { "handler": "search", "handlerArgs": { - "index": "/index-kv/", + "index": "/index/", "owner": "sha1-f2b0b7da718b97ce8c31591d8ed4645c777f3ef4", "slurpToMemory": true } @@ -108,7 +111,7 @@ "file": "/tmp/blobs/sync-to-index-queue.kv", "type": "kv" }, - "to": "/index-kv/" + "to": "/index/" } }, "/ui/": { diff --git a/pkg/serverinit/testdata/with_gallery-want.json b/pkg/serverinit/testdata/with_gallery-want.json index 53511d591..c596bf76b 100644 --- a/pkg/serverinit/testdata/with_gallery-want.json +++ b/pkg/serverinit/testdata/with_gallery-want.json @@ -17,7 +17,7 @@ "handlerArgs": { "backends": [ "/bs/", - "/index-kv/" + "/index/" ] } }, @@ -44,17 +44,20 @@ "path": "/tmp/blobs/cache" } }, - "/index-kv/": { - "handler": "storage-kvfileindexer", + "/index/": { + "handler": "storage-index", "handlerArgs": { "blobSource": "/bs/", - "file": "/path/to/indexkv.db" + "storage": { + "type": "kv", + "file": "/path/to/indexkv.db" + } } }, "/my-search/": { "handler": "search", "handlerArgs": { - "index": "/index-kv/", + "index": "/index/", "owner": "sha1-f2b0b7da718b97ce8c31591d8ed4645c777f3ef4", "slurpToMemory": true } @@ -111,7 +114,7 @@ "file": "/tmp/blobs/sync-to-index-queue.kv", "type": "kv" }, - "to": "/index-kv/" + "to": "/index/" } }, "/ui/": { diff --git a/pkg/serverinit/testdata/with_sourceroot-want.json b/pkg/serverinit/testdata/with_sourceroot-want.json index 615bcd72c..2d6f49770 100644 --- a/pkg/serverinit/testdata/with_sourceroot-want.json +++ b/pkg/serverinit/testdata/with_sourceroot-want.json @@ -17,7 +17,7 @@ "handlerArgs": { "backends": [ "/bs/", - "/index-kv/" + "/index/" ] } }, @@ -44,17 +44,20 @@ "path": "/tmp/blobs/cache" } }, - "/index-kv/": { - "handler": "storage-kvfileindexer", + "/index/": { + "handler": "storage-index", "handlerArgs": { "blobSource": "/bs/", - "file": "/path/to/indexkv.db" + "storage": { + "type": "kv", + "file": "/path/to/indexkv.db" + } } }, "/my-search/": { "handler": "search", "handlerArgs": { - "index": "/index-kv/", + "index": "/index/", "owner": "sha1-f2b0b7da718b97ce8c31591d8ed4645c777f3ef4", "slurpToMemory": true } @@ -106,7 +109,7 @@ "file": "/tmp/blobs/sync-to-index-queue.kv", "type": "kv" }, - "to": "/index-kv/" + "to": "/index/" } }, "/ui/": { diff --git a/server/camlistored/camlistored.go b/server/camlistored/camlistored.go index 450c50f0c..ee9417928 100644 --- a/server/camlistored/camlistored.go +++ b/server/camlistored/camlistored.go @@ -22,7 +22,6 @@ import ( "crypto/rsa" "crypto/x509" "crypto/x509/pkix" - "encoding/json" "encoding/pem" "flag" "fmt" @@ -41,11 +40,9 @@ import ( "time" "camlistore.org/pkg/buildinfo" - "camlistore.org/pkg/jsonsign" "camlistore.org/pkg/misc" "camlistore.org/pkg/osutil" "camlistore.org/pkg/serverinit" - "camlistore.org/pkg/types/serverconfig" "camlistore.org/pkg/webserver" // Storage options: @@ -61,19 +58,13 @@ import ( _ "camlistore.org/pkg/blobserver/s3" _ "camlistore.org/pkg/blobserver/shard" // Indexers: (also present themselves as storage targets) - // sqlite is taken care of in option_sqlite.go - "camlistore.org/pkg/index" // base indexer + in-memory dev index - _ "camlistore.org/pkg/index/kvfile" - _ "camlistore.org/pkg/index/mongo" - _ "camlistore.org/pkg/index/mysql" - _ "camlistore.org/pkg/index/postgres" + "camlistore.org/pkg/index" // KeyValue implementations: - _ "camlistore.org/pkg/sorted" _ "camlistore.org/pkg/sorted/kvfile" _ "camlistore.org/pkg/sorted/mongo" _ "camlistore.org/pkg/sorted/mysql" _ "camlistore.org/pkg/sorted/postgres" - "camlistore.org/pkg/sorted/sqlite" + "camlistore.org/pkg/sorted/sqlite" // for sqlite.CompiledIn() // Handlers: _ "camlistore.org/pkg/search" @@ -212,7 +203,7 @@ func findConfigFile(file string) (absPath string, isNewConfig bool, err error) { return } log.Printf("Generating template config file %s", absPath) - if err = newDefaultConfigFile(absPath); err == nil { + if err = serverinit.WriteDefaultConfigFile(absPath, sqlite.CompiledIn()); err == nil { isNewConfig = true } } @@ -226,63 +217,6 @@ func findConfigFile(file string) (absPath string, isNewConfig bool, err error) { return } -var defaultListenAddr = ":3179" - -// TODO(mpl): move this func to pkg/types/serverconfig as well. - -func newDefaultConfigFile(path string) error { - conf := serverconfig.Config{ - Listen: defaultListenAddr, - HTTPS: false, - Auth: "localhost", - ReplicateTo: make([]interface{}, 0), - } - blobDir := osutil.CamliBlobRoot() - if err := os.MkdirAll(blobDir, 0700); err != nil { - return fmt.Errorf("Could not create default blobs directory: %v", err) - } - conf.BlobPath = blobDir - if sqlite.CompiledIn() { - conf.SQLite = filepath.Join(osutil.CamliVarDir(), "camli-index.db") - } else { - conf.KVFile = filepath.Join(osutil.CamliVarDir(), "camli-index.kvdb") - } - - var keyId string - secRing := osutil.SecretRingFile() - _, err := os.Stat(secRing) - switch { - case err == nil: - keyId, err = jsonsign.KeyIdFromRing(secRing) - if err != nil { - return fmt.Errorf("Could not find any keyId in file %q: %v", secRing, err) - } - log.Printf("Re-using identity with keyId %q found in file %s", keyId, secRing) - case os.IsNotExist(err): - keyId, err = jsonsign.GenerateNewSecRing(secRing) - if err != nil { - return fmt.Errorf("Could not generate new secRing at file %q: %v", secRing, err) - } - log.Printf("Generated new identity with keyId %q in file %s", keyId, secRing) - } - if err != nil { - return fmt.Errorf("Could not stat secret ring %q: %v", secRing, err) - } - conf.Identity = keyId - conf.IdentitySecretRing = secRing - - confData, err := json.MarshalIndent(conf, "", " ") - if err != nil { - return fmt.Errorf("Could not json encode config file : %v", err) - } - - if err := ioutil.WriteFile(path, confData, 0600); err != nil { - return fmt.Errorf("Could not create or write default server config: %v", err) - } - - return nil -} - func setupTLS(ws *webserver.Server, config *serverinit.Config, listen string) { cert, key := config.OptionalString("httpsCert", ""), config.OptionalString("httpsKey", "") if !config.OptionalBool("https", true) { diff --git a/server/camlistored/option_sqlite.go b/server/camlistored/option_sqlite.go deleted file mode 100644 index 23ad78ebb..000000000 --- a/server/camlistored/option_sqlite.go +++ /dev/null @@ -1,16 +0,0 @@ -// +build with_sqlite - -// If the "with_sqlite" build tag is specified, the sqlite index driver -// is also built & loaded: -// -// go install -tags=with_sqlite camlistore.org/server/camlistored -// -// This is an option because the sqlite3 SQL driver requires cgo & the -// SQLite3 C library available. We want it to still be possible to -// have a pure Go server too. - -package main - -import ( - _ "camlistore.org/pkg/index/sqlite" -) diff --git a/server/camlistored/run_test.go b/server/camlistored/run_test.go index 96b85e079..f83c09712 100644 --- a/server/camlistored/run_test.go +++ b/server/camlistored/run_test.go @@ -57,7 +57,7 @@ func TestStarts(t *testing.T) { mkdir(t, confDir) *flagOpenBrowser = false - defaultListenAddr = ":0" + *listenFlag = ":0" up := make(chan struct{}) down := make(chan struct{})