Rooba
/
perkeep
mirror of https://github.com/perkeep/perkeep.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

pkg/index: cleanup, refactor sub pkgs 

http://camlistore.org/issue/263

Change-Id: I319bb097f0ce30b2bd5271b5c3bbff92b8dcc318
This commit is contained in:
mpl 2014-04-01 18:03:20 +02:00
parent 929411281d
commit 291320d451
38 changed files with 879 additions and 1348 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

557
config/dev-server-config.json
View File

@ -1,304 +1,319 @@
{ "_for-emacs": "-*- mode: js2;-*-",
"handlerConfig": true,
"baseURL": ["_env", "${CAMLI_BASEURL}"],
"auth": ["_env", "${CAMLI_AUTH}"],
"https": ["_env", "${CAMLI_TLS}", false],
"httpsCert": "config/selfgen_pem.crt",
"httpsKey": "config/selfgen_pem.key",
"prefixes": {
"/": {
"handler": "root",
"handlerArgs": {
"ownerName": ["_env", "${USER}-dev"],
"blobRoot": "/bs-recv/",
"statusRoot": "/status/",
"searchRoot": "/my-search/",
"stealth": false
}
},
"handlerConfig": true,
"baseURL": ["_env", "${CAMLI_BASEURL}"],
"auth": ["_env", "${CAMLI_AUTH}"],
"https": ["_env", "${CAMLI_TLS}", false],
"httpsCert": "config/selfgen_pem.crt",
"httpsKey": "config/selfgen_pem.key",
"prefixes": {
"/": {
"handler": "root",
"handlerArgs": {
"ownerName": ["_env", "${USER}-dev"],
"blobRoot": "/bs-recv/",
"statusRoot": "/status/",
"searchRoot": "/my-search/",
"stealth": false
}
},
"/blog/": {
"enabled": ["_env", "${CAMLI_PUBLISH_ENABLED}"],
"handler": "publish",
"handlerArgs": {
"rootName": "dev-blog-root",
"blobRoot": "/bs/",
"searchRoot": "/my-search/",
"cache": "/cache/",
"goTemplate": "blog.html",
"devBootstrapPermanodeUsing": "/sighelper/"
}
},
"/blog/": {
"enabled": ["_env", "${CAMLI_PUBLISH_ENABLED}"],
"handler": "publish",
"handlerArgs": {
"rootName": "dev-blog-root",
"blobRoot": "/bs/",
"searchRoot": "/my-search/",
"cache": "/cache/",
"goTemplate": "blog.html",
"devBootstrapPermanodeUsing": "/sighelper/"
}
},
"/pics/": {
"enabled": ["_env", "${CAMLI_PUBLISH_ENABLED}"],
"handler": "publish",
"handlerArgs": {
"rootName": "dev-pics-root",
"blobRoot": "/bs/",
"searchRoot": "/my-search/",
"cache": "/cache/",
"css": ["pics.css"],
"js": ["pics.js"],
"goTemplate": "gallery.html",
"devBootstrapPermanodeUsing": "/sighelper/"
}
},
"/pics/": {
"enabled": ["_env", "${CAMLI_PUBLISH_ENABLED}"],
"handler": "publish",
"handlerArgs": {
"rootName": "dev-pics-root",
"blobRoot": "/bs/",
"searchRoot": "/my-search/",
"cache": "/cache/",
"css": ["pics.css"],
"js": ["pics.js"],
"goTemplate": "gallery.html",
"devBootstrapPermanodeUsing": "/sighelper/"
}
},
"/stub-test-disable/": {
"handler": "publish",
"enabled": false,
"handlerArgs": {
}
},
"/stub-test-disable/": {
"handler": "publish",
"enabled": false,
"handlerArgs": {
}
},
"/ui/": {
"handler": "ui",
"handlerArgs": {
"sourceRoot": ["_env", "${CAMLI_DEV_CAMLI_ROOT}", ""],
"jsonSignRoot": "/sighelper/",
"cache": "/cache/",
"scaledImage": {
"type": "kv",
"file": ["_env", "${CAMLI_ROOT_CACHE}/thumbnails.kv", ""]
},
"publishRoots": ["/blog/", "/pics/"]
}
},
"/ui/": {
"handler": "ui",
"handlerArgs": {
"sourceRoot": ["_env", "${CAMLI_DEV_CAMLI_ROOT}", ""],
"jsonSignRoot": "/sighelper/",
"cache": "/cache/",
"scaledImage": {
"type": "kv",
"file": ["_env", "${CAMLI_ROOT_CACHE}/thumbnails.kv", ""]
},
"publishRoots": ["/blog/", "/pics/"]
}
},
"/status/": {
"handler": "status"
},
"/status/": {
"handler": "status"
},
"/sync-index/": {
"handler": "sync",
"handlerArgs": {
"from": "/bs/",
"to": ["_env", "${CAMLI_INDEXER_PATH}"],
"queue": { "type": "memory" },
"fullSyncOnStart": ["_env", "${CAMLI_FULL_INDEX_SYNC_ON_START}"],
"blockingFullSyncOnStart": ["_env", "${CAMLI_FULL_INDEX_SYNC_ON_START}"]
}
},
"/sync-index/": {
"handler": "sync",
"handlerArgs": {
"from": "/bs/",
"to": ["_env", "${CAMLI_INDEXER_PATH}"],
"queue": { "type": "memory" },
"fullSyncOnStart": ["_env", "${CAMLI_FULL_INDEX_SYNC_ON_START}"],
"blockingFullSyncOnStart": ["_env", "${CAMLI_FULL_INDEX_SYNC_ON_START}"]
}
},
"/sync-r1/": {
"handler": "sync",
"handlerArgs": {
"from": "/bs/",
"to": "/r1/",
"queue": { "type": "memory" }
}
},
"/sync-r1/": {
"handler": "sync",
"handlerArgs": {
"from": "/bs/",
"to": "/r1/",
"queue": { "type": "memory" }
}
},
"/sighelper/": {
"handler": "jsonsign",
"handlerArgs": {
"secretRing": ["_env", "${CAMLI_SECRET_RING}"],
"keyId": ["_env", "${CAMLI_KEYID}"],
"publicKeyDest": "/bs/"
}
},
"/sighelper/": {
"handler": "jsonsign",
"handlerArgs": {
"secretRing": ["_env", "${CAMLI_SECRET_RING}"],
"keyId": ["_env", "${CAMLI_KEYID}"],
"publicKeyDest": "/bs/"
}
},
"/bs-recv/": {
"handler": "storage-replica",
"handlerArgs": {
"minWritesForSuccess": 2,
"backends": ["/bs/", ["_env", "${CAMLI_INDEXER_PATH}"]],
"readBackends": ["/bs/"]
}
},
"/bs-recv/": {
"handler": "storage-replica",
"handlerArgs": {
"minWritesForSuccess": 2,
"backends": ["/bs/", ["_env", "${CAMLI_INDEXER_PATH}"]],
"readBackends": ["/bs/"]
}
},
"/cond-unused/": {
"handler": "storage-cond",
"handlerArgs": {
"write": {
"if": "isSchema",
"then": "/bs-recv/",
"else": "/bs/"
},
"read": "/bs/"
}
},
"/cond-unused/": {
"handler": "storage-cond",
"handlerArgs": {
"write": {
"if": "isSchema",
"then": "/bs-recv/",
"else": "/bs/"
},
"read": "/bs/"
}
},
"/bs/": {
"handler": "storage-filesystem",
"handlerArgs": {
"path": ["_env", "${CAMLI_ROOT}"]
}
},
"/bs/": {
"handler": "storage-filesystem",
"handlerArgs": {
"path": ["_env", "${CAMLI_ROOT}"]
}
},
"/cache/": {
"handler": "storage-filesystem",
"handlerArgs": {
"path": ["_env", "${CAMLI_ROOT_CACHE}"]
}
},
"/cache/": {
"handler": "storage-filesystem",
"handlerArgs": {
"path": ["_env", "${CAMLI_ROOT_CACHE}"]
}
},
"/sharder/": {
"handler": "storage-shard",
"handlerArgs": {
"backends": ["/s1/", "/s2/"]
}
},
"/sharder/": {
"handler": "storage-shard",
"handlerArgs": {
"backends": ["/s1/", "/s2/"]
}
},
"/s1/": {
"handler": "storage-filesystem",
"handlerArgs": {
"path": ["_env", "${CAMLI_ROOT_SHARD1}"]
}
},
"/s1/": {
"handler": "storage-filesystem",
"handlerArgs": {
"path": ["_env", "${CAMLI_ROOT_SHARD1}"]
}
},
"/s2/": {
"handler": "storage-filesystem",
"handlerArgs": {
"path": ["_env", "${CAMLI_ROOT_SHARD2}"]
}
},
"/s2/": {
"handler": "storage-filesystem",
"handlerArgs": {
"path": ["_env", "${CAMLI_ROOT_SHARD2}"]
}
},
"/repl/": {
"handler": "storage-replica",
"handlerArgs": {
"backends": ["/r1/", "/r2/", "/r3/"],
"minWritesForSuccess": 2
}
},
"/repl/": {
"handler": "storage-replica",
"handlerArgs": {
"backends": ["/r1/", "/r2/", "/r3/"],
"minWritesForSuccess": 2
}
},
"/r1/": {
"handler": "storage-diskpacked",
"handlerArgs": {
"path": ["_env", "${CAMLI_ROOT_REPLICA1}"]
}
},
"/r1/": {
"handler": "storage-diskpacked",
"handlerArgs": {
"path": ["_env", "${CAMLI_ROOT_REPLICA1}"]
}
},
"/r2/": {
"handler": "storage-filesystem",
"handlerArgs": {
"path": ["_env", "${CAMLI_ROOT_REPLICA2}"]
}
},
"/r2/": {
"handler": "storage-filesystem",
"handlerArgs": {
"path": ["_env", "${CAMLI_ROOT_REPLICA2}"]
}
},
"/r3/": {
"handler": "storage-filesystem",
"handlerArgs": {
"path": ["_env", "${CAMLI_ROOT_REPLICA3}"]
}
},
"/r3/": {
"handler": "storage-filesystem",
"handlerArgs": {
"path": ["_env", "${CAMLI_ROOT_REPLICA3}"]
}
},
"/enc/": {
"handler": "storage-encrypt",
"handlerArgs": {
"I_AGREE": "that encryption support hasn't been peer-reviewed, isn't finished, and its format might change.",
"meta": "/encmeta/",
"blobs": "/encblob/",
"metaIndex": { "type": "memory" },
"key": "000102030405060708090a0b0c0d0e0f"
}
},
"/enc/": {
"handler": "storage-encrypt",
"handlerArgs": {
"I_AGREE": "that encryption support hasn't been peer-reviewed, isn't finished, and its format might change.",
"meta": "/encmeta/",
"blobs": "/encblob/",
"metaIndex": { "type": "memory" },
"key": "000102030405060708090a0b0c0d0e0f"
}
},
"/encmeta/": {
"handler": "storage-filesystem",
"handlerArgs": {
"path": ["_env", "${CAMLI_ROOT_ENCMETA}"]
}
},
"/encmeta/": {
"handler": "storage-filesystem",
"handlerArgs": {
"path": ["_env", "${CAMLI_ROOT_ENCMETA}"]
}
},
"/encblob/": {
"handler": "storage-filesystem",
"handlerArgs": {
"path": ["_env", "${CAMLI_ROOT_ENCBLOB}"]
}
},
"/encblob/": {
"handler": "storage-filesystem",
"handlerArgs": {
"path": ["_env", "${CAMLI_ROOT_ENCBLOB}"]
}
},
"/index-kv/": {
"enabled": ["_env", "${CAMLI_KVINDEX_ENABLED}"],
"handler": "storage-kvfileindexer",
"handlerArgs": {
"blobSource": "/bs/",
"file": ["_env", "${CAMLI_DBNAME}", ""]
}
},
"/index-kv/": {
"enabled": ["_env", "${CAMLI_KVINDEX_ENABLED}"],
"handler": "storage-index",
"handlerArgs": {
"blobSource": "/bs/",
"storage": {
"type": "kv",
"file": ["_env", "${CAMLI_DBNAME}", ""]
}
}
},
"/index-mongo/": {
"enabled": ["_env", "${CAMLI_MONGO_ENABLED}", true],
"handler": "storage-mongodbindexer",
"handlerArgs": {
"host": "localhost",
"database": ["_env", "${CAMLI_DBNAME}"],
"blobSource": "/bs/"
}
},
"/index-mongo/": {
"enabled": ["_env", "${CAMLI_MONGO_ENABLED}", true],
"handler": "storage-index",
"handlerArgs": {
"blobSource": "/bs/",
"storage": {
"type": "mongo",
"host": "localhost",
"database": ["_env", "${CAMLI_DBNAME}"]
}
}
},
"/index-mysql/": {
"enabled": ["_env", "${CAMLI_MYSQL_ENABLED}", true],
"handler": "storage-mysqlindexer",
"handlerArgs": {
"database": ["_env", "${CAMLI_DBNAME}"],
"user": "root",
"password": "root",
"host": "127.0.0.1",
"blobSource": "/bs/"
}
},
"/index-mysql/": {
"enabled": ["_env", "${CAMLI_MYSQL_ENABLED}", true],
"handler": "storage-index",
"handlerArgs": {
"blobSource": "/bs/",
"storage": {
"type": "mysql",
"database": ["_env", "${CAMLI_DBNAME}"],
"user": "root",
"password": "root",
"host": "127.0.0.1"
}
}
},
"/index-postgres/": {
"enabled": ["_env", "${CAMLI_POSTGRES_ENABLED}", true],
"handler": "storage-postgresindexer",
"handlerArgs": {
"database": ["_env", "${CAMLI_DBNAME}"],
"user": "postgres",
"password": "postgres",
"host": "127.0.0.1",
"blobSource": "/bs/"
}
},
"/index-postgres/": {
"enabled": ["_env", "${CAMLI_POSTGRES_ENABLED}", true],
"handler": "storage-index",
"handlerArgs": {
"blobSource": "/bs/",
"storage": {
"type": "postgres",
"database": ["_env", "${CAMLI_DBNAME}"],
"user": "postgres",
"password": "postgres",
"host": "127.0.0.1"
}
}
},
"/index-sqlite/": {
"enabled": ["_env", "${CAMLI_SQLITE_ENABLED}", true],
"handler": "storage-sqliteindexer",
"handlerArgs": {
"file": ["_env", "${CAMLI_DBNAME}"],
"blobSource": "/bs/"
}
},
"/index-sqlite/": {
"enabled": ["_env", "${CAMLI_SQLITE_ENABLED}", true],
"handler": "storage-index",
"handlerArgs": {
"blobSource": "/bs/",
"storage": {
"type": "sqlite",
"file": ["_env", "${CAMLI_DBNAME}"]
}
}
},
"/my-search/": {
"handler": "search",
"handlerArgs": {
"index": ["_env", "${CAMLI_INDEXER_PATH}"],
"owner": ["_env", "${CAMLI_PUBKEY_BLOBREF}"],
"slurpToMemory": true,
"devBlockStartupOn": "/sync-index/"
}
},
"/my-search/": {
"handler": "search",
"handlerArgs": {
"index": ["_env", "${CAMLI_INDEXER_PATH}"],
"owner": ["_env", "${CAMLI_PUBKEY_BLOBREF}"],
"slurpToMemory": true,
"devBlockStartupOn": "/sync-index/"
}
},
"/importer/": {
"handler": "importer",
"handlerArgs": {
"dummy": {
"clientID": "dummyID",
"clientSecret": "foobar"
},
"flickr": {
"clientSecret": ["_env", "${CAMLI_FLICKR_API_KEY}", ""]
},
"foursquare": {
"clientSecret": ["_env", "${CAMLI_FOURSQUARE_API_KEY}", ""]
},
"picasa": {
"clientSecret": ["_env", "${CAMLI_PICASA_API_KEY}", ""]
},
"twitter": {
"clientSecret": ["_env", "${CAMLI_TWITTER_API_KEY}", ""]
}
}
},
"/importer/": {
"handler": "importer",
"handlerArgs": {
"dummy": {
"clientID": "dummyID",
"clientSecret": "foobar"
},
"flickr": {
"clientSecret": ["_env", "${CAMLI_FLICKR_API_KEY}", ""]
},
"foursquare": {
"clientSecret": ["_env", "${CAMLI_FOURSQUARE_API_KEY}", ""]
},
"picasa": {
"clientSecret": ["_env", "${CAMLI_PICASA_API_KEY}", ""]
},
"twitter": {
"clientSecret": ["_env", "${CAMLI_TWITTER_API_KEY}", ""]
}
}
},
"/share/": {
"handler": "share",
"handlerArgs": {
"blobRoot": "/bs/"
}
}
}
"/share/": {
"handler": "share",
"handlerArgs": {
"blobRoot": "/bs/"
}
}
}
}

34
pkg/index/index.go
View File

@ -32,6 +32,7 @@ import (
"camlistore.org/pkg/blob"
"camlistore.org/pkg/blobserver"
"camlistore.org/pkg/context"
"camlistore.org/pkg/jsonconfig"
"camlistore.org/pkg/schema"
"camlistore.org/pkg/sorted"
"camlistore.org/pkg/strutil"
@ -39,6 +40,10 @@ import (
"camlistore.org/pkg/types/camtypes"
)
func init() {
blobserver.RegisterStorageConstructor("index", newFromConfig)
}
type Index struct {
*blobserver.NoImplStorage
@ -141,6 +146,35 @@ func New(s sorted.KeyValue) (*Index, error) {
return idx, nil
}
func newFromConfig(ld blobserver.Loader, config jsonconfig.Obj) (blobserver.Storage, error) {
blobPrefix := config.RequiredString("blobSource")
kvConfig := config.RequiredObject("storage")
if err := config.Validate(); err != nil {
return nil, err
}
kv, err := sorted.NewKeyValue(kvConfig)
if err != nil {
return nil, err
}
ix, err := New(kv)
if err != nil {
return nil, err
}
sto, err := ld.GetStorage(blobPrefix)
if err != nil {
ix.Close()
return nil, err
}
ix.BlobSource = sto
// Good enough, for now:
ix.KeyFetcher = ix.BlobSource
return ix, err
}
func (x *Index) String() string {
return fmt.Sprintf("Camlistore index, using key/value implementation %T", x.s)
}

80
pkg/index/index_test.go
View File

@ -69,9 +69,15 @@ func TestDelete_Memory(t *testing.T) {
}
var (
// those dirs are not packages implementing indexers,
// those test files are not specific to an indexer implementation
// hence we do not want to check them.
excludedDirs = []string{"indextest", "testdata", "sqlindex"}
notAnIndexer = []string{
"corpus_bench_test.go",
"corpus_test.go",
"export_test.go",
"index_test.go",
"keys_test.go",
}
// A map is used in hasAllRequiredTests to note which required
// tests have been found in a package, by setting the corresponding
// booleans to true. Those are the keys for this map.
@ -81,54 +87,43 @@ var (
// This function checks that all the functions using the tests
// defined in indextest, namely:
// TestIndex_, TestPathOfSignerTarget_, TestFiles_
// do exist in the provided package.
func hasAllRequiredTests(path string, t *testing.T) error {
// do exist in the provided test file.
func hasAllRequiredTests(name string, t *testing.T) error {
tests := make(map[string]bool)
for _, v := range requiredTests {
tests[v] = false
}
dir, err := os.Open(path)
if err != nil {
t.Fatal(err)
}
names, err := dir.Readdirnames(-1)
if err != nil {
t.Fatal(err)
}
defer dir.Close()
for _, name := range names {
if strings.HasPrefix(name, ".") || !strings.HasSuffix(name, "_test.go") {
continue
}
fset := token.NewFileSet()
f, err := parser.ParseFile(fset, filepath.Join(path, name), nil, 0)
if err != nil {
t.Fatalf("%v: %v", filepath.Join(path, name), err)
}
ast.Inspect(f, func(n ast.Node) bool {
switch x := n.(type) {
case *ast.FuncDecl:
name := x.Name.Name
for k, _ := range tests {
if strings.HasPrefix(name, k) {
tests[k] = true
}
if !strings.HasSuffix(name, "_test.go") || skipFromList(name) {
return nil
}
fset := token.NewFileSet()
f, err := parser.ParseFile(fset, name, nil, 0)
if err != nil {
t.Fatalf("%v: %v", name, err)
}
ast.Inspect(f, func(n ast.Node) bool {
switch x := n.(type) {
case *ast.FuncDecl:
name := x.Name.Name
for k, _ := range tests {
if strings.HasPrefix(name, k) {
tests[k] = true
}
}
return true
})
}
}
return true
})
for k, v := range tests {
if !v {
return fmt.Errorf("%v not implemented in %v", k, path)
return fmt.Errorf("%v not implemented in %v", k, name)
}
}
return nil
}
// For each package implementing an indexer, this checks that
// For each test file dedicated to an indexer implementation, this checks that
// all the required tests are present in its test suite.
func TestIndexerTestsCompleteness(t *testing.T) {
cwd, err := os.Open(".")
@ -143,21 +138,22 @@ func TestIndexerTestsCompleteness(t *testing.T) {
for _, file := range files {
name := file.Name()
if !file.IsDir() || skipDir(name) {
if file.IsDir() || strings.HasPrefix(name, ".") {
continue
}
if err := hasAllRequiredTests(name, t); err != nil {
t.Error(err)
}
}
// special case for sqlite as it is the only one left in its own package
if err := hasAllRequiredTests(filepath.FromSlash("sqlite/sqlite_test.go"), t); err != nil {
t.Error(err)
}
}
func skipDir(name string) bool {
if strings.HasPrefix(name, "_") {
return true
}
for _, v := range excludedDirs {
if v == name {
func skipFromList(name string) bool {
for _, v := range notAnIndexer {
if name == v {
return true
}
}

57
pkg/index/kvfile/kvfile.go
View File

@ -1,57 +0,0 @@
/*
Copyright 2013 The Camlistore Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package kvfile implements the Camlistore index storage abstraction
// on top of a single mutable database file on disk using
// github.com/cznic/kv.
package kvfile
import (
"camlistore.org/pkg/blobserver"
"camlistore.org/pkg/index"
"camlistore.org/pkg/jsonconfig"
"camlistore.org/pkg/sorted/kvfile"
)
func init() {
blobserver.RegisterStorageConstructor("kvfileindexer",
blobserver.StorageConstructor(newFromConfig))
}
func newFromConfig(ld blobserver.Loader, config jsonconfig.Obj) (blobserver.Storage, error) {
blobPrefix := config.RequiredString("blobSource")
kv, err := kvfile.NewKeyValue(config)
if err != nil {
return nil, err
}
ix, err := index.New(kv)
if err != nil {
return nil, err
}
sto, err := ld.GetStorage(blobPrefix)
if err != nil {
ix.Close()
return nil, err
}
ix.BlobSource = sto
// Good enough, for now:
ix.KeyFetcher = ix.BlobSource
return ix, err
}

49
pkg/index/kvfile/kvfile_test.go → pkg/index/kvfile_test.go
View File

@ -14,12 +14,13 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
package kvfile_test
package index_test
import (
"io/ioutil"
"os"
"path/filepath"
"sync"
"testing"
"camlistore.org/pkg/index"
@ -30,7 +31,7 @@ import (
"camlistore.org/pkg/test"
)
func newSorted(t *testing.T) (kv sorted.KeyValue, cleanup func()) {
func newKvfileSorted(t *testing.T) (kv sorted.KeyValue, cleanup func()) {
td, err := ioutil.TempDir("", "kvfile-test")
if err != nil {
t.Fatal(err)
@ -46,48 +47,50 @@ func newSorted(t *testing.T) (kv sorted.KeyValue, cleanup func()) {
}
}
func TestSortedKV(t *testing.T) {
kv, cleanup := newSorted(t)
func TestSorted_Kvfile(t *testing.T) {
kv, cleanup := newKvfileSorted(t)
defer cleanup()
kvtest.TestSorted(t, kv)
}
type tester struct{}
func (tester) test(t *testing.T, tfn func(*testing.T, func() *index.Index)) {
func indexTest(t *testing.T,
sortedGenfn func(t *testing.T) (sorted.KeyValue, func()),
tfn func(*testing.T, func() *index.Index)) {
defer test.TLog(t)()
var mu sync.Mutex // guards cleanups
var cleanups []func()
defer func() {
mu.Lock() // never unlocked
for _, fn := range cleanups {
fn()
}
}()
initIndex := func() *index.Index {
kv, cleanup := newSorted(t)
makeIndex := func() *index.Index {
s, cleanup := sortedGenfn(t)
mu.Lock()
cleanups = append(cleanups, cleanup)
return index.MustNew(t, kv)
mu.Unlock()
return index.MustNew(t, s)
}
tfn(t, initIndex)
tfn(t, makeIndex)
}
func TestIndex_KV(t *testing.T) {
tester{}.test(t, indextest.Index)
func TestIndex_Kvfile(t *testing.T) {
indexTest(t, newKvfileSorted, indextest.Index)
}
func TestPathsOfSignerTarget_KV(t *testing.T) {
tester{}.test(t, indextest.PathsOfSignerTarget)
func TestPathsOfSignerTarget_Kvfile(t *testing.T) {
indexTest(t, newKvfileSorted, indextest.PathsOfSignerTarget)
}
func TestFiles_KV(t *testing.T) {
tester{}.test(t, indextest.Files)
func TestFiles_Kvfile(t *testing.T) {
indexTest(t, newKvfileSorted, indextest.Files)
}
func TestEdgesTo_KV(t *testing.T) {
tester{}.test(t, indextest.EdgesTo)
func TestEdgesTo_Kvfile(t *testing.T) {
indexTest(t, newKvfileSorted, indextest.EdgesTo)
}
func TestDelete_KV(t *testing.T) {
tester{}.test(t, indextest.Delete)
func TestDelete_Kvfile(t *testing.T) {
indexTest(t, newKvfileSorted, indextest.Delete)
}

58
pkg/index/mongo/mongoindex.go
View File

@ -1,58 +0,0 @@
/*
Copyright 2011 The Camlistore Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package mongo implements the Camlistore index storage abstraction
// on top of MongoDB.
package mongo
import (
"camlistore.org/pkg/blobserver"
"camlistore.org/pkg/index"
"camlistore.org/pkg/jsonconfig"
"camlistore.org/pkg/sorted/mongo"
)
func init() {
blobserver.RegisterStorageConstructor("mongodbindexer", newFromConfig)
}
func newFromConfig(ld blobserver.Loader, config jsonconfig.Obj) (blobserver.Storage, error) {
blobPrefix := config.RequiredString("blobSource")
mongoConf, err := mongo.ConfigFromJSON(config)
if err != nil {
return nil, err
}
kv, err := mongo.NewKeyValue(mongoConf)
if err != nil {
return nil, err
}
ix, err := index.New(kv)
if err != nil {
return nil, err
}
sto, err := ld.GetStorage(blobPrefix)
if err != nil {
return nil, err
}
ix.BlobSource = sto
// Good enough, for now:
ix.KeyFetcher = ix.BlobSource
return ix, err
}

123
pkg/index/mongo/mongoindex_test.go
View File

@ -1,123 +0,0 @@
/*
Copyright 2011 Google Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package mongo_test
import (
"errors"
"sync"
"testing"
"time"
"camlistore.org/pkg/index"
"camlistore.org/pkg/index/indextest"
"camlistore.org/pkg/sorted"
"camlistore.org/pkg/sorted/kvtest"
"camlistore.org/pkg/sorted/mongo"
"camlistore.org/pkg/test"
)
var (
once sync.Once
mongoNotAvailable bool
)
func checkMongoUp() {
mongoNotAvailable = !mongo.Ping("localhost", 500*time.Millisecond)
}
func skipOrFailIfNoMongo(t *testing.T) {
once.Do(checkMongoUp)
if mongoNotAvailable {
err := errors.New("Not running; start a mongoDB daemon on the standard port (27017). The \"keys\" collection in the \"camlitest\" database will be used.")
test.DependencyErrorOrSkip(t)
t.Fatalf("Mongo not available locally for testing: %v", err)
}
}
func newSorted(t *testing.T) (kv sorted.KeyValue, cleanup func()) {
skipOrFailIfNoMongo(t)
// connect without credentials and wipe the database
cfg := mongo.Config{
Server: "localhost",
Database: "camlitest",
}
var err error
kv, err = mongo.NewKeyValue(cfg)
if err != nil {
t.Fatal(err)
}
wiper, ok := kv.(sorted.Wiper)
if !ok {
panic("mongo KeyValue not a Wiper")
}
err = wiper.Wipe()
if err != nil {
t.Fatal(err)
}
return kv, func() {
kv.Close()
}
}
func TestSortedKV(t *testing.T) {
kv, cleanup := newSorted(t)
defer cleanup()
kvtest.TestSorted(t, kv)
}
type mongoTester struct{}
func (mongoTester) test(t *testing.T, tfn func(*testing.T, func() *index.Index)) {
defer test.TLog(t)()
var mu sync.Mutex // guards cleanups
var cleanups []func()
defer func() {
mu.Lock() // never unlocked
for _, fn := range cleanups {
fn()
}
}()
initIndex := func() *index.Index {
kv, cleanup := newSorted(t)
mu.Lock()
cleanups = append(cleanups, cleanup)
mu.Unlock()
return index.MustNew(t, kv)
}
tfn(t, initIndex)
}
func TestIndex_Mongo(t *testing.T) {
mongoTester{}.test(t, indextest.Index)
}
func TestPathsOfSignerTarget_Mongo(t *testing.T) {
mongoTester{}.test(t, indextest.PathsOfSignerTarget)
}
func TestFiles_Mongo(t *testing.T) {
mongoTester{}.test(t, indextest.Files)
}
func TestEdgesTo_Mongo(t *testing.T) {
mongoTester{}.test(t, indextest.EdgesTo)
}
func TestDelete_Mongo(t *testing.T) {
mongoTester{}.test(t, indextest.Delete)
}

72
pkg/index/mongo_test.go Normal file
View File

@ -0,0 +1,72 @@
/*
Copyright 2011 Google Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package index_test
import (
"testing"
"camlistore.org/pkg/index/indextest"
"camlistore.org/pkg/osutil"
"camlistore.org/pkg/sorted"
"camlistore.org/pkg/sorted/kvtest"
"camlistore.org/pkg/sorted/mongo"
"camlistore.org/pkg/test/dockertest"
)
func newMongoSorted(t *testing.T) (kv sorted.KeyValue, cleanup func()) {
dbname := "camlitest_" + osutil.Username()
containerID, ip := dockertest.SetupMongoContainer(t)
kv, err := mongo.NewKeyValue(mongo.Config{
Server: ip,
Database: dbname,
})
if err != nil {
containerID.Kill()
t.Fatal(err)
}
return kv, func() {
kv.Close()
containerID.Kill()
}
}
func TestSorted_Mongo(t *testing.T) {
kv, cleanup := newMongoSorted(t)
defer cleanup()
kvtest.TestSorted(t, kv)
}
func TestIndex_Mongo(t *testing.T) {
indexTest(t, newMongoSorted, indextest.Index)
}
func TestPathsOfSignerTarget_Mongo(t *testing.T) {
indexTest(t, newMongoSorted, indextest.PathsOfSignerTarget)
}
func TestFiles_Mongo(t *testing.T) {
indexTest(t, newMongoSorted, indextest.Files)
}
func TestEdgesTo_Mongo(t *testing.T) {
indexTest(t, newMongoSorted, indextest.EdgesTo)
}
func TestDelete_Mongo(t *testing.T) {
indexTest(t, newMongoSorted, indextest.Delete)
}

142
pkg/index/mysql/mysql_test.go
View File

@ -1,142 +0,0 @@
/*
Copyright 2012 The Camlistore Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package mysql_test
import (
"database/sql"
"errors"
"log"
"sync"
"testing"
"camlistore.org/pkg/index"
"camlistore.org/pkg/index/indextest"
"camlistore.org/pkg/osutil"
"camlistore.org/pkg/sorted"
"camlistore.org/pkg/sorted/kvtest"
"camlistore.org/pkg/sorted/mysql"
"camlistore.org/pkg/test"
_ "camlistore.org/third_party/github.com/ziutek/mymysql/godrv"
)
var (
once sync.Once
dbAvailable bool
rootdb *sql.DB
)
func checkDB() {
var err error
rootdb, err = sql.Open("mymysql", "mysql/root/root")
if err != nil {
log.Printf("Could not open rootdb: %v", err)
return
}
var n int
err = rootdb.QueryRow("SELECT COUNT(*) FROM user").Scan(&n)
if err == nil {
dbAvailable = true
}
}
func skipOrFailIfNoMySQL(t *testing.T) {
once.Do(checkDB)
if !dbAvailable {
// TODO(bradfitz): accept somehow other passwords than
// 'root', and/or try localhost unix socket
// connections rather than using TCP localhost?
err := errors.New("Not running; start a MySQL daemon on the standard port (3306) with root password 'root'")
test.DependencyErrorOrSkip(t)
t.Fatalf("MySQL not available locally for testing: %v", err)
}
}
func do(db *sql.DB, sql string) {
_, err := db.Exec(sql)
if err == nil {
return
}
log.Fatalf("Error %v running SQL: %s", err, sql)
}
func newSorted(t *testing.T) (kv sorted.KeyValue, clean func()) {
skipOrFailIfNoMySQL(t)
dbname := "camlitest_" + osutil.Username()
do(rootdb, "DROP DATABASE IF EXISTS "+dbname)
do(rootdb, "CREATE DATABASE "+dbname)
kv, err := mysql.NewKeyValue(mysql.Config{
Host: "localhost:3306",
Database: dbname,
User: "root",
Password: "root",
})
if err != nil {
t.Fatal(err)
}
return kv, func() {
kv.Close()
}
}
func TestSortedKV(t *testing.T) {
kv, clean := newSorted(t)
defer clean()
kvtest.TestSorted(t, kv)
}
type mysqlTester struct{}
func (mysqlTester) test(t *testing.T, tfn func(*testing.T, func() *index.Index)) {
var mu sync.Mutex // guards cleanups
var cleanups []func()
defer func() {
mu.Lock() // never unlocked
for _, fn := range cleanups {
fn()
}
}()
makeIndex := func() *index.Index {
s, cleanup := newSorted(t)
mu.Lock()
cleanups = append(cleanups, cleanup)
mu.Unlock()
return index.MustNew(t, s)
}
tfn(t, makeIndex)
}
func TestIndex_MySQL(t *testing.T) {
mysqlTester{}.test(t, indextest.Index)
}
func TestPathsOfSignerTarget_MySQL(t *testing.T) {
mysqlTester{}.test(t, indextest.PathsOfSignerTarget)
}
func TestFiles_MySQL(t *testing.T) {
mysqlTester{}.test(t, indextest.Files)
}
func TestEdgesTo_MySQL(t *testing.T) {
mysqlTester{}.test(t, indextest.EdgesTo)
}
func TestDelete_MySQL(t *testing.T) {
mysqlTester{}.test(t, indextest.Delete)
}

60
pkg/index/mysql/mysqlindexer.go
View File

@ -1,60 +0,0 @@
/*
Copyright 2011 The Camlistore Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package mysql implements the Camlistore index storage abstraction
// on top of MySQL.
package mysql
import (
"camlistore.org/pkg/blobserver"
"camlistore.org/pkg/index"
"camlistore.org/pkg/jsonconfig"
"camlistore.org/pkg/sorted/mysql"
_ "camlistore.org/third_party/github.com/ziutek/mymysql/godrv"
)
func init() {
blobserver.RegisterStorageConstructor("mysqlindexer", newFromConfig)
}
func newFromConfig(ld blobserver.Loader, config jsonconfig.Obj) (blobserver.Storage, error) {
blobPrefix := config.RequiredString("blobSource")
mysqlConf, err := mysql.ConfigFromJSON(config)
if err != nil {
return nil, err
}
kv, err := mysql.NewKeyValue(mysqlConf)
if err != nil {
return nil, err
}
ix, err := index.New(kv)
if err != nil {
return nil, err
}
sto, err := ld.GetStorage(blobPrefix)
if err != nil {
ix.Close()
return nil, err
}
ix.BlobSource = sto
// Good enough, for now:
ix.KeyFetcher = ix.BlobSource
return ix, nil
}

74
pkg/index/mysql_test.go Normal file
View File

@ -0,0 +1,74 @@
/*
Copyright 2012 The Camlistore Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package index_test
import (
"testing"
"camlistore.org/pkg/index/indextest"
"camlistore.org/pkg/osutil"
"camlistore.org/pkg/sorted"
"camlistore.org/pkg/sorted/kvtest"
"camlistore.org/pkg/sorted/mysql"
"camlistore.org/pkg/test/dockertest"
)
func newMySQLSorted(t *testing.T) (kv sorted.KeyValue, clean func()) {
dbname := "camlitest_" + osutil.Username()
containerID, ip := dockertest.SetupMySQLContainer(t, dbname)
kv, err := mysql.NewKeyValue(mysql.Config{
Host: ip + ":3306",
Database: dbname,
User: dockertest.MySQLUsername,
Password: dockertest.MySQLPassword,
})
if err != nil {
containerID.Kill()
t.Fatal(err)
}
return kv, func() {
kv.Close()
containerID.Kill()
}
}
func TestSorted_MySQL(t *testing.T) {
kv, clean := newMySQLSorted(t)
defer clean()
kvtest.TestSorted(t, kv)
}
func TestIndex_MySQL(t *testing.T) {
indexTest(t, newMySQLSorted, indextest.Index)
}
func TestPathsOfSignerTarget_MySQL(t *testing.T) {
indexTest(t, newMySQLSorted, indextest.PathsOfSignerTarget)
}
func TestFiles_MySQL(t *testing.T) {
indexTest(t, newMySQLSorted, indextest.Files)
}
func TestEdgesTo_MySQL(t *testing.T) {
indexTest(t, newMySQLSorted, indextest.EdgesTo)
}
func TestDelete_MySQL(t *testing.T) {
indexTest(t, newMySQLSorted, indextest.Delete)
}

219
pkg/index/postgres/postgres_test.go
View File

@ -1,219 +0,0 @@
/*
Copyright 2012 The Camlistore Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package postgres_test
import (
"database/sql"
"errors"
"fmt"
"log"
"strings"
"sync"
"testing"
"camlistore.org/pkg/index"
"camlistore.org/pkg/index/indextest"
"camlistore.org/pkg/osutil"
"camlistore.org/pkg/sorted"
"camlistore.org/pkg/sorted/kvtest"
"camlistore.org/pkg/sorted/postgres"
"camlistore.org/pkg/test"
_ "camlistore.org/third_party/github.com/lib/pq"
)
var (
once sync.Once
dbAvailable bool
rootdb *sql.DB
)
func checkDB() {
var err error
rootdb, err = sql.Open("postgres", "user=postgres password=postgres host=localhost dbname=postgres")
if err != nil {
log.Printf("Could not open postgres rootdb: %v", err)
return
}
var n int
err = rootdb.QueryRow("SELECT COUNT(*) FROM user").Scan(&n)
if err == nil {
dbAvailable = true
}
}
func skipOrFailIfNoPostgreSQL(t *testing.T) {
once.Do(checkDB)
if !dbAvailable {
err := errors.New("Not running; start a postgres daemon on the standard port (5432) with password 'postgres' for postgres user")
test.DependencyErrorOrSkip(t)
t.Fatalf("PostGreSQL not available locally for testing: %v", err)
}
}
func do(db *sql.DB, sql string) {
_, err := db.Exec(sql)
if err == nil {
return
}
log.Fatalf("Error %v running SQL: %s", err, sql)
}
func doQuery(db *sql.DB, sql string) {
r, err := db.Query(sql)
if err == nil {
r.Close()
return
}
log.Fatalf("Error %v running SQL query: %s", err, sql)
}
// closeAllSessions connects to the "postgres" DB on cfg.Host, and closes all connections to cfg.Database.
func closeAllSessions(cfg postgres.Config) error {
conninfo := fmt.Sprintf("user=%s dbname=postgres host=%s password=%s sslmode=%s",
cfg.User, cfg.Host, cfg.Password, cfg.SSLMode)
rootdb, err := sql.Open("postgres", conninfo)
if err != nil {
return fmt.Errorf("Could not open root db: %v", err)
}
defer rootdb.Close()
query := `
SELECT
pg_terminate_backend(pg_stat_activity.pid)
FROM
pg_stat_activity
WHERE
datname = '` + cfg.Database +
`' AND pid <> pg_backend_pid()`
// They changed procpid to pid in 9.2
if version(rootdb) < "9.2" {
query = strings.Replace(query, ".pid", ".procpid", 1)
query = strings.Replace(query, "AND pid", "AND procpid", 1)
}
r, err := rootdb.Query(query)
if err != nil {
return fmt.Errorf("Error running SQL query\n %v\n: %s", query, err)
}
r.Close()
return nil
}
func version(db *sql.DB) string {
version := ""
if err := db.QueryRow("SELECT version()").Scan(&version); err != nil {
log.Fatalf("Could not get PostgreSQL version: %v", err)
}
fields := strings.Fields(version)
if len(fields) < 2 {
log.Fatalf("Could not get PostgreSQL version because bogus answer: %q", version)
}
return fields[1]
}
func newSorted(t *testing.T) (kv sorted.KeyValue, clean func()) {
skipOrFailIfNoPostgreSQL(t)
dbname := "camlitest_" + osutil.Username()
if err := closeAllSessions(postgres.Config{
User: "postgres",
Password: "postgres",
SSLMode: "require",
Database: dbname,
Host: "localhost",
}); err != nil {
t.Fatalf("Could not close all old sessions to %q: %v", dbname, err)
}
do(rootdb, "DROP DATABASE IF EXISTS "+dbname)
do(rootdb, "CREATE DATABASE "+dbname+" LC_COLLATE = 'C' TEMPLATE = template0")
kv, err := postgres.NewKeyValue(postgres.Config{
Host: "localhost",
Database: dbname,
User: "postgres",
Password: "postgres",
SSLMode: "require",
})
if err != nil {
t.Fatal(err)
}
return kv, func() {
kv.Close()
}
}
func TestSortedKV(t *testing.T) {
kv, clean := newSorted(t)
defer clean()
kvtest.TestSorted(t, kv)
}
type postgresTester struct{}
func (postgresTester) test(t *testing.T, tfn func(*testing.T, func() *index.Index)) {
var mu sync.Mutex // guards cleanups
var cleanups []func()
defer func() {
mu.Lock() // never unlocked
for _, fn := range cleanups {
fn()
}
}()
makeIndex := func() *index.Index {
s, cleanup := newSorted(t)
mu.Lock()
cleanups = append(cleanups, cleanup)
mu.Unlock()
return index.MustNew(t, s)
}
tfn(t, makeIndex)
}
func TestIndex_Postgres(t *testing.T) {
if testing.Short() {
t.Logf("skipping test in short mode")
return
}
postgresTester{}.test(t, indextest.Index)
}
func TestPathsOfSignerTarget_Postgres(t *testing.T) {
if testing.Short() {
t.Logf("skipping test in short mode")
return
}
postgresTester{}.test(t, indextest.PathsOfSignerTarget)
}
func TestFiles_Postgres(t *testing.T) {
if testing.Short() {
t.Logf("skipping test in short mode")
return
}
postgresTester{}.test(t, indextest.Files)
}
func TestEdgesTo_Postgres(t *testing.T) {
if testing.Short() {
t.Logf("skipping test in short mode")
return
}
postgresTester{}.test(t, indextest.EdgesTo)
}
func TestDelete_Postgres(t *testing.T) {
postgresTester{}.test(t, indextest.Delete)
}

60
pkg/index/postgres/postgresindexer.go
View File

@ -1,60 +0,0 @@
/*
Copyright 2012 The Camlistore Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package postgres implements the Camlistore index storage abstraction
// on top of PostgreSQL.
package postgres
import (
"camlistore.org/pkg/blobserver"
"camlistore.org/pkg/index"
"camlistore.org/pkg/jsonconfig"
"camlistore.org/pkg/sorted/postgres"
_ "camlistore.org/third_party/github.com/lib/pq"
)
func init() {
blobserver.RegisterStorageConstructor("postgresindexer", newFromConfig)
}
func newFromConfig(ld blobserver.Loader, config jsonconfig.Obj) (blobserver.Storage, error) {
blobPrefix := config.RequiredString("blobSource")
postgresConf, err := postgres.ConfigFromJSON(config)
if err != nil {
return nil, err
}
kv, err := postgres.NewKeyValue(postgresConf)
if err != nil {
return nil, err
}
ix, err := index.New(kv)
if err != nil {
return nil, err
}
sto, err := ld.GetStorage(blobPrefix)
if err != nil {
ix.Close()
return nil, err
}
ix.BlobSource = sto
// Good enough, for now:
ix.KeyFetcher = ix.BlobSource
return ix, nil
}

75
pkg/index/postgres_test.go Normal file
View File

@ -0,0 +1,75 @@
/*
Copyright 2012 The Camlistore Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package index_test
import (
"testing"
"camlistore.org/pkg/index/indextest"
"camlistore.org/pkg/osutil"
"camlistore.org/pkg/sorted"
"camlistore.org/pkg/sorted/kvtest"
"camlistore.org/pkg/sorted/postgres"
"camlistore.org/pkg/test/dockertest"
)
func newPostgresSorted(t *testing.T) (kv sorted.KeyValue, clean func()) {
dbname := "camlitest_" + osutil.Username()
containerID, ip := dockertest.SetupPostgreSQLContainer(t, dbname)
kv, err := postgres.NewKeyValue(postgres.Config{
Host: ip,
Database: dbname,
User: dockertest.PostgresUsername,
Password: dockertest.PostgresPassword,
SSLMode: "disable",
})
if err != nil {
containerID.Kill()
t.Fatal(err)
}
return kv, func() {
kv.Close()
containerID.Kill()
}
}
func TestSorted_Postgres(t *testing.T) {
kv, clean := newPostgresSorted(t)
defer clean()
kvtest.TestSorted(t, kv)
}
func TestIndex_Postgres(t *testing.T) {
indexTest(t, newPostgresSorted, indextest.Index)
}
func TestPathsOfSignerTarget_Postgres(t *testing.T) {
indexTest(t, newPostgresSorted, indextest.PathsOfSignerTarget)
}
func TestFiles_Postgres(t *testing.T) {
indexTest(t, newPostgresSorted, indextest.Files)
}
func TestEdgesTo_Postgres(t *testing.T) {
indexTest(t, newPostgresSorted, indextest.EdgesTo)
}
func TestDelete_Postgres(t *testing.T) {
indexTest(t, newPostgresSorted, indextest.Delete)
}

58
pkg/index/sqlite/sqlite.go
View File

@ -1,58 +0,0 @@
/*
Copyright 2012 The Camlistore Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package sqlite implements the Camlistore index storage abstraction
// using an SQLite database file.
package sqlite
import (
"camlistore.org/pkg/blobserver"
"camlistore.org/pkg/index"
"camlistore.org/pkg/jsonconfig"
"camlistore.org/pkg/sorted/sqlite"
)
func init() {
blobserver.RegisterStorageConstructor("sqliteindexer", newFromConfig)
}
func newFromConfig(ld blobserver.Loader, config jsonconfig.Obj) (blobserver.Storage, error) {
blobPrefix := config.RequiredString("blobSource")
file := config.RequiredString("file")
if err := config.Validate(); err != nil {
return nil, err
}
kv, err := sqlite.NewKeyValue(file)
if err != nil {
return nil, err
}
ix, err := index.New(kv)
if err != nil {
return nil, err
}
sto, err := ld.GetStorage(blobPrefix)
if err != nil {
ix.Close()
return nil, err
}
ix.BlobSource = sto
// Good enough, for now:
ix.KeyFetcher = ix.BlobSource
return ix, nil
}

2
pkg/index/sqlite/sqlite_test.go
View File

@ -72,7 +72,7 @@ func newSorted(t *testing.T) (kv sorted.KeyValue, clean func()) {
}
}
func TestSortedKV(t *testing.T) {
func TestSorted_SQLite(t *testing.T) {
kv, clean := newSorted(t)
defer clean()
kvtest.TestSorted(t, kv)

135
pkg/serverinit/genconfig.go
View File

@ -17,8 +17,11 @@ limitations under the License.
package serverinit
import (
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"log"
"net/url"
"os"
"path/filepath"
@ -37,7 +40,8 @@ import (
type configPrefixesParams struct {
secretRing string
keyId string
indexerPath string
haveIndex bool
haveSQLite bool
blobPath string
packBlobs bool
searchOwner blob.Ref
@ -144,15 +148,18 @@ func addMongoConfig(prefixes jsonconfig.Obj, dbname string, dbinfo string) {
}
ob := map[string]interface{}{}
ob["enabled"] = true
ob["handler"] = "storage-mongodbindexer"
ob["handler"] = "storage-index"
ob["handlerArgs"] = map[string]interface{}{
"host": host,
"user": fields[0],
"password": fields[1],
"database": dbname,
"blobSource": "/bs/",
"storage": map[string]interface{}{
"type": "mongo",
"host": host,
"user": fields[0],
"password": fields[1],
"database": dbname,
},
}
prefixes["/index-mongo/"] = ob
prefixes["/index/"] = ob
}
func addSQLConfig(rdbms string, prefixes jsonconfig.Obj, dbname string, dbinfo string) {
@ -167,15 +174,18 @@ func addSQLConfig(rdbms string, prefixes jsonconfig.Obj, dbname string, dbinfo s
}
ob := map[string]interface{}{}
ob["enabled"] = true
ob["handler"] = "storage-" + rdbms + "indexer"
ob["handler"] = "storage-index"
ob["handlerArgs"] = map[string]interface{}{
"host": fields[0],
"user": user,
"password": fields[1],
"database": dbname,
"blobSource": "/bs/",
"storage": map[string]interface{}{
"type": rdbms,
"host": fields[0],
"user": user,
"password": fields[1],
"database": dbname,
},
}
prefixes["/index-"+rdbms+"/"] = ob
prefixes["/index/"] = ob
}
func addPostgresConfig(prefixes jsonconfig.Obj, dbname string, dbinfo string) {
@ -188,20 +198,26 @@ func addMySQLConfig(prefixes jsonconfig.Obj, dbname string, dbinfo string) {
func addSQLiteConfig(prefixes jsonconfig.Obj, file string) {
ob := map[string]interface{}{}
ob["handler"] = "storage-sqliteindexer"
ob["handler"] = "storage-index"
ob["handlerArgs"] = map[string]interface{}{
"blobSource": "/bs/",
"file": file,
"storage": map[string]interface{}{
"type": "sqlite",
"file": file,
},
}
prefixes["/index-sqlite/"] = ob
prefixes["/index/"] = ob
}
func addKVConfig(prefixes jsonconfig.Obj, file string) {
prefixes["/index-kv/"] = map[string]interface{}{
"handler": "storage-kvfileindexer",
prefixes["/index/"] = map[string]interface{}{
"handler": "storage-index",
"handlerArgs": map[string]interface{}{
"blobSource": "/bs/",
"file": file,
"storage": map[string]interface{}{
"type": "kv",
"file": file,
},
},
}
}
@ -384,7 +400,7 @@ func addGoogleCloudStorageConfig(params *configPrefixesParams, prefixes jsonconf
func genLowLevelPrefixes(params *configPrefixesParams, ownerName string) (m jsonconfig.Obj) {
m = make(jsonconfig.Obj)
haveIndex := params.indexerPath != ""
haveIndex := params.haveIndex
root := "/bs/"
pubKeyDest := root
if haveIndex {
@ -474,7 +490,7 @@ func genLowLevelPrefixes(params *configPrefixesParams, ownerName string) (m json
if haveIndex {
syncArgs := map[string]interface{}{
"from": "/bs/",
"to": params.indexerPath,
"to": "/index/",
}
// TODO: currently when using s3, the index must be
@ -491,7 +507,7 @@ func genLowLevelPrefixes(params *configPrefixesParams, ownerName string) (m json
dir = params.indexFileDir
}
typ := "kv"
if params.indexerPath == "/index-sqlite/" {
if params.haveSQLite {
typ = "sqlite"
}
syncArgs["queue"] = map[string]interface{}{
@ -507,7 +523,7 @@ func genLowLevelPrefixes(params *configPrefixesParams, ownerName string) (m json
m["/bs-and-index/"] = map[string]interface{}{
"handler": "storage-replica",
"handlerArgs": map[string]interface{}{
"backends": []interface{}{"/bs/", params.indexerPath},
"backends": []interface{}{"/bs/", "/index/"},
},
}
@ -524,7 +540,7 @@ func genLowLevelPrefixes(params *configPrefixesParams, ownerName string) (m json
}
searchArgs := map[string]interface{}{
"index": params.indexerPath,
"index": "/index/",
"owner": params.searchOwner.String(),
}
if params.memoryIndex {
@ -581,10 +597,11 @@ func genLowLevelConfig(conf *serverconfig.Config) (lowLevelConf *Config, err err
conf.DBName = "camli" + username
}
var indexerPath string // e.g. "/index-kv/"
var haveSQLite bool
var indexFileDir string // filesystem directory of sqlite, kv, or similar
numIndexers := numSet(conf.Mongo, conf.MySQL, conf.PostgreSQL, conf.SQLite, conf.KVFile)
runIndex := conf.RunIndex.Get()
switch {
case runIndex && numIndexers == 0:
return nil, fmt.Errorf("Unless runIndex is set to false, you must specify an index option (kvIndexFile, mongo, mysql, postgres, sqlite).")
@ -592,17 +609,10 @@ func genLowLevelConfig(conf *serverconfig.Config) (lowLevelConf *Config, err err
return nil, fmt.Errorf("With runIndex set true, you can only pick exactly one indexer (mongo, mysql, postgres, sqlite).")
case !runIndex && numIndexers != 0:
return nil, fmt.Errorf("With runIndex disabled, you can't specify any of mongo, mysql, postgres, sqlite.")
case conf.MySQL != "":
indexerPath = "/index-mysql/"
case conf.PostgreSQL != "":
indexerPath = "/index-postgres/"
case conf.Mongo != "":
indexerPath = "/index-mongo/"
case conf.SQLite != "":
indexerPath = "/index-sqlite/"
haveSQLite = true
indexFileDir = filepath.Dir(conf.SQLite)
case conf.KVFile != "":
indexerPath = "/index-kv/"
indexFileDir = filepath.Dir(conf.KVFile)
}
@ -632,7 +642,8 @@ func genLowLevelConfig(conf *serverconfig.Config) (lowLevelConf *Config, err err
prefixesParams := &configPrefixesParams{
secretRing: conf.IdentitySecretRing,
keyId: conf.Identity,
indexerPath: indexerPath,
haveIndex: runIndex,
haveSQLite: haveSQLite,
blobPath: conf.BlobPath,
packBlobs: conf.PackBlobs,
searchOwner: blob.SHA1FromString(armoredPublicKey),
@ -742,3 +753,59 @@ func setMap(m map[string]interface{}, v ...interface{}) {
}
setMap(m[v[0].(string)].(map[string]interface{}), v[1:]...)
}
// WriteDefaultConfigFile generates a new default high-level server configuration
// file at filePath. If useSQLite, the default indexer will use SQLite, otherwise
// kv. If filePath already exists, it is overwritten.
func WriteDefaultConfigFile(filePath string, useSQLite bool) error {
conf := serverconfig.Config{
Listen: ":3179",
HTTPS: false,
Auth: "localhost",
ReplicateTo: make([]interface{}, 0),
}
blobDir := osutil.CamliBlobRoot()
if err := os.MkdirAll(blobDir, 0700); err != nil {
return fmt.Errorf("Could not create default blobs directory: %v", err)
}
conf.BlobPath = blobDir
if useSQLite {
conf.SQLite = filepath.Join(osutil.CamliVarDir(), "camli-index.db")
} else {
conf.KVFile = filepath.Join(osutil.CamliVarDir(), "camli-index.kvdb")
}
var keyId string
secRing := osutil.SecretRingFile()
_, err := os.Stat(secRing)
switch {
case err == nil:
keyId, err = jsonsign.KeyIdFromRing(secRing)
if err != nil {
return fmt.Errorf("Could not find any keyId in file %q: %v", secRing, err)
}
log.Printf("Re-using identity with keyId %q found in file %s", keyId, secRing)
case os.IsNotExist(err):
keyId, err = jsonsign.GenerateNewSecRing(secRing)
if err != nil {
return fmt.Errorf("Could not generate new secRing at file %q: %v", secRing, err)
}
log.Printf("Generated new identity with keyId %q in file %s", keyId, secRing)
}
if err != nil {
return fmt.Errorf("Could not stat secret ring %q: %v", secRing, err)
}
conf.Identity = keyId
conf.IdentitySecretRing = secRing
confData, err := json.MarshalIndent(conf, "", " ")
if err != nil {
return fmt.Errorf("Could not json encode config file : %v", err)
}
if err := ioutil.WriteFile(filePath, confData, 0600); err != nil {
return fmt.Errorf("Could not create or write default server config: %v", err)
}
return nil
}

15
pkg/serverinit/testdata/baseurl-want.json vendored
View File

@ -19,7 +19,7 @@
"handlerArgs": {
"backends": [
"/bs/",
"/index-kv/"
"/index/"
]
}
},
@ -46,17 +46,20 @@
"path": "/tmp/blobs/cache"
}
},
"/index-kv/": {
"handler": "storage-kvfileindexer",
"/index/": {
"handler": "storage-index",
"handlerArgs": {
"blobSource": "/bs/",
"file": "/path/to/indexkv.db"
"storage": {
"type": "kv",
"file": "/path/to/indexkv.db"
}
}
},
"/my-search/": {
"handler": "search",
"handlerArgs": {
"index": "/index-kv/",
"index": "/index/",
"owner": "sha1-f2b0b7da718b97ce8c31591d8ed4645c777f3ef4",
"slurpToMemory": true
}
@ -89,7 +92,7 @@
"file": "/tmp/blobs/sync-to-index-queue.kv",
"type": "kv"
},
"to": "/index-kv/"
"to": "/index/"
}
},
"/ui/": {

15
pkg/serverinit/testdata/default-want.json vendored
View File

@ -18,7 +18,7 @@
"handlerArgs": {
"backends": [
"/bs/",
"/index-kv/"
"/index/"
]
}
},
@ -45,17 +45,20 @@
"path": "/tmp/blobs/cache"
}
},
"/index-kv/": {
"handler": "storage-kvfileindexer",
"/index/": {
"handler": "storage-index",
"handlerArgs": {
"blobSource": "/bs/",
"file": "/path/to/indexkv.db"
"storage": {
"type": "kv",
"file": "/path/to/indexkv.db"
}
}
},
"/my-search/": {
"handler": "search",
"handlerArgs": {
"index": "/index-kv/",
"index": "/index/",
"owner": "sha1-f2b0b7da718b97ce8c31591d8ed4645c777f3ef4",
"slurpToMemory": true
}
@ -88,7 +91,7 @@
"file": "/tmp/blobs/sync-to-index-queue.kv",
"type": "kv"
},
"to": "/index-kv/"
"to": "/index/"
}
},
"/ui/": {

15
pkg/serverinit/testdata/diskpacked-want.json vendored
View File

@ -18,7 +18,7 @@
"handlerArgs": {
"backends": [
"/bs/",
"/index-kv/"
"/index/"
]
}
},
@ -45,17 +45,20 @@
"path": "/tmp/blobs/cache"
}
},
"/index-kv/": {
"handler": "storage-kvfileindexer",
"/index/": {
"handler": "storage-index",
"handlerArgs": {
"blobSource": "/bs/",
"file": "/path/to/indexkv.db"
"storage": {
"type": "kv",
"file": "/path/to/indexkv.db"
}
}
},
"/my-search/": {
"handler": "search",
"handlerArgs": {
"index": "/index-kv/",
"index": "/index/",
"owner": "sha1-f2b0b7da718b97ce8c31591d8ed4645c777f3ef4",
"slurpToMemory": true
}
@ -88,7 +91,7 @@
"file": "/tmp/blobs/sync-to-index-queue.kv",
"type": "kv"
},
"to": "/index-kv/"
"to": "/index/"
}
},
"/ui/": {

15
pkg/serverinit/testdata/flickr-want.json vendored
View File

@ -18,7 +18,7 @@
"handlerArgs": {
"backends": [
"/bs/",
"/index-kv/"
"/index/"
]
}
},
@ -51,17 +51,20 @@
"apiKey": "monkey:balls"
}
},
"/index-kv/": {
"handler": "storage-kvfileindexer",
"/index/": {
"handler": "storage-index",
"handlerArgs": {
"blobSource": "/bs/",
"file": "/path/to/indexkv.db"
"storage": {
"type": "kv",
"file": "/path/to/indexkv.db"
}
}
},
"/my-search/": {
"handler": "search",
"handlerArgs": {
"index": "/index-kv/",
"index": "/index/",
"owner": "sha1-f2b0b7da718b97ce8c31591d8ed4645c777f3ef4",
"slurpToMemory": true
}
@ -94,7 +97,7 @@
"file": "/tmp/blobs/sync-to-index-queue.kv",
"type": "kv"
},
"to": "/index-kv/"
"to": "/index/"
}
},
"/ui/": {

15
pkg/serverinit/testdata/google_nolocaldisk-want.json vendored
View File

@ -17,7 +17,7 @@
"handlerArgs": {
"backends": [
"/bs/",
"/index-kv/"
"/index/"
]
}
},
@ -49,17 +49,20 @@
"path": "/tmp/camli-cache"
}
},
"/index-kv/": {
"handler": "storage-kvfileindexer",
"/index/": {
"handler": "storage-index",
"handlerArgs": {
"blobSource": "/bs/",
"file": "/path/to/indexkv.db"
"storage": {
"type": "kv",
"file": "/path/to/indexkv.db"
}
}
},
"/my-search/": {
"handler": "search",
"handlerArgs": {
"index": "/index-kv/",
"index": "/index/",
"owner": "sha1-f2b0b7da718b97ce8c31591d8ed4645c777f3ef4",
"slurpToMemory": true
}
@ -92,7 +95,7 @@
"file": "/path/to/sync-to-index-queue.kv",
"type": "kv"
},
"to": "/index-kv/"
"to": "/index/"
}
},
"/ui/": {

15
pkg/serverinit/testdata/listenbase-want.json vendored
View File

@ -18,7 +18,7 @@
"handlerArgs": {
"backends": [
"/bs/",
"/index-kv/"
"/index/"
]
}
},
@ -45,17 +45,20 @@
"path": "/tmp/blobs/cache"
}
},
"/index-kv/": {
"handler": "storage-kvfileindexer",
"/index/": {
"handler": "storage-index",
"handlerArgs": {
"blobSource": "/bs/",
"file": "/path/to/indexkv.db"
"storage": {
"type": "kv",
"file": "/path/to/indexkv.db"
}
}
},
"/my-search/": {
"handler": "search",
"handlerArgs": {
"index": "/index-kv/",
"index": "/index/",
"owner": "sha1-f2b0b7da718b97ce8c31591d8ed4645c777f3ef4",
"slurpToMemory": true
}
@ -88,7 +91,7 @@
"file": "/tmp/blobs/sync-to-index-queue.kv",
"type": "kv"
},
"to": "/index-kv/"
"to": "/index/"
}
},
"/ui/": {

15
pkg/serverinit/testdata/mem-want.json vendored
View File

@ -18,7 +18,7 @@
"handlerArgs": {
"backends": [
"/bs/",
"/index-kv/"
"/index/"
]
}
},
@ -45,17 +45,20 @@
"path": "/tmp/blobs/cache"
}
},
"/index-kv/": {
"handler": "storage-kvfileindexer",
"/index/": {
"handler": "storage-index",
"handlerArgs": {
"blobSource": "/bs/",
"file": "/path/to/indexkv.db"
"storage": {
"type": "kv",
"file": "/path/to/indexkv.db"
}
}
},
"/my-search/": {
"handler": "search",
"handlerArgs": {
"index": "/index-kv/",
"index": "/index/",
"owner": "sha1-f2b0b7da718b97ce8c31591d8ed4645c777f3ef4",
"slurpToMemory": true
}
@ -151,7 +154,7 @@
"file": "/tmp/blobs/sync-to-index-queue.kv",
"type": "kv"
},
"to": "/index-kv/"
"to": "/index/"
}
},
"/ui/": {

15
pkg/serverinit/testdata/memindex-want.json vendored
View File

@ -18,7 +18,7 @@
"handlerArgs": {
"backends": [
"/bs/",
"/index-kv/"
"/index/"
]
}
},
@ -45,17 +45,20 @@
"path": "/tmp/blobs/cache"
}
},
"/index-kv/": {
"handler": "storage-kvfileindexer",
"/index/": {
"handler": "storage-index",
"handlerArgs": {
"blobSource": "/bs/",
"file": "/path/to/indexkv.db"
"storage": {
"type": "kv",
"file": "/path/to/indexkv.db"
}
}
},
"/my-search/": {
"handler": "search",
"handlerArgs": {
"index": "/index-kv/",
"index": "/index/",
"owner": "sha1-f2b0b7da718b97ce8c31591d8ed4645c777f3ef4",
"slurpToMemory": true
}
@ -88,7 +91,7 @@
"file": "/tmp/blobs/sync-to-index-queue.kv",
"type": "kv"
},
"to": "/index-kv/"
"to": "/index/"
}
},
"/ui/": {

21
pkg/serverinit/testdata/mongo-want.json vendored
View File

@ -18,7 +18,7 @@
"handlerArgs": {
"backends": [
"/bs/",
"/index-mongo/"
"/index/"
]
}
},
@ -45,21 +45,24 @@
"path": "/tmp/blobs/cache"
}
},
"/index-mongo/": {
"/index/": {
"enabled": true,
"handler": "storage-mongodbindexer",
"handler": "storage-index",
"handlerArgs": {
"blobSource": "/bs/",
"database": "camlitest",
"host": "localhost",
"password": "",
"user": ""
"storage": {
"type": "mongo",
"database": "camlitest",
"host": "localhost",
"password": "",
"user": ""
}
}
},
"/my-search/": {
"handler": "search",
"handlerArgs": {
"index": "/index-mongo/",
"index": "/index/",
"owner": "sha1-f2b0b7da718b97ce8c31591d8ed4645c777f3ef4",
"slurpToMemory": true
}
@ -92,7 +95,7 @@
"file": "/tmp/blobs/sync-to-index-queue.kv",
"type": "kv"
},
"to": "/index-mongo/"
"to": "/index/"
}
},
"/ui/": {

15
pkg/serverinit/testdata/multipublish-want.json vendored
View File

@ -18,7 +18,7 @@
"handlerArgs": {
"backends": [
"/bs/",
"/index-kv/"
"/index/"
]
}
},
@ -45,17 +45,20 @@
"path": "/tmp/blobs/cache"
}
},
"/index-kv/": {
"handler": "storage-kvfileindexer",
"/index/": {
"handler": "storage-index",
"handlerArgs": {
"blobSource": "/bs/",
"file": "/path/to/indexkv.db"
"storage": {
"type": "kv",
"file": "/path/to/indexkv.db"
}
}
},
"/my-search/": {
"handler": "search",
"handlerArgs": {
"index": "/index-kv/",
"index": "/index/",
"owner": "sha1-f2b0b7da718b97ce8c31591d8ed4645c777f3ef4",
"slurpToMemory": true
}
@ -137,7 +140,7 @@
"file": "/tmp/blobs/sync-to-index-queue.kv",
"type": "kv"
},
"to": "/index-kv/"
"to": "/index/"
}
},
"/ui/": {

15
pkg/serverinit/testdata/s3_alt_host-want.json vendored
View File

@ -17,7 +17,7 @@
"handlerArgs": {
"backends": [
"/bs/",
"/index-kv/"
"/index/"
]
}
},
@ -47,17 +47,20 @@
"path": "/tmp/camli-cache"
}
},
"/index-kv/": {
"handler": "storage-kvfileindexer",
"/index/": {
"handler": "storage-index",
"handlerArgs": {
"blobSource": "/bs/",
"file": "/path/to/indexkv.db"
"storage": {
"type": "kv",
"file": "/path/to/indexkv.db"
}
}
},
"/my-search/": {
"handler": "search",
"handlerArgs": {
"index": "/index-kv/",
"index": "/index/",
"owner": "sha1-f2b0b7da718b97ce8c31591d8ed4645c777f3ef4",
"slurpToMemory": true
}
@ -90,7 +93,7 @@
"file": "/path/to/sync-to-index-queue.kv",
"type": "kv"
},
"to": "/index-kv/"
"to": "/index/"
}
},
"/ui/": {

15
pkg/serverinit/testdata/s3_nolocaldisk-want.json vendored
View File

@ -17,7 +17,7 @@
"handlerArgs": {
"backends": [
"/bs/",
"/index-kv/"
"/index/"
]
}
},
@ -46,17 +46,20 @@
"path": "/tmp/camli-cache"
}
},
"/index-kv/": {
"handler": "storage-kvfileindexer",
"/index/": {
"handler": "storage-index",
"handlerArgs": {
"blobSource": "/bs/",
"file": "/path/to/indexkv.db"
"storage": {
"type": "kv",
"file": "/path/to/indexkv.db"
}
}
},
"/my-search/": {
"handler": "search",
"handlerArgs": {
"index": "/index-kv/",
"index": "/index/",
"owner": "sha1-f2b0b7da718b97ce8c31591d8ed4645c777f3ef4",
"slurpToMemory": true
}
@ -89,7 +92,7 @@
"file": "/path/to/sync-to-index-queue.kv",
"type": "kv"
},
"to": "/index-kv/"
"to": "/index/"
}
},
"/ui/": {

21
pkg/serverinit/testdata/s3_nolocaldisk_mysql-want.json vendored
View File

@ -17,7 +17,7 @@
"handlerArgs": {
"backends": [
"/bs/",
"/index-mysql/"
"/index/"
]
}
},
@ -46,21 +46,24 @@
"path": "/tmp/camli-cache"
}
},
"/index-mysql/": {
"/index/": {
"enabled": true,
"handler": "storage-mysqlindexer",
"handler": "storage-index",
"handlerArgs": {
"blobSource": "/bs/",
"database": "camlitest",
"host": "localhost",
"password": "password",
"user": "user"
"storage": {
"type": "mysql",
"database": "camlitest",
"host": "localhost",
"password": "password",
"user": "user"
}
}
},
"/my-search/": {
"handler": "search",
"handlerArgs": {
"index": "/index-mysql/",
"index": "/index/",
"owner": "sha1-f2b0b7da718b97ce8c31591d8ed4645c777f3ef4",
"slurpToMemory": true
}
@ -90,7 +93,7 @@
"handlerArgs": {
"from": "/bs/",
"idle": true,
"to": "/index-mysql/"
"to": "/index/"
}
},
"/ui/": {

15
pkg/serverinit/testdata/sqlite-want.json vendored
View File

@ -17,7 +17,7 @@
"handlerArgs": {
"backends": [
"/bs/",
"/index-sqlite/"
"/index/"
]
}
},
@ -44,17 +44,20 @@
"path": "/tmp/blobs/cache"
}
},
"/index-sqlite/": {
"handler": "storage-sqliteindexer",
"/index/": {
"handler": "storage-index",
"handlerArgs": {
"blobSource": "/bs/",
"file": "/tmp/camli.db"
"storage": {
"type": "sqlite",
"file": "/tmp/camli.db"
}
}
},
"/my-search/": {
"handler": "search",
"handlerArgs": {
"index": "/index-sqlite/",
"index": "/index/",
"owner": "sha1-f2b0b7da718b97ce8c31591d8ed4645c777f3ef4",
"slurpToMemory": true
}
@ -87,7 +90,7 @@
"file": "/tmp/blobs/sync-to-index-queue.sqlite",
"type": "sqlite"
},
"to": "/index-sqlite/"
"to": "/index/"
}
},
"/ui/": {

15
pkg/serverinit/testdata/tls-want.json vendored
View File

@ -19,7 +19,7 @@
"handlerArgs": {
"backends": [
"/bs/",
"/index-kv/"
"/index/"
]
}
},
@ -46,17 +46,20 @@
"path": "/tmp/blobs/cache"
}
},
"/index-kv/": {
"handler": "storage-kvfileindexer",
"/index/": {
"handler": "storage-index",
"handlerArgs": {
"blobSource": "/bs/",
"file": "/path/to/indexkv.db"
"storage": {
"type": "kv",
"file": "/path/to/indexkv.db"
}
}
},
"/my-search/": {
"handler": "search",
"handlerArgs": {
"index": "/index-kv/",
"index": "/index/",
"owner": "sha1-f2b0b7da718b97ce8c31591d8ed4645c777f3ef4",
"slurpToMemory": true
}
@ -89,7 +92,7 @@
"file": "/tmp/blobs/sync-to-index-queue.kv",
"type": "kv"
},
"to": "/index-kv/"
"to": "/index/"
}
},
"/ui/": {

15
pkg/serverinit/testdata/with_blog-want.json vendored
View File

@ -38,7 +38,7 @@
"handlerArgs": {
"backends": [
"/bs/",
"/index-kv/"
"/index/"
]
}
},
@ -65,17 +65,20 @@
"path": "/tmp/blobs/cache"
}
},
"/index-kv/": {
"handler": "storage-kvfileindexer",
"/index/": {
"handler": "storage-index",
"handlerArgs": {
"blobSource": "/bs/",
"file": "/path/to/indexkv.db"
"storage": {
"type": "kv",
"file": "/path/to/indexkv.db"
}
}
},
"/my-search/": {
"handler": "search",
"handlerArgs": {
"index": "/index-kv/",
"index": "/index/",
"owner": "sha1-f2b0b7da718b97ce8c31591d8ed4645c777f3ef4",
"slurpToMemory": true
}
@ -108,7 +111,7 @@
"file": "/tmp/blobs/sync-to-index-queue.kv",
"type": "kv"
},
"to": "/index-kv/"
"to": "/index/"
}
},
"/ui/": {

15
pkg/serverinit/testdata/with_gallery-want.json vendored
View File

@ -17,7 +17,7 @@
"handlerArgs": {
"backends": [
"/bs/",
"/index-kv/"
"/index/"
]
}
},
@ -44,17 +44,20 @@
"path": "/tmp/blobs/cache"
}
},
"/index-kv/": {
"handler": "storage-kvfileindexer",
"/index/": {
"handler": "storage-index",
"handlerArgs": {
"blobSource": "/bs/",
"file": "/path/to/indexkv.db"
"storage": {
"type": "kv",
"file": "/path/to/indexkv.db"
}
}
},
"/my-search/": {
"handler": "search",
"handlerArgs": {
"index": "/index-kv/",
"index": "/index/",
"owner": "sha1-f2b0b7da718b97ce8c31591d8ed4645c777f3ef4",
"slurpToMemory": true
}
@ -111,7 +114,7 @@
"file": "/tmp/blobs/sync-to-index-queue.kv",
"type": "kv"
},
"to": "/index-kv/"
"to": "/index/"
}
},
"/ui/": {

15
pkg/serverinit/testdata/with_sourceroot-want.json vendored
View File

@ -17,7 +17,7 @@
"handlerArgs": {
"backends": [
"/bs/",
"/index-kv/"
"/index/"
]
}
},
@ -44,17 +44,20 @@
"path": "/tmp/blobs/cache"
}
},
"/index-kv/": {
"handler": "storage-kvfileindexer",
"/index/": {
"handler": "storage-index",
"handlerArgs": {
"blobSource": "/bs/",
"file": "/path/to/indexkv.db"
"storage": {
"type": "kv",
"file": "/path/to/indexkv.db"
}
}
},
"/my-search/": {
"handler": "search",
"handlerArgs": {
"index": "/index-kv/",
"index": "/index/",
"owner": "sha1-f2b0b7da718b97ce8c31591d8ed4645c777f3ef4",
"slurpToMemory": true
}
@ -106,7 +109,7 @@
"file": "/tmp/blobs/sync-to-index-queue.kv",
"type": "kv"
},
"to": "/index-kv/"
"to": "/index/"
}
},
"/ui/": {

72
server/camlistored/camlistored.go
View File

@ -22,7 +22,6 @@ import (
"crypto/rsa"
"crypto/x509"
"crypto/x509/pkix"
"encoding/json"
"encoding/pem"
"flag"
"fmt"
@ -41,11 +40,9 @@ import (
"time"
"camlistore.org/pkg/buildinfo"
"camlistore.org/pkg/jsonsign"
"camlistore.org/pkg/misc"
"camlistore.org/pkg/osutil"
"camlistore.org/pkg/serverinit"
"camlistore.org/pkg/types/serverconfig"
"camlistore.org/pkg/webserver"
// Storage options:
@ -61,19 +58,13 @@ import (
_ "camlistore.org/pkg/blobserver/s3"
_ "camlistore.org/pkg/blobserver/shard"
// Indexers: (also present themselves as storage targets)
// sqlite is taken care of in option_sqlite.go
"camlistore.org/pkg/index" // base indexer + in-memory dev index
_ "camlistore.org/pkg/index/kvfile"
_ "camlistore.org/pkg/index/mongo"
_ "camlistore.org/pkg/index/mysql"
_ "camlistore.org/pkg/index/postgres"
"camlistore.org/pkg/index"
// KeyValue implementations:
_ "camlistore.org/pkg/sorted"
_ "camlistore.org/pkg/sorted/kvfile"
_ "camlistore.org/pkg/sorted/mongo"
_ "camlistore.org/pkg/sorted/mysql"
_ "camlistore.org/pkg/sorted/postgres"
"camlistore.org/pkg/sorted/sqlite"
"camlistore.org/pkg/sorted/sqlite" // for sqlite.CompiledIn()
// Handlers:
_ "camlistore.org/pkg/search"
@ -212,7 +203,7 @@ func findConfigFile(file string) (absPath string, isNewConfig bool, err error) {
return
}
log.Printf("Generating template config file %s", absPath)
if err = newDefaultConfigFile(absPath); err == nil {
if err = serverinit.WriteDefaultConfigFile(absPath, sqlite.CompiledIn()); err == nil {
isNewConfig = true
}
}
@ -226,63 +217,6 @@ func findConfigFile(file string) (absPath string, isNewConfig bool, err error) {
return
}
var defaultListenAddr = ":3179"
// TODO(mpl): move this func to pkg/types/serverconfig as well.
func newDefaultConfigFile(path string) error {
conf := serverconfig.Config{
Listen: defaultListenAddr,
HTTPS: false,
Auth: "localhost",
ReplicateTo: make([]interface{}, 0),
}
blobDir := osutil.CamliBlobRoot()
if err := os.MkdirAll(blobDir, 0700); err != nil {
return fmt.Errorf("Could not create default blobs directory: %v", err)
}
conf.BlobPath = blobDir
if sqlite.CompiledIn() {
conf.SQLite = filepath.Join(osutil.CamliVarDir(), "camli-index.db")
} else {
conf.KVFile = filepath.Join(osutil.CamliVarDir(), "camli-index.kvdb")
}
var keyId string
secRing := osutil.SecretRingFile()
_, err := os.Stat(secRing)
switch {
case err == nil:
keyId, err = jsonsign.KeyIdFromRing(secRing)
if err != nil {
return fmt.Errorf("Could not find any keyId in file %q: %v", secRing, err)
}
log.Printf("Re-using identity with keyId %q found in file %s", keyId, secRing)
case os.IsNotExist(err):
keyId, err = jsonsign.GenerateNewSecRing(secRing)
if err != nil {
return fmt.Errorf("Could not generate new secRing at file %q: %v", secRing, err)
}
log.Printf("Generated new identity with keyId %q in file %s", keyId, secRing)
}
if err != nil {
return fmt.Errorf("Could not stat secret ring %q: %v", secRing, err)
}
conf.Identity = keyId
conf.IdentitySecretRing = secRing
confData, err := json.MarshalIndent(conf, "", " ")
if err != nil {
return fmt.Errorf("Could not json encode config file : %v", err)
}
if err := ioutil.WriteFile(path, confData, 0600); err != nil {
return fmt.Errorf("Could not create or write default server config: %v", err)
}
return nil
}
func setupTLS(ws *webserver.Server, config *serverinit.Config, listen string) {
cert, key := config.OptionalString("httpsCert", ""), config.OptionalString("httpsKey", "")
if !config.OptionalBool("https", true) {

16
server/camlistored/option_sqlite.go
View File

@ -1,16 +0,0 @@
// +build with_sqlite
// If the "with_sqlite" build tag is specified, the sqlite index driver
// is also built & loaded:
//
// go install -tags=with_sqlite camlistore.org/server/camlistored
//
// This is an option because the sqlite3 SQL driver requires cgo & the
// SQLite3 C library available. We want it to still be possible to
// have a pure Go server too.
package main
import (
_ "camlistore.org/pkg/index/sqlite"
)

2
server/camlistored/run_test.go
View File

@ -57,7 +57,7 @@ func TestStarts(t *testing.T) {
mkdir(t, confDir)
*flagOpenBrowser = false
defaultListenAddr = ":0"
*listenFlag = ":0"
up := make(chan struct{})
down := make(chan struct{})