index/sqlite: new indexer, using sqlite3

Change-Id: If1405fe3ee7c632c05431f4b0be1077d6d42c4a3

pkg/index/sqlite/dbschema.go

@@ -0,0 +1,35 @@
+/*
+Copyright 2012 Google Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package sqlite
+
+const requiredSchemaVersion = 1
+
+func SchemaVersion() int {
+	return requiredSchemaVersion
+}
+
+func SQLCreateTables() []string {
+	return []string{
+		`CREATE TABLE rows (
+ k VARCHAR(255) NOT NULL PRIMARY KEY,
+ v VARCHAR(255))`,
+
+		`CREATE TABLE meta (
+ metakey VARCHAR(255) NOT NULL PRIMARY KEY,
+ value VARCHAR(255) NOT NULL)`,
+	}
+}

pkg/index/sqlite/sqlite.go

@@ -0,0 +1,114 @@
+/*
+Copyright 2012 Google Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package sqlite
+
+import (
+	"database/sql"
+	"fmt"
+	"os"
+
+	"camlistore.org/pkg/blobserver"
+	"camlistore.org/pkg/index"
+	"camlistore.org/pkg/index/sqlindex"
+	"camlistore.org/pkg/jsonconfig"
+
+	_ "camlistore.org/third_party/github.com/mattn/go-sqlite3"
+)
+
+type storage struct {
+	*sqlindex.Storage
+
+	file string
+	db   *sql.DB
+}
+
+var _ index.IndexStorage = (*storage)(nil)
+
+// NewStorage returns an IndexStorage implementation of the described SQLite database.
+// This exists mostly for testing and does not initialize the schema.
+func NewStorage(file string) (index.IndexStorage, error) {
+	db, err := sql.Open("sqlite3", file)
+	if err != nil {
+		return nil, err
+	}
+	// TODO(bradfitz): ping db, check that it's reachable.
+	return &storage{
+		file: file,
+		db:   db,
+		Storage: &sqlindex.Storage{
+			DB: db,
+		},
+	}, nil
+}
+
+func newFromConfig(ld blobserver.Loader, config jsonconfig.Obj) (blobserver.Storage, error) {
+	var (
+		blobPrefix = config.RequiredString("blobSource")
+		file       = config.RequiredString("file")
+	)
+	if err := config.Validate(); err != nil {
+		return nil, err
+	}
+	sto, err := ld.GetStorage(blobPrefix)
+	if err != nil {
+		return nil, err
+	}
+	isto, err := NewStorage(file)
+	if err != nil {
+		return nil, err
+	}
+	is := isto.(*storage)
+	if err := is.ping(); err != nil {
+		return nil, err
+	}
+
+	version, err := is.SchemaVersion()
+	if err != nil {
+		return nil, fmt.Errorf("error getting schema version (need to init database?): %v", err)
+	}
+	if version != requiredSchemaVersion {
+		if os.Getenv("CAMLI_ADVERTISED_PASSWORD") != "" {
+			// Good signal that we're using the dev-server script, so help out
+			// the user with a more useful tip:
+			return nil, fmt.Errorf("database schema version is %d; expect %d (run \"./dev-server --wipe\" to wipe both your blobs and re-populate the database schema)", version, requiredSchemaVersion)
+		}
+		return nil, fmt.Errorf("database schema version is %d; expect %d (need to re-init/upgrade database?)",
+			version, requiredSchemaVersion)
+	}
+
+	ix := index.New(is)
+	ix.BlobSource = sto
+	// Good enough, for now:
+	ix.KeyFetcher = ix.BlobSource
+
+	return ix, nil
+}
+
+func init() {
+	blobserver.RegisterStorageConstructor("sqliteindexer", blobserver.StorageConstructor(newFromConfig))
+}
+
+func (mi *storage) ping() error {
+	// TODO(bradfitz): something more efficient here?
+	_, err := mi.SchemaVersion()
+	return err
+}
+
+func (mi *storage) SchemaVersion() (version int, err error) {
+	err = mi.db.QueryRow("SELECT value FROM meta WHERE metakey='version'").Scan(&version)
+	return
+}

pkg/index/sqlite/sqlite_test.go

@@ -0,0 +1,97 @@
+/*
+Copyright 2012 Google Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package sqlite_test
+
+import (
+	"database/sql"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"sync"
+	"testing"
+
+	"camlistore.org/pkg/index"
+	"camlistore.org/pkg/index/indextest"
+	"camlistore.org/pkg/index/sqlite"
+
+	_ "camlistore.org/third_party/github.com/mattn/go-sqlite3"
+)
+
+var (
+	once        sync.Once
+	dbAvailable bool
+	rootdb      *sql.DB
+)
+
+func checkDB() {
+	var err error
+	if rootdb, err = sql.Open("mymysql", "mysql/root/root"); err == nil {
+		var n int
+		err := rootdb.QueryRow("SELECT COUNT(*) FROM user").Scan(&n)
+		if err == nil {
+			dbAvailable = true
+		}
+	}
+}
+
+func do(db *sql.DB, sql string) {
+	_, err := db.Exec(sql)
+	if err == nil {
+		return
+	}
+	panic(fmt.Sprintf("Error %v running SQL: %s", err, sql))
+}
+
+type sqliteTester struct{}
+
+func (sqliteTester) test(t *testing.T, tfn func(*testing.T, func() *index.Index)) {
+	once.Do(checkDB)
+	f, err := ioutil.TempFile("", "sqlite-test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.Remove(f.Name())
+	makeIndex := func() *index.Index {
+		db, err := sql.Open("sqlite3", f.Name())
+		if err != nil {
+			t.Fatalf("opening test database: %v", err)
+			return nil
+		}
+		for _, tableSql := range sqlite.SQLCreateTables() {
+			do(db, tableSql)
+		}
+		do(db, fmt.Sprintf(`REPLACE INTO meta VALUES ('version', '%d')`, sqlite.SchemaVersion()))
+		s, err := sqlite.NewStorage(f.Name())
+		if err != nil {
+			panic(err)
+		}
+		return index.New(s)
+	}
+	tfn(t, makeIndex)
+}
+
+func TestIndex_SQLite(t *testing.T) {
+	sqliteTester{}.test(t, indextest.Index)
+}
+
+func TestPathsOfSignerTarget_SQLite(t *testing.T) {
+	sqliteTester{}.test(t, indextest.PathsOfSignerTarget)
+}
+
+func TestFiles_SQLite(t *testing.T) {
+	sqliteTester{}.test(t, indextest.Files)
+}