From a519105585dc160d9bb9f28c47e2739d2531fe5f Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Fri, 13 Apr 2012 16:11:16 -0700 Subject: [PATCH] index/sqlite: new indexer, using sqlite3 Change-Id: If1405fe3ee7c632c05431f4b0be1077d6d42c4a3 --- pkg/index/sqlite/dbschema.go | 35 ++++++++++ pkg/index/sqlite/sqlite.go | 114 ++++++++++++++++++++++++++++++++ pkg/index/sqlite/sqlite_test.go | 97 +++++++++++++++++++++++++++ 3 files changed, 246 insertions(+) create mode 100644 pkg/index/sqlite/dbschema.go create mode 100644 pkg/index/sqlite/sqlite.go create mode 100644 pkg/index/sqlite/sqlite_test.go diff --git a/pkg/index/sqlite/dbschema.go b/pkg/index/sqlite/dbschema.go new file mode 100644 index 000000000..cfb1f1fc0 --- /dev/null +++ b/pkg/index/sqlite/dbschema.go @@ -0,0 +1,35 @@ +/* +Copyright 2012 Google Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package sqlite + +const requiredSchemaVersion = 1 + +func SchemaVersion() int { + return requiredSchemaVersion +} + +func SQLCreateTables() []string { + return []string{ + `CREATE TABLE rows ( + k VARCHAR(255) NOT NULL PRIMARY KEY, + v VARCHAR(255))`, + + `CREATE TABLE meta ( + metakey VARCHAR(255) NOT NULL PRIMARY KEY, + value VARCHAR(255) NOT NULL)`, + } +} diff --git a/pkg/index/sqlite/sqlite.go b/pkg/index/sqlite/sqlite.go new file mode 100644 index 000000000..19f60a1a9 --- /dev/null +++ b/pkg/index/sqlite/sqlite.go @@ -0,0 +1,114 @@ +/* +Copyright 2012 Google Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package sqlite + +import ( + "database/sql" + "fmt" + "os" + + "camlistore.org/pkg/blobserver" + "camlistore.org/pkg/index" + "camlistore.org/pkg/index/sqlindex" + "camlistore.org/pkg/jsonconfig" + + _ "camlistore.org/third_party/github.com/mattn/go-sqlite3" +) + +type storage struct { + *sqlindex.Storage + + file string + db *sql.DB +} + +var _ index.IndexStorage = (*storage)(nil) + +// NewStorage returns an IndexStorage implementation of the described SQLite database. +// This exists mostly for testing and does not initialize the schema. +func NewStorage(file string) (index.IndexStorage, error) { + db, err := sql.Open("sqlite3", file) + if err != nil { + return nil, err + } + // TODO(bradfitz): ping db, check that it's reachable. + return &storage{ + file: file, + db: db, + Storage: &sqlindex.Storage{ + DB: db, + }, + }, nil +} + +func newFromConfig(ld blobserver.Loader, config jsonconfig.Obj) (blobserver.Storage, error) { + var ( + blobPrefix = config.RequiredString("blobSource") + file = config.RequiredString("file") + ) + if err := config.Validate(); err != nil { + return nil, err + } + sto, err := ld.GetStorage(blobPrefix) + if err != nil { + return nil, err + } + isto, err := NewStorage(file) + if err != nil { + return nil, err + } + is := isto.(*storage) + if err := is.ping(); err != nil { + return nil, err + } + + version, err := is.SchemaVersion() + if err != nil { + return nil, fmt.Errorf("error getting schema version (need to init database?): %v", err) + } + if version != requiredSchemaVersion { + if os.Getenv("CAMLI_ADVERTISED_PASSWORD") != "" { + // Good signal that we're using the dev-server script, so help out + // the user with a more useful tip: + return nil, fmt.Errorf("database schema version is %d; expect %d (run \"./dev-server --wipe\" to wipe both your blobs and re-populate the database schema)", version, requiredSchemaVersion) + } + return nil, fmt.Errorf("database schema version is %d; expect %d (need to re-init/upgrade database?)", + version, requiredSchemaVersion) + } + + ix := index.New(is) + ix.BlobSource = sto + // Good enough, for now: + ix.KeyFetcher = ix.BlobSource + + return ix, nil +} + +func init() { + blobserver.RegisterStorageConstructor("sqliteindexer", blobserver.StorageConstructor(newFromConfig)) +} + +func (mi *storage) ping() error { + // TODO(bradfitz): something more efficient here? + _, err := mi.SchemaVersion() + return err +} + +func (mi *storage) SchemaVersion() (version int, err error) { + err = mi.db.QueryRow("SELECT value FROM meta WHERE metakey='version'").Scan(&version) + return +} diff --git a/pkg/index/sqlite/sqlite_test.go b/pkg/index/sqlite/sqlite_test.go new file mode 100644 index 000000000..bb6a781ff --- /dev/null +++ b/pkg/index/sqlite/sqlite_test.go @@ -0,0 +1,97 @@ +/* +Copyright 2012 Google Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package sqlite_test + +import ( + "database/sql" + "fmt" + "io/ioutil" + "os" + "sync" + "testing" + + "camlistore.org/pkg/index" + "camlistore.org/pkg/index/indextest" + "camlistore.org/pkg/index/sqlite" + + _ "camlistore.org/third_party/github.com/mattn/go-sqlite3" +) + +var ( + once sync.Once + dbAvailable bool + rootdb *sql.DB +) + +func checkDB() { + var err error + if rootdb, err = sql.Open("mymysql", "mysql/root/root"); err == nil { + var n int + err := rootdb.QueryRow("SELECT COUNT(*) FROM user").Scan(&n) + if err == nil { + dbAvailable = true + } + } +} + +func do(db *sql.DB, sql string) { + _, err := db.Exec(sql) + if err == nil { + return + } + panic(fmt.Sprintf("Error %v running SQL: %s", err, sql)) +} + +type sqliteTester struct{} + +func (sqliteTester) test(t *testing.T, tfn func(*testing.T, func() *index.Index)) { + once.Do(checkDB) + f, err := ioutil.TempFile("", "sqlite-test") + if err != nil { + t.Fatal(err) + } + defer os.Remove(f.Name()) + makeIndex := func() *index.Index { + db, err := sql.Open("sqlite3", f.Name()) + if err != nil { + t.Fatalf("opening test database: %v", err) + return nil + } + for _, tableSql := range sqlite.SQLCreateTables() { + do(db, tableSql) + } + do(db, fmt.Sprintf(`REPLACE INTO meta VALUES ('version', '%d')`, sqlite.SchemaVersion())) + s, err := sqlite.NewStorage(f.Name()) + if err != nil { + panic(err) + } + return index.New(s) + } + tfn(t, makeIndex) +} + +func TestIndex_SQLite(t *testing.T) { + sqliteTester{}.test(t, indextest.Index) +} + +func TestPathsOfSignerTarget_SQLite(t *testing.T) { + sqliteTester{}.test(t, indextest.PathsOfSignerTarget) +} + +func TestFiles_SQLite(t *testing.T) { + sqliteTester{}.test(t, indextest.Files) +}