From 6ca0efac8c8e04609e73f483347a50e63119852b Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Thu, 10 Jan 2013 16:16:10 -0800 Subject: [PATCH] camlistored: much better sqlite support, by default, and help/docs/hints --- pkg/index/sqlite/sqlite.go | 9 ++++++- pkg/index/sqlite/sqlite_test.go | 12 --------- pkg/osutil/paths.go | 12 ++++++--- pkg/serverconfig/genconfig.go | 2 +- pkg/serverconfig/testdata/sqlite-want.json | 2 +- server/camlistored/camlistored.go | 30 +++++++++++++++++++++- 6 files changed, 47 insertions(+), 20 deletions(-) diff --git a/pkg/index/sqlite/sqlite.go b/pkg/index/sqlite/sqlite.go index fe598d636..ecfc54d94 100644 --- a/pkg/index/sqlite/sqlite.go +++ b/pkg/index/sqlite/sqlite.go @@ -45,7 +45,14 @@ func CompiledIn() bool { return compiled } -var ErrNotCompiled = errors.New("camlistored was not built with SQLite support. Rebuild with go get/install --tags=with_sqlite.") +var ErrNotCompiled = errors.New("camlistored was not built with SQLite support. Rebuild with go get/install --tags=with_sqlite " + compileHint()) + +func compileHint() string { + if _, err := os.Stat("/etc/apt"); err == nil { + return " (Required: apt-get install libsqlite3-dev)" + } + return "" +} // NewStorage returns an IndexStorage implementation of the described SQLite database. // This exists mostly for testing and does not initialize the schema. diff --git a/pkg/index/sqlite/sqlite_test.go b/pkg/index/sqlite/sqlite_test.go index fbd42a3e7..2c0e584d2 100644 --- a/pkg/index/sqlite/sqlite_test.go +++ b/pkg/index/sqlite/sqlite_test.go @@ -37,17 +37,6 @@ var ( rootdb *sql.DB ) -func checkDB() { - var err error - if rootdb, err = sql.Open("mymysql", "mysql/root/root"); err == nil { - var n int - err := rootdb.QueryRow("SELECT COUNT(*) FROM user").Scan(&n) - if err == nil { - dbAvailable = true - } - } -} - func do(db *sql.DB, sql string) { _, err := db.Exec(sql) if err == nil { @@ -59,7 +48,6 @@ func do(db *sql.DB, sql string) { type sqliteTester struct{} func (sqliteTester) test(t *testing.T, tfn func(*testing.T, func() *index.Index)) { - once.Do(checkDB) f, err := ioutil.TempFile("", "sqlite-test") if err != nil { t.Fatal(err) diff --git a/pkg/osutil/paths.go b/pkg/osutil/paths.go index f2c73ebaf..60f9685a8 100644 --- a/pkg/osutil/paths.go +++ b/pkg/osutil/paths.go @@ -52,14 +52,18 @@ func makeCacheDir() { os.Mkdir(cacheDir(), 0700) } -func CamliBlobRoot() string { +func CamliVarDir() string { switch runtime.GOOS { case "windows": - return filepath.Join(os.Getenv("APPDATA"), "Camlistore", "blobs") + return filepath.Join(os.Getenv("APPDATA"), "Camlistore") case "darwin": - return filepath.Join(HomeDir(), "Library", "Camlistore", "blobs") + return filepath.Join(HomeDir(), "Library", "Camlistore") } - return filepath.Join(HomeDir(), "var", "camlistore", "blobs") + return filepath.Join(HomeDir(), "var", "camlistore") +} + +func CamliBlobRoot() string { + return filepath.Join(CamliVarDir(), "blobs") } func CamliConfigDir() string { diff --git a/pkg/serverconfig/genconfig.go b/pkg/serverconfig/genconfig.go index 50ed2f615..a520b1341 100644 --- a/pkg/serverconfig/genconfig.go +++ b/pkg/serverconfig/genconfig.go @@ -175,7 +175,7 @@ func addMemindexConfig(prefixes jsonconfig.Obj) { func addSQLiteConfig(prefixes jsonconfig.Obj, file string) { ob := map[string]interface{}{} - ob["handler"] = "storage-sqlite" + ob["handler"] = "storage-sqliteindexer" ob["handlerArgs"] = map[string]interface{}{ "blobSource": "/bs/", "file": file, diff --git a/pkg/serverconfig/testdata/sqlite-want.json b/pkg/serverconfig/testdata/sqlite-want.json index 7b9483bed..93a379d2c 100644 --- a/pkg/serverconfig/testdata/sqlite-want.json +++ b/pkg/serverconfig/testdata/sqlite-want.json @@ -76,7 +76,7 @@ }, "/index-sqlite/": { - "handler": "storage-sqlite", + "handler": "storage-sqliteindexer", "handlerArgs": { "blobSource": "/bs/", "file": "/tmp/camli.db" diff --git a/server/camlistored/camlistored.go b/server/camlistored/camlistored.go index 30890b91e..2732ce02c 100644 --- a/server/camlistored/camlistored.go +++ b/server/camlistored/camlistored.go @@ -36,6 +36,7 @@ import ( "strings" "syscall" "time" + "database/sql" "camlistore.org/pkg/jsonsign" "camlistore.org/pkg/osutil" @@ -54,6 +55,7 @@ import ( _ "camlistore.org/pkg/index/mongo" _ "camlistore.org/pkg/index/mysql" _ "camlistore.org/pkg/index/postgres" + "camlistore.org/pkg/index/sqlite" // Handlers: _ "camlistore.org/pkg/search" @@ -202,6 +204,7 @@ type defaultConfigFile struct { BlobPath string `json:"blobPath"` MySQL string `json:"mysql"` Mongo string `json:"mongo"` + SQLite string `json:"sqlite"` S3 string `json:"s3"` ReplicateTo []interface{} `json:"replicateTo"` Publish struct{} `json:"publish"` @@ -214,12 +217,12 @@ func newDefaultConfigFile(path string) error { Auth: "localhost", ReplicateTo: make([]interface{}, 0), } - blobDir := osutil.CamliBlobRoot() if err := os.MkdirAll(blobDir, 0700); err != nil { return fmt.Errorf("Could not create default blobs directory: %v", err) } conf.BlobPath = blobDir + conf.SQLite = filepath.Join(osutil.CamliVarDir(), "camli-index.db") var keyId string secRing := osutil.IdentitySecretRing() @@ -246,9 +249,34 @@ func newDefaultConfigFile(path string) error { if err := ioutil.WriteFile(path, confData, 0600); err != nil { return fmt.Errorf("Could not create or write default server config: %v", err) } + + if sqlite.CompiledIn() { + if fi, err := os.Stat(conf.SQLite); os.IsNotExist(err) || (fi != nil && fi.Size() == 0) { + if err := initSQLiteDB(conf.SQLite); err != nil { + log.Printf("Error initializing DB %s: %v", conf.SQLite, err) + } + } + } else { + log.Printf("Wrote config file assuming SQLite, but SQLite is not available. Recompile with SQLite or modify %s and pick an index type.", path) + } return nil } +func initSQLiteDB(path string) error { + db, err := sql.Open("sqlite3", path) + if err != nil { + return err + } + defer db.Close() + for _, tableSql := range sqlite.SQLCreateTables() { + if _, err := db.Exec(tableSql); err != nil { + return err + } + } + _, err = db.Exec(fmt.Sprintf(`REPLACE INTO meta VALUES ('version', '%d')`, sqlite.SchemaVersion())) + return err +} + func setupTLS(ws *webserver.Server, config *serverconfig.Config, listen string) { cert, key := config.OptionalString("TLSCertFile", ""), config.OptionalString("TLSKeyFile", "") if !config.OptionalBool("https", true) {