camlistored: much better sqlite support, by default, and help/docs/hints

pkg/index/sqlite/sqlite.go

@@ -45,7 +45,14 @@ func CompiledIn() bool {
 	return compiled
 }
 
-var ErrNotCompiled = errors.New("camlistored was not built with SQLite support. Rebuild with go get/install --tags=with_sqlite.")
+var ErrNotCompiled = errors.New("camlistored was not built with SQLite support. Rebuild with go get/install --tags=with_sqlite " + compileHint())
+
+func compileHint() string {
+	if _, err := os.Stat("/etc/apt"); err == nil {
+		return " (Required: apt-get install libsqlite3-dev)"
+	}
+	return ""
+}
 
 // NewStorage returns an IndexStorage implementation of the described SQLite database.
 // This exists mostly for testing and does not initialize the schema.

pkg/index/sqlite/sqlite_test.go

@@ -37,17 +37,6 @@ var (
 	rootdb      *sql.DB
 )
 
-func checkDB() {
-	var err error
-	if rootdb, err = sql.Open("mymysql", "mysql/root/root"); err == nil {
-		var n int
-		err := rootdb.QueryRow("SELECT COUNT(*) FROM user").Scan(&n)
-		if err == nil {
-			dbAvailable = true
-		}
-	}
-}
-
 func do(db *sql.DB, sql string) {
 	_, err := db.Exec(sql)
 	if err == nil {
@@ -59,7 +48,6 @@ func do(db *sql.DB, sql string) {
 type sqliteTester struct{}
 
 func (sqliteTester) test(t *testing.T, tfn func(*testing.T, func() *index.Index)) {
-	once.Do(checkDB)
 	f, err := ioutil.TempFile("", "sqlite-test")
 	if err != nil {
 		t.Fatal(err)

pkg/osutil/paths.go

@@ -52,14 +52,18 @@ func makeCacheDir() {
 	os.Mkdir(cacheDir(), 0700)
 }
 
-func CamliBlobRoot() string {
+func CamliVarDir() string {
 	switch runtime.GOOS {
 	case "windows":
-		return filepath.Join(os.Getenv("APPDATA"), "Camlistore", "blobs")
+		return filepath.Join(os.Getenv("APPDATA"), "Camlistore")
 	case "darwin":
-		return filepath.Join(HomeDir(), "Library", "Camlistore", "blobs")
+		return filepath.Join(HomeDir(), "Library", "Camlistore")
 	}
-	return filepath.Join(HomeDir(), "var", "camlistore", "blobs")
+	return filepath.Join(HomeDir(), "var", "camlistore")
+}
+
+func CamliBlobRoot() string {
+	return filepath.Join(CamliVarDir(), "blobs")
 }
 
 func CamliConfigDir() string {

pkg/serverconfig/genconfig.go

@@ -175,7 +175,7 @@ func addMemindexConfig(prefixes jsonconfig.Obj) {
 
 func addSQLiteConfig(prefixes jsonconfig.Obj, file string) {
 	ob := map[string]interface{}{}
-	ob["handler"] = "storage-sqlite"
+	ob["handler"] = "storage-sqliteindexer"
 	ob["handlerArgs"] = map[string]interface{}{
 		"blobSource": "/bs/",
 		"file":       file,

pkg/serverconfig/testdata/sqlite-want.json

@@ -76,7 +76,7 @@
 		},
 	
 		"/index-sqlite/": {
-			"handler": "storage-sqlite",
+			"handler": "storage-sqliteindexer",
 			"handlerArgs": {
 				"blobSource": "/bs/",
 				"file": "/tmp/camli.db"

server/camlistored/camlistored.go

@@ -36,6 +36,7 @@ import (
 	"strings"
 	"syscall"
 	"time"
+	"database/sql"
 
 	"camlistore.org/pkg/jsonsign"
 	"camlistore.org/pkg/osutil"
@@ -54,6 +55,7 @@ import (
 	_ "camlistore.org/pkg/index/mongo"
 	_ "camlistore.org/pkg/index/mysql"
 	_ "camlistore.org/pkg/index/postgres"
+	"camlistore.org/pkg/index/sqlite"
 
 	// Handlers:
 	_ "camlistore.org/pkg/search"
@@ -202,6 +204,7 @@ type defaultConfigFile struct {
 	BlobPath           string        `json:"blobPath"`
 	MySQL              string        `json:"mysql"`
 	Mongo              string        `json:"mongo"`
+	SQLite             string        `json:"sqlite"`
 	S3                 string        `json:"s3"`
 	ReplicateTo        []interface{} `json:"replicateTo"`
 	Publish            struct{}      `json:"publish"`
@@ -214,12 +217,12 @@ func newDefaultConfigFile(path string) error {
 		Auth:        "localhost",
 		ReplicateTo: make([]interface{}, 0),
 	}
-
 	blobDir := osutil.CamliBlobRoot()
 	if err := os.MkdirAll(blobDir, 0700); err != nil {
 		return fmt.Errorf("Could not create default blobs directory: %v", err)
 	}
 	conf.BlobPath = blobDir
+	conf.SQLite = filepath.Join(osutil.CamliVarDir(), "camli-index.db")
 
 	var keyId string
 	secRing := osutil.IdentitySecretRing()
@@ -246,9 +249,34 @@ func newDefaultConfigFile(path string) error {
 	if err := ioutil.WriteFile(path, confData, 0600); err != nil {
 		return fmt.Errorf("Could not create or write default server config: %v", err)
 	}
+
+	if sqlite.CompiledIn() {
+		if fi, err := os.Stat(conf.SQLite); os.IsNotExist(err) || (fi != nil && fi.Size() == 0) {
+			if err := initSQLiteDB(conf.SQLite); err != nil {
+				log.Printf("Error initializing DB %s: %v", conf.SQLite, err)
+			}
+		}
+	} else {
+		log.Printf("Wrote config file assuming SQLite, but SQLite is not available. Recompile with SQLite or modify %s and pick an index type.", path)
+	}
 	return nil
 }
 
+func initSQLiteDB(path string) error {
+	db, err := sql.Open("sqlite3", path)
+	if err != nil {
+		return err
+	}
+	defer db.Close()
+	for _, tableSql := range sqlite.SQLCreateTables() {
+		if _, err := db.Exec(tableSql); err != nil {
+			return err
+		}
+	}
+	_, err = db.Exec(fmt.Sprintf(`REPLACE INTO meta VALUES ('version', '%d')`, sqlite.SchemaVersion()))
+	return err
+}
+
 func setupTLS(ws *webserver.Server, config *serverconfig.Config, listen string) {
 	cert, key := config.OptionalString("TLSCertFile", ""), config.OptionalString("TLSKeyFile", "")
 	if !config.OptionalBool("https", true) {