From 501736e54dd2cff7f3b0b9ff901eae2c91b8e3c4 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Sun, 21 Aug 2011 14:14:04 +0400 Subject: [PATCH] db: more Go db work, including intro goals doc Change-Id: I69e914bc533b3f69fba5d869c90337ee2da2fa06 --- .last_go_version | 2 +- lib/go/camli/db/db.go | 102 +++++++++--- lib/go/camli/db/db_test.go | 10 +- lib/go/camli/db/dbimpl/dbimpl.go | 30 +++- lib/go/camli/db/doc.txt | 46 ++++++ lib/go/camli/db/fakedb_test.go | 271 +++++++++++++++++++++++++++++++ lib/go/camli/db/testdb_test.go | 143 ---------------- 7 files changed, 429 insertions(+), 175 deletions(-) create mode 100644 lib/go/camli/db/doc.txt create mode 100644 lib/go/camli/db/fakedb_test.go delete mode 100644 lib/go/camli/db/testdb_test.go diff --git a/.last_go_version b/.last_go_version index 31082b9a0..314fafa0d 100644 --- a/.last_go_version +++ b/.last_go_version @@ -1 +1 @@ -6g version weekly.2011-07-29 9335+ +6g version weekly.2011-08-10 9382 diff --git a/lib/go/camli/db/db.go b/lib/go/camli/db/db.go index dc72fa291..2310c41b7 100644 --- a/lib/go/camli/db/db.go +++ b/lib/go/camli/db/db.go @@ -21,6 +21,7 @@ package db import ( "fmt" "os" + "runtime" "camli/db/dbimpl" ) @@ -40,6 +41,8 @@ func Register(name string, driver dbimpl.Driver) { type DB struct { driver dbimpl.Driver dbarg string + + freeConn []dbimpl.Conn } func Open(driverName, database string) (*DB, os.Error) { @@ -50,11 +53,44 @@ func Open(driverName, database string) (*DB, os.Error) { return &DB{driver: driver, dbarg: database}, nil } +func (db *DB) maxIdleConns() int { + const defaultMaxIdleConns = 2 + // TODO(bradfitz): ask driver, if supported, for its default preference + // TODO(bradfitz): let users override? + return defaultMaxIdleConns +} + +// conn returns a newly-opened or cached dbimpl.Conn func (db *DB) conn() (dbimpl.Conn, os.Error) { + if n := len(db.freeConn); n > 0 { + conn := db.freeConn[n-1] + db.freeConn = db.freeConn[:n-1] + return conn, nil + } return db.driver.Open(db.dbarg) } +func (db *DB) putConn(c dbimpl.Conn) { + if n := len(db.freeConn); n < db.maxIdleConns() { + db.freeConn = append(db.freeConn, c) + return + } + db.closeConn(c) +} + +func (db *DB) closeConn(c dbimpl.Conn) { + // TODO: check to see if we need this Conn for any prepared statements + // that are active. + c.Close() +} + func (db *DB) Prepare(query string) (*Stmt, os.Error) { + // TODO: check if db.driver supports an optional + // dbimpl.Preparer interface and call that instead, if so, + // otherwise we make a prepared statement that's bound + // to a connection, and to execute this prepared statement + // we either need to use this connection (if it's free), else + // get a new connection + re-prepare + execute on that one. ci, err := db.conn() if err != nil { return nil, err @@ -64,33 +100,37 @@ func (db *DB) Prepare(query string) (*Stmt, os.Error) { return nil, err } stmt := &Stmt{ - db: db, - ci: ci, - si: si, + db: db, + query: query, + ci: ci, + si: si, } return stmt, nil } func (db *DB) Exec(query string, args ...interface{}) os.Error { + // TODO(bradfitz): check to see if db.driver implements + // optional dbimpl.Execer interface and use that instead of + // even asking for a connection. conn, err := db.conn() if err != nil { return err } - defer conn.Close() - // TODO(bradfitz): check to see if conn implements optional - // dbimpl.ConnExecer interface and use that instead of - // Prepare+Exec - stmt, err := conn.Prepare(query) + defer db.putConn(conn) + // TODO(bradfitz): check to see if db.driver implements + // optional dbimpl.ConnExecer interface and use that instead + // of Prepare+Exec + sti, err := conn.Prepare(query) if err != nil { return err } - defer stmt.Close() - _, err = stmt.Exec(args) + defer sti.Close() + _, err = sti.Exec(args) return err } func (db *DB) Query(query string, args ...interface{}) (*Rows, os.Error) { - panic("TODO: implement") + panic(todo()) } var ErrNoRows = os.NewError("db: no rows in result set") @@ -104,7 +144,7 @@ func (db *DB) QueryRow(query string, args ...interface{}) *Row { } func (db *DB) Begin() (*Tx, os.Error) { - panic("TODO: implement") + panic(todo()) } // DriverDatabase returns the database's underlying driver. @@ -120,49 +160,57 @@ type Tx struct { } func (tx *Tx) Commit() os.Error { - panic("TODO: implement") + panic(todo()) } func (tx *Tx) Rollback() os.Error { - panic("TODO: implement") + panic(todo()) } func (tx *Tx) Prepare(query string) (*Stmt, os.Error) { - panic("TODO: implement") + panic(todo()) } func (tx *Tx) Exec(query string, args ...interface{}) { - panic("TODO: implement") + panic(todo()) } func (tx *Tx) Query(query string, args ...interface{}) (*Rows, os.Error) { - panic("TODO: implement") + panic(todo()) } func (tx *Tx) QueryRow(query string, args ...interface{}) *Row { - panic("TODO: implement") + panic(todo()) } type Stmt struct { db *DB // where we came from - ci dbimpl.Conn // owned + ci dbimpl.Conn // the Conn that we're bound to. to execute, we need to wait for this Conn to be free. si dbimpl.Stmt // owned + + // query is the query that created the Stmt + query string +} + +func todo() string { + _, file, line, _ := runtime.Caller(1) + return fmt.Sprintf("%s:%d: TODO: implement", file, line) } func (s *Stmt) Exec(args ...interface{}) os.Error { - panic("TODO: implement") + panic(todo()) } func (s *Stmt) Query(args ...interface{}) (*Rows, os.Error) { - panic("TODO: implement") + panic(todo()) } func (s *Stmt) QueryRow(args ...interface{}) *Row { - panic("TODO: implement") + panic(todo()) } func (s *Stmt) Close() os.Error { - panic("TODO: implement") + panic(todo()) } type Rows struct { @@ -170,19 +218,19 @@ type Rows struct { } func (rs *Rows) Next() bool { - panic("TODO: implement") + panic(todo()) } func (rs *Rows) Error() os.Error { - panic("TODO: implement") + panic(todo()) } func (rs *Rows) Scan(dest ...interface{}) os.Error { - panic("TODO: implement") + panic(todo()) } func (rs *Rows) Close() os.Error { - panic("TODO: implement") + panic(todo()) } type Row struct { diff --git a/lib/go/camli/db/db_test.go b/lib/go/camli/db/db_test.go index a4eaa0ec1..ea81b33db 100644 --- a/lib/go/camli/db/db_test.go +++ b/lib/go/camli/db/db_test.go @@ -21,12 +21,16 @@ import ( ) func TestDb(t *testing.T) { - db, err := Open("test", "foo") + db, err := Open("test", "foo;wipe") if err != nil { t.Fatalf("Open: %v", err) } - err = db.Exec("INSERT INTO foo SET col=?", "colval") + err = db.Exec("CREATE|t1|name=string,age=int32,dead=bool") if err != nil { - t.Logf("Exec: %v", err) + t.Errorf("Exec: %v", err) + } + stmt, err := db.Prepare("INSERT|t1|name=?,age=?") + if err != nil { + t.Errorf("Stmt, err = %v, %v", stmt, err) } } diff --git a/lib/go/camli/db/dbimpl/dbimpl.go b/lib/go/camli/db/dbimpl/dbimpl.go index 6f874bb3f..1c5335658 100644 --- a/lib/go/camli/db/dbimpl/dbimpl.go +++ b/lib/go/camli/db/dbimpl/dbimpl.go @@ -28,12 +28,27 @@ import ( // []byte type Driver interface { - Open(name string) (Conn, os.Error) + // Open returns a new or cached connection to the database. + // The dsn parameter, the Data Source Name, contains a + // driver-specific string containing the database name, + // connection parameters, authentication parameters, etc. + // + // The returned connection is only used by one goroutine at a + // time. + Open(dsn string) (Conn, os.Error) } type Conn interface { + // Prepare returns a prepared statement, bound to this connection. Prepare(query string) (Stmt, os.Error) + + // Close invalidates and potentially stops any current + // prepared statements and transactions, marking this + // connection as no longer in use. The driver may cache or + // close its underlying connection to its database. Close() os.Error + + // Begin starts and returns a new transaction. Begin() (Tx, os.Error) } @@ -42,6 +57,7 @@ type Result interface { RowsAffected() (int64, os.Error) } +// Stmt is a prepared statement. It is bound to a Conn. type Stmt interface { Close() os.Error NumInput() int @@ -61,3 +77,15 @@ type Tx interface { Commit() os.Error Rollback() os.Error } + +type ddlSuccess struct{} + +var DDLSuccess Result = ddlSuccess{} + +func (ddlSuccess) AutoIncrementId() (int64, os.Error) { + return 0, os.NewError("no AutoIncrementId available after DDL statement") +} + +func (ddlSuccess) RowsAffected() (int64, os.Error) { + return 0, os.NewError("no RowsAffected available after DDL statement") +} diff --git a/lib/go/camli/db/doc.txt b/lib/go/camli/db/doc.txt new file mode 100644 index 000000000..1d0a46ed1 --- /dev/null +++ b/lib/go/camli/db/doc.txt @@ -0,0 +1,46 @@ +Goals of the db and db/dbimpl packages: + +* Provide a generic database API for a variety of SQL or SQL-like + databases. There currently exist Go libraries for SQLite, MySQL, + and Postgres, but all with a very different feel, and often + a non-Go-like feel. + +* Feel like Go. + +* Care mostly about the common cases. Common SQL should be portable. + SQL edge cases or db-specific extensions can be detected and + conditionally used by the application. It is a non-goal to care + about every particular db's extension or quirk. + +* Separate out the basic implementation of a database driver + (implementing the db/dbimpl interfaces) vs the implementation + of all the user-level types and convenience methods. + In a nutshell: + + User Code ---> db package (concrete types) ---> db/dbimpl (interfaces) + Database Driver -> db (to register) + dbimpl (implement interfaces) + +* To type casting/conversions consistently between all drivers. To + acheive this, most of the type conversions are done in the db + package, not in each driver. The drivers then only have to deal + with a smaller set of types. + +* Be flexible with type conversions, but be paranoid about silent + truncation or other loss of precision. + +* Handle concurrency well. Users shouldn't need to care about the + database's per-connection thread safety issues (or lack thereof), + and shouldn't have to maintain their own free pools of connections. + The 'db' package should deal with that bookkeeping as needed. Given + a *db.DB, it should be possible to share that instance between + multiple goroutines, without any extra synchronization. + +* Push complexity, where necessary, down into the db+dbimpl packages, + rather than exposing it to users. Said otherwise, the db package + should expose an ideal database that's not finnicky about how it's + accessed, even if that's not true. + +* Provide optional interfaces in dbimpl for drivers to implement + for special cases or fastpaths. But the only party that knows about + those is the 'db' package. To user code, some stuff just might start + working or start working slightly faster. diff --git a/lib/go/camli/db/fakedb_test.go b/lib/go/camli/db/fakedb_test.go new file mode 100644 index 000000000..bfc76b9ac --- /dev/null +++ b/lib/go/camli/db/fakedb_test.go @@ -0,0 +1,271 @@ +/* +Copyright 2011 Google Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package db + +import ( + "fmt" + "log" + "os" + "strings" + "sync" + + "camli/db/dbimpl" +) + +var _ = log.Printf + +// fakeDriver is a fake database that implements Go's dbimpl.Driver +// interface, just for testing. +// +// It speaks a query language that's semantically similar to but +// syntantically different and simpler than SQL. The syntax is as +// follows: +// +// CREATE||=,=,... +// where types are: "string", [u]int{8,16,32,64}, "bool" +// INSERT||col=val,col2=val2,col3=? +// +// When opening a a fakeDriver's database, it starts empty with no +// tables. All tables and data are stored in memory only. +type fakeDriver struct { + mu sync.Mutex + openCount int + dbs map[string]*fakeDB +} + +type fakeDB struct { + name string + + mu sync.Mutex + free []*fakeConn + tables map[string]*table +} + +type table struct { + colname []string + coltype []string +} + +type fakeConn struct { + db *fakeDB // where to return ourselves to + + currTx *fakeTx +} + +type fakeTx struct { + c *fakeConn +} + +type fakeStmt struct { + c *fakeConn + q string // just for debugging + + cmd string + table string + + colName []string // used by CREATE, INSERT + colType []string // used by CREATE + colValue []string // used by INSERT (mix of strings and "?" for bound params) +} + +var driver dbimpl.Driver = &fakeDriver{} + +func init() { + Register("test", driver) +} + +// Supports dsn forms: +// +// ;wipe +func (d *fakeDriver) Open(dsn string) (dbimpl.Conn, os.Error) { + d.mu.Lock() + defer d.mu.Unlock() + d.openCount++ + if d.dbs == nil { + d.dbs = make(map[string]*fakeDB) + } + parts := strings.Split(dsn, ";") + if len(parts) < 1 { + return nil, os.NewError("fakedb: no database name") + } + name := parts[0] + db, ok := d.dbs[name] + if !ok { + db = &fakeDB{name: name} + d.dbs[name] = db + } + if len(parts) > 1 && parts[1] == "wipe" { + db.wipe() + } + return &fakeConn{db: db}, nil +} + +func (db *fakeDB) wipe() { + db.mu.Lock() + defer db.mu.Unlock() + db.tables = nil +} + +func (db *fakeDB) createTable(name string, columnNames, columnTypes []string) os.Error { + db.mu.Lock() + defer db.mu.Unlock() + if db.tables == nil { + db.tables = make(map[string]*table) + } + if _, exist := db.tables[name]; exist { + return fmt.Errorf("table %q already exists", name) + } + if len(columnNames) != len(columnTypes) { + return fmt.Errorf("create table of %q len(names) != len(types): %d vs %d", + len(columnNames), len(columnTypes)) + } + db.tables[name] = &table{colname: columnNames, coltype: columnTypes} + return nil +} + +func (db *fakeDB) columnType(table, column string) (typ string, ok bool) { + db.mu.Lock() + defer db.mu.Unlock() + if db.tables == nil { + println("no tables exist") + return + } + t, ok := db.tables[table] + if !ok { + println("table no exist") + return + } + for n, cname := range t.colname { + if cname == column { + return t.coltype[n], true + } + } + return "", false +} + +func (c *fakeConn) Begin() (dbimpl.Tx, os.Error) { + if c.currTx != nil { + return nil, os.NewError("already in a transaction") + } + c.currTx = &fakeTx{c: c} + return c.currTx, nil +} + +func (c *fakeConn) Close() os.Error { + if c.currTx != nil { + return os.NewError("can't close; in a Transaction") + } + if c.db == nil { + return os.NewError("can't close; already closed") + } + c.db = nil + return nil +} + +func errf(msg string, args ...interface{}) os.Error { + return os.NewError("fakedb: " + fmt.Sprintf(msg, args...)) +} + +func (c *fakeConn) Prepare(query string) (dbimpl.Stmt, os.Error) { + if c.db == nil { + panic("nil c.db; conn = " + fmt.Sprintf("%#v", c)) + } + parts := strings.Split(query, "|") + if len(parts) < 1 { + return nil, errf("empty query") + } + cmd := parts[0] + stmt := &fakeStmt{q: query, c: c, cmd: cmd} + switch cmd { + case "CREATE": + if len(parts) != 3 { + return nil, errf("invalid %q syntax with %d parts; want 3", cmd, len(parts)) + } + stmt.table = parts[1] + for n, colspec := range strings.Split(parts[2], ",") { + nameType := strings.Split(colspec, "=") + if len(nameType) != 2 { + return nil, errf("CREATE table %q has invalid column spec of %q (index %d)", stmt.table, colspec, n) + } + stmt.colName = append(stmt.colName, nameType[0]) + stmt.colType = append(stmt.colType, nameType[1]) + } + case "INSERT": + if len(parts) != 3 { + return nil, errf("invalid %q syntax with %d parts; want 3", cmd, len(parts)) + } + stmt.table = parts[1] + for n, colspec := range strings.Split(parts[2], ",") { + nameVal := strings.Split(colspec, "=") + if len(nameVal) != 2 { + return nil, errf("INSERT table %q has invalid column spec of %q (index %d)", stmt.table, colspec, n) + } + column, value := nameVal[0], nameVal[1] + ctype, ok := c.db.columnType(stmt.table, column) + if !ok { + return nil, errf("INSERT table %q references non-existent column %q", stmt.table, column) + } + if value != "?" { + // TODO(bradfitz): check that + // pre-bound value type conversion is + // valid for this column type + _ = ctype + } + stmt.colName = append(stmt.colName, column) + stmt.colValue = append(stmt.colValue, value) + } + default: + return nil, errf("unsupported command type %q", cmd) + } + return stmt, nil +} + +func (s *fakeStmt) Close() os.Error { + return nil +} + +func (s *fakeStmt) Exec(args []interface{}) (dbimpl.Result, os.Error) { + switch s.cmd { + case "CREATE": + if err := s.c.db.createTable(s.table, s.colName, s.colType); err != nil { + return nil, err + } + return dbimpl.DDLSuccess, nil + } + fmt.Printf("EXEC statement, cmd=%q: %#v\n", s.cmd, s) + return nil, fmt.Errorf("unimplemented statement Exec command type of %q", s.cmd) +} + +func (s *fakeStmt) Query(args []interface{}) (dbimpl.Rows, os.Error) { + println("QUERY") + fmt.Println(args...) + return nil, os.NewError(todo()) +} + +func (s *fakeStmt) NumInput() int { + return 0 +} + +func (tx *fakeTx) Commit() os.Error { + tx.c.currTx = nil + return nil +} + +func (tx *fakeTx) Rollback() os.Error { + tx.c.currTx = nil + return nil +} diff --git a/lib/go/camli/db/testdb_test.go b/lib/go/camli/db/testdb_test.go deleted file mode 100644 index 9377083c3..000000000 --- a/lib/go/camli/db/testdb_test.go +++ /dev/null @@ -1,143 +0,0 @@ -/* -Copyright 2011 Google Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package db - -import ( - "fmt" - "os" - "sync" - - "camli/db/dbimpl" -) - -type testDriver struct { - mu sync.Mutex - dbs map[string]*testDB -} - -type testDB struct { - name string - - mu sync.Mutex - free []*testConn -} - -type testConn struct { - db *testDB // where to return ourselves to - - currTx *testTx -} - -type testTx struct { - c *testConn -} - -type testStmt struct { - q string - c *testConn -} - -var driver dbimpl.Driver = &testDriver{} - -func init() { - Register("test", driver) -} - -func (d *testDriver) Open(name string) (dbimpl.Conn, os.Error) { - d.mu.Lock() - defer d.mu.Unlock() - if d.dbs == nil { - d.dbs = make(map[string]*testDB) - } - db, ok := d.dbs[name] - if !ok { - db = &testDB{name: name} - d.dbs[name] = db - } - return db.conn() -} - -func (db *testDB) returnConn(c *testConn) { - db.mu.Lock() - defer db.mu.Unlock() - db.free = append(db.free, c) -} - -func (db *testDB) conn() (dbimpl.Conn, os.Error) { - db.mu.Lock() - defer db.mu.Unlock() - if len(db.free) > 0 { - conn := db.free[len(db.free)-1] - db.free = db.free[:len(db.free)-1] - return conn, nil - } - return &testConn{db: db}, nil -} - -func (c *testConn) Begin() (dbimpl.Tx, os.Error) { - if c.currTx != nil { - return nil, os.NewError("already in a transaction") - } - c.currTx = &testTx{c: c} - return c.currTx, nil -} - -func (c *testConn) Close() os.Error { - if c.currTx != nil { - return os.NewError("can't close; in a Transaction") - } - if c.db == nil { - return os.NewError("can't close; already closed") - } - c.db.returnConn(c) - c.db = nil - return nil -} - -func (c *testConn) Prepare(query string) (dbimpl.Stmt, os.Error) { - fmt.Printf("Prepare: %q\n", query) - return &testStmt{q: query, c: c}, nil -} - -func (s *testStmt) Close() os.Error { - return nil -} - -func (s *testStmt) Exec(args []interface{}) (dbimpl.Result, os.Error) { - fmt.Printf("EXEC(%#v)\n", args) - return nil, os.NewError("TODO: implement") -} - -func (s *testStmt) Query(args []interface{}) (dbimpl.Rows, os.Error) { - println("QUERY") - fmt.Println(args...) - return nil, os.NewError("TODO: implement") -} - -func (s *testStmt) NumInput() int { - return 0 -} - -func (tx *testTx) Commit() os.Error { - tx.c.currTx = nil - return nil -} - -func (tx *testTx) Rollback() os.Error { - tx.c.currTx = nil - return nil -}