Rooba
/
perkeep
mirror of https://github.com/perkeep/perkeep.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

db: more Go db work, including intro goals doc 

Change-Id: I69e914bc533b3f69fba5d869c90337ee2da2fa06
This commit is contained in:
Brad Fitzpatrick 2011-08-21 14:14:04 +04:00
parent 27b2c7e3eb
commit 501736e54d
7 changed files with 429 additions and 175 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.last_go_version
View File

@ -1 +1 @@
6g version weekly.2011-07-29 9335+
6g version weekly.2011-08-10 9382

102
lib/go/camli/db/db.go
View File

@ -21,6 +21,7 @@ package db
import (
"fmt"
"os"
"runtime"
"camli/db/dbimpl"
)
@ -40,6 +41,8 @@ func Register(name string, driver dbimpl.Driver) {
type DB struct {
driver dbimpl.Driver
dbarg string
freeConn []dbimpl.Conn
}
func Open(driverName, database string) (*DB, os.Error) {
@ -50,11 +53,44 @@ func Open(driverName, database string) (*DB, os.Error) {
return &DB{driver: driver, dbarg: database}, nil
}
func (db *DB) maxIdleConns() int {
const defaultMaxIdleConns = 2
// TODO(bradfitz): ask driver, if supported, for its default preference
// TODO(bradfitz): let users override?
return defaultMaxIdleConns
}
// conn returns a newly-opened or cached dbimpl.Conn
func (db *DB) conn() (dbimpl.Conn, os.Error) {
if n := len(db.freeConn); n > 0 {
conn := db.freeConn[n-1]
db.freeConn = db.freeConn[:n-1]
return conn, nil
}
return db.driver.Open(db.dbarg)
}
func (db *DB) putConn(c dbimpl.Conn) {
if n := len(db.freeConn); n < db.maxIdleConns() {
db.freeConn = append(db.freeConn, c)
return
}
db.closeConn(c)
}
func (db *DB) closeConn(c dbimpl.Conn) {
// TODO: check to see if we need this Conn for any prepared statements
// that are active.
c.Close()
}
func (db *DB) Prepare(query string) (*Stmt, os.Error) {
// TODO: check if db.driver supports an optional
// dbimpl.Preparer interface and call that instead, if so,
// otherwise we make a prepared statement that's bound
// to a connection, and to execute this prepared statement
// we either need to use this connection (if it's free), else
// get a new connection + re-prepare + execute on that one.
ci, err := db.conn()
if err != nil {
return nil, err
@ -64,33 +100,37 @@ func (db *DB) Prepare(query string) (*Stmt, os.Error) {
return nil, err
}
stmt := &Stmt{
db: db,
ci: ci,
si: si,
db: db,
query: query,
ci: ci,
si: si,
}
return stmt, nil
}
func (db *DB) Exec(query string, args ...interface{}) os.Error {
// TODO(bradfitz): check to see if db.driver implements
// optional dbimpl.Execer interface and use that instead of
// even asking for a connection.
conn, err := db.conn()
if err != nil {
return err
}
defer conn.Close()
// TODO(bradfitz): check to see if conn implements optional
// dbimpl.ConnExecer interface and use that instead of
// Prepare+Exec
stmt, err := conn.Prepare(query)
defer db.putConn(conn)
// TODO(bradfitz): check to see if db.driver implements
// optional dbimpl.ConnExecer interface and use that instead
// of Prepare+Exec
sti, err := conn.Prepare(query)
if err != nil {
return err
}
defer stmt.Close()
_, err = stmt.Exec(args)
defer sti.Close()
_, err = sti.Exec(args)
return err
}
func (db *DB) Query(query string, args ...interface{}) (*Rows, os.Error) {
panic("TODO: implement")
panic(todo())
}
var ErrNoRows = os.NewError("db: no rows in result set")
@ -104,7 +144,7 @@ func (db *DB) QueryRow(query string, args ...interface{}) *Row {
}
func (db *DB) Begin() (*Tx, os.Error) {
panic("TODO: implement")
panic(todo())
}
// DriverDatabase returns the database's underlying driver.
@ -120,49 +160,57 @@ type Tx struct {
}
func (tx *Tx) Commit() os.Error {
panic("TODO: implement")
panic(todo())
}
func (tx *Tx) Rollback() os.Error {
panic("TODO: implement")
panic(todo())
}
func (tx *Tx) Prepare(query string) (*Stmt, os.Error) {
panic("TODO: implement")
panic(todo())
}
func (tx *Tx) Exec(query string, args ...interface{}) {
panic("TODO: implement")
panic(todo())
}
func (tx *Tx) Query(query string, args ...interface{}) (*Rows, os.Error) {
panic("TODO: implement")
panic(todo())
}
func (tx *Tx) QueryRow(query string, args ...interface{}) *Row {
panic("TODO: implement")
panic(todo())
}
type Stmt struct {
db *DB // where we came from
ci dbimpl.Conn // owned
ci dbimpl.Conn // the Conn that we're bound to. to execute, we need to wait for this Conn to be free.
si dbimpl.Stmt // owned
// query is the query that created the Stmt
query string
}
func todo() string {
_, file, line, _ := runtime.Caller(1)
return fmt.Sprintf("%s:%d: TODO: implement", file, line)
}
func (s *Stmt) Exec(args ...interface{}) os.Error {
panic("TODO: implement")
panic(todo())
}
func (s *Stmt) Query(args ...interface{}) (*Rows, os.Error) {
panic("TODO: implement")
panic(todo())
}
func (s *Stmt) QueryRow(args ...interface{}) *Row {
panic("TODO: implement")
panic(todo())
}
func (s *Stmt) Close() os.Error {
panic("TODO: implement")
panic(todo())
}
type Rows struct {
@ -170,19 +218,19 @@ type Rows struct {
}
func (rs *Rows) Next() bool {
panic("TODO: implement")
panic(todo())
}
func (rs *Rows) Error() os.Error {
panic("TODO: implement")
panic(todo())
}
func (rs *Rows) Scan(dest ...interface{}) os.Error {
panic("TODO: implement")
panic(todo())
}
func (rs *Rows) Close() os.Error {
panic("TODO: implement")
panic(todo())
}
type Row struct {

10
lib/go/camli/db/db_test.go
View File

@ -21,12 +21,16 @@ import (
)
func TestDb(t *testing.T) {
db, err := Open("test", "foo")
db, err := Open("test", "foo;wipe")
if err != nil {
t.Fatalf("Open: %v", err)
}
err = db.Exec("INSERT INTO foo SET col=?", "colval")
err = db.Exec("CREATE|t1|name=string,age=int32,dead=bool")
if err != nil {
t.Logf("Exec: %v", err)
t.Errorf("Exec: %v", err)
}
stmt, err := db.Prepare("INSERT|t1|name=?,age=?")
if err != nil {
t.Errorf("Stmt, err = %v, %v", stmt, err)
}
}

30
lib/go/camli/db/dbimpl/dbimpl.go
View File

@ -28,12 +28,27 @@ import (
// []byte
type Driver interface {
Open(name string) (Conn, os.Error)
// Open returns a new or cached connection to the database.
// The dsn parameter, the Data Source Name, contains a
// driver-specific string containing the database name,
// connection parameters, authentication parameters, etc.
//
// The returned connection is only used by one goroutine at a
// time.
Open(dsn string) (Conn, os.Error)
}
type Conn interface {
// Prepare returns a prepared statement, bound to this connection.
Prepare(query string) (Stmt, os.Error)
// Close invalidates and potentially stops any current
// prepared statements and transactions, marking this
// connection as no longer in use. The driver may cache or
// close its underlying connection to its database.
Close() os.Error
// Begin starts and returns a new transaction.
Begin() (Tx, os.Error)
}
@ -42,6 +57,7 @@ type Result interface {
RowsAffected() (int64, os.Error)
}
// Stmt is a prepared statement. It is bound to a Conn.
type Stmt interface {
Close() os.Error
NumInput() int
@ -61,3 +77,15 @@ type Tx interface {
Commit() os.Error
Rollback() os.Error
}
type ddlSuccess struct{}
var DDLSuccess Result = ddlSuccess{}
func (ddlSuccess) AutoIncrementId() (int64, os.Error) {
return 0, os.NewError("no AutoIncrementId available after DDL statement")
}
func (ddlSuccess) RowsAffected() (int64, os.Error) {
return 0, os.NewError("no RowsAffected available after DDL statement")
}

46
lib/go/camli/db/doc.txt Normal file
View File

@ -0,0 +1,46 @@
Goals of the db and db/dbimpl packages:
* Provide a generic database API for a variety of SQL or SQL-like
databases. There currently exist Go libraries for SQLite, MySQL,
and Postgres, but all with a very different feel, and often
a non-Go-like feel.
* Feel like Go.
* Care mostly about the common cases. Common SQL should be portable.
SQL edge cases or db-specific extensions can be detected and
conditionally used by the application. It is a non-goal to care
about every particular db's extension or quirk.
* Separate out the basic implementation of a database driver
(implementing the db/dbimpl interfaces) vs the implementation
of all the user-level types and convenience methods.
In a nutshell:
User Code ---> db package (concrete types) ---> db/dbimpl (interfaces)
Database Driver -> db (to register) + dbimpl (implement interfaces)
* To type casting/conversions consistently between all drivers. To
acheive this, most of the type conversions are done in the db
package, not in each driver. The drivers then only have to deal
with a smaller set of types.
* Be flexible with type conversions, but be paranoid about silent
truncation or other loss of precision.
* Handle concurrency well. Users shouldn't need to care about the
database's per-connection thread safety issues (or lack thereof),
and shouldn't have to maintain their own free pools of connections.
The 'db' package should deal with that bookkeeping as needed. Given
a *db.DB, it should be possible to share that instance between
multiple goroutines, without any extra synchronization.
* Push complexity, where necessary, down into the db+dbimpl packages,
rather than exposing it to users. Said otherwise, the db package
should expose an ideal database that's not finnicky about how it's
accessed, even if that's not true.
* Provide optional interfaces in dbimpl for drivers to implement
for special cases or fastpaths. But the only party that knows about
those is the 'db' package. To user code, some stuff just might start
working or start working slightly faster.

271
lib/go/camli/db/fakedb_test.go Normal file
View File

@ -0,0 +1,271 @@
/*
Copyright 2011 Google Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package db
import (
"fmt"
"log"
"os"
"strings"
"sync"
"camli/db/dbimpl"
)
var _ = log.Printf
// fakeDriver is a fake database that implements Go's dbimpl.Driver
// interface, just for testing.
//
// It speaks a query language that's semantically similar to but
// syntantically different and simpler than SQL. The syntax is as
// follows:
//
// CREATE|<tablename>|<col>=<type>,<col>=<type>,...
// where types are: "string", [u]int{8,16,32,64}, "bool"
// INSERT|<tablename>|col=val,col2=val2,col3=?
//
// When opening a a fakeDriver's database, it starts empty with no
// tables. All tables and data are stored in memory only.
type fakeDriver struct {
mu sync.Mutex
openCount int
dbs map[string]*fakeDB
}
type fakeDB struct {
name string
mu sync.Mutex
free []*fakeConn
tables map[string]*table
}
type table struct {
colname []string
coltype []string
}
type fakeConn struct {
db *fakeDB // where to return ourselves to
currTx *fakeTx
}
type fakeTx struct {
c *fakeConn
}
type fakeStmt struct {
c *fakeConn
q string // just for debugging
cmd string
table string
colName []string // used by CREATE, INSERT
colType []string // used by CREATE
colValue []string // used by INSERT (mix of strings and "?" for bound params)
}
var driver dbimpl.Driver = &fakeDriver{}
func init() {
Register("test", driver)
}
// Supports dsn forms:
// <dbname>
// <dbname>;wipe
func (d *fakeDriver) Open(dsn string) (dbimpl.Conn, os.Error) {
d.mu.Lock()
defer d.mu.Unlock()
d.openCount++
if d.dbs == nil {
d.dbs = make(map[string]*fakeDB)
}
parts := strings.Split(dsn, ";")
if len(parts) < 1 {
return nil, os.NewError("fakedb: no database name")
}
name := parts[0]
db, ok := d.dbs[name]
if !ok {
db = &fakeDB{name: name}
d.dbs[name] = db
}
if len(parts) > 1 && parts[1] == "wipe" {
db.wipe()
}
return &fakeConn{db: db}, nil
}
func (db *fakeDB) wipe() {
db.mu.Lock()
defer db.mu.Unlock()
db.tables = nil
}
func (db *fakeDB) createTable(name string, columnNames, columnTypes []string) os.Error {
db.mu.Lock()
defer db.mu.Unlock()
if db.tables == nil {
db.tables = make(map[string]*table)
}
if _, exist := db.tables[name]; exist {
return fmt.Errorf("table %q already exists", name)
}
if len(columnNames) != len(columnTypes) {
return fmt.Errorf("create table of %q len(names) != len(types): %d vs %d",
len(columnNames), len(columnTypes))
}
db.tables[name] = &table{colname: columnNames, coltype: columnTypes}
return nil
}
func (db *fakeDB) columnType(table, column string) (typ string, ok bool) {
db.mu.Lock()
defer db.mu.Unlock()
if db.tables == nil {
println("no tables exist")
return
}
t, ok := db.tables[table]
if !ok {
println("table no exist")
return
}
for n, cname := range t.colname {
if cname == column {
return t.coltype[n], true
}
}
return "", false
}
func (c *fakeConn) Begin() (dbimpl.Tx, os.Error) {
if c.currTx != nil {
return nil, os.NewError("already in a transaction")
}
c.currTx = &fakeTx{c: c}
return c.currTx, nil
}
func (c *fakeConn) Close() os.Error {
if c.currTx != nil {
return os.NewError("can't close; in a Transaction")
}
if c.db == nil {
return os.NewError("can't close; already closed")
}
c.db = nil
return nil
}
func errf(msg string, args ...interface{}) os.Error {
return os.NewError("fakedb: " + fmt.Sprintf(msg, args...))
}
func (c *fakeConn) Prepare(query string) (dbimpl.Stmt, os.Error) {
if c.db == nil {
panic("nil c.db; conn = " + fmt.Sprintf("%#v", c))
}
parts := strings.Split(query, "|")
if len(parts) < 1 {
return nil, errf("empty query")
}
cmd := parts[0]
stmt := &fakeStmt{q: query, c: c, cmd: cmd}
switch cmd {
case "CREATE":
if len(parts) != 3 {
return nil, errf("invalid %q syntax with %d parts; want 3", cmd, len(parts))
}
stmt.table = parts[1]
for n, colspec := range strings.Split(parts[2], ",") {
nameType := strings.Split(colspec, "=")
if len(nameType) != 2 {
return nil, errf("CREATE table %q has invalid column spec of %q (index %d)", stmt.table, colspec, n)
}
stmt.colName = append(stmt.colName, nameType[0])
stmt.colType = append(stmt.colType, nameType[1])
}
case "INSERT":
if len(parts) != 3 {
return nil, errf("invalid %q syntax with %d parts; want 3", cmd, len(parts))
}
stmt.table = parts[1]
for n, colspec := range strings.Split(parts[2], ",") {
nameVal := strings.Split(colspec, "=")
if len(nameVal) != 2 {
return nil, errf("INSERT table %q has invalid column spec of %q (index %d)", stmt.table, colspec, n)
}
column, value := nameVal[0], nameVal[1]
ctype, ok := c.db.columnType(stmt.table, column)
if !ok {
return nil, errf("INSERT table %q references non-existent column %q", stmt.table, column)
}
if value != "?" {
// TODO(bradfitz): check that
// pre-bound value type conversion is
// valid for this column type
_ = ctype
}
stmt.colName = append(stmt.colName, column)
stmt.colValue = append(stmt.colValue, value)
}
default:
return nil, errf("unsupported command type %q", cmd)
}
return stmt, nil
}
func (s *fakeStmt) Close() os.Error {
return nil
}
func (s *fakeStmt) Exec(args []interface{}) (dbimpl.Result, os.Error) {
switch s.cmd {
case "CREATE":
if err := s.c.db.createTable(s.table, s.colName, s.colType); err != nil {
return nil, err
}
return dbimpl.DDLSuccess, nil
}
fmt.Printf("EXEC statement, cmd=%q: %#v\n", s.cmd, s)
return nil, fmt.Errorf("unimplemented statement Exec command type of %q", s.cmd)
}
func (s *fakeStmt) Query(args []interface{}) (dbimpl.Rows, os.Error) {
println("QUERY")
fmt.Println(args...)
return nil, os.NewError(todo())
}
func (s *fakeStmt) NumInput() int {
return 0
}
func (tx *fakeTx) Commit() os.Error {
tx.c.currTx = nil
return nil
}
func (tx *fakeTx) Rollback() os.Error {
tx.c.currTx = nil
return nil
}

143
lib/go/camli/db/testdb_test.go
View File

@ -1,143 +0,0 @@
/*
Copyright 2011 Google Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package db
import (
"fmt"
"os"
"sync"
"camli/db/dbimpl"
)
type testDriver struct {
mu sync.Mutex
dbs map[string]*testDB
}
type testDB struct {
name string
mu sync.Mutex
free []*testConn
}
type testConn struct {
db *testDB // where to return ourselves to
currTx *testTx
}
type testTx struct {
c *testConn
}
type testStmt struct {
q string
c *testConn
}
var driver dbimpl.Driver = &testDriver{}
func init() {
Register("test", driver)
}
func (d *testDriver) Open(name string) (dbimpl.Conn, os.Error) {
d.mu.Lock()
defer d.mu.Unlock()
if d.dbs == nil {
d.dbs = make(map[string]*testDB)
}
db, ok := d.dbs[name]
if !ok {
db = &testDB{name: name}
d.dbs[name] = db
}
return db.conn()
}
func (db *testDB) returnConn(c *testConn) {
db.mu.Lock()
defer db.mu.Unlock()
db.free = append(db.free, c)
}
func (db *testDB) conn() (dbimpl.Conn, os.Error) {
db.mu.Lock()
defer db.mu.Unlock()
if len(db.free) > 0 {
conn := db.free[len(db.free)-1]
db.free = db.free[:len(db.free)-1]
return conn, nil
}
return &testConn{db: db}, nil
}
func (c *testConn) Begin() (dbimpl.Tx, os.Error) {
if c.currTx != nil {
return nil, os.NewError("already in a transaction")
}
c.currTx = &testTx{c: c}
return c.currTx, nil
}
func (c *testConn) Close() os.Error {
if c.currTx != nil {
return os.NewError("can't close; in a Transaction")
}
if c.db == nil {
return os.NewError("can't close; already closed")
}
c.db.returnConn(c)
c.db = nil
return nil
}
func (c *testConn) Prepare(query string) (dbimpl.Stmt, os.Error) {
fmt.Printf("Prepare: %q\n", query)
return &testStmt{q: query, c: c}, nil
}
func (s *testStmt) Close() os.Error {
return nil
}
func (s *testStmt) Exec(args []interface{}) (dbimpl.Result, os.Error) {
fmt.Printf("EXEC(%#v)\n", args)
return nil, os.NewError("TODO: implement")
}
func (s *testStmt) Query(args []interface{}) (dbimpl.Rows, os.Error) {
println("QUERY")
fmt.Println(args...)
return nil, os.NewError("TODO: implement")
}
func (s *testStmt) NumInput() int {
return 0
}
func (tx *testTx) Commit() os.Error {
tx.c.currTx = nil
return nil
}
func (tx *testTx) Rollback() os.Error {
tx.c.currTx = nil
return nil
}