kv and deps: update to fix read problem on windows

github.com/cznic/kv rev: c966980e3e8456175d4407fcbb0287057d9d0862

http://camlistore.org/issue/214

Change-Id: I15d658710059c823b775db6cc6fa14c1a32b98fb
This commit is contained in:
mpl 2013-08-29 20:25:22 +02:00
parent 70d1bfd635
commit 2c54e30441
20 changed files with 188 additions and 67 deletions

View File

@ -24,6 +24,7 @@ import (
"time"
"camlistore.org/third_party/github.com/cznic/exp/lldb"
"camlistore.org/third_party/github.com/cznic/fileutil"
)
var (
@ -795,7 +796,7 @@ func TestSlice0(t *testing.T) {
ga = append(ga, strings.Join(a, ","))
return true, nil
}); err != nil {
if err != io.EOF {
if !fileutil.IsEOF(err) {
t.Fatal(i, err)
}
}
@ -2116,7 +2117,7 @@ func TestFileReadAtWriteAt(t *testing.T) {
from, to = 0, n2
}
n, err := f.ReadAt(b[from:to], int64(from))
if err != nil && (err != io.EOF && n != 0) {
if err != nil && (!fileutil.IsEOF(err) && n != 0) {
t.Error(fsz(), from, to, err)
return
}

View File

@ -7,10 +7,10 @@ package dbm
import (
"bytes"
"fmt"
"io"
"camlistore.org/third_party/github.com/cznic/exp/lldb"
"camlistore.org/third_party/github.com/cznic/mathutil"
"camlistore.org/third_party/github.com/cznic/fileutil"
)
type header struct {
@ -149,7 +149,7 @@ func encVal(val interface{}) (r []byte, err error) {
}
func noEof(e error) (err error) {
if e != io.EOF {
if !fileutil.IsEOF(e) {
err = e
}
return

View File

@ -11,6 +11,7 @@ import (
"camlistore.org/third_party/github.com/cznic/exp/lldb"
"camlistore.org/third_party/github.com/cznic/mathutil"
"camlistore.org/third_party/github.com/cznic/fileutil"
)
/*
@ -375,7 +376,7 @@ func (f *File) ReadFrom(r io.Reader) (n int64, err error) {
n += int64(rn)
}
}
if rerr != io.EOF {
if !fileutil.IsEOF(rerr) {
err = rerr
}
return
@ -439,7 +440,7 @@ func (f *File) WriteTo(w io.Writer) (n int64, err error) {
n += int64(wn)
}
}
if rerr != io.EOF {
if !fileutil.IsEOF(rerr) {
err = rerr
}
return

View File

@ -13,6 +13,7 @@ import (
"io"
"os"
"camlistore.org/third_party/github.com/cznic/fileutil"
"camlistore.org/third_party/github.com/cznic/mathutil"
)
@ -161,7 +162,7 @@ func NewACIDFiler(db Filer, wal *os.File) (r *ACIDFiler0, err error) {
for {
k, v, err := enum.current()
if err != nil {
if err == io.EOF {
if fileutil.IsEOF(err) {
break
}
@ -173,7 +174,7 @@ func NewACIDFiler(db Filer, wal *os.File) (r *ACIDFiler0, err error) {
}
if err = enum.next(); err != nil {
if err == io.EOF {
if fileutil.IsEOF(err) {
break
}
@ -299,7 +300,7 @@ func (a *ACIDFiler0) recoverDb(db Filer) (err error) {
for {
k, v, err := enum.current()
if err != nil {
if err == io.EOF {
if fileutil.IsEOF(err) {
break
}
@ -311,7 +312,7 @@ func (a *ACIDFiler0) recoverDb(db Filer) (err error) {
}
if err = enum.next(); err != nil {
if err == io.EOF {
if fileutil.IsEOF(err) {
break
}

View File

@ -13,6 +13,7 @@ import (
"strings"
"camlistore.org/third_party/github.com/cznic/bufs"
"camlistore.org/third_party/github.com/cznic/fileutil"
"camlistore.org/third_party/github.com/cznic/sortutil"
)
@ -184,7 +185,7 @@ func (t *BTree) Dump(w io.Writer) (err error) {
err = enum.next()
if err != nil {
if err == io.EOF {
if fileutil.IsEOF(err) {
err = nil
break
}

View File

@ -10,13 +10,13 @@ import (
"encoding/hex"
"flag"
"fmt"
"io"
"math"
"math/rand"
"os"
"runtime"
"testing"
"camlistore.org/third_party/github.com/cznic/fileutil"
"camlistore.org/third_party/github.com/cznic/mathutil"
)
@ -632,15 +632,15 @@ func TestbTreeNext(t *testing.T) {
t.Fatal(err)
}
if _, _, err = enum.current(); err != io.EOF {
if _, _, err = enum.current(); !fileutil.IsEOF(err) {
t.Fatal(err)
}
if err = enum.next(); err != io.EOF {
if err = enum.next(); !fileutil.IsEOF(err) {
t.Fatal(err)
}
if err = enum.prev(); err != io.EOF {
if err = enum.prev(); !fileutil.IsEOF(err) {
t.Fatal(err)
}
@ -683,7 +683,7 @@ func TestbTreeNext(t *testing.T) {
t.Fatal(err)
}
if err = enum.next(); N > 1 && err != io.EOF {
if err = enum.next(); N > 1 && !fileutil.IsEOF(err) {
t.Fatal(err)
}
@ -697,11 +697,11 @@ func TestbTreeNext(t *testing.T) {
}
// index: N
if _, _, err = enum.current(); err != io.EOF {
if _, _, err = enum.current(); !fileutil.IsEOF(err) {
t.Fatal(err)
}
if err = enum.next(); N > 1 && err != io.EOF {
if err = enum.next(); N > 1 && !fileutil.IsEOF(err) {
t.Fatal(err)
}
@ -727,7 +727,7 @@ func TestbTreeNext(t *testing.T) {
switch {
case i == N:
if err := enum.next(); err != io.EOF {
if err := enum.next(); !fileutil.IsEOF(err) {
t.Fatal(err)
}
default:
@ -747,15 +747,15 @@ func TestbTreePrev(t *testing.T) {
t.Fatal(err)
}
if _, _, err = enum.current(); err != io.EOF {
if _, _, err = enum.current(); !fileutil.IsEOF(err) {
t.Fatal(err)
}
if err = enum.next(); err != io.EOF {
if err = enum.next(); !fileutil.IsEOF(err) {
t.Fatal(err)
}
if err = enum.prev(); err != io.EOF {
if err = enum.prev(); !fileutil.IsEOF(err) {
t.Fatal(err)
}
@ -780,7 +780,7 @@ func TestbTreePrev(t *testing.T) {
t.Fatal(err)
}
if err = enum.prev(); err != io.EOF {
if err = enum.prev(); !fileutil.IsEOF(err) {
t.Fatal(err)
}
@ -812,7 +812,7 @@ func TestbTreePrev(t *testing.T) {
}
// index: N
if _, _, err = enum.current(); err != io.EOF {
if _, _, err = enum.current(); !fileutil.IsEOF(err) {
t.Fatal(err)
}
@ -842,7 +842,7 @@ func TestbTreePrev(t *testing.T) {
switch {
case i == 1:
if err := enum.prev(); err != io.EOF {
if err := enum.prev(); !fileutil.IsEOF(err) {
t.Fatal(err)
}
default:
@ -1095,7 +1095,7 @@ func TestseekFirst(t *testing.T) {
bt := NewBTree(nil)
enum, err := bt.seekFirst()
if err != io.EOF {
if !fileutil.IsEOF(err) {
t.Fatal(err)
}
@ -1106,12 +1106,12 @@ func TestseekFirst(t *testing.T) {
}
err = enum.prev()
if err != io.EOF {
if !fileutil.IsEOF(err) {
t.Fatal(err)
}
err = enum.next()
if err != io.EOF {
if !fileutil.IsEOF(err) {
t.Fatal(err)
}
@ -1131,7 +1131,7 @@ func TestseekFirst(t *testing.T) {
}
err = enum.prev()
if err != io.EOF {
if !fileutil.IsEOF(err) {
t.Fatal(err)
}
@ -1163,7 +1163,7 @@ func TestseekLast(t *testing.T) {
bt := NewBTree(nil)
enum, err := bt.seekFirst()
if err != io.EOF {
if !fileutil.IsEOF(err) {
t.Fatal(err)
}
@ -1174,12 +1174,12 @@ func TestseekLast(t *testing.T) {
}
err = enum.prev()
if err != io.EOF {
if !fileutil.IsEOF(err) {
t.Fatal(err)
}
err = enum.next()
if err != io.EOF {
if !fileutil.IsEOF(err) {
t.Fatal(err)
}
@ -1199,7 +1199,7 @@ func TestseekLast(t *testing.T) {
}
err = enum.next()
if err != io.EOF {
if !fileutil.IsEOF(err) {
t.Fatal(err)
}
@ -1597,7 +1597,7 @@ func TestBTreeSeekNext(t *testing.T) {
k, v, err := en.Next()
if err != nil {
if err != io.EOF {
if !fileutil.IsEOF(err) {
t.Fatal(i, err)
}
@ -1687,7 +1687,7 @@ func TestBTreeSeekPrev(t *testing.T) {
k, v, err := en.Prev()
if err != nil {
if err != io.EOF {
if !fileutil.IsEOF(err) {
t.Fatal(i, err)
}

View File

@ -7,12 +7,13 @@ package lldb
import (
"bytes"
"encoding/hex"
"io"
"io/ioutil"
"math/rand"
"os"
"runtime"
"testing"
"camlistore.org/third_party/github.com/cznic/fileutil"
)
// Bench knobs.
@ -284,7 +285,7 @@ func testFilerReadAtWriteAt(t *testing.T, nf newFunc) {
from, to = 0, n2
}
n, err := f.ReadAt(b[from:to], int64(from))
if err != nil && (err != io.EOF && n != 0) {
if err != nil && (!fileutil.IsEOF(err) && n != 0) {
fsz, err = f.Size()
if err != nil {
t.Error(err)

View File

@ -79,8 +79,10 @@ package lldb
import (
"bytes"
"fmt"
"camlistore.org/third_party/github.com/cznic/mathutil"
"io"
"camlistore.org/third_party/github.com/cznic/fileutil"
"camlistore.org/third_party/github.com/cznic/mathutil"
)
const (
@ -213,7 +215,7 @@ func (f *MemFiler) ReadFrom(r io.Reader) (n int64, err error) {
n += int64(rn)
}
}
if rerr != io.EOF {
if !fileutil.IsEOF(rerr) {
err = rerr
}
return
@ -335,7 +337,7 @@ func (f *MemFiler) WriteTo(w io.Writer) (n int64, err error) {
n += int64(wn)
}
}
if rerr != io.EOF {
if !fileutil.IsEOF(rerr) {
err = rerr
}
return

View File

@ -54,6 +54,7 @@ import (
"fmt"
"io"
"camlistore.org/third_party/github.com/cznic/fileutil"
"camlistore.org/third_party/github.com/cznic/mathutil"
)
@ -151,7 +152,7 @@ func (f *bitFiler) ReadAt(b []byte, off int64) (n int, err error) {
pg = &bitPage{}
if f.parent != nil {
_, err = f.parent.ReadAt(pg.data[:], off&^bfMask)
if err != nil && err != io.EOF {
if err != nil && !fileutil.IsEOF(err) {
return
}
@ -209,7 +210,7 @@ func (f *bitFiler) WriteAt(b []byte, off int64) (n int, err error) {
pg = &bitPage{}
if f.parent != nil {
_, err = f.parent.ReadAt(pg.data[:], off&^bfMask)
if err != nil && err != io.EOF {
if err != nil && !fileutil.IsEOF(err) {
return
}

View File

@ -12,6 +12,7 @@ import (
"math/rand"
"testing"
"camlistore.org/third_party/github.com/cznic/fileutil"
"camlistore.org/third_party/github.com/cznic/mathutil"
)
@ -182,12 +183,12 @@ func TestRollbackFiler3(t *testing.T) {
}
n, err := r.ReadAt([]byte{0}, 0)
if n != 0 || err != io.EOF {
if n != 0 || !fileutil.IsEOF(err) {
t.Fatal(n, err)
}
n, err = r.ReadAt([]byte{0}, 1e6)
if n != 0 || err != io.EOF {
if n != 0 || !fileutil.IsEOF(err) {
t.Fatal(n, err)
}

View File

@ -7,6 +7,7 @@
package fileutil
import (
"io"
"os"
)
@ -21,3 +22,6 @@ func PunchHole(f *os.File, off, len int64) error {
func Fadvise(f *os.File, off, len int64, advice FadviseAdvice) error {
return nil
}
// IsEOF reports whether err is an EOF condition.
func IsEOF(err error) bool { return err == io.EOF }

View File

@ -7,6 +7,7 @@
package fileutil
import (
"io"
"os"
)
@ -21,3 +22,6 @@ func PunchHole(f *os.File, off, len int64) error {
func Fadvise(f *os.File, off, len int64, advice FadviseAdvice) error {
return nil
}
// IsEOF reports whether err is an EOF condition.
func IsEOF(err error) bool { return err == io.EOF }

View File

@ -1,23 +1,27 @@
// Copyright (c) 2011 CZ.NIC z.s.p.o. All rights reserved.
// Copyright (c) 2013 wathiede[0]. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// blame: jnml, labs.nic.cz
//
// [0]: https://github.com/wathiede
package fileutil
import (
"io"
"os"
)
// PunchHole deallocates space inside a file in the byte range starting at
// offset and continuing for len bytes. Unimplemented on FreeBSD
// offset and continuing for len bytes. Unimplemented on FreeBSD.
func PunchHole(f *os.File, off, len int64) error {
return nil
}
// Fadvise predeclares an access pattern for file data. See also 'man 2
// posix_fadvise'. Unimplemented on FreeBSD
// posix_fadvise'. Unimplemented on FreeBSD.
func Fadvise(f *os.File, off, len int64, advice FadviseAdvice) error {
return nil
}
// IsEOF reports whether err is an EOF condition.
func IsEOF(err error) bool { return err == io.EOF }

View File

@ -8,6 +8,7 @@ package fileutil
import (
"bytes"
"io"
"io/ioutil"
"os"
"strconv"
@ -32,6 +33,9 @@ func init() {
}
tokens := bytes.Split(b, []byte("."))
if len(tokens) > 3 {
tokens = tokens[:3]
}
switch len(tokens) {
case 3:
// Supported since kernel 2.6.38
@ -43,7 +47,7 @@ func init() {
puncher = func(*os.File, int64, int64) error { return nil }
}
default:
panic(n)
puncher = func(*os.File, int64, int64) error { return nil }
}
}
@ -84,3 +88,6 @@ func Fadvise(f *os.File, off, len int64, advice FadviseAdvice) error {
0, 0)
return os.NewSyscallError("SYS_FADVISE64", errno)
}
// IsEOF reports whether err is an EOF condition.
func IsEOF(err error) bool { return err == io.EOF }

View File

@ -5,7 +5,9 @@
package fileutil
import (
"io"
"os"
"syscall"
)
// PunchHole deallocates space inside a file in the byte range starting at
@ -19,3 +21,14 @@ func PunchHole(f *os.File, off, len int64) error {
func Fadvise(f *os.File, off, len int64, advice FadviseAdvice) error {
return nil
}
// IsEOF reports whether err is an EOF condition.
func IsEOF(err error) bool {
if err == io.EOF {
return true
}
// http://social.technet.microsoft.com/Forums/windowsserver/en-US/1a16311b-c625-46cf-830b-6a26af488435/how-to-solve-error-38-0x26-errorhandleeof-using-fsctlgetretrievalpointers
x, ok := err.(*os.PathError)
return ok && x.Op == "read" && x.Err.(syscall.Errno) == 0x26
}

View File

@ -38,6 +38,7 @@ func (p *Probe) assert(t *testing.T, msg int, opsRd, opsWr, bytesRd, bytesWr, se
}
func TestProbe(t *testing.T) {
return //TODO disabled due to atomic.AddInt64 failing on W32
const fn = "test.tmp"
store, err := NewFile(fn, os.O_CREATE|os.O_RDWR|os.O_CREATE, 0666)

View File

@ -8,15 +8,17 @@ import (
"encoding/binary"
"flag"
"fmt"
"io"
"io/ioutil"
"math"
"os"
"path"
"path/filepath"
"runtime"
"strings"
"sync"
"testing"
"camlistore.org/third_party/github.com/cznic/fileutil"
"camlistore.org/third_party/github.com/cznic/mathutil"
)
@ -146,9 +148,9 @@ func TestName(t *testing.T) {
}(db.Name())
if n := db.Name(); n == "" ||
!strings.Contains(n, "_testdata/") ||
!strings.HasPrefix(path.Base(n), "temp") ||
!strings.HasSuffix(path.Base(n), ".db") ||
!strings.Contains(n, "_testdata") ||
!strings.HasPrefix(filepath.Base(n), "temp") ||
!strings.HasSuffix(filepath.Base(n), ".db") ||
path.Base(n) == "temp.db" {
t.Error(n)
}
@ -1393,7 +1395,7 @@ func TestSeekNext(t *testing.T) {
k, v, err := en.Next()
if err != nil {
if err != io.EOF {
if !fileutil.IsEOF(err) {
t.Fatal(i, err)
}
@ -1487,7 +1489,7 @@ func TestSeekPrev(t *testing.T) {
k, v, err := en.Prev()
if err != nil {
if err != io.EOF {
if !fileutil.IsEOF(err) {
t.Fatal(i, err)
}
@ -1808,3 +1810,64 @@ func TestWALName(t *testing.T) {
t.Error(n)
}
}
func TestCreateWithEmptyWAL(t *testing.T) {
dir, err := ioutil.TempDir("", "kv-test-create")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(dir)
dbName := filepath.Join(dir, "test.db")
var o Options
walName := o.walName(dbName, "")
wal, err := os.Create(walName)
if err != nil {
t.Error(err)
return
}
wal.Close()
defer os.Remove(walName)
db, err := Create(dbName, &Options{})
if err != nil {
t.Error(err)
return
}
if err = db.Set([]byte("foo"), []byte("bar")); err != nil {
t.Error(err)
}
db.Close()
}
func TestCreateWithNonEmptyWAL(t *testing.T) {
dir, err := ioutil.TempDir("", "kv-test-create")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(dir)
dbName := filepath.Join(dir, "test.db")
var o Options
walName := o.walName(dbName, "")
wal, err := os.Create(walName)
if err != nil {
t.Error(err)
return
}
if n, err := wal.Write([]byte{0}); n != 1 || err != nil {
t.Error(n, err)
return
}
wal.Close()
defer os.Remove(walName)
if _, err = Create(dbName, &Options{}); err == nil {
t.Error("Unexpected success")
return
}
}

View File

@ -7,7 +7,8 @@ package kv
import (
"bytes"
"fmt"
"io"
"camlistore.org/third_party/github.com/cznic/fileutil"
)
type header struct {
@ -48,7 +49,7 @@ func h2b(b []byte, h int64) []byte {
}
func noEof(e error) (err error) {
if e != io.EOF {
if !fileutil.IsEOF(e) {
err = e
}
return

View File

@ -291,6 +291,13 @@ func (db *DB) Close() (err error) {
err = e1
}
}
if wal := db.wal; wal != nil {
e := wal.Close()
db.wal = nil
if err == nil {
err = e
}
}
return
}
@ -664,12 +671,10 @@ func (db *DB) Put(dst, key []byte, upd func(key, old []byte) (new []byte, write
return
}
// Seek returns an enumerator or an error if any. Normally the enumerator is
// positioned on a KV pair such that 'key' >= KV.key and 'hit' is key ==
// KV.key. If 'key' collates after any existing key in the DB then the
// enumerator's position is effectively "after" the last KV pair, but that is
// not an error. However, such enumerator will return err set to io.EOF from
// its Next/Prev methods.
// Seek returns an enumerator positioned on the first key/value pair whose key
// is 'greater than or equal to' the given key. There may be no such pair, in
// which case the Next,Prev methods of the returned enumerator will always
// return io.EOF.
//
// Seek is atomic and it is safe for concurrent use by multiple goroutines.
func (db *DB) Seek(key []byte) (enum *Enumerator, hit bool, err error) {

View File

@ -85,8 +85,9 @@ type Options struct {
// used. If it is of zero size then a clean shutdown of the DB is
// assumed, otherwise an automatic DB recovery is performed.
//
// On creating a new DB the WAL file must not exist. It's not safe to
// write to a WAL file as it may contain unprocessed DB recovery data.
// On creating a new DB the WAL file must not exist or it must be
// empty. It's not safe to write to a non empty WAL file as it may
// contain unprocessed DB recovery data.
_WAL string
// Time to collect transactions before committing them into the WAL.
@ -142,7 +143,16 @@ func (o *Options) check(dbname string, new, lock bool) (err error) {
case true:
if o.wal, err = os.OpenFile(o._WAL, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0666); err != nil {
if os.IsExist(err) {
err = fmt.Errorf("cannot create DB %q: WAL file %q exists", dbname, o._WAL)
fi, e := os.Stat(o._WAL)
if e != nil {
return e
}
if sz := fi.Size(); sz != 0 {
return fmt.Errorf("cannot create DB %q: non empty WAL file %q (size %d) exists", dbname, o._WAL, sz)
}
o.wal, err = os.OpenFile(o._WAL, os.O_RDWR, 0666)
}
return
}