storagetest: more tests, and simplify a bit

Change-Id: Ic0d6c9718ef79ff82f05cf142286022862f30236
This commit is contained in:
Brad Fitzpatrick 2013-12-02 11:57:02 -08:00
parent 6478e256f7
commit 7466336f44
1 changed files with 89 additions and 49 deletions

View File

@ -18,15 +18,19 @@ limitations under the License.
package storagetest
import (
"crypto/sha1"
"errors"
"fmt"
"io"
"reflect"
"sort"
"strconv"
"strings"
"testing"
"time"
"camlistore.org/pkg/blob"
"camlistore.org/pkg/blobserver"
"camlistore.org/pkg/syncutil"
"camlistore.org/pkg/test"
"camlistore.org/pkg/types"
)
@ -42,10 +46,6 @@ func Test(t *testing.T, fn func(*testing.T) (sto blobserver.Storage, cleanup fun
}()
t.Logf("Testing blobserver storage %T", sto)
blobs := make([]*test.Blob, 0, 5)
blobRefs := make([]blob.Ref, 0, cap(blobs))
blobSizedRefs := make([]blob.SizedRef, 0, cap(blobs))
t.Logf("Testing Enumerate for empty")
dest := make(chan blob.SizedRef)
go func() {
@ -53,7 +53,11 @@ func Test(t *testing.T, fn func(*testing.T) (sto blobserver.Storage, cleanup fun
t.Fatalf("EnumerateBlob: %v", err)
}
}()
testEnumerate(t, dest, blobSizedRefs)
testEnumerate(t, sto, nil)
var blobs []*test.Blob
var blobRefs []blob.Ref
var blobSizedRefs []blob.SizedRef
contents := []string{"foo", "quux", "asdf", "qwerty", "0123456789"}
if !testing.Short() {
@ -78,13 +82,7 @@ func Test(t *testing.T, fn func(*testing.T) (sto blobserver.Storage, cleanup fun
switch len(blobSizedRefs) {
case 1, 5, 100:
t.Logf("Testing Enumerate for %d blobs", len(blobSizedRefs))
dest = make(chan blob.SizedRef)
go func() {
if err := sto.EnumerateBlobs(dest, "", 1000); err != nil {
t.Fatalf("EnumerateBlob: %v", err)
}
}()
testEnumerate(t, dest, blobSizedRefs)
testEnumerate(t, sto, blobSizedRefs)
}
}
b1 := blobs[0]
@ -94,7 +92,6 @@ func Test(t *testing.T, fn func(*testing.T) (sto blobserver.Storage, cleanup fun
t.Logf("Testing FetchStreaming")
for i, b2 := range blobs {
t.Logf("%d. fetching %s", i, b2.BlobRef())
rc, size, err := sto.FetchStreaming(b2.BlobRef())
if err != nil {
t.Fatalf("error fetching %d. %s: %v", i, b2, err)
@ -104,7 +101,6 @@ func Test(t *testing.T, fn func(*testing.T) (sto blobserver.Storage, cleanup fun
}
if fetcher, ok := sto.(fetcher); ok {
t.Logf("fetching %s", b1.BlobRef())
rsc, size, err := fetcher.Fetch(b1.BlobRef())
if err != nil {
t.Fatalf("error fetching %s: %v", b1, err)
@ -129,14 +125,28 @@ func Test(t *testing.T, fn func(*testing.T) (sto blobserver.Storage, cleanup fun
}()
testStat(t, dest, blobSizedRefs)
t.Logf("Testing Enumerate")
dest = make(chan blob.SizedRef)
go func() {
if err := sto.EnumerateBlobs(dest, "", 1000); err != nil {
t.Fatalf("EnumerateBlob: %v", err)
}
}()
testEnumerate(t, dest, blobSizedRefs)
// Enumerate tests.
sort.Sort(blob.SizedByRef(blobSizedRefs))
t.Logf("Testing Enumerate on all")
testEnumerate(t, sto, blobSizedRefs)
t.Logf("Testing Enumerate 'limit' param")
testEnumerate(t, sto, blobSizedRefs[:3], 3)
// Enumerate 'after'
{
after := blobSizedRefs[2].Ref.String()
t.Logf("Testing Enumerate 'after' param; after %q", after)
testEnumerate(t, sto, blobSizedRefs[3:], after)
}
// Enumerate 'after' + limit
{
after := blobSizedRefs[2].Ref.String()
t.Logf("Testing Enumerate 'after' + 'limit' param; after %q, limit 1", after)
testEnumerate(t, sto, blobSizedRefs[3:4], after, 1)
}
t.Logf("Testing Remove")
if err := sto.RemoveBlobs(blobRefs); err != nil {
@ -153,7 +163,7 @@ type fetcher interface {
}
func testSizedBlob(t *testing.T, r io.Reader, b1 blob.Ref, size int64) {
h := sha1.New()
h := b1.Hash()
n, err := io.Copy(h, r)
if err != nil {
t.Fatalf("error reading from %s: %v", r, err)
@ -167,37 +177,68 @@ func testSizedBlob(t *testing.T, r io.Reader, b1 blob.Ref, size int64) {
}
}
func testEnumerate(t *testing.T, dest <-chan blob.SizedRef, want []blob.SizedRef) {
// must sort want slice!
m := make(map[string]int, len(want))
refs := make([]string, 0, len(want))
for i, sb := range want {
m[sb.Ref.String()] = i
refs = append(refs, sb.Ref.String())
func testEnumerate(t *testing.T, sto blobserver.Storage, wantUnsorted []blob.SizedRef, opts ...interface{}) {
var after string
var n = 1000
for _, opt := range opts {
switch v := opt.(type) {
case string:
after = v
case int:
n = v
default:
panic("bad option of type " + fmt.Sprint("%T", v))
}
}
sort.Strings(refs)
i := 0
for sb := range dest {
t.Logf("enumerated %s", sb)
if !sb.Valid() {
break
want := append([]blob.SizedRef(nil), wantUnsorted...)
sort.Sort(blob.SizedByRef(want))
sbc := make(chan blob.SizedRef, 10)
var got []blob.SizedRef
var grp syncutil.Group
sawEnd := make(chan bool, 1)
grp.Go(func() error {
if err := sto.EnumerateBlobs(sbc, after, n); err != nil {
return fmt.Errorf("EnumerateBlobs(%q, %d): %v", after, n)
}
wanted := want[m[refs[i]]]
if wanted.Size != sb.Size {
t.Fatalf("received blob size is %d, wanted %d for &%d", sb.Size, wanted.Size, i)
return nil
})
grp.Go(func() error {
for sb := range sbc {
if !sb.Valid() {
return fmt.Errorf("invalid blobref %#v received in enumerate", sb)
}
got = append(got, sb)
}
if wanted.Ref != sb.Ref {
t.Fatalf("received blob ref mismatch &%d: wanted %s, got %s", i, sb.Ref, wanted.Ref)
}
i++
if i >= len(want) {
break
sawEnd <- true
return nil
})
grp.Go(func() error {
select {
case <-sawEnd:
return nil
case <-time.After(10 * time.Second):
return errors.New("timeout waiting for EnumerateBlobs to close its channel")
}
})
if err := grp.Err(); err != nil {
t.Fatalf("Enumerate error: %v", err)
return
}
if len(got) == 0 && len(want) == 0 {
return
}
if !reflect.DeepEqual(got, want) {
t.Fatalf("Enumerate mismatch. Got %d; want %d.\n Got: %v\nWant: %v\n",
len(got), len(want), got, want)
}
}
func testStat(t *testing.T, dest <-chan blob.SizedRef, want []blob.SizedRef) {
func testStat(t *testing.T, enum <-chan blob.SizedRef, want []blob.SizedRef) {
// blobs may arrive in ANY order
m := make(map[string]int, len(want))
for i, sb := range want {
@ -205,8 +246,7 @@ func testStat(t *testing.T, dest <-chan blob.SizedRef, want []blob.SizedRef) {
}
i := 0
for sb := range dest {
t.Logf("statted %s", sb)
for sb := range enum {
if !sb.Valid() {
break
}