mirror of https://github.com/perkeep/perkeep.git
storagetest: more tests, and simplify a bit
Change-Id: Ic0d6c9718ef79ff82f05cf142286022862f30236
This commit is contained in:
parent
6478e256f7
commit
7466336f44
|
@ -18,15 +18,19 @@ limitations under the License.
|
|||
package storagetest
|
||||
|
||||
import (
|
||||
"crypto/sha1"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"camlistore.org/pkg/blob"
|
||||
"camlistore.org/pkg/blobserver"
|
||||
"camlistore.org/pkg/syncutil"
|
||||
"camlistore.org/pkg/test"
|
||||
"camlistore.org/pkg/types"
|
||||
)
|
||||
|
@ -42,10 +46,6 @@ func Test(t *testing.T, fn func(*testing.T) (sto blobserver.Storage, cleanup fun
|
|||
}()
|
||||
t.Logf("Testing blobserver storage %T", sto)
|
||||
|
||||
blobs := make([]*test.Blob, 0, 5)
|
||||
blobRefs := make([]blob.Ref, 0, cap(blobs))
|
||||
blobSizedRefs := make([]blob.SizedRef, 0, cap(blobs))
|
||||
|
||||
t.Logf("Testing Enumerate for empty")
|
||||
dest := make(chan blob.SizedRef)
|
||||
go func() {
|
||||
|
@ -53,7 +53,11 @@ func Test(t *testing.T, fn func(*testing.T) (sto blobserver.Storage, cleanup fun
|
|||
t.Fatalf("EnumerateBlob: %v", err)
|
||||
}
|
||||
}()
|
||||
testEnumerate(t, dest, blobSizedRefs)
|
||||
testEnumerate(t, sto, nil)
|
||||
|
||||
var blobs []*test.Blob
|
||||
var blobRefs []blob.Ref
|
||||
var blobSizedRefs []blob.SizedRef
|
||||
|
||||
contents := []string{"foo", "quux", "asdf", "qwerty", "0123456789"}
|
||||
if !testing.Short() {
|
||||
|
@ -78,13 +82,7 @@ func Test(t *testing.T, fn func(*testing.T) (sto blobserver.Storage, cleanup fun
|
|||
switch len(blobSizedRefs) {
|
||||
case 1, 5, 100:
|
||||
t.Logf("Testing Enumerate for %d blobs", len(blobSizedRefs))
|
||||
dest = make(chan blob.SizedRef)
|
||||
go func() {
|
||||
if err := sto.EnumerateBlobs(dest, "", 1000); err != nil {
|
||||
t.Fatalf("EnumerateBlob: %v", err)
|
||||
}
|
||||
}()
|
||||
testEnumerate(t, dest, blobSizedRefs)
|
||||
testEnumerate(t, sto, blobSizedRefs)
|
||||
}
|
||||
}
|
||||
b1 := blobs[0]
|
||||
|
@ -94,7 +92,6 @@ func Test(t *testing.T, fn func(*testing.T) (sto blobserver.Storage, cleanup fun
|
|||
|
||||
t.Logf("Testing FetchStreaming")
|
||||
for i, b2 := range blobs {
|
||||
t.Logf("%d. fetching %s", i, b2.BlobRef())
|
||||
rc, size, err := sto.FetchStreaming(b2.BlobRef())
|
||||
if err != nil {
|
||||
t.Fatalf("error fetching %d. %s: %v", i, b2, err)
|
||||
|
@ -104,7 +101,6 @@ func Test(t *testing.T, fn func(*testing.T) (sto blobserver.Storage, cleanup fun
|
|||
}
|
||||
|
||||
if fetcher, ok := sto.(fetcher); ok {
|
||||
t.Logf("fetching %s", b1.BlobRef())
|
||||
rsc, size, err := fetcher.Fetch(b1.BlobRef())
|
||||
if err != nil {
|
||||
t.Fatalf("error fetching %s: %v", b1, err)
|
||||
|
@ -129,14 +125,28 @@ func Test(t *testing.T, fn func(*testing.T) (sto blobserver.Storage, cleanup fun
|
|||
}()
|
||||
testStat(t, dest, blobSizedRefs)
|
||||
|
||||
t.Logf("Testing Enumerate")
|
||||
dest = make(chan blob.SizedRef)
|
||||
go func() {
|
||||
if err := sto.EnumerateBlobs(dest, "", 1000); err != nil {
|
||||
t.Fatalf("EnumerateBlob: %v", err)
|
||||
}
|
||||
}()
|
||||
testEnumerate(t, dest, blobSizedRefs)
|
||||
// Enumerate tests.
|
||||
sort.Sort(blob.SizedByRef(blobSizedRefs))
|
||||
|
||||
t.Logf("Testing Enumerate on all")
|
||||
testEnumerate(t, sto, blobSizedRefs)
|
||||
|
||||
t.Logf("Testing Enumerate 'limit' param")
|
||||
testEnumerate(t, sto, blobSizedRefs[:3], 3)
|
||||
|
||||
// Enumerate 'after'
|
||||
{
|
||||
after := blobSizedRefs[2].Ref.String()
|
||||
t.Logf("Testing Enumerate 'after' param; after %q", after)
|
||||
testEnumerate(t, sto, blobSizedRefs[3:], after)
|
||||
}
|
||||
|
||||
// Enumerate 'after' + limit
|
||||
{
|
||||
after := blobSizedRefs[2].Ref.String()
|
||||
t.Logf("Testing Enumerate 'after' + 'limit' param; after %q, limit 1", after)
|
||||
testEnumerate(t, sto, blobSizedRefs[3:4], after, 1)
|
||||
}
|
||||
|
||||
t.Logf("Testing Remove")
|
||||
if err := sto.RemoveBlobs(blobRefs); err != nil {
|
||||
|
@ -153,7 +163,7 @@ type fetcher interface {
|
|||
}
|
||||
|
||||
func testSizedBlob(t *testing.T, r io.Reader, b1 blob.Ref, size int64) {
|
||||
h := sha1.New()
|
||||
h := b1.Hash()
|
||||
n, err := io.Copy(h, r)
|
||||
if err != nil {
|
||||
t.Fatalf("error reading from %s: %v", r, err)
|
||||
|
@ -167,37 +177,68 @@ func testSizedBlob(t *testing.T, r io.Reader, b1 blob.Ref, size int64) {
|
|||
}
|
||||
}
|
||||
|
||||
func testEnumerate(t *testing.T, dest <-chan blob.SizedRef, want []blob.SizedRef) {
|
||||
// must sort want slice!
|
||||
m := make(map[string]int, len(want))
|
||||
refs := make([]string, 0, len(want))
|
||||
for i, sb := range want {
|
||||
m[sb.Ref.String()] = i
|
||||
refs = append(refs, sb.Ref.String())
|
||||
func testEnumerate(t *testing.T, sto blobserver.Storage, wantUnsorted []blob.SizedRef, opts ...interface{}) {
|
||||
var after string
|
||||
var n = 1000
|
||||
for _, opt := range opts {
|
||||
switch v := opt.(type) {
|
||||
case string:
|
||||
after = v
|
||||
case int:
|
||||
n = v
|
||||
default:
|
||||
panic("bad option of type " + fmt.Sprint("%T", v))
|
||||
}
|
||||
}
|
||||
sort.Strings(refs)
|
||||
|
||||
i := 0
|
||||
for sb := range dest {
|
||||
t.Logf("enumerated %s", sb)
|
||||
if !sb.Valid() {
|
||||
break
|
||||
want := append([]blob.SizedRef(nil), wantUnsorted...)
|
||||
sort.Sort(blob.SizedByRef(want))
|
||||
|
||||
sbc := make(chan blob.SizedRef, 10)
|
||||
|
||||
var got []blob.SizedRef
|
||||
var grp syncutil.Group
|
||||
sawEnd := make(chan bool, 1)
|
||||
grp.Go(func() error {
|
||||
if err := sto.EnumerateBlobs(sbc, after, n); err != nil {
|
||||
return fmt.Errorf("EnumerateBlobs(%q, %d): %v", after, n)
|
||||
}
|
||||
wanted := want[m[refs[i]]]
|
||||
if wanted.Size != sb.Size {
|
||||
t.Fatalf("received blob size is %d, wanted %d for &%d", sb.Size, wanted.Size, i)
|
||||
return nil
|
||||
})
|
||||
grp.Go(func() error {
|
||||
for sb := range sbc {
|
||||
if !sb.Valid() {
|
||||
return fmt.Errorf("invalid blobref %#v received in enumerate", sb)
|
||||
}
|
||||
got = append(got, sb)
|
||||
}
|
||||
if wanted.Ref != sb.Ref {
|
||||
t.Fatalf("received blob ref mismatch &%d: wanted %s, got %s", i, sb.Ref, wanted.Ref)
|
||||
}
|
||||
i++
|
||||
if i >= len(want) {
|
||||
break
|
||||
sawEnd <- true
|
||||
return nil
|
||||
|
||||
})
|
||||
grp.Go(func() error {
|
||||
select {
|
||||
case <-sawEnd:
|
||||
return nil
|
||||
case <-time.After(10 * time.Second):
|
||||
return errors.New("timeout waiting for EnumerateBlobs to close its channel")
|
||||
}
|
||||
|
||||
})
|
||||
if err := grp.Err(); err != nil {
|
||||
t.Fatalf("Enumerate error: %v", err)
|
||||
return
|
||||
}
|
||||
if len(got) == 0 && len(want) == 0 {
|
||||
return
|
||||
}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("Enumerate mismatch. Got %d; want %d.\n Got: %v\nWant: %v\n",
|
||||
len(got), len(want), got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func testStat(t *testing.T, dest <-chan blob.SizedRef, want []blob.SizedRef) {
|
||||
func testStat(t *testing.T, enum <-chan blob.SizedRef, want []blob.SizedRef) {
|
||||
// blobs may arrive in ANY order
|
||||
m := make(map[string]int, len(want))
|
||||
for i, sb := range want {
|
||||
|
@ -205,8 +246,7 @@ func testStat(t *testing.T, dest <-chan blob.SizedRef, want []blob.SizedRef) {
|
|||
}
|
||||
|
||||
i := 0
|
||||
for sb := range dest {
|
||||
t.Logf("statted %s", sb)
|
||||
for sb := range enum {
|
||||
if !sb.Valid() {
|
||||
break
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue