camget: do 10 at once. better, much pref work remains.

Change-Id: Ib2e159a8a290aff249fbec109d4e76099fa0491a
This commit is contained in:
Brad Fitzpatrick 2013-01-02 15:13:22 -08:00
parent 8e44c62047
commit 8866e88db0
1 changed files with 21 additions and 2 deletions

View File

@ -203,13 +203,32 @@ func smartFetch(cl *client.Client, targ string, br *blobref.BlobRef) error {
}
// directory entries
const numWorkers = 10
type work struct {
br *blobref.BlobRef
errc chan<- error
}
workc := make(chan work)
defer close(workc)
for i := 0; i < numWorkers; i++ {
go func() {
for wi := range workc {
wi.errc <- smartFetch(cl, targ, wi.br)
}
}()
}
var errcs []<-chan error
for _, m := range sc.Members {
// TODO: do n at a time
dref := blobref.Parse(m)
if dref == nil {
return fmt.Errorf("bad member blobref: %v", m)
}
if err := smartFetch(cl, targ, dref); err != nil {
errc := make(chan error, 1)
errcs = append(errcs, errc)
workc <- work{dref, errc}
}
for _, errc := range errcs {
if err := <-errc; err != nil {
return err
}
}