diff --git a/pkg/blobserver/s3/s3_test.go b/pkg/blobserver/s3/s3_test.go index 51637247e..1d7ba13b4 100644 --- a/pkg/blobserver/s3/s3_test.go +++ b/pkg/blobserver/s3/s3_test.go @@ -51,6 +51,7 @@ func TestNextStr(t *testing.T) { {"abc", "abd"}, {"ab\xff", "ac\x00"}, {"a\xff\xff", "b\x00\x00"}, + {"sha1-da39a3ee5e6b4b0d3255bfef95601890afd80709", "sha1-da39a3ee5e6b4b0d3255bfef95601890afd8070:"}, } for _, tt := range tests { if got := nextStr(tt.s); got != tt.want { diff --git a/pkg/misc/amazon/s3/client.go b/pkg/misc/amazon/s3/client.go index 88a1c5a33..9cebf4b27 100644 --- a/pkg/misc/amazon/s3/client.go +++ b/pkg/misc/amazon/s3/client.go @@ -27,10 +27,12 @@ import ( "hash" "io" "io/ioutil" + "log" "net/http" "net/url" "os" "strconv" + "time" "camlistore.org/pkg/httputil" ) @@ -145,6 +147,9 @@ type Item struct { type listBucketResults struct { Contents []*Item IsTruncated bool + MaxKeys int + Name string // bucket name + Marker string } // ListBucket returns 0 to maxKeys (inclusive) items from the provided @@ -163,18 +168,47 @@ func (c *Client) ListBucket(bucket string, startAt string, maxKeys int) (items [ fetchN = maxList } var bres listBucketResults + url_ := fmt.Sprintf("https://%s.%s/?marker=%s&max-keys=%d", bucket, c.hostname(), url.QueryEscape(marker), fetchN) - req := newReq(url_) - c.Auth.SignRequest(req) - res, err := c.httpClient().Do(req) - if err != nil { - return nil, err - } - err = xml.NewDecoder(res.Body).Decode(&bres) - httputil.CloseBody(res.Body) - if err != nil { - return nil, err + + // Try the enumerate three times, since Amazon likes to close + // https connections a lot, and Go sucks at dealing with it: + // https://code.google.com/p/go/issues/detail?id=3514 + const maxTries = 5 + for try := 1; try <= maxTries; try++ { + time.Sleep(time.Duration(try-1) * 100 * time.Millisecond) + req := newReq(url_) + c.Auth.SignRequest(req) + res, err := c.httpClient().Do(req) + if err != nil { + if try < maxTries { + continue + } + return nil, err + } + if res.StatusCode != 200 { + err = fmt.Errorf("s3.enumerate: status code %v", res.StatusCode) + } else { + bres = listBucketResults{} + var logbuf bytes.Buffer + err = xml.NewDecoder(io.TeeReader(res.Body, &logbuf)).Decode(&bres) + if err != nil { + log.Printf("Error parsing s3 XML response: %v for %q", err, logbuf.Bytes()) + } else if bres.MaxKeys != fetchN || bres.Name != bucket || bres.Marker != marker { + err = fmt.Errorf("Unexpected parse from server: %#v from: %s", bres, logbuf.Bytes()) + log.Print(err) + } + } + httputil.CloseBody(res.Body) + if err != nil { + if try < maxTries-1 { + continue + } + log.Print(err) + return nil, err + } + break } for _, it := range bres.Contents { if it.Key == marker && it.Key != startAt { @@ -188,6 +222,7 @@ func (c *Client) ListBucket(bucket string, startAt string, maxKeys int) (items [ marker = it.Key } if !bres.IsTruncated { + // log.Printf("Not truncated. so breaking. items = %d; len Contents = %d, url = %s", len(items), len(bres.Contents), url_) break } }