From 59050a893eb1f15c217b0e3b9accf829b166ec6e Mon Sep 17 00:00:00 2001 From: Bill Thiede Date: Tue, 8 Jul 2014 17:13:47 -0700 Subject: [PATCH] Revert "diskpacked: bug fix walkPack and add test." This reverts commit 83d2546d21e27426f1fe5ea1886bb87f6be42d15. Change-Id: Ia2d622601d68ca5795141de1314244f1e4cd14cb --- pkg/blobserver/diskpacked/diskpacked.go | 13 ++-- pkg/blobserver/diskpacked/reindex.go | 28 +++++++- pkg/blobserver/diskpacked/reindex_test.go | 68 ------------------ .../diskpacked/testdata/pack-00000.blobs | Bin 218 -> 0 bytes .../diskpacked/testdata/pack-00001.blobs | Bin 158 -> 0 bytes 5 files changed, 32 insertions(+), 77 deletions(-) delete mode 100644 pkg/blobserver/diskpacked/reindex_test.go delete mode 100644 pkg/blobserver/diskpacked/testdata/pack-00000.blobs delete mode 100644 pkg/blobserver/diskpacked/testdata/pack-00001.blobs diff --git a/pkg/blobserver/diskpacked/diskpacked.go b/pkg/blobserver/diskpacked/diskpacked.go index d6d204914..e16071594 100644 --- a/pkg/blobserver/diskpacked/diskpacked.go +++ b/pkg/blobserver/diskpacked/diskpacked.go @@ -41,7 +41,6 @@ import ( "log" "os" "path/filepath" - "regexp" "strings" "sync" @@ -469,6 +468,12 @@ func headerLength(digest string, size uint32) int { return len(fmt.Sprintf("[%s %d]", digest, size)) } +// The header of deleted blobs has a digest in which the hash type is +// set to all 'x', but the correct size. +func isDeletedRef(digest string) bool { + return strings.HasPrefix(digest, "x") +} + // Type readSeekNopCloser is an io.ReadSeeker with a no-op Close method. type readSeekNopCloser struct { io.ReadSeeker @@ -480,10 +485,6 @@ func newReadSeekNopCloser(rs io.ReadSeeker) types.ReadSeekCloser { return readSeekNopCloser{rs} } -// The header of deleted blobs has a digest in which the hash type is -// set to all 'x', the hash value is all '0', and has the correct size. -var deletedBlobRef = regexp.MustCompile(`^x+-0+$`) - // StreamBlobs Implements the blobserver.StreamBlobs interface. func (s *storage) StreamBlobs(ctx *context.Context, dest chan<- *blob.Blob, contToken string, limitBytes int64) (nextContinueToken string, err error) { defer close(dest) @@ -555,7 +556,7 @@ func (s *storage) StreamBlobs(ctx *context.Context, dest chan<- *blob.Blob, cont } offsetToAdd += int64(headerLength(digest, size)) - if deletedBlobRef.MatchString(digest) { + if isDeletedRef(digest) { // Skip over deletion padding _, err = io.CopyN(ioutil.Discard, r, int64(size)) if err != nil { diff --git a/pkg/blobserver/diskpacked/reindex.go b/pkg/blobserver/diskpacked/reindex.go index f3efe6ee9..bfb61773d 100644 --- a/pkg/blobserver/diskpacked/reindex.go +++ b/pkg/blobserver/diskpacked/reindex.go @@ -203,14 +203,36 @@ func (s *storage) walkPack(verbose bool, packID int, } size = uint32(size64) - if deletedBlobRef.Match(chunk[:i]) { + // maybe deleted? + state, deleted := 0, true + if chunk[0] == 'x' { + Loop: + for _, c := range chunk[:i] { + switch state { + case 0: + if c != 'x' { + if c == '-' { + state++ + } else { + deleted = false + break Loop + } + } + case 1: + if c != '0' { + deleted = false + break Loop + } + } + } + } + if deleted { ref = blob.Ref{} if verbose { log.Printf("found deleted at %d", pos) } } else { - var ok bool - ref, ok = blob.Parse(string(chunk[:i])) + ref, ok := blob.Parse(string(chunk[:i])) if !ok { return errAt("", fmt.Sprintf("cannot parse %q as blobref", chunk[:i])) } diff --git a/pkg/blobserver/diskpacked/reindex_test.go b/pkg/blobserver/diskpacked/reindex_test.go deleted file mode 100644 index 16cf967bf..000000000 --- a/pkg/blobserver/diskpacked/reindex_test.go +++ /dev/null @@ -1,68 +0,0 @@ -/* -Copyright 2014 Google Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package diskpacked - -import ( - "testing" - - "camlistore.org/pkg/blob" -) - -type blobStat struct { - id int - ref string - offset int64 - size uint32 -} - -func TestWalkPack(t *testing.T) { - want := []blobStat{ - {0, "sha1-f7ff9e8b7bb2e09b70935a5d785e0cc5d9d0abf0", 49, 5}, - {0, "sha1-70c07ec18ef89c5309bbb0937f3a6342411e1fdd", 103, 5}, - {0, "", 157, 7}, - {0, "sha1-70c07ec18ef89c5309bbb0937f3a6342411e1fdd", 213, 6}, - {1, "sha1-fe05bcdcdc4928012781a5f1a2a77cbb5398e106", 49, 3}, - {1, "sha1-ad782ecdac770fc6eb9a62e44f90873fb97fb26b", 101, 3}, - {1, "sha1-b802f384302cb24fbab0a44997e820bf2e8507bb", 153, 5}, - } - var got []blobStat - s := storage{root: "testdata"} - walk := func(packID int, ref blob.Ref, offset int64, size uint32) error { - t.Log(packID, ref, offset, size) - got = append(got, blobStat{ - id: packID, - ref: ref.String(), - offset: offset, - size: size, - }) - return nil - } - - if err := s.Walk(nil, walk); err != nil { - t.Fatal(err) - } - - if len(got) != len(want) { - t.Errorf("Got len %q want len %q", got, want) - } - for i, g := range got { - w := want[i] - if g.id != w.id || g.ref != w.ref || g.offset != w.offset || g.size != w.size { - t.Errorf("%d: got %d, %q, %d, %d want %d, %q, %d, %d", i, g.id, g.ref, g.offset, g.size, w.id, w.ref, w.offset, w.size) - } - } -} diff --git a/pkg/blobserver/diskpacked/testdata/pack-00000.blobs b/pkg/blobserver/diskpacked/testdata/pack-00000.blobs deleted file mode 100644 index f7ea31c8baf561f4b841c814b65fd8ba7d2b5973..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 218 zcmb8oy$QoG5J2Hoynx|k`Fzp@$p$He3*8?s1cIbVwvTWYh+qHl%Im0Mc5=^_n3xxX z%mg`NMWuuWBB)uRvcrV?-CFDAR0INK5|Vk!f^k6RA<1Li;<~IMSa8*$9{t(0yhk(q Q*+l!Tcjb>=ylubR7l*+-`Tzg` diff --git a/pkg/blobserver/diskpacked/testdata/pack-00001.blobs b/pkg/blobserver/diskpacked/testdata/pack-00001.blobs deleted file mode 100644 index 6fba9e6b40463298db2aa4904cb7acc5aa602e05..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 158 zcmXBL%L&6k3