diff --git a/clients/go/camsync/camsync.go b/clients/go/camsync/camsync.go index 02a52ae69..ba2976f58 100644 --- a/clients/go/camsync/camsync.go +++ b/clients/go/camsync/camsync.go @@ -42,63 +42,6 @@ func usage(err string) { os.Exit(2) } -// TODO: use Generics if/when available -type chanPeeker struct { - ch chan *blobref.SizedBlobRef - peek *blobref.SizedBlobRef - Closed bool -} - -func (cp *chanPeeker) Peek() *blobref.SizedBlobRef { - if cp.Closed { - return nil - } - if cp.peek != nil { - return cp.peek - } - cp.peek = <-cp.ch - if closed(cp.ch) { - cp.Closed = true - return nil - } - return cp.peek -} - -func (cp *chanPeeker) Take() *blobref.SizedBlobRef { - v := cp.Peek() - cp.peek = nil - return v -} - -func yieldMissingDestinationBlobs(destMissing, srcch, dstch chan *blobref.SizedBlobRef) { - defer close(destMissing) - - src := &chanPeeker{ch: srcch} - dst := &chanPeeker{ch: dstch} - - for src.Peek() != nil { - // If the destination has reached its end, anything - // remaining in the source is needed. - if dst.Peek() == nil { - destMissing <- src.Take() - continue - } - - srcStr := src.Peek().BlobRef.String() - dstStr := dst.Peek().BlobRef.String() - switch { - case srcStr == dstStr: - // Skip both - src.Take() - dst.Take() - case srcStr < dstStr: - src.Take() - case srcStr > dstStr: - destMissing <- src.Take() - } - } -} - func main() { flag.Parse() @@ -138,7 +81,7 @@ func main() { // Merge sort srcBlobs and destBlobs destNotHaveBlobs := make(chan *blobref.SizedBlobRef, 100) - go yieldMissingDestinationBlobs(destNotHaveBlobs, srcBlobs, destBlobs) + go client.ListMissingDestinationBlobs(destNotHaveBlobs, srcBlobs, destBlobs) for sb := range destNotHaveBlobs { fmt.Printf("Destination needs blob: %s\n", sb) diff --git a/lib/go/client/Makefile b/lib/go/client/Makefile index e0fbc87fe..58eed30a6 100644 --- a/lib/go/client/Makefile +++ b/lib/go/client/Makefile @@ -9,5 +9,6 @@ GOFILES=\ enumerate.go\ get.go\ upload.go\ + sync.go\ include $(GOROOT)/src/Make.pkg diff --git a/lib/go/client/sync.go b/lib/go/client/sync.go new file mode 100644 index 000000000..497a4aa02 --- /dev/null +++ b/lib/go/client/sync.go @@ -0,0 +1,83 @@ +/* +Copyright 2011 Google Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package client + +import ( + "camli/blobref" +) + +// TODO: use Generics if/when available +type chanPeeker struct { + ch chan *blobref.SizedBlobRef + peek *blobref.SizedBlobRef + Closed bool +} + +func (cp *chanPeeker) Peek() *blobref.SizedBlobRef { + if cp.Closed { + return nil + } + if cp.peek != nil { + return cp.peek + } + cp.peek = <-cp.ch + if closed(cp.ch) { + cp.Closed = true + return nil + } + return cp.peek +} + +func (cp *chanPeeker) Take() *blobref.SizedBlobRef { + v := cp.Peek() + cp.peek = nil + return v +} + +// ListMissingDestinationBlobs reads from 'srcch' and 'dstch' (sorted +// enumerations of blobs from two blob servers) and sends to +// 'destMissing' any blobs which appear on the source but not at the +// destination. destMissing is closed at the end. +func ListMissingDestinationBlobs(destMissing, srcch, dstch chan *blobref.SizedBlobRef) { + defer close(destMissing) + + src := &chanPeeker{ch: srcch} + dst := &chanPeeker{ch: dstch} + + for src.Peek() != nil { + // If the destination has reached its end, anything + // remaining in the source is needed. + if dst.Peek() == nil { + destMissing <- src.Take() + continue + } + + srcStr := src.Peek().BlobRef.String() + dstStr := dst.Peek().BlobRef.String() + + switch { + case srcStr == dstStr: + // Skip both + src.Take() + dst.Take() + case srcStr < dstStr: + destMissing <- src.Take() + case srcStr > dstStr: + dst.Take() + } + } +} diff --git a/lib/go/client/sync_test.go b/lib/go/client/sync_test.go new file mode 100644 index 000000000..2eb75b49a --- /dev/null +++ b/lib/go/client/sync_test.go @@ -0,0 +1,78 @@ +/* +Copyright 2011 Google Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package client + +import ( + "camli/blobref" + "strings" + "testing" +) + +type lmdbTest struct { + source, dest, expectedMissing string // comma-separated list of blobref strings +} + +func (lt *lmdbTest) run(t *testing.T) { + srcBlobs := make(chan *blobref.SizedBlobRef, 100) + destBlobs := make(chan *blobref.SizedBlobRef, 100) + sendTestBlobs(srcBlobs, lt.source) + sendTestBlobs(destBlobs, lt.dest) + + missing := make(chan *blobref.SizedBlobRef, 100) + got := make([]string, 0) + go ListMissingDestinationBlobs(missing, srcBlobs, destBlobs) + for sb := range missing { + got = append(got, sb.BlobRef.String()) + } + gotJoined := strings.Join(got, ",") + if gotJoined != lt.expectedMissing { + t.Errorf("For %q and %q expected %q, got %q", + lt.source, lt.dest, lt.expectedMissing, gotJoined) + } +} + +func sendTestBlobs(ch chan *blobref.SizedBlobRef, list string) { + defer close(ch) + if list == "" { + return + } + for _, b := range strings.Split(list, ",", -1) { + br := blobref.Parse(b) + if br == nil { + panic("Invalid blobref: " + b) + } + ch <- &blobref.SizedBlobRef{BlobRef: br, Size: 123} + } +} + +func TestListMissingDestinationBlobs(t *testing.T) { + tests := []lmdbTest{ + { "foo-a,foo-b,foo-c", "", "foo-a,foo-b,foo-c" }, + { "foo-a,foo-b,foo-c", "foo-a", "foo-b,foo-c" }, + { "foo-a,foo-b,foo-c", "foo-b", "foo-a,foo-c" }, + { "foo-a,foo-b,foo-c", "foo-c", "foo-a,foo-b" }, + { "foo-a,foo-b,foo-c", "foo-a,foo-b", "foo-c" }, + { "foo-a,foo-b,foo-c", "foo-b,foo-c", "foo-a" }, + { "foo-a,foo-b,foo-c", "foo-a,foo-b,foo-c", "" }, + { "", "foo-a,foo-b,foo-c", "" }, + { "foo-f", "foo-a,foo-b,foo-c", "foo-f" }, + } + + for _, test := range tests { + test.run(t) + } +}