sync: move to library, add tests, fix merge bug

clients/go/camsync/camsync.go

@@ -42,63 +42,6 @@ func usage(err string) {
 	os.Exit(2)
 }
 
-// TODO: use Generics if/when available
-type chanPeeker struct {
-	ch     chan *blobref.SizedBlobRef
-	peek   *blobref.SizedBlobRef
-	Closed bool
-}
-
-func (cp *chanPeeker) Peek() *blobref.SizedBlobRef {
-	if cp.Closed {
-		return nil
-	}
-	if cp.peek != nil {
-		return cp.peek
-	}
-	cp.peek = <-cp.ch
-	if closed(cp.ch) {
-		cp.Closed = true
-		return nil
-	}
-	return cp.peek
-}
-
-func (cp *chanPeeker) Take() *blobref.SizedBlobRef {
-	v := cp.Peek()
-	cp.peek = nil
-	return v
-}
-
-func yieldMissingDestinationBlobs(destMissing, srcch, dstch chan *blobref.SizedBlobRef) {
-	defer close(destMissing)
-
-	src := &chanPeeker{ch: srcch}
-	dst := &chanPeeker{ch: dstch}
-
-	for src.Peek() != nil {
-		// If the destination has reached its end, anything
-		// remaining in the source is needed.
-		if dst.Peek() == nil {
-			destMissing <- src.Take()
-			continue
-		}
-
-		srcStr := src.Peek().BlobRef.String()
-		dstStr := dst.Peek().BlobRef.String()
-		switch {
-		case srcStr == dstStr:
-			// Skip both
-			src.Take()
-			dst.Take()
-		case srcStr < dstStr:
-			src.Take()
-		case srcStr > dstStr:
-			destMissing <- src.Take()
-		}
-	}
-}
-
 func main() {
 	flag.Parse()
 
@@ -138,7 +81,7 @@ func main() {
 
 		// Merge sort srcBlobs and destBlobs
 		destNotHaveBlobs := make(chan *blobref.SizedBlobRef, 100)
-		go yieldMissingDestinationBlobs(destNotHaveBlobs, srcBlobs, destBlobs)
+		go client.ListMissingDestinationBlobs(destNotHaveBlobs, srcBlobs, destBlobs)
 		for sb := range destNotHaveBlobs {
 			fmt.Printf("Destination needs blob: %s\n", sb)
 

lib/go/client/Makefile

@@ -9,5 +9,6 @@ GOFILES=\
 	enumerate.go\
 	get.go\
 	upload.go\
+	sync.go\
 
 include $(GOROOT)/src/Make.pkg

lib/go/client/sync.go

@@ -0,0 +1,83 @@
+/*
+Copyright 2011 Google Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package client
+
+import (
+	"camli/blobref"
+)
+
+// TODO: use Generics if/when available
+type chanPeeker struct {
+	ch     chan *blobref.SizedBlobRef
+	peek   *blobref.SizedBlobRef
+	Closed bool
+}
+
+func (cp *chanPeeker) Peek() *blobref.SizedBlobRef {
+	if cp.Closed {
+		return nil
+	}
+	if cp.peek != nil {
+		return cp.peek
+	}
+	cp.peek = <-cp.ch
+	if closed(cp.ch) {
+		cp.Closed = true
+		return nil
+	}
+	return cp.peek
+}
+
+func (cp *chanPeeker) Take() *blobref.SizedBlobRef {
+	v := cp.Peek()
+	cp.peek = nil
+	return v
+}
+
+// ListMissingDestinationBlobs reads from 'srcch' and 'dstch' (sorted
+// enumerations of blobs from two blob servers) and sends to
+// 'destMissing' any blobs which appear on the source but not at the
+// destination.  destMissing is closed at the end.
+func ListMissingDestinationBlobs(destMissing, srcch, dstch chan *blobref.SizedBlobRef) {
+	defer close(destMissing)
+
+	src := &chanPeeker{ch: srcch}
+	dst := &chanPeeker{ch: dstch}
+
+	for src.Peek() != nil {
+		// If the destination has reached its end, anything
+		// remaining in the source is needed.
+		if dst.Peek() == nil {
+			destMissing <- src.Take()
+			continue
+		}
+
+		srcStr := src.Peek().BlobRef.String()
+		dstStr := dst.Peek().BlobRef.String()
+
+		switch {
+		case srcStr == dstStr:
+			// Skip both
+			src.Take()
+			dst.Take()
+		case srcStr < dstStr:
+			destMissing <- src.Take()
+		case srcStr > dstStr:
+			dst.Take()
+		}
+	}
+}

lib/go/client/sync_test.go

@@ -0,0 +1,78 @@
+/*
+Copyright 2011 Google Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package client
+
+import (
+	"camli/blobref"
+	"strings"
+	"testing"
+)
+
+type lmdbTest struct {
+	source, dest, expectedMissing string // comma-separated list of blobref strings
+}
+
+func (lt *lmdbTest) run(t *testing.T) {
+	srcBlobs := make(chan *blobref.SizedBlobRef, 100)
+	destBlobs := make(chan *blobref.SizedBlobRef, 100)
+	sendTestBlobs(srcBlobs, lt.source)
+	sendTestBlobs(destBlobs, lt.dest)
+
+	missing := make(chan *blobref.SizedBlobRef, 100)
+	got := make([]string, 0)
+	go ListMissingDestinationBlobs(missing, srcBlobs, destBlobs)
+	for sb := range missing {
+		got = append(got, sb.BlobRef.String())
+	}
+	gotJoined := strings.Join(got, ",")
+	if gotJoined != lt.expectedMissing {
+		t.Errorf("For %q and %q expected %q, got %q",
+			lt.source, lt.dest, lt.expectedMissing, gotJoined)
+	}
+}
+
+func sendTestBlobs(ch chan *blobref.SizedBlobRef, list string) {
+	defer close(ch)
+	if list == "" {
+		return
+	}
+	for _, b := range strings.Split(list, ",", -1) {
+		br := blobref.Parse(b)
+		if br == nil {
+			panic("Invalid blobref: " + b)
+		}
+		ch <- &blobref.SizedBlobRef{BlobRef: br, Size: 123}
+	}
+}
+
+func TestListMissingDestinationBlobs(t *testing.T) {
+	tests := []lmdbTest{
+		{ "foo-a,foo-b,foo-c", "", "foo-a,foo-b,foo-c" },
+		{ "foo-a,foo-b,foo-c", "foo-a", "foo-b,foo-c" },
+		{ "foo-a,foo-b,foo-c", "foo-b", "foo-a,foo-c" },
+		{ "foo-a,foo-b,foo-c", "foo-c", "foo-a,foo-b" },
+		{ "foo-a,foo-b,foo-c", "foo-a,foo-b", "foo-c" },
+		{ "foo-a,foo-b,foo-c", "foo-b,foo-c", "foo-a" },
+		{ "foo-a,foo-b,foo-c", "foo-a,foo-b,foo-c", "" },
+		{ "", "foo-a,foo-b,foo-c", "" },
+		{ "foo-f", "foo-a,foo-b,foo-c", "foo-f" },
+	}
+
+	for _, test := range tests {
+		test.run(t)
+	}
+}