pkg/misc/amazon/s3: test against fake-s3 in docker

Fixes #424

Change-Id: Ib13946df3a5d868e10519576725e4d365ce27f64

misc/docker/fakes3/Dockerfile

@@ -0,0 +1,15 @@
+# Copyright 2016 The Camlistore Authors.
+FROM debian:jessie
+
+ENV DEBIAN_FRONTEND noninteractive
+
+RUN apt-get update && apt-get install -yqq git ruby ruby-builder ruby-thor
+WORKDIR /usr/local/src/github.com/jubos
+RUN git clone https://github.com/jubos/fake-s3.git
+WORKDIR fake-s3
+RUN git reset --hard 8f7ba5512acba8072654dc7d8964a9a5bebce8a9
+
+RUN mkdir -p /fakes3_root
+ENTRYPOINT ["/usr/local/src/github.com/jubos/fake-s3/bin/fakes3"]
+CMD ["-r",  "/fakes3_root", "-p",  "4567"]
+EXPOSE 4567

misc/docker/fakes3/Makefile

@@ -0,0 +1,9 @@
+fakes3:
+	docker build -t camlistore/fakes3 .
+
+upload: fakes3
+	docker save camlistore/fakes3 | gzip > fakes3.tar.gz
+	gsutil cp fakes3.tar.gz gs://camlistore-docker/
+
+clean:
+	rm fakes3.tar.gz

pkg/misc/amazon/s3/client.go

@@ -51,6 +51,7 @@ type Client struct {
 	// apparently S3 throttles us if there are too many. No limit if nil.
 	// Default in S3 blobserver is 5.
 	PutGate *syncutil.Gate
+	NoSSL   bool // disable SSL. For testing against fake-s3.
 }
 
 type Bucket struct {
@@ -65,12 +66,19 @@ func (c *Client) transport() http.RoundTripper {
 	return http.DefaultTransport
 }
 
+func (c *Client) scheme() string {
+	if c.NoSSL {
+		return "http://"
+	}
+	return "https://"
+}
+
 // bucketURL returns the URL prefix of the bucket, with trailing slash
 func (c *Client) bucketURL(bucket string) string {
 	if IsValidBucket(bucket) && !strings.Contains(bucket, ".") {
-		return fmt.Sprintf("https://%s.%s/", bucket, c.hostname())
+		return fmt.Sprintf("%s%s.%s/", c.scheme(), bucket, c.hostname())
 	}
-	return fmt.Sprintf("https://%s/%s/", c.hostname(), bucket)
+	return fmt.Sprintf("%s%s/%s/", c.scheme(), c.hostname(), bucket)
 }
 
 func (c *Client) keyURL(bucket, key string) string {
@@ -87,7 +95,7 @@ func newReq(url_ string) *http.Request {
 }
 
 func (c *Client) Buckets() ([]*Bucket, error) {
-	req := newReq("https://" + c.hostname() + "/")
+	req := newReq(c.scheme() + c.hostname() + "/")
 	c.Auth.SignRequest(req)
 	res, err := c.transport().RoundTrip(req)
 	if err != nil {

pkg/misc/amazon/s3/client_test.go

@@ -17,37 +17,55 @@ limitations under the License.
 package s3
 
 import (
+	"bytes"
+	"crypto/md5"
+	"io"
 	"net/http"
 	"os"
 	"reflect"
 	"strings"
 	"testing"
 
+	"camlistore.org/pkg/test/dockertest"
+
 	"go4.org/syncutil"
 )
 
-var tc *Client
+var (
+	tc          *Client
+	containerID dockertest.ContainerID // for running fake-s3
+)
 
-func getTestClient(t *testing.T) bool {
+func getTestClient(t *testing.T) {
 	accessKey := os.Getenv("AWS_ACCESS_KEY_ID")
 	secret := os.Getenv("AWS_ACCESS_KEY_SECRET")
-	if accessKey == "" || secret == "" {
+	if accessKey != "" && secret != "" {
-		t.Logf("Skipping test; no AWS_ACCESS_KEY_ID or AWS_ACCESS_KEY_SECRET set in environment")
-		return false
-	}
 		tc = &Client{
 			Auth:      &Auth{AccessKey: accessKey, SecretAccessKey: secret},
 			Transport: http.DefaultTransport,
 			PutGate:   syncutil.NewGate(5),
 		}
-	return true
+		return
+	}
+	t.Logf("no AWS_ACCESS_KEY_ID or AWS_ACCESS_KEY_SECRET set in environment; trying against local fakes3 instead.")
+	var ip string
+	containerID, ip = dockertest.SetupFakeS3Container(t)
+	hostname := ip + ":4567"
+	tc = &Client{
+		Auth:      &Auth{AccessKey: "foo", SecretAccessKey: "bar", Hostname: hostname},
+		Transport: http.DefaultTransport,
+		PutGate:   syncutil.NewGate(5),
+		NoSSL:     true,
+	}
 }
 
 func TestBuckets(t *testing.T) {
-	if !getTestClient(t) {
+	getTestClient(t)
-		return
+	defer containerID.KillRemove(t)
+	_, err := tc.Buckets()
+	if err != nil {
+		t.Fatal(err)
 	}
-	tc.Buckets()
 }
 
 func TestParseBuckets(t *testing.T) {
@@ -99,3 +117,20 @@ func TestValidBucketNames(t *testing.T) {
 		}
 	}
 }
+
+func TestPutObject(t *testing.T) {
+	getTestClient(t)
+	defer containerID.KillRemove(t)
+	var buf bytes.Buffer
+	md5h := md5.New()
+
+	size, err := io.Copy(io.MultiWriter(&buf, md5h), strings.NewReader("hello world"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	// TODO(mpl): figure how to make fake-s3 work with buckets.
+	if err = tc.PutObject("hello.txt", "", md5h, size, &buf); err != nil {
+		t.Fatal(err)
+	}
+	// TODO(mpl): figure out why Stat of newly uploaded object does not match size from above.
+}

pkg/test/dockertest/docker.go

@@ -21,11 +21,15 @@ package dockertest // import "camlistore.org/pkg/test/dockertest"
 
 import (
 	"bytes"
+	"compress/gzip"
 	"database/sql"
 	"encoding/json"
 	"errors"
 	"fmt"
+	"io"
 	"log"
+	"net/http"
+	"os"
 	"os/exec"
 	"strings"
 	"testing"
@@ -51,12 +55,75 @@ func runLongTest(t *testing.T, image string) {
 			t.Skipf("Error running docker to check for %s: %v", image, err)
 		}
 		log.Printf("Pulling docker image %s ...", image)
+		if strings.HasPrefix(image, "camlistore/") {
+			if err := loadCamliHubImage(image); err != nil {
+				t.Skipf("Error pulling %s: %v", image, err)
+			}
+			return
+		}
 		if err := Pull(image); err != nil {
 			t.Skipf("Error pulling %s: %v", image, err)
 		}
 	}
 }
 
+// loadCamliHubImage fetches a docker image saved as a .tar.gz in the
+// camlistore-docker bucket, and loads it in docker.
+func loadCamliHubImage(image string) error {
+	if !strings.HasPrefix(image, "camlistore/") {
+		return fmt.Errorf("not an image hosted on camlistore-docker")
+	}
+	imgURL := camliHub + strings.TrimPrefix(image, "camlistore/") + ".tar.gz"
+	resp, err := http.Get(imgURL)
+	if err != nil {
+		return fmt.Errorf("error fetching image %s: %v", image, err)
+	}
+	defer resp.Body.Close()
+
+	dockerLoad := exec.Command("docker", "load")
+	dockerLoad.Stderr = os.Stderr
+	tar, err := dockerLoad.StdinPipe()
+	if err != nil {
+		return err
+	}
+	errc1 := make(chan error)
+	errc2 := make(chan error)
+	go func() {
+		defer tar.Close()
+		zr, err := gzip.NewReader(resp.Body)
+		if err != nil {
+			errc1 <- fmt.Errorf("gzip reader error for image %s: %v", image, err)
+			return
+		}
+		defer zr.Close()
+		if _, err = io.Copy(tar, zr); err != nil {
+			errc1 <- fmt.Errorf("error gunzipping image %s: %v", image, err)
+			return
+		}
+		errc1 <- nil
+	}()
+	go func() {
+		if err := dockerLoad.Run(); err != nil {
+			errc2 <- fmt.Errorf("error running docker load %v: %v", image, err)
+			return
+		}
+		errc2 <- nil
+	}()
+	select {
+	case err := <-errc1:
+		if err != nil {
+			return err
+		}
+		return <-errc2
+	case err := <-errc2:
+		if err != nil {
+			return err
+		}
+		return <-errc1
+	}
+	return nil
+}
+
 // haveDocker returns whether the "docker" command was found.
 func haveDocker() bool {
 	_, err := exec.LookPath("docker")
@@ -139,6 +206,9 @@ func (c ContainerID) IP() (string, error) {
 }
 
 func (c ContainerID) Kill() error {
+	if string(c) == "" {
+		return nil
+	}
 	return KillContainer(string(c))
 }
 
@@ -147,6 +217,9 @@ func (c ContainerID) Remove() error {
 	if Debug {
 		return nil
 	}
+	if string(c) == "" {
+		return nil
+	}
 	return exec.Command("docker", "rm", "-v", string(c)).Run()
 }
 
@@ -204,8 +277,16 @@ const (
 	postgresImage    = "nornagon/postgres"
 	PostgresUsername = "docker" // set up by the dockerfile of postgresImage
 	PostgresPassword = "docker" // set up by the dockerfile of postgresImage
+	camliHub         = "https://storage.googleapis.com/camlistore-docker/"
+	fakeS3Image      = "camlistore/fakes3"
 )
 
+func SetupFakeS3Container(t *testing.T) (c ContainerID, ip string) {
+	return setupContainer(t, fakeS3Image, 4567, 10*time.Second, func() (string, error) {
+		return run("-d", fakeS3Image)
+	})
+}
+
 // SetupMongoContainer sets up a real MongoDB instance for testing purposes,
 // using a Docker container. It returns the container ID and its IP address,
 // or makes the test fail on error.