From 7254e8156097e6315a242d04fd4c036df69f8105 Mon Sep 17 00:00:00 2001 From: mpl Date: Sat, 12 Nov 2016 01:45:42 +0100 Subject: [PATCH] pkg/misc/amazon/s3: test against fake-s3 in docker Fixes #424 Change-Id: Ib13946df3a5d868e10519576725e4d365ce27f64 --- misc/docker/fakes3/Dockerfile | 15 ++++++ misc/docker/fakes3/Makefile | 9 ++++ pkg/misc/amazon/s3/client.go | 14 ++++-- pkg/misc/amazon/s3/client_test.go | 55 +++++++++++++++++---- pkg/test/dockertest/docker.go | 81 +++++++++++++++++++++++++++++++ 5 files changed, 161 insertions(+), 13 deletions(-) create mode 100644 misc/docker/fakes3/Dockerfile create mode 100644 misc/docker/fakes3/Makefile diff --git a/misc/docker/fakes3/Dockerfile b/misc/docker/fakes3/Dockerfile new file mode 100644 index 000000000..96ec32915 --- /dev/null +++ b/misc/docker/fakes3/Dockerfile @@ -0,0 +1,15 @@ +# Copyright 2016 The Camlistore Authors. +FROM debian:jessie + +ENV DEBIAN_FRONTEND noninteractive + +RUN apt-get update && apt-get install -yqq git ruby ruby-builder ruby-thor +WORKDIR /usr/local/src/github.com/jubos +RUN git clone https://github.com/jubos/fake-s3.git +WORKDIR fake-s3 +RUN git reset --hard 8f7ba5512acba8072654dc7d8964a9a5bebce8a9 + +RUN mkdir -p /fakes3_root +ENTRYPOINT ["/usr/local/src/github.com/jubos/fake-s3/bin/fakes3"] +CMD ["-r", "/fakes3_root", "-p", "4567"] +EXPOSE 4567 diff --git a/misc/docker/fakes3/Makefile b/misc/docker/fakes3/Makefile new file mode 100644 index 000000000..64524237e --- /dev/null +++ b/misc/docker/fakes3/Makefile @@ -0,0 +1,9 @@ +fakes3: + docker build -t camlistore/fakes3 . + +upload: fakes3 + docker save camlistore/fakes3 | gzip > fakes3.tar.gz + gsutil cp fakes3.tar.gz gs://camlistore-docker/ + +clean: + rm fakes3.tar.gz diff --git a/pkg/misc/amazon/s3/client.go b/pkg/misc/amazon/s3/client.go index f64bc7a18..ccaa4c9b3 100644 --- a/pkg/misc/amazon/s3/client.go +++ b/pkg/misc/amazon/s3/client.go @@ -51,6 +51,7 @@ type Client struct { // apparently S3 throttles us if there are too many. No limit if nil. // Default in S3 blobserver is 5. PutGate *syncutil.Gate + NoSSL bool // disable SSL. For testing against fake-s3. } type Bucket struct { @@ -65,12 +66,19 @@ func (c *Client) transport() http.RoundTripper { return http.DefaultTransport } +func (c *Client) scheme() string { + if c.NoSSL { + return "http://" + } + return "https://" +} + // bucketURL returns the URL prefix of the bucket, with trailing slash func (c *Client) bucketURL(bucket string) string { if IsValidBucket(bucket) && !strings.Contains(bucket, ".") { - return fmt.Sprintf("https://%s.%s/", bucket, c.hostname()) + return fmt.Sprintf("%s%s.%s/", c.scheme(), bucket, c.hostname()) } - return fmt.Sprintf("https://%s/%s/", c.hostname(), bucket) + return fmt.Sprintf("%s%s/%s/", c.scheme(), c.hostname(), bucket) } func (c *Client) keyURL(bucket, key string) string { @@ -87,7 +95,7 @@ func newReq(url_ string) *http.Request { } func (c *Client) Buckets() ([]*Bucket, error) { - req := newReq("https://" + c.hostname() + "/") + req := newReq(c.scheme() + c.hostname() + "/") c.Auth.SignRequest(req) res, err := c.transport().RoundTrip(req) if err != nil { diff --git a/pkg/misc/amazon/s3/client_test.go b/pkg/misc/amazon/s3/client_test.go index 2e6f63def..bdd2a7733 100644 --- a/pkg/misc/amazon/s3/client_test.go +++ b/pkg/misc/amazon/s3/client_test.go @@ -17,37 +17,55 @@ limitations under the License. package s3 import ( + "bytes" + "crypto/md5" + "io" "net/http" "os" "reflect" "strings" "testing" + "camlistore.org/pkg/test/dockertest" + "go4.org/syncutil" ) -var tc *Client +var ( + tc *Client + containerID dockertest.ContainerID // for running fake-s3 +) -func getTestClient(t *testing.T) bool { +func getTestClient(t *testing.T) { accessKey := os.Getenv("AWS_ACCESS_KEY_ID") secret := os.Getenv("AWS_ACCESS_KEY_SECRET") - if accessKey == "" || secret == "" { - t.Logf("Skipping test; no AWS_ACCESS_KEY_ID or AWS_ACCESS_KEY_SECRET set in environment") - return false + if accessKey != "" && secret != "" { + tc = &Client{ + Auth: &Auth{AccessKey: accessKey, SecretAccessKey: secret}, + Transport: http.DefaultTransport, + PutGate: syncutil.NewGate(5), + } + return } + t.Logf("no AWS_ACCESS_KEY_ID or AWS_ACCESS_KEY_SECRET set in environment; trying against local fakes3 instead.") + var ip string + containerID, ip = dockertest.SetupFakeS3Container(t) + hostname := ip + ":4567" tc = &Client{ - Auth: &Auth{AccessKey: accessKey, SecretAccessKey: secret}, + Auth: &Auth{AccessKey: "foo", SecretAccessKey: "bar", Hostname: hostname}, Transport: http.DefaultTransport, PutGate: syncutil.NewGate(5), + NoSSL: true, } - return true } func TestBuckets(t *testing.T) { - if !getTestClient(t) { - return + getTestClient(t) + defer containerID.KillRemove(t) + _, err := tc.Buckets() + if err != nil { + t.Fatal(err) } - tc.Buckets() } func TestParseBuckets(t *testing.T) { @@ -99,3 +117,20 @@ func TestValidBucketNames(t *testing.T) { } } } + +func TestPutObject(t *testing.T) { + getTestClient(t) + defer containerID.KillRemove(t) + var buf bytes.Buffer + md5h := md5.New() + + size, err := io.Copy(io.MultiWriter(&buf, md5h), strings.NewReader("hello world")) + if err != nil { + t.Fatal(err) + } + // TODO(mpl): figure how to make fake-s3 work with buckets. + if err = tc.PutObject("hello.txt", "", md5h, size, &buf); err != nil { + t.Fatal(err) + } + // TODO(mpl): figure out why Stat of newly uploaded object does not match size from above. +} diff --git a/pkg/test/dockertest/docker.go b/pkg/test/dockertest/docker.go index e84d8af67..9cf14ea5a 100644 --- a/pkg/test/dockertest/docker.go +++ b/pkg/test/dockertest/docker.go @@ -21,11 +21,15 @@ package dockertest // import "camlistore.org/pkg/test/dockertest" import ( "bytes" + "compress/gzip" "database/sql" "encoding/json" "errors" "fmt" + "io" "log" + "net/http" + "os" "os/exec" "strings" "testing" @@ -51,12 +55,75 @@ func runLongTest(t *testing.T, image string) { t.Skipf("Error running docker to check for %s: %v", image, err) } log.Printf("Pulling docker image %s ...", image) + if strings.HasPrefix(image, "camlistore/") { + if err := loadCamliHubImage(image); err != nil { + t.Skipf("Error pulling %s: %v", image, err) + } + return + } if err := Pull(image); err != nil { t.Skipf("Error pulling %s: %v", image, err) } } } +// loadCamliHubImage fetches a docker image saved as a .tar.gz in the +// camlistore-docker bucket, and loads it in docker. +func loadCamliHubImage(image string) error { + if !strings.HasPrefix(image, "camlistore/") { + return fmt.Errorf("not an image hosted on camlistore-docker") + } + imgURL := camliHub + strings.TrimPrefix(image, "camlistore/") + ".tar.gz" + resp, err := http.Get(imgURL) + if err != nil { + return fmt.Errorf("error fetching image %s: %v", image, err) + } + defer resp.Body.Close() + + dockerLoad := exec.Command("docker", "load") + dockerLoad.Stderr = os.Stderr + tar, err := dockerLoad.StdinPipe() + if err != nil { + return err + } + errc1 := make(chan error) + errc2 := make(chan error) + go func() { + defer tar.Close() + zr, err := gzip.NewReader(resp.Body) + if err != nil { + errc1 <- fmt.Errorf("gzip reader error for image %s: %v", image, err) + return + } + defer zr.Close() + if _, err = io.Copy(tar, zr); err != nil { + errc1 <- fmt.Errorf("error gunzipping image %s: %v", image, err) + return + } + errc1 <- nil + }() + go func() { + if err := dockerLoad.Run(); err != nil { + errc2 <- fmt.Errorf("error running docker load %v: %v", image, err) + return + } + errc2 <- nil + }() + select { + case err := <-errc1: + if err != nil { + return err + } + return <-errc2 + case err := <-errc2: + if err != nil { + return err + } + return <-errc1 + } + return nil +} + // haveDocker returns whether the "docker" command was found. func haveDocker() bool { _, err := exec.LookPath("docker") @@ -139,6 +206,9 @@ func (c ContainerID) IP() (string, error) { } func (c ContainerID) Kill() error { + if string(c) == "" { + return nil + } return KillContainer(string(c)) } @@ -147,6 +217,9 @@ func (c ContainerID) Remove() error { if Debug { return nil } + if string(c) == "" { + return nil + } return exec.Command("docker", "rm", "-v", string(c)).Run() } @@ -204,8 +277,16 @@ const ( postgresImage = "nornagon/postgres" PostgresUsername = "docker" // set up by the dockerfile of postgresImage PostgresPassword = "docker" // set up by the dockerfile of postgresImage + camliHub = "https://storage.googleapis.com/camlistore-docker/" + fakeS3Image = "camlistore/fakes3" ) +func SetupFakeS3Container(t *testing.T) (c ContainerID, ip string) { + return setupContainer(t, fakeS3Image, 4567, 10*time.Second, func() (string, error) { + return run("-d", fakeS3Image) + }) +} + // SetupMongoContainer sets up a real MongoDB instance for testing purposes, // using a Docker container. It returns the container ID and its IP address, // or makes the test fail on error.