From cb742f43f92f72a9508fcc7697094bcbf576e3d1 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Sun, 27 Mar 2011 15:25:21 -0700 Subject: [PATCH] Start of Amazon S3 library. For now signing + tests. --- build.pl | 1 + lib/go/camli/misc/amazon/s3/auth.go | 141 +++++++++++++++++++++++ lib/go/camli/misc/amazon/s3/auth_test.go | 111 ++++++++++++++++++ 3 files changed, 253 insertions(+) create mode 100644 lib/go/camli/misc/amazon/s3/auth.go create mode 100644 lib/go/camli/misc/amazon/s3/auth_test.go diff --git a/build.pl b/build.pl index 95b8879ad..b0b4d142a 100755 --- a/build.pl +++ b/build.pl @@ -406,6 +406,7 @@ TARGET: lib/go/camli/jsonsign TARGET: lib/go/camli/lru TARGET: lib/go/camli/magic TARGET: lib/go/camli/misc/httprange +TARGET: lib/go/camli/misc/amazon/s3 TARGET: lib/go/camli/mysqlindexer TARGET: lib/go/camli/schema TARGET: lib/go/camli/search diff --git a/lib/go/camli/misc/amazon/s3/auth.go b/lib/go/camli/misc/amazon/s3/auth.go new file mode 100644 index 000000000..f3ef32e17 --- /dev/null +++ b/lib/go/camli/misc/amazon/s3/auth.go @@ -0,0 +1,141 @@ +/* +Copyright 2011 Google Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package s3 + +import ( + "bytes" + "http" + "sort" + "strings" +) + +// See http://docs.amazonwebservices.com/AmazonS3/latest/dev/index.html?RESTAuthentication.html + +type Auth struct { + AccessKey string + SecretAccessKey string +} + +func firstNonEmptyString(strs ...string) string { + for _, s := range strs { + if s != "" { + return s + } + } + return "" +} + +// From the Amazon docs: +// +// StringToSign = HTTP-Verb + "\n" + +// Content-MD5 + "\n" + +// Content-Type + "\n" + +// Date + "\n" + +// CanonicalizedAmzHeaders + +// CanonicalizedResource; +func stringToSign(req *http.Request) string { + buf := new(bytes.Buffer) + buf.WriteString(req.Method) + buf.WriteByte('\n') + buf.WriteString(req.Header.Get("Content-MD5")) + buf.WriteByte('\n') + buf.WriteString(req.Header.Get("Content-Type")) + buf.WriteByte('\n') + if req.Header.Get("x-amz-date") == "" { + buf.WriteString(req.Header.Get("Date")) + } + buf.WriteByte('\n') + writeCanonicalizedAmzHeaders(buf, req) + writeCanonicalizedResource(buf, req) + return buf.String() +} + +func hasPrefixCaseInsensitive(s, pfx string) bool { + if len(pfx) > len(s) { + return false + } + shead := s[:len(pfx)] + if shead == pfx { + return true + } + shead = strings.ToLower(shead) + return shead == pfx || shead == strings.ToLower(pfx) +} + +func writeCanonicalizedAmzHeaders(buf *bytes.Buffer, req *http.Request) { + amzHeaders := make([]string, 0) + vals := make(map[string][]string) + for k, vv := range req.Header { + if hasPrefixCaseInsensitive(k, "x-amz-") { + lk := strings.ToLower(k) + amzHeaders = append(amzHeaders, lk) + vals[lk] = vv + } + } + sort.SortStrings(amzHeaders) + for _, k := range amzHeaders { + buf.WriteString(k) + buf.WriteByte(':') + for idx, v := range vals[k] { + if idx > 0 { + buf.WriteByte(',') + } + if strings.Contains(v, "\n") { + // TODO: "Unfold" long headers that + // span multiple lines (as allowed by + // RFC 2616, section 4.2) by replacing + // the folding white-space (including + // new-line) by a single space. + buf.WriteString(v) + } else { + buf.WriteString(v) + } + } + buf.WriteByte('\n') + } +} + +// From the Amazon docs: +// +// CanonicalizedResource = [ "/" + Bucket ] + +// + +// [ sub-resource, if present. For example "?acl", "?location", "?logging", or "?torrent"]; +func writeCanonicalizedResource(buf *bytes.Buffer, req *http.Request) { + if bucket := bucketFromHostname(req); bucket != "" { + buf.WriteByte('/') + buf.WriteString(bucket) + } + buf.WriteString(req.URL.Path) + // TODO: subresource +} + +const standardUSRegionAWSSuffix = ".s3.amazonaws.com" +const standardUSRegionAWS = "s3.amazonaws.com" + +func bucketFromHostname(req *http.Request) string { + host := req.Host + if host == standardUSRegionAWS { + return "" + } + if strings.HasSuffix(host, standardUSRegionAWSSuffix) { + return host[:len(host)-len(standardUSRegionAWSSuffix)] + } + if lastColon := strings.LastIndex(host, ":"); lastColon != -1 { + return host[:lastColon] + } + return host +} diff --git a/lib/go/camli/misc/amazon/s3/auth_test.go b/lib/go/camli/misc/amazon/s3/auth_test.go new file mode 100644 index 000000000..43f87b00b --- /dev/null +++ b/lib/go/camli/misc/amazon/s3/auth_test.go @@ -0,0 +1,111 @@ +/* +Copyright 2011 Google Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package s3 + +import ( + "bufio" + "fmt" + "http" + "strings" + "testing" +) + +type reqAndExpected struct { + req, expected string +} + +func req(s string) *http.Request { + req, err := http.ReadRequest(bufio.NewReader(strings.NewReader(s))) + if err != nil { + panic(fmt.Sprintf("bad request in test: %q (error: %v)", req, err)) + } + return req +} + +func TestStringToSign(t *testing.T) { + tests := []reqAndExpected{ + {`GET /photos/puppy.jpg HTTP/1.1 +Host: johnsmith.s3.amazonaws.com +Date: Tue, 27 Mar 2007 19:36:42 +0000 + +`, + "GET\n\n\nTue, 27 Mar 2007 19:36:42 +0000\n/johnsmith/photos/puppy.jpg"}, + {`PUT /photos/puppy.jpg HTTP/1.1 +Content-Type: image/jpeg +Content-Length: 94328 +Host: johnsmith.s3.amazonaws.com +Date: Tue, 27 Mar 2007 21:15:45 +0000 + +`, + "PUT\n\nimage/jpeg\nTue, 27 Mar 2007 21:15:45 +0000\n/johnsmith/photos/puppy.jpg"}, + {`GET /?prefix=photos&max-keys=50&marker=puppy HTTP/1.1 +User-Agent: Mozilla/5.0 +Host: johnsmith.s3.amazonaws.com +Date: Tue, 27 Mar 2007 19:42:41 +0000 + +`, + "GET\n\n\nTue, 27 Mar 2007 19:42:41 +0000\n/johnsmith/"}, + {`DELETE /johnsmith/photos/puppy.jpg HTTP/1.1 +User-Agent: dotnet +Host: s3.amazonaws.com +Date: Tue, 27 Mar 2007 21:20:27 +0000 +x-amz-date: Tue, 27 Mar 2007 21:20:26 +0000 + +`, + "DELETE\n\n\n\nx-amz-date:Tue, 27 Mar 2007 21:20:26 +0000\n/johnsmith/photos/puppy.jpg"}, + {`PUT /db-backup.dat.gz HTTP/1.1 +User-Agent: curl/7.15.5 +Host: static.johnsmith.net:8080 +Date: Tue, 27 Mar 2007 21:06:08 +0000 +x-amz-acl: public-read +content-type: application/x-download +Content-MD5: 4gJE4saaMU4BqNR0kLY+lw== +X-Amz-Meta-ReviewedBy: joe@johnsmith.net +X-Amz-Meta-ReviewedBy: jane@johnsmith.net +X-Amz-Meta-FileChecksum: 0x02661779 +X-Amz-Meta-ChecksumAlgorithm: crc32 +Content-Disposition: attachment; filename=database.dat +Content-Encoding: gzip +Content-Length: 5913339 + +`, + "PUT\n4gJE4saaMU4BqNR0kLY+lw==\napplication/x-download\nTue, 27 Mar 2007 21:06:08 +0000\nx-amz-acl:public-read\nx-amz-meta-checksumalgorithm:crc32\nx-amz-meta-filechecksum:0x02661779\nx-amz-meta-reviewedby:joe@johnsmith.net,jane@johnsmith.net\n/static.johnsmith.net/db-backup.dat.gz"}, + } + for idx, test := range tests { + got := stringToSign(req(test.req)) + if got != test.expected { + t.Errorf("test %d: expected %q", idx, test.expected) + t.Errorf("test %d: got %q", idx, got) + } + } +} + +func TestBucketFromHostname(t *testing.T) { + tests := []reqAndExpected{ + {"GET / HTTP/1.0\n\n", ""}, + {"GET / HTTP/1.0\nHost: s3.amazonaws.com\n\n", ""}, + {"GET / HTTP/1.0\nHost: foo.s3.amazonaws.com\n\n", "foo"}, + {"GET / HTTP/1.0\nHost: foo.com:123\n\n", "foo.com"}, + {"GET / HTTP/1.0\nHost: bar.com\n\n", "bar.com"}, + } + for idx, test := range tests { + got := bucketFromHostname(req(test.req)) + if got != test.expected { + t.Errorf("test %d: expected %q; got %q", idx, test.expected, got) + } + } +}