Start of Amazon S3 library. For now signing + tests.

build.pl

@@ -406,6 +406,7 @@ TARGET: lib/go/camli/jsonsign
 TARGET: lib/go/camli/lru
 TARGET: lib/go/camli/magic
 TARGET: lib/go/camli/misc/httprange
+TARGET: lib/go/camli/misc/amazon/s3
 TARGET: lib/go/camli/mysqlindexer
 TARGET: lib/go/camli/schema
 TARGET: lib/go/camli/search

lib/go/camli/misc/amazon/s3/auth.go

@@ -0,0 +1,141 @@
+/*
+Copyright 2011 Google Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package s3
+
+import (
+	"bytes"
+	"http"
+	"sort"
+	"strings"
+)
+
+// See http://docs.amazonwebservices.com/AmazonS3/latest/dev/index.html?RESTAuthentication.html
+
+type Auth struct {
+	AccessKey       string
+	SecretAccessKey string
+}
+
+func firstNonEmptyString(strs ...string) string {
+	for _, s := range strs {
+		if s != "" {
+			return s
+		}
+	}
+	return ""
+}
+
+// From the Amazon docs:
+//
+// StringToSign = HTTP-Verb + "\n" +
+// 	 Content-MD5 + "\n" +
+//	 Content-Type + "\n" +
+//	 Date + "\n" +
+//	 CanonicalizedAmzHeaders +
+//	 CanonicalizedResource;
+func stringToSign(req *http.Request) string {
+	buf := new(bytes.Buffer)
+	buf.WriteString(req.Method)
+	buf.WriteByte('\n')
+	buf.WriteString(req.Header.Get("Content-MD5"))
+	buf.WriteByte('\n')
+	buf.WriteString(req.Header.Get("Content-Type"))
+	buf.WriteByte('\n')
+	if req.Header.Get("x-amz-date") == "" {
+		buf.WriteString(req.Header.Get("Date"))
+	}
+	buf.WriteByte('\n')
+	writeCanonicalizedAmzHeaders(buf, req)
+	writeCanonicalizedResource(buf, req)
+	return buf.String()
+}
+
+func hasPrefixCaseInsensitive(s, pfx string) bool {
+	if len(pfx) > len(s) {
+		return false
+	}
+	shead := s[:len(pfx)]
+	if shead == pfx {
+		return true
+	}
+	shead = strings.ToLower(shead)
+	return shead == pfx || shead == strings.ToLower(pfx)
+}
+
+func writeCanonicalizedAmzHeaders(buf *bytes.Buffer, req *http.Request) {
+	amzHeaders := make([]string, 0)
+	vals := make(map[string][]string)
+	for k, vv := range req.Header {
+		if hasPrefixCaseInsensitive(k, "x-amz-") {
+			lk := strings.ToLower(k)
+			amzHeaders = append(amzHeaders, lk)
+			vals[lk] = vv
+		}
+	}
+	sort.SortStrings(amzHeaders)
+	for _, k := range amzHeaders {
+		buf.WriteString(k)
+		buf.WriteByte(':')
+		for idx, v := range vals[k] {
+			if idx > 0 {
+				buf.WriteByte(',')
+			}
+			if strings.Contains(v, "\n") {
+				// TODO: "Unfold" long headers that
+				// span multiple lines (as allowed by
+				// RFC 2616, section 4.2) by replacing
+				// the folding white-space (including
+				// new-line) by a single space.
+				buf.WriteString(v)
+			} else {
+				buf.WriteString(v)
+			}
+		}
+		buf.WriteByte('\n')
+	}
+}
+
+// From the Amazon docs:
+//
+// CanonicalizedResource = [ "/" + Bucket ] +
+// 	  <HTTP-Request-URI, from the protocol name up to the query string> +
+// 	  [ sub-resource, if present. For example "?acl", "?location", "?logging", or "?torrent"];
+func writeCanonicalizedResource(buf *bytes.Buffer, req *http.Request) {
+	if bucket := bucketFromHostname(req); bucket != "" {
+		buf.WriteByte('/')
+		buf.WriteString(bucket)
+	}
+	buf.WriteString(req.URL.Path)
+	// TODO: subresource
+}
+
+const standardUSRegionAWSSuffix = ".s3.amazonaws.com"
+const standardUSRegionAWS = "s3.amazonaws.com"
+
+func bucketFromHostname(req *http.Request) string {
+	host := req.Host
+	if host == standardUSRegionAWS {
+		return ""
+	}
+	if strings.HasSuffix(host, standardUSRegionAWSSuffix) {
+		return host[:len(host)-len(standardUSRegionAWSSuffix)]
+	}
+	if lastColon := strings.LastIndex(host, ":"); lastColon != -1 {
+		return host[:lastColon]
+	}
+	return host
+}

lib/go/camli/misc/amazon/s3/auth_test.go

@@ -0,0 +1,111 @@
+/*
+Copyright 2011 Google Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package s3
+
+import (
+	"bufio"
+	"fmt"
+	"http"
+	"strings"
+	"testing"
+)
+
+type reqAndExpected struct {
+	req, expected string
+}
+
+func req(s string) *http.Request {
+	req, err := http.ReadRequest(bufio.NewReader(strings.NewReader(s)))
+	if err != nil {
+		panic(fmt.Sprintf("bad request in test: %q (error: %v)", req, err))
+	}
+	return req
+}
+
+func TestStringToSign(t *testing.T) {
+	tests := []reqAndExpected{
+		{`GET /photos/puppy.jpg HTTP/1.1
+Host: johnsmith.s3.amazonaws.com
+Date: Tue, 27 Mar 2007 19:36:42 +0000
+
+`,
+			"GET\n\n\nTue, 27 Mar 2007 19:36:42 +0000\n/johnsmith/photos/puppy.jpg"},
+		{`PUT /photos/puppy.jpg HTTP/1.1
+Content-Type: image/jpeg
+Content-Length: 94328
+Host: johnsmith.s3.amazonaws.com
+Date: Tue, 27 Mar 2007 21:15:45 +0000
+
+`,
+			"PUT\n\nimage/jpeg\nTue, 27 Mar 2007 21:15:45 +0000\n/johnsmith/photos/puppy.jpg"},
+		{`GET /?prefix=photos&max-keys=50&marker=puppy HTTP/1.1
+User-Agent: Mozilla/5.0
+Host: johnsmith.s3.amazonaws.com
+Date: Tue, 27 Mar 2007 19:42:41 +0000
+
+`,
+			"GET\n\n\nTue, 27 Mar 2007 19:42:41 +0000\n/johnsmith/"},
+		{`DELETE /johnsmith/photos/puppy.jpg HTTP/1.1
+User-Agent: dotnet
+Host: s3.amazonaws.com
+Date: Tue, 27 Mar 2007 21:20:27 +0000
+x-amz-date: Tue, 27 Mar 2007 21:20:26 +0000
+
+`,
+			"DELETE\n\n\n\nx-amz-date:Tue, 27 Mar 2007 21:20:26 +0000\n/johnsmith/photos/puppy.jpg"},
+		{`PUT /db-backup.dat.gz HTTP/1.1
+User-Agent: curl/7.15.5
+Host: static.johnsmith.net:8080
+Date: Tue, 27 Mar 2007 21:06:08 +0000
+x-amz-acl: public-read
+content-type: application/x-download
+Content-MD5: 4gJE4saaMU4BqNR0kLY+lw==
+X-Amz-Meta-ReviewedBy: joe@johnsmith.net
+X-Amz-Meta-ReviewedBy: jane@johnsmith.net
+X-Amz-Meta-FileChecksum: 0x02661779
+X-Amz-Meta-ChecksumAlgorithm: crc32
+Content-Disposition: attachment; filename=database.dat
+Content-Encoding: gzip
+Content-Length: 5913339
+
+`,
+			"PUT\n4gJE4saaMU4BqNR0kLY+lw==\napplication/x-download\nTue, 27 Mar 2007 21:06:08 +0000\nx-amz-acl:public-read\nx-amz-meta-checksumalgorithm:crc32\nx-amz-meta-filechecksum:0x02661779\nx-amz-meta-reviewedby:joe@johnsmith.net,jane@johnsmith.net\n/static.johnsmith.net/db-backup.dat.gz"},
+	}
+	for idx, test := range tests {
+		got := stringToSign(req(test.req))
+		if got != test.expected {
+			t.Errorf("test %d: expected %q", idx, test.expected)
+			t.Errorf("test %d:      got %q", idx, got)
+		}
+	}
+}
+
+func TestBucketFromHostname(t *testing.T) {
+	tests := []reqAndExpected{
+		{"GET / HTTP/1.0\n\n", ""},
+		{"GET / HTTP/1.0\nHost: s3.amazonaws.com\n\n", ""},
+		{"GET / HTTP/1.0\nHost: foo.s3.amazonaws.com\n\n", "foo"},
+		{"GET / HTTP/1.0\nHost: foo.com:123\n\n", "foo.com"},
+		{"GET / HTTP/1.0\nHost: bar.com\n\n", "bar.com"},
+	}
+	for idx, test := range tests {
+		got := bucketFromHostname(req(test.req))
+		if got != test.expected {
+			t.Errorf("test %d: expected %q; got %q", idx, test.expected, got)
+		}
+	}
+}