Initial work on google storage for developers.

Brought in goauth2 client library, added a utility for obtaining
tokens for the first time.

Change-Id: I7c8301912a086df55732c1a1bc4ddf619438d66c

build.pl

@@ -628,6 +628,7 @@ __DATA__
 
 TARGET: clients/go/camdbinit
 TARGET: clients/go/camget
+TARGET: clients/go/camgsinit
 TARGET: clients/go/camput
 TARGET: clients/go/cammount
     =only_os_linux
@@ -636,6 +637,7 @@ TARGET: lib/go/camli/auth
 TARGET: lib/go/camli/blobref
 TARGET: lib/go/camli/blobserver
 TARGET: lib/go/camli/blobserver/cond
+TARGET: lib/go/camli/blobserver/googlestorage
 TARGET: lib/go/camli/blobserver/handlers
 TARGET: lib/go/camli/blobserver/localdisk
 TARGET: lib/go/camli/blobserver/remote
@@ -664,6 +666,7 @@ TARGET: lib/go/camli/schema
 TARGET: lib/go/camli/search
 TARGET: lib/go/camli/test
 TARGET: lib/go/camli/test/asserts
+TARGET: lib/go/camli/third_party/code.google.com/goauth2/oauth
 TARGET: lib/go/camli/third_party/github.com/bradfitz/gomemcache
 TARGET: lib/go/camli/third_party/github.com/hanwen/go-fuse/fuse
     =skip_tests

clients/go/camgsinit/camgsinit.go

@@ -0,0 +1,96 @@
+/*
+Copyright 2011 Google Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"bufio"
+	"fmt"
+	"json"
+	"os"
+	"strings"
+
+	"camli/blobserver/googlestorage"
+	"camli/third_party/code.google.com/goauth2/oauth"
+)
+
+func main() {
+	var (
+		err          os.Error
+		clientId     string
+		clientSecret string
+	)
+
+	if clientId, clientSecret, err = getClientInfo(); err != nil {
+		panic(err)
+	}
+	transport := googlestorage.MakeOauthTransport(clientId, clientSecret, "")
+
+	var accessCode string
+	if accessCode, err = getAccessCode(transport.Config); err != nil {
+		panic(err)
+	}
+	if _, err = transport.Exchange(accessCode); err != nil {
+		panic(err)
+	}
+
+	fmt.Printf("\nYour Google Storage auth object:\n\n")
+	enc := json.NewEncoder(os.Stdout)
+	authObj := map[string]string{
+		"client_id":     transport.ClientId,
+		"client_secret": transport.ClientSecret,
+		"refresh_token": transport.RefreshToken,
+	}
+	enc.Encode(authObj)
+	fmt.Print("\n")
+}
+
+// Prompt the user for an input line.  Return the given input.
+func prompt(promptText string) (string, os.Error) {
+	fmt.Print(promptText)
+	input := bufio.NewReader(os.Stdin)
+	line, _, err := input.ReadLine()
+	if err != nil {
+		return "", fmt.Errorf("Failed to read line: %v", err)
+	}
+	return strings.TrimSpace(string(line)), nil
+}
+
+// Provide the authorization link, then prompt for the resulting access code
+func getAccessCode(config *oauth.Config) (string, os.Error) {
+	fmt.Printf("In order to obtain a storage access code, you will need to naviage to the following URL:\n\n")
+	fmt.Printf("https://accounts.google.com/o/oauth2/auth?client_id=%s&redirect_uri=urn:ietf:wg:oauth:2.0:oob&scope=%s&response_type=code\n\n",
+		config.ClientId, config.Scope)
+	return prompt("Please enter the access code provided by that page:")
+}
+
+// Prompt for client id / secret
+func getClientInfo() (string, string, os.Error) {
+	fmt.Printf("Please provide the client id and client secret for your google storage account\n")
+	fmt.Printf("(You can find these at http://code.google.com/apis/console > your project > API Access)\n")
+	var (
+		err          os.Error
+		clientId     string
+		clientSecret string
+	)
+	if clientId, err = prompt("Client ID:"); err != nil {
+		return "", "", err
+	}
+	if clientSecret, err = prompt("Client Secret:"); err != nil {
+		return "", "", err
+	}
+	return clientId, clientSecret, nil
+}

lib/go/camli/blobserver/googlestorage/auth.go

@@ -0,0 +1,47 @@
+/*
+Copyright 2011 Google Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package googlestorage
+
+import (
+	"camli/third_party/code.google.com/goauth2/oauth"
+)
+
+const (
+	Scope       = "https://www.googleapis.com/auth/devstorage.read_write"
+	AuthURL     = "https://accounts.google.com/o/oauth2/auth"
+	TokenURL    = "https://accounts.google.com/o/oauth2/token"
+	RedirectURL = "urn:ietf:wg:oauth:2.0:oob"
+)
+
+func MakeOauthTransport(clientId string, clientSecret string, refreshToken string) *oauth.Transport {
+	return &oauth.Transport{
+		&oauth.Config{
+			ClientId:     clientId,
+			ClientSecret: clientSecret,
+			Scope:        Scope,
+			AuthURL:      AuthURL,
+			TokenURL:     TokenURL,
+			RedirectURL:  RedirectURL,
+		},
+		&oauth.Token{
+			AccessToken:  "",
+			RefreshToken: refreshToken,
+			TokenExpiry:  0,
+		},
+		nil,
+	}
+}

lib/go/camli/third_party/code.google.com/goauth2/oauth/oauth.go

@@ -0,0 +1,198 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// The oauth package provides support for making
+// OAuth2-authenticated HTTP requests.
+//
+// Example usage:
+//
+//	// Specify your configuration. (typically as a global variable)
+//	var config = &oauth.Config{
+//		ClientId:     YOUR_CLIENT_ID,
+//		ClientSecret: YOUR_CLIENT_SECRET,
+//		Scope:        "https://www.googleapis.com/auth/buzz",
+//		AuthURL:      "https://accounts.google.com/o/oauth2/auth",
+//		TokenURL:     "https://accounts.google.com/o/oauth2/token",
+//		RedirectURL:  "http://you.example.org/handler",
+//	}
+//
+//	// A landing page redirects to the OAuth provider to get the auth code.
+//	func landing(w http.ResponseWriter, r *http.Request) {
+//		http.Redirect(w, r, config.AuthCodeURL("foo"), http.StatusFound)
+//	}
+//
+//	// The user will be redirected back to this handler, that takes the
+//	// "code" query parameter and Exchanges it for an access token.
+//	func handler(w http.ResponseWriter, r *http.Request) {
+//		t := &oauth.Transport{Config: config}
+//		t.Exchange(r.FormValue("code"))
+//		// The Transport now has a valid Token. Create an *http.Client
+//		// with which we can make authenticated API requests.
+//		c := t.Client()
+//		c.Post(...)
+//		// ...
+//		// btw, r.FormValue("state") == "foo"
+//	}
+//
+package oauth
+
+// TODO(adg): A means of automatically saving credentials when updated.
+
+import (
+	"http"
+	"json"
+	"os"
+	"time"
+)
+
+// Config is the configuration of an OAuth consumer.
+type Config struct {
+	ClientId     string
+	ClientSecret string
+	Scope        string
+	AuthURL      string
+	TokenURL     string
+	RedirectURL  string // Defaults to out-of-band mode if empty.
+}
+
+func (c *Config) redirectURL() string {
+	if c.RedirectURL != "" {
+		return c.RedirectURL
+	}
+	return "oob"
+}
+
+// Token contains an end-user's tokens.
+// This is the data you must store to persist authentication.
+type Token struct {
+	AccessToken  string `json:"access_token"`
+	RefreshToken string `json:"refresh_token"`
+	TokenExpiry  int64  `json:"expires_in"`
+}
+
+// Transport implements http.RoundTripper. When configured with a valid
+// Config and Token it can be used to make authenticated HTTP requests.
+//
+//	t := &oauth.Transport{config}
+//      t.Exchange(code)
+//      // t now contains a valid Token
+//	r, _, err := t.Client().Get("http://example.org/url/requiring/auth")
+//
+// It will automatically refresh the Token if it can,
+// updating the supplied Token in place.
+type Transport struct {
+	*Config
+	*Token
+
+	// Transport is the HTTP transport to use when making requests.
+	// It will default to http.DefaultTransport if nil.
+	// (It should never be an oauth.Transport.)
+	Transport http.RoundTripper
+}
+
+// Client returns an *http.Client that makes OAuth-authenticated requests.
+func (t *Transport) Client() *http.Client {
+	return &http.Client{Transport: t}
+}
+
+func (t *Transport) transport() http.RoundTripper {
+	if t.Transport != nil {
+		return t.Transport
+	}
+	return http.DefaultTransport
+}
+
+// AuthCodeURL returns a URL that the end-user should be redirected to,
+// so that they may obtain an authorization code.
+func (c *Config) AuthCodeURL(state string) string {
+	url, err := http.ParseURL(c.AuthURL)
+	if err != nil {
+		panic("AuthURL malformed: " + err.String())
+	}
+	q := http.Values{
+		"response_type": {"code"},
+		"client_id":     {c.ClientId},
+		"redirect_uri":  {c.redirectURL()},
+		"scope":         {c.Scope},
+		"state":         {state},
+	}.Encode()
+	if url.RawQuery == "" {
+		url.RawQuery = q
+	} else {
+		url.RawQuery += "&" + q
+	}
+	return url.String()
+}
+
+// Exchange takes a code and gets access Token from the remote server.
+func (t *Transport) Exchange(code string) (tok *Token, err os.Error) {
+	if t.Config == nil {
+		return nil, os.NewError("no Config supplied")
+	}
+	tok = new(Token)
+	err = t.updateToken(tok, http.Values{
+		"grant_type":   {"authorization_code"},
+		"redirect_uri": {t.redirectURL()},
+		"scope":        {t.Scope},
+		"code":         {code},
+	})
+	if err == nil {
+		t.Token = tok
+	}
+	return
+}
+
+// RoundTrip executes a single HTTP transaction using the Transport's
+// Token as authorization headers.
+func (t *Transport) RoundTrip(req *http.Request) (resp *http.Response, err os.Error) {
+	if t.Config == nil {
+		return nil, os.NewError("no Config supplied")
+	}
+	if t.Token == nil {
+		return nil, os.NewError("no Token supplied")
+	}
+
+	// Make the HTTP request
+	req.Header.Set("Authorization", "OAuth "+t.AccessToken)
+	if resp, err = t.transport().RoundTrip(req); err != nil {
+		return
+	}
+
+	// Refresh credentials if they're stale and try again
+	if resp.StatusCode == 401 {
+		if err = t.refresh(); err != nil {
+			return
+		}
+		resp, err = t.transport().RoundTrip(req)
+	}
+
+	return
+}
+
+func (t *Transport) refresh() os.Error {
+	return t.updateToken(t.Token, http.Values{
+		"grant_type":    {"refresh_token"},
+		"refresh_token": {t.RefreshToken},
+	})
+}
+
+func (t *Transport) updateToken(tok *Token, v http.Values) os.Error {
+	v.Set("client_id", t.ClientId)
+	v.Set("client_secret", t.ClientSecret)
+	r, err := (&http.Client{Transport: t.transport()}).PostForm(t.TokenURL, v)
+	if err != nil {
+		return err
+	}
+	defer r.Body.Close()
+	if r.StatusCode != 200 {
+		return os.NewError("invalid response: " + r.Status)
+	}
+	if err = json.NewDecoder(r.Body).Decode(tok); err != nil {
+		return err
+	}
+	if tok.TokenExpiry != 0 {
+		tok.TokenExpiry = time.Seconds() + tok.TokenExpiry
+	}
+	return nil
+}

lib/go/camli/third_party/update.pl

@@ -32,6 +32,13 @@ my %proj = (
             [ "*.go", "github.com/camlistore/GoMySQL" ]
         ],
     },
+    "goauth" => {
+        hg => "https://code.google.com/p/goauth2/",
+        copies => [
+            # File glob => target directory
+            [ "oauth/*.go", "code.google.com/goauth2/oauth" ]
+        ],
+    },
     "gomemcache" => {
         git => "https://github.com/bradfitz/gomemcache/",
         copies => [
@@ -48,27 +55,64 @@ my %proj = (
     },
 );
 
+# Copies the file globs specified by a project 'copies' value.
+sub copy_files {
+  my $name = shift;
+  my $p = $proj{$name};
+  for my $cp (@{$p->{copies}}) {
+      my $glob = $cp->[0] or die;
+      my $target_dir = $cp->[1] or die;
+      system("mkdir", "-p", "$Bin/$target_dir") and die "Failed to make $Bin/$target_dir";
+      my @files = glob($glob) or die "Glob '$glob' didn't match any files for project '$name'";
+      system("cp", "-p", @files, "$Bin/$target_dir") and die "Copy failed.";
+  }
+}
+
+# Fetches most recent project sources from git
+sub update_git {
+  my $name = shift;
+  my $p = $proj{$name};
+
+  chdir($workdir) or die;
+  unless (-d $name) {
+    print STDERR "Cloning $name ...\n";
+    system("git", "clone", $p->{git}, $name) and die "git clone failure";
+  }
+
+  chdir($name) or die;
+  print STDERR "Updating $name ...\n";
+  system("git", "pull");
+  copy_files($name);
+}
+
+# Fetches most recent project sources from mercurial
+sub update_hg {
+  my $name = shift;
+  my $p = $proj{$name};
+
+  chdir($workdir) or die;
+  unless (-d $name) {
+    print STDERR "Cloning $name ...\n";
+    system("hg", "clone", $p->{hg}, $name) and die "hg clone failure";
+  }
+
+  chdir($name) or die;
+  print STDERR "Updating $name ...\n";
+  system("hg", "pull");
+  system("hg", "update");
+  copy_files($name);
+}
+
 foreach my $name (sort keys %proj) {
     next if @ARGV && $name !~ /\Q$ARGV[0]\E/;
     my $p = $proj{$name};
 
-    chdir($workdir) or die;
-    $p->{git} or die "no git key defined for $name";
-    unless (-d $name) {
-        print STDERR "Cloning $name ...\n";
-        system("git", "clone", $p->{git}, $name) and die "git clone failure";
-    }
-    chdir($name) or die;
-    print STDERR "Updating $name ...\n";
-    system("git", "pull");
-    for my $cp (@{$p->{copies}}) {
-        my $glob = $cp->[0] or die;
-        my $target_dir = $cp->[1] or die;
-        system("mkdir", "-p", "$Bin/$target_dir") and die "Failed to make $Bin/$target_dir";
-        my @files = glob($glob) or die "Glob '$glob' didn't match any files for project '$name'";
-        system("cp", "-p", @files, "$Bin/$target_dir") and die "Copy failed.";
+    if ($p->{git}) {
+      update_git($name);
+    } elsif ($p->{hg}) {
+      update_hg($name);
+    } else {
+      die "No known VCS defined for $name";
     }
 }
 
-
-