Rooba
/
perkeep
mirror of https://github.com/perkeep/perkeep.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge "google blobservers: migrate to golang.org/x/oauth2"

This commit is contained in:
mpl 2015-04-20 21:21:50 +00:00 committed by Gerrit Code Review
parent 529dca734a 5147aff319
commit 40f99f5f6e
14 changed files with 301 additions and 337 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

55
cmd/camdeploy/gce.go
View File

@ -18,8 +18,6 @@ package main
import (
"bufio"
"encoding/json"
"errors"
"flag"
"fmt"
"io/ioutil"
@ -30,6 +28,7 @@ import (
"camlistore.org/pkg/cmdmain"
"camlistore.org/pkg/context"
"camlistore.org/pkg/deploy/gce"
"camlistore.org/pkg/oauthutil"
"camlistore.org/third_party/golang.org/x/oauth2"
)
@ -117,8 +116,18 @@ func (c *gceCmd) RunCommand(args []string) error {
}
depl := &gce.Deployer{
Client: oauth2.NewClient(oauth2.NoContext, oauth2.ReuseTokenSource(nil,
&tokenSource{config: config, cacheFile: c.project + "-token.json"})),
Client: oauth2.NewClient(oauth2.NoContext, oauth2.ReuseTokenSource(nil, &oauthutil.TokenSource{
Config: config,
CacheFile: c.project + "-token.json",
AuthCode: func() string {
fmt.Println("Get auth code from:")
fmt.Printf("%v\n", config.AuthCodeURL("my-state", oauth2.AccessTypeOffline, oauth2.ApprovalForce))
fmt.Println("Enter auth code:")
sc := bufio.NewScanner(os.Stdin)
sc.Scan()
return strings.TrimSpace(sc.Text())
},
})),
Conf: instConf,
}
inst, err := depl.Create(context.TODO())
@ -144,41 +153,3 @@ func readFile(v string) string {
}
return strings.TrimSpace(string(slurp))
}
type tokenSource struct {
config *oauth2.Config
cacheFile string
}
func (src tokenSource) Token() (*oauth2.Token, error) {
tok := new(oauth2.Token)
tokenData, err := ioutil.ReadFile(src.cacheFile)
if err == nil {
err = json.Unmarshal(tokenData, tok)
if err == nil {
if tok.Valid() {
return tok, nil
}
err = errors.New("invalid token")
}
}
fmt.Printf("Error getting token from %s: %v\n", src.cacheFile, err)
fmt.Println("Get auth code from:")
fmt.Printf("%v\n", src.config.AuthCodeURL("my-state", oauth2.AccessTypeOffline, oauth2.ApprovalForce))
fmt.Println("Enter auth code:")
sc := bufio.NewScanner(os.Stdin)
sc.Scan()
authCode := strings.TrimSpace(sc.Text())
tok, err = src.config.Exchange(oauth2.NoContext, authCode)
if err != nil {
return nil, fmt.Errorf("could not exchange auth code for a token: %v", err)
}
tokenData, err = json.Marshal(&tok)
if err != nil {
return nil, fmt.Errorf("could not encode token as json: %v", err)
}
if err := ioutil.WriteFile(src.cacheFile, tokenData, 0600); err != nil {
return nil, fmt.Errorf("could not cache token in %v: %v", src.cacheFile, err)
}
return tok, nil
}

81
cmd/camtool/googinit.go
View File

@ -25,8 +25,11 @@ import (
"camlistore.org/pkg/blobserver/google/drive"
"camlistore.org/pkg/cmdmain"
"camlistore.org/pkg/constants/google"
"camlistore.org/pkg/googlestorage"
"camlistore.org/third_party/code.google.com/p/goauth2/oauth"
"camlistore.org/pkg/oauthutil"
"camlistore.org/third_party/golang.org/x/oauth2"
)
type googinitCmd struct {
@ -54,38 +57,52 @@ func (c *googinitCmd) RunCommand(args []string) error {
err error
clientId string
clientSecret string
transport *oauth.Transport
oauthConfig *oauth2.Config
)
if c.storageType != "drive" && c.storageType != "cloud" {
return cmdmain.UsageError("Invalid storage type: must be drive for Google Drive or cloud for Google Cloud Storage.")
}
if clientId, clientSecret, err = getClientInfo(); err != nil {
return err
}
clientId, clientSecret = getClientInfo()
switch c.storageType {
case "drive":
transport = drive.MakeOauthTransport(clientId, clientSecret, "")
oauthConfig = &oauth2.Config{
Scopes: []string{drive.Scope},
Endpoint: google.Endpoint,
ClientID: clientId,
ClientSecret: clientSecret,
RedirectURL: oauthutil.TitleBarRedirectURL,
}
case "cloud":
transport = googlestorage.MakeOauthTransport(clientId, clientSecret, "")
oauthConfig = &oauth2.Config{
Scopes: []string{googlestorage.Scope},
Endpoint: google.Endpoint,
ClientID: clientId,
ClientSecret: clientSecret,
RedirectURL: oauthutil.TitleBarRedirectURL,
}
}
var accessCode string
if accessCode, err = getAccessCode(transport.Config); err != nil {
return err
}
if _, err = transport.Exchange(accessCode); err != nil {
return err
token, err := oauth2.ReuseTokenSource(nil, &oauthutil.TokenSource{
Config: oauthConfig,
AuthCode: func() string {
fmt.Fprintf(cmdmain.Stdout, "Get auth code from:\n\n")
fmt.Fprintf(cmdmain.Stdout, "%v\n\n", oauthConfig.AuthCodeURL("", oauth2.AccessTypeOffline, oauth2.ApprovalForce))
return prompt("Enter auth code:")
},
}).Token()
if err != nil {
return fmt.Errorf("could not acquire token: %v", err)
}
fmt.Fprintf(cmdmain.Stdout, "\nYour Google auth object:\n\n")
enc := json.NewEncoder(cmdmain.Stdout)
authObj := map[string]string{
"client_id": transport.ClientId,
"client_secret": transport.ClientSecret,
"refresh_token": transport.RefreshToken,
"client_id": clientId,
"client_secret": clientSecret,
"refresh_token": token.RefreshToken,
}
enc.Encode(authObj)
fmt.Fprint(cmdmain.Stdout, "\n")
@ -93,38 +110,22 @@ func (c *googinitCmd) RunCommand(args []string) error {
}
// Prompt the user for an input line. Return the given input.
func prompt(promptText string) (string, error) {
func prompt(promptText string) string {
fmt.Fprint(cmdmain.Stdout, promptText)
input := bufio.NewReader(cmdmain.Stdin)
line, _, err := input.ReadLine()
if err != nil {
return "", fmt.Errorf("Failed to read line: %v", err)
}
return strings.TrimSpace(string(line)), nil
}
// Provide the authorization link, then prompt for the resulting access code
func getAccessCode(config *oauth.Config) (string, error) {
fmt.Fprintf(cmdmain.Stdout, "In order to obtain an access code, you will need to navigate to the following URL:\n\n")
fmt.Fprintf(cmdmain.Stdout, "https://accounts.google.com/o/oauth2/auth?client_id=%s&redirect_uri=urn:ietf:wg:oauth:2.0:oob&scope=%s&response_type=code\n\n",
config.ClientId, config.Scope)
return prompt("Please enter the access code provided by that page:")
sc := bufio.NewScanner(cmdmain.Stdin)
sc.Scan()
return strings.TrimSpace(sc.Text())
}
// Prompt for client id / secret
func getClientInfo() (string, string, error) {
func getClientInfo() (string, string) {
fmt.Fprintf(cmdmain.Stdout, "Please provide the client id and client secret \n")
fmt.Fprintf(cmdmain.Stdout, "(You can find these at http://code.google.com/apis/console > your project > API Access)\n")
var (
err error
clientId string
clientSecret string
)
if clientId, err = prompt("Client ID:"); err != nil {
return "", "", err
}
if clientSecret, err = prompt("Client Secret:"); err != nil {
return "", "", err
}
return clientId, clientSecret, nil
clientId = prompt("Client ID:")
clientSecret = prompt("Client Secret:")
return clientId, clientSecret
}

60
pkg/blobserver/google/cloudstorage/cloudstorage_test.go
View File

@ -26,13 +26,18 @@ import (
"camlistore.org/pkg/blob"
"camlistore.org/pkg/blobserver"
"camlistore.org/pkg/blobserver/storagetest"
"camlistore.org/pkg/constants/google"
"camlistore.org/pkg/context"
"camlistore.org/pkg/googlestorage"
"camlistore.org/pkg/jsonconfig"
"camlistore.org/third_party/code.google.com/p/goauth2/oauth"
"camlistore.org/pkg/oauthutil"
"camlistore.org/third_party/golang.org/x/oauth2"
)
var (
// TODO(mpl): use a config file generated with the help of googinit, like for googlestorage tests.
// And remove the 'camlistore-*-test' naming requirement ?
bucket = flag.String("bucket", "", "Bucket name to use for testing. If empty, testing is skipped. If non-empty, it must begin with 'camlistore-' and end in '-test' and have zero items in it.")
clientID = flag.String("client_id", "", "OAuth2 client_id for testing")
clientSecret = flag.String("client_secret", "", "OAuth2 client secret for testing")
@ -59,31 +64,28 @@ func testStorage(t *testing.T, bucketDir string) {
t.Fatal("--client_id and --client_secret required. Obtain from https://console.developers.google.com/ > Project > APIs & Auth > Credentials. Should be a 'native' or 'Installed application'")
}
tokenCache := oauth.CacheFile(*tokenCache)
token, err := tokenCache.Token()
config := &oauth2.Config{
Scopes: []string{googlestorage.Scope},
Endpoint: google.Endpoint,
ClientID: *clientID,
ClientSecret: *clientSecret,
RedirectURL: oauthutil.TitleBarRedirectURL,
}
token, err := oauth2.ReuseTokenSource(nil,
&oauthutil.TokenSource{
Config: config,
CacheFile: *tokenCache,
AuthCode: func() string {
if *authCode == "" {
t.Skipf("Re-run using --auth_code= with the value obtained from %s",
config.AuthCodeURL("", oauth2.AccessTypeOffline, oauth2.ApprovalForce))
return ""
}
return *authCode
},
}).Token()
if err != nil {
config := &oauth.Config{
// The client-id and secret should be for an "Installed Application" when using
// the CLI. Later we'll use a web application with a callback.
ClientId: *clientID,
ClientSecret: *clientSecret,
Scope: "https://www.googleapis.com/auth/devstorage.full_control",
AuthURL: "https://accounts.google.com/o/oauth2/auth",
TokenURL: "https://accounts.google.com/o/oauth2/token",
RedirectURL: "urn:ietf:wg:oauth:2.0:oob",
}
if *authCode != "" {
tr := &oauth.Transport{
Config: config,
}
token, err = tr.Exchange(*authCode)
if err != nil {
t.Fatalf("Error getting a token using auth code: %v", err)
}
tokenCache.PutToken(token)
} else {
t.Skipf("Re-run using --auth_code= with the value obtained from %s", config.AuthCodeURL(""))
}
t.Fatalf("could not acquire token: %v", err)
}
bucketWithDir := path.Join(*bucket, bucketDir)
@ -108,11 +110,9 @@ func testStorage(t *testing.T, bucketDir string) {
// Adding "a", and "c" objects in the bucket to make sure objects out of the
// "directory" are not touched and have no influence.
for _, key := range []string{"a", "c"} {
for tries, shouldRetry := 0, true; tries < 2 && shouldRetry; tries++ {
shouldRetry, err = sto.(*Storage).client.PutObject(
&googlestorage.Object{Bucket: sto.(*Storage).bucket, Key: key},
strings.NewReader(key))
}
err := sto.(*Storage).client.PutObject(
&googlestorage.Object{Bucket: sto.(*Storage).bucket, Key: key},
strings.NewReader(key))
if err != nil {
t.Fatalf("could not insert object %s in bucket %v: %v", key, sto.(*Storage).bucket, err)
}

22
pkg/blobserver/google/cloudstorage/storage.go
View File

@ -35,10 +35,14 @@ import (
"camlistore.org/pkg/blobserver"
"camlistore.org/pkg/blobserver/memory"
"camlistore.org/pkg/constants"
"camlistore.org/pkg/constants/google"
"camlistore.org/pkg/context"
"camlistore.org/pkg/googlestorage"
"camlistore.org/pkg/jsonconfig"
"camlistore.org/pkg/oauthutil"
"camlistore.org/pkg/syncutil"
"camlistore.org/third_party/golang.org/x/oauth2"
)
type Storage struct {
@ -113,8 +117,14 @@ func newFromConfig(_ blobserver.Loader, config jsonconfig.Obj) (blobserver.Stora
if refreshToken == "" {
return nil, errors.New("missing required parameter 'refresh_token'")
}
gs.client = googlestorage.NewClient(googlestorage.MakeOauthTransport(
clientID, clientSecret, refreshToken))
oAuthClient := oauth2.NewClient(oauth2.NoContext, oauthutil.NewRefreshTokenSource(&oauth2.Config{
Scopes: []string{googlestorage.Scope},
Endpoint: google.Endpoint,
ClientID: clientID,
ClientSecret: clientSecret,
RedirectURL: oauthutil.TitleBarRedirectURL,
}, refreshToken))
gs.client = googlestorage.NewClient(oAuthClient)
}
if cacheSize != 0 {
@ -165,11 +175,9 @@ func (s *Storage) ReceiveBlob(br blob.Ref, source io.Reader) (blob.SizedRef, err
return blob.SizedRef{}, err
}
for tries, shouldRetry := 0, true; tries < 2 && shouldRetry; tries++ {
shouldRetry, err = s.client.PutObject(
&googlestorage.Object{Bucket: s.bucket, Key: s.dirPrefix + br.String()},
ioutil.NopCloser(bytes.NewReader(buf.Bytes())))
}
err = s.client.PutObject(
&googlestorage.Object{Bucket: s.bucket, Key: s.dirPrefix + br.String()},
ioutil.NopCloser(bytes.NewReader(buf.Bytes())))
if err != nil {
return blob.SizedRef{}, err
}

48
pkg/blobserver/google/drive/auth.go
View File

@ -1,48 +0,0 @@
/*
Copyright 2014 The Camlistore Authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package drive
import (
"time"
"camlistore.org/third_party/code.google.com/p/goauth2/oauth"
)
const (
Scope = "https://www.googleapis.com/auth/drive"
AuthURL = "https://accounts.google.com/o/oauth2/auth"
TokenURL = "https://accounts.google.com/o/oauth2/token"
RedirectURL = "urn:ietf:wg:oauth:2.0:oob"
)
func MakeOauthTransport(clientId string, clientSecret string, refreshToken string) *oauth.Transport {
return &oauth.Transport{
Config: &oauth.Config{
ClientId: clientId,
ClientSecret: clientSecret,
Scope: Scope,
AuthURL: AuthURL,
TokenURL: TokenURL,
RedirectURL: RedirectURL,
},
Token: &oauth.Token{
AccessToken: "",
RefreshToken: refreshToken,
Expiry: time.Time{}, // no expiry
},
}
}

39
pkg/blobserver/google/drive/drive.go
View File

@ -35,19 +35,17 @@ Example low-level config:
package drive
import (
"net/http"
"time"
"camlistore.org/pkg/blobserver"
"camlistore.org/pkg/blobserver/google/drive/service"
"camlistore.org/pkg/constants/google"
"camlistore.org/pkg/jsonconfig"
"camlistore.org/third_party/code.google.com/p/goauth2/oauth"
"camlistore.org/pkg/oauthutil"
"camlistore.org/third_party/golang.org/x/oauth2"
)
const (
GoogleOAuth2AuthURL = "https://accounts.google.com/o/oauth2/auth"
GoogleOAuth2TokenURL = "https://accounts.google.com/o/oauth2/token"
)
// Scope is the OAuth2 scope used for Google Drive.
const Scope = "https://www.googleapis.com/auth/drive"
type driveStorage struct {
service *service.DriveService
@ -55,29 +53,18 @@ type driveStorage struct {
func newFromConfig(_ blobserver.Loader, config jsonconfig.Obj) (blobserver.Storage, error) {
auth := config.RequiredObject("auth")
oauthConf := &oauth.Config{
ClientId: auth.RequiredString("client_id"),
oAuthClient := oauth2.NewClient(oauth2.NoContext, oauthutil.NewRefreshTokenSource(&oauth2.Config{
Scopes: []string{Scope},
Endpoint: google.Endpoint,
ClientID: auth.RequiredString("client_id"),
ClientSecret: auth.RequiredString("client_secret"),
AuthURL: GoogleOAuth2AuthURL,
TokenURL: GoogleOAuth2TokenURL,
}
// force refreshes the access token on start, make sure
// refresh request in parallel are being started
transport := &oauth.Transport{
Token: &oauth.Token{
AccessToken: "",
RefreshToken: auth.RequiredString("refresh_token"),
Expiry: time.Now(),
},
Config: oauthConf,
Transport: http.DefaultTransport,
}
RedirectURL: oauthutil.TitleBarRedirectURL,
}, auth.RequiredString("refresh_token")))
parent := config.RequiredString("parent_id")
if err := config.Validate(); err != nil {
return nil, err
}
service, err := service.New(transport, parent)
service, err := service.New(oAuthClient, parent)
sto := &driveStorage{
service: service,
}

40
pkg/blobserver/google/drive/drive_test.go
View File

@ -23,11 +23,15 @@ import (
"camlistore.org/pkg/blobserver"
"camlistore.org/pkg/blobserver/storagetest"
"camlistore.org/pkg/constants/google"
"camlistore.org/pkg/jsonconfig"
"camlistore.org/third_party/code.google.com/p/goauth2/oauth"
"camlistore.org/pkg/oauthutil"
"camlistore.org/third_party/golang.org/x/oauth2"
)
var (
// TODO(mpl): use a config file generated with the help of googinit, like for googlestorage tests.
parentId = flag.String("parentDir", "", "id of the directory on google drive to use for testing. If empty or \"root\", testing is skipped.")
clientID = flag.String("client_id", "", "OAuth2 client_id for testing")
clientSecret = flag.String("client_secret", "", "OAuth2 client secret for testing")
@ -43,20 +47,28 @@ func TestStorage(t *testing.T) {
t.Fatal("--client_id and --client_secret required. Obtain from https://console.developers.google.com/ > Project > APIs & Auth > Credentials. Should be a 'native' or 'Installed application'")
}
tokenCache := oauth.CacheFile(*tokenCache)
token, err := tokenCache.Token()
config := &oauth2.Config{
Scopes: []string{Scope},
Endpoint: google.Endpoint,
ClientID: *clientID,
ClientSecret: *clientSecret,
RedirectURL: oauthutil.TitleBarRedirectURL,
}
token, err := oauth2.ReuseTokenSource(nil,
&oauthutil.TokenSource{
Config: config,
CacheFile: *tokenCache,
AuthCode: func() string {
if *authCode == "" {
t.Skipf("Re-run using --auth_code= with the value obtained from %s",
config.AuthCodeURL("", oauth2.AccessTypeOffline, oauth2.ApprovalForce))
return ""
}
return *authCode
},
}).Token()
if err != nil {
tr := MakeOauthTransport(*clientID, *clientSecret, "")
config := tr.Config
if *authCode != "" {
token, err = tr.Exchange(*authCode)
if err != nil {
t.Fatalf("Error getting a token using auth code: %v", err)
}
tokenCache.PutToken(token)
} else {
t.Skipf("Re-run using --auth_code= with the value obtained from %s", config.AuthCodeURL(""))
}
t.Fatalf("could not acquire token: %v", err)
}
storagetest.TestOpt(t, storagetest.Opts{

11
pkg/blobserver/google/drive/service/service.go
View File

@ -26,7 +26,6 @@ import (
"net/http"
"os"
"camlistore.org/third_party/code.google.com/p/goauth2/oauth"
client "camlistore.org/third_party/google.golang.org/api/drive/v2"
)
@ -38,7 +37,7 @@ const (
// DriveService wraps Google Drive API to implement utility methods to
// be performed on the root Drive destination folder.
type DriveService struct {
transport *oauth.Transport
client *http.Client
apiservice *client.Service
parentId string
}
@ -47,8 +46,8 @@ type DriveService struct {
// that will be used as the current directory in methods on the returned
// DriveService (such as Get). If empty, it defaults to the root of the
// drive.
func New(transport *oauth.Transport, parentId string) (*DriveService, error) {
apiservice, err := client.New(transport.Client())
func New(oauthClient *http.Client, parentId string) (*DriveService, error) {
apiservice, err := client.New(oauthClient)
if err != nil {
return nil, err
}
@ -56,7 +55,7 @@ func New(transport *oauth.Transport, parentId string) (*DriveService, error) {
// because "root" is known as a special alias for the root directory in drive.
parentId = "root"
}
service := &DriveService{transport: transport, apiservice: apiservice, parentId: parentId}
service := &DriveService{client: oauthClient, apiservice: apiservice, parentId: parentId}
return service, err
}
@ -141,7 +140,7 @@ func (s *DriveService) Fetch(title string) (body io.ReadCloser, size uint32, err
req, _ := http.NewRequest("GET", file.DownloadUrl, nil)
var resp *http.Response
if resp, err = s.transport.RoundTrip(req); err != nil {
if resp, err = s.client.Transport.RoundTrip(req); err != nil {
return
}
if file.FileSize > math.MaxUint32 || file.FileSize < 0 {

2
pkg/googlestorage/README
View File

@ -38,4 +38,4 @@ In order to run these tests properly, you will need to:
}
You can use 'camtool gsinit' to help obtain the auth config object.
You can use 'camtool googinit' to help obtain the auth config object.

48
pkg/googlestorage/auth.go
View File

@ -1,48 +0,0 @@
/*
Copyright 2013 Google Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package googlestorage
import (
"time"
"camlistore.org/third_party/code.google.com/p/goauth2/oauth"
)
const (
Scope = "https://www.googleapis.com/auth/devstorage.read_write"
AuthURL = "https://accounts.google.com/o/oauth2/auth"
TokenURL = "https://accounts.google.com/o/oauth2/token"
RedirectURL = "urn:ietf:wg:oauth:2.0:oob"
)
func MakeOauthTransport(clientId string, clientSecret string, refreshToken string) *oauth.Transport {
return &oauth.Transport{
Config: &oauth.Config{
ClientId: clientId,
ClientSecret: clientSecret,
Scope: Scope,
AuthURL: AuthURL,
TokenURL: TokenURL,
RedirectURL: RedirectURL,
},
Token: &oauth.Token{
AccessToken: "",
RefreshToken: refreshToken,
Expiry: time.Time{}, // no expiry
},
}
}

74
pkg/googlestorage/googlestorage.go
View File

@ -35,7 +35,7 @@ import (
"camlistore.org/pkg/blob"
"camlistore.org/pkg/httputil"
"camlistore.org/third_party/code.google.com/p/goauth2/oauth"
"camlistore.org/third_party/golang.org/x/net/context"
"camlistore.org/third_party/golang.org/x/oauth2"
"camlistore.org/third_party/golang.org/x/oauth2/google"
@ -45,13 +45,14 @@ import (
const (
gsAccessURL = "https://storage.googleapis.com"
// Scope is the OAuth2 scope used for Google Cloud Storage.
Scope = "https://www.googleapis.com/auth/devstorage.read_write"
)
// A Client provides access to Google Cloud Storage.
type Client struct {
client *http.Client
transport *oauth.Transport // nil for service clients
service *api.Service
client *http.Client
service *api.Service
}
// An Object holds the name of an object (its bucket and key) within
@ -105,13 +106,11 @@ func NewServiceClient() (*Client, error) {
return &Client{client: client, service: service}, nil
}
func NewClient(transport *oauth.Transport) *Client {
client := transport.Client()
service, _ := api.New(client)
func NewClient(oauthClient *http.Client) *Client {
service, _ := api.New(oauthClient)
return &Client{
client: transport.Client(),
transport: transport,
service: service,
client: oauthClient,
service: service,
}
}
@ -126,40 +125,6 @@ func (so SizedObject) String() string {
return fmt.Sprintf("%v/%v (%vB)", so.Bucket, so.Key, so.Size)
}
// A close relative to http.Client.Do(), helping with token refresh logic.
// If canResend is true and the initial request's response is an auth error
// (401 or 403), oauth credentials will be refreshed and the request sent
// again. This should only be done for requests with empty bodies, since the
// Body will be consumed on the first attempt if it exists.
// If canResend is false, and req would have been resent if canResend were
// true, then shouldRetry will be true.
// One of resp or err will always be nil.
func (gsa *Client) doRequest(req *http.Request, canResend bool) (resp *http.Response, err error, shouldRetry bool) {
resp, err = gsa.client.Do(req)
if err != nil {
return
}
if gsa.transport == nil {
return
}
if resp.StatusCode == 401 || resp.StatusCode == 403 {
// Unauth. Perhaps tokens need refreshing?
if err = gsa.transport.Refresh(); err != nil {
return
}
// Refresh succeeded. req should be resent
if !canResend {
return resp, nil, true
}
// Resend req. First, need to close the soon-overwritten response Body
resp.Body.Close()
resp, err = gsa.client.Do(req)
}
return
}
// Makes a simple body-less google storage request
func (gsa *Client) simpleRequest(method, url_ string) (resp *http.Response, err error) {
// Construct the request
@ -169,8 +134,7 @@ func (gsa *Client) simpleRequest(method, url_ string) (resp *http.Response, err
}
req.Header.Set("x-goog-api-version", "2")
resp, err, _ = gsa.doRequest(req, true)
return
return gsa.client.Do(req)
}
// GetObject fetches a Google Cloud Storage object.
@ -217,7 +181,7 @@ func (c *Client) GetPartialObject(obj Object, offset, length int64) (rc io.ReadC
req.Header.Set("Range", fmt.Sprintf("bytes=%d-", offset))
}
resp, err, _ := c.doRequest(req, true)
resp, err := c.client.Do(req)
if err != nil {
return nil, fmt.Errorf("GS GET request failed: %v\n", err)
}
@ -265,15 +229,15 @@ func (gsa *Client) StatObject(obj *Object) (size int64, exists bool, err error)
// shouldRetry will be true if the put failed due to authorization, but
// credentials have been refreshed and another attempt is likely to succeed.
// In this case, content will have been consumed.
func (gsa *Client) PutObject(obj *Object, content io.Reader) (shouldRetry bool, err error) {
func (gsa *Client) PutObject(obj *Object, content io.Reader) error {
if err := obj.valid(); err != nil {
return false, err
return err
}
const maxSlurp = 2 << 20
var buf bytes.Buffer
n, err := io.CopyN(&buf, content, maxSlurp)
if err != nil && err != io.EOF {
return false, err
return err
}
contentType := http.DetectContentType(buf.Bytes())
if contentType == "application/octet-stream" && n < maxSlurp && utf8.Valid(buf.Bytes()) {
@ -283,20 +247,20 @@ func (gsa *Client) PutObject(obj *Object, content io.Reader) (shouldRetry bool,
objURL := gsAccessURL + "/" + obj.Bucket + "/" + obj.Key
var req *http.Request
if req, err = http.NewRequest("PUT", objURL, ioutil.NopCloser(io.MultiReader(&buf, content))); err != nil {
return
return err
}
req.Header.Set("x-goog-api-version", "2")
req.Header.Set("Content-Type", contentType)
var resp *http.Response
if resp, err, shouldRetry = gsa.doRequest(req, false); err != nil {
return
if resp, err = gsa.client.Do(req); err != nil {
return err
}
if resp.StatusCode != http.StatusOK {
return shouldRetry, fmt.Errorf("Bad put response code: %v", resp.Status)
return fmt.Errorf("Bad put response code: %v", resp.Status)
}
return
return nil
}
// DeleteObject removes an object.

22
pkg/googlestorage/googlestorage_test.go
View File

@ -27,7 +27,11 @@ import (
"testing"
"time"
"camlistore.org/pkg/constants/google"
"camlistore.org/pkg/jsonconfig"
"camlistore.org/pkg/oauthutil"
"camlistore.org/third_party/golang.org/x/oauth2"
)
const testObjectContent = "Google Storage Test\n"
@ -66,9 +70,13 @@ func doConfig(t *testing.T) (gsa *Client, bucket string) {
t.Fatalf("Invalid config: %v", err)
}
gsa = NewClient(MakeOauthTransport(auth.RequiredString("client_id"),
auth.RequiredString("client_secret"),
auth.RequiredString("refresh_token")))
gsa = NewClient(oauth2.NewClient(oauth2.NoContext, oauthutil.NewRefreshTokenSource(&oauth2.Config{
Scopes: []string{Scope},
Endpoint: google.Endpoint,
ClientID: auth.RequiredString("client_id"),
ClientSecret: auth.RequiredString("client_secret"),
RedirectURL: oauthutil.TitleBarRedirectURL,
}, auth.RequiredString("refresh_token"))))
if err := auth.Validate(); err != nil {
t.Fatalf("Invalid config: %v", err)
@ -158,12 +166,8 @@ func TestPutObject(t *testing.T) {
testKey := fmt.Sprintf("test-put-%v.%v.%v-%v.%v.%v",
now.Year(), now.Month(), now.Day(), now.Hour(), now.Minute(), now.Second())
shouldRetry, err := gs.PutObject(&Object{bucket, testKey},
err := gs.PutObject(&Object{bucket, testKey},
&BufferCloser{bytes.NewBufferString(testObjectContent)})
if shouldRetry {
shouldRetry, err = gs.PutObject(&Object{bucket, testKey},
&BufferCloser{bytes.NewBufferString(testObjectContent)})
}
if err != nil {
t.Fatalf("Failed to put object: %v", err)
}
@ -192,7 +196,7 @@ func TestDeleteObject(t *testing.T) {
now := time.Now()
testKey := fmt.Sprintf("test-delete-%v.%v.%v-%v.%v.%v",
now.Year(), now.Month(), now.Day(), now.Hour(), now.Minute(), now.Second())
_, err = gs.PutObject(&Object{bucket, testKey},
err = gs.PutObject(&Object{bucket, testKey},
&BufferCloser{bytes.NewBufferString("Delete Me")})
if err != nil {
t.Fatalf("Failed to put file to delete.")

121
pkg/oauthutil/oauth.go Normal file
View File

@ -0,0 +1,121 @@
/*
Copyright 2015 The Camlistore Authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package oauthutil
import (
"encoding/json"
"errors"
"fmt"
"time"
"camlistore.org/pkg/wkfs"
"camlistore.org/third_party/golang.org/x/oauth2"
)
// TitleBarRedirectURL is the OAuth2 redirect URL to use when the authorization
// code should be returned in the title bar of the browser, with the page text
// prompting the user to copy the code and paste it in the application.
const TitleBarRedirectURL = "urn:ietf:wg:oauth:2.0:oob"
// ErrNoAuthCode is returned when Token() has not found any valid cached token
// and TokenSource does not have an AuthCode for getting a new token.
var ErrNoAuthCode = errors.New("oauthutil: unspecified TokenSource.AuthCode")
// TokenSource is an implementation of oauth2.TokenSource. It uses CacheFile to store and
// reuse the the acquired token, and AuthCode to provide the authorization code that will be
// exchanged for a token otherwise.
type TokenSource struct {
Config *oauth2.Config
// CacheFile is where the token will be stored JSON-encoded. Any call to Token
// first tries to read a valid token from CacheFile.
CacheFile string
// AuthCode provides the authorization code that Token will exchange for a token.
// It usually is a way to prompt the user for the code. If CacheFile does not provide
// a token and AuthCode is nil, Token returns ErrNoAuthCode.
AuthCode func() string
}
var errExpiredToken = errors.New("expired token")
// cachedToken returns the token saved in cacheFile. It specifically returns
// errTokenExpired if the token is expired.
func cachedToken(cacheFile string) (*oauth2.Token, error) {
tok := new(oauth2.Token)
tokenData, err := wkfs.ReadFile(cacheFile)
if err != nil {
return nil, err
}
if err = json.Unmarshal(tokenData, tok); err != nil {
return nil, err
}
if !tok.Valid() {
if tok != nil && time.Now().After(tok.Expiry) {
return nil, errExpiredToken
}
return nil, errors.New("invalid token")
}
return tok, nil
}
// Token first tries to find a valid token in CacheFile, and otherwise uses
// Config and AuthCode to fetch a new token. This new token is saved in CacheFile
// (if not blank). If CacheFile did not provide a token and AuthCode is nil,
// ErrNoAuthCode is returned.
func (src TokenSource) Token() (*oauth2.Token, error) {
var tok *oauth2.Token
var err error
if src.CacheFile != "" {
tok, err = cachedToken(src.CacheFile)
if err == nil {
return tok, nil
}
if err != errExpiredToken {
fmt.Printf("Error getting token from %s: %v\n", src.CacheFile, err)
}
}
if src.AuthCode == nil {
return nil, ErrNoAuthCode
}
tok, err = src.Config.Exchange(oauth2.NoContext, src.AuthCode())
if err != nil {
return nil, fmt.Errorf("could not exchange auth code for a token: %v", err)
}
if src.CacheFile == "" {
return tok, nil
}
tokenData, err := json.Marshal(&tok)
if err != nil {
return nil, fmt.Errorf("could not encode token as json: %v", err)
}
if err := wkfs.WriteFile(src.CacheFile, tokenData, 0600); err != nil {
return nil, fmt.Errorf("could not cache token in %v: %v", src.CacheFile, err)
}
return tok, nil
}
// NewRefreshTokenSource returns a token source that obtains its initial token
// based on the provided config and the refresh token.
func NewRefreshTokenSource(config *oauth2.Config, refreshToken string) oauth2.TokenSource {
var noInitialToken *oauth2.Token = nil
return oauth2.ReuseTokenSource(noInitialToken, config.TokenSource(
oauth2.NoContext, // TODO: maybe accept a context later.
&oauth2.Token{RefreshToken: refreshToken},
))
}

15
pkg/wkfs/gcs/gcs.go
View File

@ -172,17 +172,10 @@ func (w *fileWriter) Close() (err error) {
return nil
}
w.closed = true
var retry bool
for tries := 0; tries < 2; tries++ {
retry, err = w.fs.client.PutObject(&googlestorage.Object{
Bucket: w.bucket,
Key: w.key,
}, ioutil.NopCloser(bytes.NewReader(w.buf.Bytes())))
if retry {
continue
}
}
return err
return w.fs.client.PutObject(&googlestorage.Object{
Bucket: w.bucket,
Key: w.key,
}, ioutil.NopCloser(bytes.NewReader(w.buf.Bytes())))
}
type statInfo struct {