From 5147aff319514e598510beb1aaeae1d1b4598e1d Mon Sep 17 00:00:00 2001 From: mpl Date: Tue, 7 Apr 2015 18:02:33 +0200 Subject: [PATCH] google blobservers: migrate to golang.org/x/oauth2 Previous oauth2 (code.google.com/p/goauth2/oauth) still used in: third_party/github.com/tgulacsi/picago pkg/importer/picasa pkg/importer/foursquare Final cleanup in a subsequent CL. Change-Id: I805d81fcaa651e03a17823c78abe5040d51346c3 --- cmd/camdeploy/gce.go | 55 ++------ cmd/camtool/googinit.go | 81 ++++++------ .../google/cloudstorage/cloudstorage_test.go | 60 ++++----- pkg/blobserver/google/cloudstorage/storage.go | 22 +++- pkg/blobserver/google/drive/auth.go | 48 ------- pkg/blobserver/google/drive/drive.go | 39 ++---- pkg/blobserver/google/drive/drive_test.go | 40 ++++-- .../google/drive/service/service.go | 11 +- pkg/googlestorage/README | 2 +- pkg/googlestorage/auth.go | 48 ------- pkg/googlestorage/googlestorage.go | 74 +++-------- pkg/googlestorage/googlestorage_test.go | 22 ++-- pkg/oauthutil/oauth.go | 121 ++++++++++++++++++ pkg/wkfs/gcs/gcs.go | 15 +-- 14 files changed, 301 insertions(+), 337 deletions(-) delete mode 100644 pkg/blobserver/google/drive/auth.go delete mode 100644 pkg/googlestorage/auth.go create mode 100644 pkg/oauthutil/oauth.go diff --git a/cmd/camdeploy/gce.go b/cmd/camdeploy/gce.go index 2936c85bf..9c00900a3 100644 --- a/cmd/camdeploy/gce.go +++ b/cmd/camdeploy/gce.go @@ -18,8 +18,6 @@ package main import ( "bufio" - "encoding/json" - "errors" "flag" "fmt" "io/ioutil" @@ -30,6 +28,7 @@ import ( "camlistore.org/pkg/cmdmain" "camlistore.org/pkg/context" "camlistore.org/pkg/deploy/gce" + "camlistore.org/pkg/oauthutil" "camlistore.org/third_party/golang.org/x/oauth2" ) @@ -117,8 +116,18 @@ func (c *gceCmd) RunCommand(args []string) error { } depl := &gce.Deployer{ - Client: oauth2.NewClient(oauth2.NoContext, oauth2.ReuseTokenSource(nil, - &tokenSource{config: config, cacheFile: c.project + "-token.json"})), + Client: oauth2.NewClient(oauth2.NoContext, oauth2.ReuseTokenSource(nil, &oauthutil.TokenSource{ + Config: config, + CacheFile: c.project + "-token.json", + AuthCode: func() string { + fmt.Println("Get auth code from:") + fmt.Printf("%v\n", config.AuthCodeURL("my-state", oauth2.AccessTypeOffline, oauth2.ApprovalForce)) + fmt.Println("Enter auth code:") + sc := bufio.NewScanner(os.Stdin) + sc.Scan() + return strings.TrimSpace(sc.Text()) + }, + })), Conf: instConf, } inst, err := depl.Create(context.TODO()) @@ -144,41 +153,3 @@ func readFile(v string) string { } return strings.TrimSpace(string(slurp)) } - -type tokenSource struct { - config *oauth2.Config - cacheFile string -} - -func (src tokenSource) Token() (*oauth2.Token, error) { - tok := new(oauth2.Token) - tokenData, err := ioutil.ReadFile(src.cacheFile) - if err == nil { - err = json.Unmarshal(tokenData, tok) - if err == nil { - if tok.Valid() { - return tok, nil - } - err = errors.New("invalid token") - } - } - fmt.Printf("Error getting token from %s: %v\n", src.cacheFile, err) - fmt.Println("Get auth code from:") - fmt.Printf("%v\n", src.config.AuthCodeURL("my-state", oauth2.AccessTypeOffline, oauth2.ApprovalForce)) - fmt.Println("Enter auth code:") - sc := bufio.NewScanner(os.Stdin) - sc.Scan() - authCode := strings.TrimSpace(sc.Text()) - tok, err = src.config.Exchange(oauth2.NoContext, authCode) - if err != nil { - return nil, fmt.Errorf("could not exchange auth code for a token: %v", err) - } - tokenData, err = json.Marshal(&tok) - if err != nil { - return nil, fmt.Errorf("could not encode token as json: %v", err) - } - if err := ioutil.WriteFile(src.cacheFile, tokenData, 0600); err != nil { - return nil, fmt.Errorf("could not cache token in %v: %v", src.cacheFile, err) - } - return tok, nil -} diff --git a/cmd/camtool/googinit.go b/cmd/camtool/googinit.go index 77b15602d..00fc41b22 100644 --- a/cmd/camtool/googinit.go +++ b/cmd/camtool/googinit.go @@ -25,8 +25,11 @@ import ( "camlistore.org/pkg/blobserver/google/drive" "camlistore.org/pkg/cmdmain" + "camlistore.org/pkg/constants/google" "camlistore.org/pkg/googlestorage" - "camlistore.org/third_party/code.google.com/p/goauth2/oauth" + "camlistore.org/pkg/oauthutil" + + "camlistore.org/third_party/golang.org/x/oauth2" ) type googinitCmd struct { @@ -54,38 +57,52 @@ func (c *googinitCmd) RunCommand(args []string) error { err error clientId string clientSecret string - transport *oauth.Transport + oauthConfig *oauth2.Config ) if c.storageType != "drive" && c.storageType != "cloud" { return cmdmain.UsageError("Invalid storage type: must be drive for Google Drive or cloud for Google Cloud Storage.") } - if clientId, clientSecret, err = getClientInfo(); err != nil { - return err - } + clientId, clientSecret = getClientInfo() switch c.storageType { case "drive": - transport = drive.MakeOauthTransport(clientId, clientSecret, "") + oauthConfig = &oauth2.Config{ + Scopes: []string{drive.Scope}, + Endpoint: google.Endpoint, + ClientID: clientId, + ClientSecret: clientSecret, + RedirectURL: oauthutil.TitleBarRedirectURL, + } case "cloud": - transport = googlestorage.MakeOauthTransport(clientId, clientSecret, "") + oauthConfig = &oauth2.Config{ + Scopes: []string{googlestorage.Scope}, + Endpoint: google.Endpoint, + ClientID: clientId, + ClientSecret: clientSecret, + RedirectURL: oauthutil.TitleBarRedirectURL, + } } - var accessCode string - if accessCode, err = getAccessCode(transport.Config); err != nil { - return err - } - if _, err = transport.Exchange(accessCode); err != nil { - return err + token, err := oauth2.ReuseTokenSource(nil, &oauthutil.TokenSource{ + Config: oauthConfig, + AuthCode: func() string { + fmt.Fprintf(cmdmain.Stdout, "Get auth code from:\n\n") + fmt.Fprintf(cmdmain.Stdout, "%v\n\n", oauthConfig.AuthCodeURL("", oauth2.AccessTypeOffline, oauth2.ApprovalForce)) + return prompt("Enter auth code:") + }, + }).Token() + if err != nil { + return fmt.Errorf("could not acquire token: %v", err) } fmt.Fprintf(cmdmain.Stdout, "\nYour Google auth object:\n\n") enc := json.NewEncoder(cmdmain.Stdout) authObj := map[string]string{ - "client_id": transport.ClientId, - "client_secret": transport.ClientSecret, - "refresh_token": transport.RefreshToken, + "client_id": clientId, + "client_secret": clientSecret, + "refresh_token": token.RefreshToken, } enc.Encode(authObj) fmt.Fprint(cmdmain.Stdout, "\n") @@ -93,38 +110,22 @@ func (c *googinitCmd) RunCommand(args []string) error { } // Prompt the user for an input line. Return the given input. -func prompt(promptText string) (string, error) { +func prompt(promptText string) string { fmt.Fprint(cmdmain.Stdout, promptText) - input := bufio.NewReader(cmdmain.Stdin) - line, _, err := input.ReadLine() - if err != nil { - return "", fmt.Errorf("Failed to read line: %v", err) - } - return strings.TrimSpace(string(line)), nil -} - -// Provide the authorization link, then prompt for the resulting access code -func getAccessCode(config *oauth.Config) (string, error) { - fmt.Fprintf(cmdmain.Stdout, "In order to obtain an access code, you will need to navigate to the following URL:\n\n") - fmt.Fprintf(cmdmain.Stdout, "https://accounts.google.com/o/oauth2/auth?client_id=%s&redirect_uri=urn:ietf:wg:oauth:2.0:oob&scope=%s&response_type=code\n\n", - config.ClientId, config.Scope) - return prompt("Please enter the access code provided by that page:") + sc := bufio.NewScanner(cmdmain.Stdin) + sc.Scan() + return strings.TrimSpace(sc.Text()) } // Prompt for client id / secret -func getClientInfo() (string, string, error) { +func getClientInfo() (string, string) { fmt.Fprintf(cmdmain.Stdout, "Please provide the client id and client secret \n") fmt.Fprintf(cmdmain.Stdout, "(You can find these at http://code.google.com/apis/console > your project > API Access)\n") var ( - err error clientId string clientSecret string ) - if clientId, err = prompt("Client ID:"); err != nil { - return "", "", err - } - if clientSecret, err = prompt("Client Secret:"); err != nil { - return "", "", err - } - return clientId, clientSecret, nil + clientId = prompt("Client ID:") + clientSecret = prompt("Client Secret:") + return clientId, clientSecret } diff --git a/pkg/blobserver/google/cloudstorage/cloudstorage_test.go b/pkg/blobserver/google/cloudstorage/cloudstorage_test.go index 304ec9f8c..f039c7d59 100644 --- a/pkg/blobserver/google/cloudstorage/cloudstorage_test.go +++ b/pkg/blobserver/google/cloudstorage/cloudstorage_test.go @@ -26,13 +26,18 @@ import ( "camlistore.org/pkg/blob" "camlistore.org/pkg/blobserver" "camlistore.org/pkg/blobserver/storagetest" + "camlistore.org/pkg/constants/google" "camlistore.org/pkg/context" "camlistore.org/pkg/googlestorage" "camlistore.org/pkg/jsonconfig" - "camlistore.org/third_party/code.google.com/p/goauth2/oauth" + "camlistore.org/pkg/oauthutil" + + "camlistore.org/third_party/golang.org/x/oauth2" ) var ( + // TODO(mpl): use a config file generated with the help of googinit, like for googlestorage tests. + // And remove the 'camlistore-*-test' naming requirement ? bucket = flag.String("bucket", "", "Bucket name to use for testing. If empty, testing is skipped. If non-empty, it must begin with 'camlistore-' and end in '-test' and have zero items in it.") clientID = flag.String("client_id", "", "OAuth2 client_id for testing") clientSecret = flag.String("client_secret", "", "OAuth2 client secret for testing") @@ -59,31 +64,28 @@ func testStorage(t *testing.T, bucketDir string) { t.Fatal("--client_id and --client_secret required. Obtain from https://console.developers.google.com/ > Project > APIs & Auth > Credentials. Should be a 'native' or 'Installed application'") } - tokenCache := oauth.CacheFile(*tokenCache) - token, err := tokenCache.Token() + config := &oauth2.Config{ + Scopes: []string{googlestorage.Scope}, + Endpoint: google.Endpoint, + ClientID: *clientID, + ClientSecret: *clientSecret, + RedirectURL: oauthutil.TitleBarRedirectURL, + } + token, err := oauth2.ReuseTokenSource(nil, + &oauthutil.TokenSource{ + Config: config, + CacheFile: *tokenCache, + AuthCode: func() string { + if *authCode == "" { + t.Skipf("Re-run using --auth_code= with the value obtained from %s", + config.AuthCodeURL("", oauth2.AccessTypeOffline, oauth2.ApprovalForce)) + return "" + } + return *authCode + }, + }).Token() if err != nil { - config := &oauth.Config{ - // The client-id and secret should be for an "Installed Application" when using - // the CLI. Later we'll use a web application with a callback. - ClientId: *clientID, - ClientSecret: *clientSecret, - Scope: "https://www.googleapis.com/auth/devstorage.full_control", - AuthURL: "https://accounts.google.com/o/oauth2/auth", - TokenURL: "https://accounts.google.com/o/oauth2/token", - RedirectURL: "urn:ietf:wg:oauth:2.0:oob", - } - if *authCode != "" { - tr := &oauth.Transport{ - Config: config, - } - token, err = tr.Exchange(*authCode) - if err != nil { - t.Fatalf("Error getting a token using auth code: %v", err) - } - tokenCache.PutToken(token) - } else { - t.Skipf("Re-run using --auth_code= with the value obtained from %s", config.AuthCodeURL("")) - } + t.Fatalf("could not acquire token: %v", err) } bucketWithDir := path.Join(*bucket, bucketDir) @@ -108,11 +110,9 @@ func testStorage(t *testing.T, bucketDir string) { // Adding "a", and "c" objects in the bucket to make sure objects out of the // "directory" are not touched and have no influence. for _, key := range []string{"a", "c"} { - for tries, shouldRetry := 0, true; tries < 2 && shouldRetry; tries++ { - shouldRetry, err = sto.(*Storage).client.PutObject( - &googlestorage.Object{Bucket: sto.(*Storage).bucket, Key: key}, - strings.NewReader(key)) - } + err := sto.(*Storage).client.PutObject( + &googlestorage.Object{Bucket: sto.(*Storage).bucket, Key: key}, + strings.NewReader(key)) if err != nil { t.Fatalf("could not insert object %s in bucket %v: %v", key, sto.(*Storage).bucket, err) } diff --git a/pkg/blobserver/google/cloudstorage/storage.go b/pkg/blobserver/google/cloudstorage/storage.go index 2cde60936..0225a066c 100644 --- a/pkg/blobserver/google/cloudstorage/storage.go +++ b/pkg/blobserver/google/cloudstorage/storage.go @@ -35,10 +35,14 @@ import ( "camlistore.org/pkg/blobserver" "camlistore.org/pkg/blobserver/memory" "camlistore.org/pkg/constants" + "camlistore.org/pkg/constants/google" "camlistore.org/pkg/context" "camlistore.org/pkg/googlestorage" "camlistore.org/pkg/jsonconfig" + "camlistore.org/pkg/oauthutil" "camlistore.org/pkg/syncutil" + + "camlistore.org/third_party/golang.org/x/oauth2" ) type Storage struct { @@ -113,8 +117,14 @@ func newFromConfig(_ blobserver.Loader, config jsonconfig.Obj) (blobserver.Stora if refreshToken == "" { return nil, errors.New("missing required parameter 'refresh_token'") } - gs.client = googlestorage.NewClient(googlestorage.MakeOauthTransport( - clientID, clientSecret, refreshToken)) + oAuthClient := oauth2.NewClient(oauth2.NoContext, oauthutil.NewRefreshTokenSource(&oauth2.Config{ + Scopes: []string{googlestorage.Scope}, + Endpoint: google.Endpoint, + ClientID: clientID, + ClientSecret: clientSecret, + RedirectURL: oauthutil.TitleBarRedirectURL, + }, refreshToken)) + gs.client = googlestorage.NewClient(oAuthClient) } if cacheSize != 0 { @@ -165,11 +175,9 @@ func (s *Storage) ReceiveBlob(br blob.Ref, source io.Reader) (blob.SizedRef, err return blob.SizedRef{}, err } - for tries, shouldRetry := 0, true; tries < 2 && shouldRetry; tries++ { - shouldRetry, err = s.client.PutObject( - &googlestorage.Object{Bucket: s.bucket, Key: s.dirPrefix + br.String()}, - ioutil.NopCloser(bytes.NewReader(buf.Bytes()))) - } + err = s.client.PutObject( + &googlestorage.Object{Bucket: s.bucket, Key: s.dirPrefix + br.String()}, + ioutil.NopCloser(bytes.NewReader(buf.Bytes()))) if err != nil { return blob.SizedRef{}, err } diff --git a/pkg/blobserver/google/drive/auth.go b/pkg/blobserver/google/drive/auth.go deleted file mode 100644 index c8aa7d85e..000000000 --- a/pkg/blobserver/google/drive/auth.go +++ /dev/null @@ -1,48 +0,0 @@ -/* -Copyright 2014 The Camlistore Authors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package drive - -import ( - "time" - - "camlistore.org/third_party/code.google.com/p/goauth2/oauth" -) - -const ( - Scope = "https://www.googleapis.com/auth/drive" - AuthURL = "https://accounts.google.com/o/oauth2/auth" - TokenURL = "https://accounts.google.com/o/oauth2/token" - RedirectURL = "urn:ietf:wg:oauth:2.0:oob" -) - -func MakeOauthTransport(clientId string, clientSecret string, refreshToken string) *oauth.Transport { - return &oauth.Transport{ - Config: &oauth.Config{ - ClientId: clientId, - ClientSecret: clientSecret, - Scope: Scope, - AuthURL: AuthURL, - TokenURL: TokenURL, - RedirectURL: RedirectURL, - }, - Token: &oauth.Token{ - AccessToken: "", - RefreshToken: refreshToken, - Expiry: time.Time{}, // no expiry - }, - } -} diff --git a/pkg/blobserver/google/drive/drive.go b/pkg/blobserver/google/drive/drive.go index 7da7d14cd..00d2f384a 100644 --- a/pkg/blobserver/google/drive/drive.go +++ b/pkg/blobserver/google/drive/drive.go @@ -35,19 +35,17 @@ Example low-level config: package drive import ( - "net/http" - "time" - "camlistore.org/pkg/blobserver" "camlistore.org/pkg/blobserver/google/drive/service" + "camlistore.org/pkg/constants/google" "camlistore.org/pkg/jsonconfig" - "camlistore.org/third_party/code.google.com/p/goauth2/oauth" + "camlistore.org/pkg/oauthutil" + + "camlistore.org/third_party/golang.org/x/oauth2" ) -const ( - GoogleOAuth2AuthURL = "https://accounts.google.com/o/oauth2/auth" - GoogleOAuth2TokenURL = "https://accounts.google.com/o/oauth2/token" -) +// Scope is the OAuth2 scope used for Google Drive. +const Scope = "https://www.googleapis.com/auth/drive" type driveStorage struct { service *service.DriveService @@ -55,29 +53,18 @@ type driveStorage struct { func newFromConfig(_ blobserver.Loader, config jsonconfig.Obj) (blobserver.Storage, error) { auth := config.RequiredObject("auth") - oauthConf := &oauth.Config{ - ClientId: auth.RequiredString("client_id"), + oAuthClient := oauth2.NewClient(oauth2.NoContext, oauthutil.NewRefreshTokenSource(&oauth2.Config{ + Scopes: []string{Scope}, + Endpoint: google.Endpoint, + ClientID: auth.RequiredString("client_id"), ClientSecret: auth.RequiredString("client_secret"), - AuthURL: GoogleOAuth2AuthURL, - TokenURL: GoogleOAuth2TokenURL, - } - - // force refreshes the access token on start, make sure - // refresh request in parallel are being started - transport := &oauth.Transport{ - Token: &oauth.Token{ - AccessToken: "", - RefreshToken: auth.RequiredString("refresh_token"), - Expiry: time.Now(), - }, - Config: oauthConf, - Transport: http.DefaultTransport, - } + RedirectURL: oauthutil.TitleBarRedirectURL, + }, auth.RequiredString("refresh_token"))) parent := config.RequiredString("parent_id") if err := config.Validate(); err != nil { return nil, err } - service, err := service.New(transport, parent) + service, err := service.New(oAuthClient, parent) sto := &driveStorage{ service: service, } diff --git a/pkg/blobserver/google/drive/drive_test.go b/pkg/blobserver/google/drive/drive_test.go index 494857b53..e3d2c1b4d 100644 --- a/pkg/blobserver/google/drive/drive_test.go +++ b/pkg/blobserver/google/drive/drive_test.go @@ -23,11 +23,15 @@ import ( "camlistore.org/pkg/blobserver" "camlistore.org/pkg/blobserver/storagetest" + "camlistore.org/pkg/constants/google" "camlistore.org/pkg/jsonconfig" - "camlistore.org/third_party/code.google.com/p/goauth2/oauth" + "camlistore.org/pkg/oauthutil" + + "camlistore.org/third_party/golang.org/x/oauth2" ) var ( + // TODO(mpl): use a config file generated with the help of googinit, like for googlestorage tests. parentId = flag.String("parentDir", "", "id of the directory on google drive to use for testing. If empty or \"root\", testing is skipped.") clientID = flag.String("client_id", "", "OAuth2 client_id for testing") clientSecret = flag.String("client_secret", "", "OAuth2 client secret for testing") @@ -43,20 +47,28 @@ func TestStorage(t *testing.T) { t.Fatal("--client_id and --client_secret required. Obtain from https://console.developers.google.com/ > Project > APIs & Auth > Credentials. Should be a 'native' or 'Installed application'") } - tokenCache := oauth.CacheFile(*tokenCache) - token, err := tokenCache.Token() + config := &oauth2.Config{ + Scopes: []string{Scope}, + Endpoint: google.Endpoint, + ClientID: *clientID, + ClientSecret: *clientSecret, + RedirectURL: oauthutil.TitleBarRedirectURL, + } + token, err := oauth2.ReuseTokenSource(nil, + &oauthutil.TokenSource{ + Config: config, + CacheFile: *tokenCache, + AuthCode: func() string { + if *authCode == "" { + t.Skipf("Re-run using --auth_code= with the value obtained from %s", + config.AuthCodeURL("", oauth2.AccessTypeOffline, oauth2.ApprovalForce)) + return "" + } + return *authCode + }, + }).Token() if err != nil { - tr := MakeOauthTransport(*clientID, *clientSecret, "") - config := tr.Config - if *authCode != "" { - token, err = tr.Exchange(*authCode) - if err != nil { - t.Fatalf("Error getting a token using auth code: %v", err) - } - tokenCache.PutToken(token) - } else { - t.Skipf("Re-run using --auth_code= with the value obtained from %s", config.AuthCodeURL("")) - } + t.Fatalf("could not acquire token: %v", err) } storagetest.TestOpt(t, storagetest.Opts{ diff --git a/pkg/blobserver/google/drive/service/service.go b/pkg/blobserver/google/drive/service/service.go index 3fac3cf8b..063367b08 100644 --- a/pkg/blobserver/google/drive/service/service.go +++ b/pkg/blobserver/google/drive/service/service.go @@ -26,7 +26,6 @@ import ( "net/http" "os" - "camlistore.org/third_party/code.google.com/p/goauth2/oauth" client "camlistore.org/third_party/google.golang.org/api/drive/v2" ) @@ -38,7 +37,7 @@ const ( // DriveService wraps Google Drive API to implement utility methods to // be performed on the root Drive destination folder. type DriveService struct { - transport *oauth.Transport + client *http.Client apiservice *client.Service parentId string } @@ -47,8 +46,8 @@ type DriveService struct { // that will be used as the current directory in methods on the returned // DriveService (such as Get). If empty, it defaults to the root of the // drive. -func New(transport *oauth.Transport, parentId string) (*DriveService, error) { - apiservice, err := client.New(transport.Client()) +func New(oauthClient *http.Client, parentId string) (*DriveService, error) { + apiservice, err := client.New(oauthClient) if err != nil { return nil, err } @@ -56,7 +55,7 @@ func New(transport *oauth.Transport, parentId string) (*DriveService, error) { // because "root" is known as a special alias for the root directory in drive. parentId = "root" } - service := &DriveService{transport: transport, apiservice: apiservice, parentId: parentId} + service := &DriveService{client: oauthClient, apiservice: apiservice, parentId: parentId} return service, err } @@ -141,7 +140,7 @@ func (s *DriveService) Fetch(title string) (body io.ReadCloser, size uint32, err req, _ := http.NewRequest("GET", file.DownloadUrl, nil) var resp *http.Response - if resp, err = s.transport.RoundTrip(req); err != nil { + if resp, err = s.client.Transport.RoundTrip(req); err != nil { return } if file.FileSize > math.MaxUint32 || file.FileSize < 0 { diff --git a/pkg/googlestorage/README b/pkg/googlestorage/README index e8870e9e5..b13900591 100644 --- a/pkg/googlestorage/README +++ b/pkg/googlestorage/README @@ -38,4 +38,4 @@ In order to run these tests properly, you will need to: } - You can use 'camtool gsinit' to help obtain the auth config object. + You can use 'camtool googinit' to help obtain the auth config object. diff --git a/pkg/googlestorage/auth.go b/pkg/googlestorage/auth.go deleted file mode 100644 index 8b6e4f2fb..000000000 --- a/pkg/googlestorage/auth.go +++ /dev/null @@ -1,48 +0,0 @@ -/* -Copyright 2013 Google Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package googlestorage - -import ( - "time" - - "camlistore.org/third_party/code.google.com/p/goauth2/oauth" -) - -const ( - Scope = "https://www.googleapis.com/auth/devstorage.read_write" - AuthURL = "https://accounts.google.com/o/oauth2/auth" - TokenURL = "https://accounts.google.com/o/oauth2/token" - RedirectURL = "urn:ietf:wg:oauth:2.0:oob" -) - -func MakeOauthTransport(clientId string, clientSecret string, refreshToken string) *oauth.Transport { - return &oauth.Transport{ - Config: &oauth.Config{ - ClientId: clientId, - ClientSecret: clientSecret, - Scope: Scope, - AuthURL: AuthURL, - TokenURL: TokenURL, - RedirectURL: RedirectURL, - }, - Token: &oauth.Token{ - AccessToken: "", - RefreshToken: refreshToken, - Expiry: time.Time{}, // no expiry - }, - } -} diff --git a/pkg/googlestorage/googlestorage.go b/pkg/googlestorage/googlestorage.go index 323b141c0..4b080be7d 100644 --- a/pkg/googlestorage/googlestorage.go +++ b/pkg/googlestorage/googlestorage.go @@ -35,7 +35,7 @@ import ( "camlistore.org/pkg/blob" "camlistore.org/pkg/httputil" - "camlistore.org/third_party/code.google.com/p/goauth2/oauth" + "camlistore.org/third_party/golang.org/x/net/context" "camlistore.org/third_party/golang.org/x/oauth2" "camlistore.org/third_party/golang.org/x/oauth2/google" @@ -45,13 +45,14 @@ import ( const ( gsAccessURL = "https://storage.googleapis.com" + // Scope is the OAuth2 scope used for Google Cloud Storage. + Scope = "https://www.googleapis.com/auth/devstorage.read_write" ) // A Client provides access to Google Cloud Storage. type Client struct { - client *http.Client - transport *oauth.Transport // nil for service clients - service *api.Service + client *http.Client + service *api.Service } // An Object holds the name of an object (its bucket and key) within @@ -105,13 +106,11 @@ func NewServiceClient() (*Client, error) { return &Client{client: client, service: service}, nil } -func NewClient(transport *oauth.Transport) *Client { - client := transport.Client() - service, _ := api.New(client) +func NewClient(oauthClient *http.Client) *Client { + service, _ := api.New(oauthClient) return &Client{ - client: transport.Client(), - transport: transport, - service: service, + client: oauthClient, + service: service, } } @@ -126,40 +125,6 @@ func (so SizedObject) String() string { return fmt.Sprintf("%v/%v (%vB)", so.Bucket, so.Key, so.Size) } -// A close relative to http.Client.Do(), helping with token refresh logic. -// If canResend is true and the initial request's response is an auth error -// (401 or 403), oauth credentials will be refreshed and the request sent -// again. This should only be done for requests with empty bodies, since the -// Body will be consumed on the first attempt if it exists. -// If canResend is false, and req would have been resent if canResend were -// true, then shouldRetry will be true. -// One of resp or err will always be nil. -func (gsa *Client) doRequest(req *http.Request, canResend bool) (resp *http.Response, err error, shouldRetry bool) { - resp, err = gsa.client.Do(req) - if err != nil { - return - } - if gsa.transport == nil { - return - } - - if resp.StatusCode == 401 || resp.StatusCode == 403 { - // Unauth. Perhaps tokens need refreshing? - if err = gsa.transport.Refresh(); err != nil { - return - } - // Refresh succeeded. req should be resent - if !canResend { - return resp, nil, true - } - // Resend req. First, need to close the soon-overwritten response Body - resp.Body.Close() - resp, err = gsa.client.Do(req) - } - - return -} - // Makes a simple body-less google storage request func (gsa *Client) simpleRequest(method, url_ string) (resp *http.Response, err error) { // Construct the request @@ -169,8 +134,7 @@ func (gsa *Client) simpleRequest(method, url_ string) (resp *http.Response, err } req.Header.Set("x-goog-api-version", "2") - resp, err, _ = gsa.doRequest(req, true) - return + return gsa.client.Do(req) } // GetObject fetches a Google Cloud Storage object. @@ -217,7 +181,7 @@ func (c *Client) GetPartialObject(obj Object, offset, length int64) (rc io.ReadC req.Header.Set("Range", fmt.Sprintf("bytes=%d-", offset)) } - resp, err, _ := c.doRequest(req, true) + resp, err := c.client.Do(req) if err != nil { return nil, fmt.Errorf("GS GET request failed: %v\n", err) } @@ -265,15 +229,15 @@ func (gsa *Client) StatObject(obj *Object) (size int64, exists bool, err error) // shouldRetry will be true if the put failed due to authorization, but // credentials have been refreshed and another attempt is likely to succeed. // In this case, content will have been consumed. -func (gsa *Client) PutObject(obj *Object, content io.Reader) (shouldRetry bool, err error) { +func (gsa *Client) PutObject(obj *Object, content io.Reader) error { if err := obj.valid(); err != nil { - return false, err + return err } const maxSlurp = 2 << 20 var buf bytes.Buffer n, err := io.CopyN(&buf, content, maxSlurp) if err != nil && err != io.EOF { - return false, err + return err } contentType := http.DetectContentType(buf.Bytes()) if contentType == "application/octet-stream" && n < maxSlurp && utf8.Valid(buf.Bytes()) { @@ -283,20 +247,20 @@ func (gsa *Client) PutObject(obj *Object, content io.Reader) (shouldRetry bool, objURL := gsAccessURL + "/" + obj.Bucket + "/" + obj.Key var req *http.Request if req, err = http.NewRequest("PUT", objURL, ioutil.NopCloser(io.MultiReader(&buf, content))); err != nil { - return + return err } req.Header.Set("x-goog-api-version", "2") req.Header.Set("Content-Type", contentType) var resp *http.Response - if resp, err, shouldRetry = gsa.doRequest(req, false); err != nil { - return + if resp, err = gsa.client.Do(req); err != nil { + return err } if resp.StatusCode != http.StatusOK { - return shouldRetry, fmt.Errorf("Bad put response code: %v", resp.Status) + return fmt.Errorf("Bad put response code: %v", resp.Status) } - return + return nil } // DeleteObject removes an object. diff --git a/pkg/googlestorage/googlestorage_test.go b/pkg/googlestorage/googlestorage_test.go index 8e87dfa02..32a65c46a 100644 --- a/pkg/googlestorage/googlestorage_test.go +++ b/pkg/googlestorage/googlestorage_test.go @@ -27,7 +27,11 @@ import ( "testing" "time" + "camlistore.org/pkg/constants/google" "camlistore.org/pkg/jsonconfig" + "camlistore.org/pkg/oauthutil" + + "camlistore.org/third_party/golang.org/x/oauth2" ) const testObjectContent = "Google Storage Test\n" @@ -66,9 +70,13 @@ func doConfig(t *testing.T) (gsa *Client, bucket string) { t.Fatalf("Invalid config: %v", err) } - gsa = NewClient(MakeOauthTransport(auth.RequiredString("client_id"), - auth.RequiredString("client_secret"), - auth.RequiredString("refresh_token"))) + gsa = NewClient(oauth2.NewClient(oauth2.NoContext, oauthutil.NewRefreshTokenSource(&oauth2.Config{ + Scopes: []string{Scope}, + Endpoint: google.Endpoint, + ClientID: auth.RequiredString("client_id"), + ClientSecret: auth.RequiredString("client_secret"), + RedirectURL: oauthutil.TitleBarRedirectURL, + }, auth.RequiredString("refresh_token")))) if err := auth.Validate(); err != nil { t.Fatalf("Invalid config: %v", err) @@ -158,12 +166,8 @@ func TestPutObject(t *testing.T) { testKey := fmt.Sprintf("test-put-%v.%v.%v-%v.%v.%v", now.Year(), now.Month(), now.Day(), now.Hour(), now.Minute(), now.Second()) - shouldRetry, err := gs.PutObject(&Object{bucket, testKey}, + err := gs.PutObject(&Object{bucket, testKey}, &BufferCloser{bytes.NewBufferString(testObjectContent)}) - if shouldRetry { - shouldRetry, err = gs.PutObject(&Object{bucket, testKey}, - &BufferCloser{bytes.NewBufferString(testObjectContent)}) - } if err != nil { t.Fatalf("Failed to put object: %v", err) } @@ -192,7 +196,7 @@ func TestDeleteObject(t *testing.T) { now := time.Now() testKey := fmt.Sprintf("test-delete-%v.%v.%v-%v.%v.%v", now.Year(), now.Month(), now.Day(), now.Hour(), now.Minute(), now.Second()) - _, err = gs.PutObject(&Object{bucket, testKey}, + err = gs.PutObject(&Object{bucket, testKey}, &BufferCloser{bytes.NewBufferString("Delete Me")}) if err != nil { t.Fatalf("Failed to put file to delete.") diff --git a/pkg/oauthutil/oauth.go b/pkg/oauthutil/oauth.go new file mode 100644 index 000000000..3f8e5fc6b --- /dev/null +++ b/pkg/oauthutil/oauth.go @@ -0,0 +1,121 @@ +/* +Copyright 2015 The Camlistore Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package oauthutil + +import ( + "encoding/json" + "errors" + "fmt" + "time" + + "camlistore.org/pkg/wkfs" + + "camlistore.org/third_party/golang.org/x/oauth2" +) + +// TitleBarRedirectURL is the OAuth2 redirect URL to use when the authorization +// code should be returned in the title bar of the browser, with the page text +// prompting the user to copy the code and paste it in the application. +const TitleBarRedirectURL = "urn:ietf:wg:oauth:2.0:oob" + +// ErrNoAuthCode is returned when Token() has not found any valid cached token +// and TokenSource does not have an AuthCode for getting a new token. +var ErrNoAuthCode = errors.New("oauthutil: unspecified TokenSource.AuthCode") + +// TokenSource is an implementation of oauth2.TokenSource. It uses CacheFile to store and +// reuse the the acquired token, and AuthCode to provide the authorization code that will be +// exchanged for a token otherwise. +type TokenSource struct { + Config *oauth2.Config + + // CacheFile is where the token will be stored JSON-encoded. Any call to Token + // first tries to read a valid token from CacheFile. + CacheFile string + + // AuthCode provides the authorization code that Token will exchange for a token. + // It usually is a way to prompt the user for the code. If CacheFile does not provide + // a token and AuthCode is nil, Token returns ErrNoAuthCode. + AuthCode func() string +} + +var errExpiredToken = errors.New("expired token") + +// cachedToken returns the token saved in cacheFile. It specifically returns +// errTokenExpired if the token is expired. +func cachedToken(cacheFile string) (*oauth2.Token, error) { + tok := new(oauth2.Token) + tokenData, err := wkfs.ReadFile(cacheFile) + if err != nil { + return nil, err + } + if err = json.Unmarshal(tokenData, tok); err != nil { + return nil, err + } + if !tok.Valid() { + if tok != nil && time.Now().After(tok.Expiry) { + return nil, errExpiredToken + } + return nil, errors.New("invalid token") + } + return tok, nil +} + +// Token first tries to find a valid token in CacheFile, and otherwise uses +// Config and AuthCode to fetch a new token. This new token is saved in CacheFile +// (if not blank). If CacheFile did not provide a token and AuthCode is nil, +// ErrNoAuthCode is returned. +func (src TokenSource) Token() (*oauth2.Token, error) { + var tok *oauth2.Token + var err error + if src.CacheFile != "" { + tok, err = cachedToken(src.CacheFile) + if err == nil { + return tok, nil + } + if err != errExpiredToken { + fmt.Printf("Error getting token from %s: %v\n", src.CacheFile, err) + } + } + if src.AuthCode == nil { + return nil, ErrNoAuthCode + } + tok, err = src.Config.Exchange(oauth2.NoContext, src.AuthCode()) + if err != nil { + return nil, fmt.Errorf("could not exchange auth code for a token: %v", err) + } + if src.CacheFile == "" { + return tok, nil + } + tokenData, err := json.Marshal(&tok) + if err != nil { + return nil, fmt.Errorf("could not encode token as json: %v", err) + } + if err := wkfs.WriteFile(src.CacheFile, tokenData, 0600); err != nil { + return nil, fmt.Errorf("could not cache token in %v: %v", src.CacheFile, err) + } + return tok, nil +} + +// NewRefreshTokenSource returns a token source that obtains its initial token +// based on the provided config and the refresh token. +func NewRefreshTokenSource(config *oauth2.Config, refreshToken string) oauth2.TokenSource { + var noInitialToken *oauth2.Token = nil + return oauth2.ReuseTokenSource(noInitialToken, config.TokenSource( + oauth2.NoContext, // TODO: maybe accept a context later. + &oauth2.Token{RefreshToken: refreshToken}, + )) +} diff --git a/pkg/wkfs/gcs/gcs.go b/pkg/wkfs/gcs/gcs.go index d545f164b..e65fa27eb 100644 --- a/pkg/wkfs/gcs/gcs.go +++ b/pkg/wkfs/gcs/gcs.go @@ -172,17 +172,10 @@ func (w *fileWriter) Close() (err error) { return nil } w.closed = true - var retry bool - for tries := 0; tries < 2; tries++ { - retry, err = w.fs.client.PutObject(&googlestorage.Object{ - Bucket: w.bucket, - Key: w.key, - }, ioutil.NopCloser(bytes.NewReader(w.buf.Bytes()))) - if retry { - continue - } - } - return err + return w.fs.client.PutObject(&googlestorage.Object{ + Bucket: w.bucket, + Key: w.key, + }, ioutil.NopCloser(bytes.NewReader(w.buf.Bytes()))) } type statInfo struct {