From c9a0beae4547fc775184b72ebb2ea57c36d9008a Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Wed, 1 Apr 2015 08:37:32 -0700 Subject: [PATCH] Add new 'env' package to detect the type of environment. Also, delete my old gce package from third_party and only use the google metadata package (which my gce package became, and which was also already vendored into third_party) Fixes #596 Change-Id: I64fd6f1e9dc6f433466f91f81efd2ecbf039334f --- pkg/env/env.go | 46 +++ pkg/googlestorage/googlestorage.go | 28 +- pkg/osutil/gce/gce.go | 9 +- pkg/serverinit/env.go | 17 +- pkg/sorted/mysql/cloudsql.go | 11 +- pkg/wkfs/gcs/gcs.go | 4 +- server/camlistored/camlistored.go | 4 +- third_party/github.com/bradfitz/gce/LICENSE | 7 - third_party/github.com/bradfitz/gce/README | 5 - third_party/github.com/bradfitz/gce/gce.go | 312 -------------------- 10 files changed, 91 insertions(+), 352 deletions(-) create mode 100644 pkg/env/env.go delete mode 100644 third_party/github.com/bradfitz/gce/LICENSE delete mode 100644 third_party/github.com/bradfitz/gce/README delete mode 100644 third_party/github.com/bradfitz/gce/gce.go diff --git a/pkg/env/env.go b/pkg/env/env.go new file mode 100644 index 000000000..44da8090a --- /dev/null +++ b/pkg/env/env.go @@ -0,0 +1,46 @@ +/* +Copyright 2015 The Camlistore Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package env detects what sort of environment Camlistore is running in. +package env + +import ( + "sync" + + "camlistore.org/third_party/google.golang.org/cloud/compute/metadata" +) + +// OsGCE reports whether this process is running in a Google Compute +// Engine (GCE) environment. This only returns true if the +// "camlistore-config-dir" instance metadata value is defined. +// Instances running in custom configs on GCE will be unaffected. +func OnGCE() bool { + gceOnce.Do(detectGCE) + return isGCE +} + +var ( + gceOnce sync.Once + isGCE bool +) + +func detectGCE() { + if !metadata.OnGCE() { + return + } + v, _ := metadata.InstanceAttributeValue("camlistore-config-dir") + isGCE = v != "" +} diff --git a/pkg/googlestorage/googlestorage.go b/pkg/googlestorage/googlestorage.go index a3beb4b9f..323b141c0 100644 --- a/pkg/googlestorage/googlestorage.go +++ b/pkg/googlestorage/googlestorage.go @@ -36,8 +36,11 @@ import ( "camlistore.org/pkg/blob" "camlistore.org/pkg/httputil" "camlistore.org/third_party/code.google.com/p/goauth2/oauth" - "camlistore.org/third_party/github.com/bradfitz/gce" + "camlistore.org/third_party/golang.org/x/net/context" + "camlistore.org/third_party/golang.org/x/oauth2" + "camlistore.org/third_party/golang.org/x/oauth2/google" api "camlistore.org/third_party/google.golang.org/api/storage/v1" + "camlistore.org/third_party/google.golang.org/cloud/compute/metadata" ) const ( @@ -78,19 +81,28 @@ type SizedObject struct { } // NewServiceClient returns a Client for use when running on Google -// Compute Engine. This client can access buckets owned by the samre +// Compute Engine. This client can access buckets owned by the same // project ID as the VM. func NewServiceClient() (*Client, error) { - if !gce.OnGCE() { + if !metadata.OnGCE() { return nil, errors.New("not running on Google Compute Engine") } - scopes, _ := gce.Scopes("default") - if !scopes.Contains("https://www.googleapis.com/auth/devstorage.full_control") && - !scopes.Contains("https://www.googleapis.com/auth/devstorage.read_write") { + scopes, _ := metadata.Scopes("default") + haveScope := func(scope string) bool { + for _, x := range scopes { + if x == scope { + return true + } + } + return false + } + if !haveScope("https://www.googleapis.com/auth/devstorage.full_control") && + !haveScope("https://www.googleapis.com/auth/devstorage.read_write") { return nil, errors.New("when this Google Compute Engine VM instance was created, it wasn't granted access to Cloud Storage") } - service, _ := api.New(gce.Client) - return &Client{client: gce.Client, service: service}, nil + client := oauth2.NewClient(context.Background(), google.ComputeTokenSource("")) + service, _ := api.New(client) + return &Client{client: client, service: service}, nil } func NewClient(transport *oauth.Transport) *Client { diff --git a/pkg/osutil/gce/gce.go b/pkg/osutil/gce/gce.go index 5191e2621..c438d1109 100644 --- a/pkg/osutil/gce/gce.go +++ b/pkg/osutil/gce/gce.go @@ -23,18 +23,19 @@ import ( "path" "strings" + "camlistore.org/pkg/env" "camlistore.org/pkg/jsonconfig" "camlistore.org/pkg/osutil" _ "camlistore.org/pkg/wkfs/gcs" - "camlistore.org/third_party/github.com/bradfitz/gce" + "camlistore.org/third_party/google.golang.org/cloud/compute/metadata" ) func init() { - if !gce.OnGCE() { + if !env.OnGCE() { return } osutil.RegisterConfigDirFunc(func() string { - v, _ := gce.InstanceAttributeValue("camlistore-config-dir") + v, _ := metadata.InstanceAttributeValue("camlistore-config-dir") if v == "" { return v } @@ -48,7 +49,7 @@ func init() { if !ok { return nil, errors.New("expected argument after _gce_instance_meta to be a string") } - val, err := gce.InstanceAttributeValue(attr) + val, err := metadata.InstanceAttributeValue(attr) if err != nil { return nil, fmt.Errorf("error reading GCE instance attribute %q: %v", attr, err) } diff --git a/pkg/serverinit/env.go b/pkg/serverinit/env.go index 0388d45ef..fb2ba670f 100644 --- a/pkg/serverinit/env.go +++ b/pkg/serverinit/env.go @@ -21,26 +21,27 @@ import ( "os" "strings" + "camlistore.org/pkg/env" "camlistore.org/pkg/osutil" "camlistore.org/pkg/types/serverconfig" - "camlistore.org/third_party/github.com/bradfitz/gce" + "camlistore.org/third_party/google.golang.org/cloud/compute/metadata" ) // DefaultEnvConfig returns the default configuration when running on a known // environment. Currently this just includes Google Compute Engine. // If the environment isn't known (nil, nil) is returned. func DefaultEnvConfig() (*Config, error) { - if !gce.OnGCE() { + if !env.OnGCE() { return nil, nil } auth := "none" - user, _ := gce.InstanceAttributeValue("camlistore-username") - pass, _ := gce.InstanceAttributeValue("camlistore-password") - confBucket, err := gce.InstanceAttributeValue("camlistore-config-dir") + user, _ := metadata.InstanceAttributeValue("camlistore-username") + pass, _ := metadata.InstanceAttributeValue("camlistore-password") + confBucket, err := metadata.InstanceAttributeValue("camlistore-config-dir") if confBucket == "" || err != nil { return nil, fmt.Errorf("VM instance metadata key 'camlistore-config-dir' not set: %v", err) } - blobBucket, err := gce.InstanceAttributeValue("camlistore-blob-dir") + blobBucket, err := metadata.InstanceAttributeValue("camlistore-blob-dir") if blobBucket == "" || err != nil { return nil, fmt.Errorf("VM instance metadata key 'camlistore-blob-dir' not set: %v", err) } @@ -56,8 +57,8 @@ func DefaultEnvConfig() (*Config, error) { return nil, err } - ipOrHost, _ := gce.ExternalIP() - host, _ := gce.InstanceAttributeValue("camlistore-hostname") + ipOrHost, _ := metadata.ExternalIP() + host, _ := metadata.InstanceAttributeValue("camlistore-hostname") if host != "" { ipOrHost = host } diff --git a/pkg/sorted/mysql/cloudsql.go b/pkg/sorted/mysql/cloudsql.go index 6cd8b1ba6..add111ca4 100644 --- a/pkg/sorted/mysql/cloudsql.go +++ b/pkg/sorted/mysql/cloudsql.go @@ -23,8 +23,11 @@ import ( "log" "strings" + "camlistore.org/third_party/golang.org/x/net/context" + "camlistore.org/third_party/golang.org/x/oauth2" + "camlistore.org/third_party/golang.org/x/oauth2/google" sqladmin "camlistore.org/third_party/google.golang.org/api/sqladmin/v1beta3" - "camlistore.org/third_party/github.com/bradfitz/gce" + "camlistore.org/third_party/google.golang.org/cloud/compute/metadata" ) const cloudSQLSuffix = ".cloudsql.google.internal" @@ -34,15 +37,15 @@ func maybeRemapCloudSQL(host string) (out string, err error) { return host, nil } inst := strings.TrimSuffix(host, cloudSQLSuffix) - if !gce.OnGCE() { + if !metadata.OnGCE() { return "", errors.New("CloudSQL support only available when running on Google Compute Engine.") } - proj, err := gce.ProjectID() + proj, err := metadata.ProjectID() if err != nil { return "", fmt.Errorf("Failed to lookup GCE project ID: %v", err) } - admin, _ := sqladmin.New(gce.Client) + admin, _ := sqladmin.New(oauth2.NewClient(context.Background(), google.ComputeTokenSource(""))) listRes, err := admin.Instances.List(proj).Do() if err != nil { return "", fmt.Errorf("error enumerating Cloud SQL instances: %v", err) diff --git a/pkg/wkfs/gcs/gcs.go b/pkg/wkfs/gcs/gcs.go index 927667618..d545f164b 100644 --- a/pkg/wkfs/gcs/gcs.go +++ b/pkg/wkfs/gcs/gcs.go @@ -33,7 +33,7 @@ import ( "camlistore.org/pkg/googlestorage" "camlistore.org/pkg/wkfs" - "camlistore.org/third_party/github.com/bradfitz/gce" + "camlistore.org/third_party/google.golang.org/cloud/compute/metadata" ) // Max size for all files read or written. This filesystem is only @@ -42,7 +42,7 @@ import ( const maxSize = 1 << 20 func init() { - if !gce.OnGCE() { + if !metadata.OnGCE() { return } client, err := googlestorage.NewServiceClient() diff --git a/server/camlistored/camlistored.go b/server/camlistored/camlistored.go index 9b329d08c..56af4057f 100644 --- a/server/camlistored/camlistored.go +++ b/server/camlistored/camlistored.go @@ -35,6 +35,7 @@ import ( "time" "camlistore.org/pkg/buildinfo" + "camlistore.org/pkg/env" "camlistore.org/pkg/httputil" "camlistore.org/pkg/legal/legalprint" "camlistore.org/pkg/netutil" @@ -45,7 +46,6 @@ import ( // VM environments: _ "camlistore.org/pkg/osutil/gce" - "camlistore.org/third_party/github.com/bradfitz/gce" // Storage options: _ "camlistore.org/pkg/blobserver/blobpacked" @@ -400,7 +400,7 @@ func Main(up chan<- struct{}, down <-chan struct{}) { } log.Printf("Available on %s", urlToOpen) - if gce.OnGCE() && strings.HasPrefix(baseURL, "https://") { + if env.OnGCE() && strings.HasPrefix(baseURL, "https://") { go redirectFromHTTP(baseURL) } diff --git a/third_party/github.com/bradfitz/gce/LICENSE b/third_party/github.com/bradfitz/gce/LICENSE deleted file mode 100644 index 2dc6853ca..000000000 --- a/third_party/github.com/bradfitz/gce/LICENSE +++ /dev/null @@ -1,7 +0,0 @@ -Copyright 2014 Google & the Go AUTHORS - -Go AUTHORS are: -See https://code.google.com/p/go/source/browse/AUTHORS - -Licensed under the terms of Go itself: -https://code.google.com/p/go/source/browse/LICENSE diff --git a/third_party/github.com/bradfitz/gce/README b/third_party/github.com/bradfitz/gce/README deleted file mode 100644 index f113e47c3..000000000 --- a/third_party/github.com/bradfitz/gce/README +++ /dev/null @@ -1,5 +0,0 @@ -Package gce provides access to Google Compute Engine (GCE) metadata and -API service accounts. - -See the code for docs, or as HTML at http://godoc.org/github.com/bradfitz/gce . - diff --git a/third_party/github.com/bradfitz/gce/gce.go b/third_party/github.com/bradfitz/gce/gce.go deleted file mode 100644 index e765945b8..000000000 --- a/third_party/github.com/bradfitz/gce/gce.go +++ /dev/null @@ -1,312 +0,0 @@ -/* -Copyright 2014 Google & the Go AUTHORS - -Go AUTHORS are: -See https://code.google.com/p/go/source/browse/AUTHORS - -Licensed under the terms of Go itself: -https://code.google.com/p/go/source/browse/LICENSE -*/ - -// Package gce provides access to Google Compute Engine (GCE) metadata and -// API service accounts. -// -// Most of this package is a wrapper around the GCE metadata service, -// as documented at https://developers.google.com/compute/docs/metadata. -package gce - -import ( - "encoding/json" - "errors" - "fmt" - "io/ioutil" - "net" - "net/http" - "strings" - "sync" - "time" -) - -// Strings is a list of strings. -type Strings []string - -// Contains reports whether v is contained in s. -func (s Strings) Contains(v string) bool { - for _, sv := range s { - if v == sv { - return true - } - } - return false -} - -var metaClient = &http.Client{ - Transport: &http.Transport{ - Dial: (&net.Dialer{ - Timeout: 750 * time.Millisecond, - KeepAlive: 30 * time.Second, - }).Dial, - ResponseHeaderTimeout: 750 * time.Millisecond, - }, -} - -// MetadataValue returns a value from the metadata service. -// The suffix is appended to "http://metadata/computeMetadata/v1/". -func MetadataValue(suffix string) (string, error) { - // Using 169.254.169.254 instead of "metadata" here because Go - // binaries built with the "netgo" tag and without cgo won't - // know the search suffix for "metadata" is - // ".google.internal", and this IP address is documented as - // being stable anyway. - url := "http://169.254.169.254/computeMetadata/v1/" + suffix - req, _ := http.NewRequest("GET", url, nil) - req.Header.Set("Metadata-Flavor", "Google") - res, err := metaClient.Do(req) - if err != nil { - return "", err - } - defer res.Body.Close() - if res.StatusCode != 200 { - return "", fmt.Errorf("status code %d trying to fetch %s", res.StatusCode, url) - } - all, err := ioutil.ReadAll(res.Body) - if err != nil { - return "", err - } - return string(all), nil -} - -func metaValueTrim(suffix string) (s string, err error) { - s, err = MetadataValue(suffix) - s = strings.TrimSpace(s) - return -} - -type cachedValue struct { - k string - trim bool - mu sync.Mutex - v string -} - -var ( - proj = &cachedValue{k: "project/project-id", trim: true} - projID = &cachedValue{k: "project/numeric-project-id", trim: true} - instID = &cachedValue{k: "instance/id", trim: true} -) - -func (c *cachedValue) get() (v string, err error) { - defer c.mu.Unlock() - c.mu.Lock() - if c.v != "" { - return c.v, nil - } - if c.trim { - v, err = metaValueTrim(c.k) - } else { - v, err = MetadataValue(c.k) - } - if err == nil { - c.v = v - } - return -} - -var onGCE struct { - sync.Mutex - set bool - v bool -} - -// OnGCE reports whether this process is running on Google Compute Engine. -func OnGCE() bool { - defer onGCE.Unlock() - onGCE.Lock() - if onGCE.set { - return onGCE.v - } - onGCE.set = true - - res, err := metaClient.Get("http://metadata.google.internal") - if err != nil { - return false - } - onGCE.v = res.Header.Get("Metadata-Flavor") == "Google" - return onGCE.v -} - -// ProjectID returns the current instance's project ID string. -func ProjectID() (string, error) { return proj.get() } - -// NumericProjectID returns the current instance's numeric project ID. -func NumericProjectID() (string, error) { return projID.get() } - -// InternalIP returns the instance's primary internal IP address. -func InternalIP() (string, error) { - return metaValueTrim("instance/network-interfaces/0/ip") -} - -// ExternalIP returns the instance's primary external (public) IP address. -func ExternalIP() (string, error) { - return metaValueTrim("instance/network-interfaces/0/access-configs/0/external-ip") -} - -// Hostname returns the instance's hostname. This will probably be of -// the form "INSTANCENAME.c.PROJECT.internal" but that isn't -// guaranteed. -// -// TODO: what is this defined to be? Docs say "The host name of the -// instance." -func Hostname() (string, error) { - return metaValueTrim("network-interfaces/0/ip") -} - -// InstanceTags returns the list of user-defined instance tags, -// assigned when initially creating a GCE instance. -func InstanceTags() (Strings, error) { - var s Strings - j, err := MetadataValue("instance/tags") - if err != nil { - return nil, err - } - if err := json.NewDecoder(strings.NewReader(j)).Decode(&s); err != nil { - return nil, err - } - return s, nil -} - -// InstanceID returns the current VM's numeric instance ID. -func InstanceID() (string, error) { - return instID.get() -} - -// InstanceAttributes returns the list of user-defined attributes, -// assigned when initially creating a GCE VM instance. The value of an -// attribute can be obtained with InstanceAttributeValue. -func InstanceAttributes() (Strings, error) { return lines("instance/attributes/") } - -// ProjectAttributes returns the list of user-defined attributes -// applying to the project as a whole, not just this VM. The value of -// an attribute can be obtained with ProjectAttributeValue. -func ProjectAttributes() (Strings, error) { return lines("project/attributes/") } - -func lines(suffix string) (Strings, error) { - j, err := MetadataValue(suffix) - if err != nil { - return nil, err - } - s := strings.Split(strings.TrimSpace(j), "\n") - for i := range s { - s[i] = strings.TrimSpace(s[i]) - } - return Strings(s), nil -} - -// InstanceAttributeValue returns the value of the provided VM -// instance attribute. -func InstanceAttributeValue(attr string) (string, error) { - return MetadataValue("instance/attributes/" + attr) -} - -// ProjectAttributeValue returns the value of the provided -// project attribute. -func ProjectAttributeValue(attr string) (string, error) { - return MetadataValue("project/attributes/" + attr) -} - -// Scopes returns the service account scopes for the given account. -// The account may be empty or the string "default" to use the instance's -// main account. -func Scopes(serviceAccount string) (Strings, error) { - if serviceAccount == "" { - serviceAccount = "default" - } - return lines("instance/service-accounts/" + serviceAccount + "/scopes") -} - -// Transport is an HTTP transport that adds authentication headers to -// the request using the default GCE service account and forwards the -// requests to the http package's default transport. -var Transport = NewTransport("default", http.DefaultTransport) - -// Client is an http Client that uses the default GCE transport. -var Client = &http.Client{Transport: Transport} - -// NewTransport returns a transport that uses the provided GCE -// serviceAccount (optional) to add authentication headers and then -// uses the provided underlying "base" transport. -// -// For more information on Service Accounts, see -// https://developers.google.com/compute/docs/authentication. -func NewTransport(serviceAccount string, base http.RoundTripper) http.RoundTripper { - if serviceAccount == "" { - serviceAccount = "default" - } - return &transport{base: base, acct: serviceAccount} -} - -type transport struct { - base http.RoundTripper - acct string - - mu sync.Mutex - token string - expires time.Time -} - -func (t *transport) getToken() (string, error) { - t.mu.Lock() - defer t.mu.Unlock() - if t.token != "" && t.expires.After(time.Now().Add(2*time.Second)) { - return t.token, nil - } - tokenJSON, err := MetadataValue("instance/service-accounts/" + t.acct + "/token") - if err != nil { - return "", err - } - var token struct { - AccessToken string `json:"access_token"` - ExpiresIn int `json:"expires_in"` - } - if err := json.NewDecoder(strings.NewReader(tokenJSON)).Decode(&token); err != nil { - return "", err - } - if token.AccessToken == "" { - return "", errors.New("no access token returned") - } - t.token = token.AccessToken - t.expires = time.Now().Add(time.Duration(token.ExpiresIn) * time.Second) - return t.token, nil -} - -// cloneRequest returns a clone of the provided *http.Request. -// The clone is a shallow copy of the struct and its Header map. -func cloneRequest(r *http.Request) *http.Request { - // shallow copy of the struct - r2 := new(http.Request) - *r2 = *r - // deep copy of the Header - r2.Header = make(http.Header) - for k, s := range r.Header { - r2.Header[k] = s - } - return r2 -} - -func (t *transport) RoundTrip(req *http.Request) (*http.Response, error) { - token, err := t.getToken() - if err != nil { - return nil, err - } - - newReq := cloneRequest(req) - newReq.Header.Set("Authorization", "Bearer "+token) - - // Needed for some APIs, like Google Cloud Storage? - // See https://developers.google.com/storage/docs/projects - // Which despite saying XML, also seems to fix JSON API? - projID, _ := ProjectID() - newReq.Header["x-goog-project-id"] = []string{projID} - - return t.base.RoundTrip(newReq) -}