Add new 'env' package to detect the type of environment.

Also, delete my old gce package from third_party and only use the
google metadata package (which my gce package became, and which was
also already vendored into third_party)

Fixes #596

Change-Id: I64fd6f1e9dc6f433466f91f81efd2ecbf039334f

pkg/env/env.go

@@ -0,0 +1,46 @@
+/*
+Copyright 2015 The Camlistore Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package env detects what sort of environment Camlistore is running in.
+package env
+
+import (
+	"sync"
+
+	"camlistore.org/third_party/google.golang.org/cloud/compute/metadata"
+)
+
+// OsGCE reports whether this process is running in a Google Compute
+// Engine (GCE) environment.  This only returns true if the
+// "camlistore-config-dir" instance metadata value is defined.
+// Instances running in custom configs on GCE will be unaffected.
+func OnGCE() bool {
+	gceOnce.Do(detectGCE)
+	return isGCE
+}
+
+var (
+	gceOnce sync.Once
+	isGCE   bool
+)
+
+func detectGCE() {
+	if !metadata.OnGCE() {
+		return
+	}
+	v, _ := metadata.InstanceAttributeValue("camlistore-config-dir")
+	isGCE = v != ""
+}

pkg/googlestorage/googlestorage.go

@@ -36,8 +36,11 @@ import (
 	"camlistore.org/pkg/blob"
 	"camlistore.org/pkg/httputil"
 	"camlistore.org/third_party/code.google.com/p/goauth2/oauth"
-	"camlistore.org/third_party/github.com/bradfitz/gce"
+	"camlistore.org/third_party/golang.org/x/net/context"
+	"camlistore.org/third_party/golang.org/x/oauth2"
+	"camlistore.org/third_party/golang.org/x/oauth2/google"
 	api "camlistore.org/third_party/google.golang.org/api/storage/v1"
+	"camlistore.org/third_party/google.golang.org/cloud/compute/metadata"
 )
 
 const (
@@ -78,19 +81,28 @@ type SizedObject struct {
 }
 
 // NewServiceClient returns a Client for use when running on Google
-// Compute Engine.  This client can access buckets owned by the samre
+// Compute Engine.  This client can access buckets owned by the same
 // project ID as the VM.
 func NewServiceClient() (*Client, error) {
-	if !gce.OnGCE() {
+	if !metadata.OnGCE() {
 		return nil, errors.New("not running on Google Compute Engine")
 	}
-	scopes, _ := gce.Scopes("default")
+	scopes, _ := metadata.Scopes("default")
-	if !scopes.Contains("https://www.googleapis.com/auth/devstorage.full_control") &&
+	haveScope := func(scope string) bool {
-		!scopes.Contains("https://www.googleapis.com/auth/devstorage.read_write") {
+		for _, x := range scopes {
+			if x == scope {
+				return true
+			}
+		}
+		return false
+	}
+	if !haveScope("https://www.googleapis.com/auth/devstorage.full_control") &&
+		!haveScope("https://www.googleapis.com/auth/devstorage.read_write") {
 		return nil, errors.New("when this Google Compute Engine VM instance was created, it wasn't granted access to Cloud Storage")
 	}
-	service, _ := api.New(gce.Client)
+	client := oauth2.NewClient(context.Background(), google.ComputeTokenSource(""))
-	return &Client{client: gce.Client, service: service}, nil
+	service, _ := api.New(client)
+	return &Client{client: client, service: service}, nil
 }
 
 func NewClient(transport *oauth.Transport) *Client {

pkg/osutil/gce/gce.go

@@ -23,18 +23,19 @@ import (
 	"path"
 	"strings"
 
+	"camlistore.org/pkg/env"
 	"camlistore.org/pkg/jsonconfig"
 	"camlistore.org/pkg/osutil"
 	_ "camlistore.org/pkg/wkfs/gcs"
-	"camlistore.org/third_party/github.com/bradfitz/gce"
+	"camlistore.org/third_party/google.golang.org/cloud/compute/metadata"
 )
 
 func init() {
-	if !gce.OnGCE() {
+	if !env.OnGCE() {
 		return
 	}
 	osutil.RegisterConfigDirFunc(func() string {
-		v, _ := gce.InstanceAttributeValue("camlistore-config-dir")
+		v, _ := metadata.InstanceAttributeValue("camlistore-config-dir")
 		if v == "" {
 			return v
 		}
@@ -48,7 +49,7 @@ func init() {
 		if !ok {
 			return nil, errors.New("expected argument after _gce_instance_meta to be a string")
 		}
-		val, err := gce.InstanceAttributeValue(attr)
+		val, err := metadata.InstanceAttributeValue(attr)
 		if err != nil {
 			return nil, fmt.Errorf("error reading GCE instance attribute %q: %v", attr, err)
 		}

pkg/serverinit/env.go

@@ -21,26 +21,27 @@ import (
 	"os"
 	"strings"
 
+	"camlistore.org/pkg/env"
 	"camlistore.org/pkg/osutil"
 	"camlistore.org/pkg/types/serverconfig"
-	"camlistore.org/third_party/github.com/bradfitz/gce"
+	"camlistore.org/third_party/google.golang.org/cloud/compute/metadata"
 )
 
 // DefaultEnvConfig returns the default configuration when running on a known
 // environment. Currently this just includes Google Compute Engine.
 // If the environment isn't known (nil, nil) is returned.
 func DefaultEnvConfig() (*Config, error) {
-	if !gce.OnGCE() {
+	if !env.OnGCE() {
 		return nil, nil
 	}
 	auth := "none"
-	user, _ := gce.InstanceAttributeValue("camlistore-username")
+	user, _ := metadata.InstanceAttributeValue("camlistore-username")
-	pass, _ := gce.InstanceAttributeValue("camlistore-password")
+	pass, _ := metadata.InstanceAttributeValue("camlistore-password")
-	confBucket, err := gce.InstanceAttributeValue("camlistore-config-dir")
+	confBucket, err := metadata.InstanceAttributeValue("camlistore-config-dir")
 	if confBucket == "" || err != nil {
 		return nil, fmt.Errorf("VM instance metadata key 'camlistore-config-dir' not set: %v", err)
 	}
-	blobBucket, err := gce.InstanceAttributeValue("camlistore-blob-dir")
+	blobBucket, err := metadata.InstanceAttributeValue("camlistore-blob-dir")
 	if blobBucket == "" || err != nil {
 		return nil, fmt.Errorf("VM instance metadata key 'camlistore-blob-dir' not set: %v", err)
 	}
@@ -56,8 +57,8 @@ func DefaultEnvConfig() (*Config, error) {
 		return nil, err
 	}
 
-	ipOrHost, _ := gce.ExternalIP()
+	ipOrHost, _ := metadata.ExternalIP()
-	host, _ := gce.InstanceAttributeValue("camlistore-hostname")
+	host, _ := metadata.InstanceAttributeValue("camlistore-hostname")
 	if host != "" {
 		ipOrHost = host
 	}

pkg/sorted/mysql/cloudsql.go

@@ -23,8 +23,11 @@ import (
 	"log"
 	"strings"
 
+	"camlistore.org/third_party/golang.org/x/net/context"
+	"camlistore.org/third_party/golang.org/x/oauth2"
+	"camlistore.org/third_party/golang.org/x/oauth2/google"
 	sqladmin "camlistore.org/third_party/google.golang.org/api/sqladmin/v1beta3"
-	"camlistore.org/third_party/github.com/bradfitz/gce"
+	"camlistore.org/third_party/google.golang.org/cloud/compute/metadata"
 )
 
 const cloudSQLSuffix = ".cloudsql.google.internal"
@@ -34,15 +37,15 @@ func maybeRemapCloudSQL(host string) (out string, err error) {
 		return host, nil
 	}
 	inst := strings.TrimSuffix(host, cloudSQLSuffix)
-	if !gce.OnGCE() {
+	if !metadata.OnGCE() {
 		return "", errors.New("CloudSQL support only available when running on Google Compute Engine.")
 	}
-	proj, err := gce.ProjectID()
+	proj, err := metadata.ProjectID()
 	if err != nil {
 		return "", fmt.Errorf("Failed to lookup GCE project ID: %v", err)
 	}
 
-	admin, _ := sqladmin.New(gce.Client)
+	admin, _ := sqladmin.New(oauth2.NewClient(context.Background(), google.ComputeTokenSource("")))
 	listRes, err := admin.Instances.List(proj).Do()
 	if err != nil {
 		return "", fmt.Errorf("error enumerating Cloud SQL instances: %v", err)

pkg/wkfs/gcs/gcs.go

@@ -33,7 +33,7 @@ import (
 
 	"camlistore.org/pkg/googlestorage"
 	"camlistore.org/pkg/wkfs"
-	"camlistore.org/third_party/github.com/bradfitz/gce"
+	"camlistore.org/third_party/google.golang.org/cloud/compute/metadata"
 )
 
 // Max size for all files read or written. This filesystem is only
@@ -42,7 +42,7 @@ import (
 const maxSize = 1 << 20
 
 func init() {
-	if !gce.OnGCE() {
+	if !metadata.OnGCE() {
 		return
 	}
 	client, err := googlestorage.NewServiceClient()

server/camlistored/camlistored.go

@@ -35,6 +35,7 @@ import (
 	"time"
 
 	"camlistore.org/pkg/buildinfo"
+	"camlistore.org/pkg/env"
 	"camlistore.org/pkg/httputil"
 	"camlistore.org/pkg/legal/legalprint"
 	"camlistore.org/pkg/netutil"
@@ -45,7 +46,6 @@ import (
 
 	// VM environments:
 	_ "camlistore.org/pkg/osutil/gce"
-	"camlistore.org/third_party/github.com/bradfitz/gce"
 
 	// Storage options:
 	_ "camlistore.org/pkg/blobserver/blobpacked"
@@ -400,7 +400,7 @@ func Main(up chan<- struct{}, down <-chan struct{}) {
 	}
 	log.Printf("Available on %s", urlToOpen)
 
-	if gce.OnGCE() && strings.HasPrefix(baseURL, "https://") {
+	if env.OnGCE() && strings.HasPrefix(baseURL, "https://") {
 		go redirectFromHTTP(baseURL)
 	}
 

third_party/github.com/bradfitz/gce/LICENSE

@@ -1,7 +0,0 @@
-Copyright 2014 Google & the Go AUTHORS
-
-Go AUTHORS are:
-See https://code.google.com/p/go/source/browse/AUTHORS
-
-Licensed under the terms of Go itself:
-https://code.google.com/p/go/source/browse/LICENSE

third_party/github.com/bradfitz/gce/README

@@ -1,5 +0,0 @@
-Package gce provides access to Google Compute Engine (GCE) metadata and
-API service accounts.
-
-See the code for docs, or as HTML at http://godoc.org/github.com/bradfitz/gce .
-

third_party/github.com/bradfitz/gce/gce.go

@@ -1,312 +0,0 @@
-/*
-Copyright 2014 Google & the Go AUTHORS
-
-Go AUTHORS are:
-See https://code.google.com/p/go/source/browse/AUTHORS
-
-Licensed under the terms of Go itself:
-https://code.google.com/p/go/source/browse/LICENSE
-*/
-
-// Package gce provides access to Google Compute Engine (GCE) metadata and
-// API service accounts.
-//
-// Most of this package is a wrapper around the GCE metadata service,
-// as documented at https://developers.google.com/compute/docs/metadata.
-package gce
-
-import (
-	"encoding/json"
-	"errors"
-	"fmt"
-	"io/ioutil"
-	"net"
-	"net/http"
-	"strings"
-	"sync"
-	"time"
-)
-
-// Strings is a list of strings.
-type Strings []string
-
-// Contains reports whether v is contained in s.
-func (s Strings) Contains(v string) bool {
-	for _, sv := range s {
-		if v == sv {
-			return true
-		}
-	}
-	return false
-}
-
-var metaClient = &http.Client{
-	Transport: &http.Transport{
-		Dial: (&net.Dialer{
-			Timeout:   750 * time.Millisecond,
-			KeepAlive: 30 * time.Second,
-		}).Dial,
-		ResponseHeaderTimeout: 750 * time.Millisecond,
-	},
-}
-
-// MetadataValue returns a value from the metadata service.
-// The suffix is appended to "http://metadata/computeMetadata/v1/".
-func MetadataValue(suffix string) (string, error) {
-	// Using 169.254.169.254 instead of "metadata" here because Go
-	// binaries built with the "netgo" tag and without cgo won't
-	// know the search suffix for "metadata" is
-	// ".google.internal", and this IP address is documented as
-	// being stable anyway.
-	url := "http://169.254.169.254/computeMetadata/v1/" + suffix
-	req, _ := http.NewRequest("GET", url, nil)
-	req.Header.Set("Metadata-Flavor", "Google")
-	res, err := metaClient.Do(req)
-	if err != nil {
-		return "", err
-	}
-	defer res.Body.Close()
-	if res.StatusCode != 200 {
-		return "", fmt.Errorf("status code %d trying to fetch %s", res.StatusCode, url)
-	}
-	all, err := ioutil.ReadAll(res.Body)
-	if err != nil {
-		return "", err
-	}
-	return string(all), nil
-}
-
-func metaValueTrim(suffix string) (s string, err error) {
-	s, err = MetadataValue(suffix)
-	s = strings.TrimSpace(s)
-	return
-}
-
-type cachedValue struct {
-	k    string
-	trim bool
-	mu   sync.Mutex
-	v    string
-}
-
-var (
-	proj   = &cachedValue{k: "project/project-id", trim: true}
-	projID = &cachedValue{k: "project/numeric-project-id", trim: true}
-	instID = &cachedValue{k: "instance/id", trim: true}
-)
-
-func (c *cachedValue) get() (v string, err error) {
-	defer c.mu.Unlock()
-	c.mu.Lock()
-	if c.v != "" {
-		return c.v, nil
-	}
-	if c.trim {
-		v, err = metaValueTrim(c.k)
-	} else {
-		v, err = MetadataValue(c.k)
-	}
-	if err == nil {
-		c.v = v
-	}
-	return
-}
-
-var onGCE struct {
-	sync.Mutex
-	set bool
-	v   bool
-}
-
-// OnGCE reports whether this process is running on Google Compute Engine.
-func OnGCE() bool {
-	defer onGCE.Unlock()
-	onGCE.Lock()
-	if onGCE.set {
-		return onGCE.v
-	}
-	onGCE.set = true
-
-	res, err := metaClient.Get("http://metadata.google.internal")
-	if err != nil {
-		return false
-	}
-	onGCE.v = res.Header.Get("Metadata-Flavor") == "Google"
-	return onGCE.v
-}
-
-// ProjectID returns the current instance's project ID string.
-func ProjectID() (string, error) { return proj.get() }
-
-// NumericProjectID returns the current instance's numeric project ID.
-func NumericProjectID() (string, error) { return projID.get() }
-
-// InternalIP returns the instance's primary internal IP address.
-func InternalIP() (string, error) {
-	return metaValueTrim("instance/network-interfaces/0/ip")
-}
-
-// ExternalIP returns the instance's primary external (public) IP address.
-func ExternalIP() (string, error) {
-	return metaValueTrim("instance/network-interfaces/0/access-configs/0/external-ip")
-}
-
-// Hostname returns the instance's hostname. This will probably be of
-// the form "INSTANCENAME.c.PROJECT.internal" but that isn't
-// guaranteed.
-//
-// TODO: what is this defined to be? Docs say "The host name of the
-// instance."
-func Hostname() (string, error) {
-	return metaValueTrim("network-interfaces/0/ip")
-}
-
-// InstanceTags returns the list of user-defined instance tags,
-// assigned when initially creating a GCE instance.
-func InstanceTags() (Strings, error) {
-	var s Strings
-	j, err := MetadataValue("instance/tags")
-	if err != nil {
-		return nil, err
-	}
-	if err := json.NewDecoder(strings.NewReader(j)).Decode(&s); err != nil {
-		return nil, err
-	}
-	return s, nil
-}
-
-// InstanceID returns the current VM's numeric instance ID.
-func InstanceID() (string, error) {
-	return instID.get()
-}
-
-// InstanceAttributes returns the list of user-defined attributes,
-// assigned when initially creating a GCE VM instance. The value of an
-// attribute can be obtained with InstanceAttributeValue.
-func InstanceAttributes() (Strings, error) { return lines("instance/attributes/") }
-
-// ProjectAttributes returns the list of user-defined attributes
-// applying to the project as a whole, not just this VM.  The value of
-// an attribute can be obtained with ProjectAttributeValue.
-func ProjectAttributes() (Strings, error) { return lines("project/attributes/") }
-
-func lines(suffix string) (Strings, error) {
-	j, err := MetadataValue(suffix)
-	if err != nil {
-		return nil, err
-	}
-	s := strings.Split(strings.TrimSpace(j), "\n")
-	for i := range s {
-		s[i] = strings.TrimSpace(s[i])
-	}
-	return Strings(s), nil
-}
-
-// InstanceAttributeValue returns the value of the provided VM
-// instance attribute.
-func InstanceAttributeValue(attr string) (string, error) {
-	return MetadataValue("instance/attributes/" + attr)
-}
-
-// ProjectAttributeValue returns the value of the provided
-// project attribute.
-func ProjectAttributeValue(attr string) (string, error) {
-	return MetadataValue("project/attributes/" + attr)
-}
-
-// Scopes returns the service account scopes for the given account.
-// The account may be empty or the string "default" to use the instance's
-// main account.
-func Scopes(serviceAccount string) (Strings, error) {
-	if serviceAccount == "" {
-		serviceAccount = "default"
-	}
-	return lines("instance/service-accounts/" + serviceAccount + "/scopes")
-}
-
-// Transport is an HTTP transport that adds authentication headers to
-// the request using the default GCE service account and forwards the
-// requests to the http package's default transport.
-var Transport = NewTransport("default", http.DefaultTransport)
-
-// Client is an http Client that uses the default GCE transport.
-var Client = &http.Client{Transport: Transport}
-
-// NewTransport returns a transport that uses the provided GCE
-// serviceAccount (optional) to add authentication headers and then
-// uses the provided underlying "base" transport.
-//
-// For more information on Service Accounts, see
-// https://developers.google.com/compute/docs/authentication.
-func NewTransport(serviceAccount string, base http.RoundTripper) http.RoundTripper {
-	if serviceAccount == "" {
-		serviceAccount = "default"
-	}
-	return &transport{base: base, acct: serviceAccount}
-}
-
-type transport struct {
-	base http.RoundTripper
-	acct string
-
-	mu      sync.Mutex
-	token   string
-	expires time.Time
-}
-
-func (t *transport) getToken() (string, error) {
-	t.mu.Lock()
-	defer t.mu.Unlock()
-	if t.token != "" && t.expires.After(time.Now().Add(2*time.Second)) {
-		return t.token, nil
-	}
-	tokenJSON, err := MetadataValue("instance/service-accounts/" + t.acct + "/token")
-	if err != nil {
-		return "", err
-	}
-	var token struct {
-		AccessToken string `json:"access_token"`
-		ExpiresIn   int    `json:"expires_in"`
-	}
-	if err := json.NewDecoder(strings.NewReader(tokenJSON)).Decode(&token); err != nil {
-		return "", err
-	}
-	if token.AccessToken == "" {
-		return "", errors.New("no access token returned")
-	}
-	t.token = token.AccessToken
-	t.expires = time.Now().Add(time.Duration(token.ExpiresIn) * time.Second)
-	return t.token, nil
-}
-
-// cloneRequest returns a clone of the provided *http.Request.
-// The clone is a shallow copy of the struct and its Header map.
-func cloneRequest(r *http.Request) *http.Request {
-	// shallow copy of the struct
-	r2 := new(http.Request)
-	*r2 = *r
-	// deep copy of the Header
-	r2.Header = make(http.Header)
-	for k, s := range r.Header {
-		r2.Header[k] = s
-	}
-	return r2
-}
-
-func (t *transport) RoundTrip(req *http.Request) (*http.Response, error) {
-	token, err := t.getToken()
-	if err != nil {
-		return nil, err
-	}
-
-	newReq := cloneRequest(req)
-	newReq.Header.Set("Authorization", "Bearer "+token)
-
-	// Needed for some APIs, like Google Cloud Storage?
-	// See https://developers.google.com/storage/docs/projects
-	// Which despite saying XML, also seems to fix JSON API?
-	projID, _ := ProjectID()
-	newReq.Header["x-goog-project-id"] = []string{projID}
-
-	return t.base.RoundTrip(newReq)
-}