added support for using GOOGLE_CREDENTIALS for gs:// filestate authentication (#2906)

* added support for using GOOGLE_CREDENTIALS environment variable for authenticating with gs:// file state

* modified the change to fix #2791 as well

* fixed a small bug

* fixed linter error

* added code comments

* Update pkg/backend/filestate/gcpauth.go

Co-Authored-By: CyrusNajmabadi <cyrus.najmabadi@gmail.com>

* Parse provided backend url to check if scheme is gs://

* Update changelog

CHANGELOG.md

@@ -1,6 +1,17 @@
 CHANGELOG
 =========
 
+## HEAD (Unreleased)
+
+- Add support for GOOGLE_CREDENTIALS when using Google Cloud Storage backend. i.e.:
+
+  ```sh
+   export GOOGLE_CREDENTIALS="$(cat ~/service-account-credentials.json)"
+   pulumi login gs://my-bucket
+  ```
+
+  [#2906](https://github.com/pulumi/pulumi/pull/2906) (Fixes [#2790](https://github.com/pulumi/pulumi/issues/2790), [#2791](https://github.com/pulumi/pulumi/issues/2791))
+
 ## 1.7.1 (2019-12-13)
 
 - Fix [SxS issue](https://github.com/pulumi/pulumi/issues/3652) introduced in 1.7.0 when assigning

go.mod

@@ -57,6 +57,7 @@ require (
 	gocloud.dev/secrets/hashivault v0.18.0
 	golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5
 	golang.org/x/net v0.0.0-20190619014844-b5b0513f8c1b
+	golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45
 	golang.org/x/sync v0.0.0-20190423024810-112230192c58
 	golang.org/x/sys v0.0.0-20190620070143-6f217b454f45
 	google.golang.org/api v0.6.0

pkg/backend/filestate/backend.go

@@ -30,7 +30,7 @@ import (
 	"gocloud.dev/blob"
 	_ "gocloud.dev/blob/azureblob" // driver for azblob://
 	_ "gocloud.dev/blob/fileblob"  // driver for file://
-	_ "gocloud.dev/blob/gcsblob"   // driver for gs://
+	"gocloud.dev/blob/gcsblob"     // driver for gs://
 	_ "gocloud.dev/blob/s3blob"    // driver for s3://
 	"gocloud.dev/gcerrors"
 
@@ -47,6 +47,7 @@ import (
 	"github.com/pulumi/pulumi/pkg/resource/edit"
 	"github.com/pulumi/pulumi/pkg/resource/stack"
 	"github.com/pulumi/pulumi/pkg/tokens"
+	"github.com/pulumi/pulumi/pkg/util/cmdutil"
 	"github.com/pulumi/pulumi/pkg/util/contract"
 	"github.com/pulumi/pulumi/pkg/util/logging"
 	"github.com/pulumi/pulumi/pkg/util/result"
@@ -106,16 +107,28 @@ func New(d diag.Sink, originalURL string) (Backend, error) {
 		return nil, err
 	}
 
-	bucket, err := blob.OpenBucket(context.TODO(), u)
+	p, err := url.Parse(u)
+	if err != nil {
+		return nil, err
+	}
+
+	blobmux := blob.DefaultURLMux()
+
+	// for gcp we want to support additional credentials
+	// schemes on top of go-cloud's default credentials mux.
+	if p.Scheme == gcsblob.Scheme {
+		blobmux, err = GoogleCredentialsMux(context.TODO())
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	bucket, err := blobmux.OpenBucket(context.TODO(), u)
 	if err != nil {
 		return nil, errors.Wrapf(err, "unable to open bucket %s", u)
 	}
 
 	if !strings.HasPrefix(u, FilePathPrefix) {
-		p, err := url.Parse(u)
-		if err != nil {
-			return nil, err
-		}
 		bucketSubDir := strings.TrimLeft(p.Path, "/")
 		if bucketSubDir != "" {
 			if !strings.HasSuffix(bucketSubDir, "/") {
@@ -541,7 +554,12 @@ func (b *localBackend) apply(
 		} else {
 			link, err = b.bucket.SignedURL(context.TODO(), b.stackPath(stackName), nil)
 			if err != nil {
-				return changes, result.FromError(errors.Wrap(err, "Could not get signed url for stack location"))
+				// we log a warning here rather then returning an error to avoid exiting
+				// pulumi with an error code.
+				// printing a statefile perma link happens after all the providers have finished
+				// deploying the infrastructure, failing the pulumi update because there was a
+				// problem printing a statefile perma link can be missleading in automated CI environments.
+				cmdutil.Diag().Warningf(diag.Message("", "Could not get signed url for stack location: %v"), err)
 			}
 		}
 

pkg/backend/filestate/gcpauth.go

@@ -0,0 +1,85 @@
+package filestate
+
+import (
+	"context"
+	"encoding/json"
+	"os"
+
+	"github.com/pulumi/pulumi/pkg/diag"
+	"github.com/pulumi/pulumi/pkg/util/cmdutil"
+
+	"golang.org/x/oauth2/google"
+
+	"gocloud.dev/blob/gcsblob"
+
+	"cloud.google.com/go/storage"
+
+	"github.com/pkg/errors"
+	"gocloud.dev/blob"
+	"gocloud.dev/gcp"
+)
+
+type GoogleCredentials struct {
+	PrivateKeyID string `json:"private_key_id"`
+	PrivateKey   string `json:"private_key"`
+	ClientEmail  string `json:"client_email"`
+	ClientID     string `json:"client_id"`
+}
+
+func googleCredentials(ctx context.Context) (*google.Credentials, error) {
+	// GOOGLE_CREDENTIALS aren't part of the gcloud standard authorization variables
+	// but the GCP terraform provider uses this variable to allow users to authenticate
+	// with the contents of a credentials.json file instead of just a file path.
+	// https://www.terraform.io/docs/backends/types/gcs.html
+	if creds := os.Getenv("GOOGLE_CREDENTIALS"); creds != "" {
+		// We try $GOOGLE_CREDENTIALS before gcp.DefaultCredentials
+		// so that users can override the default creds
+		credentials, err := google.CredentialsFromJSON(ctx, []byte(creds), storage.ScopeReadWrite)
+		if err != nil {
+			return nil, errors.Wrap(err, "unable to parse credentials from $GOOGLE_CREDENTIALS")
+		}
+		return credentials, nil
+	}
+
+	// DefaultCredentials will attempt to load creds in the following order:
+	// 1. a file located at $GOOGLE_APPLICATION_CREDENTIALS
+	// 2. application_default_credentials.json file in ~/.config/gcloud or $APPDATA\gcloud
+	credentials, err := gcp.DefaultCredentials(ctx)
+	if err != nil {
+		return nil, errors.Wrap(err, "unable to find gcp credentials")
+	}
+	return credentials, nil
+}
+
+func GoogleCredentialsMux(ctx context.Context) (*blob.URLMux, error) {
+	credentials, err := googleCredentials(ctx)
+	if err != nil {
+		return nil, errors.New("missing google credentials")
+	}
+
+	client, err := gcp.NewHTTPClient(gcp.DefaultTransport(), credentials.TokenSource)
+	if err != nil {
+		return nil, err
+	}
+
+	options := gcsblob.Options{}
+	account := GoogleCredentials{}
+	err = json.Unmarshal(credentials.JSON, &account)
+	if err == nil && account.ClientEmail != "" && account.PrivateKey != "" {
+		options.GoogleAccessID = account.ClientEmail
+		options.PrivateKey = []byte(account.PrivateKey)
+	} else {
+		cmdutil.Diag().Warningf(diag.Message("",
+			"Pulumi will not be able to print a statefile permalink using these credentials. "+
+				"Neither a GoogleAccessID or PrivateKey are available. "+
+				"Try using a GCP Service Account."))
+	}
+
+	blobmux := &blob.URLMux{}
+	blobmux.RegisterBucket(gcsblob.Scheme, &gcsblob.URLOpener{
+		Client:  client,
+		Options: options,
+	})
+
+	return blobmux, nil
+}