Merge pull request #1552 from lawrencejones/use-auto-auth

Automatically load Google auth

Gopkg.lock

@@ -4,7 +4,11 @@
 [[projects]]
   branch = "master"
   name = "bazil.org/fuse"
-  packages = [".","fs","fuseutil"]
+  packages = [
+    ".",
+    "fs",
+    "fuseutil"
+  ]
   revision = "371fbbdaa8987b715bdd21d6adc4c9b20155f748"
 
 [[projects]]
@@ -21,7 +25,12 @@
 
 [[projects]]
   name = "github.com/Azure/go-autorest"
-  packages = ["autorest","autorest/adal","autorest/azure","autorest/date"]
+  packages = [
+    "autorest",
+    "autorest/adal",
+    "autorest/azure",
+    "autorest/date"
+  ]
   revision = "c2a68353555b68de3ee8455a4fd3e890a0ac6d99"
   version = "v9.8.1"
 
@@ -87,7 +96,12 @@
 
 [[projects]]
   name = "github.com/kurin/blazer"
-  packages = ["b2","base","internal/b2types","internal/blog"]
+  packages = [
+    "b2",
+    "base",
+    "internal/b2types",
+    "internal/blog"
+  ]
   revision = "cd0304efa98725679cf68422cefa328d3d96f2f4"
   version = "v0.3.0"
 
@@ -98,7 +112,15 @@
 
 [[projects]]
   name = "github.com/minio/minio-go"
-  packages = [".","pkg/credentials","pkg/encrypt","pkg/policy","pkg/s3signer","pkg/s3utils","pkg/set"]
+  packages = [
+    ".",
+    "pkg/credentials",
+    "pkg/encrypt",
+    "pkg/policy",
+    "pkg/s3signer",
+    "pkg/s3utils",
+    "pkg/set"
+  ]
   revision = "14f1d472d115bac5ca4804094aa87484a72ced61"
   version = "4.0.6"
 
@@ -164,7 +186,10 @@
 
 [[projects]]
   name = "github.com/spf13/cobra"
-  packages = [".","doc"]
+  packages = [
+    ".",
+    "doc"
+  ]
   revision = "7b2c5ac9fc04fc5efafb60700713d4fa609b777b"
   version = "v0.0.1"
 
@@ -177,19 +202,40 @@
 [[projects]]
   branch = "master"
   name = "golang.org/x/crypto"
-  packages = ["curve25519","ed25519","ed25519/internal/edwards25519","internal/chacha20","pbkdf2","poly1305","scrypt","ssh","ssh/terminal"]
+  packages = [
+    "curve25519",
+    "ed25519",
+    "ed25519/internal/edwards25519",
+    "internal/chacha20",
+    "pbkdf2",
+    "poly1305",
+    "scrypt",
+    "ssh",
+    "ssh/terminal"
+  ]
   revision = "3d37316aaa6bd9929127ac9a527abf408178ea7b"
 
 [[projects]]
   branch = "master"
   name = "golang.org/x/net"
-  packages = ["context","context/ctxhttp","idna","lex/httplex"]
+  packages = [
+    "context",
+    "context/ctxhttp",
+    "idna",
+    "lex/httplex"
+  ]
   revision = "5ccada7d0a7ba9aeb5d3aca8d3501b4c2a509fec"
 
 [[projects]]
   branch = "master"
   name = "golang.org/x/oauth2"
-  packages = [".","google","internal","jws","jwt"]
+  packages = [
+    ".",
+    "google",
+    "internal",
+    "jws",
+    "jwt"
+  ]
   revision = "b28fcf2b08a19742b43084fb40ab78ac6c3d8067"
 
 [[projects]]
@@ -201,24 +247,58 @@
 [[projects]]
   branch = "master"
   name = "golang.org/x/sys"
-  packages = ["unix","windows"]
+  packages = [
+    "unix",
+    "windows"
+  ]
   revision = "af50095a40f9041b3b38960738837185c26e9419"
 
 [[projects]]
   branch = "master"
   name = "golang.org/x/text"
-  packages = ["collate","collate/build","internal/colltab","internal/gen","internal/tag","internal/triegen","internal/ucd","language","secure/bidirule","transform","unicode/bidi","unicode/cldr","unicode/norm","unicode/rangetable"]
+  packages = [
+    "collate",
+    "collate/build",
+    "internal/colltab",
+    "internal/gen",
+    "internal/tag",
+    "internal/triegen",
+    "internal/ucd",
+    "language",
+    "secure/bidirule",
+    "transform",
+    "unicode/bidi",
+    "unicode/cldr",
+    "unicode/norm",
+    "unicode/rangetable"
+  ]
   revision = "e19ae1496984b1c655b8044a65c0300a3c878dd3"
 
 [[projects]]
   branch = "master"
   name = "google.golang.org/api"
-  packages = ["gensupport","googleapi","googleapi/internal/uritemplates","storage/v1"]
+  packages = [
+    "gensupport",
+    "googleapi",
+    "googleapi/internal/uritemplates",
+    "storage/v1"
+  ]
   revision = "65b0d8655182691ad23b4fac11e6f7b897d9b634"
 
 [[projects]]
   name = "google.golang.org/appengine"
-  packages = [".","internal","internal/app_identity","internal/base","internal/datastore","internal/log","internal/modules","internal/remote_api","internal/urlfetch","urlfetch"]
+  packages = [
+    ".",
+    "internal",
+    "internal/app_identity",
+    "internal/base",
+    "internal/datastore",
+    "internal/log",
+    "internal/modules",
+    "internal/remote_api",
+    "internal/urlfetch",
+    "urlfetch"
+  ]
   revision = "150dc57a1b433e64154302bdc40b6bb8aefa313a"
   version = "v1.0.0"
 
@@ -231,6 +311,6 @@
 [solve-meta]
   analyzer-name = "dep"
   analyzer-version = 1
-  inputs-digest = "336ac5c261c174cac89f9a7102b493f08edfbd51fd61d1673d1d2ec4132d80ab"
+  inputs-digest = "a7d099b3ce195ffc37adedb05a4386be38e6158925a1c0fe579efdc20fa11f6a"
   solver-name = "gps-cdcl"
   solver-version = 1

changelog/unreleased/pull-1552

@@ -0,0 +1,12 @@
+Feature: Use Google Application Default credentials
+
+Google provide libraries to generate appropriate credentials with various
+fallback sources. This change uses the library to generate our GCS client, which
+allows us to make use of these extra methods.
+
+This should be backward compatible with previous restic behaviour while adding
+the additional capabilities to auth from Google's internal metadata endpoints.
+For users running restic in GCP this can make authentication far easier than it
+was before.
+
+https://developers.google.com/identity/protocols/application-default-credentials

cmd/restic/global.go

@@ -440,18 +440,6 @@ func parseConfig(loc location.Location, opts options.Options) (interface{}, erro
 			cfg.ProjectID = os.Getenv("GOOGLE_PROJECT_ID")
 		}
 
-		if cfg.JSONKeyPath == "" {
-			if path := os.Getenv("GOOGLE_APPLICATION_CREDENTIALS"); path != "" {
-				// Check read access
-				if _, err := ioutil.ReadFile(path); err != nil {
-					return nil, errors.Fatalf("Failed to read google credential from file %v: %v", path, err)
-				}
-				cfg.JSONKeyPath = path
-			} else {
-				return nil, errors.Fatal("No credential file path is set")
-			}
-		}
-
 		if err := opts.Apply(loc.Scheme, &cfg); err != nil {
 			return nil, err
 		}

doc/030_preparing_a_new_repo.rst

@@ -362,8 +362,14 @@ key file and the project ID as follows:
     $ export GOOGLE_PROJECT_ID=123123123123
     $ export GOOGLE_APPLICATION_CREDENTIALS=$HOME/.config/gs-secret-restic-key.json
 
-Then you can use the ``gs:`` backend type to create a new repository in the
-bucket `foo` at the root path:
+We use Google's client library to generate [default authentication
+material](https://developers.google.com/identity/protocols/application-default-credentials),
+which means if you're running in Google Container Engine or are otherwise
+located on an instance with default service accounts then these should work out
+the box.
+
+Once authenticated, you can use the ``gs:`` backend type to create a new
+repository in the bucket `foo` at the root path:
 
 .. code-block:: console
 

internal/backend/gs/config.go

@@ -8,13 +8,13 @@ import (
 	"github.com/restic/restic/internal/options"
 )
 
-// Config contains all configuration necessary to connect to a Google Cloud
-// Storage bucket.
+// Config contains all configuration necessary to connect to a Google Cloud Storage
+// bucket. We use Google's default application credentials to acquire an access token, so
+// we don't require that calling code supply any authentication material here.
 type Config struct {
-	ProjectID   string
-	JSONKeyPath string
-	Bucket      string
-	Prefix      string
+	ProjectID string
+	Bucket    string
+	Prefix    string
 
 	Connections uint `option:"connections" help:"set a limit for the number of concurrent connections (default: 20)"`
 }

internal/backend/gs/gs.go

@@ -15,9 +15,6 @@ import (
 	"github.com/restic/restic/internal/debug"
 	"github.com/restic/restic/internal/restic"
 
-	"io/ioutil"
-
-	"golang.org/x/oauth2"
 	"golang.org/x/oauth2/google"
 	"google.golang.org/api/googleapi"
 	storage "google.golang.org/api/storage/v1"
@@ -43,30 +40,12 @@ type Backend struct {
 // Ensure that *Backend implements restic.Backend.
 var _ restic.Backend = &Backend{}
 
-func getStorageService(jsonKeyPath string, rt http.RoundTripper) (*storage.Service, error) {
-
-	raw, err := ioutil.ReadFile(jsonKeyPath)
-	if err != nil {
-		return nil, errors.Wrap(err, "ReadFile")
-	}
-
-	conf, err := google.JWTConfigFromJSON(raw, storage.DevstorageReadWriteScope)
+func getStorageService() (*storage.Service, error) {
+	client, err := google.DefaultClient(context.TODO(), storage.DevstorageReadWriteScope)
 	if err != nil {
 		return nil, err
 	}
 
-	// create a new HTTP client
-	httpClient := &http.Client{
-		Transport: rt,
-	}
-
-	// create a now context with the HTTP client stored at the oauth2.HTTPClient key
-	ctx := context.WithValue(context.Background(), oauth2.HTTPClient, httpClient)
-
-	// then pass this context to Client(), which returns a new HTTP client
-	client := conf.Client(ctx)
-
-	// that we can then pass to New()
 	service, err := storage.New(client)
 	if err != nil {
 		return nil, err
@@ -80,7 +59,7 @@ const defaultListMaxItems = 1000
 func open(cfg Config, rt http.RoundTripper) (*Backend, error) {
 	debug.Log("open, config %#v", cfg)
 
-	service, err := getStorageService(cfg.JSONKeyPath, rt)
+	service, err := getStorageService()
 	if err != nil {
 		return nil, errors.Wrap(err, "getStorageService")
 	}

internal/backend/gs/gs_test.go

@@ -34,7 +34,6 @@ func newGSTestSuite(t testing.TB) *test.Suite {
 
 			cfg := gscfg.(gs.Config)
 			cfg.ProjectID = os.Getenv("RESTIC_TEST_GS_PROJECT_ID")
-			cfg.JSONKeyPath = os.Getenv("RESTIC_TEST_GS_APPLICATION_CREDENTIALS")
 			cfg.Prefix = fmt.Sprintf("test-%d", time.Now().UnixNano())
 			return cfg, nil
 		},
@@ -88,8 +87,8 @@ func TestBackendGS(t *testing.T) {
 	}()
 
 	vars := []string{
+		"GOOGLE_APPLICATION_CREDENTIALS",
 		"RESTIC_TEST_GS_PROJECT_ID",
-		"RESTIC_TEST_GS_APPLICATION_CREDENTIALS",
 		"RESTIC_TEST_GS_REPOSITORY",
 	}
 
@@ -106,8 +105,8 @@ func TestBackendGS(t *testing.T) {
 
 func BenchmarkBackendGS(t *testing.B) {
 	vars := []string{
+		"GOOGLE_APPLICATION_CREDENTIALS",
 		"RESTIC_TEST_GS_PROJECT_ID",
-		"RESTIC_TEST_GS_APPLICATION_CREDENTIALS",
 		"RESTIC_TEST_GS_REPOSITORY",
 	}
 

run_integration_tests.go

@@ -183,7 +183,7 @@ func (env *TravisEnvironment) RunTests() error {
 
 	env.env["GOPATH"] = os.Getenv("GOPATH")
 	if env.gcsCredentialsFile != "" {
-		env.env["RESTIC_TEST_GS_APPLICATION_CREDENTIALS"] = env.gcsCredentialsFile
+		env.env["GOOGLE_APPLICATION_CREDENTIALS"] = env.gcsCredentialsFile
 	}
 
 	// ensure that the following tests cannot be silently skipped on Travis