From 0dfdc11ed909f3ec8f8c0f0fdcbc342ed4602608 Mon Sep 17 00:00:00 2001 From: Lawrence Jones Date: Fri, 12 Jan 2018 17:25:02 +0000 Subject: [PATCH 1/3] Automatically load Google auth This change removes the hardcoded Google auth mechanism for the GCS backend, instead using Google's provided client library to discover and generate credential material. Google recommend that client libraries use their common auth mechanism in order to authorise requests against Google services. Doing so means you automatically support various types of authentication, from the standard GOOGLE_APPLICATION_CREDENTIALS environment variable to making use of Google's metadata API if running within Google Container Engine. --- cmd/restic/global.go | 12 ------------ internal/backend/gs/config.go | 12 ++++++------ internal/backend/gs/gs.go | 27 +++------------------------ internal/backend/gs/gs_test.go | 5 ++--- run_integration_tests.go | 2 +- 5 files changed, 12 insertions(+), 46 deletions(-) diff --git a/cmd/restic/global.go b/cmd/restic/global.go index 6055132bd..5d8e5110f 100644 --- a/cmd/restic/global.go +++ b/cmd/restic/global.go @@ -440,18 +440,6 @@ func parseConfig(loc location.Location, opts options.Options) (interface{}, erro cfg.ProjectID = os.Getenv("GOOGLE_PROJECT_ID") } - if cfg.JSONKeyPath == "" { - if path := os.Getenv("GOOGLE_APPLICATION_CREDENTIALS"); path != "" { - // Check read access - if _, err := ioutil.ReadFile(path); err != nil { - return nil, errors.Fatalf("Failed to read google credential from file %v: %v", path, err) - } - cfg.JSONKeyPath = path - } else { - return nil, errors.Fatal("No credential file path is set") - } - } - if err := opts.Apply(loc.Scheme, &cfg); err != nil { return nil, err } diff --git a/internal/backend/gs/config.go b/internal/backend/gs/config.go index eebd4fe23..e18066ef9 100644 --- a/internal/backend/gs/config.go +++ b/internal/backend/gs/config.go @@ -8,13 +8,13 @@ import ( "github.com/restic/restic/internal/options" ) -// Config contains all configuration necessary to connect to a Google Cloud -// Storage bucket. +// Config contains all configuration necessary to connect to a Google Cloud Storage +// bucket. We use Google's default application credentials to acquire an access token, so +// we don't require that calling code supply any authentication material here. type Config struct { - ProjectID string - JSONKeyPath string - Bucket string - Prefix string + ProjectID string + Bucket string + Prefix string Connections uint `option:"connections" help:"set a limit for the number of concurrent connections (default: 20)"` } diff --git a/internal/backend/gs/gs.go b/internal/backend/gs/gs.go index 7cdced796..c68efe01d 100644 --- a/internal/backend/gs/gs.go +++ b/internal/backend/gs/gs.go @@ -15,9 +15,6 @@ import ( "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/restic" - "io/ioutil" - - "golang.org/x/oauth2" "golang.org/x/oauth2/google" "google.golang.org/api/googleapi" storage "google.golang.org/api/storage/v1" @@ -43,30 +40,12 @@ type Backend struct { // Ensure that *Backend implements restic.Backend. var _ restic.Backend = &Backend{} -func getStorageService(jsonKeyPath string, rt http.RoundTripper) (*storage.Service, error) { - - raw, err := ioutil.ReadFile(jsonKeyPath) - if err != nil { - return nil, errors.Wrap(err, "ReadFile") - } - - conf, err := google.JWTConfigFromJSON(raw, storage.DevstorageReadWriteScope) +func getStorageService() (*storage.Service, error) { + client, err := google.DefaultClient(context.TODO(), storage.DevstorageReadWriteScope) if err != nil { return nil, err } - // create a new HTTP client - httpClient := &http.Client{ - Transport: rt, - } - - // create a now context with the HTTP client stored at the oauth2.HTTPClient key - ctx := context.WithValue(context.Background(), oauth2.HTTPClient, httpClient) - - // then pass this context to Client(), which returns a new HTTP client - client := conf.Client(ctx) - - // that we can then pass to New() service, err := storage.New(client) if err != nil { return nil, err @@ -80,7 +59,7 @@ const defaultListMaxItems = 1000 func open(cfg Config, rt http.RoundTripper) (*Backend, error) { debug.Log("open, config %#v", cfg) - service, err := getStorageService(cfg.JSONKeyPath, rt) + service, err := getStorageService() if err != nil { return nil, errors.Wrap(err, "getStorageService") } diff --git a/internal/backend/gs/gs_test.go b/internal/backend/gs/gs_test.go index a7bf97848..27ff809ff 100644 --- a/internal/backend/gs/gs_test.go +++ b/internal/backend/gs/gs_test.go @@ -34,7 +34,6 @@ func newGSTestSuite(t testing.TB) *test.Suite { cfg := gscfg.(gs.Config) cfg.ProjectID = os.Getenv("RESTIC_TEST_GS_PROJECT_ID") - cfg.JSONKeyPath = os.Getenv("RESTIC_TEST_GS_APPLICATION_CREDENTIALS") cfg.Prefix = fmt.Sprintf("test-%d", time.Now().UnixNano()) return cfg, nil }, @@ -88,8 +87,8 @@ func TestBackendGS(t *testing.T) { }() vars := []string{ + "GOOGLE_APPLICATION_CREDENTIALS", "RESTIC_TEST_GS_PROJECT_ID", - "RESTIC_TEST_GS_APPLICATION_CREDENTIALS", "RESTIC_TEST_GS_REPOSITORY", } @@ -106,8 +105,8 @@ func TestBackendGS(t *testing.T) { func BenchmarkBackendGS(t *testing.B) { vars := []string{ + "GOOGLE_APPLICATION_CREDENTIALS", "RESTIC_TEST_GS_PROJECT_ID", - "RESTIC_TEST_GS_APPLICATION_CREDENTIALS", "RESTIC_TEST_GS_REPOSITORY", } diff --git a/run_integration_tests.go b/run_integration_tests.go index 7e182c9e8..91bd1fb95 100644 --- a/run_integration_tests.go +++ b/run_integration_tests.go @@ -183,7 +183,7 @@ func (env *TravisEnvironment) RunTests() error { env.env["GOPATH"] = os.Getenv("GOPATH") if env.gcsCredentialsFile != "" { - env.env["RESTIC_TEST_GS_APPLICATION_CREDENTIALS"] = env.gcsCredentialsFile + env.env["GOOGLE_APPLICATION_CREDENTIALS"] = env.gcsCredentialsFile } // ensure that the following tests cannot be silently skipped on Travis From 492baf991ffb588a6ceb3738ac6d0282fa9d8d95 Mon Sep 17 00:00:00 2001 From: Lawrence Jones Date: Fri, 12 Jan 2018 17:36:57 +0000 Subject: [PATCH 2/3] Update docs and add changelog entry: Google auth Add documentation around using default Google application credentials, along with a changelog extra that describes the feature and the potential impact on existing restic uses (read: none). --- changelog/unreleased/pull-1552 | 12 ++++++++++++ doc/030_preparing_a_new_repo.rst | 10 ++++++++-- 2 files changed, 20 insertions(+), 2 deletions(-) create mode 100644 changelog/unreleased/pull-1552 diff --git a/changelog/unreleased/pull-1552 b/changelog/unreleased/pull-1552 new file mode 100644 index 000000000..f97a60f06 --- /dev/null +++ b/changelog/unreleased/pull-1552 @@ -0,0 +1,12 @@ +Feature: Use Google Application Default credentials + +Google provide libraries to generate appropriate credentials with various +fallback sources. This change uses the library to generate our GCS client, which +allows us to make use of these extra methods. + +This should be backward compatible with previous restic behaviour while adding +the additional capabilities to auth from Google's internal metadata endpoints. +For users running restic in GCP this can make authentication far easier than it +was before. + +https://developers.google.com/identity/protocols/application-default-credentials diff --git a/doc/030_preparing_a_new_repo.rst b/doc/030_preparing_a_new_repo.rst index dd8e9f7c7..1b984c4a7 100644 --- a/doc/030_preparing_a_new_repo.rst +++ b/doc/030_preparing_a_new_repo.rst @@ -362,8 +362,14 @@ key file and the project ID as follows: $ export GOOGLE_PROJECT_ID=123123123123 $ export GOOGLE_APPLICATION_CREDENTIALS=$HOME/.config/gs-secret-restic-key.json -Then you can use the ``gs:`` backend type to create a new repository in the -bucket `foo` at the root path: +We use Google's client library to generate [default authentication +material](https://developers.google.com/identity/protocols/application-default-credentials), +which means if you're running in Google Container Engine or are otherwise +located on an instance with default service accounts then these should work out +the box. + +Once authenticated, you can use the ``gs:`` backend type to create a new +repository in the bucket `foo` at the root path: .. code-block:: console From 57c623398256c1b9d4134af790c53d1cad93fe38 Mon Sep 17 00:00:00 2001 From: Lawrence Jones Date: Fri, 16 Mar 2018 10:31:30 +0000 Subject: [PATCH 3/3] dep ensure --- Gopkg.lock | 106 ++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 93 insertions(+), 13 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index 54a3f7a1f..92f61a847 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -4,7 +4,11 @@ [[projects]] branch = "master" name = "bazil.org/fuse" - packages = [".","fs","fuseutil"] + packages = [ + ".", + "fs", + "fuseutil" + ] revision = "371fbbdaa8987b715bdd21d6adc4c9b20155f748" [[projects]] @@ -21,7 +25,12 @@ [[projects]] name = "github.com/Azure/go-autorest" - packages = ["autorest","autorest/adal","autorest/azure","autorest/date"] + packages = [ + "autorest", + "autorest/adal", + "autorest/azure", + "autorest/date" + ] revision = "c2a68353555b68de3ee8455a4fd3e890a0ac6d99" version = "v9.8.1" @@ -87,7 +96,12 @@ [[projects]] name = "github.com/kurin/blazer" - packages = ["b2","base","internal/b2types","internal/blog"] + packages = [ + "b2", + "base", + "internal/b2types", + "internal/blog" + ] revision = "cd0304efa98725679cf68422cefa328d3d96f2f4" version = "v0.3.0" @@ -98,7 +112,15 @@ [[projects]] name = "github.com/minio/minio-go" - packages = [".","pkg/credentials","pkg/encrypt","pkg/policy","pkg/s3signer","pkg/s3utils","pkg/set"] + packages = [ + ".", + "pkg/credentials", + "pkg/encrypt", + "pkg/policy", + "pkg/s3signer", + "pkg/s3utils", + "pkg/set" + ] revision = "14f1d472d115bac5ca4804094aa87484a72ced61" version = "4.0.6" @@ -164,7 +186,10 @@ [[projects]] name = "github.com/spf13/cobra" - packages = [".","doc"] + packages = [ + ".", + "doc" + ] revision = "7b2c5ac9fc04fc5efafb60700713d4fa609b777b" version = "v0.0.1" @@ -177,19 +202,40 @@ [[projects]] branch = "master" name = "golang.org/x/crypto" - packages = ["curve25519","ed25519","ed25519/internal/edwards25519","internal/chacha20","pbkdf2","poly1305","scrypt","ssh","ssh/terminal"] + packages = [ + "curve25519", + "ed25519", + "ed25519/internal/edwards25519", + "internal/chacha20", + "pbkdf2", + "poly1305", + "scrypt", + "ssh", + "ssh/terminal" + ] revision = "3d37316aaa6bd9929127ac9a527abf408178ea7b" [[projects]] branch = "master" name = "golang.org/x/net" - packages = ["context","context/ctxhttp","idna","lex/httplex"] + packages = [ + "context", + "context/ctxhttp", + "idna", + "lex/httplex" + ] revision = "5ccada7d0a7ba9aeb5d3aca8d3501b4c2a509fec" [[projects]] branch = "master" name = "golang.org/x/oauth2" - packages = [".","google","internal","jws","jwt"] + packages = [ + ".", + "google", + "internal", + "jws", + "jwt" + ] revision = "b28fcf2b08a19742b43084fb40ab78ac6c3d8067" [[projects]] @@ -201,24 +247,58 @@ [[projects]] branch = "master" name = "golang.org/x/sys" - packages = ["unix","windows"] + packages = [ + "unix", + "windows" + ] revision = "af50095a40f9041b3b38960738837185c26e9419" [[projects]] branch = "master" name = "golang.org/x/text" - packages = ["collate","collate/build","internal/colltab","internal/gen","internal/tag","internal/triegen","internal/ucd","language","secure/bidirule","transform","unicode/bidi","unicode/cldr","unicode/norm","unicode/rangetable"] + packages = [ + "collate", + "collate/build", + "internal/colltab", + "internal/gen", + "internal/tag", + "internal/triegen", + "internal/ucd", + "language", + "secure/bidirule", + "transform", + "unicode/bidi", + "unicode/cldr", + "unicode/norm", + "unicode/rangetable" + ] revision = "e19ae1496984b1c655b8044a65c0300a3c878dd3" [[projects]] branch = "master" name = "google.golang.org/api" - packages = ["gensupport","googleapi","googleapi/internal/uritemplates","storage/v1"] + packages = [ + "gensupport", + "googleapi", + "googleapi/internal/uritemplates", + "storage/v1" + ] revision = "65b0d8655182691ad23b4fac11e6f7b897d9b634" [[projects]] name = "google.golang.org/appengine" - packages = [".","internal","internal/app_identity","internal/base","internal/datastore","internal/log","internal/modules","internal/remote_api","internal/urlfetch","urlfetch"] + packages = [ + ".", + "internal", + "internal/app_identity", + "internal/base", + "internal/datastore", + "internal/log", + "internal/modules", + "internal/remote_api", + "internal/urlfetch", + "urlfetch" + ] revision = "150dc57a1b433e64154302bdc40b6bb8aefa313a" version = "v1.0.0" @@ -231,6 +311,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "336ac5c261c174cac89f9a7102b493f08edfbd51fd61d1673d1d2ec4132d80ab" + inputs-digest = "a7d099b3ce195ffc37adedb05a4386be38e6158925a1c0fe579efdc20fa11f6a" solver-name = "gps-cdcl" solver-version = 1