diff --git a/Gopkg.lock b/Gopkg.lock index 54a3f7a1f..92f61a847 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -4,7 +4,11 @@ [[projects]] branch = "master" name = "bazil.org/fuse" - packages = [".","fs","fuseutil"] + packages = [ + ".", + "fs", + "fuseutil" + ] revision = "371fbbdaa8987b715bdd21d6adc4c9b20155f748" [[projects]] @@ -21,7 +25,12 @@ [[projects]] name = "github.com/Azure/go-autorest" - packages = ["autorest","autorest/adal","autorest/azure","autorest/date"] + packages = [ + "autorest", + "autorest/adal", + "autorest/azure", + "autorest/date" + ] revision = "c2a68353555b68de3ee8455a4fd3e890a0ac6d99" version = "v9.8.1" @@ -87,7 +96,12 @@ [[projects]] name = "github.com/kurin/blazer" - packages = ["b2","base","internal/b2types","internal/blog"] + packages = [ + "b2", + "base", + "internal/b2types", + "internal/blog" + ] revision = "cd0304efa98725679cf68422cefa328d3d96f2f4" version = "v0.3.0" @@ -98,7 +112,15 @@ [[projects]] name = "github.com/minio/minio-go" - packages = [".","pkg/credentials","pkg/encrypt","pkg/policy","pkg/s3signer","pkg/s3utils","pkg/set"] + packages = [ + ".", + "pkg/credentials", + "pkg/encrypt", + "pkg/policy", + "pkg/s3signer", + "pkg/s3utils", + "pkg/set" + ] revision = "14f1d472d115bac5ca4804094aa87484a72ced61" version = "4.0.6" @@ -164,7 +186,10 @@ [[projects]] name = "github.com/spf13/cobra" - packages = [".","doc"] + packages = [ + ".", + "doc" + ] revision = "7b2c5ac9fc04fc5efafb60700713d4fa609b777b" version = "v0.0.1" @@ -177,19 +202,40 @@ [[projects]] branch = "master" name = "golang.org/x/crypto" - packages = ["curve25519","ed25519","ed25519/internal/edwards25519","internal/chacha20","pbkdf2","poly1305","scrypt","ssh","ssh/terminal"] + packages = [ + "curve25519", + "ed25519", + "ed25519/internal/edwards25519", + "internal/chacha20", + "pbkdf2", + "poly1305", + "scrypt", + "ssh", + "ssh/terminal" + ] revision = "3d37316aaa6bd9929127ac9a527abf408178ea7b" [[projects]] branch = "master" name = "golang.org/x/net" - packages = ["context","context/ctxhttp","idna","lex/httplex"] + packages = [ + "context", + "context/ctxhttp", + "idna", + "lex/httplex" + ] revision = "5ccada7d0a7ba9aeb5d3aca8d3501b4c2a509fec" [[projects]] branch = "master" name = "golang.org/x/oauth2" - packages = [".","google","internal","jws","jwt"] + packages = [ + ".", + "google", + "internal", + "jws", + "jwt" + ] revision = "b28fcf2b08a19742b43084fb40ab78ac6c3d8067" [[projects]] @@ -201,24 +247,58 @@ [[projects]] branch = "master" name = "golang.org/x/sys" - packages = ["unix","windows"] + packages = [ + "unix", + "windows" + ] revision = "af50095a40f9041b3b38960738837185c26e9419" [[projects]] branch = "master" name = "golang.org/x/text" - packages = ["collate","collate/build","internal/colltab","internal/gen","internal/tag","internal/triegen","internal/ucd","language","secure/bidirule","transform","unicode/bidi","unicode/cldr","unicode/norm","unicode/rangetable"] + packages = [ + "collate", + "collate/build", + "internal/colltab", + "internal/gen", + "internal/tag", + "internal/triegen", + "internal/ucd", + "language", + "secure/bidirule", + "transform", + "unicode/bidi", + "unicode/cldr", + "unicode/norm", + "unicode/rangetable" + ] revision = "e19ae1496984b1c655b8044a65c0300a3c878dd3" [[projects]] branch = "master" name = "google.golang.org/api" - packages = ["gensupport","googleapi","googleapi/internal/uritemplates","storage/v1"] + packages = [ + "gensupport", + "googleapi", + "googleapi/internal/uritemplates", + "storage/v1" + ] revision = "65b0d8655182691ad23b4fac11e6f7b897d9b634" [[projects]] name = "google.golang.org/appengine" - packages = [".","internal","internal/app_identity","internal/base","internal/datastore","internal/log","internal/modules","internal/remote_api","internal/urlfetch","urlfetch"] + packages = [ + ".", + "internal", + "internal/app_identity", + "internal/base", + "internal/datastore", + "internal/log", + "internal/modules", + "internal/remote_api", + "internal/urlfetch", + "urlfetch" + ] revision = "150dc57a1b433e64154302bdc40b6bb8aefa313a" version = "v1.0.0" @@ -231,6 +311,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "336ac5c261c174cac89f9a7102b493f08edfbd51fd61d1673d1d2ec4132d80ab" + inputs-digest = "a7d099b3ce195ffc37adedb05a4386be38e6158925a1c0fe579efdc20fa11f6a" solver-name = "gps-cdcl" solver-version = 1 diff --git a/changelog/unreleased/pull-1552 b/changelog/unreleased/pull-1552 new file mode 100644 index 000000000..f97a60f06 --- /dev/null +++ b/changelog/unreleased/pull-1552 @@ -0,0 +1,12 @@ +Feature: Use Google Application Default credentials + +Google provide libraries to generate appropriate credentials with various +fallback sources. This change uses the library to generate our GCS client, which +allows us to make use of these extra methods. + +This should be backward compatible with previous restic behaviour while adding +the additional capabilities to auth from Google's internal metadata endpoints. +For users running restic in GCP this can make authentication far easier than it +was before. + +https://developers.google.com/identity/protocols/application-default-credentials diff --git a/cmd/restic/global.go b/cmd/restic/global.go index 6055132bd..5d8e5110f 100644 --- a/cmd/restic/global.go +++ b/cmd/restic/global.go @@ -440,18 +440,6 @@ func parseConfig(loc location.Location, opts options.Options) (interface{}, erro cfg.ProjectID = os.Getenv("GOOGLE_PROJECT_ID") } - if cfg.JSONKeyPath == "" { - if path := os.Getenv("GOOGLE_APPLICATION_CREDENTIALS"); path != "" { - // Check read access - if _, err := ioutil.ReadFile(path); err != nil { - return nil, errors.Fatalf("Failed to read google credential from file %v: %v", path, err) - } - cfg.JSONKeyPath = path - } else { - return nil, errors.Fatal("No credential file path is set") - } - } - if err := opts.Apply(loc.Scheme, &cfg); err != nil { return nil, err } diff --git a/doc/030_preparing_a_new_repo.rst b/doc/030_preparing_a_new_repo.rst index dd8e9f7c7..1b984c4a7 100644 --- a/doc/030_preparing_a_new_repo.rst +++ b/doc/030_preparing_a_new_repo.rst @@ -362,8 +362,14 @@ key file and the project ID as follows: $ export GOOGLE_PROJECT_ID=123123123123 $ export GOOGLE_APPLICATION_CREDENTIALS=$HOME/.config/gs-secret-restic-key.json -Then you can use the ``gs:`` backend type to create a new repository in the -bucket `foo` at the root path: +We use Google's client library to generate [default authentication +material](https://developers.google.com/identity/protocols/application-default-credentials), +which means if you're running in Google Container Engine or are otherwise +located on an instance with default service accounts then these should work out +the box. + +Once authenticated, you can use the ``gs:`` backend type to create a new +repository in the bucket `foo` at the root path: .. code-block:: console diff --git a/internal/backend/gs/config.go b/internal/backend/gs/config.go index eebd4fe23..e18066ef9 100644 --- a/internal/backend/gs/config.go +++ b/internal/backend/gs/config.go @@ -8,13 +8,13 @@ import ( "github.com/restic/restic/internal/options" ) -// Config contains all configuration necessary to connect to a Google Cloud -// Storage bucket. +// Config contains all configuration necessary to connect to a Google Cloud Storage +// bucket. We use Google's default application credentials to acquire an access token, so +// we don't require that calling code supply any authentication material here. type Config struct { - ProjectID string - JSONKeyPath string - Bucket string - Prefix string + ProjectID string + Bucket string + Prefix string Connections uint `option:"connections" help:"set a limit for the number of concurrent connections (default: 20)"` } diff --git a/internal/backend/gs/gs.go b/internal/backend/gs/gs.go index 7cdced796..c68efe01d 100644 --- a/internal/backend/gs/gs.go +++ b/internal/backend/gs/gs.go @@ -15,9 +15,6 @@ import ( "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/restic" - "io/ioutil" - - "golang.org/x/oauth2" "golang.org/x/oauth2/google" "google.golang.org/api/googleapi" storage "google.golang.org/api/storage/v1" @@ -43,30 +40,12 @@ type Backend struct { // Ensure that *Backend implements restic.Backend. var _ restic.Backend = &Backend{} -func getStorageService(jsonKeyPath string, rt http.RoundTripper) (*storage.Service, error) { - - raw, err := ioutil.ReadFile(jsonKeyPath) - if err != nil { - return nil, errors.Wrap(err, "ReadFile") - } - - conf, err := google.JWTConfigFromJSON(raw, storage.DevstorageReadWriteScope) +func getStorageService() (*storage.Service, error) { + client, err := google.DefaultClient(context.TODO(), storage.DevstorageReadWriteScope) if err != nil { return nil, err } - // create a new HTTP client - httpClient := &http.Client{ - Transport: rt, - } - - // create a now context with the HTTP client stored at the oauth2.HTTPClient key - ctx := context.WithValue(context.Background(), oauth2.HTTPClient, httpClient) - - // then pass this context to Client(), which returns a new HTTP client - client := conf.Client(ctx) - - // that we can then pass to New() service, err := storage.New(client) if err != nil { return nil, err @@ -80,7 +59,7 @@ const defaultListMaxItems = 1000 func open(cfg Config, rt http.RoundTripper) (*Backend, error) { debug.Log("open, config %#v", cfg) - service, err := getStorageService(cfg.JSONKeyPath, rt) + service, err := getStorageService() if err != nil { return nil, errors.Wrap(err, "getStorageService") } diff --git a/internal/backend/gs/gs_test.go b/internal/backend/gs/gs_test.go index a7bf97848..27ff809ff 100644 --- a/internal/backend/gs/gs_test.go +++ b/internal/backend/gs/gs_test.go @@ -34,7 +34,6 @@ func newGSTestSuite(t testing.TB) *test.Suite { cfg := gscfg.(gs.Config) cfg.ProjectID = os.Getenv("RESTIC_TEST_GS_PROJECT_ID") - cfg.JSONKeyPath = os.Getenv("RESTIC_TEST_GS_APPLICATION_CREDENTIALS") cfg.Prefix = fmt.Sprintf("test-%d", time.Now().UnixNano()) return cfg, nil }, @@ -88,8 +87,8 @@ func TestBackendGS(t *testing.T) { }() vars := []string{ + "GOOGLE_APPLICATION_CREDENTIALS", "RESTIC_TEST_GS_PROJECT_ID", - "RESTIC_TEST_GS_APPLICATION_CREDENTIALS", "RESTIC_TEST_GS_REPOSITORY", } @@ -106,8 +105,8 @@ func TestBackendGS(t *testing.T) { func BenchmarkBackendGS(t *testing.B) { vars := []string{ + "GOOGLE_APPLICATION_CREDENTIALS", "RESTIC_TEST_GS_PROJECT_ID", - "RESTIC_TEST_GS_APPLICATION_CREDENTIALS", "RESTIC_TEST_GS_REPOSITORY", } diff --git a/run_integration_tests.go b/run_integration_tests.go index 7e182c9e8..91bd1fb95 100644 --- a/run_integration_tests.go +++ b/run_integration_tests.go @@ -183,7 +183,7 @@ func (env *TravisEnvironment) RunTests() error { env.env["GOPATH"] = os.Getenv("GOPATH") if env.gcsCredentialsFile != "" { - env.env["RESTIC_TEST_GS_APPLICATION_CREDENTIALS"] = env.gcsCredentialsFile + env.env["GOOGLE_APPLICATION_CREDENTIALS"] = env.gcsCredentialsFile } // ensure that the following tests cannot be silently skipped on Travis