Merge pull request #1281 from prattmic/gcs_perms

gs: allow backend creation without storage.buckets.get

CHANGELOG.md

@@ -22,6 +22,11 @@ Important Changes in 0.X.Y
    untangle mixed files automatically.
    https://github.com/restic/restic/pull/1265
 
+ * The Google Cloud Storage backend no longer requires the service account to
+   have the `storage.buckets.get` permission ("Storage Admin" role) in `restic
+   init` if the bucket already exists.
+   https://github.com/restic/restic/pull/1281
+
 Small changes
 -------------
 

doc/manual.rst

@@ -411,16 +411,18 @@ Restic connects to Google Cloud Storage via a `service account`_.
 
 For normal restic operation, the service account must have the
 ``storage.objects.{create,delete,get,list}`` permissions for the bucket. These
-are included in the "Storage Object Admin" role. For ``restic init``, the
-service account must also have the ``storage.buckets.get`` and
-``storage.buckets.create`` (if the bucket does not exist) permissions. These
-are included in the "Storage Admin" role.
+are included in the "Storage Object Admin" role.
 
-`Create a service account key`_ and download the JSON credentials file.
+``restic init`` can create the repository bucket. Doing so requires the
+``storage.buckets.create`` permission ("Storage Admin" role). If the bucket
+already exists that permission is unnecessary.
 
-In addition, you need the Google Project ID that you can see in the Google
-Cloud Platform console at the "Storage/Settings" menu. Export the path to the
-JSON key file and the project ID as follows:
+To use the Google Cloud Storage backend, first `create a service account key`_
+and download the JSON credentials file.
+
+Second, find the Google Project ID that you can see in the Google Cloud
+Platform console at the "Storage/Settings" menu. Export the path to the JSON
+key file and the project ID as follows:
 
 .. code-block:: console
 
@@ -436,7 +438,7 @@ bucket `foo` at the root path:
     enter password for new backend:
     enter password again:
 
-    created restic backend bde47d6254 at gs:restic-dev-an:foo2
+    created restic backend bde47d6254 at gs:foo:/
     [...]
 
 The number of concurrent connections to the GCS service can be set with the
@@ -444,7 +446,7 @@ The number of concurrent connections to the GCS service can be set with the
 established.
 
 .. _service account: https://cloud.google.com/storage/docs/authentication#service_accounts
-.. _Create a service account key: https://cloud.google.com/storage/docs/authentication#generating-a-private-key
+.. _create a service account key: https://cloud.google.com/storage/docs/authentication#generating-a-private-key
 
 
 Password prompt on Windows

internal/backend/gs/gs.go

@@ -99,27 +99,58 @@ func Open(cfg Config) (restic.Backend, error) {
 	return open(cfg)
 }
 
-// Create opens the gs backend at the specified bucket and creates the bucket
-// if it does not exist yet.
+// Create opens the gs backend at the specified bucket and attempts to creates
+// the bucket if it does not exist yet.
 //
-// In addition to the permissions required by Backend, Create requires these
-// permissions:
-//  * storage.buckets.get
-//  * storage.buckets.create (if the bucket doesn't exist)
+// The service account must have the "storage.buckets.create" permission to
+// create a bucket the does not yet exist.
 func Create(cfg Config) (restic.Backend, error) {
 	be, err := open(cfg)
 	if err != nil {
 		return nil, errors.Wrap(err, "open")
 	}
 
-	// Create bucket if it doesn't exist.
+	// Try to determine if the bucket exists. If it does not, try to create it.
+	//
+	// A Get call has three typical error cases:
+	//
+	// * nil: Bucket exists and we have access to the metadata (returned).
+	//
+	// * 403: Bucket exists and we do not have access to the metadata. We
+	// don't have storage.buckets.get permission to the bucket, but we may
+	// still be able to access objects in the bucket.
+	//
+	// * 404: Bucket doesn't exist.
+	//
+	// Determining if the bucket is accessible is best-effort because the
+	// 403 case is ambiguous.
 	if _, err := be.service.Buckets.Get(be.bucketName).Do(); err != nil {
-		bucket := &storage.Bucket{
-			Name: be.bucketName,
+		gerr, ok := err.(*googleapi.Error)
+		if !ok {
+			// Don't know what to do with this error.
+			return nil, errors.Wrap(err, "service.Buckets.Get")
 		}
 
-		if _, err := be.service.Buckets.Insert(be.projectID, bucket).Do(); err != nil {
-			return nil, errors.Wrap(err, "service.Buckets.Insert")
+		switch gerr.Code {
+		case 403:
+			// Bucket exists, but we don't know if it is
+			// accessible. Optimistically assume it is; if not,
+			// future Backend calls will fail.
+			debug.Log("Unable to determine if bucket %s is accessible (err %v). Continuing as if it is.", be.bucketName, err)
+		case 404:
+			// Bucket doesn't exist, try to create it.
+			bucket := &storage.Bucket{
+				Name: be.bucketName,
+			}
+
+			if _, err := be.service.Buckets.Insert(be.projectID, bucket).Do(); err != nil {
+				// Always an error, as the bucket definitely
+				// doesn't exist.
+				return nil, errors.Wrap(err, "service.Buckets.Insert")
+			}
+		default:
+			// Don't know what to do with this error.
+			return nil, errors.Wrap(err, "service.Buckets.Get")
 		}
 	}