From 4ddcc17135fe73cd71ab22cb4715ddd4d0f404b8 Mon Sep 17 00:00:00 2001 From: Nick Douma Date: Wed, 11 Nov 2020 11:48:42 +0100 Subject: [PATCH 1/9] Add support for boolean extended options --- internal/options/options.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/internal/options/options.go b/internal/options/options.go index f03eb6097..43d9b63da 100644 --- a/internal/options/options.go +++ b/internal/options/options.go @@ -199,6 +199,14 @@ func (o Options) Apply(ns string, dst interface{}) error { v.Field(i).SetUint(vi) + case "bool": + vi, err := strconv.ParseBool(value) + if err != nil { + return err + } + + v.Field(i).SetBool(vi) + case "Duration": d, err := time.ParseDuration(value) if err != nil { From ccd55d529dd7dcd6db2d39e16215457c7cdd02e7 Mon Sep 17 00:00:00 2001 From: Nick Douma Date: Wed, 11 Nov 2020 11:48:57 +0100 Subject: [PATCH 2/9] Add s3.list-objects-v1 extended option and default to false --- internal/backend/s3/config.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/internal/backend/s3/config.go b/internal/backend/s3/config.go index 93de42152..a0aa95609 100644 --- a/internal/backend/s3/config.go +++ b/internal/backend/s3/config.go @@ -20,16 +20,18 @@ type Config struct { Layout string `option:"layout" help:"use this backend layout (default: auto-detect)"` StorageClass string `option:"storage-class" help:"set S3 storage class (STANDARD, STANDARD_IA, ONEZONE_IA, INTELLIGENT_TIERING or REDUCED_REDUNDANCY)"` - Connections uint `option:"connections" help:"set a limit for the number of concurrent connections (default: 5)"` - MaxRetries uint `option:"retries" help:"set the number of retries attempted"` - Region string `option:"region" help:"set region"` - BucketLookup string `option:"bucket-lookup" help:"bucket lookup style: 'auto', 'dns', or 'path'."` + Connections uint `option:"connections" help:"set a limit for the number of concurrent connections (default: 5)"` + MaxRetries uint `option:"retries" help:"set the number of retries attempted"` + Region string `option:"region" help:"set region"` + BucketLookup string `option:"bucket-lookup" help:"bucket lookup style: 'auto', 'dns', or 'path'."` + ListObjectsV1 bool `option:"list-objects-v1" help:"use deprecated V1 api for ListObjects calls."` } // NewConfig returns a new Config with the default values filled in. func NewConfig() Config { return Config{ - Connections: 5, + Connections: 5, + ListObjectsV1: false, } } From 829959390aeea8b63b4c7be9f8ae9050e2584194 Mon Sep 17 00:00:00 2001 From: Nick Douma Date: Wed, 11 Nov 2020 11:49:20 +0100 Subject: [PATCH 3/9] Provide UseV1 parameter to minio.ListObjectsOptions based on s3.list-objects-v1 --- internal/backend/s3/s3.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/internal/backend/s3/s3.go b/internal/backend/s3/s3.go index 3c40f7c49..08d3b7aa7 100644 --- a/internal/backend/s3/s3.go +++ b/internal/backend/s3/s3.go @@ -205,9 +205,12 @@ func (be *Backend) ReadDir(ctx context.Context, dir string) (list []os.FileInfo, ctx, cancel := context.WithCancel(ctx) defer cancel() + debug.Log("using ListObjectsV1(%v)", be.cfg.ListObjectsV1) + for obj := range be.client.ListObjects(ctx, be.cfg.Bucket, minio.ListObjectsOptions{ Prefix: dir, Recursive: false, + UseV1: be.cfg.ListObjectsV1, }) { if obj.Err != nil { return nil, err @@ -427,12 +430,15 @@ func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.F ctx, cancel := context.WithCancel(ctx) defer cancel() + debug.Log("using ListObjectsV1(%v)", be.cfg.ListObjectsV1) + // NB: unfortunately we can't protect this with be.sem.GetToken() here. // Doing so would enable a deadlock situation (gh-1399), as ListObjects() // starts its own goroutine and returns results via a channel. listresp := be.client.ListObjects(ctx, be.cfg.Bucket, minio.ListObjectsOptions{ Prefix: prefix, Recursive: recursive, + UseV1: be.cfg.ListObjectsV1, }) for obj := range listresp { From 354b7e89cc7b5b8777d305a3114c86f462982d2c Mon Sep 17 00:00:00 2001 From: Nick Douma Date: Wed, 11 Nov 2020 12:32:46 +0100 Subject: [PATCH 4/9] Document the extended s3.list-objects-v1 flag in a new Ceph section --- doc/030_preparing_a_new_repo.rst | 33 ++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/doc/030_preparing_a_new_repo.rst b/doc/030_preparing_a_new_repo.rst index f470cf383..5e9cf1c4a 100644 --- a/doc/030_preparing_a_new_repo.rst +++ b/doc/030_preparing_a_new_repo.rst @@ -339,6 +339,39 @@ For example with an actual endpoint: $ restic -o s3.bucket-lookup=dns -o s3.region=oss-eu-west-1 -r s3:https://oss-eu-west-1.aliyuncs.com/bucketname init +Ceph +**** + +`Ceph Object Gateway `__ is an object +storage interface built on top of librados to provide applications with a RESTful +gateway to Ceph Storage Clusters. + +Ceph Rados Gateway (RGW) provides a S3 interface that is compatible with a large +subset of the Amazon S3 RESTful API. + +You will need to setup the following environment variables with the proper credentials: + +.. code-block:: console + + $ export AWS_ACCESS_KEY_ID= + $ export AWS_SECRET_ACCESS_KEY= + +Now you can easily initialize restic to use Ceph RGW as a backend with +this command. + +.. code-block:: console + + $ restic -r s3:/// init + enter password for new backend: + enter password again: + created restic backend xxxxxxxxxx at s3:/// + Please note that knowledge of your password is required to access + the repository. Losing your password means that your data is irrecoverably lost. + +.. note:: Version of Ceph before v14.2.5 do not implement the ListObjectsV2 API properly. + On these backends you need to provide the ``-o s3.list-objects-v1=true`` option + to use the ListObjects API instead. + OpenStack Swift *************** From 4320ff2bbfd344067d9438518e905ed7bb4e4394 Mon Sep 17 00:00:00 2001 From: Nick Douma Date: Wed, 11 Nov 2020 12:33:22 +0100 Subject: [PATCH 5/9] Add changelog entry for s3.list-objects-v1 --- changelog/unreleased/issue-3083 | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 changelog/unreleased/issue-3083 diff --git a/changelog/unreleased/issue-3083 b/changelog/unreleased/issue-3083 new file mode 100644 index 000000000..00e34167b --- /dev/null +++ b/changelog/unreleased/issue-3083 @@ -0,0 +1,13 @@ +Enhancement: Allow usage of deprecated S3 ListObjectsV1 API + +Restic didn't work with some S3 API implementations that don't have a +working ListObjectsV2 API endpoint. This concerns mainly Ceph versions +before v14.2.5 , but may impact other S3 API implementations. + +Restic now allows selection of the older ListObjectsV1 endpoint by using +the 's3.list-objects-v1' extended option, for instance: + +`restic -o s3.list-objects-v1=true snapshots` + +https://github.com/restic/restic/issues/3083 +https://github.com/restic/restic/pull/3085 \ No newline at end of file From f013662e3fb89b3b67cf1319d7a7db20e9dd2f2e Mon Sep 17 00:00:00 2001 From: Nick Douma Date: Wed, 11 Nov 2020 15:11:14 +0100 Subject: [PATCH 6/9] Remove separate section on Ceph, and move s3.list-objects-v1 note to S3 section --- doc/030_preparing_a_new_repo.rst | 39 +++++--------------------------- 1 file changed, 6 insertions(+), 33 deletions(-) diff --git a/doc/030_preparing_a_new_repo.rst b/doc/030_preparing_a_new_repo.rst index 5e9cf1c4a..164cb7007 100644 --- a/doc/030_preparing_a_new_repo.rst +++ b/doc/030_preparing_a_new_repo.rst @@ -239,6 +239,12 @@ For an S3-compatible server that is not Amazon (like Minio, see below), or is only available via HTTP, you can specify the URL to the server like this: ``s3:http://server:port/bucket_name``. + +.. note:: Certain S3-compatible servers do not properly implement the ListObjectsV2 API, + most notably Ceph versions before v14.2.5. On these backends you need to + provide the ``-o s3.list-objects-v1=true`` option to use the ListObjects API instead. + + Minio Server ************ @@ -339,39 +345,6 @@ For example with an actual endpoint: $ restic -o s3.bucket-lookup=dns -o s3.region=oss-eu-west-1 -r s3:https://oss-eu-west-1.aliyuncs.com/bucketname init -Ceph -**** - -`Ceph Object Gateway `__ is an object -storage interface built on top of librados to provide applications with a RESTful -gateway to Ceph Storage Clusters. - -Ceph Rados Gateway (RGW) provides a S3 interface that is compatible with a large -subset of the Amazon S3 RESTful API. - -You will need to setup the following environment variables with the proper credentials: - -.. code-block:: console - - $ export AWS_ACCESS_KEY_ID= - $ export AWS_SECRET_ACCESS_KEY= - -Now you can easily initialize restic to use Ceph RGW as a backend with -this command. - -.. code-block:: console - - $ restic -r s3:/// init - enter password for new backend: - enter password again: - created restic backend xxxxxxxxxx at s3:/// - Please note that knowledge of your password is required to access - the repository. Losing your password means that your data is irrecoverably lost. - -.. note:: Version of Ceph before v14.2.5 do not implement the ListObjectsV2 API properly. - On these backends you need to provide the ``-o s3.list-objects-v1=true`` option - to use the ListObjects API instead. - OpenStack Swift *************** From 3e0456d88b0eea9e2c4865d6d7d027630a9b64fd Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Wed, 11 Nov 2020 20:00:34 +0100 Subject: [PATCH 7/9] Highlight that s3.list-objects-v1 is a temporary --- changelog/unreleased/issue-3083 | 18 ++++++++++++------ doc/030_preparing_a_new_repo.rst | 9 ++++++--- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/changelog/unreleased/issue-3083 b/changelog/unreleased/issue-3083 index 00e34167b..5790a0094 100644 --- a/changelog/unreleased/issue-3083 +++ b/changelog/unreleased/issue-3083 @@ -1,13 +1,19 @@ Enhancement: Allow usage of deprecated S3 ListObjectsV1 API Restic didn't work with some S3 API implementations that don't have a -working ListObjectsV2 API endpoint. This concerns mainly Ceph versions -before v14.2.5 , but may impact other S3 API implementations. +broken `ListObjectsV2` API endpoint. This concerns mainly Ceph versions +before v14.2.5 , but may impact other S3 API implementations. When a broken +server implementation is used, restic prints errors similar to the following: -Restic now allows selection of the older ListObjectsV1 endpoint by using -the 's3.list-objects-v1' extended option, for instance: + List() returned error: Truncated response should have continuation token set -`restic -o s3.list-objects-v1=true snapshots` +As a temporary workaround, restic now allows selection of the older +`ListObjects` endpoint by using the `s3.list-objects-v1` extended option, for +instance: + + restic -o s3.list-objects-v1=true snapshots + +This option may be removed in future versions of restic. https://github.com/restic/restic/issues/3083 -https://github.com/restic/restic/pull/3085 \ No newline at end of file +https://github.com/restic/restic/pull/3085 diff --git a/doc/030_preparing_a_new_repo.rst b/doc/030_preparing_a_new_repo.rst index 164cb7007..e6cd35757 100644 --- a/doc/030_preparing_a_new_repo.rst +++ b/doc/030_preparing_a_new_repo.rst @@ -240,9 +240,12 @@ or is only available via HTTP, you can specify the URL to the server like this: ``s3:http://server:port/bucket_name``. -.. note:: Certain S3-compatible servers do not properly implement the ListObjectsV2 API, - most notably Ceph versions before v14.2.5. On these backends you need to - provide the ``-o s3.list-objects-v1=true`` option to use the ListObjects API instead. +.. note:: Certain S3-compatible servers do not properly implement the + ``ListObjectsV2`` API, most notably Ceph versions before v14.2.5. On these + backends, as a temporary workaround, you can provide the + ``-o s3.list-objects-v1=true`` option to use the older + ``ListObjects`` API instead. This option may be removed in future + versions of restic. Minio Server From 4a0b7328eccfcd4aa6a36dd32a8c4fd312fbb2ac Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Wed, 11 Nov 2020 20:20:35 +0100 Subject: [PATCH 8/9] s3: Remove dots for config description --- internal/backend/s3/config.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/backend/s3/config.go b/internal/backend/s3/config.go index a0aa95609..77b712fc7 100644 --- a/internal/backend/s3/config.go +++ b/internal/backend/s3/config.go @@ -23,8 +23,8 @@ type Config struct { Connections uint `option:"connections" help:"set a limit for the number of concurrent connections (default: 5)"` MaxRetries uint `option:"retries" help:"set the number of retries attempted"` Region string `option:"region" help:"set region"` - BucketLookup string `option:"bucket-lookup" help:"bucket lookup style: 'auto', 'dns', or 'path'."` - ListObjectsV1 bool `option:"list-objects-v1" help:"use deprecated V1 api for ListObjects calls."` + BucketLookup string `option:"bucket-lookup" help:"bucket lookup style: 'auto', 'dns', or 'path'"` + ListObjectsV1 bool `option:"list-objects-v1" help:"use deprecated V1 api for ListObjects calls"` } // NewConfig returns a new Config with the default values filled in. From b1bbdcb637a9d4122fbe51efd26341be63a4acdb Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Wed, 11 Nov 2020 20:30:30 +0100 Subject: [PATCH 9/9] Improve changelog --- changelog/unreleased/issue-3083 | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/changelog/unreleased/issue-3083 b/changelog/unreleased/issue-3083 index 5790a0094..c1653346a 100644 --- a/changelog/unreleased/issue-3083 +++ b/changelog/unreleased/issue-3083 @@ -1,15 +1,14 @@ Enhancement: Allow usage of deprecated S3 ListObjectsV1 API -Restic didn't work with some S3 API implementations that don't have a -broken `ListObjectsV2` API endpoint. This concerns mainly Ceph versions -before v14.2.5 , but may impact other S3 API implementations. When a broken -server implementation is used, restic prints errors similar to the following: +Some S3 API implementations, e.g. Ceph before version 14.2.5, have a broken +`ListObjectsV2` implementation which cause problems for restic when using their +API endpoints. When a broken server implementation is used, restic prints +errors similar to the following: List() returned error: Truncated response should have continuation token set -As a temporary workaround, restic now allows selection of the older -`ListObjects` endpoint by using the `s3.list-objects-v1` extended option, for -instance: +As a temporary workaround, restic now allows using the older `ListObjects` +endpoint by setting the `s3.list-objects-v1` extended option, for instance: restic -o s3.list-objects-v1=true snapshots