diff --git a/changelog/unreleased/issue-3083 b/changelog/unreleased/issue-3083 new file mode 100644 index 000000000..c1653346a --- /dev/null +++ b/changelog/unreleased/issue-3083 @@ -0,0 +1,18 @@ +Enhancement: Allow usage of deprecated S3 ListObjectsV1 API + +Some S3 API implementations, e.g. Ceph before version 14.2.5, have a broken +`ListObjectsV2` implementation which cause problems for restic when using their +API endpoints. When a broken server implementation is used, restic prints +errors similar to the following: + + List() returned error: Truncated response should have continuation token set + +As a temporary workaround, restic now allows using the older `ListObjects` +endpoint by setting the `s3.list-objects-v1` extended option, for instance: + + restic -o s3.list-objects-v1=true snapshots + +This option may be removed in future versions of restic. + +https://github.com/restic/restic/issues/3083 +https://github.com/restic/restic/pull/3085 diff --git a/doc/030_preparing_a_new_repo.rst b/doc/030_preparing_a_new_repo.rst index f470cf383..e6cd35757 100644 --- a/doc/030_preparing_a_new_repo.rst +++ b/doc/030_preparing_a_new_repo.rst @@ -239,6 +239,15 @@ For an S3-compatible server that is not Amazon (like Minio, see below), or is only available via HTTP, you can specify the URL to the server like this: ``s3:http://server:port/bucket_name``. + +.. note:: Certain S3-compatible servers do not properly implement the + ``ListObjectsV2`` API, most notably Ceph versions before v14.2.5. On these + backends, as a temporary workaround, you can provide the + ``-o s3.list-objects-v1=true`` option to use the older + ``ListObjects`` API instead. This option may be removed in future + versions of restic. + + Minio Server ************ diff --git a/internal/backend/s3/config.go b/internal/backend/s3/config.go index 93de42152..77b712fc7 100644 --- a/internal/backend/s3/config.go +++ b/internal/backend/s3/config.go @@ -20,16 +20,18 @@ type Config struct { Layout string `option:"layout" help:"use this backend layout (default: auto-detect)"` StorageClass string `option:"storage-class" help:"set S3 storage class (STANDARD, STANDARD_IA, ONEZONE_IA, INTELLIGENT_TIERING or REDUCED_REDUNDANCY)"` - Connections uint `option:"connections" help:"set a limit for the number of concurrent connections (default: 5)"` - MaxRetries uint `option:"retries" help:"set the number of retries attempted"` - Region string `option:"region" help:"set region"` - BucketLookup string `option:"bucket-lookup" help:"bucket lookup style: 'auto', 'dns', or 'path'."` + Connections uint `option:"connections" help:"set a limit for the number of concurrent connections (default: 5)"` + MaxRetries uint `option:"retries" help:"set the number of retries attempted"` + Region string `option:"region" help:"set region"` + BucketLookup string `option:"bucket-lookup" help:"bucket lookup style: 'auto', 'dns', or 'path'"` + ListObjectsV1 bool `option:"list-objects-v1" help:"use deprecated V1 api for ListObjects calls"` } // NewConfig returns a new Config with the default values filled in. func NewConfig() Config { return Config{ - Connections: 5, + Connections: 5, + ListObjectsV1: false, } } diff --git a/internal/backend/s3/s3.go b/internal/backend/s3/s3.go index 3c40f7c49..08d3b7aa7 100644 --- a/internal/backend/s3/s3.go +++ b/internal/backend/s3/s3.go @@ -205,9 +205,12 @@ func (be *Backend) ReadDir(ctx context.Context, dir string) (list []os.FileInfo, ctx, cancel := context.WithCancel(ctx) defer cancel() + debug.Log("using ListObjectsV1(%v)", be.cfg.ListObjectsV1) + for obj := range be.client.ListObjects(ctx, be.cfg.Bucket, minio.ListObjectsOptions{ Prefix: dir, Recursive: false, + UseV1: be.cfg.ListObjectsV1, }) { if obj.Err != nil { return nil, err @@ -427,12 +430,15 @@ func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.F ctx, cancel := context.WithCancel(ctx) defer cancel() + debug.Log("using ListObjectsV1(%v)", be.cfg.ListObjectsV1) + // NB: unfortunately we can't protect this with be.sem.GetToken() here. // Doing so would enable a deadlock situation (gh-1399), as ListObjects() // starts its own goroutine and returns results via a channel. listresp := be.client.ListObjects(ctx, be.cfg.Bucket, minio.ListObjectsOptions{ Prefix: prefix, Recursive: recursive, + UseV1: be.cfg.ListObjectsV1, }) for obj := range listresp { diff --git a/internal/options/options.go b/internal/options/options.go index f03eb6097..43d9b63da 100644 --- a/internal/options/options.go +++ b/internal/options/options.go @@ -199,6 +199,14 @@ func (o Options) Apply(ns string, dst interface{}) error { v.Field(i).SetUint(vi) + case "bool": + vi, err := strconv.ParseBool(value) + if err != nil { + return err + } + + v.Field(i).SetBool(vi) + case "Duration": d, err := time.ParseDuration(value) if err != nil {