2
2
mirror of https://github.com/octoleo/restic.git synced 2024-11-26 14:56:29 +00:00

Merge pull request #3085 from LordGaav/s3-list-objects-v1-flag

Extended option to select V1 API for ListObjects on S3 backend
This commit is contained in:
rawtaz 2020-11-11 20:52:44 +01:00 committed by GitHub
commit dfb9326b1b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 48 additions and 5 deletions

View File

@ -0,0 +1,18 @@
Enhancement: Allow usage of deprecated S3 ListObjectsV1 API
Some S3 API implementations, e.g. Ceph before version 14.2.5, have a broken
`ListObjectsV2` implementation which cause problems for restic when using their
API endpoints. When a broken server implementation is used, restic prints
errors similar to the following:
List() returned error: Truncated response should have continuation token set
As a temporary workaround, restic now allows using the older `ListObjects`
endpoint by setting the `s3.list-objects-v1` extended option, for instance:
restic -o s3.list-objects-v1=true snapshots
This option may be removed in future versions of restic.
https://github.com/restic/restic/issues/3083
https://github.com/restic/restic/pull/3085

View File

@ -239,6 +239,15 @@ For an S3-compatible server that is not Amazon (like Minio, see below),
or is only available via HTTP, you can specify the URL to the server
like this: ``s3:http://server:port/bucket_name``.
.. note:: Certain S3-compatible servers do not properly implement the
``ListObjectsV2`` API, most notably Ceph versions before v14.2.5. On these
backends, as a temporary workaround, you can provide the
``-o s3.list-objects-v1=true`` option to use the older
``ListObjects`` API instead. This option may be removed in future
versions of restic.
Minio Server
************

View File

@ -23,13 +23,15 @@ type Config struct {
Connections uint `option:"connections" help:"set a limit for the number of concurrent connections (default: 5)"`
MaxRetries uint `option:"retries" help:"set the number of retries attempted"`
Region string `option:"region" help:"set region"`
BucketLookup string `option:"bucket-lookup" help:"bucket lookup style: 'auto', 'dns', or 'path'."`
BucketLookup string `option:"bucket-lookup" help:"bucket lookup style: 'auto', 'dns', or 'path'"`
ListObjectsV1 bool `option:"list-objects-v1" help:"use deprecated V1 api for ListObjects calls"`
}
// NewConfig returns a new Config with the default values filled in.
func NewConfig() Config {
return Config{
Connections: 5,
ListObjectsV1: false,
}
}

View File

@ -205,9 +205,12 @@ func (be *Backend) ReadDir(ctx context.Context, dir string) (list []os.FileInfo,
ctx, cancel := context.WithCancel(ctx)
defer cancel()
debug.Log("using ListObjectsV1(%v)", be.cfg.ListObjectsV1)
for obj := range be.client.ListObjects(ctx, be.cfg.Bucket, minio.ListObjectsOptions{
Prefix: dir,
Recursive: false,
UseV1: be.cfg.ListObjectsV1,
}) {
if obj.Err != nil {
return nil, err
@ -427,12 +430,15 @@ func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.F
ctx, cancel := context.WithCancel(ctx)
defer cancel()
debug.Log("using ListObjectsV1(%v)", be.cfg.ListObjectsV1)
// NB: unfortunately we can't protect this with be.sem.GetToken() here.
// Doing so would enable a deadlock situation (gh-1399), as ListObjects()
// starts its own goroutine and returns results via a channel.
listresp := be.client.ListObjects(ctx, be.cfg.Bucket, minio.ListObjectsOptions{
Prefix: prefix,
Recursive: recursive,
UseV1: be.cfg.ListObjectsV1,
})
for obj := range listresp {

View File

@ -199,6 +199,14 @@ func (o Options) Apply(ns string, dst interface{}) error {
v.Field(i).SetUint(vi)
case "bool":
vi, err := strconv.ParseBool(value)
if err != nil {
return err
}
v.Field(i).SetBool(vi)
case "Duration":
d, err := time.ParseDuration(value)
if err != nil {