mirror of
https://github.com/octoleo/restic.git
synced 2024-12-23 03:18:55 +00:00
Merge pull request #3085 from LordGaav/s3-list-objects-v1-flag
Extended option to select V1 API for ListObjects on S3 backend
This commit is contained in:
commit
dfb9326b1b
18
changelog/unreleased/issue-3083
Normal file
18
changelog/unreleased/issue-3083
Normal file
@ -0,0 +1,18 @@
|
||||
Enhancement: Allow usage of deprecated S3 ListObjectsV1 API
|
||||
|
||||
Some S3 API implementations, e.g. Ceph before version 14.2.5, have a broken
|
||||
`ListObjectsV2` implementation which cause problems for restic when using their
|
||||
API endpoints. When a broken server implementation is used, restic prints
|
||||
errors similar to the following:
|
||||
|
||||
List() returned error: Truncated response should have continuation token set
|
||||
|
||||
As a temporary workaround, restic now allows using the older `ListObjects`
|
||||
endpoint by setting the `s3.list-objects-v1` extended option, for instance:
|
||||
|
||||
restic -o s3.list-objects-v1=true snapshots
|
||||
|
||||
This option may be removed in future versions of restic.
|
||||
|
||||
https://github.com/restic/restic/issues/3083
|
||||
https://github.com/restic/restic/pull/3085
|
@ -239,6 +239,15 @@ For an S3-compatible server that is not Amazon (like Minio, see below),
|
||||
or is only available via HTTP, you can specify the URL to the server
|
||||
like this: ``s3:http://server:port/bucket_name``.
|
||||
|
||||
|
||||
.. note:: Certain S3-compatible servers do not properly implement the
|
||||
``ListObjectsV2`` API, most notably Ceph versions before v14.2.5. On these
|
||||
backends, as a temporary workaround, you can provide the
|
||||
``-o s3.list-objects-v1=true`` option to use the older
|
||||
``ListObjects`` API instead. This option may be removed in future
|
||||
versions of restic.
|
||||
|
||||
|
||||
Minio Server
|
||||
************
|
||||
|
||||
|
@ -20,16 +20,18 @@ type Config struct {
|
||||
Layout string `option:"layout" help:"use this backend layout (default: auto-detect)"`
|
||||
StorageClass string `option:"storage-class" help:"set S3 storage class (STANDARD, STANDARD_IA, ONEZONE_IA, INTELLIGENT_TIERING or REDUCED_REDUNDANCY)"`
|
||||
|
||||
Connections uint `option:"connections" help:"set a limit for the number of concurrent connections (default: 5)"`
|
||||
MaxRetries uint `option:"retries" help:"set the number of retries attempted"`
|
||||
Region string `option:"region" help:"set region"`
|
||||
BucketLookup string `option:"bucket-lookup" help:"bucket lookup style: 'auto', 'dns', or 'path'."`
|
||||
Connections uint `option:"connections" help:"set a limit for the number of concurrent connections (default: 5)"`
|
||||
MaxRetries uint `option:"retries" help:"set the number of retries attempted"`
|
||||
Region string `option:"region" help:"set region"`
|
||||
BucketLookup string `option:"bucket-lookup" help:"bucket lookup style: 'auto', 'dns', or 'path'"`
|
||||
ListObjectsV1 bool `option:"list-objects-v1" help:"use deprecated V1 api for ListObjects calls"`
|
||||
}
|
||||
|
||||
// NewConfig returns a new Config with the default values filled in.
|
||||
func NewConfig() Config {
|
||||
return Config{
|
||||
Connections: 5,
|
||||
Connections: 5,
|
||||
ListObjectsV1: false,
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -205,9 +205,12 @@ func (be *Backend) ReadDir(ctx context.Context, dir string) (list []os.FileInfo,
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
|
||||
debug.Log("using ListObjectsV1(%v)", be.cfg.ListObjectsV1)
|
||||
|
||||
for obj := range be.client.ListObjects(ctx, be.cfg.Bucket, minio.ListObjectsOptions{
|
||||
Prefix: dir,
|
||||
Recursive: false,
|
||||
UseV1: be.cfg.ListObjectsV1,
|
||||
}) {
|
||||
if obj.Err != nil {
|
||||
return nil, err
|
||||
@ -427,12 +430,15 @@ func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.F
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
|
||||
debug.Log("using ListObjectsV1(%v)", be.cfg.ListObjectsV1)
|
||||
|
||||
// NB: unfortunately we can't protect this with be.sem.GetToken() here.
|
||||
// Doing so would enable a deadlock situation (gh-1399), as ListObjects()
|
||||
// starts its own goroutine and returns results via a channel.
|
||||
listresp := be.client.ListObjects(ctx, be.cfg.Bucket, minio.ListObjectsOptions{
|
||||
Prefix: prefix,
|
||||
Recursive: recursive,
|
||||
UseV1: be.cfg.ListObjectsV1,
|
||||
})
|
||||
|
||||
for obj := range listresp {
|
||||
|
@ -199,6 +199,14 @@ func (o Options) Apply(ns string, dst interface{}) error {
|
||||
|
||||
v.Field(i).SetUint(vi)
|
||||
|
||||
case "bool":
|
||||
vi, err := strconv.ParseBool(value)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
v.Field(i).SetBool(vi)
|
||||
|
||||
case "Duration":
|
||||
d, err := time.ParseDuration(value)
|
||||
if err != nil {
|
||||
|
Loading…
Reference in New Issue
Block a user