From 897c923cc954518fa4c45224ac3e99296ae9340b Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 14 Oct 2017 14:51:00 +0200 Subject: [PATCH] Retry failed backend requests --- cmd/restic/global.go | 5 ++ internal/backend/backend_error.go | 73 +++++++++++++++++++++++++++++ internal/backend/backend_retry.go | 77 +++++++++++++++++++++++++++++++ 3 files changed, 155 insertions(+) create mode 100644 internal/backend/backend_error.go create mode 100644 internal/backend/backend_retry.go diff --git a/cmd/restic/global.go b/cmd/restic/global.go index 631de553f..70d031289 100644 --- a/cmd/restic/global.go +++ b/cmd/restic/global.go @@ -9,6 +9,7 @@ import ( "runtime" "strings" "syscall" + "time" "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/azure" @@ -328,6 +329,10 @@ func OpenRepository(opts GlobalOptions) (*repository.Repository, error) { be = limiter.LimitBackend(be, limiter.NewStaticLimiter(opts.LimitUploadKb, opts.LimitDownloadKb)) } + be = backend.NewRetryBackend(be, 10, func(msg string, err error, d time.Duration) { + Warnf("%v returned error, retrying after %v: %v\n", msg, d, err) + }) + s := repository.New(be) opts.password, err = ReadPassword(opts, "enter password for repository: ") diff --git a/internal/backend/backend_error.go b/internal/backend/backend_error.go new file mode 100644 index 000000000..2c9a616cc --- /dev/null +++ b/internal/backend/backend_error.go @@ -0,0 +1,73 @@ +package backend + +import ( + "context" + "io" + "math/rand" + "sync" + + "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/restic" +) + +// ErrorBackend is used to induce errors into various function calls and test +// the retry functions. +type ErrorBackend struct { + FailSave float32 + FailLoad float32 + FailStat float32 + restic.Backend + + r *rand.Rand + m sync.Mutex +} + +// statically ensure that ErrorBackend implements restic.Backend. +var _ restic.Backend = &ErrorBackend{} + +// NewErrorBackend wraps be with a backend that returns errors according to +// given probabilities. +func NewErrorBackend(be restic.Backend, seed int64) *ErrorBackend { + return &ErrorBackend{ + Backend: be, + r: rand.New(rand.NewSource(seed)), + } +} + +func (be *ErrorBackend) fail(p float32) bool { + be.m.Lock() + v := be.r.Float32() + be.m.Unlock() + + return v < p +} + +// Save stores the data in the backend under the given handle. +func (be *ErrorBackend) Save(ctx context.Context, h restic.Handle, rd io.Reader) error { + if be.fail(be.FailSave) { + return errors.Errorf("Save(%v) random error induced", h) + } + + return be.Backend.Save(ctx, h, rd) +} + +// Load returns a reader that yields the contents of the file at h at the +// given offset. If length is larger than zero, only a portion of the file +// is returned. rd must be closed after use. If an error is returned, the +// ReadCloser must be nil. +func (be *ErrorBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { + if be.fail(be.FailLoad) { + return nil, errors.Errorf("Load(%v, %v, %v) random error induced", h, length, offset) + } + + return be.Backend.Load(ctx, h, length, offset) +} + +// Stat returns information about the File identified by h. +func (be *ErrorBackend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) { + if be.fail(be.FailLoad) { + return restic.FileInfo{}, errors.Errorf("Stat(%v) random error induced", h) + } + + return be.Stat(ctx, h) +} diff --git a/internal/backend/backend_retry.go b/internal/backend/backend_retry.go new file mode 100644 index 000000000..66db447ca --- /dev/null +++ b/internal/backend/backend_retry.go @@ -0,0 +1,77 @@ +package backend + +import ( + "context" + "fmt" + "io" + "time" + + "github.com/cenkalti/backoff" + "github.com/restic/restic/internal/restic" +) + +// RetryBackend retries operations on the backend in case of an error with a +// backoff. +type RetryBackend struct { + restic.Backend + MaxTries int + Report func(string, error, time.Duration) +} + +// statically ensure that RetryBackend implements restic.Backend. +var _ restic.Backend = &RetryBackend{} + +// NewRetryBackend wraps be with a backend that retries operations after a +// backoff. report is called with a description and the error, if one occurred. +func NewRetryBackend(be restic.Backend, maxTries int, report func(string, error, time.Duration)) *RetryBackend { + return &RetryBackend{ + Backend: be, + MaxTries: maxTries, + Report: report, + } +} + +func (be *RetryBackend) retry(msg string, f func() error) error { + return backoff.RetryNotify(f, + backoff.WithMaxTries(backoff.NewExponentialBackOff(), uint64(be.MaxTries)), + func(err error, d time.Duration) { + if be.Report != nil { + be.Report(msg, err, d) + } + }, + ) +} + +// Save stores the data in the backend under the given handle. +func (be *RetryBackend) Save(ctx context.Context, h restic.Handle, rd io.Reader) error { + return be.retry(fmt.Sprintf("Save(%v)", h), func() error { + return be.Backend.Save(ctx, h, rd) + }) +} + +// Load returns a reader that yields the contents of the file at h at the +// given offset. If length is larger than zero, only a portion of the file +// is returned. rd must be closed after use. If an error is returned, the +// ReadCloser must be nil. +func (be *RetryBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64) (rd io.ReadCloser, err error) { + err = be.retry(fmt.Sprintf("Load(%v, %v, %v)", h, length, offset), + func() error { + var innerError error + rd, innerError = be.Backend.Load(ctx, h, length, offset) + + return innerError + }) + return rd, err +} + +// Stat returns information about the File identified by h. +func (be *RetryBackend) Stat(ctx context.Context, h restic.Handle) (fi restic.FileInfo, err error) { + err = be.retry(fmt.Sprintf("Stat(%v)", h), + func() error { + var innerError error + fi, innerError = be.Backend.Stat(ctx, h) + + return innerError + }) + return fi, err +}