From 0e722efb0914a6125f640eeca3f003401eae2cbd Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 14 Oct 2017 13:38:17 +0200 Subject: [PATCH 1/8] backend: Add Delete() to restic.Backend interface --- internal/backend/local/local.go | 2 +- internal/backend/rest/rest.go | 31 +++++++++++++++++++++++++++++++ internal/backend/sftp/sftp.go | 5 +++++ internal/restic/backend.go | 3 +++ 4 files changed, 40 insertions(+), 1 deletion(-) diff --git a/internal/backend/local/local.go b/internal/backend/local/local.go index b3cd9127f..b738bff13 100644 --- a/internal/backend/local/local.go +++ b/internal/backend/local/local.go @@ -275,7 +275,7 @@ func (b *Local) List(ctx context.Context, t restic.FileType) <-chan string { } // Delete removes the repository and all files. -func (b *Local) Delete() error { +func (b *Local) Delete(ctx context.Context) error { debug.Log("Delete()") return fs.RemoveAll(b.Path) } diff --git a/internal/backend/rest/rest.go b/internal/backend/rest/rest.go index c5d1140c2..cd283a8c9 100644 --- a/internal/backend/rest/rest.go +++ b/internal/backend/rest/rest.go @@ -349,3 +349,34 @@ func (b *restBackend) Close() error { // same function. return nil } + +// Remove keys for a specified backend type. +func (b *restBackend) removeKeys(ctx context.Context, t restic.FileType) error { + for key := range b.List(ctx, restic.DataFile) { + err := b.Remove(ctx, restic.Handle{Type: restic.DataFile, Name: key}) + if err != nil { + return err + } + } + + return nil +} + +// Delete removes all data in the backend. +func (b *restBackend) Delete(ctx context.Context) error { + alltypes := []restic.FileType{ + restic.DataFile, + restic.KeyFile, + restic.LockFile, + restic.SnapshotFile, + restic.IndexFile} + + for _, t := range alltypes { + err := b.removeKeys(ctx, t) + if err != nil { + return nil + } + } + + return b.Remove(ctx, restic.Handle{Type: restic.ConfigFile}) +} diff --git a/internal/backend/sftp/sftp.go b/internal/backend/sftp/sftp.go index 27ecabd81..ef006ce6f 100644 --- a/internal/backend/sftp/sftp.go +++ b/internal/backend/sftp/sftp.go @@ -499,3 +499,8 @@ func (r *SFTP) Close() error { <-r.result return nil } + +// Delete removes all data in the backend. +func (r *SFTP) Delete(context.Context) error { + return r.c.RemoveDirectory(r.p) +} diff --git a/internal/restic/backend.go b/internal/restic/backend.go index b3d91cc62..906ebf6d7 100644 --- a/internal/restic/backend.go +++ b/internal/restic/backend.go @@ -40,6 +40,9 @@ type Backend interface { // IsNotExist returns true if the error was caused by a non-existing file // in the backend. IsNotExist(err error) bool + + // Delete removes all data in the backend. + Delete(ctx context.Context) error } // FileInfo is returned by Stat() and contains information about a file in the From 67193e3debf47c1256a9f611b06f9c53f9e5add2 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 14 Oct 2017 14:04:14 +0200 Subject: [PATCH 2/8] Improve error message when creating lock failed --- cmd/restic/lock.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/restic/lock.go b/cmd/restic/lock.go index c147a9cb5..d0f44ee77 100644 --- a/cmd/restic/lock.go +++ b/cmd/restic/lock.go @@ -8,6 +8,7 @@ import ( "time" "github.com/restic/restic/internal/debug" + "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" ) @@ -35,7 +36,7 @@ func lockRepository(repo *repository.Repository, exclusive bool) (*restic.Lock, lock, err := lockFn(context.TODO(), repo) if err != nil { - return nil, err + return nil, errors.Fatalf("unable to create lock in backend: %v", err) } debug.Log("create lock %p (exclusive %v)", lock, exclusive) From 897c923cc954518fa4c45224ac3e99296ae9340b Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 14 Oct 2017 14:51:00 +0200 Subject: [PATCH 3/8] Retry failed backend requests --- cmd/restic/global.go | 5 ++ internal/backend/backend_error.go | 73 +++++++++++++++++++++++++++++ internal/backend/backend_retry.go | 77 +++++++++++++++++++++++++++++++ 3 files changed, 155 insertions(+) create mode 100644 internal/backend/backend_error.go create mode 100644 internal/backend/backend_retry.go diff --git a/cmd/restic/global.go b/cmd/restic/global.go index 631de553f..70d031289 100644 --- a/cmd/restic/global.go +++ b/cmd/restic/global.go @@ -9,6 +9,7 @@ import ( "runtime" "strings" "syscall" + "time" "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/azure" @@ -328,6 +329,10 @@ func OpenRepository(opts GlobalOptions) (*repository.Repository, error) { be = limiter.LimitBackend(be, limiter.NewStaticLimiter(opts.LimitUploadKb, opts.LimitDownloadKb)) } + be = backend.NewRetryBackend(be, 10, func(msg string, err error, d time.Duration) { + Warnf("%v returned error, retrying after %v: %v\n", msg, d, err) + }) + s := repository.New(be) opts.password, err = ReadPassword(opts, "enter password for repository: ") diff --git a/internal/backend/backend_error.go b/internal/backend/backend_error.go new file mode 100644 index 000000000..2c9a616cc --- /dev/null +++ b/internal/backend/backend_error.go @@ -0,0 +1,73 @@ +package backend + +import ( + "context" + "io" + "math/rand" + "sync" + + "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/restic" +) + +// ErrorBackend is used to induce errors into various function calls and test +// the retry functions. +type ErrorBackend struct { + FailSave float32 + FailLoad float32 + FailStat float32 + restic.Backend + + r *rand.Rand + m sync.Mutex +} + +// statically ensure that ErrorBackend implements restic.Backend. +var _ restic.Backend = &ErrorBackend{} + +// NewErrorBackend wraps be with a backend that returns errors according to +// given probabilities. +func NewErrorBackend(be restic.Backend, seed int64) *ErrorBackend { + return &ErrorBackend{ + Backend: be, + r: rand.New(rand.NewSource(seed)), + } +} + +func (be *ErrorBackend) fail(p float32) bool { + be.m.Lock() + v := be.r.Float32() + be.m.Unlock() + + return v < p +} + +// Save stores the data in the backend under the given handle. +func (be *ErrorBackend) Save(ctx context.Context, h restic.Handle, rd io.Reader) error { + if be.fail(be.FailSave) { + return errors.Errorf("Save(%v) random error induced", h) + } + + return be.Backend.Save(ctx, h, rd) +} + +// Load returns a reader that yields the contents of the file at h at the +// given offset. If length is larger than zero, only a portion of the file +// is returned. rd must be closed after use. If an error is returned, the +// ReadCloser must be nil. +func (be *ErrorBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { + if be.fail(be.FailLoad) { + return nil, errors.Errorf("Load(%v, %v, %v) random error induced", h, length, offset) + } + + return be.Backend.Load(ctx, h, length, offset) +} + +// Stat returns information about the File identified by h. +func (be *ErrorBackend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) { + if be.fail(be.FailLoad) { + return restic.FileInfo{}, errors.Errorf("Stat(%v) random error induced", h) + } + + return be.Stat(ctx, h) +} diff --git a/internal/backend/backend_retry.go b/internal/backend/backend_retry.go new file mode 100644 index 000000000..66db447ca --- /dev/null +++ b/internal/backend/backend_retry.go @@ -0,0 +1,77 @@ +package backend + +import ( + "context" + "fmt" + "io" + "time" + + "github.com/cenkalti/backoff" + "github.com/restic/restic/internal/restic" +) + +// RetryBackend retries operations on the backend in case of an error with a +// backoff. +type RetryBackend struct { + restic.Backend + MaxTries int + Report func(string, error, time.Duration) +} + +// statically ensure that RetryBackend implements restic.Backend. +var _ restic.Backend = &RetryBackend{} + +// NewRetryBackend wraps be with a backend that retries operations after a +// backoff. report is called with a description and the error, if one occurred. +func NewRetryBackend(be restic.Backend, maxTries int, report func(string, error, time.Duration)) *RetryBackend { + return &RetryBackend{ + Backend: be, + MaxTries: maxTries, + Report: report, + } +} + +func (be *RetryBackend) retry(msg string, f func() error) error { + return backoff.RetryNotify(f, + backoff.WithMaxTries(backoff.NewExponentialBackOff(), uint64(be.MaxTries)), + func(err error, d time.Duration) { + if be.Report != nil { + be.Report(msg, err, d) + } + }, + ) +} + +// Save stores the data in the backend under the given handle. +func (be *RetryBackend) Save(ctx context.Context, h restic.Handle, rd io.Reader) error { + return be.retry(fmt.Sprintf("Save(%v)", h), func() error { + return be.Backend.Save(ctx, h, rd) + }) +} + +// Load returns a reader that yields the contents of the file at h at the +// given offset. If length is larger than zero, only a portion of the file +// is returned. rd must be closed after use. If an error is returned, the +// ReadCloser must be nil. +func (be *RetryBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64) (rd io.ReadCloser, err error) { + err = be.retry(fmt.Sprintf("Load(%v, %v, %v)", h, length, offset), + func() error { + var innerError error + rd, innerError = be.Backend.Load(ctx, h, length, offset) + + return innerError + }) + return rd, err +} + +// Stat returns information about the File identified by h. +func (be *RetryBackend) Stat(ctx context.Context, h restic.Handle) (fi restic.FileInfo, err error) { + err = be.retry(fmt.Sprintf("Stat(%v)", h), + func() error { + var innerError error + fi, innerError = be.Backend.Stat(ctx, h) + + return innerError + }) + return fi, err +} From 3eea555155baf13a6686d2f47490f2c113a21937 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 14 Oct 2017 15:07:19 +0200 Subject: [PATCH 4/8] Add dependency cenkalti/backoff --- Gopkg.lock | 8 +- vendor/github.com/cenkalti/backoff/.gitignore | 22 +++ .../github.com/cenkalti/backoff/.travis.yml | 9 + vendor/github.com/cenkalti/backoff/LICENSE | 20 +++ vendor/github.com/cenkalti/backoff/README.md | 30 ++++ vendor/github.com/cenkalti/backoff/backoff.go | 66 ++++++++ .../cenkalti/backoff/backoff_test.go | 27 +++ vendor/github.com/cenkalti/backoff/context.go | 60 +++++++ .../cenkalti/backoff/context_test.go | 26 +++ .../cenkalti/backoff/example_test.go | 73 ++++++++ .../cenkalti/backoff/exponential.go | 156 ++++++++++++++++++ .../cenkalti/backoff/exponential_test.go | 108 ++++++++++++ vendor/github.com/cenkalti/backoff/retry.go | 78 +++++++++ .../github.com/cenkalti/backoff/retry_test.go | 99 +++++++++++ vendor/github.com/cenkalti/backoff/ticker.go | 81 +++++++++ .../cenkalti/backoff/ticker_test.go | 94 +++++++++++ vendor/github.com/cenkalti/backoff/tries.go | 35 ++++ .../github.com/cenkalti/backoff/tries_test.go | 55 ++++++ 18 files changed, 1046 insertions(+), 1 deletion(-) create mode 100644 vendor/github.com/cenkalti/backoff/.gitignore create mode 100644 vendor/github.com/cenkalti/backoff/.travis.yml create mode 100644 vendor/github.com/cenkalti/backoff/LICENSE create mode 100644 vendor/github.com/cenkalti/backoff/README.md create mode 100644 vendor/github.com/cenkalti/backoff/backoff.go create mode 100644 vendor/github.com/cenkalti/backoff/backoff_test.go create mode 100644 vendor/github.com/cenkalti/backoff/context.go create mode 100644 vendor/github.com/cenkalti/backoff/context_test.go create mode 100644 vendor/github.com/cenkalti/backoff/example_test.go create mode 100644 vendor/github.com/cenkalti/backoff/exponential.go create mode 100644 vendor/github.com/cenkalti/backoff/exponential_test.go create mode 100644 vendor/github.com/cenkalti/backoff/retry.go create mode 100644 vendor/github.com/cenkalti/backoff/retry_test.go create mode 100644 vendor/github.com/cenkalti/backoff/ticker.go create mode 100644 vendor/github.com/cenkalti/backoff/ticker_test.go create mode 100644 vendor/github.com/cenkalti/backoff/tries.go create mode 100644 vendor/github.com/cenkalti/backoff/tries_test.go diff --git a/Gopkg.lock b/Gopkg.lock index 37f0ebe7a..a7b7f672f 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -25,6 +25,12 @@ revision = "f6be1abbb5abd0517522f850dd785990d373da7e" version = "v8.4.0" +[[projects]] + name = "github.com/cenkalti/backoff" + packages = ["."] + revision = "61153c768f31ee5f130071d08fc82b85208528de" + version = "v1.1.0" + [[projects]] name = "github.com/cpuguy83/go-md2man" packages = ["md2man"] @@ -208,6 +214,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "1cd40a4f4ee9009890482c6076d6725360d8b4475a8106cc5150d825989ba163" + inputs-digest = "ea711bd1a9bfc8902b973a4de3a840f42536b9091fd8558980f44d6ca1622227" solver-name = "gps-cdcl" solver-version = 1 diff --git a/vendor/github.com/cenkalti/backoff/.gitignore b/vendor/github.com/cenkalti/backoff/.gitignore new file mode 100644 index 000000000..00268614f --- /dev/null +++ b/vendor/github.com/cenkalti/backoff/.gitignore @@ -0,0 +1,22 @@ +# Compiled Object files, Static and Dynamic libs (Shared Objects) +*.o +*.a +*.so + +# Folders +_obj +_test + +# Architecture specific extensions/prefixes +*.[568vq] +[568vq].out + +*.cgo1.go +*.cgo2.c +_cgo_defun.c +_cgo_gotypes.go +_cgo_export.* + +_testmain.go + +*.exe diff --git a/vendor/github.com/cenkalti/backoff/.travis.yml b/vendor/github.com/cenkalti/backoff/.travis.yml new file mode 100644 index 000000000..1040404bf --- /dev/null +++ b/vendor/github.com/cenkalti/backoff/.travis.yml @@ -0,0 +1,9 @@ +language: go +go: + - 1.3.3 + - tip +before_install: + - go get github.com/mattn/goveralls + - go get golang.org/x/tools/cmd/cover +script: + - $HOME/gopath/bin/goveralls -service=travis-ci diff --git a/vendor/github.com/cenkalti/backoff/LICENSE b/vendor/github.com/cenkalti/backoff/LICENSE new file mode 100644 index 000000000..89b817996 --- /dev/null +++ b/vendor/github.com/cenkalti/backoff/LICENSE @@ -0,0 +1,20 @@ +The MIT License (MIT) + +Copyright (c) 2014 Cenk Altı + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/cenkalti/backoff/README.md b/vendor/github.com/cenkalti/backoff/README.md new file mode 100644 index 000000000..13b347fb9 --- /dev/null +++ b/vendor/github.com/cenkalti/backoff/README.md @@ -0,0 +1,30 @@ +# Exponential Backoff [![GoDoc][godoc image]][godoc] [![Build Status][travis image]][travis] [![Coverage Status][coveralls image]][coveralls] + +This is a Go port of the exponential backoff algorithm from [Google's HTTP Client Library for Java][google-http-java-client]. + +[Exponential backoff][exponential backoff wiki] +is an algorithm that uses feedback to multiplicatively decrease the rate of some process, +in order to gradually find an acceptable rate. +The retries exponentially increase and stop increasing when a certain threshold is met. + +## Usage + +See https://godoc.org/github.com/cenkalti/backoff#pkg-examples + +## Contributing + +* I would like to keep this library as small as possible. +* Please don't send a PR without opening an issue and discussing it first. +* If proposed change is not a common use case, I will probably not accept it. + +[godoc]: https://godoc.org/github.com/cenkalti/backoff +[godoc image]: https://godoc.org/github.com/cenkalti/backoff?status.png +[travis]: https://travis-ci.org/cenkalti/backoff +[travis image]: https://travis-ci.org/cenkalti/backoff.png?branch=master +[coveralls]: https://coveralls.io/github/cenkalti/backoff?branch=master +[coveralls image]: https://coveralls.io/repos/github/cenkalti/backoff/badge.svg?branch=master + +[google-http-java-client]: https://github.com/google/google-http-java-client +[exponential backoff wiki]: http://en.wikipedia.org/wiki/Exponential_backoff + +[advanced example]: https://godoc.org/github.com/cenkalti/backoff#example_ diff --git a/vendor/github.com/cenkalti/backoff/backoff.go b/vendor/github.com/cenkalti/backoff/backoff.go new file mode 100644 index 000000000..2102c5f2d --- /dev/null +++ b/vendor/github.com/cenkalti/backoff/backoff.go @@ -0,0 +1,66 @@ +// Package backoff implements backoff algorithms for retrying operations. +// +// Use Retry function for retrying operations that may fail. +// If Retry does not meet your needs, +// copy/paste the function into your project and modify as you wish. +// +// There is also Ticker type similar to time.Ticker. +// You can use it if you need to work with channels. +// +// See Examples section below for usage examples. +package backoff + +import "time" + +// BackOff is a backoff policy for retrying an operation. +type BackOff interface { + // NextBackOff returns the duration to wait before retrying the operation, + // or backoff.Stop to indicate that no more retries should be made. + // + // Example usage: + // + // duration := backoff.NextBackOff(); + // if (duration == backoff.Stop) { + // // Do not retry operation. + // } else { + // // Sleep for duration and retry operation. + // } + // + NextBackOff() time.Duration + + // Reset to initial state. + Reset() +} + +// Stop indicates that no more retries should be made for use in NextBackOff(). +const Stop time.Duration = -1 + +// ZeroBackOff is a fixed backoff policy whose backoff time is always zero, +// meaning that the operation is retried immediately without waiting, indefinitely. +type ZeroBackOff struct{} + +func (b *ZeroBackOff) Reset() {} + +func (b *ZeroBackOff) NextBackOff() time.Duration { return 0 } + +// StopBackOff is a fixed backoff policy that always returns backoff.Stop for +// NextBackOff(), meaning that the operation should never be retried. +type StopBackOff struct{} + +func (b *StopBackOff) Reset() {} + +func (b *StopBackOff) NextBackOff() time.Duration { return Stop } + +// ConstantBackOff is a backoff policy that always returns the same backoff delay. +// This is in contrast to an exponential backoff policy, +// which returns a delay that grows longer as you call NextBackOff() over and over again. +type ConstantBackOff struct { + Interval time.Duration +} + +func (b *ConstantBackOff) Reset() {} +func (b *ConstantBackOff) NextBackOff() time.Duration { return b.Interval } + +func NewConstantBackOff(d time.Duration) *ConstantBackOff { + return &ConstantBackOff{Interval: d} +} diff --git a/vendor/github.com/cenkalti/backoff/backoff_test.go b/vendor/github.com/cenkalti/backoff/backoff_test.go new file mode 100644 index 000000000..91f27c4f1 --- /dev/null +++ b/vendor/github.com/cenkalti/backoff/backoff_test.go @@ -0,0 +1,27 @@ +package backoff + +import ( + "testing" + "time" +) + +func TestNextBackOffMillis(t *testing.T) { + subtestNextBackOff(t, 0, new(ZeroBackOff)) + subtestNextBackOff(t, Stop, new(StopBackOff)) +} + +func subtestNextBackOff(t *testing.T, expectedValue time.Duration, backOffPolicy BackOff) { + for i := 0; i < 10; i++ { + next := backOffPolicy.NextBackOff() + if next != expectedValue { + t.Errorf("got: %d expected: %d", next, expectedValue) + } + } +} + +func TestConstantBackOff(t *testing.T) { + backoff := NewConstantBackOff(time.Second) + if backoff.NextBackOff() != time.Second { + t.Error("invalid interval") + } +} diff --git a/vendor/github.com/cenkalti/backoff/context.go b/vendor/github.com/cenkalti/backoff/context.go new file mode 100644 index 000000000..5d1570925 --- /dev/null +++ b/vendor/github.com/cenkalti/backoff/context.go @@ -0,0 +1,60 @@ +package backoff + +import ( + "time" + + "golang.org/x/net/context" +) + +// BackOffContext is a backoff policy that stops retrying after the context +// is canceled. +type BackOffContext interface { + BackOff + Context() context.Context +} + +type backOffContext struct { + BackOff + ctx context.Context +} + +// WithContext returns a BackOffContext with context ctx +// +// ctx must not be nil +func WithContext(b BackOff, ctx context.Context) BackOffContext { + if ctx == nil { + panic("nil context") + } + + if b, ok := b.(*backOffContext); ok { + return &backOffContext{ + BackOff: b.BackOff, + ctx: ctx, + } + } + + return &backOffContext{ + BackOff: b, + ctx: ctx, + } +} + +func ensureContext(b BackOff) BackOffContext { + if cb, ok := b.(BackOffContext); ok { + return cb + } + return WithContext(b, context.Background()) +} + +func (b *backOffContext) Context() context.Context { + return b.ctx +} + +func (b *backOffContext) NextBackOff() time.Duration { + select { + case <-b.Context().Done(): + return Stop + default: + return b.BackOff.NextBackOff() + } +} diff --git a/vendor/github.com/cenkalti/backoff/context_test.go b/vendor/github.com/cenkalti/backoff/context_test.go new file mode 100644 index 000000000..993fa6149 --- /dev/null +++ b/vendor/github.com/cenkalti/backoff/context_test.go @@ -0,0 +1,26 @@ +package backoff + +import ( + "testing" + "time" + + "golang.org/x/net/context" +) + +func TestContext(t *testing.T) { + b := NewConstantBackOff(time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + cb := WithContext(b, ctx) + + if cb.Context() != ctx { + t.Error("invalid context") + } + + cancel() + + if cb.NextBackOff() != Stop { + t.Error("invalid next back off") + } +} diff --git a/vendor/github.com/cenkalti/backoff/example_test.go b/vendor/github.com/cenkalti/backoff/example_test.go new file mode 100644 index 000000000..d97a8db8e --- /dev/null +++ b/vendor/github.com/cenkalti/backoff/example_test.go @@ -0,0 +1,73 @@ +package backoff + +import ( + "log" + + "golang.org/x/net/context" +) + +func ExampleRetry() { + // An operation that may fail. + operation := func() error { + return nil // or an error + } + + err := Retry(operation, NewExponentialBackOff()) + if err != nil { + // Handle error. + return + } + + // Operation is successful. +} + +func ExampleRetryContext() { + // A context + ctx := context.Background() + + // An operation that may fail. + operation := func() error { + return nil // or an error + } + + b := WithContext(NewExponentialBackOff(), ctx) + + err := Retry(operation, b) + if err != nil { + // Handle error. + return + } + + // Operation is successful. +} + +func ExampleTicker() { + // An operation that may fail. + operation := func() error { + return nil // or an error + } + + ticker := NewTicker(NewExponentialBackOff()) + + var err error + + // Ticks will continue to arrive when the previous operation is still running, + // so operations that take a while to fail could run in quick succession. + for _ = range ticker.C { + if err = operation(); err != nil { + log.Println(err, "will retry...") + continue + } + + ticker.Stop() + break + } + + if err != nil { + // Operation has failed. + return + } + + // Operation is successful. + return +} diff --git a/vendor/github.com/cenkalti/backoff/exponential.go b/vendor/github.com/cenkalti/backoff/exponential.go new file mode 100644 index 000000000..9a6addf07 --- /dev/null +++ b/vendor/github.com/cenkalti/backoff/exponential.go @@ -0,0 +1,156 @@ +package backoff + +import ( + "math/rand" + "time" +) + +/* +ExponentialBackOff is a backoff implementation that increases the backoff +period for each retry attempt using a randomization function that grows exponentially. + +NextBackOff() is calculated using the following formula: + + randomized interval = + RetryInterval * (random value in range [1 - RandomizationFactor, 1 + RandomizationFactor]) + +In other words NextBackOff() will range between the randomization factor +percentage below and above the retry interval. + +For example, given the following parameters: + + RetryInterval = 2 + RandomizationFactor = 0.5 + Multiplier = 2 + +the actual backoff period used in the next retry attempt will range between 1 and 3 seconds, +multiplied by the exponential, that is, between 2 and 6 seconds. + +Note: MaxInterval caps the RetryInterval and not the randomized interval. + +If the time elapsed since an ExponentialBackOff instance is created goes past the +MaxElapsedTime, then the method NextBackOff() starts returning backoff.Stop. + +The elapsed time can be reset by calling Reset(). + +Example: Given the following default arguments, for 10 tries the sequence will be, +and assuming we go over the MaxElapsedTime on the 10th try: + + Request # RetryInterval (seconds) Randomized Interval (seconds) + + 1 0.5 [0.25, 0.75] + 2 0.75 [0.375, 1.125] + 3 1.125 [0.562, 1.687] + 4 1.687 [0.8435, 2.53] + 5 2.53 [1.265, 3.795] + 6 3.795 [1.897, 5.692] + 7 5.692 [2.846, 8.538] + 8 8.538 [4.269, 12.807] + 9 12.807 [6.403, 19.210] + 10 19.210 backoff.Stop + +Note: Implementation is not thread-safe. +*/ +type ExponentialBackOff struct { + InitialInterval time.Duration + RandomizationFactor float64 + Multiplier float64 + MaxInterval time.Duration + // After MaxElapsedTime the ExponentialBackOff stops. + // It never stops if MaxElapsedTime == 0. + MaxElapsedTime time.Duration + Clock Clock + + currentInterval time.Duration + startTime time.Time + random *rand.Rand +} + +// Clock is an interface that returns current time for BackOff. +type Clock interface { + Now() time.Time +} + +// Default values for ExponentialBackOff. +const ( + DefaultInitialInterval = 500 * time.Millisecond + DefaultRandomizationFactor = 0.5 + DefaultMultiplier = 1.5 + DefaultMaxInterval = 60 * time.Second + DefaultMaxElapsedTime = 15 * time.Minute +) + +// NewExponentialBackOff creates an instance of ExponentialBackOff using default values. +func NewExponentialBackOff() *ExponentialBackOff { + b := &ExponentialBackOff{ + InitialInterval: DefaultInitialInterval, + RandomizationFactor: DefaultRandomizationFactor, + Multiplier: DefaultMultiplier, + MaxInterval: DefaultMaxInterval, + MaxElapsedTime: DefaultMaxElapsedTime, + Clock: SystemClock, + random: rand.New(rand.NewSource(time.Now().UnixNano())), + } + b.Reset() + return b +} + +type systemClock struct{} + +func (t systemClock) Now() time.Time { + return time.Now() +} + +// SystemClock implements Clock interface that uses time.Now(). +var SystemClock = systemClock{} + +// Reset the interval back to the initial retry interval and restarts the timer. +func (b *ExponentialBackOff) Reset() { + b.currentInterval = b.InitialInterval + b.startTime = b.Clock.Now() +} + +// NextBackOff calculates the next backoff interval using the formula: +// Randomized interval = RetryInterval +/- (RandomizationFactor * RetryInterval) +func (b *ExponentialBackOff) NextBackOff() time.Duration { + // Make sure we have not gone over the maximum elapsed time. + if b.MaxElapsedTime != 0 && b.GetElapsedTime() > b.MaxElapsedTime { + return Stop + } + defer b.incrementCurrentInterval() + if b.random == nil { + b.random = rand.New(rand.NewSource(time.Now().UnixNano())) + } + return getRandomValueFromInterval(b.RandomizationFactor, b.random.Float64(), b.currentInterval) +} + +// GetElapsedTime returns the elapsed time since an ExponentialBackOff instance +// is created and is reset when Reset() is called. +// +// The elapsed time is computed using time.Now().UnixNano(). +func (b *ExponentialBackOff) GetElapsedTime() time.Duration { + return b.Clock.Now().Sub(b.startTime) +} + +// Increments the current interval by multiplying it with the multiplier. +func (b *ExponentialBackOff) incrementCurrentInterval() { + // Check for overflow, if overflow is detected set the current interval to the max interval. + if float64(b.currentInterval) >= float64(b.MaxInterval)/b.Multiplier { + b.currentInterval = b.MaxInterval + } else { + b.currentInterval = time.Duration(float64(b.currentInterval) * b.Multiplier) + } +} + +// Returns a random value from the following interval: +// [randomizationFactor * currentInterval, randomizationFactor * currentInterval]. +func getRandomValueFromInterval(randomizationFactor, random float64, currentInterval time.Duration) time.Duration { + var delta = randomizationFactor * float64(currentInterval) + var minInterval = float64(currentInterval) - delta + var maxInterval = float64(currentInterval) + delta + + // Get a random value from the range [minInterval, maxInterval]. + // The formula used below has a +1 because if the minInterval is 1 and the maxInterval is 3 then + // we want a 33% chance for selecting either 1, 2 or 3. + return time.Duration(minInterval + (random * (maxInterval - minInterval + 1))) +} diff --git a/vendor/github.com/cenkalti/backoff/exponential_test.go b/vendor/github.com/cenkalti/backoff/exponential_test.go new file mode 100644 index 000000000..11b95e4f6 --- /dev/null +++ b/vendor/github.com/cenkalti/backoff/exponential_test.go @@ -0,0 +1,108 @@ +package backoff + +import ( + "math" + "testing" + "time" +) + +func TestBackOff(t *testing.T) { + var ( + testInitialInterval = 500 * time.Millisecond + testRandomizationFactor = 0.1 + testMultiplier = 2.0 + testMaxInterval = 5 * time.Second + testMaxElapsedTime = 15 * time.Minute + ) + + exp := NewExponentialBackOff() + exp.InitialInterval = testInitialInterval + exp.RandomizationFactor = testRandomizationFactor + exp.Multiplier = testMultiplier + exp.MaxInterval = testMaxInterval + exp.MaxElapsedTime = testMaxElapsedTime + exp.Reset() + + var expectedResults = []time.Duration{500, 1000, 2000, 4000, 5000, 5000, 5000, 5000, 5000, 5000} + for i, d := range expectedResults { + expectedResults[i] = d * time.Millisecond + } + + for _, expected := range expectedResults { + assertEquals(t, expected, exp.currentInterval) + // Assert that the next backoff falls in the expected range. + var minInterval = expected - time.Duration(testRandomizationFactor*float64(expected)) + var maxInterval = expected + time.Duration(testRandomizationFactor*float64(expected)) + var actualInterval = exp.NextBackOff() + if !(minInterval <= actualInterval && actualInterval <= maxInterval) { + t.Error("error") + } + } +} + +func TestGetRandomizedInterval(t *testing.T) { + // 33% chance of being 1. + assertEquals(t, 1, getRandomValueFromInterval(0.5, 0, 2)) + assertEquals(t, 1, getRandomValueFromInterval(0.5, 0.33, 2)) + // 33% chance of being 2. + assertEquals(t, 2, getRandomValueFromInterval(0.5, 0.34, 2)) + assertEquals(t, 2, getRandomValueFromInterval(0.5, 0.66, 2)) + // 33% chance of being 3. + assertEquals(t, 3, getRandomValueFromInterval(0.5, 0.67, 2)) + assertEquals(t, 3, getRandomValueFromInterval(0.5, 0.99, 2)) +} + +type TestClock struct { + i time.Duration + start time.Time +} + +func (c *TestClock) Now() time.Time { + t := c.start.Add(c.i) + c.i += time.Second + return t +} + +func TestGetElapsedTime(t *testing.T) { + var exp = NewExponentialBackOff() + exp.Clock = &TestClock{} + exp.Reset() + + var elapsedTime = exp.GetElapsedTime() + if elapsedTime != time.Second { + t.Errorf("elapsedTime=%d", elapsedTime) + } +} + +func TestMaxElapsedTime(t *testing.T) { + var exp = NewExponentialBackOff() + exp.Clock = &TestClock{start: time.Time{}.Add(10000 * time.Second)} + // Change the currentElapsedTime to be 0 ensuring that the elapsed time will be greater + // than the max elapsed time. + exp.startTime = time.Time{} + assertEquals(t, Stop, exp.NextBackOff()) +} + +func TestBackOffOverflow(t *testing.T) { + var ( + testInitialInterval time.Duration = math.MaxInt64 / 2 + testMaxInterval time.Duration = math.MaxInt64 + testMultiplier = 2.1 + ) + + exp := NewExponentialBackOff() + exp.InitialInterval = testInitialInterval + exp.Multiplier = testMultiplier + exp.MaxInterval = testMaxInterval + exp.Reset() + + exp.NextBackOff() + // Assert that when an overflow is possible the current varerval time.Duration is set to the max varerval time.Duration . + assertEquals(t, testMaxInterval, exp.currentInterval) +} + +func assertEquals(t *testing.T, expected, value time.Duration) { + if expected != value { + t.Errorf("got: %d, expected: %d", value, expected) + } +} diff --git a/vendor/github.com/cenkalti/backoff/retry.go b/vendor/github.com/cenkalti/backoff/retry.go new file mode 100644 index 000000000..5dbd825b5 --- /dev/null +++ b/vendor/github.com/cenkalti/backoff/retry.go @@ -0,0 +1,78 @@ +package backoff + +import "time" + +// An Operation is executing by Retry() or RetryNotify(). +// The operation will be retried using a backoff policy if it returns an error. +type Operation func() error + +// Notify is a notify-on-error function. It receives an operation error and +// backoff delay if the operation failed (with an error). +// +// NOTE that if the backoff policy stated to stop retrying, +// the notify function isn't called. +type Notify func(error, time.Duration) + +// Retry the operation o until it does not return error or BackOff stops. +// o is guaranteed to be run at least once. +// It is the caller's responsibility to reset b after Retry returns. +// +// If o returns a *PermanentError, the operation is not retried, and the +// wrapped error is returned. +// +// Retry sleeps the goroutine for the duration returned by BackOff after a +// failed operation returns. +func Retry(o Operation, b BackOff) error { return RetryNotify(o, b, nil) } + +// RetryNotify calls notify function with the error and wait duration +// for each failed attempt before sleep. +func RetryNotify(operation Operation, b BackOff, notify Notify) error { + var err error + var next time.Duration + + cb := ensureContext(b) + + b.Reset() + for { + if err = operation(); err == nil { + return nil + } + + if permanent, ok := err.(*PermanentError); ok { + return permanent.Err + } + + if next = b.NextBackOff(); next == Stop { + return err + } + + if notify != nil { + notify(err, next) + } + + t := time.NewTimer(next) + + select { + case <-cb.Context().Done(): + t.Stop() + return err + case <-t.C: + } + } +} + +// PermanentError signals that the operation should not be retried. +type PermanentError struct { + Err error +} + +func (e *PermanentError) Error() string { + return e.Err.Error() +} + +// Permanent wraps the given err in a *PermanentError. +func Permanent(err error) *PermanentError { + return &PermanentError{ + Err: err, + } +} diff --git a/vendor/github.com/cenkalti/backoff/retry_test.go b/vendor/github.com/cenkalti/backoff/retry_test.go new file mode 100644 index 000000000..5af288897 --- /dev/null +++ b/vendor/github.com/cenkalti/backoff/retry_test.go @@ -0,0 +1,99 @@ +package backoff + +import ( + "errors" + "fmt" + "log" + "testing" + "time" + + "golang.org/x/net/context" +) + +func TestRetry(t *testing.T) { + const successOn = 3 + var i = 0 + + // This function is successful on "successOn" calls. + f := func() error { + i++ + log.Printf("function is called %d. time\n", i) + + if i == successOn { + log.Println("OK") + return nil + } + + log.Println("error") + return errors.New("error") + } + + err := Retry(f, NewExponentialBackOff()) + if err != nil { + t.Errorf("unexpected error: %s", err.Error()) + } + if i != successOn { + t.Errorf("invalid number of retries: %d", i) + } +} + +func TestRetryContext(t *testing.T) { + var cancelOn = 3 + var i = 0 + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // This function cancels context on "cancelOn" calls. + f := func() error { + i++ + log.Printf("function is called %d. time\n", i) + + // cancelling the context in the operation function is not a typical + // use-case, however it allows to get predictable test results. + if i == cancelOn { + cancel() + } + + log.Println("error") + return fmt.Errorf("error (%d)", i) + } + + err := Retry(f, WithContext(NewConstantBackOff(time.Millisecond), ctx)) + if err == nil { + t.Errorf("error is unexpectedly nil") + } + if err.Error() != "error (3)" { + t.Errorf("unexpected error: %s", err.Error()) + } + if i != cancelOn { + t.Errorf("invalid number of retries: %d", i) + } +} + +func TestRetryPermenent(t *testing.T) { + const permanentOn = 3 + var i = 0 + + // This function fails permanently after permanentOn tries + f := func() error { + i++ + log.Printf("function is called %d. time\n", i) + + if i == permanentOn { + log.Println("permanent error") + return Permanent(errors.New("permanent error")) + } + + log.Println("error") + return errors.New("error") + } + + err := Retry(f, NewExponentialBackOff()) + if err == nil || err.Error() != "permanent error" { + t.Errorf("unexpected error: %s", err) + } + if i != permanentOn { + t.Errorf("invalid number of retries: %d", i) + } +} diff --git a/vendor/github.com/cenkalti/backoff/ticker.go b/vendor/github.com/cenkalti/backoff/ticker.go new file mode 100644 index 000000000..49a99718d --- /dev/null +++ b/vendor/github.com/cenkalti/backoff/ticker.go @@ -0,0 +1,81 @@ +package backoff + +import ( + "runtime" + "sync" + "time" +) + +// Ticker holds a channel that delivers `ticks' of a clock at times reported by a BackOff. +// +// Ticks will continue to arrive when the previous operation is still running, +// so operations that take a while to fail could run in quick succession. +type Ticker struct { + C <-chan time.Time + c chan time.Time + b BackOffContext + stop chan struct{} + stopOnce sync.Once +} + +// NewTicker returns a new Ticker containing a channel that will send the time at times +// specified by the BackOff argument. Ticker is guaranteed to tick at least once. +// The channel is closed when Stop method is called or BackOff stops. +func NewTicker(b BackOff) *Ticker { + c := make(chan time.Time) + t := &Ticker{ + C: c, + c: c, + b: ensureContext(b), + stop: make(chan struct{}), + } + go t.run() + runtime.SetFinalizer(t, (*Ticker).Stop) + return t +} + +// Stop turns off a ticker. After Stop, no more ticks will be sent. +func (t *Ticker) Stop() { + t.stopOnce.Do(func() { close(t.stop) }) +} + +func (t *Ticker) run() { + c := t.c + defer close(c) + t.b.Reset() + + // Ticker is guaranteed to tick at least once. + afterC := t.send(time.Now()) + + for { + if afterC == nil { + return + } + + select { + case tick := <-afterC: + afterC = t.send(tick) + case <-t.stop: + t.c = nil // Prevent future ticks from being sent to the channel. + return + case <-t.b.Context().Done(): + return + } + } +} + +func (t *Ticker) send(tick time.Time) <-chan time.Time { + select { + case t.c <- tick: + case <-t.stop: + return nil + } + + next := t.b.NextBackOff() + if next == Stop { + t.Stop() + return nil + } + + return time.After(next) +} diff --git a/vendor/github.com/cenkalti/backoff/ticker_test.go b/vendor/github.com/cenkalti/backoff/ticker_test.go new file mode 100644 index 000000000..085828cca --- /dev/null +++ b/vendor/github.com/cenkalti/backoff/ticker_test.go @@ -0,0 +1,94 @@ +package backoff + +import ( + "errors" + "fmt" + "log" + "testing" + "time" + + "golang.org/x/net/context" +) + +func TestTicker(t *testing.T) { + const successOn = 3 + var i = 0 + + // This function is successful on "successOn" calls. + f := func() error { + i++ + log.Printf("function is called %d. time\n", i) + + if i == successOn { + log.Println("OK") + return nil + } + + log.Println("error") + return errors.New("error") + } + + b := NewExponentialBackOff() + ticker := NewTicker(b) + + var err error + for _ = range ticker.C { + if err = f(); err != nil { + t.Log(err) + continue + } + + break + } + if err != nil { + t.Errorf("unexpected error: %s", err.Error()) + } + if i != successOn { + t.Errorf("invalid number of retries: %d", i) + } +} + +func TestTickerContext(t *testing.T) { + const cancelOn = 3 + var i = 0 + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // This function cancels context on "cancelOn" calls. + f := func() error { + i++ + log.Printf("function is called %d. time\n", i) + + // cancelling the context in the operation function is not a typical + // use-case, however it allows to get predictable test results. + if i == cancelOn { + cancel() + } + + log.Println("error") + return fmt.Errorf("error (%d)", i) + } + + b := WithContext(NewConstantBackOff(time.Millisecond), ctx) + ticker := NewTicker(b) + + var err error + for _ = range ticker.C { + if err = f(); err != nil { + t.Log(err) + continue + } + + break + } + if err == nil { + t.Errorf("error is unexpectedly nil") + } + if err.Error() != "error (3)" { + t.Errorf("unexpected error: %s", err.Error()) + } + if i != cancelOn { + t.Errorf("invalid number of retries: %d", i) + } +} diff --git a/vendor/github.com/cenkalti/backoff/tries.go b/vendor/github.com/cenkalti/backoff/tries.go new file mode 100644 index 000000000..d2da7308b --- /dev/null +++ b/vendor/github.com/cenkalti/backoff/tries.go @@ -0,0 +1,35 @@ +package backoff + +import "time" + +/* +WithMaxTries creates a wrapper around another BackOff, which will +return Stop if NextBackOff() has been called too many times since +the last time Reset() was called + +Note: Implementation is not thread-safe. +*/ +func WithMaxTries(b BackOff, max uint64) BackOff { + return &backOffTries{delegate: b, maxTries: max} +} + +type backOffTries struct { + delegate BackOff + maxTries uint64 + numTries uint64 +} + +func (b *backOffTries) NextBackOff() time.Duration { + if b.maxTries > 0 { + if b.maxTries <= b.numTries { + return Stop + } + b.numTries++ + } + return b.delegate.NextBackOff() +} + +func (b *backOffTries) Reset() { + b.numTries = 0 + b.delegate.Reset() +} diff --git a/vendor/github.com/cenkalti/backoff/tries_test.go b/vendor/github.com/cenkalti/backoff/tries_test.go new file mode 100644 index 000000000..bd1021143 --- /dev/null +++ b/vendor/github.com/cenkalti/backoff/tries_test.go @@ -0,0 +1,55 @@ +package backoff + +import ( + "math/rand" + "testing" + "time" +) + +func TestMaxTriesHappy(t *testing.T) { + r := rand.New(rand.NewSource(time.Now().UnixNano())) + max := 17 + r.Intn(13) + bo := WithMaxTries(&ZeroBackOff{}, uint64(max)) + + // Load up the tries count, but reset should clear the record + for ix := 0; ix < max/2; ix++ { + bo.NextBackOff() + } + bo.Reset() + + // Now fill the tries count all the way up + for ix := 0; ix < max; ix++ { + d := bo.NextBackOff() + if d == Stop { + t.Errorf("returned Stop on try %d", ix) + } + } + + // We have now called the BackOff max number of times, we expect + // the next result to be Stop, even if we try it multiple times + for ix := 0; ix < 7; ix++ { + d := bo.NextBackOff() + if d != Stop { + t.Error("invalid next back off") + } + } + + // Reset makes it all work again + bo.Reset() + d := bo.NextBackOff() + if d == Stop { + t.Error("returned Stop after reset") + } + +} + +func TestMaxTriesZero(t *testing.T) { + // It might not make sense, but its okay to send a zero + bo := WithMaxTries(&ZeroBackOff{}, uint64(0)) + for ix := 0; ix < 11; ix++ { + d := bo.NextBackOff() + if d == Stop { + t.Errorf("returned Stop on try %d", ix) + } + } +} From b8af7f63a0c55117371ded5beb6bd68d57997efc Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 14 Oct 2017 15:53:32 +0200 Subject: [PATCH 5/8] backend test: Always remove files for TestList --- internal/backend/test/tests.go | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/internal/backend/test/tests.go b/internal/backend/test/tests.go index e32e2c421..2b4b31b8a 100644 --- a/internal/backend/test/tests.go +++ b/internal/backend/test/tests.go @@ -675,24 +675,21 @@ func (s *Suite) TestBackend(t *testing.T) { t.Fatalf("lists aren't equal, want:\n %v\n got:\n%v\n", IDs, list) } - // remove content if requested - if test.TestCleanupTempDirs { - var handles []restic.Handle - for _, ts := range testStrings { - id, err := restic.ParseID(ts.id) - test.OK(t, err) + var handles []restic.Handle + for _, ts := range testStrings { + id, err := restic.ParseID(ts.id) + test.OK(t, err) - h := restic.Handle{Type: tpe, Name: id.String()} + h := restic.Handle{Type: tpe, Name: id.String()} - found, err := b.Test(context.TODO(), h) - test.OK(t, err) - test.Assert(t, found, fmt.Sprintf("id %q not found", id)) + found, err := b.Test(context.TODO(), h) + test.OK(t, err) + test.Assert(t, found, fmt.Sprintf("id %q not found", id)) - handles = append(handles, h) - } - - test.OK(t, s.delayedRemove(t, b, handles...)) + handles = append(handles, h) } + + test.OK(t, s.delayedRemove(t, b, handles...)) } } From e56370eb5bef81429db79034165b4cfd21a03f7b Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 14 Oct 2017 15:56:38 +0200 Subject: [PATCH 6/8] Remove Deleter interface --- internal/backend/azure/azure_test.go | 6 +----- internal/backend/b2/b2_test.go | 6 +----- internal/backend/gs/gs_test.go | 6 +----- internal/backend/rest/rest.go | 6 +++++- internal/backend/s3/s3_test.go | 6 +----- internal/backend/swift/swift_test.go | 6 +----- internal/repository/repository.go | 6 +----- internal/restic/repository.go | 5 ----- 8 files changed, 11 insertions(+), 36 deletions(-) diff --git a/internal/backend/azure/azure_test.go b/internal/backend/azure/azure_test.go index 326e20183..fe3b3e83c 100644 --- a/internal/backend/azure/azure_test.go +++ b/internal/backend/azure/azure_test.go @@ -76,11 +76,7 @@ func newAzureTestSuite(t testing.TB) *test.Suite { return err } - if err := be.(restic.Deleter).Delete(context.TODO()); err != nil { - return err - } - - return nil + return be.Delete(context.TODO()) }, } } diff --git a/internal/backend/b2/b2_test.go b/internal/backend/b2/b2_test.go index 675a5c864..5784bd9a2 100644 --- a/internal/backend/b2/b2_test.go +++ b/internal/backend/b2/b2_test.go @@ -62,11 +62,7 @@ func newB2TestSuite(t testing.TB) *test.Suite { return err } - if err := be.(restic.Deleter).Delete(context.TODO()); err != nil { - return err - } - - return nil + return be.Delete(context.TODO()) }, } } diff --git a/internal/backend/gs/gs_test.go b/internal/backend/gs/gs_test.go index 739aaacf3..1c55697f3 100644 --- a/internal/backend/gs/gs_test.go +++ b/internal/backend/gs/gs_test.go @@ -69,11 +69,7 @@ func newGSTestSuite(t testing.TB) *test.Suite { return err } - if err := be.(restic.Deleter).Delete(context.TODO()); err != nil { - return err - } - - return nil + return be.Delete(context.TODO()) }, } } diff --git a/internal/backend/rest/rest.go b/internal/backend/rest/rest.go index cd283a8c9..51c8e0c5c 100644 --- a/internal/backend/rest/rest.go +++ b/internal/backend/rest/rest.go @@ -378,5 +378,9 @@ func (b *restBackend) Delete(ctx context.Context) error { } } - return b.Remove(ctx, restic.Handle{Type: restic.ConfigFile}) + err := b.Remove(ctx, restic.Handle{Type: restic.ConfigFile}) + if err != nil && b.IsNotExist(err) { + return nil + } + return err } diff --git a/internal/backend/s3/s3_test.go b/internal/backend/s3/s3_test.go index b9cbc4ef3..fe5e92299 100644 --- a/internal/backend/s3/s3_test.go +++ b/internal/backend/s3/s3_test.go @@ -280,11 +280,7 @@ func newS3TestSuite(t testing.TB) *test.Suite { return err } - if err := be.(restic.Deleter).Delete(context.TODO()); err != nil { - return err - } - - return nil + return be.Delete(context.TODO()) }, } } diff --git a/internal/backend/swift/swift_test.go b/internal/backend/swift/swift_test.go index f4cf28aaf..30a61ea7b 100644 --- a/internal/backend/swift/swift_test.go +++ b/internal/backend/swift/swift_test.go @@ -93,11 +93,7 @@ func newSwiftTestSuite(t testing.TB) *test.Suite { return err } - if err := be.(restic.Deleter).Delete(context.TODO()); err != nil { - return err - } - - return nil + return be.Delete(context.TODO()) }, } } diff --git a/internal/repository/repository.go b/internal/repository/repository.go index e2d870780..161eb8734 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -591,11 +591,7 @@ func (r *Repository) ListPack(ctx context.Context, id restic.ID) ([]restic.Blob, // Delete calls backend.Delete() if implemented, and returns an error // otherwise. func (r *Repository) Delete(ctx context.Context) error { - if b, ok := r.be.(restic.Deleter); ok { - return b.Delete(ctx) - } - - return errors.New("Delete() called for backend that does not implement this method") + return r.be.Delete(ctx) } // Close closes the repository by closing the backend. diff --git a/internal/restic/repository.go b/internal/restic/repository.go index 9be667fa7..6c8cad863 100644 --- a/internal/restic/repository.go +++ b/internal/restic/repository.go @@ -44,11 +44,6 @@ type Repository interface { SaveTree(context.Context, *Tree) (ID, error) } -// Deleter removes all data stored in a backend/repo. -type Deleter interface { - Delete(context.Context) error -} - // Lister allows listing files in a backend. type Lister interface { List(context.Context, FileType) <-chan string From 7fe496f983e324dd128fa64abdbd224b62603d68 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 14 Oct 2017 15:56:46 +0200 Subject: [PATCH 7/8] Ensure TestDelete runs last --- internal/backend/test/tests.go | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/internal/backend/test/tests.go b/internal/backend/test/tests.go index 2b4b31b8a..9e88a12ba 100644 --- a/internal/backend/test/tests.go +++ b/internal/backend/test/tests.go @@ -693,8 +693,8 @@ func (s *Suite) TestBackend(t *testing.T) { } } -// TestDelete tests the Delete function. -func (s *Suite) TestDelete(t *testing.T) { +// TestZZZDelete tests the Delete function. The name ensures that this test is executed last. +func (s *Suite) TestZZZDelete(t *testing.T) { if !test.TestCleanupTempDirs { t.Skipf("not removing backend, TestCleanupTempDirs is false") } @@ -702,12 +702,7 @@ func (s *Suite) TestDelete(t *testing.T) { b := s.open(t) defer s.close(t, b) - be, ok := b.(restic.Deleter) - if !ok { - return - } - - err := be.Delete(context.TODO()) + err := b.Delete(context.TODO()) if err != nil { t.Fatalf("error deleting backend: %+v", err) } From 4a995105a99a000d28fd4b22ed2f62fa67d39094 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 14 Oct 2017 16:08:15 +0200 Subject: [PATCH 8/8] sftp: Fix Delete() --- internal/backend/sftp/sftp.go | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/internal/backend/sftp/sftp.go b/internal/backend/sftp/sftp.go index ef006ce6f..116a1c20e 100644 --- a/internal/backend/sftp/sftp.go +++ b/internal/backend/sftp/sftp.go @@ -500,7 +500,38 @@ func (r *SFTP) Close() error { return nil } +func (r *SFTP) deleteRecursive(name string) error { + entries, err := r.c.ReadDir(name) + if err != nil { + return errors.Wrap(err, "ReadDir") + } + + for _, fi := range entries { + itemName := r.Join(name, fi.Name()) + if fi.IsDir() { + err := r.deleteRecursive(itemName) + if err != nil { + return errors.Wrap(err, "ReadDir") + } + + err = r.c.RemoveDirectory(itemName) + if err != nil { + return errors.Wrap(err, "RemoveDirectory") + } + + continue + } + + err := r.c.Remove(itemName) + if err != nil { + return errors.Wrap(err, "ReadDir") + } + } + + return nil +} + // Delete removes all data in the backend. func (r *SFTP) Delete(context.Context) error { - return r.c.RemoveDirectory(r.p) + return r.deleteRecursive(r.p) }