From e10420553bcc4c0066e64dc9ffa4e0814a3abced Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 15 Oct 2022 22:29:58 +0200 Subject: [PATCH 1/4] speed-up integration tests by reducing the RetryBackend timeout On my machine this decreases the runtime for `./cmd/restic` from 9.5s to 6.5s. --- cmd/restic/integration_helpers_test.go | 2 ++ internal/backend/backend_retry.go | 10 +++++++++- internal/backend/testing.go | 8 ++++++++ 3 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 internal/backend/testing.go diff --git a/cmd/restic/integration_helpers_test.go b/cmd/restic/integration_helpers_test.go index 17a3c29c3..6a4d064a8 100644 --- a/cmd/restic/integration_helpers_test.go +++ b/cmd/restic/integration_helpers_test.go @@ -9,6 +9,7 @@ import ( "runtime" "testing" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/options" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" @@ -171,6 +172,7 @@ func withTestEnvironment(t testing.TB) (env *testEnvironment, cleanup func()) { repository.TestUseLowSecurityKDFParameters(t) restic.TestDisableCheckPolynomial(t) + backend.TestFastRetries(t) tempdir, err := ioutil.TempDir(rtest.TestTempDir, "restic-test-") rtest.OK(t, err) diff --git a/internal/backend/backend_retry.go b/internal/backend/backend_retry.go index 1910bec59..e9a22d75f 100644 --- a/internal/backend/backend_retry.go +++ b/internal/backend/backend_retry.go @@ -55,6 +55,8 @@ func retryNotifyErrorWithSuccess(operation backoff.Operation, b backoff.BackOff, return backoff.RetryNotify(operationWrapper, b, notify) } +var fastRetries = false + func (be *RetryBackend) retry(ctx context.Context, msg string, f func() error) error { // Don't do anything when called with an already cancelled context. There would be // no retries in that case either, so be consistent and abort always. @@ -66,8 +68,14 @@ func (be *RetryBackend) retry(ctx context.Context, msg string, f func() error) e return ctx.Err() } + bo := backoff.NewExponentialBackOff() + if fastRetries { + // speed up integration tests + bo.InitialInterval = 1 * time.Millisecond + } + err := retryNotifyErrorWithSuccess(f, - backoff.WithContext(backoff.WithMaxRetries(backoff.NewExponentialBackOff(), uint64(be.MaxTries)), ctx), + backoff.WithContext(backoff.WithMaxRetries(bo, uint64(be.MaxTries)), ctx), func(err error, d time.Duration) { if be.Report != nil { be.Report(msg, err, d) diff --git a/internal/backend/testing.go b/internal/backend/testing.go new file mode 100644 index 000000000..0d7fa1d76 --- /dev/null +++ b/internal/backend/testing.go @@ -0,0 +1,8 @@ +package backend + +import "testing" + +// TestFastRetries reduces the initial retry delay to 1 millisecond +func TestFastRetries(t testing.TB) { + fastRetries = true +} From c3400d3c55b79e394b68f2748d5cea262737d694 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 15 Oct 2022 23:13:44 +0200 Subject: [PATCH 2/4] backend: speedup RetryBackend tests --- internal/backend/backend_retry_test.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/internal/backend/backend_retry_test.go b/internal/backend/backend_retry_test.go index 7890eab42..aca30f61d 100644 --- a/internal/backend/backend_retry_test.go +++ b/internal/backend/backend_retry_test.go @@ -35,6 +35,7 @@ func TestBackendSaveRetry(t *testing.T) { }, } + TestFastRetries(t) retryBackend := NewRetryBackend(be, 10, nil, nil) data := test.Random(23, 5*1024*1024+11241) @@ -70,6 +71,7 @@ func TestBackendSaveRetryAtomic(t *testing.T) { HasAtomicReplaceFn: func() bool { return true }, } + TestFastRetries(t) retryBackend := NewRetryBackend(be, 10, nil, nil) data := test.Random(23, 5*1024*1024+11241) @@ -103,6 +105,7 @@ func TestBackendListRetry(t *testing.T) { }, } + TestFastRetries(t) retryBackend := NewRetryBackend(be, 10, nil, nil) var listed []string @@ -132,6 +135,7 @@ func TestBackendListRetryErrorFn(t *testing.T) { }, } + TestFastRetries(t) retryBackend := NewRetryBackend(be, 10, nil, nil) var ErrTest = errors.New("test error") @@ -187,6 +191,7 @@ func TestBackendListRetryErrorBackend(t *testing.T) { }, } + TestFastRetries(t) const maxRetries = 2 retryBackend := NewRetryBackend(be, maxRetries, nil, nil) @@ -257,6 +262,7 @@ func TestBackendLoadRetry(t *testing.T) { return failingReader{data: data, limit: limit}, nil } + TestFastRetries(t) retryBackend := NewRetryBackend(be, 10, nil, nil) var buf []byte @@ -276,6 +282,7 @@ func assertIsCanceled(t *testing.T, err error) { func TestBackendCanceledContext(t *testing.T) { // unimplemented mock backend functions return an error by default // check that we received the expected context canceled error instead + TestFastRetries(t) retryBackend := NewRetryBackend(mock.NewBackend(), 2, nil, nil) h := restic.Handle{Type: restic.PackFile, Name: restic.NewRandomID().String()} From 28e1c4574b3f4fbe9bec99847a5022ec9b8ba1a2 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 15 Oct 2022 23:14:33 +0200 Subject: [PATCH 3/4] mem: use cheaper hash for backend --- internal/backend/mem/mem_backend.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/backend/mem/mem_backend.go b/internal/backend/mem/mem_backend.go index 7e8ae5356..4d1e70387 100644 --- a/internal/backend/mem/mem_backend.go +++ b/internal/backend/mem/mem_backend.go @@ -3,13 +3,13 @@ package mem import ( "bytes" "context" - "crypto/md5" "encoding/base64" "hash" "io" "io/ioutil" "sync" + "github.com/cespare/xxhash/v2" "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/sema" "github.com/restic/restic/internal/debug" @@ -266,7 +266,7 @@ func (be *MemoryBackend) Location() string { // Hasher may return a hash function for calculating a content hash for the backend func (be *MemoryBackend) Hasher() hash.Hash { - return md5.New() + return xxhash.New() } // HasAtomicReplace returns whether Save() can atomically replace files From aa39bf3cf606286b899c27c0ddea49a8b9a8755a Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 15 Oct 2022 23:15:03 +0200 Subject: [PATCH 4/4] backend/test: remove duplicate test The test is identical to the tests for the mem backend. --- internal/backend/test/tests_test.go | 66 ----------------------------- 1 file changed, 66 deletions(-) delete mode 100644 internal/backend/test/tests_test.go diff --git a/internal/backend/test/tests_test.go b/internal/backend/test/tests_test.go deleted file mode 100644 index 8e52e3c59..000000000 --- a/internal/backend/test/tests_test.go +++ /dev/null @@ -1,66 +0,0 @@ -package test_test - -import ( - "context" - "testing" - - "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/restic" - - "github.com/restic/restic/internal/backend/mem" - "github.com/restic/restic/internal/backend/test" -) - -type memConfig struct { - be restic.Backend -} - -func newTestSuite(t testing.TB) *test.Suite { - return &test.Suite{ - // NewConfig returns a config for a new temporary backend that will be used in tests. - NewConfig: func() (interface{}, error) { - return &memConfig{}, nil - }, - - // CreateFn is a function that creates a temporary repository for the tests. - Create: func(cfg interface{}) (restic.Backend, error) { - c := cfg.(*memConfig) - if c.be != nil { - ok, err := c.be.Test(context.TODO(), restic.Handle{Type: restic.ConfigFile}) - if err != nil { - return nil, err - } - - if ok { - return nil, errors.New("config already exists") - } - } - - c.be = mem.New() - return c.be, nil - }, - - // OpenFn is a function that opens a previously created temporary repository. - Open: func(cfg interface{}) (restic.Backend, error) { - c := cfg.(*memConfig) - if c.be == nil { - c.be = mem.New() - } - return c.be, nil - }, - - // CleanupFn removes data created during the tests. - Cleanup: func(cfg interface{}) error { - // no cleanup needed - return nil - }, - } -} - -func TestSuiteBackendMem(t *testing.T) { - newTestSuite(t).RunTests(t) -} - -func BenchmarkSuiteBackendMem(b *testing.B) { - newTestSuite(b).RunBenchmarks(b) -}