Merge pull request #3973 from MichaelEischer/speedup-integration-tests

speed-up integration tests by reducing the RetryBackend timeout

cmd/restic/integration_helpers_test.go

@@ -9,6 +9,7 @@ import (
 	"runtime"
 	"testing"
 
+	"github.com/restic/restic/internal/backend"
 	"github.com/restic/restic/internal/options"
 	"github.com/restic/restic/internal/repository"
 	"github.com/restic/restic/internal/restic"
@@ -171,6 +172,7 @@ func withTestEnvironment(t testing.TB) (env *testEnvironment, cleanup func()) {
 
 	repository.TestUseLowSecurityKDFParameters(t)
 	restic.TestDisableCheckPolynomial(t)
+	backend.TestFastRetries(t)
 
 	tempdir, err := ioutil.TempDir(rtest.TestTempDir, "restic-test-")
 	rtest.OK(t, err)

internal/backend/backend_retry.go

@@ -55,6 +55,8 @@ func retryNotifyErrorWithSuccess(operation backoff.Operation, b backoff.BackOff,
 	return backoff.RetryNotify(operationWrapper, b, notify)
 }
 
+var fastRetries = false
+
 func (be *RetryBackend) retry(ctx context.Context, msg string, f func() error) error {
 	// Don't do anything when called with an already cancelled context. There would be
 	// no retries in that case either, so be consistent and abort always.
@@ -66,8 +68,14 @@ func (be *RetryBackend) retry(ctx context.Context, msg string, f func() error) e
 		return ctx.Err()
 	}
 
+	bo := backoff.NewExponentialBackOff()
+	if fastRetries {
+		// speed up integration tests
+		bo.InitialInterval = 1 * time.Millisecond
+	}
+
 	err := retryNotifyErrorWithSuccess(f,
-		backoff.WithContext(backoff.WithMaxRetries(backoff.NewExponentialBackOff(), uint64(be.MaxTries)), ctx),
+		backoff.WithContext(backoff.WithMaxRetries(bo, uint64(be.MaxTries)), ctx),
 		func(err error, d time.Duration) {
 			if be.Report != nil {
 				be.Report(msg, err, d)

internal/backend/backend_retry_test.go

@@ -35,6 +35,7 @@ func TestBackendSaveRetry(t *testing.T) {
 		},
 	}
 
+	TestFastRetries(t)
 	retryBackend := NewRetryBackend(be, 10, nil, nil)
 
 	data := test.Random(23, 5*1024*1024+11241)
@@ -70,6 +71,7 @@ func TestBackendSaveRetryAtomic(t *testing.T) {
 		HasAtomicReplaceFn: func() bool { return true },
 	}
 
+	TestFastRetries(t)
 	retryBackend := NewRetryBackend(be, 10, nil, nil)
 
 	data := test.Random(23, 5*1024*1024+11241)
@@ -103,6 +105,7 @@ func TestBackendListRetry(t *testing.T) {
 		},
 	}
 
+	TestFastRetries(t)
 	retryBackend := NewRetryBackend(be, 10, nil, nil)
 
 	var listed []string
@@ -132,6 +135,7 @@ func TestBackendListRetryErrorFn(t *testing.T) {
 		},
 	}
 
+	TestFastRetries(t)
 	retryBackend := NewRetryBackend(be, 10, nil, nil)
 
 	var ErrTest = errors.New("test error")
@@ -187,6 +191,7 @@ func TestBackendListRetryErrorBackend(t *testing.T) {
 		},
 	}
 
+	TestFastRetries(t)
 	const maxRetries = 2
 	retryBackend := NewRetryBackend(be, maxRetries, nil, nil)
 
@@ -257,6 +262,7 @@ func TestBackendLoadRetry(t *testing.T) {
 		return failingReader{data: data, limit: limit}, nil
 	}
 
+	TestFastRetries(t)
 	retryBackend := NewRetryBackend(be, 10, nil, nil)
 
 	var buf []byte
@@ -276,6 +282,7 @@ func assertIsCanceled(t *testing.T, err error) {
 func TestBackendCanceledContext(t *testing.T) {
 	// unimplemented mock backend functions return an error by default
 	// check that we received the expected context canceled error instead
+	TestFastRetries(t)
 	retryBackend := NewRetryBackend(mock.NewBackend(), 2, nil, nil)
 	h := restic.Handle{Type: restic.PackFile, Name: restic.NewRandomID().String()}
 

internal/backend/mem/mem_backend.go

@@ -3,13 +3,13 @@ package mem
 import (
 	"bytes"
 	"context"
-	"crypto/md5"
 	"encoding/base64"
 	"hash"
 	"io"
 	"io/ioutil"
 	"sync"
 
+	"github.com/cespare/xxhash/v2"
 	"github.com/restic/restic/internal/backend"
 	"github.com/restic/restic/internal/backend/sema"
 	"github.com/restic/restic/internal/debug"
@@ -266,7 +266,7 @@ func (be *MemoryBackend) Location() string {
 
 // Hasher may return a hash function for calculating a content hash for the backend
 func (be *MemoryBackend) Hasher() hash.Hash {
-	return md5.New()
+	return xxhash.New()
 }
 
 // HasAtomicReplace returns whether Save() can atomically replace files

internal/backend/test/tests_test.go

@@ -1,66 +0,0 @@
-package test_test
-
-import (
-	"context"
-	"testing"
-
-	"github.com/restic/restic/internal/errors"
-	"github.com/restic/restic/internal/restic"
-
-	"github.com/restic/restic/internal/backend/mem"
-	"github.com/restic/restic/internal/backend/test"
-)
-
-type memConfig struct {
-	be restic.Backend
-}
-
-func newTestSuite(t testing.TB) *test.Suite {
-	return &test.Suite{
-		// NewConfig returns a config for a new temporary backend that will be used in tests.
-		NewConfig: func() (interface{}, error) {
-			return &memConfig{}, nil
-		},
-
-		// CreateFn is a function that creates a temporary repository for the tests.
-		Create: func(cfg interface{}) (restic.Backend, error) {
-			c := cfg.(*memConfig)
-			if c.be != nil {
-				ok, err := c.be.Test(context.TODO(), restic.Handle{Type: restic.ConfigFile})
-				if err != nil {
-					return nil, err
-				}
-
-				if ok {
-					return nil, errors.New("config already exists")
-				}
-			}
-
-			c.be = mem.New()
-			return c.be, nil
-		},
-
-		// OpenFn is a function that opens a previously created temporary repository.
-		Open: func(cfg interface{}) (restic.Backend, error) {
-			c := cfg.(*memConfig)
-			if c.be == nil {
-				c.be = mem.New()
-			}
-			return c.be, nil
-		},
-
-		// CleanupFn removes data created during the tests.
-		Cleanup: func(cfg interface{}) error {
-			// no cleanup needed
-			return nil
-		},
-	}
-}
-
-func TestSuiteBackendMem(t *testing.T) {
-	newTestSuite(t).RunTests(t)
-}
-
-func BenchmarkSuiteBackendMem(b *testing.B) {
-	newTestSuite(b).RunBenchmarks(b)
-}

internal/backend/testing.go

@@ -0,0 +1,8 @@
+package backend
+
+import "testing"
+
+// TestFastRetries reduces the initial retry delay to 1 millisecond
+func TestFastRetries(t testing.TB) {
+	fastRetries = true
+}