diff --git a/cmd/restic/integration_helpers_test.go b/cmd/restic/integration_helpers_test.go
index 17a3c29c3..6a4d064a8 100644
--- a/cmd/restic/integration_helpers_test.go
+++ b/cmd/restic/integration_helpers_test.go
@@ -9,6 +9,7 @@ import (
 	"runtime"
 	"testing"
 
+	"github.com/restic/restic/internal/backend"
 	"github.com/restic/restic/internal/options"
 	"github.com/restic/restic/internal/repository"
 	"github.com/restic/restic/internal/restic"
@@ -171,6 +172,7 @@ func withTestEnvironment(t testing.TB) (env *testEnvironment, cleanup func()) {
 
 	repository.TestUseLowSecurityKDFParameters(t)
 	restic.TestDisableCheckPolynomial(t)
+	backend.TestFastRetries(t)
 
 	tempdir, err := ioutil.TempDir(rtest.TestTempDir, "restic-test-")
 	rtest.OK(t, err)
diff --git a/internal/backend/backend_retry.go b/internal/backend/backend_retry.go
index 1910bec59..e9a22d75f 100644
--- a/internal/backend/backend_retry.go
+++ b/internal/backend/backend_retry.go
@@ -55,6 +55,8 @@ func retryNotifyErrorWithSuccess(operation backoff.Operation, b backoff.BackOff,
 	return backoff.RetryNotify(operationWrapper, b, notify)
 }
 
+var fastRetries = false
+
 func (be *RetryBackend) retry(ctx context.Context, msg string, f func() error) error {
 	// Don't do anything when called with an already cancelled context. There would be
 	// no retries in that case either, so be consistent and abort always.
@@ -66,8 +68,14 @@ func (be *RetryBackend) retry(ctx context.Context, msg string, f func() error) e
 		return ctx.Err()
 	}
 
+	bo := backoff.NewExponentialBackOff()
+	if fastRetries {
+		// speed up integration tests
+		bo.InitialInterval = 1 * time.Millisecond
+	}
+
 	err := retryNotifyErrorWithSuccess(f,
-		backoff.WithContext(backoff.WithMaxRetries(backoff.NewExponentialBackOff(), uint64(be.MaxTries)), ctx),
+		backoff.WithContext(backoff.WithMaxRetries(bo, uint64(be.MaxTries)), ctx),
 		func(err error, d time.Duration) {
 			if be.Report != nil {
 				be.Report(msg, err, d)
diff --git a/internal/backend/testing.go b/internal/backend/testing.go
new file mode 100644
index 000000000..0d7fa1d76
--- /dev/null
+++ b/internal/backend/testing.go
@@ -0,0 +1,8 @@
+package backend
+
+import "testing"
+
+// TestFastRetries reduces the initial retry delay to 1 millisecond
+func TestFastRetries(t testing.TB) {
+	fastRetries = true
+}