Merge pull request #961 from restic/add-backend-benchmarks

Add backend benchmarks

src/restic/backend/local/local_test.go

@@ -10,8 +10,8 @@ import (
 	. "restic/test"
 )
 
-func TestBackend(t *testing.T) {
+func newTestSuite(t testing.TB) *test.Suite {
-	suite := test.Suite{
+	return &test.Suite{
 		// NewConfig returns a config for a new temporary backend that will be used in tests.
 		NewConfig: func() (interface{}, error) {
 			dir, err := ioutil.TempDir(TestTempDir, "restic-test-local-")
@@ -50,6 +50,12 @@ func TestBackend(t *testing.T) {
 			return nil
 		},
 	}
-
+}
-	suite.RunTests(t)
+
+func TestBackend(t *testing.T) {
+	newTestSuite(t).RunTests(t)
+}
+
+func BenchmarkBackend(t *testing.B) {
+	newTestSuite(t).RunBenchmarks(t)
 }

src/restic/backend/mem/mem_backend_test.go

@@ -14,8 +14,8 @@ type memConfig struct {
 	be restic.Backend
 }
 
-func TestSuiteBackendMem(t *testing.T) {
+func newTestSuite() *test.Suite {
-	suite := test.Suite{
+	return &test.Suite{
 		// NewConfig returns a config for a new temporary backend that will be used in tests.
 		NewConfig: func() (interface{}, error) {
 			return &memConfig{}, nil
@@ -54,6 +54,12 @@ func TestSuiteBackendMem(t *testing.T) {
 			return nil
 		},
 	}
-
+}
-	suite.RunTests(t)
+
+func TestSuiteBackendMem(t *testing.T) {
+	newTestSuite().RunTests(t)
+}
+
+func BenchmarkSuiteBackendMem(t *testing.B) {
+	newTestSuite().RunBenchmarks(t)
 }

src/restic/backend/rest/rest_test.go

@@ -60,23 +60,8 @@ func runRESTServer(ctx context.Context, t testing.TB, dir string) func() {
 	}
 }
 
-func TestBackendREST(t *testing.T) {
+func newTestSuite(ctx context.Context, t testing.TB) *test.Suite {
-	defer func() {
+	return &test.Suite{
-		if t.Skipped() {
-			SkipDisallowed(t, "restic/backend/rest.TestBackendREST")
-		}
-	}()
-
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-
-	dir, cleanup := TempDir(t)
-	defer cleanup()
-
-	cleanup = runRESTServer(ctx, t, dir)
-	defer cleanup()
-
-	suite := test.Suite{
 		// NewConfig returns a config for a new temporary backend that will be used in tests.
 		NewConfig: func() (interface{}, error) {
 			dir, err := ioutil.TempDir(TestTempDir, "restic-test-rest-")
@@ -114,6 +99,36 @@ func TestBackendREST(t *testing.T) {
 			return nil
 		},
 	}
-
+}
-	suite.RunTests(t)
+
+func TestBackendREST(t *testing.T) {
+	defer func() {
+		if t.Skipped() {
+			SkipDisallowed(t, "restic/backend/rest.TestBackendREST")
+		}
+	}()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	dir, cleanup := TempDir(t)
+	defer cleanup()
+
+	cleanup = runRESTServer(ctx, t, dir)
+	defer cleanup()
+
+	newTestSuite(ctx, t).RunTests(t)
+}
+
+func BenchmarkBackendREST(t *testing.B) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	dir, cleanup := TempDir(t)
+	defer cleanup()
+
+	cleanup = runRESTServer(ctx, t, dir)
+	defer cleanup()
+
+	newTestSuite(ctx, t).RunBenchmarks(t)
 }

src/restic/backend/s3/s3_test.go

@@ -79,7 +79,7 @@ func runMinio(ctx context.Context, t testing.TB, dir, key, secret string) func()
 	}
 }
 
-func newCredentials(t testing.TB) (key, secret string) {
+func newRandomCredentials(t testing.TB) (key, secret string) {
 	buf := make([]byte, 10)
 	_, err := io.ReadFull(rand.Reader, buf)
 	if err != nil {
@@ -96,38 +96,22 @@ func newCredentials(t testing.TB) (key, secret string) {
 	return key, secret
 }
 
-func TestBackendMinio(t *testing.T) {
+type MinioTestConfig struct {
-	defer func() {
-		if t.Skipped() {
-			SkipDisallowed(t, "restic/backend/s3.TestBackendMinio")
-		}
-	}()
-
-	// try to find a minio binary
-	_, err := exec.LookPath("minio")
-	if err != nil {
-		t.Skip(err)
-		return
-	}
-
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-
-	type Config struct {
 	s3.Config
 
 	tempdir       string
 	removeTempdir func()
 	stopServer    func()
-	}
+}
 
-	suite := test.Suite{
+func newMinioTestSuite(ctx context.Context, t testing.TB) *test.Suite {
+	return &test.Suite{
 		// NewConfig returns a config for a new temporary backend that will be used in tests.
 		NewConfig: func() (interface{}, error) {
-			cfg := Config{}
+			cfg := MinioTestConfig{}
 
 			cfg.tempdir, cfg.removeTempdir = TempDir(t)
-			key, secret := newCredentials(t)
+			key, secret := newRandomCredentials(t)
 			cfg.stopServer = runMinio(ctx, t, cfg.tempdir, key, secret)
 
 			cfg.Config = s3.Config{
@@ -143,7 +127,7 @@ func TestBackendMinio(t *testing.T) {
 
 		// CreateFn is a function that creates a temporary repository for the tests.
 		Create: func(config interface{}) (restic.Backend, error) {
-			cfg := config.(Config)
+			cfg := config.(MinioTestConfig)
 
 			be, err := s3.Open(cfg.Config)
 			if err != nil {
@@ -164,13 +148,13 @@ func TestBackendMinio(t *testing.T) {
 
 		// OpenFn is a function that opens a previously created temporary repository.
 		Open: func(config interface{}) (restic.Backend, error) {
-			cfg := config.(Config)
+			cfg := config.(MinioTestConfig)
 			return s3.Open(cfg.Config)
 		},
 
 		// CleanupFn removes data created during the tests.
 		Cleanup: func(config interface{}) error {
-			cfg := config.(Config)
+			cfg := config.(MinioTestConfig)
 			if cfg.stopServer != nil {
 				cfg.stopServer()
 			}
@@ -180,31 +164,44 @@ func TestBackendMinio(t *testing.T) {
 			return nil
 		},
 	}
-
-	suite.RunTests(t)
 }
 
-func TestBackendS3(t *testing.T) {
+func TestBackendMinio(t *testing.T) {
 	defer func() {
 		if t.Skipped() {
-			SkipDisallowed(t, "restic/backend/s3.TestBackendS3")
+			SkipDisallowed(t, "restic/backend/s3.TestBackendMinio")
 		}
 	}()
 
-	vars := []string{
+	// try to find a minio binary
-		"RESTIC_TEST_S3_KEY",
+	_, err := exec.LookPath("minio")
-		"RESTIC_TEST_S3_SECRET",
+	if err != nil {
-		"RESTIC_TEST_S3_REPOSITORY",
+		t.Skip(err)
-	}
-
-	for _, v := range vars {
-		if os.Getenv(v) == "" {
-			t.Skipf("environment variable %v not set", v)
 		return
 	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	newMinioTestSuite(ctx, t).RunTests(t)
+}
+
+func BenchmarkBackendMinio(t *testing.B) {
+	// try to find a minio binary
+	_, err := exec.LookPath("minio")
+	if err != nil {
+		t.Skip(err)
+		return
 	}
 
-	suite := test.Suite{
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	newMinioTestSuite(ctx, t).RunBenchmarks(t)
+}
+
+func newS3TestSuite(t testing.TB) *test.Suite {
+	return &test.Suite{
 		// do not use excessive data
 		MinimalData: true,
 
@@ -265,7 +262,46 @@ func TestBackendS3(t *testing.T) {
 			return nil
 		},
 	}
+}
+
+func TestBackendS3(t *testing.T) {
+	defer func() {
+		if t.Skipped() {
+			SkipDisallowed(t, "restic/backend/s3.TestBackendS3")
+		}
+	}()
+
+	vars := []string{
+		"RESTIC_TEST_S3_KEY",
+		"RESTIC_TEST_S3_SECRET",
+		"RESTIC_TEST_S3_REPOSITORY",
+	}
+
+	for _, v := range vars {
+		if os.Getenv(v) == "" {
+			t.Skipf("environment variable %v not set", v)
+			return
+		}
+	}
 
 	t.Logf("run tests")
-	suite.RunTests(t)
+	newS3TestSuite(t).RunTests(t)
+}
+
+func BenchmarkBackendS3(t *testing.B) {
+	vars := []string{
+		"RESTIC_TEST_S3_KEY",
+		"RESTIC_TEST_S3_SECRET",
+		"RESTIC_TEST_S3_REPOSITORY",
+	}
+
+	for _, v := range vars {
+		if os.Getenv(v) == "" {
+			t.Skipf("environment variable %v not set", v)
+			return
+		}
+	}
+
+	t.Logf("run tests")
+	newS3TestSuite(t).RunBenchmarks(t)
 }

src/restic/backend/sftp/sftp_test.go

@@ -29,18 +29,8 @@ func findSFTPServerBinary() string {
 
 var sftpServer = findSFTPServerBinary()
 
-func TestBackendSFTP(t *testing.T) {
+func newTestSuite(t testing.TB) *test.Suite {
-	defer func() {
+	return &test.Suite{
-		if t.Skipped() {
-			SkipDisallowed(t, "restic/backend/sftp.TestBackendSFTP")
-		}
-	}()
-
-	if sftpServer == "" {
-		t.Skip("sftp server binary not found")
-	}
-
-	suite := test.Suite{
 		// NewConfig returns a config for a new temporary backend that will be used in tests.
 		NewConfig: func() (interface{}, error) {
 			dir, err := ioutil.TempDir(TestTempDir, "restic-test-sftp-")
@@ -80,6 +70,26 @@ func TestBackendSFTP(t *testing.T) {
 			return nil
 		},
 	}
-
+}
-	suite.RunTests(t)
+
+func TestBackendSFTP(t *testing.T) {
+	defer func() {
+		if t.Skipped() {
+			SkipDisallowed(t, "restic/backend/sftp.TestBackendSFTP")
+		}
+	}()
+
+	if sftpServer == "" {
+		t.Skip("sftp server binary not found")
+	}
+
+	newTestSuite(t).RunTests(t)
+}
+
+func BenchmarkBackendSFTP(t *testing.B) {
+	if sftpServer == "" {
+		t.Skip("sftp server binary not found")
+	}
+
+	newTestSuite(t).RunBenchmarks(t)
 }

src/restic/backend/test/benchmarks.go

@@ -0,0 +1,182 @@
+package test
+
+import (
+	"bytes"
+	"io"
+	"restic"
+	"restic/test"
+	"testing"
+)
+
+func saveRandomFile(t testing.TB, be restic.Backend, length int) ([]byte, restic.Handle) {
+	data := test.Random(23, length)
+	id := restic.Hash(data)
+	handle := restic.Handle{Type: restic.DataFile, Name: id.String()}
+	if err := be.Save(handle, bytes.NewReader(data)); err != nil {
+		t.Fatalf("Save() error: %+v", err)
+	}
+	return data, handle
+}
+
+func remove(t testing.TB, be restic.Backend, h restic.Handle) {
+	if err := be.Remove(h); err != nil {
+		t.Fatalf("Remove() returned error: %v", err)
+	}
+}
+
+// BenchmarkLoadFile benchmarks the Load() method of a backend by
+// loading a complete file.
+func (s *Suite) BenchmarkLoadFile(t *testing.B) {
+	be := s.open(t)
+	defer s.close(t, be)
+
+	length := 1<<24 + 2123
+	data, handle := saveRandomFile(t, be, length)
+	defer remove(t, be, handle)
+
+	buf := make([]byte, length)
+
+	t.SetBytes(int64(length))
+	t.ResetTimer()
+
+	for i := 0; i < t.N; i++ {
+		rd, err := be.Load(handle, 0, 0)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		n, err := io.ReadFull(rd, buf)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if err = rd.Close(); err != nil {
+			t.Fatalf("Close() returned error: %v", err)
+		}
+
+		if n != length {
+			t.Fatalf("wrong number of bytes read: want %v, got %v", length, n)
+		}
+
+		if !bytes.Equal(data, buf) {
+			t.Fatalf("wrong bytes returned")
+		}
+	}
+}
+
+// BenchmarkLoadPartialFile benchmarks the Load() method of a backend by
+// loading the remainder of a file starting at a given offset.
+func (s *Suite) BenchmarkLoadPartialFile(t *testing.B) {
+	be := s.open(t)
+	defer s.close(t, be)
+
+	datalength := 1<<24 + 2123
+	data, handle := saveRandomFile(t, be, datalength)
+	defer remove(t, be, handle)
+
+	testLength := datalength/4 + 555
+
+	buf := make([]byte, testLength)
+
+	t.SetBytes(int64(testLength))
+	t.ResetTimer()
+
+	for i := 0; i < t.N; i++ {
+		rd, err := be.Load(handle, testLength, 0)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		n, err := io.ReadFull(rd, buf)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if err = rd.Close(); err != nil {
+			t.Fatalf("Close() returned error: %v", err)
+		}
+
+		if n != testLength {
+			t.Fatalf("wrong number of bytes read: want %v, got %v", testLength, n)
+		}
+
+		if !bytes.Equal(data[:testLength], buf) {
+			t.Fatalf("wrong bytes returned")
+		}
+
+	}
+}
+
+// BenchmarkLoadPartialFileOffset benchmarks the Load() method of a
+// backend by loading a number of bytes of a file starting at a given offset.
+func (s *Suite) BenchmarkLoadPartialFileOffset(t *testing.B) {
+	be := s.open(t)
+	defer s.close(t, be)
+
+	datalength := 1<<24 + 2123
+	data, handle := saveRandomFile(t, be, datalength)
+	defer remove(t, be, handle)
+
+	testLength := datalength/4 + 555
+	testOffset := 8273
+
+	buf := make([]byte, testLength)
+
+	t.SetBytes(int64(testLength))
+	t.ResetTimer()
+
+	for i := 0; i < t.N; i++ {
+		rd, err := be.Load(handle, testLength, int64(testOffset))
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		n, err := io.ReadFull(rd, buf)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if err = rd.Close(); err != nil {
+			t.Fatalf("Close() returned error: %v", err)
+		}
+
+		if n != testLength {
+			t.Fatalf("wrong number of bytes read: want %v, got %v", testLength, n)
+		}
+
+		if !bytes.Equal(data[testOffset:testOffset+testLength], buf) {
+			t.Fatalf("wrong bytes returned")
+		}
+
+	}
+}
+
+// BenchmarkSave benchmarks the Save() method of a backend.
+func (s *Suite) BenchmarkSave(t *testing.B) {
+	be := s.open(t)
+	defer s.close(t, be)
+
+	length := 1<<24 + 2123
+	data := test.Random(23, length)
+	id := restic.Hash(data)
+	handle := restic.Handle{Type: restic.DataFile, Name: id.String()}
+
+	rd := bytes.NewReader(data)
+
+	t.SetBytes(int64(length))
+	t.ResetTimer()
+
+	for i := 0; i < t.N; i++ {
+		if _, err := rd.Seek(0, 0); err != nil {
+			t.Fatal(err)
+		}
+
+		if err := be.Save(handle, rd); err != nil {
+			t.Fatal(err)
+		}
+
+		if err := be.Remove(handle); err != nil {
+			t.Fatal(err)
+		}
+	}
+}

src/restic/backend/test/doc.go

@@ -0,0 +1,42 @@
+// Package test contains a test suite with benchmarks for restic backends.
+//
+// Overview
+//
+// For the test suite to work a few functions need to be implemented to create
+// new config, create a backend, open it and run cleanup tasks afterwards. The
+// Suite struct has fields for each function.
+//
+// So for a new backend, a Suite needs to be built with callback functions,
+// then the methods RunTests() and RunBenchmarks() can be used to run the
+// individual tests and benchmarks as subtests/subbenchmarks.
+//
+// Example
+//
+// Assuming a *Suite is returned by newTestSuite(), the tests and benchmarks
+// can be run like this:
+//   func newTestSuite(t testing.TB) *test.Suite {
+//   	return &test.Suite{
+//   		Create: func(cfg interface{}) (restic.Backend, error) {
+//   			[...]
+//   		},
+//   		[...]
+//   	}
+//   }
+//
+//   func TestSuiteBackendMem(t *testing.T) {
+//   	newTestSuite(t).RunTests(t)
+//   }
+//
+//   func BenchmarkSuiteBackendMem(b *testing.B) {
+//   	newTestSuite(b).RunBenchmarks(b)
+//   }
+//
+// The functions are run in alphabetical order.
+//
+// Add new tests
+//
+// A new test or benchmark can be added by implementing a method on *Suite
+// with the name starting with "Test" and a single *testing.T parameter for
+// test. For benchmarks, the name must start with "Benchmark" and the parameter
+// is a *testing.B
+package test

src/restic/backend/test/funcs.go

@@ -1,21 +0,0 @@
-// DO NOT EDIT, AUTOMATICALLY GENERATED
-
-package test
-
-import (
-	"testing"
-)
-
-var testFunctions = []struct {
-	Name string
-	Fn   func(t testing.TB, suite *Suite)
-}{
-	{"CreateWithConfig", BackendTestCreateWithConfig},
-	{"Location", BackendTestLocation},
-	{"Config", BackendTestConfig},
-	{"Load", BackendTestLoad},
-	{"Save", BackendTestSave},
-	{"SaveFilenames", BackendTestSaveFilenames},
-	{"Backend", BackendTestBackend},
-	{"Delete", BackendTestDelete},
-}

src/restic/backend/test/generate_test_list.go

@@ -1,131 +0,0 @@
-// +build ignore
-
-package main
-
-import (
-	"bufio"
-	"flag"
-	"fmt"
-	"io"
-	"log"
-	"os"
-	"os/exec"
-	"path/filepath"
-	"regexp"
-	"text/template"
-	"unicode"
-	"unicode/utf8"
-)
-
-var data struct {
-	Package string
-	Funcs   []string
-}
-
-var testTemplate = `
-// DO NOT EDIT, AUTOMATICALLY GENERATED
-
-package {{ .Package }}
-
-import (
-	"testing"
-)
-
-var testFunctions = []struct {
-	Name string
-	Fn   func(t testing.TB, suite *Suite)
-}{
-{{ range $f := .Funcs -}}
-	{"{{ $f }}", BackendTest{{ $f }},},
-{{ end }}
-}
-`
-
-var testFile = flag.String("testfile", "tests.go", "file to search test functions in")
-var outputFile = flag.String("output", "funcs.go", "output file to write generated code to")
-var packageName = flag.String("package", "", "the package name to use")
-var prefix = flag.String("prefix", "", "test function prefix")
-var quiet = flag.Bool("quiet", false, "be quiet")
-
-func errx(err error) {
-	if err == nil {
-		return
-	}
-
-	fmt.Fprintf(os.Stderr, "error: %v\n", err)
-	os.Exit(1)
-}
-
-var funcRegex = regexp.MustCompile(`^func\s+BackendTest(.+)\s*\(`)
-
-func findTestFunctions() (funcs []string) {
-	f, err := os.Open(*testFile)
-	errx(err)
-
-	sc := bufio.NewScanner(f)
-	for sc.Scan() {
-		match := funcRegex.FindStringSubmatch(sc.Text())
-		if len(match) > 0 {
-			funcs = append(funcs, match[1])
-		}
-	}
-
-	if err := sc.Err(); err != nil {
-		log.Fatalf("Error scanning file: %v", err)
-	}
-
-	errx(f.Close())
-	return funcs
-}
-
-func generateOutput(wr io.Writer, data interface{}) {
-	t := template.Must(template.New("backendtest").Parse(testTemplate))
-
-	cmd := exec.Command("gofmt")
-	cmd.Stdout = wr
-	in, err := cmd.StdinPipe()
-	errx(err)
-	errx(cmd.Start())
-	errx(t.Execute(in, data))
-	errx(in.Close())
-	errx(cmd.Wait())
-}
-
-func packageTestFunctionPrefix(pkg string) string {
-	if pkg == "" {
-		return ""
-	}
-
-	r, n := utf8.DecodeRuneInString(pkg)
-	return string(unicode.ToUpper(r)) + pkg[n:]
-}
-
-func init() {
-	flag.Parse()
-}
-
-func main() {
-	dir, err := os.Getwd()
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "Getwd() %v\n", err)
-		os.Exit(1)
-	}
-
-	pkg := *packageName
-	if pkg == "" {
-		pkg = filepath.Base(dir)
-	}
-
-	f, err := os.Create(*outputFile)
-	errx(err)
-
-	data.Package = pkg
-	data.Funcs = findTestFunctions()
-	generateOutput(f, data)
-
-	errx(f.Close())
-
-	if !*quiet {
-		fmt.Printf("wrote backend tests for package %v to %v\n", data.Package, *outputFile)
-	}
-}

src/restic/backend/test/suite.go

@@ -0,0 +1,174 @@
+package test
+
+import (
+	"reflect"
+	"restic"
+	"restic/test"
+	"strings"
+	"testing"
+)
+
+// Suite implements a test suite for restic backends.
+type Suite struct {
+	Config interface{}
+
+	// NewConfig returns a config for a new temporary backend that will be used in tests.
+	NewConfig func() (interface{}, error)
+
+	// CreateFn is a function that creates a temporary repository for the tests.
+	Create func(cfg interface{}) (restic.Backend, error)
+
+	// OpenFn is a function that opens a previously created temporary repository.
+	Open func(cfg interface{}) (restic.Backend, error)
+
+	// CleanupFn removes data created during the tests.
+	Cleanup func(cfg interface{}) error
+
+	// MinimalData instructs the tests to not use excessive data.
+	MinimalData bool
+}
+
+// RunTests executes all defined tests as subtests of t.
+func (s *Suite) RunTests(t *testing.T) {
+	var err error
+	s.Config, err = s.NewConfig()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// test create/open functions first
+	be := s.create(t)
+	s.close(t, be)
+
+	for _, test := range s.testFuncs(t) {
+		t.Run(test.Name, test.Fn)
+	}
+
+	if !test.TestCleanupTempDirs {
+		t.Logf("not cleaning up backend")
+		return
+	}
+
+	if err = s.Cleanup(s.Config); err != nil {
+		t.Fatal(err)
+	}
+}
+
+type testFunction struct {
+	Name string
+	Fn   func(*testing.T)
+}
+
+func (s *Suite) testFuncs(t testing.TB) (funcs []testFunction) {
+	tpe := reflect.TypeOf(s)
+	v := reflect.ValueOf(s)
+
+	for i := 0; i < tpe.NumMethod(); i++ {
+		methodType := tpe.Method(i)
+		name := methodType.Name
+
+		// discard functions which do not have the right name
+		if !strings.HasPrefix(name, "Test") {
+			continue
+		}
+
+		iface := v.Method(i).Interface()
+		f, ok := iface.(func(*testing.T))
+		if !ok {
+			t.Logf("warning: function %v of *Suite has the wrong signature for a test function\nwant: func(*testing.T),\nhave: %T",
+				name, iface)
+			continue
+		}
+
+		funcs = append(funcs, testFunction{
+			Name: name,
+			Fn:   f,
+		})
+	}
+
+	return funcs
+}
+
+type benchmarkFunction struct {
+	Name string
+	Fn   func(*testing.B)
+}
+
+func (s *Suite) benchmarkFuncs(t testing.TB) (funcs []benchmarkFunction) {
+	tpe := reflect.TypeOf(s)
+	v := reflect.ValueOf(s)
+
+	for i := 0; i < tpe.NumMethod(); i++ {
+		methodType := tpe.Method(i)
+		name := methodType.Name
+
+		// discard functions which do not have the right name
+		if !strings.HasPrefix(name, "Benchmark") {
+			continue
+		}
+
+		iface := v.Method(i).Interface()
+		f, ok := iface.(func(*testing.B))
+		if !ok {
+			t.Logf("warning: function %v of *Suite has the wrong signature for a test function\nwant: func(*testing.T),\nhave: %T",
+				name, iface)
+			continue
+		}
+
+		funcs = append(funcs, benchmarkFunction{
+			Name: name,
+			Fn:   f,
+		})
+	}
+
+	return funcs
+}
+
+// RunBenchmarks executes all defined benchmarks as subtests of b.
+func (s *Suite) RunBenchmarks(b *testing.B) {
+	var err error
+	s.Config, err = s.NewConfig()
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	// test create/open functions first
+	be := s.create(b)
+	s.close(b, be)
+
+	for _, test := range s.benchmarkFuncs(b) {
+		b.Run(test.Name, test.Fn)
+	}
+
+	if !test.TestCleanupTempDirs {
+		b.Logf("not cleaning up backend")
+		return
+	}
+
+	if err = s.Cleanup(s.Config); err != nil {
+		b.Fatal(err)
+	}
+}
+
+func (s *Suite) create(t testing.TB) restic.Backend {
+	be, err := s.Create(s.Config)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return be
+}
+
+func (s *Suite) open(t testing.TB) restic.Backend {
+	be, err := s.Open(s.Config)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return be
+}
+
+func (s *Suite) close(t testing.TB, be restic.Backend) {
+	err := be.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+}

src/restic/backend/test/tests.go

@@ -1,4 +1,3 @@
-// Package test contains a test suite for restic backends.
 package test
 
 import (
@@ -20,88 +19,28 @@ import (
 	"restic/backend"
 )
 
-// Suite implements a test suite for restic backends.
+// TestCreateWithConfig tests that creating a backend in a location which already
-type Suite struct {
-	Config interface{}
-
-	// NewConfig returns a config for a new temporary backend that will be used in tests.
-	NewConfig func() (interface{}, error)
-
-	// CreateFn is a function that creates a temporary repository for the tests.
-	Create func(cfg interface{}) (restic.Backend, error)
-
-	// OpenFn is a function that opens a previously created temporary repository.
-	Open func(cfg interface{}) (restic.Backend, error)
-
-	// CleanupFn removes data created during the tests.
-	Cleanup func(cfg interface{}) error
-
-	// MinimalData instructs the tests to not use excessive data.
-	MinimalData bool
-}
-
-// RunTests executes all defined tests as subtests of t.
-func (s *Suite) RunTests(t *testing.T) {
-	var err error
-	s.Config, err = s.NewConfig()
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// test create/open functions first
-	be := s.create(t)
-	s.close(t, be)
-
-	for _, test := range testFunctions {
-		t.Run(test.Name, func(t *testing.T) {
-			test.Fn(t, s)
-		})
-	}
-
-	if !test.TestCleanupTempDirs {
-		t.Logf("not cleaning up backend")
-		return
-	}
-
-	if err = s.Cleanup(s.Config); err != nil {
-		t.Fatal(err)
-	}
-}
-
-func (s *Suite) create(t testing.TB) restic.Backend {
-	be, err := s.Create(s.Config)
-	if err != nil {
-		t.Fatal(err)
-	}
-	return be
-}
-
-func (s *Suite) open(t testing.TB) restic.Backend {
-	be, err := s.Open(s.Config)
-	if err != nil {
-		t.Fatal(err)
-	}
-	return be
-}
-
-func (s *Suite) close(t testing.TB, be restic.Backend) {
-	err := be.Close()
-	if err != nil {
-		t.Fatal(err)
-	}
-}
-
-// BackendTestCreateWithConfig tests that creating a backend in a location which already
 // has a config file fails.
-func BackendTestCreateWithConfig(t testing.TB, s *Suite) {
+func (s *Suite) TestCreateWithConfig(t *testing.T) {
 	b := s.open(t)
 	defer s.close(t, b)
 
+	// remove a config if present
+	cfgHandle := restic.Handle{Type: restic.ConfigFile}
+	cfgPresent, err := b.Test(cfgHandle)
+	if err != nil {
+		t.Fatalf("unable to test for config: %+v", err)
+	}
+
+	if cfgPresent {
+		remove(t, b, cfgHandle)
+	}
+
 	// save a config
 	store(t, b, restic.ConfigFile, []byte("test config"))
 
 	// now create the backend again, this must fail
-	_, err := s.Create(s.Config)
+	_, err = s.Create(s.Config)
 	if err == nil {
 		t.Fatalf("expected error not found for creating a backend with an existing config file")
 	}
@@ -113,8 +52,8 @@ func BackendTestCreateWithConfig(t testing.TB, s *Suite) {
 	}
 }
 
-// BackendTestLocation tests that a location string is returned.
+// TestLocation tests that a location string is returned.
-func BackendTestLocation(t testing.TB, s *Suite) {
+func (s *Suite) TestLocation(t *testing.T) {
 	b := s.open(t)
 	defer s.close(t, b)
 
@@ -124,8 +63,8 @@ func BackendTestLocation(t testing.TB, s *Suite) {
 	}
 }
 
-// BackendTestConfig saves and loads a config from the backend.
+// TestConfig saves and loads a config from the backend.
-func BackendTestConfig(t testing.TB, s *Suite) {
+func (s *Suite) TestConfig(t *testing.T) {
 	b := s.open(t)
 	defer s.close(t, b)
 
@@ -155,10 +94,13 @@ func BackendTestConfig(t testing.TB, s *Suite) {
 			t.Fatalf("wrong data returned, want %q, got %q", testString, string(buf))
 		}
 	}
+
+	// remove the config
+	remove(t, b, restic.Handle{Type: restic.ConfigFile})
 }
 
-// BackendTestLoad tests the backend's Load function.
+// TestLoad tests the backend's Load function.
-func BackendTestLoad(t testing.TB, s *Suite) {
+func (s *Suite) TestLoad(t *testing.T) {
 	b := s.open(t)
 	defer s.close(t, b)
 
@@ -290,8 +232,8 @@ func (ec errorCloser) Size() int64 {
 	return ec.size
 }
 
-// BackendTestSave tests saving data in the backend.
+// TestSave tests saving data in the backend.
-func BackendTestSave(t testing.TB, s *Suite) {
+func (s *Suite) TestSave(t *testing.T) {
 	b := s.open(t)
 	defer s.close(t, b)
 	var id restic.ID
@@ -390,8 +332,8 @@ var filenameTests = []struct {
 	},
 }
 
-// BackendTestSaveFilenames tests saving data with various file names in the backend.
+// TestSaveFilenames tests saving data with various file names in the backend.
-func BackendTestSaveFilenames(t testing.TB, s *Suite) {
+func (s *Suite) TestSaveFilenames(t *testing.T) {
 	b := s.open(t)
 	defer s.close(t, b)
 
@@ -439,8 +381,8 @@ func store(t testing.TB, b restic.Backend, tpe restic.FileType, data []byte) res
 	return h
 }
 
-// BackendTestBackend tests all functions of the backend.
+// TestBackend tests all functions of the backend.
-func BackendTestBackend(t testing.TB, s *Suite) {
+func (s *Suite) TestBackend(t *testing.T) {
 	b := s.open(t)
 	defer s.close(t, b)
 
@@ -573,8 +515,8 @@ func BackendTestBackend(t testing.TB, s *Suite) {
 	}
 }
 
-// BackendTestDelete tests the Delete function.
+// TestDelete tests the Delete function.
-func BackendTestDelete(t testing.TB, s *Suite) {
+func (s *Suite) TestDelete(t *testing.T) {
 	if !test.TestCleanupTempDirs {
 		t.Skipf("not removing backend, TestCleanupTempDirs is false")
 	}

src/restic/backend/test/tests_test.go

@@ -15,8 +15,8 @@ type memConfig struct {
 	be restic.Backend
 }
 
-func TestSuiteBackendMem(t *testing.T) {
+func newTestSuite(t testing.TB) *test.Suite {
-	suite := test.Suite{
+	return &test.Suite{
 		// NewConfig returns a config for a new temporary backend that will be used in tests.
 		NewConfig: func() (interface{}, error) {
 			return &memConfig{}, nil
@@ -55,6 +55,12 @@ func TestSuiteBackendMem(t *testing.T) {
 			return nil
 		},
 	}
-
+}
-	suite.RunTests(t)
+
+func TestSuiteBackendMem(t *testing.T) {
+	newTestSuite(t).RunTests(t)
+}
+
+func BenchmarkSuiteBackendMem(b *testing.B) {
+	newTestSuite(b).RunBenchmarks(b)
 }