Wire context into backend layout detection

cmd/restic/global.go

@@ -687,15 +687,15 @@ func open(s string, gopts GlobalOptions, opts options.Options) (restic.Backend,
 
 	switch loc.Scheme {
 	case "local":
-		be, err = local.Open(cfg.(local.Config))
+		be, err = local.Open(globalOptions.ctx, cfg.(local.Config))
 		// wrap the backend in a LimitBackend so that the throughput is limited
 		be = limiter.LimitBackend(be, lim)
 	case "sftp":
-		be, err = sftp.Open(cfg.(sftp.Config))
+		be, err = sftp.Open(globalOptions.ctx, cfg.(sftp.Config))
 		// wrap the backend in a LimitBackend so that the throughput is limited
 		be = limiter.LimitBackend(be, lim)
 	case "s3":
-		be, err = s3.Open(cfg.(s3.Config), rt)
+		be, err = s3.Open(globalOptions.ctx, cfg.(s3.Config), rt)
 	case "gs":
 		be, err = gs.Open(cfg.(gs.Config), rt)
 	case "azure":
@@ -754,9 +754,9 @@ func create(s string, opts options.Options) (restic.Backend, error) {
 
 	switch loc.Scheme {
 	case "local":
-		return local.Create(cfg.(local.Config))
+		return local.Create(globalOptions.ctx, cfg.(local.Config))
 	case "sftp":
-		return sftp.Create(cfg.(sftp.Config))
+		return sftp.Create(globalOptions.ctx, cfg.(sftp.Config))
 	case "s3":
 		return s3.Create(globalOptions.ctx, cfg.(s3.Config), rt)
 	case "gs":

internal/backend/layout.go

@@ -1,6 +1,7 @@
 package backend
 
 import (
+	"context"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -24,7 +25,7 @@ type Layout interface {
 // Filesystem is the abstraction of a file system used for a backend.
 type Filesystem interface {
 	Join(...string) string
-	ReadDir(string) ([]os.FileInfo, error)
+	ReadDir(context.Context, string) ([]os.FileInfo, error)
 	IsNotExist(error) bool
 }
 
@@ -36,7 +37,7 @@ type LocalFilesystem struct {
 }
 
 // ReadDir returns all entries of a directory.
-func (l *LocalFilesystem) ReadDir(dir string) ([]os.FileInfo, error) {
+func (l *LocalFilesystem) ReadDir(ctx context.Context, dir string) ([]os.FileInfo, error) {
 	f, err := fs.Open(dir)
 	if err != nil {
 		return nil, err
@@ -68,8 +69,8 @@ func (l *LocalFilesystem) IsNotExist(err error) bool {
 var backendFilenameLength = len(restic.ID{}) * 2
 var backendFilename = regexp.MustCompile(fmt.Sprintf("^[a-fA-F0-9]{%d}$", backendFilenameLength))
 
-func hasBackendFile(fs Filesystem, dir string) (bool, error) {
+func hasBackendFile(ctx context.Context, fs Filesystem, dir string) (bool, error) {
-	entries, err := fs.ReadDir(dir)
+	entries, err := fs.ReadDir(ctx, dir)
 	if err != nil && fs.IsNotExist(errors.Cause(err)) {
 		return false, nil
 	}
@@ -94,20 +95,20 @@ var ErrLayoutDetectionFailed = errors.New("auto-detecting the filesystem layout
 // DetectLayout tries to find out which layout is used in a local (or sftp)
 // filesystem at the given path. If repo is nil, an instance of LocalFilesystem
 // is used.
-func DetectLayout(repo Filesystem, dir string) (Layout, error) {
+func DetectLayout(ctx context.Context, repo Filesystem, dir string) (Layout, error) {
 	debug.Log("detect layout at %v", dir)
 	if repo == nil {
 		repo = &LocalFilesystem{}
 	}
 
 	// key file in the "keys" dir (DefaultLayout)
-	foundKeysFile, err := hasBackendFile(repo, repo.Join(dir, defaultLayoutPaths[restic.KeyFile]))
+	foundKeysFile, err := hasBackendFile(ctx, repo, repo.Join(dir, defaultLayoutPaths[restic.KeyFile]))
 	if err != nil {
 		return nil, err
 	}
 
 	// key file in the "key" dir (S3LegacyLayout)
-	foundKeyFile, err := hasBackendFile(repo, repo.Join(dir, s3LayoutPaths[restic.KeyFile]))
+	foundKeyFile, err := hasBackendFile(ctx, repo, repo.Join(dir, s3LayoutPaths[restic.KeyFile]))
 	if err != nil {
 		return nil, err
 	}
@@ -134,7 +135,7 @@ func DetectLayout(repo Filesystem, dir string) (Layout, error) {
 
 // ParseLayout parses the config string and returns a Layout. When layout is
 // the empty string, DetectLayout is used. If that fails, defaultLayout is used.
-func ParseLayout(repo Filesystem, layout, defaultLayout, path string) (l Layout, err error) {
+func ParseLayout(ctx context.Context, repo Filesystem, layout, defaultLayout, path string) (l Layout, err error) {
 	debug.Log("parse layout string %q for backend at %v", layout, path)
 	switch layout {
 	case "default":
@@ -148,12 +149,12 @@ func ParseLayout(repo Filesystem, layout, defaultLayout, path string) (l Layout,
 			Join: repo.Join,
 		}
 	case "":
-		l, err = DetectLayout(repo, path)
+		l, err = DetectLayout(ctx, repo, path)
 
 		// use the default layout if auto detection failed
 		if errors.Cause(err) == ErrLayoutDetectionFailed && defaultLayout != "" {
 			debug.Log("error: %v, use default layout %v", err, defaultLayout)
-			return ParseLayout(repo, defaultLayout, "", path)
+			return ParseLayout(ctx, repo, defaultLayout, "", path)
 		}
 
 		if err != nil {

internal/backend/layout_test.go

@@ -1,6 +1,7 @@
 package backend
 
 import (
+	"context"
 	"fmt"
 	"path"
 	"path/filepath"
@@ -371,7 +372,7 @@ func TestDetectLayout(t *testing.T) {
 			t.Run(fmt.Sprintf("%v/fs-%T", test.filename, fs), func(t *testing.T) {
 				rtest.SetupTarTestFixture(t, path, filepath.Join("testdata", test.filename))
 
-				layout, err := DetectLayout(fs, filepath.Join(path, "repo"))
+				layout, err := DetectLayout(context.TODO(), fs, filepath.Join(path, "repo"))
 				if err != nil {
 					t.Fatal(err)
 				}
@@ -409,7 +410,7 @@ func TestParseLayout(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.layoutName, func(t *testing.T) {
-			layout, err := ParseLayout(&LocalFilesystem{}, test.layoutName, test.defaultLayoutName, filepath.Join(path, "repo"))
+			layout, err := ParseLayout(context.TODO(), &LocalFilesystem{}, test.layoutName, test.defaultLayoutName, filepath.Join(path, "repo"))
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -441,7 +442,7 @@ func TestParseLayoutInvalid(t *testing.T) {
 
 	for _, name := range invalidNames {
 		t.Run(name, func(t *testing.T) {
-			layout, err := ParseLayout(nil, name, "", path)
+			layout, err := ParseLayout(context.TODO(), nil, name, "", path)
 			if err == nil {
 				t.Fatalf("expected error not found for layout name %v, layout is %v", name, layout)
 			}

internal/backend/local/layout_test.go

@@ -36,7 +36,7 @@ func TestLayout(t *testing.T) {
 			rtest.SetupTarTestFixture(t, path, filepath.Join("..", "testdata", test.filename))
 
 			repo := filepath.Join(path, "repo")
-			be, err := Open(Config{
+			be, err := Open(context.TODO(), Config{
 				Path:   repo,
 				Layout: test.layout,
 			})

internal/backend/local/local.go

@@ -27,9 +27,9 @@ var _ restic.Backend = &Local{}
 const defaultLayout = "default"
 
 // Open opens the local backend as specified by config.
-func Open(cfg Config) (*Local, error) {
+func Open(ctx context.Context, cfg Config) (*Local, error) {
 	debug.Log("open local backend at %v (layout %q)", cfg.Path, cfg.Layout)
-	l, err := backend.ParseLayout(&backend.LocalFilesystem{}, cfg.Layout, defaultLayout, cfg.Path)
+	l, err := backend.ParseLayout(ctx, &backend.LocalFilesystem{}, cfg.Layout, defaultLayout, cfg.Path)
 	if err != nil {
 		return nil, err
 	}
@@ -39,10 +39,10 @@ func Open(cfg Config) (*Local, error) {
 
 // Create creates all the necessary files and directories for a new local
 // backend at dir. Afterwards a new config blob should be created.
-func Create(cfg Config) (*Local, error) {
+func Create(ctx context.Context, cfg Config) (*Local, error) {
 	debug.Log("create local backend at %v (layout %q)", cfg.Path, cfg.Layout)
 
-	l, err := backend.ParseLayout(&backend.LocalFilesystem{}, cfg.Layout, defaultLayout, cfg.Path)
+	l, err := backend.ParseLayout(ctx, &backend.LocalFilesystem{}, cfg.Layout, defaultLayout, cfg.Path)
 	if err != nil {
 		return nil, err
 	}

internal/backend/local/local_test.go

@@ -1,6 +1,7 @@
 package local_test
 
 import (
+	"context"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -32,13 +33,13 @@ func newTestSuite(t testing.TB) *test.Suite {
 		// CreateFn is a function that creates a temporary repository for the tests.
 		Create: func(config interface{}) (restic.Backend, error) {
 			cfg := config.(local.Config)
-			return local.Create(cfg)
+			return local.Create(context.TODO(), cfg)
 		},
 
 		// OpenFn is a function that opens a previously created temporary repository.
 		Open: func(config interface{}) (restic.Backend, error) {
 			cfg := config.(local.Config)
-			return local.Open(cfg)
+			return local.Open(context.TODO(), cfg)
 		},
 
 		// CleanupFn removes data created during the tests.
@@ -91,7 +92,7 @@ func empty(t testing.TB, dir string) {
 func openclose(t testing.TB, dir string) {
 	cfg := local.Config{Path: dir}
 
-	be, err := local.Open(cfg)
+	be, err := local.Open(context.TODO(), cfg)
 	if err != nil {
 		t.Logf("Open returned error %v", err)
 	}

internal/backend/s3/s3.go

@@ -33,7 +33,7 @@ var _ restic.Backend = &Backend{}
 
 const defaultLayout = "default"
 
-func open(cfg Config, rt http.RoundTripper) (*Backend, error) {
+func open(ctx context.Context, cfg Config, rt http.RoundTripper) (*Backend, error) {
 	debug.Log("open, config %#v", cfg)
 
 	if cfg.MaxRetries > 0 {
@@ -87,7 +87,7 @@ func open(cfg Config, rt http.RoundTripper) (*Backend, error) {
 		cfg:    cfg,
 	}
 
-	l, err := backend.ParseLayout(be, cfg.Layout, defaultLayout, cfg.Prefix)
+	l, err := backend.ParseLayout(ctx, be, cfg.Layout, defaultLayout, cfg.Prefix)
 	if err != nil {
 		return nil, err
 	}
@@ -99,14 +99,14 @@ func open(cfg Config, rt http.RoundTripper) (*Backend, error) {
 
 // Open opens the S3 backend at bucket and region. The bucket is created if it
 // does not exist yet.
-func Open(cfg Config, rt http.RoundTripper) (restic.Backend, error) {
+func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend, error) {
-	return open(cfg, rt)
+	return open(ctx, cfg, rt)
 }
 
 // Create opens the S3 backend at bucket and region and creates the bucket if
 // it does not exist yet.
 func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend, error) {
-	be, err := open(cfg, rt)
+	be, err := open(ctx, cfg, rt)
 	if err != nil {
 		return nil, errors.Wrap(err, "open")
 	}
@@ -179,7 +179,7 @@ func (fi fileInfo) IsDir() bool        { return fi.isDir }   // abbreviation for
 func (fi fileInfo) Sys() interface{}   { return nil }        // underlying data source (can return nil)
 
 // ReadDir returns the entries for a directory.
-func (be *Backend) ReadDir(dir string) (list []os.FileInfo, err error) {
+func (be *Backend) ReadDir(ctx context.Context, dir string) (list []os.FileInfo, err error) {
 	debug.Log("ReadDir(%v)", dir)
 
 	// make sure dir ends with a slash
@@ -187,7 +187,7 @@ func (be *Backend) ReadDir(dir string) (list []os.FileInfo, err error) {
 		dir += "/"
 	}
 
-	ctx, cancel := context.WithCancel(context.TODO())
+	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
 
 	for obj := range be.client.ListObjects(ctx, be.cfg.Bucket, minio.ListObjectsOptions{

internal/backend/sftp/layout_test.go

@@ -42,7 +42,7 @@ func TestLayout(t *testing.T) {
 			rtest.SetupTarTestFixture(t, path, filepath.Join("..", "testdata", test.filename))
 
 			repo := filepath.Join(path, "repo")
-			be, err := sftp.Open(sftp.Config{
+			be, err := sftp.Open(context.TODO(), sftp.Config{
 				Command: fmt.Sprintf("%q -e", sftpServer),
 				Path:    repo,
 				Layout:  test.layout,

internal/backend/sftp/sftp.go

@@ -109,7 +109,7 @@ func (r *SFTP) clientError() error {
 // Open opens an sftp backend as described by the config by running
 // "ssh" with the appropriate arguments (or cfg.Command, if set). The function
 // preExec is run just before, postExec just after starting a program.
-func Open(cfg Config) (*SFTP, error) {
+func Open(ctx context.Context, cfg Config) (*SFTP, error) {
 	debug.Log("open backend with config %#v", cfg)
 
 	cmd, args, err := buildSSHCommand(cfg)
@@ -123,7 +123,7 @@ func Open(cfg Config) (*SFTP, error) {
 		return nil, err
 	}
 
-	sftp.Layout, err = backend.ParseLayout(sftp, cfg.Layout, defaultLayout, cfg.Path)
+	sftp.Layout, err = backend.ParseLayout(ctx, sftp, cfg.Layout, defaultLayout, cfg.Path)
 	if err != nil {
 		return nil, err
 	}
@@ -152,7 +152,7 @@ func (r *SFTP) Join(p ...string) string {
 }
 
 // ReadDir returns the entries for a directory.
-func (r *SFTP) ReadDir(dir string) ([]os.FileInfo, error) {
+func (r *SFTP) ReadDir(ctx context.Context, dir string) ([]os.FileInfo, error) {
 	fi, err := r.c.ReadDir(dir)
 
 	// sftp client does not specify dir name on error, so add it here
@@ -207,7 +207,7 @@ func buildSSHCommand(cfg Config) (cmd string, args []string, err error) {
 // Create creates an sftp backend as described by the config by running "ssh"
 // with the appropriate arguments (or cfg.Command, if set). The function
 // preExec is run just before, postExec just after starting a program.
-func Create(cfg Config) (*SFTP, error) {
+func Create(ctx context.Context, cfg Config) (*SFTP, error) {
 	cmd, args, err := buildSSHCommand(cfg)
 	if err != nil {
 		return nil, err
@@ -219,7 +219,7 @@ func Create(cfg Config) (*SFTP, error) {
 		return nil, err
 	}
 
-	sftp.Layout, err = backend.ParseLayout(sftp, cfg.Layout, defaultLayout, cfg.Path)
+	sftp.Layout, err = backend.ParseLayout(ctx, sftp, cfg.Layout, defaultLayout, cfg.Path)
 	if err != nil {
 		return nil, err
 	}
@@ -241,7 +241,7 @@ func Create(cfg Config) (*SFTP, error) {
 	}
 
 	// open backend
-	return Open(cfg)
+	return Open(ctx, cfg)
 }
 
 // Location returns this backend's location (the directory name).
@@ -467,8 +467,8 @@ func (r *SFTP) Close() error {
 	return nil
 }
 
-func (r *SFTP) deleteRecursive(name string) error {
+func (r *SFTP) deleteRecursive(ctx context.Context, name string) error {
-	entries, err := r.ReadDir(name)
+	entries, err := r.ReadDir(ctx, name)
 	if err != nil {
 		return errors.Wrap(err, "ReadDir")
 	}
@@ -476,7 +476,7 @@ func (r *SFTP) deleteRecursive(name string) error {
 	for _, fi := range entries {
 		itemName := r.Join(name, fi.Name())
 		if fi.IsDir() {
-			err := r.deleteRecursive(itemName)
+			err := r.deleteRecursive(ctx, itemName)
 			if err != nil {
 				return errors.Wrap(err, "ReadDir")
 			}
@@ -499,6 +499,6 @@ func (r *SFTP) deleteRecursive(name string) error {
 }
 
 // Delete removes all data in the backend.
-func (r *SFTP) Delete(context.Context) error {
+func (r *SFTP) Delete(ctx context.Context) error {
-	return r.deleteRecursive(r.p)
+	return r.deleteRecursive(ctx, r.p)
 }

internal/backend/sftp/sftp_test.go

@@ -1,6 +1,7 @@
 package sftp_test
 
 import (
+	"context"
 	"fmt"
 	"io/ioutil"
 	"os"
@@ -50,13 +51,13 @@ func newTestSuite(t testing.TB) *test.Suite {
 		// CreateFn is a function that creates a temporary repository for the tests.
 		Create: func(config interface{}) (restic.Backend, error) {
 			cfg := config.(sftp.Config)
-			return sftp.Create(cfg)
+			return sftp.Create(context.TODO(), cfg)
 		},
 
 		// OpenFn is a function that opens a previously created temporary repository.
 		Open: func(config interface{}) (restic.Backend, error) {
 			cfg := config.(sftp.Config)
-			return sftp.Open(cfg)
+			return sftp.Open(context.TODO(), cfg)
 		},
 
 		// CleanupFn removes data created during the tests.

internal/repository/testing.go

@@ -76,7 +76,7 @@ func TestRepository(t testing.TB) (r restic.Repository, cleanup func()) {
 	if dir != "" {
 		_, err := os.Stat(dir)
 		if err != nil {
-			be, err := local.Create(local.Config{Path: dir})
+			be, err := local.Create(context.TODO(), local.Config{Path: dir})
 			if err != nil {
 				t.Fatalf("error creating local backend at %v: %v", dir, err)
 			}
@@ -93,7 +93,7 @@ func TestRepository(t testing.TB) (r restic.Repository, cleanup func()) {
 
 // TestOpenLocal opens a local repository.
 func TestOpenLocal(t testing.TB, dir string) (r restic.Repository) {
-	be, err := local.Open(local.Config{Path: dir})
+	be, err := local.Open(context.TODO(), local.Config{Path: dir})
 	if err != nil {
 		t.Fatal(err)
 	}