diff --git a/cmd/restic/global.go b/cmd/restic/global.go index 08c8860a0..663f68e96 100644 --- a/cmd/restic/global.go +++ b/cmd/restic/global.go @@ -687,15 +687,15 @@ func open(s string, gopts GlobalOptions, opts options.Options) (restic.Backend, switch loc.Scheme { case "local": - be, err = local.Open(cfg.(local.Config)) + be, err = local.Open(globalOptions.ctx, cfg.(local.Config)) // wrap the backend in a LimitBackend so that the throughput is limited be = limiter.LimitBackend(be, lim) case "sftp": - be, err = sftp.Open(cfg.(sftp.Config)) + be, err = sftp.Open(globalOptions.ctx, cfg.(sftp.Config)) // wrap the backend in a LimitBackend so that the throughput is limited be = limiter.LimitBackend(be, lim) case "s3": - be, err = s3.Open(cfg.(s3.Config), rt) + be, err = s3.Open(globalOptions.ctx, cfg.(s3.Config), rt) case "gs": be, err = gs.Open(cfg.(gs.Config), rt) case "azure": @@ -754,9 +754,9 @@ func create(s string, opts options.Options) (restic.Backend, error) { switch loc.Scheme { case "local": - return local.Create(cfg.(local.Config)) + return local.Create(globalOptions.ctx, cfg.(local.Config)) case "sftp": - return sftp.Create(cfg.(sftp.Config)) + return sftp.Create(globalOptions.ctx, cfg.(sftp.Config)) case "s3": return s3.Create(globalOptions.ctx, cfg.(s3.Config), rt) case "gs": diff --git a/internal/backend/layout.go b/internal/backend/layout.go index cc376da03..e916df705 100644 --- a/internal/backend/layout.go +++ b/internal/backend/layout.go @@ -1,6 +1,7 @@ package backend import ( + "context" "fmt" "os" "path/filepath" @@ -24,7 +25,7 @@ type Layout interface { // Filesystem is the abstraction of a file system used for a backend. type Filesystem interface { Join(...string) string - ReadDir(string) ([]os.FileInfo, error) + ReadDir(context.Context, string) ([]os.FileInfo, error) IsNotExist(error) bool } @@ -36,7 +37,7 @@ type LocalFilesystem struct { } // ReadDir returns all entries of a directory. -func (l *LocalFilesystem) ReadDir(dir string) ([]os.FileInfo, error) { +func (l *LocalFilesystem) ReadDir(ctx context.Context, dir string) ([]os.FileInfo, error) { f, err := fs.Open(dir) if err != nil { return nil, err @@ -68,8 +69,8 @@ func (l *LocalFilesystem) IsNotExist(err error) bool { var backendFilenameLength = len(restic.ID{}) * 2 var backendFilename = regexp.MustCompile(fmt.Sprintf("^[a-fA-F0-9]{%d}$", backendFilenameLength)) -func hasBackendFile(fs Filesystem, dir string) (bool, error) { - entries, err := fs.ReadDir(dir) +func hasBackendFile(ctx context.Context, fs Filesystem, dir string) (bool, error) { + entries, err := fs.ReadDir(ctx, dir) if err != nil && fs.IsNotExist(errors.Cause(err)) { return false, nil } @@ -94,20 +95,20 @@ var ErrLayoutDetectionFailed = errors.New("auto-detecting the filesystem layout // DetectLayout tries to find out which layout is used in a local (or sftp) // filesystem at the given path. If repo is nil, an instance of LocalFilesystem // is used. -func DetectLayout(repo Filesystem, dir string) (Layout, error) { +func DetectLayout(ctx context.Context, repo Filesystem, dir string) (Layout, error) { debug.Log("detect layout at %v", dir) if repo == nil { repo = &LocalFilesystem{} } // key file in the "keys" dir (DefaultLayout) - foundKeysFile, err := hasBackendFile(repo, repo.Join(dir, defaultLayoutPaths[restic.KeyFile])) + foundKeysFile, err := hasBackendFile(ctx, repo, repo.Join(dir, defaultLayoutPaths[restic.KeyFile])) if err != nil { return nil, err } // key file in the "key" dir (S3LegacyLayout) - foundKeyFile, err := hasBackendFile(repo, repo.Join(dir, s3LayoutPaths[restic.KeyFile])) + foundKeyFile, err := hasBackendFile(ctx, repo, repo.Join(dir, s3LayoutPaths[restic.KeyFile])) if err != nil { return nil, err } @@ -134,7 +135,7 @@ func DetectLayout(repo Filesystem, dir string) (Layout, error) { // ParseLayout parses the config string and returns a Layout. When layout is // the empty string, DetectLayout is used. If that fails, defaultLayout is used. -func ParseLayout(repo Filesystem, layout, defaultLayout, path string) (l Layout, err error) { +func ParseLayout(ctx context.Context, repo Filesystem, layout, defaultLayout, path string) (l Layout, err error) { debug.Log("parse layout string %q for backend at %v", layout, path) switch layout { case "default": @@ -148,12 +149,12 @@ func ParseLayout(repo Filesystem, layout, defaultLayout, path string) (l Layout, Join: repo.Join, } case "": - l, err = DetectLayout(repo, path) + l, err = DetectLayout(ctx, repo, path) // use the default layout if auto detection failed if errors.Cause(err) == ErrLayoutDetectionFailed && defaultLayout != "" { debug.Log("error: %v, use default layout %v", err, defaultLayout) - return ParseLayout(repo, defaultLayout, "", path) + return ParseLayout(ctx, repo, defaultLayout, "", path) } if err != nil { diff --git a/internal/backend/layout_test.go b/internal/backend/layout_test.go index 104e54aa3..d319a0b2d 100644 --- a/internal/backend/layout_test.go +++ b/internal/backend/layout_test.go @@ -1,6 +1,7 @@ package backend import ( + "context" "fmt" "path" "path/filepath" @@ -371,7 +372,7 @@ func TestDetectLayout(t *testing.T) { t.Run(fmt.Sprintf("%v/fs-%T", test.filename, fs), func(t *testing.T) { rtest.SetupTarTestFixture(t, path, filepath.Join("testdata", test.filename)) - layout, err := DetectLayout(fs, filepath.Join(path, "repo")) + layout, err := DetectLayout(context.TODO(), fs, filepath.Join(path, "repo")) if err != nil { t.Fatal(err) } @@ -409,7 +410,7 @@ func TestParseLayout(t *testing.T) { for _, test := range tests { t.Run(test.layoutName, func(t *testing.T) { - layout, err := ParseLayout(&LocalFilesystem{}, test.layoutName, test.defaultLayoutName, filepath.Join(path, "repo")) + layout, err := ParseLayout(context.TODO(), &LocalFilesystem{}, test.layoutName, test.defaultLayoutName, filepath.Join(path, "repo")) if err != nil { t.Fatal(err) } @@ -441,7 +442,7 @@ func TestParseLayoutInvalid(t *testing.T) { for _, name := range invalidNames { t.Run(name, func(t *testing.T) { - layout, err := ParseLayout(nil, name, "", path) + layout, err := ParseLayout(context.TODO(), nil, name, "", path) if err == nil { t.Fatalf("expected error not found for layout name %v, layout is %v", name, layout) } diff --git a/internal/backend/local/layout_test.go b/internal/backend/local/layout_test.go index 8cd0be031..5b1135253 100644 --- a/internal/backend/local/layout_test.go +++ b/internal/backend/local/layout_test.go @@ -36,7 +36,7 @@ func TestLayout(t *testing.T) { rtest.SetupTarTestFixture(t, path, filepath.Join("..", "testdata", test.filename)) repo := filepath.Join(path, "repo") - be, err := Open(Config{ + be, err := Open(context.TODO(), Config{ Path: repo, Layout: test.layout, }) diff --git a/internal/backend/local/local.go b/internal/backend/local/local.go index 34ac3a20e..5261a0852 100644 --- a/internal/backend/local/local.go +++ b/internal/backend/local/local.go @@ -27,9 +27,9 @@ var _ restic.Backend = &Local{} const defaultLayout = "default" // Open opens the local backend as specified by config. -func Open(cfg Config) (*Local, error) { +func Open(ctx context.Context, cfg Config) (*Local, error) { debug.Log("open local backend at %v (layout %q)", cfg.Path, cfg.Layout) - l, err := backend.ParseLayout(&backend.LocalFilesystem{}, cfg.Layout, defaultLayout, cfg.Path) + l, err := backend.ParseLayout(ctx, &backend.LocalFilesystem{}, cfg.Layout, defaultLayout, cfg.Path) if err != nil { return nil, err } @@ -39,10 +39,10 @@ func Open(cfg Config) (*Local, error) { // Create creates all the necessary files and directories for a new local // backend at dir. Afterwards a new config blob should be created. -func Create(cfg Config) (*Local, error) { +func Create(ctx context.Context, cfg Config) (*Local, error) { debug.Log("create local backend at %v (layout %q)", cfg.Path, cfg.Layout) - l, err := backend.ParseLayout(&backend.LocalFilesystem{}, cfg.Layout, defaultLayout, cfg.Path) + l, err := backend.ParseLayout(ctx, &backend.LocalFilesystem{}, cfg.Layout, defaultLayout, cfg.Path) if err != nil { return nil, err } diff --git a/internal/backend/local/local_test.go b/internal/backend/local/local_test.go index 4ca3fdb71..70b5e771e 100644 --- a/internal/backend/local/local_test.go +++ b/internal/backend/local/local_test.go @@ -1,6 +1,7 @@ package local_test import ( + "context" "io/ioutil" "os" "path/filepath" @@ -32,13 +33,13 @@ func newTestSuite(t testing.TB) *test.Suite { // CreateFn is a function that creates a temporary repository for the tests. Create: func(config interface{}) (restic.Backend, error) { cfg := config.(local.Config) - return local.Create(cfg) + return local.Create(context.TODO(), cfg) }, // OpenFn is a function that opens a previously created temporary repository. Open: func(config interface{}) (restic.Backend, error) { cfg := config.(local.Config) - return local.Open(cfg) + return local.Open(context.TODO(), cfg) }, // CleanupFn removes data created during the tests. @@ -91,7 +92,7 @@ func empty(t testing.TB, dir string) { func openclose(t testing.TB, dir string) { cfg := local.Config{Path: dir} - be, err := local.Open(cfg) + be, err := local.Open(context.TODO(), cfg) if err != nil { t.Logf("Open returned error %v", err) } diff --git a/internal/backend/s3/s3.go b/internal/backend/s3/s3.go index b8f8c833d..d1243bfd5 100644 --- a/internal/backend/s3/s3.go +++ b/internal/backend/s3/s3.go @@ -33,7 +33,7 @@ var _ restic.Backend = &Backend{} const defaultLayout = "default" -func open(cfg Config, rt http.RoundTripper) (*Backend, error) { +func open(ctx context.Context, cfg Config, rt http.RoundTripper) (*Backend, error) { debug.Log("open, config %#v", cfg) if cfg.MaxRetries > 0 { @@ -87,7 +87,7 @@ func open(cfg Config, rt http.RoundTripper) (*Backend, error) { cfg: cfg, } - l, err := backend.ParseLayout(be, cfg.Layout, defaultLayout, cfg.Prefix) + l, err := backend.ParseLayout(ctx, be, cfg.Layout, defaultLayout, cfg.Prefix) if err != nil { return nil, err } @@ -99,14 +99,14 @@ func open(cfg Config, rt http.RoundTripper) (*Backend, error) { // Open opens the S3 backend at bucket and region. The bucket is created if it // does not exist yet. -func Open(cfg Config, rt http.RoundTripper) (restic.Backend, error) { - return open(cfg, rt) +func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend, error) { + return open(ctx, cfg, rt) } // Create opens the S3 backend at bucket and region and creates the bucket if // it does not exist yet. func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend, error) { - be, err := open(cfg, rt) + be, err := open(ctx, cfg, rt) if err != nil { return nil, errors.Wrap(err, "open") } @@ -179,7 +179,7 @@ func (fi fileInfo) IsDir() bool { return fi.isDir } // abbreviation for func (fi fileInfo) Sys() interface{} { return nil } // underlying data source (can return nil) // ReadDir returns the entries for a directory. -func (be *Backend) ReadDir(dir string) (list []os.FileInfo, err error) { +func (be *Backend) ReadDir(ctx context.Context, dir string) (list []os.FileInfo, err error) { debug.Log("ReadDir(%v)", dir) // make sure dir ends with a slash @@ -187,7 +187,7 @@ func (be *Backend) ReadDir(dir string) (list []os.FileInfo, err error) { dir += "/" } - ctx, cancel := context.WithCancel(context.TODO()) + ctx, cancel := context.WithCancel(ctx) defer cancel() for obj := range be.client.ListObjects(ctx, be.cfg.Bucket, minio.ListObjectsOptions{ diff --git a/internal/backend/sftp/layout_test.go b/internal/backend/sftp/layout_test.go index 50878c778..0d0214669 100644 --- a/internal/backend/sftp/layout_test.go +++ b/internal/backend/sftp/layout_test.go @@ -42,7 +42,7 @@ func TestLayout(t *testing.T) { rtest.SetupTarTestFixture(t, path, filepath.Join("..", "testdata", test.filename)) repo := filepath.Join(path, "repo") - be, err := sftp.Open(sftp.Config{ + be, err := sftp.Open(context.TODO(), sftp.Config{ Command: fmt.Sprintf("%q -e", sftpServer), Path: repo, Layout: test.layout, diff --git a/internal/backend/sftp/sftp.go b/internal/backend/sftp/sftp.go index fc2de79d8..0a2c15e75 100644 --- a/internal/backend/sftp/sftp.go +++ b/internal/backend/sftp/sftp.go @@ -109,7 +109,7 @@ func (r *SFTP) clientError() error { // Open opens an sftp backend as described by the config by running // "ssh" with the appropriate arguments (or cfg.Command, if set). The function // preExec is run just before, postExec just after starting a program. -func Open(cfg Config) (*SFTP, error) { +func Open(ctx context.Context, cfg Config) (*SFTP, error) { debug.Log("open backend with config %#v", cfg) cmd, args, err := buildSSHCommand(cfg) @@ -123,7 +123,7 @@ func Open(cfg Config) (*SFTP, error) { return nil, err } - sftp.Layout, err = backend.ParseLayout(sftp, cfg.Layout, defaultLayout, cfg.Path) + sftp.Layout, err = backend.ParseLayout(ctx, sftp, cfg.Layout, defaultLayout, cfg.Path) if err != nil { return nil, err } @@ -152,7 +152,7 @@ func (r *SFTP) Join(p ...string) string { } // ReadDir returns the entries for a directory. -func (r *SFTP) ReadDir(dir string) ([]os.FileInfo, error) { +func (r *SFTP) ReadDir(ctx context.Context, dir string) ([]os.FileInfo, error) { fi, err := r.c.ReadDir(dir) // sftp client does not specify dir name on error, so add it here @@ -207,7 +207,7 @@ func buildSSHCommand(cfg Config) (cmd string, args []string, err error) { // Create creates an sftp backend as described by the config by running "ssh" // with the appropriate arguments (or cfg.Command, if set). The function // preExec is run just before, postExec just after starting a program. -func Create(cfg Config) (*SFTP, error) { +func Create(ctx context.Context, cfg Config) (*SFTP, error) { cmd, args, err := buildSSHCommand(cfg) if err != nil { return nil, err @@ -219,7 +219,7 @@ func Create(cfg Config) (*SFTP, error) { return nil, err } - sftp.Layout, err = backend.ParseLayout(sftp, cfg.Layout, defaultLayout, cfg.Path) + sftp.Layout, err = backend.ParseLayout(ctx, sftp, cfg.Layout, defaultLayout, cfg.Path) if err != nil { return nil, err } @@ -241,7 +241,7 @@ func Create(cfg Config) (*SFTP, error) { } // open backend - return Open(cfg) + return Open(ctx, cfg) } // Location returns this backend's location (the directory name). @@ -467,8 +467,8 @@ func (r *SFTP) Close() error { return nil } -func (r *SFTP) deleteRecursive(name string) error { - entries, err := r.ReadDir(name) +func (r *SFTP) deleteRecursive(ctx context.Context, name string) error { + entries, err := r.ReadDir(ctx, name) if err != nil { return errors.Wrap(err, "ReadDir") } @@ -476,7 +476,7 @@ func (r *SFTP) deleteRecursive(name string) error { for _, fi := range entries { itemName := r.Join(name, fi.Name()) if fi.IsDir() { - err := r.deleteRecursive(itemName) + err := r.deleteRecursive(ctx, itemName) if err != nil { return errors.Wrap(err, "ReadDir") } @@ -499,6 +499,6 @@ func (r *SFTP) deleteRecursive(name string) error { } // Delete removes all data in the backend. -func (r *SFTP) Delete(context.Context) error { - return r.deleteRecursive(r.p) +func (r *SFTP) Delete(ctx context.Context) error { + return r.deleteRecursive(ctx, r.p) } diff --git a/internal/backend/sftp/sftp_test.go b/internal/backend/sftp/sftp_test.go index f32e04499..61bc49dc8 100644 --- a/internal/backend/sftp/sftp_test.go +++ b/internal/backend/sftp/sftp_test.go @@ -1,6 +1,7 @@ package sftp_test import ( + "context" "fmt" "io/ioutil" "os" @@ -50,13 +51,13 @@ func newTestSuite(t testing.TB) *test.Suite { // CreateFn is a function that creates a temporary repository for the tests. Create: func(config interface{}) (restic.Backend, error) { cfg := config.(sftp.Config) - return sftp.Create(cfg) + return sftp.Create(context.TODO(), cfg) }, // OpenFn is a function that opens a previously created temporary repository. Open: func(config interface{}) (restic.Backend, error) { cfg := config.(sftp.Config) - return sftp.Open(cfg) + return sftp.Open(context.TODO(), cfg) }, // CleanupFn removes data created during the tests. diff --git a/internal/repository/testing.go b/internal/repository/testing.go index e070cdc8a..c5211fea6 100644 --- a/internal/repository/testing.go +++ b/internal/repository/testing.go @@ -76,7 +76,7 @@ func TestRepository(t testing.TB) (r restic.Repository, cleanup func()) { if dir != "" { _, err := os.Stat(dir) if err != nil { - be, err := local.Create(local.Config{Path: dir}) + be, err := local.Create(context.TODO(), local.Config{Path: dir}) if err != nil { t.Fatalf("error creating local backend at %v: %v", dir, err) } @@ -93,7 +93,7 @@ func TestRepository(t testing.TB) (r restic.Repository, cleanup func()) { // TestOpenLocal opens a local repository. func TestOpenLocal(t testing.TB, dir string) (r restic.Repository) { - be, err := local.Open(local.Config{Path: dir}) + be, err := local.Open(context.TODO(), local.Config{Path: dir}) if err != nil { t.Fatal(err) }