From d0668b695d00381cdd5190cefeb38678a2ae34f0 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 31 Oct 2021 22:56:57 +0100 Subject: [PATCH 01/11] Remove unnecessary context.WithCancel calls The gopts.ctx is cancelled when the main() method of restic exits. --- cmd/restic/cmd_copy.go | 4 +--- cmd/restic/cmd_diff.go | 4 +--- cmd/restic/cmd_find.go | 4 +--- cmd/restic/cmd_forget.go | 5 +---- cmd/restic/cmd_key.go | 4 +--- cmd/restic/cmd_ls.go | 5 +---- cmd/restic/cmd_migrate.go | 13 ++++++------- cmd/restic/cmd_snapshots.go | 5 +---- cmd/restic/cmd_stats.go | 4 +--- cmd/restic/cmd_tag.go | 3 +-- 10 files changed, 15 insertions(+), 36 deletions(-) diff --git a/cmd/restic/cmd_copy.go b/cmd/restic/cmd_copy.go index bd56d1182..0324468a3 100644 --- a/cmd/restic/cmd_copy.go +++ b/cmd/restic/cmd_copy.go @@ -62,9 +62,7 @@ func runCopy(opts CopyOptions, gopts GlobalOptions, args []string) error { gopts, secondaryGopts = secondaryGopts, gopts } - ctx, cancel := context.WithCancel(gopts.ctx) - defer cancel() - + ctx := gopts.ctx srcRepo, err := OpenRepository(gopts) if err != nil { return err diff --git a/cmd/restic/cmd_diff.go b/cmd/restic/cmd_diff.go index 5fdd28d97..76b377928 100644 --- a/cmd/restic/cmd_diff.go +++ b/cmd/restic/cmd_diff.go @@ -326,9 +326,7 @@ func runDiff(opts DiffOptions, gopts GlobalOptions, args []string) error { return errors.Fatalf("specify two snapshot IDs") } - ctx, cancel := context.WithCancel(gopts.ctx) - defer cancel() - + ctx := gopts.ctx repo, err := OpenRepository(gopts) if err != nil { return err diff --git a/cmd/restic/cmd_find.go b/cmd/restic/cmd_find.go index 5f1c07ab2..96c11e655 100644 --- a/cmd/restic/cmd_find.go +++ b/cmd/restic/cmd_find.go @@ -590,9 +590,7 @@ func runFind(opts FindOptions, gopts GlobalOptions, args []string) error { return err } - ctx, cancel := context.WithCancel(gopts.ctx) - defer cancel() - + ctx := gopts.ctx f := &Finder{ repo: repo, pat: pat, diff --git a/cmd/restic/cmd_forget.go b/cmd/restic/cmd_forget.go index 29d3c81ff..da7681867 100644 --- a/cmd/restic/cmd_forget.go +++ b/cmd/restic/cmd_forget.go @@ -1,7 +1,6 @@ package main import ( - "context" "encoding/json" "io" @@ -122,9 +121,7 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error { } } - ctx, cancel := context.WithCancel(gopts.ctx) - defer cancel() - + ctx := gopts.ctx var snapshots restic.Snapshots removeSnIDs := restic.NewIDSet() diff --git a/cmd/restic/cmd_key.go b/cmd/restic/cmd_key.go index 69c2542b7..ac64078f3 100644 --- a/cmd/restic/cmd_key.go +++ b/cmd/restic/cmd_key.go @@ -202,9 +202,7 @@ func runKey(gopts GlobalOptions, args []string) error { return errors.Fatal("wrong number of arguments") } - ctx, cancel := context.WithCancel(gopts.ctx) - defer cancel() - + ctx := gopts.ctx repo, err := OpenRepository(gopts) if err != nil { return err diff --git a/cmd/restic/cmd_ls.go b/cmd/restic/cmd_ls.go index df7fd7b18..ddd02e65d 100644 --- a/cmd/restic/cmd_ls.go +++ b/cmd/restic/cmd_ls.go @@ -1,7 +1,6 @@ package main import ( - "context" "encoding/json" "os" "strings" @@ -175,9 +174,7 @@ func runLs(opts LsOptions, gopts GlobalOptions, args []string) error { return err } - ctx, cancel := context.WithCancel(gopts.ctx) - defer cancel() - + ctx := gopts.ctx var ( printSnapshot func(sn *restic.Snapshot) printNode func(path string, node *restic.Node) diff --git a/cmd/restic/cmd_migrate.go b/cmd/restic/cmd_migrate.go index b0a5319ea..85ebc2e99 100644 --- a/cmd/restic/cmd_migrate.go +++ b/cmd/restic/cmd_migrate.go @@ -1,6 +1,8 @@ package main import ( + "context" + "github.com/restic/restic/internal/migrations" "github.com/restic/restic/internal/restic" @@ -39,8 +41,7 @@ func init() { f.BoolVarP(&migrateOptions.Force, "force", "f", false, `apply a migration a second time`) } -func checkMigrations(opts MigrateOptions, gopts GlobalOptions, repo restic.Repository) error { - ctx := gopts.ctx +func checkMigrations(ctx context.Context, repo restic.Repository) error { Printf("available migrations:\n") found := false @@ -63,9 +64,7 @@ func checkMigrations(opts MigrateOptions, gopts GlobalOptions, repo restic.Repos return nil } -func applyMigrations(opts MigrateOptions, gopts GlobalOptions, repo restic.Repository, args []string) error { - ctx := gopts.ctx - +func applyMigrations(ctx context.Context, opts MigrateOptions, gopts GlobalOptions, repo restic.Repository, args []string) error { var firsterr error for _, name := range args { for _, m := range migrations.All { @@ -130,8 +129,8 @@ func runMigrate(opts MigrateOptions, gopts GlobalOptions, args []string) error { } if len(args) == 0 { - return checkMigrations(opts, gopts, repo) + return checkMigrations(gopts.ctx, repo) } - return applyMigrations(opts, gopts, repo, args) + return applyMigrations(gopts.ctx, opts, gopts, repo, args) } diff --git a/cmd/restic/cmd_snapshots.go b/cmd/restic/cmd_snapshots.go index 80a205dcf..b394a89af 100644 --- a/cmd/restic/cmd_snapshots.go +++ b/cmd/restic/cmd_snapshots.go @@ -1,7 +1,6 @@ package main import ( - "context" "encoding/json" "fmt" "io" @@ -71,9 +70,7 @@ func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) erro } } - ctx, cancel := context.WithCancel(gopts.ctx) - defer cancel() - + ctx := gopts.ctx var snapshots restic.Snapshots for sn := range FindFilteredSnapshots(ctx, repo.Backend(), repo, opts.Hosts, opts.Tags, opts.Paths, args) { snapshots = append(snapshots, sn) diff --git a/cmd/restic/cmd_stats.go b/cmd/restic/cmd_stats.go index d845cb223..dbc16f91e 100644 --- a/cmd/restic/cmd_stats.go +++ b/cmd/restic/cmd_stats.go @@ -74,9 +74,7 @@ func runStats(gopts GlobalOptions, args []string) error { return err } - ctx, cancel := context.WithCancel(gopts.ctx) - defer cancel() - + ctx := gopts.ctx repo, err := OpenRepository(gopts) if err != nil { return err diff --git a/cmd/restic/cmd_tag.go b/cmd/restic/cmd_tag.go index 7a2561235..cd52e7241 100644 --- a/cmd/restic/cmd_tag.go +++ b/cmd/restic/cmd_tag.go @@ -118,8 +118,7 @@ func runTag(opts TagOptions, gopts GlobalOptions, args []string) error { } changeCnt := 0 - ctx, cancel := context.WithCancel(gopts.ctx) - defer cancel() + ctx := gopts.ctx for sn := range FindFilteredSnapshots(ctx, repo.Backend(), repo, opts.Hosts, opts.Tags, opts.Paths, args) { changed, err := changeTags(ctx, repo, sn, opts.SetTags.Flatten(), opts.AddTags.Flatten(), opts.RemoveTags.Flatten()) if err != nil { From ab819b23444203ff5aa2bca0c2aa0d6d75a808ae Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 31 Oct 2021 23:01:47 +0100 Subject: [PATCH 02/11] key: Cleanup method signatures --- cmd/restic/cmd_key.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/cmd/restic/cmd_key.go b/cmd/restic/cmd_key.go index ac64078f3..6a18e8b9e 100644 --- a/cmd/restic/cmd_key.go +++ b/cmd/restic/cmd_key.go @@ -120,18 +120,18 @@ func getNewPassword(gopts GlobalOptions) (string, error) { "enter password again: ") } -func addKey(gopts GlobalOptions, repo *repository.Repository) error { +func addKey(ctx context.Context, repo *repository.Repository, gopts GlobalOptions) error { pw, err := getNewPassword(gopts) if err != nil { return err } - id, err := repository.AddKey(gopts.ctx, repo, pw, keyUsername, keyHostname, repo.Key()) + id, err := repository.AddKey(ctx, repo, pw, keyUsername, keyHostname, repo.Key()) if err != nil { return errors.Fatalf("creating new key failed: %v\n", err) } - err = switchToNewKeyAndRemoveIfBroken(gopts.ctx, repo, id, pw) + err = switchToNewKeyAndRemoveIfBroken(ctx, repo, id, pw) if err != nil { return err } @@ -156,25 +156,25 @@ func deleteKey(ctx context.Context, repo *repository.Repository, name string) er return nil } -func changePassword(gopts GlobalOptions, repo *repository.Repository) error { +func changePassword(ctx context.Context, repo *repository.Repository, gopts GlobalOptions) error { pw, err := getNewPassword(gopts) if err != nil { return err } - id, err := repository.AddKey(gopts.ctx, repo, pw, "", "", repo.Key()) + id, err := repository.AddKey(ctx, repo, pw, "", "", repo.Key()) if err != nil { return errors.Fatalf("creating new key failed: %v\n", err) } oldID := repo.KeyName() - err = switchToNewKeyAndRemoveIfBroken(gopts.ctx, repo, id, pw) + err = switchToNewKeyAndRemoveIfBroken(ctx, repo, id, pw) if err != nil { return err } h := restic.Handle{Type: restic.KeyFile, Name: oldID} - err = repo.Backend().Remove(gopts.ctx, h) + err = repo.Backend().Remove(ctx, h) if err != nil { return err } @@ -198,11 +198,11 @@ func switchToNewKeyAndRemoveIfBroken(ctx context.Context, repo *repository.Repos } func runKey(gopts GlobalOptions, args []string) error { + ctx := gopts.ctx if len(args) < 1 || (args[0] == "remove" && len(args) != 2) || (args[0] != "remove" && len(args) != 1) { return errors.Fatal("wrong number of arguments") } - ctx := gopts.ctx repo, err := OpenRepository(gopts) if err != nil { return err @@ -224,7 +224,7 @@ func runKey(gopts GlobalOptions, args []string) error { return err } - return addKey(gopts, repo) + return addKey(ctx, repo, gopts) case "remove": lock, err := lockRepoExclusive(ctx, repo) defer unlockRepo(lock) @@ -237,7 +237,7 @@ func runKey(gopts GlobalOptions, args []string) error { return err } - return deleteKey(gopts.ctx, repo, id) + return deleteKey(ctx, repo, id) case "passwd": lock, err := lockRepoExclusive(ctx, repo) defer unlockRepo(lock) @@ -245,7 +245,7 @@ func runKey(gopts GlobalOptions, args []string) error { return err } - return changePassword(gopts, repo) + return changePassword(ctx, repo, gopts) } return nil From 985722b102057c2ed167c01dbcf5cb7ecc9fe6c3 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 31 Oct 2021 23:08:13 +0100 Subject: [PATCH 03/11] Remove ctx from globalOptions Previously the global context was either accessed via gopts.ctx, stored in a local variable and then used within that function or sometimes both. This makes it very hard to follow which ctx or a wrapped version of it reaches which method. Thus just drop the context from the globalOptions struct and pass it explicitly to every command line handler method. --- cmd/restic/cmd_backup.go | 19 +++-- cmd/restic/cmd_cat.go | 25 +++--- cmd/restic/cmd_check.go | 21 ++--- cmd/restic/cmd_copy.go | 9 +- cmd/restic/cmd_debug.go | 32 +++---- cmd/restic/cmd_diff.go | 7 +- cmd/restic/cmd_dump.go | 10 +-- cmd/restic/cmd_find.go | 13 ++- cmd/restic/cmd_forget.go | 14 ++-- cmd/restic/cmd_init.go | 15 ++-- cmd/restic/cmd_key.go | 7 +- cmd/restic/cmd_list.go | 16 ++-- cmd/restic/cmd_ls.go | 12 +-- cmd/restic/cmd_migrate.go | 14 ++-- cmd/restic/cmd_mount.go | 11 +-- cmd/restic/cmd_prune.go | 53 ++++++------ cmd/restic/cmd_rebuild_index.go | 18 ++-- cmd/restic/cmd_recover.go | 28 +++---- cmd/restic/cmd_restore.go | 8 +- cmd/restic/cmd_self_update.go | 7 +- cmd/restic/cmd_snapshots.go | 10 +-- cmd/restic/cmd_stats.go | 9 +- cmd/restic/cmd_tag.go | 9 +- cmd/restic/cmd_unlock.go | 10 ++- cmd/restic/delete.go | 14 ++-- cmd/restic/global.go | 44 +++++----- cmd/restic/integration_fuse_test.go | 9 +- cmd/restic/integration_helpers_test.go | 2 - cmd/restic/integration_test.go | 112 ++++++++++++------------- 29 files changed, 280 insertions(+), 278 deletions(-) diff --git a/cmd/restic/cmd_backup.go b/cmd/restic/cmd_backup.go index e36471762..1bd86ceeb 100644 --- a/cmd/restic/cmd_backup.go +++ b/cmd/restic/cmd_backup.go @@ -57,8 +57,9 @@ Exit status is 3 if some source data could not be read (incomplete snapshot crea }, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { + ctx := globalCtx() var wg sync.WaitGroup - cancelCtx, cancel := context.WithCancel(globalOptions.ctx) + cancelCtx, cancel := context.WithCancel(ctx) defer func() { // shutdown termstatus cancel() @@ -72,7 +73,7 @@ Exit status is 3 if some source data could not be read (incomplete snapshot crea term.Run(cancelCtx) }() - return runBackup(backupOptions, globalOptions, term, args) + return runBackup(ctx, backupOptions, globalOptions, term, args) }, } @@ -527,7 +528,7 @@ func findParentSnapshot(ctx context.Context, repo restic.Repository, opts Backup return parentID, nil } -func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Terminal, args []string) error { +func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, term *termstatus.Terminal, args []string) error { err := opts.Check(gopts, args) if err != nil { return err @@ -550,7 +551,7 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina Verbosef("open repository\n") } - repo, err := OpenRepository(gopts) + repo, err := OpenRepository(ctx, gopts) if err != nil { return err } @@ -577,7 +578,7 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina progressReporter.SetMinUpdatePause(calculateProgressInterval(!gopts.Quiet, gopts.JSON)) - wg, wgCtx := errgroup.WithContext(gopts.ctx) + wg, wgCtx := errgroup.WithContext(ctx) cancelCtx, cancel := context.WithCancel(wgCtx) defer cancel() wg.Go(func() error { return progressReporter.Run(cancelCtx) }) @@ -585,7 +586,7 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina if !gopts.JSON { progressPrinter.V("lock repository") } - lock, err := lockRepo(gopts.ctx, repo) + lock, err := lockRepo(ctx, repo) defer unlockRepo(lock) if err != nil { return err @@ -605,7 +606,7 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina var parentSnapshotID *restic.ID if !opts.Stdin { - parentSnapshotID, err = findParentSnapshot(gopts.ctx, repo, opts, targets, timeStamp) + parentSnapshotID, err = findParentSnapshot(ctx, repo, opts, targets, timeStamp) if err != nil { return err } @@ -622,7 +623,7 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina if !gopts.JSON { progressPrinter.V("load index files") } - err = repo.LoadIndex(gopts.ctx) + err = repo.LoadIndex(ctx) if err != nil { return err } @@ -727,7 +728,7 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina if !gopts.JSON { progressPrinter.V("start backup on %v", targets) } - _, id, err := arch.Snapshot(gopts.ctx, targets, snapshotOpts) + _, id, err := arch.Snapshot(ctx, targets, snapshotOpts) // cleanly shutdown all running goroutines cancel() diff --git a/cmd/restic/cmd_cat.go b/cmd/restic/cmd_cat.go index 297c4cc85..dcdfc6e8c 100644 --- a/cmd/restic/cmd_cat.go +++ b/cmd/restic/cmd_cat.go @@ -1,6 +1,7 @@ package main import ( + "context" "encoding/json" "github.com/spf13/cobra" @@ -24,7 +25,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - return runCat(globalOptions, args) + return runCat(globalCtx(), globalOptions, args) }, } @@ -32,18 +33,18 @@ func init() { cmdRoot.AddCommand(cmdCat) } -func runCat(gopts GlobalOptions, args []string) error { +func runCat(ctx context.Context, gopts GlobalOptions, args []string) error { if len(args) < 1 || (args[0] != "masterkey" && args[0] != "config" && len(args) != 2) { return errors.Fatal("type or ID not specified") } - repo, err := OpenRepository(gopts) + repo, err := OpenRepository(ctx, gopts) if err != nil { return err } if !gopts.NoLock { - lock, err := lockRepo(gopts.ctx, repo) + lock, err := lockRepo(ctx, repo) if err != nil { return err } @@ -62,7 +63,7 @@ func runCat(gopts GlobalOptions, args []string) error { } // find snapshot id with prefix - id, err = restic.FindSnapshot(gopts.ctx, repo.Backend(), args[1]) + id, err = restic.FindSnapshot(ctx, repo.Backend(), args[1]) if err != nil { return errors.Fatalf("could not find snapshot: %v\n", err) } @@ -79,7 +80,7 @@ func runCat(gopts GlobalOptions, args []string) error { Println(string(buf)) return nil case "index": - buf, err := repo.LoadUnpacked(gopts.ctx, restic.IndexFile, id, nil) + buf, err := repo.LoadUnpacked(ctx, restic.IndexFile, id, nil) if err != nil { return err } @@ -87,7 +88,7 @@ func runCat(gopts GlobalOptions, args []string) error { Println(string(buf)) return nil case "snapshot": - sn, err := restic.LoadSnapshot(gopts.ctx, repo, id) + sn, err := restic.LoadSnapshot(ctx, repo, id) if err != nil { return err } @@ -100,7 +101,7 @@ func runCat(gopts GlobalOptions, args []string) error { Println(string(buf)) return nil case "key": - key, err := repository.LoadKey(gopts.ctx, repo, id.String()) + key, err := repository.LoadKey(ctx, repo, id.String()) if err != nil { return err } @@ -121,7 +122,7 @@ func runCat(gopts GlobalOptions, args []string) error { Println(string(buf)) return nil case "lock": - lock, err := restic.LoadLock(gopts.ctx, repo, id) + lock, err := restic.LoadLock(ctx, repo, id) if err != nil { return err } @@ -136,7 +137,7 @@ func runCat(gopts GlobalOptions, args []string) error { case "pack": h := restic.Handle{Type: restic.PackFile, Name: id.String()} - buf, err := backend.LoadAll(gopts.ctx, nil, repo.Backend(), h) + buf, err := backend.LoadAll(ctx, nil, repo.Backend(), h) if err != nil { return err } @@ -150,7 +151,7 @@ func runCat(gopts GlobalOptions, args []string) error { return err case "blob": - err = repo.LoadIndex(gopts.ctx) + err = repo.LoadIndex(ctx) if err != nil { return err } @@ -161,7 +162,7 @@ func runCat(gopts GlobalOptions, args []string) error { continue } - buf, err := repo.LoadBlob(gopts.ctx, t, id, nil) + buf, err := repo.LoadBlob(ctx, t, id, nil) if err != nil { return err } diff --git a/cmd/restic/cmd_check.go b/cmd/restic/cmd_check.go index ab871e645..899a6a232 100644 --- a/cmd/restic/cmd_check.go +++ b/cmd/restic/cmd_check.go @@ -1,6 +1,7 @@ package main import ( + "context" "io/ioutil" "math/rand" "strconv" @@ -34,7 +35,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - return runCheck(checkOptions, globalOptions, args) + return runCheck(globalCtx(), checkOptions, globalOptions, args) }, PreRunE: func(cmd *cobra.Command, args []string) error { return checkFlags(checkOptions) @@ -191,7 +192,7 @@ func prepareCheckCache(opts CheckOptions, gopts *GlobalOptions) (cleanup func()) return cleanup } -func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error { +func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args []string) error { if len(args) != 0 { return errors.Fatal("the check command expects no arguments, only options - please see `restic help check` for usage and flags") } @@ -202,14 +203,14 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error { return code, nil }) - repo, err := OpenRepository(gopts) + repo, err := OpenRepository(ctx, gopts) if err != nil { return err } if !gopts.NoLock { Verbosef("create exclusive lock for repository\n") - lock, err := lockRepoExclusive(gopts.ctx, repo) + lock, err := lockRepoExclusive(ctx, repo) defer unlockRepo(lock) if err != nil { return err @@ -217,13 +218,13 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error { } chkr := checker.New(repo, opts.CheckUnused) - err = chkr.LoadSnapshots(gopts.ctx) + err = chkr.LoadSnapshots(ctx) if err != nil { return err } Verbosef("load indexes\n") - hints, errs := chkr.LoadIndex(gopts.ctx) + hints, errs := chkr.LoadIndex(ctx) errorsFound := false suggestIndexRebuild := false @@ -260,7 +261,7 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error { errChan := make(chan error) Verbosef("check all packs\n") - go chkr.Packs(gopts.ctx, errChan) + go chkr.Packs(ctx, errChan) for err := range errChan { if checker.IsOrphanedPack(err) { @@ -287,7 +288,7 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error { defer wg.Done() bar := newProgressMax(!gopts.Quiet, 0, "snapshots") defer bar.Done() - chkr.Structure(gopts.ctx, bar, errChan) + chkr.Structure(ctx, bar, errChan) }() for err := range errChan { @@ -308,7 +309,7 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error { wg.Wait() if opts.CheckUnused { - for _, id := range chkr.UnusedBlobs(gopts.ctx) { + for _, id := range chkr.UnusedBlobs(ctx) { Verbosef("unused blob %v\n", id) errorsFound = true } @@ -320,7 +321,7 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error { p := newProgressMax(!gopts.Quiet, packCount, "packs") errChan := make(chan error) - go chkr.ReadPacks(gopts.ctx, packs, p, errChan) + go chkr.ReadPacks(ctx, packs, p, errChan) for err := range errChan { errorsFound = true diff --git a/cmd/restic/cmd_copy.go b/cmd/restic/cmd_copy.go index 0324468a3..3aae94b13 100644 --- a/cmd/restic/cmd_copy.go +++ b/cmd/restic/cmd_copy.go @@ -32,7 +32,7 @@ This can be mitigated by the "--copy-chunker-params" option when initializing a new destination repository using the "init" command. `, RunE: func(cmd *cobra.Command, args []string) error { - return runCopy(copyOptions, globalOptions, args) + return runCopy(globalCtx(), copyOptions, globalOptions, args) }, } @@ -52,7 +52,7 @@ func init() { initMultiSnapshotFilterOptions(f, ©Options.snapshotFilterOptions, true) } -func runCopy(opts CopyOptions, gopts GlobalOptions, args []string) error { +func runCopy(ctx context.Context, opts CopyOptions, gopts GlobalOptions, args []string) error { secondaryGopts, isFromRepo, err := fillSecondaryGlobalOpts(opts.secondaryRepoOptions, gopts, "destination") if err != nil { return err @@ -62,13 +62,12 @@ func runCopy(opts CopyOptions, gopts GlobalOptions, args []string) error { gopts, secondaryGopts = secondaryGopts, gopts } - ctx := gopts.ctx - srcRepo, err := OpenRepository(gopts) + srcRepo, err := OpenRepository(ctx, gopts) if err != nil { return err } - dstRepo, err := OpenRepository(secondaryGopts) + dstRepo, err := OpenRepository(ctx, secondaryGopts) if err != nil { return err } diff --git a/cmd/restic/cmd_debug.go b/cmd/restic/cmd_debug.go index ac4996b7c..f7cb30381 100644 --- a/cmd/restic/cmd_debug.go +++ b/cmd/restic/cmd_debug.go @@ -46,7 +46,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - return runDebugDump(globalOptions, args) + return runDebugDump(globalCtx(), globalOptions, args) }, } @@ -141,18 +141,18 @@ func dumpIndexes(ctx context.Context, repo restic.Repository, wr io.Writer) erro }) } -func runDebugDump(gopts GlobalOptions, args []string) error { +func runDebugDump(ctx context.Context, gopts GlobalOptions, args []string) error { if len(args) != 1 { return errors.Fatal("type not specified") } - repo, err := OpenRepository(gopts) + repo, err := OpenRepository(ctx, gopts) if err != nil { return err } if !gopts.NoLock { - lock, err := lockRepo(gopts.ctx, repo) + lock, err := lockRepo(ctx, repo) defer unlockRepo(lock) if err != nil { return err @@ -163,20 +163,20 @@ func runDebugDump(gopts GlobalOptions, args []string) error { switch tpe { case "indexes": - return dumpIndexes(gopts.ctx, repo, gopts.stdout) + return dumpIndexes(ctx, repo, gopts.stdout) case "snapshots": - return debugPrintSnapshots(gopts.ctx, repo, gopts.stdout) + return debugPrintSnapshots(ctx, repo, gopts.stdout) case "packs": - return printPacks(gopts.ctx, repo, gopts.stdout) + return printPacks(ctx, repo, gopts.stdout) case "all": Printf("snapshots:\n") - err := debugPrintSnapshots(gopts.ctx, repo, gopts.stdout) + err := debugPrintSnapshots(ctx, repo, gopts.stdout) if err != nil { return err } Printf("\nindexes:\n") - err = dumpIndexes(gopts.ctx, repo, gopts.stdout) + err = dumpIndexes(ctx, repo, gopts.stdout) if err != nil { return err } @@ -192,7 +192,7 @@ var cmdDebugExamine = &cobra.Command{ Short: "Examine a pack file", DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - return runDebugExamine(globalOptions, args) + return runDebugExamine(globalCtx(), globalOptions, args) }, } @@ -426,8 +426,8 @@ func storePlainBlob(id restic.ID, prefix string, plain []byte) error { return nil } -func runDebugExamine(gopts GlobalOptions, args []string) error { - repo, err := OpenRepository(gopts) +func runDebugExamine(ctx context.Context, gopts GlobalOptions, args []string) error { + repo, err := OpenRepository(ctx, gopts) if err != nil { return err } @@ -436,7 +436,7 @@ func runDebugExamine(gopts GlobalOptions, args []string) error { for _, name := range args { id, err := restic.ParseID(name) if err != nil { - name, err = restic.Find(gopts.ctx, repo.Backend(), restic.PackFile, name) + name, err = restic.Find(ctx, repo.Backend(), restic.PackFile, name) if err == nil { id, err = restic.ParseID(name) } @@ -453,20 +453,20 @@ func runDebugExamine(gopts GlobalOptions, args []string) error { } if !gopts.NoLock { - lock, err := lockRepo(gopts.ctx, repo) + lock, err := lockRepo(ctx, repo) defer unlockRepo(lock) if err != nil { return err } } - err = repo.LoadIndex(gopts.ctx) + err = repo.LoadIndex(ctx) if err != nil { return err } for _, id := range ids { - err := examinePack(gopts.ctx, repo, id) + err := examinePack(ctx, repo, id) if err != nil { Warnf("error: %v\n", err) } diff --git a/cmd/restic/cmd_diff.go b/cmd/restic/cmd_diff.go index 76b377928..e5fd95915 100644 --- a/cmd/restic/cmd_diff.go +++ b/cmd/restic/cmd_diff.go @@ -35,7 +35,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - return runDiff(diffOptions, globalOptions, args) + return runDiff(globalCtx(), diffOptions, globalOptions, args) }, } @@ -321,13 +321,12 @@ func (c *Comparer) diffTree(ctx context.Context, stats *DiffStatsContainer, pref return nil } -func runDiff(opts DiffOptions, gopts GlobalOptions, args []string) error { +func runDiff(ctx context.Context, opts DiffOptions, gopts GlobalOptions, args []string) error { if len(args) != 2 { return errors.Fatalf("specify two snapshot IDs") } - ctx := gopts.ctx - repo, err := OpenRepository(gopts) + repo, err := OpenRepository(ctx, gopts) if err != nil { return err } diff --git a/cmd/restic/cmd_dump.go b/cmd/restic/cmd_dump.go index 44dff2783..d0c795969 100644 --- a/cmd/restic/cmd_dump.go +++ b/cmd/restic/cmd_dump.go @@ -34,7 +34,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - return runDump(dumpOptions, globalOptions, args) + return runDump(globalCtx(), dumpOptions, globalOptions, args) }, } @@ -107,9 +107,7 @@ func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.Repositor return fmt.Errorf("path %q not found in snapshot", item) } -func runDump(opts DumpOptions, gopts GlobalOptions, args []string) error { - ctx := gopts.ctx - +func runDump(ctx context.Context, opts DumpOptions, gopts GlobalOptions, args []string) error { if len(args) != 2 { return errors.Fatal("no file and no snapshot ID specified") } @@ -127,7 +125,7 @@ func runDump(opts DumpOptions, gopts GlobalOptions, args []string) error { splittedPath := splitPath(path.Clean(pathToPrint)) - repo, err := OpenRepository(gopts) + repo, err := OpenRepository(ctx, gopts) if err != nil { return err } @@ -154,7 +152,7 @@ func runDump(opts DumpOptions, gopts GlobalOptions, args []string) error { } } - sn, err := restic.LoadSnapshot(gopts.ctx, repo, id) + sn, err := restic.LoadSnapshot(ctx, repo, id) if err != nil { Exitf(2, "loading snapshot %q failed: %v", snapshotIDString, err) } diff --git a/cmd/restic/cmd_find.go b/cmd/restic/cmd_find.go index 96c11e655..5b4cf503f 100644 --- a/cmd/restic/cmd_find.go +++ b/cmd/restic/cmd_find.go @@ -38,7 +38,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - return runFind(findOptions, globalOptions, args) + return runFind(globalCtx(), findOptions, globalOptions, args) }, } @@ -534,7 +534,7 @@ func (f *Finder) findObjectsPacks(ctx context.Context) { } } -func runFind(opts FindOptions, gopts GlobalOptions, args []string) error { +func runFind(ctx context.Context, opts FindOptions, gopts GlobalOptions, args []string) error { if len(args) == 0 { return errors.Fatal("wrong number of arguments") } @@ -568,29 +568,28 @@ func runFind(opts FindOptions, gopts GlobalOptions, args []string) error { return errors.Fatal("cannot have several ID types") } - repo, err := OpenRepository(gopts) + repo, err := OpenRepository(ctx, gopts) if err != nil { return err } if !gopts.NoLock { - lock, err := lockRepo(gopts.ctx, repo) + lock, err := lockRepo(ctx, repo) defer unlockRepo(lock) if err != nil { return err } } - snapshotLister, err := backend.MemorizeList(gopts.ctx, repo.Backend(), restic.SnapshotFile) + snapshotLister, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile) if err != nil { return err } - if err = repo.LoadIndex(gopts.ctx); err != nil { + if err = repo.LoadIndex(ctx); err != nil { return err } - ctx := gopts.ctx f := &Finder{ repo: repo, pat: pat, diff --git a/cmd/restic/cmd_forget.go b/cmd/restic/cmd_forget.go index da7681867..112618bd1 100644 --- a/cmd/restic/cmd_forget.go +++ b/cmd/restic/cmd_forget.go @@ -1,6 +1,7 @@ package main import ( + "context" "encoding/json" "io" @@ -31,7 +32,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - return runForget(forgetOptions, globalOptions, args) + return runForget(globalCtx(), forgetOptions, globalOptions, args) }, } @@ -98,13 +99,13 @@ func init() { addPruneOptions(cmdForget) } -func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error { +func runForget(ctx context.Context, opts ForgetOptions, gopts GlobalOptions, args []string) error { err := verifyPruneOptions(&pruneOptions) if err != nil { return err } - repo, err := OpenRepository(gopts) + repo, err := OpenRepository(ctx, gopts) if err != nil { return err } @@ -114,14 +115,13 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error { } if !opts.DryRun || !gopts.NoLock { - lock, err := lockRepoExclusive(gopts.ctx, repo) + lock, err := lockRepoExclusive(ctx, repo) defer unlockRepo(lock) if err != nil { return err } } - ctx := gopts.ctx var snapshots restic.Snapshots removeSnIDs := restic.NewIDSet() @@ -216,7 +216,7 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error { if len(removeSnIDs) > 0 { if !opts.DryRun { - err := DeleteFilesChecked(gopts, repo, removeSnIDs, restic.SnapshotFile) + err := DeleteFilesChecked(ctx, gopts, repo, removeSnIDs, restic.SnapshotFile) if err != nil { return err } @@ -239,7 +239,7 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error { Verbosef("%d snapshots have been removed, running prune\n", len(removeSnIDs)) } pruneOptions.DryRun = opts.DryRun - return runPruneWithRepo(pruneOptions, gopts, repo, removeSnIDs) + return runPruneWithRepo(ctx, pruneOptions, gopts, repo, removeSnIDs) } return nil diff --git a/cmd/restic/cmd_init.go b/cmd/restic/cmd_init.go index 4c0392ff1..4922a58b1 100644 --- a/cmd/restic/cmd_init.go +++ b/cmd/restic/cmd_init.go @@ -1,6 +1,7 @@ package main import ( + "context" "strconv" "github.com/restic/chunker" @@ -25,7 +26,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - return runInit(initOptions, globalOptions, args) + return runInit(globalCtx(), initOptions, globalOptions, args) }, } @@ -47,7 +48,7 @@ func init() { f.StringVar(&initOptions.RepositoryVersion, "repository-version", "stable", "repository format version to use, allowed values are a format version, 'latest' and 'stable'") } -func runInit(opts InitOptions, gopts GlobalOptions, args []string) error { +func runInit(ctx context.Context, opts InitOptions, gopts GlobalOptions, args []string) error { var version uint if opts.RepositoryVersion == "latest" || opts.RepositoryVersion == "" { version = restic.MaxRepoVersion @@ -64,7 +65,7 @@ func runInit(opts InitOptions, gopts GlobalOptions, args []string) error { return errors.Fatalf("only repository versions between %v and %v are allowed", restic.MinRepoVersion, restic.MaxRepoVersion) } - chunkerPolynomial, err := maybeReadChunkerPolynomial(opts, gopts) + chunkerPolynomial, err := maybeReadChunkerPolynomial(ctx, opts, gopts) if err != nil { return err } @@ -81,7 +82,7 @@ func runInit(opts InitOptions, gopts GlobalOptions, args []string) error { return err } - be, err := create(repo, gopts.extended) + be, err := create(ctx, repo, gopts.extended) if err != nil { return errors.Fatalf("create repository at %s failed: %v\n", location.StripPassword(gopts.Repo), err) } @@ -94,7 +95,7 @@ func runInit(opts InitOptions, gopts GlobalOptions, args []string) error { return err } - err = s.Init(gopts.ctx, version, gopts.password, chunkerPolynomial) + err = s.Init(ctx, version, gopts.password, chunkerPolynomial) if err != nil { return errors.Fatalf("create key in repository at %s failed: %v\n", location.StripPassword(gopts.Repo), err) } @@ -108,14 +109,14 @@ func runInit(opts InitOptions, gopts GlobalOptions, args []string) error { return nil } -func maybeReadChunkerPolynomial(opts InitOptions, gopts GlobalOptions) (*chunker.Pol, error) { +func maybeReadChunkerPolynomial(ctx context.Context, opts InitOptions, gopts GlobalOptions) (*chunker.Pol, error) { if opts.CopyChunkerParameters { otherGopts, _, err := fillSecondaryGlobalOpts(opts.secondaryRepoOptions, gopts, "secondary") if err != nil { return nil, err } - otherRepo, err := OpenRepository(otherGopts) + otherRepo, err := OpenRepository(ctx, otherGopts) if err != nil { return nil, err } diff --git a/cmd/restic/cmd_key.go b/cmd/restic/cmd_key.go index 6a18e8b9e..c8588eb59 100644 --- a/cmd/restic/cmd_key.go +++ b/cmd/restic/cmd_key.go @@ -28,7 +28,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - return runKey(globalOptions, args) + return runKey(globalCtx(), globalOptions, args) }, } @@ -197,13 +197,12 @@ func switchToNewKeyAndRemoveIfBroken(ctx context.Context, repo *repository.Repos return nil } -func runKey(gopts GlobalOptions, args []string) error { - ctx := gopts.ctx +func runKey(ctx context.Context, gopts GlobalOptions, args []string) error { if len(args) < 1 || (args[0] == "remove" && len(args) != 2) || (args[0] != "remove" && len(args) != 1) { return errors.Fatal("wrong number of arguments") } - repo, err := OpenRepository(gopts) + repo, err := OpenRepository(ctx, gopts) if err != nil { return err } diff --git a/cmd/restic/cmd_list.go b/cmd/restic/cmd_list.go index 3feb8f3e7..819c1de4b 100644 --- a/cmd/restic/cmd_list.go +++ b/cmd/restic/cmd_list.go @@ -1,6 +1,8 @@ package main import ( + "context" + "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" @@ -21,7 +23,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - return runList(cmd, globalOptions, args) + return runList(globalCtx(), cmd, globalOptions, args) }, } @@ -29,18 +31,18 @@ func init() { cmdRoot.AddCommand(cmdList) } -func runList(cmd *cobra.Command, opts GlobalOptions, args []string) error { +func runList(ctx context.Context, cmd *cobra.Command, opts GlobalOptions, args []string) error { if len(args) != 1 { return errors.Fatal("type not specified, usage: " + cmd.Use) } - repo, err := OpenRepository(opts) + repo, err := OpenRepository(ctx, opts) if err != nil { return err } if !opts.NoLock && args[0] != "locks" { - lock, err := lockRepo(opts.ctx, repo) + lock, err := lockRepo(ctx, repo) defer unlockRepo(lock) if err != nil { return err @@ -60,11 +62,11 @@ func runList(cmd *cobra.Command, opts GlobalOptions, args []string) error { case "locks": t = restic.LockFile case "blobs": - return repository.ForAllIndexes(opts.ctx, repo, func(id restic.ID, idx *repository.Index, oldFormat bool, err error) error { + return repository.ForAllIndexes(ctx, repo, func(id restic.ID, idx *repository.Index, oldFormat bool, err error) error { if err != nil { return err } - idx.Each(opts.ctx, func(blobs restic.PackedBlob) { + idx.Each(ctx, func(blobs restic.PackedBlob) { Printf("%v %v\n", blobs.Type, blobs.ID) }) return nil @@ -73,7 +75,7 @@ func runList(cmd *cobra.Command, opts GlobalOptions, args []string) error { return errors.Fatal("invalid type") } - return repo.List(opts.ctx, t, func(id restic.ID, size int64) error { + return repo.List(ctx, t, func(id restic.ID, size int64) error { Printf("%s\n", id) return nil }) diff --git a/cmd/restic/cmd_ls.go b/cmd/restic/cmd_ls.go index ddd02e65d..96465c431 100644 --- a/cmd/restic/cmd_ls.go +++ b/cmd/restic/cmd_ls.go @@ -1,6 +1,7 @@ package main import ( + "context" "encoding/json" "os" "strings" @@ -41,7 +42,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - return runLs(lsOptions, globalOptions, args) + return runLs(globalCtx(), lsOptions, globalOptions, args) }, } @@ -110,7 +111,7 @@ func lsNodeJSON(enc *json.Encoder, path string, node *restic.Node) error { return enc.Encode(n) } -func runLs(opts LsOptions, gopts GlobalOptions, args []string) error { +func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []string) error { if len(args) == 0 { return errors.Fatal("no snapshot ID specified, specify snapshot ID or use special ID 'latest'") } @@ -160,21 +161,20 @@ func runLs(opts LsOptions, gopts GlobalOptions, args []string) error { return false } - repo, err := OpenRepository(gopts) + repo, err := OpenRepository(ctx, gopts) if err != nil { return err } - snapshotLister, err := backend.MemorizeList(gopts.ctx, repo.Backend(), restic.SnapshotFile) + snapshotLister, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile) if err != nil { return err } - if err = repo.LoadIndex(gopts.ctx); err != nil { + if err = repo.LoadIndex(ctx); err != nil { return err } - ctx := gopts.ctx var ( printSnapshot func(sn *restic.Snapshot) printNode func(path string, node *restic.Node) diff --git a/cmd/restic/cmd_migrate.go b/cmd/restic/cmd_migrate.go index 85ebc2e99..7a48051c7 100644 --- a/cmd/restic/cmd_migrate.go +++ b/cmd/restic/cmd_migrate.go @@ -24,7 +24,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - return runMigrate(migrateOptions, globalOptions, args) + return runMigrate(globalCtx(), migrateOptions, globalOptions, args) }, } @@ -93,7 +93,7 @@ func applyMigrations(ctx context.Context, opts MigrateOptions, gopts GlobalOptio checkGopts := gopts // the repository is already locked checkGopts.NoLock = true - err = runCheck(checkOptions, checkGopts, []string{}) + err = runCheck(ctx, checkOptions, checkGopts, []string{}) if err != nil { return err } @@ -116,21 +116,21 @@ func applyMigrations(ctx context.Context, opts MigrateOptions, gopts GlobalOptio return firsterr } -func runMigrate(opts MigrateOptions, gopts GlobalOptions, args []string) error { - repo, err := OpenRepository(gopts) +func runMigrate(ctx context.Context, opts MigrateOptions, gopts GlobalOptions, args []string) error { + repo, err := OpenRepository(ctx, gopts) if err != nil { return err } - lock, err := lockRepoExclusive(gopts.ctx, repo) + lock, err := lockRepoExclusive(ctx, repo) defer unlockRepo(lock) if err != nil { return err } if len(args) == 0 { - return checkMigrations(gopts.ctx, repo) + return checkMigrations(ctx, repo) } - return applyMigrations(gopts.ctx, opts, gopts, repo, args) + return applyMigrations(ctx, opts, gopts, repo, args) } diff --git a/cmd/restic/cmd_mount.go b/cmd/restic/cmd_mount.go index bfb236b73..ef4d9a32b 100644 --- a/cmd/restic/cmd_mount.go +++ b/cmd/restic/cmd_mount.go @@ -4,6 +4,7 @@ package main import ( + "context" "os" "strings" "time" @@ -66,7 +67,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - return runMount(mountOptions, globalOptions, args) + return runMount(globalCtx(), mountOptions, globalOptions, args) }, } @@ -98,7 +99,7 @@ func init() { _ = mountFlags.MarkDeprecated("snapshot-template", "use --time-template") } -func runMount(opts MountOptions, gopts GlobalOptions, args []string) error { +func runMount(ctx context.Context, opts MountOptions, gopts GlobalOptions, args []string) error { if opts.TimeTemplate == "" { return errors.Fatal("time template string cannot be empty") } @@ -114,20 +115,20 @@ func runMount(opts MountOptions, gopts GlobalOptions, args []string) error { debug.Log("start mount") defer debug.Log("finish mount") - repo, err := OpenRepository(gopts) + repo, err := OpenRepository(ctx, gopts) if err != nil { return err } if !gopts.NoLock { - lock, err := lockRepo(gopts.ctx, repo) + lock, err := lockRepo(ctx, repo) defer unlockRepo(lock) if err != nil { return err } } - err = repo.LoadIndex(gopts.ctx) + err = repo.LoadIndex(ctx) if err != nil { return err } diff --git a/cmd/restic/cmd_prune.go b/cmd/restic/cmd_prune.go index ffc442d18..2bfd2fc42 100644 --- a/cmd/restic/cmd_prune.go +++ b/cmd/restic/cmd_prune.go @@ -34,7 +34,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - return runPrune(pruneOptions, globalOptions) + return runPrune(globalCtx(), pruneOptions, globalOptions) }, } @@ -134,7 +134,7 @@ func verifyPruneOptions(opts *PruneOptions) error { return nil } -func runPrune(opts PruneOptions, gopts GlobalOptions) error { +func runPrune(ctx context.Context, opts PruneOptions, gopts GlobalOptions) error { err := verifyPruneOptions(&opts) if err != nil { return err @@ -144,7 +144,7 @@ func runPrune(opts PruneOptions, gopts GlobalOptions) error { return errors.Fatal("disabled compression and `--repack-uncompressed` are mutually exclusive") } - repo, err := OpenRepository(gopts) + repo, err := OpenRepository(ctx, gopts) if err != nil { return err } @@ -165,16 +165,16 @@ func runPrune(opts PruneOptions, gopts GlobalOptions) error { opts.unsafeRecovery = true } - lock, err := lockRepoExclusive(gopts.ctx, repo) + lock, err := lockRepoExclusive(ctx, repo) defer unlockRepo(lock) if err != nil { return err } - return runPruneWithRepo(opts, gopts, repo, restic.NewIDSet()) + return runPruneWithRepo(ctx, opts, gopts, repo, restic.NewIDSet()) } -func runPruneWithRepo(opts PruneOptions, gopts GlobalOptions, repo *repository.Repository, ignoreSnapshots restic.IDSet) error { +func runPruneWithRepo(ctx context.Context, opts PruneOptions, gopts GlobalOptions, repo *repository.Repository, ignoreSnapshots restic.IDSet) error { // we do not need index updates while pruning! repo.DisableAutoIndexUpdate() @@ -184,12 +184,12 @@ func runPruneWithRepo(opts PruneOptions, gopts GlobalOptions, repo *repository.R Verbosef("loading indexes...\n") // loading the index before the snapshots is ok, as we use an exclusive lock here - err := repo.LoadIndex(gopts.ctx) + err := repo.LoadIndex(ctx) if err != nil { return err } - plan, stats, err := planPrune(opts, gopts, repo, ignoreSnapshots) + plan, stats, err := planPrune(ctx, opts, gopts, repo, ignoreSnapshots) if err != nil { return err } @@ -199,7 +199,7 @@ func runPruneWithRepo(opts PruneOptions, gopts GlobalOptions, repo *repository.R return err } - return doPrune(opts, gopts, repo, plan) + return doPrune(ctx, opts, gopts, repo, plan) } type pruneStats struct { @@ -255,11 +255,10 @@ type packInfoWithID struct { // planPrune selects which files to rewrite and which to delete and which blobs to keep. // Also some summary statistics are returned. -func planPrune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, ignoreSnapshots restic.IDSet) (prunePlan, pruneStats, error) { - ctx := gopts.ctx +func planPrune(ctx context.Context, opts PruneOptions, gopts GlobalOptions, repo restic.Repository, ignoreSnapshots restic.IDSet) (prunePlan, pruneStats, error) { var stats pruneStats - usedBlobs, err := getUsedBlobs(gopts, repo, ignoreSnapshots) + usedBlobs, err := getUsedBlobs(ctx, gopts, repo, ignoreSnapshots) if err != nil { return prunePlan{}, stats, err } @@ -652,9 +651,7 @@ func printPruneStats(gopts GlobalOptions, stats pruneStats) error { // - rebuild the index while ignoring all files that will be deleted // - delete the files // plan.removePacks and plan.ignorePacks are modified in this function. -func doPrune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, plan prunePlan) (err error) { - ctx := gopts.ctx - +func doPrune(ctx context.Context, opts PruneOptions, gopts GlobalOptions, repo restic.Repository, plan prunePlan) (err error) { if opts.DryRun { if !gopts.JSON && gopts.verbosity >= 2 { if len(plan.removePacksFirst) > 0 { @@ -670,7 +667,7 @@ func doPrune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, pla // unreferenced packs can be safely deleted first if len(plan.removePacksFirst) != 0 { Verbosef("deleting unreferenced packs\n") - DeleteFiles(gopts, repo, plan.removePacksFirst, restic.PackFile) + DeleteFiles(ctx, gopts, repo, plan.removePacksFirst, restic.PackFile) } if len(plan.repackPacks) != 0 { @@ -703,12 +700,12 @@ func doPrune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, pla if opts.unsafeRecovery { Verbosef("deleting index files\n") indexFiles := repo.Index().(*repository.MasterIndex).IDs() - err = DeleteFilesChecked(gopts, repo, indexFiles, restic.IndexFile) + err = DeleteFilesChecked(ctx, gopts, repo, indexFiles, restic.IndexFile) if err != nil { return errors.Fatalf("%s", err) } } else if len(plan.ignorePacks) != 0 { - err = rebuildIndexFiles(gopts, repo, plan.ignorePacks, nil) + err = rebuildIndexFiles(ctx, gopts, repo, plan.ignorePacks, nil) if err != nil { return errors.Fatalf("%s", err) } @@ -716,11 +713,11 @@ func doPrune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, pla if len(plan.removePacks) != 0 { Verbosef("removing %d old packs\n", len(plan.removePacks)) - DeleteFiles(gopts, repo, plan.removePacks, restic.PackFile) + DeleteFiles(ctx, gopts, repo, plan.removePacks, restic.PackFile) } if opts.unsafeRecovery { - _, err = writeIndexFiles(gopts, repo, plan.ignorePacks, nil) + _, err = writeIndexFiles(ctx, gopts, repo, plan.ignorePacks, nil) if err != nil { return errors.Fatalf("%s", err) } @@ -730,31 +727,29 @@ func doPrune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, pla return nil } -func writeIndexFiles(gopts GlobalOptions, repo restic.Repository, removePacks restic.IDSet, extraObsolete restic.IDs) (restic.IDSet, error) { +func writeIndexFiles(ctx context.Context, gopts GlobalOptions, repo restic.Repository, removePacks restic.IDSet, extraObsolete restic.IDs) (restic.IDSet, error) { Verbosef("rebuilding index\n") bar := newProgressMax(!gopts.Quiet, 0, "packs processed") - obsoleteIndexes, err := repo.Index().Save(gopts.ctx, repo, removePacks, extraObsolete, bar) + obsoleteIndexes, err := repo.Index().Save(ctx, repo, removePacks, extraObsolete, bar) bar.Done() return obsoleteIndexes, err } -func rebuildIndexFiles(gopts GlobalOptions, repo restic.Repository, removePacks restic.IDSet, extraObsolete restic.IDs) error { - obsoleteIndexes, err := writeIndexFiles(gopts, repo, removePacks, extraObsolete) +func rebuildIndexFiles(ctx context.Context, gopts GlobalOptions, repo restic.Repository, removePacks restic.IDSet, extraObsolete restic.IDs) error { + obsoleteIndexes, err := writeIndexFiles(ctx, gopts, repo, removePacks, extraObsolete) if err != nil { return err } Verbosef("deleting obsolete index files\n") - return DeleteFilesChecked(gopts, repo, obsoleteIndexes, restic.IndexFile) + return DeleteFilesChecked(ctx, gopts, repo, obsoleteIndexes, restic.IndexFile) } -func getUsedBlobs(gopts GlobalOptions, repo restic.Repository, ignoreSnapshots restic.IDSet) (usedBlobs restic.BlobSet, err error) { - ctx := gopts.ctx - +func getUsedBlobs(ctx context.Context, gopts GlobalOptions, repo restic.Repository, ignoreSnapshots restic.IDSet) (usedBlobs restic.BlobSet, err error) { var snapshotTrees restic.IDs Verbosef("loading all snapshots...\n") - err = restic.ForAllSnapshots(gopts.ctx, repo.Backend(), repo, ignoreSnapshots, + err = restic.ForAllSnapshots(ctx, repo.Backend(), repo, ignoreSnapshots, func(id restic.ID, sn *restic.Snapshot, err error) error { if err != nil { debug.Log("failed to load snapshot %v (error %v)", id, err) diff --git a/cmd/restic/cmd_rebuild_index.go b/cmd/restic/cmd_rebuild_index.go index 0b3274ec4..6fd9eac24 100644 --- a/cmd/restic/cmd_rebuild_index.go +++ b/cmd/restic/cmd_rebuild_index.go @@ -1,6 +1,8 @@ package main import ( + "context" + "github.com/restic/restic/internal/pack" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" @@ -22,7 +24,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - return runRebuildIndex(rebuildIndexOptions, globalOptions) + return runRebuildIndex(globalCtx(), rebuildIndexOptions, globalOptions) }, } @@ -40,24 +42,22 @@ func init() { } -func runRebuildIndex(opts RebuildIndexOptions, gopts GlobalOptions) error { - repo, err := OpenRepository(gopts) +func runRebuildIndex(ctx context.Context, opts RebuildIndexOptions, gopts GlobalOptions) error { + repo, err := OpenRepository(ctx, gopts) if err != nil { return err } - lock, err := lockRepoExclusive(gopts.ctx, repo) + lock, err := lockRepoExclusive(ctx, repo) defer unlockRepo(lock) if err != nil { return err } - return rebuildIndex(opts, gopts, repo, restic.NewIDSet()) + return rebuildIndex(ctx, opts, gopts, repo, restic.NewIDSet()) } -func rebuildIndex(opts RebuildIndexOptions, gopts GlobalOptions, repo *repository.Repository, ignorePacks restic.IDSet) error { - ctx := gopts.ctx - +func rebuildIndex(ctx context.Context, opts RebuildIndexOptions, gopts GlobalOptions, repo *repository.Repository, ignorePacks restic.IDSet) error { var obsoleteIndexes restic.IDs packSizeFromList := make(map[restic.ID]int64) packSizeFromIndex := make(map[restic.ID]int64) @@ -141,7 +141,7 @@ func rebuildIndex(opts RebuildIndexOptions, gopts GlobalOptions, repo *repositor } } - err = rebuildIndexFiles(gopts, repo, removePacks, obsoleteIndexes) + err = rebuildIndexFiles(ctx, gopts, repo, removePacks, obsoleteIndexes) if err != nil { return err } diff --git a/cmd/restic/cmd_recover.go b/cmd/restic/cmd_recover.go index d20bde036..b78a527ab 100644 --- a/cmd/restic/cmd_recover.go +++ b/cmd/restic/cmd_recover.go @@ -27,7 +27,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - return runRecover(globalOptions) + return runRecover(globalCtx(), globalOptions) }, } @@ -35,30 +35,30 @@ func init() { cmdRoot.AddCommand(cmdRecover) } -func runRecover(gopts GlobalOptions) error { +func runRecover(ctx context.Context, gopts GlobalOptions) error { hostname, err := os.Hostname() if err != nil { return err } - repo, err := OpenRepository(gopts) + repo, err := OpenRepository(ctx, gopts) if err != nil { return err } - lock, err := lockRepo(gopts.ctx, repo) + lock, err := lockRepo(ctx, repo) defer unlockRepo(lock) if err != nil { return err } - snapshotLister, err := backend.MemorizeList(gopts.ctx, repo.Backend(), restic.SnapshotFile) + snapshotLister, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile) if err != nil { return err } Verbosef("load index files\n") - if err = repo.LoadIndex(gopts.ctx); err != nil { + if err = repo.LoadIndex(ctx); err != nil { return err } @@ -66,7 +66,7 @@ func runRecover(gopts GlobalOptions) error { // tree. If it is not referenced, we have a root tree. trees := make(map[restic.ID]bool) - repo.Index().Each(gopts.ctx, func(blob restic.PackedBlob) { + repo.Index().Each(ctx, func(blob restic.PackedBlob) { if blob.Type == restic.TreeBlob { trees[blob.Blob.ID] = false } @@ -75,7 +75,7 @@ func runRecover(gopts GlobalOptions) error { Verbosef("load %d trees\n", len(trees)) bar := newProgressMax(!gopts.Quiet, uint64(len(trees)), "trees loaded") for id := range trees { - tree, err := restic.LoadTree(gopts.ctx, repo, id) + tree, err := restic.LoadTree(ctx, repo, id) if err != nil { Warnf("unable to load tree %v: %v\n", id.Str(), err) continue @@ -91,7 +91,7 @@ func runRecover(gopts GlobalOptions) error { bar.Done() Verbosef("load snapshots\n") - err = restic.ForAllSnapshots(gopts.ctx, snapshotLister, repo, nil, func(id restic.ID, sn *restic.Snapshot, err error) error { + err = restic.ForAllSnapshots(ctx, snapshotLister, repo, nil, func(id restic.ID, sn *restic.Snapshot, err error) error { trees[*sn.Tree] = true return nil }) @@ -132,18 +132,18 @@ func runRecover(gopts GlobalOptions) error { } } - wg, ctx := errgroup.WithContext(gopts.ctx) - repo.StartPackUploader(ctx, wg) + wg, wgCtx := errgroup.WithContext(ctx) + repo.StartPackUploader(wgCtx, wg) var treeID restic.ID wg.Go(func() error { var err error - treeID, err = restic.SaveTree(ctx, repo, tree) + treeID, err = restic.SaveTree(wgCtx, repo, tree) if err != nil { return errors.Fatalf("unable to save new tree to the repository: %v", err) } - err = repo.Flush(ctx) + err = repo.Flush(wgCtx) if err != nil { return errors.Fatalf("unable to save blobs to the repository: %v", err) } @@ -154,7 +154,7 @@ func runRecover(gopts GlobalOptions) error { return err } - return createSnapshot(gopts.ctx, "/recover", hostname, []string{"recovered"}, repo, &treeID) + return createSnapshot(ctx, "/recover", hostname, []string{"recovered"}, repo, &treeID) } diff --git a/cmd/restic/cmd_restore.go b/cmd/restic/cmd_restore.go index 1da8407a4..a66d1e633 100644 --- a/cmd/restic/cmd_restore.go +++ b/cmd/restic/cmd_restore.go @@ -1,6 +1,7 @@ package main import ( + "context" "strings" "time" @@ -30,7 +31,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - return runRestore(restoreOptions, globalOptions, args) + return runRestore(globalCtx(), restoreOptions, globalOptions, args) }, } @@ -63,8 +64,7 @@ func init() { flags.BoolVar(&restoreOptions.Verify, "verify", false, "verify restored files content") } -func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error { - ctx := gopts.ctx +func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions, args []string) error { hasExcludes := len(opts.Exclude) > 0 || len(opts.InsensitiveExclude) > 0 hasIncludes := len(opts.Include) > 0 || len(opts.InsensitiveInclude) > 0 @@ -117,7 +117,7 @@ func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error { debug.Log("restore %v to %v", snapshotIDString, opts.Target) - repo, err := OpenRepository(gopts) + repo, err := OpenRepository(ctx, gopts) if err != nil { return err } diff --git a/cmd/restic/cmd_self_update.go b/cmd/restic/cmd_self_update.go index 6d604c792..cc10c4c77 100644 --- a/cmd/restic/cmd_self_update.go +++ b/cmd/restic/cmd_self_update.go @@ -3,6 +3,7 @@ package main import ( + "context" "os" "path/filepath" @@ -27,7 +28,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - return runSelfUpdate(selfUpdateOptions, globalOptions, args) + return runSelfUpdate(globalCtx(), selfUpdateOptions, globalOptions, args) }, } @@ -45,7 +46,7 @@ func init() { flags.StringVar(&selfUpdateOptions.Output, "output", "", "Save the downloaded file as `filename` (default: running binary itself)") } -func runSelfUpdate(opts SelfUpdateOptions, gopts GlobalOptions, args []string) error { +func runSelfUpdate(ctx context.Context, opts SelfUpdateOptions, gopts GlobalOptions, args []string) error { if opts.Output == "" { file, err := os.Executable() if err != nil { @@ -73,7 +74,7 @@ func runSelfUpdate(opts SelfUpdateOptions, gopts GlobalOptions, args []string) e Verbosef("writing restic to %v\n", opts.Output) - v, err := selfupdate.DownloadLatestStableRelease(gopts.ctx, opts.Output, version, Verbosef) + v, err := selfupdate.DownloadLatestStableRelease(ctx, opts.Output, version, Verbosef) if err != nil { return errors.Fatalf("unable to update restic: %v", err) } diff --git a/cmd/restic/cmd_snapshots.go b/cmd/restic/cmd_snapshots.go index b394a89af..6810be7a0 100644 --- a/cmd/restic/cmd_snapshots.go +++ b/cmd/restic/cmd_snapshots.go @@ -1,6 +1,7 @@ package main import ( + "context" "encoding/json" "fmt" "io" @@ -25,7 +26,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - return runSnapshots(snapshotOptions, globalOptions, args) + return runSnapshots(globalCtx(), snapshotOptions, globalOptions, args) }, } @@ -56,21 +57,20 @@ func init() { f.StringVarP(&snapshotOptions.GroupBy, "group-by", "g", "", "`group` snapshots by host, paths and/or tags, separated by comma") } -func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) error { - repo, err := OpenRepository(gopts) +func runSnapshots(ctx context.Context, opts SnapshotOptions, gopts GlobalOptions, args []string) error { + repo, err := OpenRepository(ctx, gopts) if err != nil { return err } if !gopts.NoLock { - lock, err := lockRepo(gopts.ctx, repo) + lock, err := lockRepo(ctx, repo) defer unlockRepo(lock) if err != nil { return err } } - ctx := gopts.ctx var snapshots restic.Snapshots for sn := range FindFilteredSnapshots(ctx, repo.Backend(), repo, opts.Hosts, opts.Tags, opts.Paths, args) { snapshots = append(snapshots, sn) diff --git a/cmd/restic/cmd_stats.go b/cmd/restic/cmd_stats.go index dbc16f91e..67a5ab542 100644 --- a/cmd/restic/cmd_stats.go +++ b/cmd/restic/cmd_stats.go @@ -47,7 +47,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - return runStats(globalOptions, args) + return runStats(globalCtx(), globalOptions, args) }, } @@ -68,14 +68,13 @@ func init() { initMultiSnapshotFilterOptions(f, &statsOptions.snapshotFilterOptions, true) } -func runStats(gopts GlobalOptions, args []string) error { +func runStats(ctx context.Context, gopts GlobalOptions, args []string) error { err := verifyStatsInput(gopts, args) if err != nil { return err } - ctx := gopts.ctx - repo, err := OpenRepository(gopts) + repo, err := OpenRepository(ctx, gopts) if err != nil { return err } @@ -88,7 +87,7 @@ func runStats(gopts GlobalOptions, args []string) error { } } - snapshotLister, err := backend.MemorizeList(gopts.ctx, repo.Backend(), restic.SnapshotFile) + snapshotLister, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile) if err != nil { return err } diff --git a/cmd/restic/cmd_tag.go b/cmd/restic/cmd_tag.go index cd52e7241..a27eb5d7b 100644 --- a/cmd/restic/cmd_tag.go +++ b/cmd/restic/cmd_tag.go @@ -29,7 +29,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - return runTag(tagOptions, globalOptions, args) + return runTag(globalCtx(), tagOptions, globalOptions, args) }, } @@ -95,7 +95,7 @@ func changeTags(ctx context.Context, repo *repository.Repository, sn *restic.Sna return changed, nil } -func runTag(opts TagOptions, gopts GlobalOptions, args []string) error { +func runTag(ctx context.Context, opts TagOptions, gopts GlobalOptions, args []string) error { if len(opts.SetTags) == 0 && len(opts.AddTags) == 0 && len(opts.RemoveTags) == 0 { return errors.Fatal("nothing to do!") } @@ -103,14 +103,14 @@ func runTag(opts TagOptions, gopts GlobalOptions, args []string) error { return errors.Fatal("--set and --add/--remove cannot be given at the same time") } - repo, err := OpenRepository(gopts) + repo, err := OpenRepository(ctx, gopts) if err != nil { return err } if !gopts.NoLock { Verbosef("create exclusive lock for repository\n") - lock, err := lockRepoExclusive(gopts.ctx, repo) + lock, err := lockRepoExclusive(ctx, repo) defer unlockRepo(lock) if err != nil { return err @@ -118,7 +118,6 @@ func runTag(opts TagOptions, gopts GlobalOptions, args []string) error { } changeCnt := 0 - ctx := gopts.ctx for sn := range FindFilteredSnapshots(ctx, repo.Backend(), repo, opts.Hosts, opts.Tags, opts.Paths, args) { changed, err := changeTags(ctx, repo, sn, opts.SetTags.Flatten(), opts.AddTags.Flatten(), opts.RemoveTags.Flatten()) if err != nil { diff --git a/cmd/restic/cmd_unlock.go b/cmd/restic/cmd_unlock.go index 16a5da579..bcdef18f6 100644 --- a/cmd/restic/cmd_unlock.go +++ b/cmd/restic/cmd_unlock.go @@ -1,6 +1,8 @@ package main import ( + "context" + "github.com/restic/restic/internal/restic" "github.com/spf13/cobra" ) @@ -18,7 +20,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - return runUnlock(unlockOptions, globalOptions) + return runUnlock(globalCtx(), unlockOptions, globalOptions) }, } @@ -35,8 +37,8 @@ func init() { unlockCmd.Flags().BoolVar(&unlockOptions.RemoveAll, "remove-all", false, "remove all locks, even non-stale ones") } -func runUnlock(opts UnlockOptions, gopts GlobalOptions) error { - repo, err := OpenRepository(gopts) +func runUnlock(ctx context.Context, opts UnlockOptions, gopts GlobalOptions) error { + repo, err := OpenRepository(ctx, gopts) if err != nil { return err } @@ -46,7 +48,7 @@ func runUnlock(opts UnlockOptions, gopts GlobalOptions) error { fn = restic.RemoveAllLocks } - processed, err := fn(gopts.ctx, repo) + processed, err := fn(ctx, repo) if err != nil { return err } diff --git a/cmd/restic/delete.go b/cmd/restic/delete.go index d97b9e617..2046ccfde 100644 --- a/cmd/restic/delete.go +++ b/cmd/restic/delete.go @@ -1,6 +1,8 @@ package main import ( + "context" + "golang.org/x/sync/errgroup" "github.com/restic/restic/internal/restic" @@ -8,22 +10,22 @@ import ( // DeleteFiles deletes the given fileList of fileType in parallel // it will print a warning if there is an error, but continue deleting the remaining files -func DeleteFiles(gopts GlobalOptions, repo restic.Repository, fileList restic.IDSet, fileType restic.FileType) { - _ = deleteFiles(gopts, true, repo, fileList, fileType) +func DeleteFiles(ctx context.Context, gopts GlobalOptions, repo restic.Repository, fileList restic.IDSet, fileType restic.FileType) { + _ = deleteFiles(ctx, gopts, true, repo, fileList, fileType) } // DeleteFilesChecked deletes the given fileList of fileType in parallel // if an error occurs, it will cancel and return this error -func DeleteFilesChecked(gopts GlobalOptions, repo restic.Repository, fileList restic.IDSet, fileType restic.FileType) error { - return deleteFiles(gopts, false, repo, fileList, fileType) +func DeleteFilesChecked(ctx context.Context, gopts GlobalOptions, repo restic.Repository, fileList restic.IDSet, fileType restic.FileType) error { + return deleteFiles(ctx, gopts, false, repo, fileList, fileType) } // deleteFiles deletes the given fileList of fileType in parallel // if ignoreError=true, it will print a warning if there was an error, else it will abort. -func deleteFiles(gopts GlobalOptions, ignoreError bool, repo restic.Repository, fileList restic.IDSet, fileType restic.FileType) error { +func deleteFiles(ctx context.Context, gopts GlobalOptions, ignoreError bool, repo restic.Repository, fileList restic.IDSet, fileType restic.FileType) error { totalCount := len(fileList) fileChan := make(chan restic.ID) - wg, ctx := errgroup.WithContext(gopts.ctx) + wg, ctx := errgroup.WithContext(ctx) wg.Go(func() error { defer close(fileChan) for id := range fileList { diff --git a/cmd/restic/global.go b/cmd/restic/global.go index a92e52725..82a4cfcca 100644 --- a/cmd/restic/global.go +++ b/cmd/restic/global.go @@ -68,7 +68,6 @@ type GlobalOptions struct { backend.TransportOptions limiter.Limits - ctx context.Context password string stdout io.Writer stderr io.Writer @@ -93,10 +92,11 @@ var globalOptions = GlobalOptions{ } var isReadingPassword bool +var internalGlobalCtx context.Context func init() { var cancel context.CancelFunc - globalOptions.ctx, cancel = context.WithCancel(context.Background()) + internalGlobalCtx, cancel = context.WithCancel(context.Background()) AddCleanupHandler(func(code int) (int, error) { // Must be called before the unlock cleanup handler to ensure that the latter is // not blocked due to limited number of backend connections, see #1434 @@ -145,6 +145,10 @@ func init() { restoreTerminal() } +func globalCtx() context.Context { + return internalGlobalCtx +} + // checkErrno returns nil when err is set to syscall.Errno(0), since this is no // error condition. func checkErrno(err error) error { @@ -428,13 +432,13 @@ func ReadRepo(opts GlobalOptions) (string, error) { const maxKeys = 20 // OpenRepository reads the password and opens the repository. -func OpenRepository(opts GlobalOptions) (*repository.Repository, error) { +func OpenRepository(ctx context.Context, opts GlobalOptions) (*repository.Repository, error) { repo, err := ReadRepo(opts) if err != nil { return nil, err } - be, err := open(repo, opts, opts.extended) + be, err := open(ctx, repo, opts, opts.extended) if err != nil { return nil, err } @@ -478,7 +482,7 @@ func OpenRepository(opts GlobalOptions) (*repository.Repository, error) { continue } - err = s.SearchKey(opts.ctx, opts.password, maxKeys, opts.KeyHint) + err = s.SearchKey(ctx, opts.password, maxKeys, opts.KeyHint) if err != nil && passwordTriesLeft > 1 { opts.password = "" fmt.Fprintf(os.Stderr, "%s. Try again\n", err) @@ -695,7 +699,7 @@ func parseConfig(loc location.Location, opts options.Options) (interface{}, erro } // Open the backend specified by a location config. -func open(s string, gopts GlobalOptions, opts options.Options) (restic.Backend, error) { +func open(ctx context.Context, s string, gopts GlobalOptions, opts options.Options) (restic.Backend, error) { debug.Log("parsing location %v", location.StripPassword(s)) loc, err := location.Parse(s) if err != nil { @@ -720,19 +724,19 @@ func open(s string, gopts GlobalOptions, opts options.Options) (restic.Backend, switch loc.Scheme { case "local": - be, err = local.Open(globalOptions.ctx, cfg.(local.Config)) + be, err = local.Open(ctx, cfg.(local.Config)) case "sftp": - be, err = sftp.Open(globalOptions.ctx, cfg.(sftp.Config)) + be, err = sftp.Open(ctx, cfg.(sftp.Config)) case "s3": - be, err = s3.Open(globalOptions.ctx, cfg.(s3.Config), rt) + be, err = s3.Open(ctx, cfg.(s3.Config), rt) case "gs": be, err = gs.Open(cfg.(gs.Config), rt) case "azure": be, err = azure.Open(cfg.(azure.Config), rt) case "swift": - be, err = swift.Open(globalOptions.ctx, cfg.(swift.Config), rt) + be, err = swift.Open(ctx, cfg.(swift.Config), rt) case "b2": - be, err = b2.Open(globalOptions.ctx, cfg.(b2.Config), rt) + be, err = b2.Open(ctx, cfg.(b2.Config), rt) case "rest": be, err = rest.Open(cfg.(rest.Config), rt) case "rclone": @@ -760,7 +764,7 @@ func open(s string, gopts GlobalOptions, opts options.Options) (restic.Backend, } // check if config is there - fi, err := be.Stat(globalOptions.ctx, restic.Handle{Type: restic.ConfigFile}) + fi, err := be.Stat(ctx, restic.Handle{Type: restic.ConfigFile}) if err != nil { return nil, errors.Fatalf("unable to open config file: %v\nIs there a repository at the following location?\n%v", err, location.StripPassword(s)) } @@ -773,7 +777,7 @@ func open(s string, gopts GlobalOptions, opts options.Options) (restic.Backend, } // Create the backend specified by URI. -func create(s string, opts options.Options) (restic.Backend, error) { +func create(ctx context.Context, s string, opts options.Options) (restic.Backend, error) { debug.Log("parsing location %v", s) loc, err := location.Parse(s) if err != nil { @@ -792,23 +796,23 @@ func create(s string, opts options.Options) (restic.Backend, error) { switch loc.Scheme { case "local": - return local.Create(globalOptions.ctx, cfg.(local.Config)) + return local.Create(ctx, cfg.(local.Config)) case "sftp": - return sftp.Create(globalOptions.ctx, cfg.(sftp.Config)) + return sftp.Create(ctx, cfg.(sftp.Config)) case "s3": - return s3.Create(globalOptions.ctx, cfg.(s3.Config), rt) + return s3.Create(ctx, cfg.(s3.Config), rt) case "gs": return gs.Create(cfg.(gs.Config), rt) case "azure": return azure.Create(cfg.(azure.Config), rt) case "swift": - return swift.Open(globalOptions.ctx, cfg.(swift.Config), rt) + return swift.Open(ctx, cfg.(swift.Config), rt) case "b2": - return b2.Create(globalOptions.ctx, cfg.(b2.Config), rt) + return b2.Create(ctx, cfg.(b2.Config), rt) case "rest": - return rest.Create(globalOptions.ctx, cfg.(rest.Config), rt) + return rest.Create(ctx, cfg.(rest.Config), rt) case "rclone": - return rclone.Create(globalOptions.ctx, cfg.(rclone.Config)) + return rclone.Create(ctx, cfg.(rclone.Config)) } debug.Log("invalid repository scheme: %v", s) diff --git a/cmd/restic/integration_fuse_test.go b/cmd/restic/integration_fuse_test.go index 6a95ac87d..c09b77f19 100644 --- a/cmd/restic/integration_fuse_test.go +++ b/cmd/restic/integration_fuse_test.go @@ -4,6 +4,7 @@ package main import ( + "context" "fmt" "os" "path/filepath" @@ -57,7 +58,7 @@ func testRunMount(t testing.TB, gopts GlobalOptions, dir string) { opts := MountOptions{ TimeTemplate: time.RFC3339, } - rtest.OK(t, runMount(opts, gopts, []string{dir})) + rtest.OK(t, runMount(context.TODO(), opts, gopts, []string{dir})) } func testRunUmount(t testing.TB, gopts GlobalOptions, dir string) { @@ -119,7 +120,7 @@ func checkSnapshots(t testing.TB, global GlobalOptions, repo *repository.Reposit } for _, id := range snapshotIDs { - snapshot, err := restic.LoadSnapshot(global.ctx, repo, id) + snapshot, err := restic.LoadSnapshot(context.TODO(), repo, id) rtest.OK(t, err) ts := snapshot.Time.Format(time.RFC3339) @@ -160,7 +161,7 @@ func TestMount(t *testing.T) { testRunInit(t, env.gopts) - repo, err := OpenRepository(env.gopts) + repo, err := OpenRepository(context.TODO(), env.gopts) rtest.OK(t, err) checkSnapshots(t, env.gopts, repo, env.mountpoint, env.repo, []restic.ID{}, 0) @@ -205,7 +206,7 @@ func TestMountSameTimestamps(t *testing.T) { rtest.SetupTarTestFixture(t, env.base, filepath.Join("testdata", "repo-same-timestamps.tar.gz")) - repo, err := OpenRepository(env.gopts) + repo, err := OpenRepository(context.TODO(), env.gopts) rtest.OK(t, err) ids := []restic.ID{ diff --git a/cmd/restic/integration_helpers_test.go b/cmd/restic/integration_helpers_test.go index e87baddca..17a3c29c3 100644 --- a/cmd/restic/integration_helpers_test.go +++ b/cmd/restic/integration_helpers_test.go @@ -2,7 +2,6 @@ package main import ( "bytes" - "context" "fmt" "io/ioutil" "os" @@ -193,7 +192,6 @@ func withTestEnvironment(t testing.TB) (env *testEnvironment, cleanup func()) { Repo: env.repo, Quiet: true, CacheDir: env.cache, - ctx: context.Background(), password: rtest.TestPassword, stdout: os.Stdout, stderr: os.Stderr, diff --git a/cmd/restic/integration_test.go b/cmd/restic/integration_test.go index 033267cc5..307937cc8 100644 --- a/cmd/restic/integration_test.go +++ b/cmd/restic/integration_test.go @@ -52,12 +52,12 @@ func testRunInit(t testing.TB, opts GlobalOptions) { restic.TestDisableCheckPolynomial(t) restic.TestSetLockTimeout(t, 0) - rtest.OK(t, runInit(InitOptions{}, opts, nil)) + rtest.OK(t, runInit(context.TODO(), InitOptions{}, opts, nil)) t.Logf("repository initialized at %v", opts.Repo) } func testRunBackupAssumeFailure(t testing.TB, dir string, target []string, opts BackupOptions, gopts GlobalOptions) error { - ctx, cancel := context.WithCancel(gopts.ctx) + ctx, cancel := context.WithCancel(context.TODO()) defer cancel() var wg errgroup.Group @@ -71,7 +71,7 @@ func testRunBackupAssumeFailure(t testing.TB, dir string, target []string, opts defer cleanup() } - backupErr := runBackup(opts, gopts, term, target) + backupErr := runBackup(ctx, opts, gopts, term, target) cancel() @@ -95,7 +95,7 @@ func testRunList(t testing.TB, tpe string, opts GlobalOptions) restic.IDs { globalOptions.stdout = os.Stdout }() - rtest.OK(t, runList(cmdList, opts, []string{tpe})) + rtest.OK(t, runList(context.TODO(), cmdList, opts, []string{tpe})) return parseIDsFromReader(t, buf) } @@ -112,7 +112,7 @@ func testRunRestoreLatest(t testing.TB, gopts GlobalOptions, dir string, paths [ }, } - rtest.OK(t, runRestore(opts, gopts, []string{"latest"})) + rtest.OK(t, runRestore(context.TODO(), opts, gopts, []string{"latest"})) } func testRunRestoreExcludes(t testing.TB, gopts GlobalOptions, dir string, snapshotID restic.ID, excludes []string) { @@ -121,7 +121,7 @@ func testRunRestoreExcludes(t testing.TB, gopts GlobalOptions, dir string, snaps Exclude: excludes, } - rtest.OK(t, runRestore(opts, gopts, []string{snapshotID.String()})) + rtest.OK(t, runRestore(context.TODO(), opts, gopts, []string{snapshotID.String()})) } func testRunRestoreIncludes(t testing.TB, gopts GlobalOptions, dir string, snapshotID restic.ID, includes []string) { @@ -130,11 +130,11 @@ func testRunRestoreIncludes(t testing.TB, gopts GlobalOptions, dir string, snaps Include: includes, } - rtest.OK(t, runRestore(opts, gopts, []string{snapshotID.String()})) + rtest.OK(t, runRestore(context.TODO(), opts, gopts, []string{snapshotID.String()})) } func testRunRestoreAssumeFailure(t testing.TB, snapshotID string, opts RestoreOptions, gopts GlobalOptions) error { - err := runRestore(opts, gopts, []string{snapshotID}) + err := runRestore(context.TODO(), opts, gopts, []string{snapshotID}) return err } @@ -144,7 +144,7 @@ func testRunCheck(t testing.TB, gopts GlobalOptions) { ReadData: true, CheckUnused: true, } - rtest.OK(t, runCheck(opts, gopts, nil)) + rtest.OK(t, runCheck(context.TODO(), opts, gopts, nil)) } func testRunCheckOutput(gopts GlobalOptions) (string, error) { @@ -159,7 +159,7 @@ func testRunCheckOutput(gopts GlobalOptions) (string, error) { ReadData: true, } - err := runCheck(opts, gopts, nil) + err := runCheck(context.TODO(), opts, gopts, nil) return buf.String(), err } @@ -177,7 +177,7 @@ func testRunDiffOutput(gopts GlobalOptions, firstSnapshotID string, secondSnapsh opts := DiffOptions{ ShowMetadata: false, } - err := runDiff(opts, gopts, []string{firstSnapshotID, secondSnapshotID}) + err := runDiff(context.TODO(), opts, gopts, []string{firstSnapshotID, secondSnapshotID}) return buf.String(), err } @@ -187,7 +187,7 @@ func testRunRebuildIndex(t testing.TB, gopts GlobalOptions) { globalOptions.stdout = os.Stdout }() - rtest.OK(t, runRebuildIndex(RebuildIndexOptions{}, gopts)) + rtest.OK(t, runRebuildIndex(context.TODO(), RebuildIndexOptions{}, gopts)) } func testRunLs(t testing.TB, gopts GlobalOptions, snapshotID string) []string { @@ -202,7 +202,7 @@ func testRunLs(t testing.TB, gopts GlobalOptions, snapshotID string) []string { opts := LsOptions{} - rtest.OK(t, runLs(opts, gopts, []string{snapshotID})) + rtest.OK(t, runLs(context.TODO(), opts, gopts, []string{snapshotID})) return strings.Split(buf.String(), "\n") } @@ -218,7 +218,7 @@ func testRunFind(t testing.TB, wantJSON bool, gopts GlobalOptions, pattern strin opts := FindOptions{} - rtest.OK(t, runFind(opts, gopts, []string{pattern})) + rtest.OK(t, runFind(context.TODO(), opts, gopts, []string{pattern})) return buf.Bytes() } @@ -234,7 +234,7 @@ func testRunSnapshots(t testing.TB, gopts GlobalOptions) (newest *Snapshot, snap opts := SnapshotOptions{} - rtest.OK(t, runSnapshots(opts, globalOptions, []string{})) + rtest.OK(t, runSnapshots(context.TODO(), opts, globalOptions, []string{})) snapshots := []Snapshot{} rtest.OK(t, json.Unmarshal(buf.Bytes(), &snapshots)) @@ -251,7 +251,7 @@ func testRunSnapshots(t testing.TB, gopts GlobalOptions) (newest *Snapshot, snap func testRunForget(t testing.TB, gopts GlobalOptions, args ...string) { opts := ForgetOptions{} - rtest.OK(t, runForget(opts, gopts, args)) + rtest.OK(t, runForget(context.TODO(), opts, gopts, args)) } func testRunForgetJSON(t testing.TB, gopts GlobalOptions, args ...string) { @@ -269,7 +269,7 @@ func testRunForgetJSON(t testing.TB, gopts GlobalOptions, args ...string) { Last: 1, } - rtest.OK(t, runForget(opts, gopts, args)) + rtest.OK(t, runForget(context.TODO(), opts, gopts, args)) var forgets []*ForgetGroup rtest.OK(t, json.Unmarshal(buf.Bytes(), &forgets)) @@ -288,7 +288,7 @@ func testRunPrune(t testing.TB, gopts GlobalOptions, opts PruneOptions) { defer func() { gopts.backendTestHook = oldHook }() - rtest.OK(t, runPrune(opts, gopts)) + rtest.OK(t, runPrune(context.TODO(), opts, gopts)) } func testSetupBackupData(t testing.TB, env *testEnvironment) string { @@ -437,11 +437,11 @@ func TestBackupNonExistingFile(t *testing.T) { } func removePacksExcept(gopts GlobalOptions, t *testing.T, keep restic.IDSet, removeTreePacks bool) { - r, err := OpenRepository(gopts) + r, err := OpenRepository(context.TODO(), gopts) rtest.OK(t, err) // Get all tree packs - rtest.OK(t, r.LoadIndex(gopts.ctx)) + rtest.OK(t, r.LoadIndex(context.TODO())) treePacks := restic.NewIDSet() r.Index().Each(context.TODO(), func(pb restic.PackedBlob) { @@ -451,11 +451,11 @@ func removePacksExcept(gopts GlobalOptions, t *testing.T, keep restic.IDSet, rem }) // remove all packs containing data blobs - rtest.OK(t, r.List(gopts.ctx, restic.PackFile, func(id restic.ID, size int64) error { + rtest.OK(t, r.List(context.TODO(), restic.PackFile, func(id restic.ID, size int64) error { if treePacks.Has(id) != removeTreePacks || keep.Has(id) { return nil } - return r.Backend().Remove(gopts.ctx, restic.Handle{Type: restic.PackFile, Name: id.String()}) + return r.Backend().Remove(context.TODO(), restic.Handle{Type: restic.PackFile, Name: id.String()}) })) } @@ -479,7 +479,7 @@ func TestBackupSelfHealing(t *testing.T) { testRunRebuildIndex(t, env.gopts) // now the repo is also missing the data blob in the index; check should report this - rtest.Assert(t, runCheck(CheckOptions{}, env.gopts, nil) != nil, + rtest.Assert(t, runCheck(context.TODO(), CheckOptions{}, env.gopts, nil) != nil, "check should have reported an error") // second backup should report an error but "heal" this situation @@ -502,9 +502,9 @@ func TestBackupTreeLoadError(t *testing.T) { // Backup a subdirectory first, such that we can remove the tree pack for the subdirectory testRunBackup(t, env.testdata, []string{"test"}, opts, env.gopts) - r, err := OpenRepository(env.gopts) + r, err := OpenRepository(context.TODO(), env.gopts) rtest.OK(t, err) - rtest.OK(t, r.LoadIndex(env.gopts.ctx)) + rtest.OK(t, r.LoadIndex(context.TODO())) treePacks := restic.NewIDSet() r.Index().Each(context.TODO(), func(pb restic.PackedBlob) { if pb.Type == restic.TreeBlob { @@ -517,11 +517,11 @@ func TestBackupTreeLoadError(t *testing.T) { // delete the subdirectory pack first for id := range treePacks { - rtest.OK(t, r.Backend().Remove(env.gopts.ctx, restic.Handle{Type: restic.PackFile, Name: id.String()})) + rtest.OK(t, r.Backend().Remove(context.TODO(), restic.Handle{Type: restic.PackFile, Name: id.String()})) } testRunRebuildIndex(t, env.gopts) // now the repo is missing the tree blob in the index; check should report this - rtest.Assert(t, runCheck(CheckOptions{}, env.gopts, nil) != nil, "check should have reported an error") + rtest.Assert(t, runCheck(context.TODO(), CheckOptions{}, env.gopts, nil) != nil, "check should have reported an error") // second backup should report an error but "heal" this situation err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{filepath.Base(env.testdata)}, opts, env.gopts) rtest.Assert(t, err != nil, "backup should have reported an error for the subdirectory") @@ -531,7 +531,7 @@ func TestBackupTreeLoadError(t *testing.T) { removePacksExcept(env.gopts, t, restic.NewIDSet(), true) testRunRebuildIndex(t, env.gopts) // now the repo is also missing the data blob in the index; check should report this - rtest.Assert(t, runCheck(CheckOptions{}, env.gopts, nil) != nil, "check should have reported an error") + rtest.Assert(t, runCheck(context.TODO(), CheckOptions{}, env.gopts, nil) != nil, "check should have reported an error") // second backup should report an error but "heal" this situation err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{filepath.Base(env.testdata)}, opts, env.gopts) rtest.Assert(t, err != nil, "backup should have reported an error") @@ -761,7 +761,7 @@ func testRunCopy(t testing.TB, srcGopts GlobalOptions, dstGopts GlobalOptions) { }, } - rtest.OK(t, runCopy(copyOpts, gopts, nil)) + rtest.OK(t, runCopy(context.TODO(), copyOpts, gopts, nil)) } func TestCopy(t *testing.T) { @@ -903,15 +903,15 @@ func TestInitCopyChunkerParams(t *testing.T) { password: env2.gopts.password, }, } - rtest.Assert(t, runInit(initOpts, env.gopts, nil) != nil, "expected invalid init options to fail") + rtest.Assert(t, runInit(context.TODO(), initOpts, env.gopts, nil) != nil, "expected invalid init options to fail") initOpts.CopyChunkerParameters = true - rtest.OK(t, runInit(initOpts, env.gopts, nil)) + rtest.OK(t, runInit(context.TODO(), initOpts, env.gopts, nil)) - repo, err := OpenRepository(env.gopts) + repo, err := OpenRepository(context.TODO(), env.gopts) rtest.OK(t, err) - otherRepo, err := OpenRepository(env2.gopts) + otherRepo, err := OpenRepository(context.TODO(), env2.gopts) rtest.OK(t, err) rtest.Assert(t, repo.Config().ChunkerPolynomial == otherRepo.Config().ChunkerPolynomial, @@ -920,7 +920,7 @@ func TestInitCopyChunkerParams(t *testing.T) { } func testRunTag(t testing.TB, opts TagOptions, gopts GlobalOptions) { - rtest.OK(t, runTag(opts, gopts, []string{})) + rtest.OK(t, runTag(context.TODO(), opts, gopts, []string{})) } func TestTag(t *testing.T) { @@ -1012,7 +1012,7 @@ func testRunKeyListOtherIDs(t testing.TB, gopts GlobalOptions) []string { globalOptions.stdout = os.Stdout }() - rtest.OK(t, runKey(gopts, []string{"list"})) + rtest.OK(t, runKey(context.TODO(), gopts, []string{"list"})) scanner := bufio.NewScanner(buf) exp := regexp.MustCompile(`^ ([a-f0-9]+) `) @@ -1033,7 +1033,7 @@ func testRunKeyAddNewKey(t testing.TB, newPassword string, gopts GlobalOptions) testKeyNewPassword = "" }() - rtest.OK(t, runKey(gopts, []string{"add"})) + rtest.OK(t, runKey(context.TODO(), gopts, []string{"add"})) } func testRunKeyAddNewKeyUserHost(t testing.TB, gopts GlobalOptions) { @@ -1047,11 +1047,11 @@ func testRunKeyAddNewKeyUserHost(t testing.TB, gopts GlobalOptions) { rtest.OK(t, cmdKey.Flags().Parse([]string{"--user=john", "--host=example.com"})) t.Log("adding key for john@example.com") - rtest.OK(t, runKey(gopts, []string{"add"})) + rtest.OK(t, runKey(context.TODO(), gopts, []string{"add"})) - repo, err := OpenRepository(gopts) + repo, err := OpenRepository(context.TODO(), gopts) rtest.OK(t, err) - key, err := repository.SearchKey(gopts.ctx, repo, testKeyNewPassword, 2, "") + key, err := repository.SearchKey(context.TODO(), repo, testKeyNewPassword, 2, "") rtest.OK(t, err) rtest.Equals(t, "john", key.Username) @@ -1064,13 +1064,13 @@ func testRunKeyPasswd(t testing.TB, newPassword string, gopts GlobalOptions) { testKeyNewPassword = "" }() - rtest.OK(t, runKey(gopts, []string{"passwd"})) + rtest.OK(t, runKey(context.TODO(), gopts, []string{"passwd"})) } func testRunKeyRemove(t testing.TB, gopts GlobalOptions, IDs []string) { t.Logf("remove %d keys: %q\n", len(IDs), IDs) for _, id := range IDs { - rtest.OK(t, runKey(gopts, []string{"remove", id})) + rtest.OK(t, runKey(context.TODO(), gopts, []string{"remove", id})) } } @@ -1100,7 +1100,7 @@ func TestKeyAddRemove(t *testing.T) { env.gopts.password = passwordList[len(passwordList)-1] t.Logf("testing access with last password %q\n", env.gopts.password) - rtest.OK(t, runKey(env.gopts, []string{"list"})) + rtest.OK(t, runKey(context.TODO(), env.gopts, []string{"list"})) testRunCheck(t, env.gopts) testRunKeyAddNewKeyUserHost(t, env.gopts) @@ -1128,16 +1128,16 @@ func TestKeyProblems(t *testing.T) { testKeyNewPassword = "" }() - err := runKey(env.gopts, []string{"passwd"}) + err := runKey(context.TODO(), env.gopts, []string{"passwd"}) t.Log(err) rtest.Assert(t, err != nil, "expected passwd change to fail") - err = runKey(env.gopts, []string{"add"}) + err = runKey(context.TODO(), env.gopts, []string{"add"}) t.Log(err) rtest.Assert(t, err != nil, "expected key adding to fail") t.Logf("testing access with initial password %q\n", env.gopts.password) - rtest.OK(t, runKey(env.gopts, []string{"list"})) + rtest.OK(t, runKey(context.TODO(), env.gopts, []string{"list"})) testRunCheck(t, env.gopts) } @@ -1549,7 +1549,7 @@ func TestRebuildIndexFailsOnAppendOnly(t *testing.T) { env.gopts.backendTestHook = func(r restic.Backend) (restic.Backend, error) { return &appendOnlyBackend{r}, nil } - err := runRebuildIndex(RebuildIndexOptions{}, env.gopts) + err := runRebuildIndex(context.TODO(), RebuildIndexOptions{}, env.gopts) if err == nil { t.Error("expected rebuildIndex to fail") } @@ -1645,18 +1645,18 @@ func testPrune(t *testing.T, pruneOpts PruneOptions, checkOpts CheckOptions) { testRunForgetJSON(t, env.gopts) testRunForget(t, env.gopts, firstSnapshot[0].String()) testRunPrune(t, env.gopts, pruneOpts) - rtest.OK(t, runCheck(checkOpts, env.gopts, nil)) + rtest.OK(t, runCheck(context.TODO(), checkOpts, env.gopts, nil)) } var pruneDefaultOptions = PruneOptions{MaxUnused: "5%"} func listPacks(gopts GlobalOptions, t *testing.T) restic.IDSet { - r, err := OpenRepository(gopts) + r, err := OpenRepository(context.TODO(), gopts) rtest.OK(t, err) packs := restic.NewIDSet() - rtest.OK(t, r.List(gopts.ctx, restic.PackFile, func(id restic.ID, size int64) error { + rtest.OK(t, r.List(context.TODO(), restic.PackFile, func(id restic.ID, size int64) error { packs.Insert(id) return nil })) @@ -1697,7 +1697,7 @@ func TestPruneWithDamagedRepository(t *testing.T) { env.gopts.backendTestHook = oldHook }() // prune should fail - rtest.Assert(t, runPrune(pruneDefaultOptions, env.gopts) == errorPacksMissing, + rtest.Assert(t, runPrune(context.TODO(), pruneDefaultOptions, env.gopts) == errorPacksMissing, "prune should have reported index not complete error") } @@ -1769,7 +1769,7 @@ func testEdgeCaseRepo(t *testing.T, tarfile string, optionsCheck CheckOptions, o if checkOK { testRunCheck(t, env.gopts) } else { - rtest.Assert(t, runCheck(optionsCheck, env.gopts, nil) != nil, + rtest.Assert(t, runCheck(context.TODO(), optionsCheck, env.gopts, nil) != nil, "check should have reported an error") } @@ -1777,7 +1777,7 @@ func testEdgeCaseRepo(t *testing.T, tarfile string, optionsCheck CheckOptions, o testRunPrune(t, env.gopts, optionsPrune) testRunCheck(t, env.gopts) } else { - rtest.Assert(t, runPrune(optionsPrune, env.gopts) != nil, + rtest.Assert(t, runPrune(context.TODO(), optionsPrune, env.gopts) != nil, "prune should have reported an error") } } @@ -1848,10 +1848,10 @@ func TestListOnce(t *testing.T) { testRunForgetJSON(t, env.gopts) testRunForget(t, env.gopts, firstSnapshot[0].String()) testRunPrune(t, env.gopts, pruneOpts) - rtest.OK(t, runCheck(checkOpts, env.gopts, nil)) + rtest.OK(t, runCheck(context.TODO(), checkOpts, env.gopts, nil)) - rtest.OK(t, runRebuildIndex(RebuildIndexOptions{}, env.gopts)) - rtest.OK(t, runRebuildIndex(RebuildIndexOptions{ReadAllPacks: true}, env.gopts)) + rtest.OK(t, runRebuildIndex(context.TODO(), RebuildIndexOptions{}, env.gopts)) + rtest.OK(t, runRebuildIndex(context.TODO(), RebuildIndexOptions{ReadAllPacks: true}, env.gopts)) } func TestHardLink(t *testing.T) { @@ -2204,7 +2204,7 @@ func TestFindListOnce(t *testing.T) { testRunBackup(t, "", []string{filepath.Join(env.testdata, "0", "0", "9", "3")}, opts, env.gopts) thirdSnapshot := restic.NewIDSet(testRunList(t, "snapshots", env.gopts)...) - repo, err := OpenRepository(env.gopts) + repo, err := OpenRepository(context.TODO(), env.gopts) rtest.OK(t, err) snapshotIDs := restic.NewIDSet() From 928914f82151c0881405fbbf567be3a78dd12924 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 31 Oct 2021 23:19:27 +0100 Subject: [PATCH 04/11] Prepare for context bound to lock lifetime --- cmd/restic/cmd_backup.go | 2 +- cmd/restic/cmd_cat.go | 6 +++--- cmd/restic/cmd_check.go | 3 ++- cmd/restic/cmd_copy.go | 5 +++-- cmd/restic/cmd_debug.go | 6 ++++-- cmd/restic/cmd_diff.go | 3 ++- cmd/restic/cmd_dump.go | 3 ++- cmd/restic/cmd_find.go | 3 ++- cmd/restic/cmd_forget.go | 3 ++- cmd/restic/cmd_key.go | 8 ++++---- cmd/restic/cmd_list.go | 3 ++- cmd/restic/cmd_migrate.go | 2 +- cmd/restic/cmd_mount.go | 4 +++- cmd/restic/cmd_prune.go | 2 +- cmd/restic/cmd_rebuild_index.go | 2 +- cmd/restic/cmd_recover.go | 2 +- cmd/restic/cmd_restore.go | 3 ++- cmd/restic/cmd_snapshots.go | 3 ++- cmd/restic/cmd_stats.go | 3 ++- cmd/restic/cmd_tag.go | 3 ++- cmd/restic/lock.go | 10 +++++----- 21 files changed, 47 insertions(+), 32 deletions(-) diff --git a/cmd/restic/cmd_backup.go b/cmd/restic/cmd_backup.go index 1bd86ceeb..cb4c25141 100644 --- a/cmd/restic/cmd_backup.go +++ b/cmd/restic/cmd_backup.go @@ -586,7 +586,7 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter if !gopts.JSON { progressPrinter.V("lock repository") } - lock, err := lockRepo(ctx, repo) + lock, ctx, err := lockRepo(ctx, repo) defer unlockRepo(lock) if err != nil { return err diff --git a/cmd/restic/cmd_cat.go b/cmd/restic/cmd_cat.go index dcdfc6e8c..c2fedfcd4 100644 --- a/cmd/restic/cmd_cat.go +++ b/cmd/restic/cmd_cat.go @@ -44,12 +44,12 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string) error { } if !gopts.NoLock { - lock, err := lockRepo(ctx, repo) + var lock *restic.Lock + lock, ctx, err = lockRepo(ctx, repo) + defer unlockRepo(lock) if err != nil { return err } - - defer unlockRepo(lock) } tpe := args[0] diff --git a/cmd/restic/cmd_check.go b/cmd/restic/cmd_check.go index 899a6a232..0b20a5061 100644 --- a/cmd/restic/cmd_check.go +++ b/cmd/restic/cmd_check.go @@ -210,7 +210,8 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args if !gopts.NoLock { Verbosef("create exclusive lock for repository\n") - lock, err := lockRepoExclusive(ctx, repo) + var lock *restic.Lock + lock, ctx, err = lockRepoExclusive(ctx, repo) defer unlockRepo(lock) if err != nil { return err diff --git a/cmd/restic/cmd_copy.go b/cmd/restic/cmd_copy.go index 3aae94b13..8acbfc288 100644 --- a/cmd/restic/cmd_copy.go +++ b/cmd/restic/cmd_copy.go @@ -73,14 +73,15 @@ func runCopy(ctx context.Context, opts CopyOptions, gopts GlobalOptions, args [] } if !gopts.NoLock { - srcLock, err := lockRepo(ctx, srcRepo) + var srcLock *restic.Lock + srcLock, ctx, err = lockRepo(ctx, srcRepo) defer unlockRepo(srcLock) if err != nil { return err } } - dstLock, err := lockRepo(ctx, dstRepo) + dstLock, ctx, err := lockRepo(ctx, dstRepo) defer unlockRepo(dstLock) if err != nil { return err diff --git a/cmd/restic/cmd_debug.go b/cmd/restic/cmd_debug.go index f7cb30381..71bad2dac 100644 --- a/cmd/restic/cmd_debug.go +++ b/cmd/restic/cmd_debug.go @@ -152,7 +152,8 @@ func runDebugDump(ctx context.Context, gopts GlobalOptions, args []string) error } if !gopts.NoLock { - lock, err := lockRepo(ctx, repo) + var lock *restic.Lock + lock, ctx, err = lockRepo(ctx, repo) defer unlockRepo(lock) if err != nil { return err @@ -453,7 +454,8 @@ func runDebugExamine(ctx context.Context, gopts GlobalOptions, args []string) er } if !gopts.NoLock { - lock, err := lockRepo(ctx, repo) + var lock *restic.Lock + lock, ctx, err = lockRepo(ctx, repo) defer unlockRepo(lock) if err != nil { return err diff --git a/cmd/restic/cmd_diff.go b/cmd/restic/cmd_diff.go index e5fd95915..967995072 100644 --- a/cmd/restic/cmd_diff.go +++ b/cmd/restic/cmd_diff.go @@ -332,7 +332,8 @@ func runDiff(ctx context.Context, opts DiffOptions, gopts GlobalOptions, args [] } if !gopts.NoLock { - lock, err := lockRepo(ctx, repo) + var lock *restic.Lock + lock, ctx, err = lockRepo(ctx, repo) defer unlockRepo(lock) if err != nil { return err diff --git a/cmd/restic/cmd_dump.go b/cmd/restic/cmd_dump.go index d0c795969..f892225cf 100644 --- a/cmd/restic/cmd_dump.go +++ b/cmd/restic/cmd_dump.go @@ -131,7 +131,8 @@ func runDump(ctx context.Context, opts DumpOptions, gopts GlobalOptions, args [] } if !gopts.NoLock { - lock, err := lockRepo(ctx, repo) + var lock *restic.Lock + lock, ctx, err = lockRepo(ctx, repo) defer unlockRepo(lock) if err != nil { return err diff --git a/cmd/restic/cmd_find.go b/cmd/restic/cmd_find.go index 5b4cf503f..d47495ed5 100644 --- a/cmd/restic/cmd_find.go +++ b/cmd/restic/cmd_find.go @@ -574,7 +574,8 @@ func runFind(ctx context.Context, opts FindOptions, gopts GlobalOptions, args [] } if !gopts.NoLock { - lock, err := lockRepo(ctx, repo) + var lock *restic.Lock + lock, ctx, err = lockRepo(ctx, repo) defer unlockRepo(lock) if err != nil { return err diff --git a/cmd/restic/cmd_forget.go b/cmd/restic/cmd_forget.go index 112618bd1..60f454e69 100644 --- a/cmd/restic/cmd_forget.go +++ b/cmd/restic/cmd_forget.go @@ -115,7 +115,8 @@ func runForget(ctx context.Context, opts ForgetOptions, gopts GlobalOptions, arg } if !opts.DryRun || !gopts.NoLock { - lock, err := lockRepoExclusive(ctx, repo) + var lock *restic.Lock + lock, ctx, err = lockRepoExclusive(ctx, repo) defer unlockRepo(lock) if err != nil { return err diff --git a/cmd/restic/cmd_key.go b/cmd/restic/cmd_key.go index c8588eb59..b497343e4 100644 --- a/cmd/restic/cmd_key.go +++ b/cmd/restic/cmd_key.go @@ -209,7 +209,7 @@ func runKey(ctx context.Context, gopts GlobalOptions, args []string) error { switch args[0] { case "list": - lock, err := lockRepo(ctx, repo) + lock, ctx, err := lockRepo(ctx, repo) defer unlockRepo(lock) if err != nil { return err @@ -217,7 +217,7 @@ func runKey(ctx context.Context, gopts GlobalOptions, args []string) error { return listKeys(ctx, repo, gopts) case "add": - lock, err := lockRepo(ctx, repo) + lock, ctx, err := lockRepo(ctx, repo) defer unlockRepo(lock) if err != nil { return err @@ -225,7 +225,7 @@ func runKey(ctx context.Context, gopts GlobalOptions, args []string) error { return addKey(ctx, repo, gopts) case "remove": - lock, err := lockRepoExclusive(ctx, repo) + lock, ctx, err := lockRepoExclusive(ctx, repo) defer unlockRepo(lock) if err != nil { return err @@ -238,7 +238,7 @@ func runKey(ctx context.Context, gopts GlobalOptions, args []string) error { return deleteKey(ctx, repo, id) case "passwd": - lock, err := lockRepoExclusive(ctx, repo) + lock, ctx, err := lockRepoExclusive(ctx, repo) defer unlockRepo(lock) if err != nil { return err diff --git a/cmd/restic/cmd_list.go b/cmd/restic/cmd_list.go index 819c1de4b..9aa713879 100644 --- a/cmd/restic/cmd_list.go +++ b/cmd/restic/cmd_list.go @@ -42,7 +42,8 @@ func runList(ctx context.Context, cmd *cobra.Command, opts GlobalOptions, args [ } if !opts.NoLock && args[0] != "locks" { - lock, err := lockRepo(ctx, repo) + var lock *restic.Lock + lock, ctx, err = lockRepo(ctx, repo) defer unlockRepo(lock) if err != nil { return err diff --git a/cmd/restic/cmd_migrate.go b/cmd/restic/cmd_migrate.go index 7a48051c7..ffd855b48 100644 --- a/cmd/restic/cmd_migrate.go +++ b/cmd/restic/cmd_migrate.go @@ -122,7 +122,7 @@ func runMigrate(ctx context.Context, opts MigrateOptions, gopts GlobalOptions, a return err } - lock, err := lockRepoExclusive(ctx, repo) + lock, ctx, err := lockRepoExclusive(ctx, repo) defer unlockRepo(lock) if err != nil { return err diff --git a/cmd/restic/cmd_mount.go b/cmd/restic/cmd_mount.go index ef4d9a32b..0bbd1e6d5 100644 --- a/cmd/restic/cmd_mount.go +++ b/cmd/restic/cmd_mount.go @@ -13,6 +13,7 @@ import ( "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/restic" resticfs "github.com/restic/restic/internal/fs" "github.com/restic/restic/internal/fuse" @@ -121,7 +122,8 @@ func runMount(ctx context.Context, opts MountOptions, gopts GlobalOptions, args } if !gopts.NoLock { - lock, err := lockRepo(ctx, repo) + var lock *restic.Lock + lock, ctx, err = lockRepo(ctx, repo) defer unlockRepo(lock) if err != nil { return err diff --git a/cmd/restic/cmd_prune.go b/cmd/restic/cmd_prune.go index 2bfd2fc42..01ce0da26 100644 --- a/cmd/restic/cmd_prune.go +++ b/cmd/restic/cmd_prune.go @@ -165,7 +165,7 @@ func runPrune(ctx context.Context, opts PruneOptions, gopts GlobalOptions) error opts.unsafeRecovery = true } - lock, err := lockRepoExclusive(ctx, repo) + lock, ctx, err := lockRepoExclusive(ctx, repo) defer unlockRepo(lock) if err != nil { return err diff --git a/cmd/restic/cmd_rebuild_index.go b/cmd/restic/cmd_rebuild_index.go index 6fd9eac24..cb2917183 100644 --- a/cmd/restic/cmd_rebuild_index.go +++ b/cmd/restic/cmd_rebuild_index.go @@ -48,7 +48,7 @@ func runRebuildIndex(ctx context.Context, opts RebuildIndexOptions, gopts Global return err } - lock, err := lockRepoExclusive(ctx, repo) + lock, ctx, err := lockRepoExclusive(ctx, repo) defer unlockRepo(lock) if err != nil { return err diff --git a/cmd/restic/cmd_recover.go b/cmd/restic/cmd_recover.go index b78a527ab..b296f7b9b 100644 --- a/cmd/restic/cmd_recover.go +++ b/cmd/restic/cmd_recover.go @@ -46,7 +46,7 @@ func runRecover(ctx context.Context, gopts GlobalOptions) error { return err } - lock, err := lockRepo(ctx, repo) + lock, ctx, err := lockRepo(ctx, repo) defer unlockRepo(lock) if err != nil { return err diff --git a/cmd/restic/cmd_restore.go b/cmd/restic/cmd_restore.go index a66d1e633..6ddec5233 100644 --- a/cmd/restic/cmd_restore.go +++ b/cmd/restic/cmd_restore.go @@ -123,7 +123,8 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions, a } if !gopts.NoLock { - lock, err := lockRepo(ctx, repo) + var lock *restic.Lock + lock, ctx, err = lockRepo(ctx, repo) defer unlockRepo(lock) if err != nil { return err diff --git a/cmd/restic/cmd_snapshots.go b/cmd/restic/cmd_snapshots.go index 6810be7a0..82561722c 100644 --- a/cmd/restic/cmd_snapshots.go +++ b/cmd/restic/cmd_snapshots.go @@ -64,7 +64,8 @@ func runSnapshots(ctx context.Context, opts SnapshotOptions, gopts GlobalOptions } if !gopts.NoLock { - lock, err := lockRepo(ctx, repo) + var lock *restic.Lock + lock, ctx, err = lockRepo(ctx, repo) defer unlockRepo(lock) if err != nil { return err diff --git a/cmd/restic/cmd_stats.go b/cmd/restic/cmd_stats.go index 67a5ab542..f03026574 100644 --- a/cmd/restic/cmd_stats.go +++ b/cmd/restic/cmd_stats.go @@ -80,7 +80,8 @@ func runStats(ctx context.Context, gopts GlobalOptions, args []string) error { } if !gopts.NoLock { - lock, err := lockRepo(ctx, repo) + var lock *restic.Lock + lock, ctx, err = lockRepo(ctx, repo) defer unlockRepo(lock) if err != nil { return err diff --git a/cmd/restic/cmd_tag.go b/cmd/restic/cmd_tag.go index a27eb5d7b..bf880dafb 100644 --- a/cmd/restic/cmd_tag.go +++ b/cmd/restic/cmd_tag.go @@ -110,7 +110,8 @@ func runTag(ctx context.Context, opts TagOptions, gopts GlobalOptions, args []st if !gopts.NoLock { Verbosef("create exclusive lock for repository\n") - lock, err := lockRepoExclusive(ctx, repo) + var lock *restic.Lock + lock, ctx, err = lockRepoExclusive(ctx, repo) defer unlockRepo(lock) if err != nil { return err diff --git a/cmd/restic/lock.go b/cmd/restic/lock.go index 89acb0aef..0cea02cfd 100644 --- a/cmd/restic/lock.go +++ b/cmd/restic/lock.go @@ -19,15 +19,15 @@ var globalLocks struct { sync.Once } -func lockRepo(ctx context.Context, repo *repository.Repository) (*restic.Lock, error) { +func lockRepo(ctx context.Context, repo *repository.Repository) (*restic.Lock, context.Context, error) { return lockRepository(ctx, repo, false) } -func lockRepoExclusive(ctx context.Context, repo *repository.Repository) (*restic.Lock, error) { +func lockRepoExclusive(ctx context.Context, repo *repository.Repository) (*restic.Lock, context.Context, error) { return lockRepository(ctx, repo, true) } -func lockRepository(ctx context.Context, repo *repository.Repository, exclusive bool) (*restic.Lock, error) { +func lockRepository(ctx context.Context, repo *repository.Repository, exclusive bool) (*restic.Lock, context.Context, error) { // make sure that a repository is unlocked properly and after cancel() was // called by the cleanup handler in global.go globalLocks.Do(func() { @@ -41,7 +41,7 @@ func lockRepository(ctx context.Context, repo *repository.Repository, exclusive lock, err := lockFn(ctx, repo) if err != nil { - return nil, errors.WithMessage(err, "unable to create lock in backend") + return nil, ctx, errors.WithMessage(err, "unable to create lock in backend") } debug.Log("create lock %p (exclusive %v)", lock, exclusive) @@ -57,7 +57,7 @@ func lockRepository(ctx context.Context, repo *repository.Repository, exclusive globalLocks.locks = append(globalLocks.locks, lock) globalLocks.Unlock() - return lock, err + return lock, ctx, err } var refreshInterval = 5 * time.Minute From d92957dd78ca01dc16a7998ca6a9e339fe23044b Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 31 Oct 2021 23:25:36 +0100 Subject: [PATCH 05/11] lock: Implement strict lock expiry monitoring Restic continued e.g. a backup task even when it failed to renew the lock or failed to do so in time. For example if a backup client enters standby during the backup this can allow other operations like `prune` to run in the meantime (after calling `unlock`). After leaving standby the backup client will continue its backup and upload indexes which refer pack files that were removed in the meantime. This commit introduces a goroutine explicitly monitoring for locks that are not refreshed in time. To simplify the implementation there's now a separate goroutine to refresh the lock and monitor for timeouts for each lock. The monitoring goroutine would now cause the backup to fail as the client has lost it's lock in the meantime. The lock refresh goroutines are bound to the context used to lock the repository initially. The context returned by `lockRepo` is also cancelled when any of the goroutines exits. This ensures that the context is cancelled whenever for any reason the lock is no longer refreshed. --- cmd/restic/lock.go | 149 ++++++++++++++++++++++++++-------------- internal/restic/lock.go | 4 +- 2 files changed, 99 insertions(+), 54 deletions(-) diff --git a/cmd/restic/lock.go b/cmd/restic/lock.go index 0cea02cfd..188bb1d59 100644 --- a/cmd/restic/lock.go +++ b/cmd/restic/lock.go @@ -11,10 +11,13 @@ import ( "github.com/restic/restic/internal/restic" ) +type lockContext struct { + cancel context.CancelFunc + refreshWG sync.WaitGroup +} + var globalLocks struct { - locks []*restic.Lock - cancelRefresh chan struct{} - refreshWG sync.WaitGroup + locks map[*restic.Lock]*lockContext sync.Mutex sync.Once } @@ -27,6 +30,8 @@ func lockRepoExclusive(ctx context.Context, repo *repository.Repository) (*resti return lockRepository(ctx, repo, true) } +// lockRepository wraps the ctx such that it is cancelled when the repository is unlocked +// cancelling the original context also stops the lock refresh func lockRepository(ctx context.Context, repo *repository.Repository, exclusive bool) (*restic.Lock, context.Context, error) { // make sure that a repository is unlocked properly and after cancel() was // called by the cleanup handler in global.go @@ -45,16 +50,17 @@ func lockRepository(ctx context.Context, repo *repository.Repository, exclusive } debug.Log("create lock %p (exclusive %v)", lock, exclusive) - globalLocks.Lock() - if globalLocks.cancelRefresh == nil { - debug.Log("start goroutine for lock refresh") - globalLocks.cancelRefresh = make(chan struct{}) - globalLocks.refreshWG = sync.WaitGroup{} - globalLocks.refreshWG.Add(1) - go refreshLocks(&globalLocks.refreshWG, globalLocks.cancelRefresh) + ctx, cancel := context.WithCancel(ctx) + lockInfo := &lockContext{ + cancel: cancel, } + lockInfo.refreshWG.Add(2) + refreshChan := make(chan struct{}) - globalLocks.locks = append(globalLocks.locks, lock) + globalLocks.Lock() + globalLocks.locks[lock] = lockInfo + go refreshLocks(ctx, lock, lockInfo, refreshChan) + go monitorLockRefresh(ctx, lock, lockInfo, refreshChan) globalLocks.Unlock() return lock, ctx, err @@ -62,32 +68,76 @@ func lockRepository(ctx context.Context, repo *repository.Repository, exclusive var refreshInterval = 5 * time.Minute -func refreshLocks(wg *sync.WaitGroup, done <-chan struct{}) { - debug.Log("start") - defer func() { - wg.Done() - globalLocks.Lock() - globalLocks.cancelRefresh = nil - globalLocks.Unlock() - }() +// consider a lock refresh failed a bit before the lock actually becomes stale +// the difference allows to compensate for a small time drift between clients. +var refreshabilityTimeout = restic.StaleLockTimeout - refreshInterval*3/2 +func refreshLocks(ctx context.Context, lock *restic.Lock, lockInfo *lockContext, refreshed chan<- struct{}) { + debug.Log("start") ticker := time.NewTicker(refreshInterval) + lastRefresh := lock.Time + + defer func() { + ticker.Stop() + // ensure that the context was cancelled before removing the lock + lockInfo.cancel() + + // remove the lock from the repo + debug.Log("unlocking repository with lock %v", lock) + if err := lock.Unlock(); err != nil { + debug.Log("error while unlocking: %v", err) + Warnf("error while unlocking: %v", err) + } + + lockInfo.refreshWG.Done() + }() for { select { - case <-done: + case <-ctx.Done(): debug.Log("terminate") return case <-ticker.C: + if time.Since(lastRefresh) > refreshabilityTimeout { + // the lock is too old, wait until the expiry monitor cancels the context + continue + } + debug.Log("refreshing locks") - globalLocks.Lock() - for _, lock := range globalLocks.locks { - err := lock.Refresh(context.TODO()) - if err != nil { - Warnf("unable to refresh lock: %v\n", err) + err := lock.Refresh(context.TODO()) + if err != nil { + Warnf("unable to refresh lock: %v\n", err) + } else { + lastRefresh = lock.Time + // inform monitor gorountine about successful refresh + select { + case <-ctx.Done(): + case refreshed <- struct{}{}: } } - globalLocks.Unlock() + } + } +} + +func monitorLockRefresh(ctx context.Context, lock *restic.Lock, lockInfo *lockContext, refreshed <-chan struct{}) { + timer := time.NewTimer(refreshabilityTimeout) + defer func() { + timer.Stop() + lockInfo.cancel() + lockInfo.refreshWG.Done() + }() + + for { + select { + case <-ctx.Done(): + debug.Log("terminate expiry monitoring") + return + case <-refreshed: + // reset timer once the lock was refreshed successfully + timer.Reset(refreshabilityTimeout) + case <-timer.C: + Warnf("Fatal: failed to refresh lock in time\n") + return } } } @@ -98,40 +148,35 @@ func unlockRepo(lock *restic.Lock) { } globalLocks.Lock() - defer globalLocks.Unlock() + lockInfo, exists := globalLocks.locks[lock] + delete(globalLocks.locks, lock) + globalLocks.Unlock() - for i := 0; i < len(globalLocks.locks); i++ { - if lock == globalLocks.locks[i] { - // remove the lock from the repo - debug.Log("unlocking repository with lock %v", lock) - if err := lock.Unlock(); err != nil { - debug.Log("error while unlocking: %v", err) - Warnf("error while unlocking: %v", err) - return - } - - // remove the lock from the list of locks - globalLocks.locks = append(globalLocks.locks[:i], globalLocks.locks[i+1:]...) - return - } + if !exists { + debug.Log("unable to find lock %v in the global list of locks, ignoring", lock) + return } - - debug.Log("unable to find lock %v in the global list of locks, ignoring", lock) + lockInfo.cancel() + lockInfo.refreshWG.Wait() } func unlockAll(code int) (int, error) { globalLocks.Lock() - defer globalLocks.Unlock() - + locks := globalLocks.locks debug.Log("unlocking %d locks", len(globalLocks.locks)) - for _, lock := range globalLocks.locks { - if err := lock.Unlock(); err != nil { - debug.Log("error while unlocking: %v", err) - return code, err - } - debug.Log("successfully removed lock") + for _, lockInfo := range globalLocks.locks { + lockInfo.cancel() + } + globalLocks.locks = make(map[*restic.Lock]*lockContext) + globalLocks.Unlock() + + for _, lockInfo := range locks { + lockInfo.refreshWG.Wait() } - globalLocks.locks = globalLocks.locks[:0] return code, nil } + +func init() { + globalLocks.locks = make(map[*restic.Lock]*lockContext) +} diff --git a/internal/restic/lock.go b/internal/restic/lock.go index 031e8755c..c8079f58d 100644 --- a/internal/restic/lock.go +++ b/internal/restic/lock.go @@ -175,14 +175,14 @@ func (l *Lock) Unlock() error { return l.repo.Backend().Remove(context.TODO(), Handle{Type: LockFile, Name: l.lockID.String()}) } -var staleTimeout = 30 * time.Minute +var StaleLockTimeout = 30 * time.Minute // Stale returns true if the lock is stale. A lock is stale if the timestamp is // older than 30 minutes or if it was created on the current machine and the // process isn't alive any more. func (l *Lock) Stale() bool { debug.Log("testing if lock %v for process %d is stale", l, l.PID) - if time.Since(l.Time) > staleTimeout { + if time.Since(l.Time) > StaleLockTimeout { debug.Log("lock is stale, timestamp is too old: %v\n", l.Time) return true } From c3538b063aa82bdcec009fbd4f937a3d41fef992 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 14 Nov 2021 16:32:03 +0100 Subject: [PATCH 06/11] lock: Use repository interface instead of struct --- cmd/restic/lock.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/cmd/restic/lock.go b/cmd/restic/lock.go index 188bb1d59..03d880670 100644 --- a/cmd/restic/lock.go +++ b/cmd/restic/lock.go @@ -7,7 +7,6 @@ import ( "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" ) @@ -22,17 +21,17 @@ var globalLocks struct { sync.Once } -func lockRepo(ctx context.Context, repo *repository.Repository) (*restic.Lock, context.Context, error) { +func lockRepo(ctx context.Context, repo restic.Repository) (*restic.Lock, context.Context, error) { return lockRepository(ctx, repo, false) } -func lockRepoExclusive(ctx context.Context, repo *repository.Repository) (*restic.Lock, context.Context, error) { +func lockRepoExclusive(ctx context.Context, repo restic.Repository) (*restic.Lock, context.Context, error) { return lockRepository(ctx, repo, true) } // lockRepository wraps the ctx such that it is cancelled when the repository is unlocked // cancelling the original context also stops the lock refresh -func lockRepository(ctx context.Context, repo *repository.Repository, exclusive bool) (*restic.Lock, context.Context, error) { +func lockRepository(ctx context.Context, repo restic.Repository, exclusive bool) (*restic.Lock, context.Context, error) { // make sure that a repository is unlocked properly and after cancel() was // called by the cleanup handler in global.go globalLocks.Do(func() { From 9959190e396814c009c2b22fee79fb2d25efff70 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 14 Nov 2021 17:38:56 +0100 Subject: [PATCH 07/11] lock: Add integration test The tests check that the wrapped context is properly canceled whenever the repository is unlock or when the lock refresh fails. --- cmd/restic/lock_test.go | 130 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 130 insertions(+) create mode 100644 cmd/restic/lock_test.go diff --git a/cmd/restic/lock_test.go b/cmd/restic/lock_test.go new file mode 100644 index 000000000..70e864448 --- /dev/null +++ b/cmd/restic/lock_test.go @@ -0,0 +1,130 @@ +package main + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/restic/restic/internal/repository" + "github.com/restic/restic/internal/restic" + rtest "github.com/restic/restic/internal/test" +) + +func openTestRepo(t *testing.T, wrapper backendWrapper) (*repository.Repository, func(), *testEnvironment) { + env, cleanup := withTestEnvironment(t) + if wrapper != nil { + env.gopts.backendTestHook = wrapper + } + testRunInit(t, env.gopts) + + repo, err := OpenRepository(context.TODO(), env.gopts) + rtest.OK(t, err) + return repo, cleanup, env +} + +func checkedLockRepo(ctx context.Context, t *testing.T, repo restic.Repository) (*restic.Lock, context.Context) { + lock, wrappedCtx, err := lockRepo(ctx, repo) + rtest.OK(t, err) + rtest.OK(t, wrappedCtx.Err()) + if lock.Stale() { + t.Fatal("lock returned stale lock") + } + return lock, wrappedCtx +} + +func TestLock(t *testing.T) { + repo, cleanup, _ := openTestRepo(t, nil) + defer cleanup() + + lock, wrappedCtx := checkedLockRepo(context.Background(), t, repo) + unlockRepo(lock) + if wrappedCtx.Err() == nil { + t.Fatal("unlock did not cancel context") + } +} + +func TestLockCancel(t *testing.T) { + repo, cleanup, _ := openTestRepo(t, nil) + defer cleanup() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + lock, wrappedCtx := checkedLockRepo(ctx, t, repo) + cancel() + if wrappedCtx.Err() == nil { + t.Fatal("canceled parent context did not cancel context") + } + + // unlockRepo should not crash + unlockRepo(lock) +} + +func TestLockUnlockAll(t *testing.T) { + repo, cleanup, _ := openTestRepo(t, nil) + defer cleanup() + + lock, wrappedCtx := checkedLockRepo(context.Background(), t, repo) + _, err := unlockAll(0) + rtest.OK(t, err) + if wrappedCtx.Err() == nil { + t.Fatal("canceled parent context did not cancel context") + } + + // unlockRepo should not crash + unlockRepo(lock) +} + +func TestLockConflict(t *testing.T) { + repo, cleanup, env := openTestRepo(t, nil) + defer cleanup() + repo2, err := OpenRepository(context.TODO(), env.gopts) + rtest.OK(t, err) + + lock, _, err := lockRepoExclusive(context.Background(), repo) + rtest.OK(t, err) + defer unlockRepo(lock) + _, _, err = lockRepo(context.Background(), repo2) + if err == nil { + t.Fatal("second lock should have failed") + } +} + +type writeOnceBackend struct { + restic.Backend + written bool +} + +func (b *writeOnceBackend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { + if b.written { + return fmt.Errorf("fail after first write") + } + b.written = true + return b.Backend.Save(ctx, h, rd) +} + +func TestLockFailedRefresh(t *testing.T) { + repo, cleanup, _ := openTestRepo(t, func(r restic.Backend) (restic.Backend, error) { + return &writeOnceBackend{Backend: r}, nil + }) + defer cleanup() + + // reduce locking intervals to be suitable for testing + ri, rt := refreshInterval, refreshabilityTimeout + refreshInterval = 20 * time.Millisecond + refreshabilityTimeout = 100 * time.Millisecond + defer func() { + refreshInterval, refreshabilityTimeout = ri, rt + }() + + lock, wrappedCtx := checkedLockRepo(context.Background(), t, repo) + + select { + case <-wrappedCtx.Done(): + // expected lock refresh failure + case <-time.After(time.Second): + t.Fatal("failed lock refresh did not cause context cancellation") + } + // unlockRepo should not crash + unlockRepo(lock) +} From aeed420e1aec8210e9a0f7c2774de18f53b3be79 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 14 Nov 2021 17:52:41 +0100 Subject: [PATCH 08/11] add changelog --- changelog/unreleased/issue-2715 | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 changelog/unreleased/issue-2715 diff --git a/changelog/unreleased/issue-2715 b/changelog/unreleased/issue-2715 new file mode 100644 index 000000000..2401ce009 --- /dev/null +++ b/changelog/unreleased/issue-2715 @@ -0,0 +1,12 @@ +Enhancement: Cancel commands if lock is not refresh in time + +Restic commands kept running even if they failed to refresh their locks in +time. This can be a problem if a concurrent call to `unlock` and `prune` +removes data from the repository. Not refreshing a lock in time can for example +be caused by a client switching to standby while running a backup. + +Lock handling is now much stricter. Commands requiring a lock are canceled if +the lock is not refreshed successfully in time. + +https://github.com/restic/restic/issues/2715 +https://github.com/restic/restic/pull/3569 From 401e432e9d5de678c2463dae8426a1b6949afcfa Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 20 Nov 2021 16:48:22 +0100 Subject: [PATCH 09/11] lock: Do not ignore invalid lock files While searching for lock file from concurrently running restic instances, restic ignored unreadable lock files. These can either be in fact invalid or just be temporarily unreadable. As it is not really possible to differentiate between both cases, just err on the side of caution and consider the repository as already locked. The code retries searching for other locks up to three times to smooth out temporarily unreadable lock files. --- changelog/unreleased/issue-2715 | 7 +++++- internal/restic/lock.go | 42 ++++++++++++++++++++++----------- internal/restic/lock_test.go | 28 ++++++++++++++++++++++ 3 files changed, 62 insertions(+), 15 deletions(-) diff --git a/changelog/unreleased/issue-2715 b/changelog/unreleased/issue-2715 index 2401ce009..2e4d67c97 100644 --- a/changelog/unreleased/issue-2715 +++ b/changelog/unreleased/issue-2715 @@ -1,4 +1,4 @@ -Enhancement: Cancel commands if lock is not refresh in time +Enhancement: Stricter repository lock handling Restic commands kept running even if they failed to refresh their locks in time. This can be a problem if a concurrent call to `unlock` and `prune` @@ -8,5 +8,10 @@ be caused by a client switching to standby while running a backup. Lock handling is now much stricter. Commands requiring a lock are canceled if the lock is not refreshed successfully in time. +In addition, if a lock file is not readable restic will not allow starting a +command. It may be necessary to remove invalid lock file manually or using +`unlock --remove-all`. Please make sure that no other restic processes are +running concurrently. + https://github.com/restic/restic/issues/2715 https://github.com/restic/restic/pull/3569 diff --git a/internal/restic/lock.go b/internal/restic/lock.go index c8079f58d..13a422e71 100644 --- a/internal/restic/lock.go +++ b/internal/restic/lock.go @@ -137,23 +137,37 @@ func (l *Lock) fillUserInfo() error { // non-exclusive lock is to be created, an error is only returned when an // exclusive lock is found. func (l *Lock) checkForOtherLocks(ctx context.Context) error { - return ForAllLocks(ctx, l.repo, l.lockID, func(id ID, lock *Lock, err error) error { - if err != nil { - // ignore locks that cannot be loaded - debug.Log("ignore lock %v: %v", id, err) + var err error + // retry locking a few times + for i := 0; i < 3; i++ { + err = ForAllLocks(ctx, l.repo, l.lockID, func(id ID, lock *Lock, err error) error { + if err != nil { + // if we cannot load a lock then it is unclear whether it can be ignored + // it could either be invalid or just unreadable due to network/permission problems + debug.Log("ignore lock %v: %v", id, err) + return errors.Fatal(err.Error()) + } + + if l.Exclusive { + return &alreadyLockedError{otherLock: lock} + } + + if !l.Exclusive && lock.Exclusive { + return &alreadyLockedError{otherLock: lock} + } + + return nil + }) + // no lock detected + if err == nil { return nil } - - if l.Exclusive { - return &alreadyLockedError{otherLock: lock} + // lock conflicts are permanent + if _, ok := err.(*alreadyLockedError); ok { + return err } - - if !l.Exclusive && lock.Exclusive { - return &alreadyLockedError{otherLock: lock} - } - - return nil - }) + } + return err } // createLock acquires the lock by creating a file in the repository. diff --git a/internal/restic/lock_test.go b/internal/restic/lock_test.go index 8a0622020..577e204aa 100644 --- a/internal/restic/lock_test.go +++ b/internal/restic/lock_test.go @@ -2,10 +2,13 @@ package restic_test import ( "context" + "fmt" + "io" "os" "testing" "time" + "github.com/restic/restic/internal/backend/mem" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" @@ -49,6 +52,31 @@ func TestMultipleLock(t *testing.T) { rtest.OK(t, lock2.Unlock()) } +type failLockLoadingBackend struct { + restic.Backend +} + +func (be *failLockLoadingBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { + if h.Type == restic.LockFile { + return fmt.Errorf("error loading lock") + } + return be.Backend.Load(ctx, h, length, offset, fn) +} + +func TestMultipleLockFailure(t *testing.T) { + be := &failLockLoadingBackend{Backend: mem.New()} + repo, cleanup := repository.TestRepositoryWithBackend(t, be, 0) + defer cleanup() + + lock1, err := restic.NewLock(context.TODO(), repo) + rtest.OK(t, err) + + _, err = restic.NewLock(context.TODO(), repo) + rtest.Assert(t, err != nil, "unreadable lock file did not result in an error") + + rtest.OK(t, lock1.Unlock()) +} + func TestLockExclusive(t *testing.T) { repo, cleanup := repository.TestRepository(t) defer cleanup() From 49126796d03213a8b57d4a5a344f09d55a16cb51 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 10 Sep 2022 19:25:06 +0200 Subject: [PATCH 10/11] lock: fix timer expiry monitoring during standby Monotonic timers are paused during standby. Thus these timers won't fire after waking up. Fall back to periodic polling to detect too large clock jumps. See https://github.com/golang/go/issues/35012 for a discussion of go timers during standby. --- cmd/restic/lock.go | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/cmd/restic/lock.go b/cmd/restic/lock.go index 03d880670..1a8386de5 100644 --- a/cmd/restic/lock.go +++ b/cmd/restic/lock.go @@ -119,7 +119,18 @@ func refreshLocks(ctx context.Context, lock *restic.Lock, lockInfo *lockContext, } func monitorLockRefresh(ctx context.Context, lock *restic.Lock, lockInfo *lockContext, refreshed <-chan struct{}) { - timer := time.NewTimer(refreshabilityTimeout) + // time.Now() might use a monotonic timer which is paused during standby + // convert to unix time to ensure we compare real time values + lastRefresh := time.Now().Unix() + pollDuration := 1 * time.Second + if refreshInterval < pollDuration { + // require for TestLockFailedRefresh + pollDuration = refreshInterval / 5 + } + // timers are paused during standby, which is a problem as the refresh timeout + // _must_ expire if the host was too long in standby. Thus fall back to periodic checks + // https://github.com/golang/go/issues/35012 + timer := time.NewTimer(pollDuration) defer func() { timer.Stop() lockInfo.cancel() @@ -132,9 +143,14 @@ func monitorLockRefresh(ctx context.Context, lock *restic.Lock, lockInfo *lockCo debug.Log("terminate expiry monitoring") return case <-refreshed: - // reset timer once the lock was refreshed successfully - timer.Reset(refreshabilityTimeout) + lastRefresh = time.Now().Unix() case <-timer.C: + if float64(time.Now().Unix()-lastRefresh) < refreshInterval.Seconds() { + // restart timer + timer.Reset(pollDuration) + continue + } + Warnf("Fatal: failed to refresh lock in time\n") return } From 6d2d297215bc4ffc332b78e725bcf32a4a70fa4c Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 2 Oct 2022 23:24:37 +0200 Subject: [PATCH 11/11] pass global context through cobra --- cmd/restic/cmd_backup.go | 2 +- cmd/restic/cmd_cat.go | 2 +- cmd/restic/cmd_check.go | 2 +- cmd/restic/cmd_copy.go | 2 +- cmd/restic/cmd_debug.go | 4 ++-- cmd/restic/cmd_diff.go | 2 +- cmd/restic/cmd_dump.go | 2 +- cmd/restic/cmd_find.go | 2 +- cmd/restic/cmd_forget.go | 2 +- cmd/restic/cmd_init.go | 2 +- cmd/restic/cmd_key.go | 2 +- cmd/restic/cmd_list.go | 2 +- cmd/restic/cmd_ls.go | 2 +- cmd/restic/cmd_migrate.go | 2 +- cmd/restic/cmd_mount.go | 2 +- cmd/restic/cmd_prune.go | 2 +- cmd/restic/cmd_rebuild_index.go | 2 +- cmd/restic/cmd_recover.go | 2 +- cmd/restic/cmd_restore.go | 2 +- cmd/restic/cmd_self_update.go | 2 +- cmd/restic/cmd_snapshots.go | 2 +- cmd/restic/cmd_stats.go | 2 +- cmd/restic/cmd_tag.go | 2 +- cmd/restic/cmd_unlock.go | 2 +- cmd/restic/global.go | 4 ---- cmd/restic/main.go | 2 +- 26 files changed, 26 insertions(+), 30 deletions(-) diff --git a/cmd/restic/cmd_backup.go b/cmd/restic/cmd_backup.go index cb4c25141..286ef51e2 100644 --- a/cmd/restic/cmd_backup.go +++ b/cmd/restic/cmd_backup.go @@ -57,7 +57,7 @@ Exit status is 3 if some source data could not be read (incomplete snapshot crea }, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - ctx := globalCtx() + ctx := cmd.Context() var wg sync.WaitGroup cancelCtx, cancel := context.WithCancel(ctx) defer func() { diff --git a/cmd/restic/cmd_cat.go b/cmd/restic/cmd_cat.go index c2fedfcd4..16fa968a8 100644 --- a/cmd/restic/cmd_cat.go +++ b/cmd/restic/cmd_cat.go @@ -25,7 +25,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - return runCat(globalCtx(), globalOptions, args) + return runCat(cmd.Context(), globalOptions, args) }, } diff --git a/cmd/restic/cmd_check.go b/cmd/restic/cmd_check.go index 0b20a5061..692a19ba1 100644 --- a/cmd/restic/cmd_check.go +++ b/cmd/restic/cmd_check.go @@ -35,7 +35,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - return runCheck(globalCtx(), checkOptions, globalOptions, args) + return runCheck(cmd.Context(), checkOptions, globalOptions, args) }, PreRunE: func(cmd *cobra.Command, args []string) error { return checkFlags(checkOptions) diff --git a/cmd/restic/cmd_copy.go b/cmd/restic/cmd_copy.go index 8acbfc288..08df7e8a7 100644 --- a/cmd/restic/cmd_copy.go +++ b/cmd/restic/cmd_copy.go @@ -32,7 +32,7 @@ This can be mitigated by the "--copy-chunker-params" option when initializing a new destination repository using the "init" command. `, RunE: func(cmd *cobra.Command, args []string) error { - return runCopy(globalCtx(), copyOptions, globalOptions, args) + return runCopy(cmd.Context(), copyOptions, globalOptions, args) }, } diff --git a/cmd/restic/cmd_debug.go b/cmd/restic/cmd_debug.go index 71bad2dac..38bc30ec8 100644 --- a/cmd/restic/cmd_debug.go +++ b/cmd/restic/cmd_debug.go @@ -46,7 +46,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - return runDebugDump(globalCtx(), globalOptions, args) + return runDebugDump(cmd.Context(), globalOptions, args) }, } @@ -193,7 +193,7 @@ var cmdDebugExamine = &cobra.Command{ Short: "Examine a pack file", DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - return runDebugExamine(globalCtx(), globalOptions, args) + return runDebugExamine(cmd.Context(), globalOptions, args) }, } diff --git a/cmd/restic/cmd_diff.go b/cmd/restic/cmd_diff.go index 967995072..2444e677b 100644 --- a/cmd/restic/cmd_diff.go +++ b/cmd/restic/cmd_diff.go @@ -35,7 +35,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - return runDiff(globalCtx(), diffOptions, globalOptions, args) + return runDiff(cmd.Context(), diffOptions, globalOptions, args) }, } diff --git a/cmd/restic/cmd_dump.go b/cmd/restic/cmd_dump.go index f892225cf..508cb5c76 100644 --- a/cmd/restic/cmd_dump.go +++ b/cmd/restic/cmd_dump.go @@ -34,7 +34,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - return runDump(globalCtx(), dumpOptions, globalOptions, args) + return runDump(cmd.Context(), dumpOptions, globalOptions, args) }, } diff --git a/cmd/restic/cmd_find.go b/cmd/restic/cmd_find.go index d47495ed5..8e5f9b604 100644 --- a/cmd/restic/cmd_find.go +++ b/cmd/restic/cmd_find.go @@ -38,7 +38,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - return runFind(globalCtx(), findOptions, globalOptions, args) + return runFind(cmd.Context(), findOptions, globalOptions, args) }, } diff --git a/cmd/restic/cmd_forget.go b/cmd/restic/cmd_forget.go index 60f454e69..f1aef75e3 100644 --- a/cmd/restic/cmd_forget.go +++ b/cmd/restic/cmd_forget.go @@ -32,7 +32,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - return runForget(globalCtx(), forgetOptions, globalOptions, args) + return runForget(cmd.Context(), forgetOptions, globalOptions, args) }, } diff --git a/cmd/restic/cmd_init.go b/cmd/restic/cmd_init.go index 4922a58b1..f833369ef 100644 --- a/cmd/restic/cmd_init.go +++ b/cmd/restic/cmd_init.go @@ -26,7 +26,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - return runInit(globalCtx(), initOptions, globalOptions, args) + return runInit(cmd.Context(), initOptions, globalOptions, args) }, } diff --git a/cmd/restic/cmd_key.go b/cmd/restic/cmd_key.go index b497343e4..52143bbc1 100644 --- a/cmd/restic/cmd_key.go +++ b/cmd/restic/cmd_key.go @@ -28,7 +28,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - return runKey(globalCtx(), globalOptions, args) + return runKey(cmd.Context(), globalOptions, args) }, } diff --git a/cmd/restic/cmd_list.go b/cmd/restic/cmd_list.go index 9aa713879..81390ecf8 100644 --- a/cmd/restic/cmd_list.go +++ b/cmd/restic/cmd_list.go @@ -23,7 +23,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - return runList(globalCtx(), cmd, globalOptions, args) + return runList(cmd.Context(), cmd, globalOptions, args) }, } diff --git a/cmd/restic/cmd_ls.go b/cmd/restic/cmd_ls.go index 96465c431..fd9a0ef0d 100644 --- a/cmd/restic/cmd_ls.go +++ b/cmd/restic/cmd_ls.go @@ -42,7 +42,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - return runLs(globalCtx(), lsOptions, globalOptions, args) + return runLs(cmd.Context(), lsOptions, globalOptions, args) }, } diff --git a/cmd/restic/cmd_migrate.go b/cmd/restic/cmd_migrate.go index ffd855b48..6d614be39 100644 --- a/cmd/restic/cmd_migrate.go +++ b/cmd/restic/cmd_migrate.go @@ -24,7 +24,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - return runMigrate(globalCtx(), migrateOptions, globalOptions, args) + return runMigrate(cmd.Context(), migrateOptions, globalOptions, args) }, } diff --git a/cmd/restic/cmd_mount.go b/cmd/restic/cmd_mount.go index 0bbd1e6d5..95ee7efcf 100644 --- a/cmd/restic/cmd_mount.go +++ b/cmd/restic/cmd_mount.go @@ -68,7 +68,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - return runMount(globalCtx(), mountOptions, globalOptions, args) + return runMount(cmd.Context(), mountOptions, globalOptions, args) }, } diff --git a/cmd/restic/cmd_prune.go b/cmd/restic/cmd_prune.go index 01ce0da26..74b0db026 100644 --- a/cmd/restic/cmd_prune.go +++ b/cmd/restic/cmd_prune.go @@ -34,7 +34,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - return runPrune(globalCtx(), pruneOptions, globalOptions) + return runPrune(cmd.Context(), pruneOptions, globalOptions) }, } diff --git a/cmd/restic/cmd_rebuild_index.go b/cmd/restic/cmd_rebuild_index.go index cb2917183..15a21df65 100644 --- a/cmd/restic/cmd_rebuild_index.go +++ b/cmd/restic/cmd_rebuild_index.go @@ -24,7 +24,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - return runRebuildIndex(globalCtx(), rebuildIndexOptions, globalOptions) + return runRebuildIndex(cmd.Context(), rebuildIndexOptions, globalOptions) }, } diff --git a/cmd/restic/cmd_recover.go b/cmd/restic/cmd_recover.go index b296f7b9b..65f4c8750 100644 --- a/cmd/restic/cmd_recover.go +++ b/cmd/restic/cmd_recover.go @@ -27,7 +27,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - return runRecover(globalCtx(), globalOptions) + return runRecover(cmd.Context(), globalOptions) }, } diff --git a/cmd/restic/cmd_restore.go b/cmd/restic/cmd_restore.go index 6ddec5233..1254174d7 100644 --- a/cmd/restic/cmd_restore.go +++ b/cmd/restic/cmd_restore.go @@ -31,7 +31,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - return runRestore(globalCtx(), restoreOptions, globalOptions, args) + return runRestore(cmd.Context(), restoreOptions, globalOptions, args) }, } diff --git a/cmd/restic/cmd_self_update.go b/cmd/restic/cmd_self_update.go index cc10c4c77..23345a97c 100644 --- a/cmd/restic/cmd_self_update.go +++ b/cmd/restic/cmd_self_update.go @@ -28,7 +28,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - return runSelfUpdate(globalCtx(), selfUpdateOptions, globalOptions, args) + return runSelfUpdate(cmd.Context(), selfUpdateOptions, globalOptions, args) }, } diff --git a/cmd/restic/cmd_snapshots.go b/cmd/restic/cmd_snapshots.go index 82561722c..0bfa4d110 100644 --- a/cmd/restic/cmd_snapshots.go +++ b/cmd/restic/cmd_snapshots.go @@ -26,7 +26,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - return runSnapshots(globalCtx(), snapshotOptions, globalOptions, args) + return runSnapshots(cmd.Context(), snapshotOptions, globalOptions, args) }, } diff --git a/cmd/restic/cmd_stats.go b/cmd/restic/cmd_stats.go index f03026574..d49b0ad34 100644 --- a/cmd/restic/cmd_stats.go +++ b/cmd/restic/cmd_stats.go @@ -47,7 +47,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - return runStats(globalCtx(), globalOptions, args) + return runStats(cmd.Context(), globalOptions, args) }, } diff --git a/cmd/restic/cmd_tag.go b/cmd/restic/cmd_tag.go index bf880dafb..222ddd04a 100644 --- a/cmd/restic/cmd_tag.go +++ b/cmd/restic/cmd_tag.go @@ -29,7 +29,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - return runTag(globalCtx(), tagOptions, globalOptions, args) + return runTag(cmd.Context(), tagOptions, globalOptions, args) }, } diff --git a/cmd/restic/cmd_unlock.go b/cmd/restic/cmd_unlock.go index bcdef18f6..7b449d949 100644 --- a/cmd/restic/cmd_unlock.go +++ b/cmd/restic/cmd_unlock.go @@ -20,7 +20,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - return runUnlock(globalCtx(), unlockOptions, globalOptions) + return runUnlock(cmd.Context(), unlockOptions, globalOptions) }, } diff --git a/cmd/restic/global.go b/cmd/restic/global.go index 82a4cfcca..fb14934f8 100644 --- a/cmd/restic/global.go +++ b/cmd/restic/global.go @@ -145,10 +145,6 @@ func init() { restoreTerminal() } -func globalCtx() context.Context { - return internalGlobalCtx -} - // checkErrno returns nil when err is set to syscall.Errno(0), since this is no // error condition. func checkErrno(err error) error { diff --git a/cmd/restic/main.go b/cmd/restic/main.go index ad3ef89d4..2582a3781 100644 --- a/cmd/restic/main.go +++ b/cmd/restic/main.go @@ -95,7 +95,7 @@ func main() { debug.Log("main %#v", os.Args) debug.Log("restic %s compiled with %v on %v/%v", version, runtime.Version(), runtime.GOOS, runtime.GOARCH) - err := cmdRoot.Execute() + err := cmdRoot.ExecuteContext(internalGlobalCtx) switch { case restic.IsAlreadyLocked(err):