octoleo/restic
2
2
Fork 0
mirror of https://github.com/octoleo/restic.git synced 2024-11-22 21:05:10 +00:00
Code Issues Releases Wiki Activity

Merge pull request #3569 from MichaelEischer/strict-locking

Browse Source 
Strict repository lock handling
This commit is contained in:
Michael Eischer 2022-10-03 00:44:44 +02:00 committed by GitHub
commit a61fbd287a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
35 changed files with 638 additions and 397 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
changelog/unreleased/issue-2715 Normal file
View File

@ -0,0 +1,17 @@
Enhancement: Stricter repository lock handling
Restic commands kept running even if they failed to refresh their locks in
time. This can be a problem if a concurrent call to `unlock` and `prune`
removes data from the repository. Not refreshing a lock in time can for example
be caused by a client switching to standby while running a backup.
Lock handling is now much stricter. Commands requiring a lock are canceled if
the lock is not refreshed successfully in time.
In addition, if a lock file is not readable restic will not allow starting a
command. It may be necessary to remove invalid lock file manually or using
`unlock --remove-all`. Please make sure that no other restic processes are
running concurrently.
https://github.com/restic/restic/issues/2715
https://github.com/restic/restic/pull/3569

19
cmd/restic/cmd_backup.go
View File

@ -57,8 +57,9 @@ Exit status is 3 if some source data could not be read (incomplete snapshot crea
},
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
ctx := cmd.Context()
var wg sync.WaitGroup
cancelCtx, cancel := context.WithCancel(globalOptions.ctx)
cancelCtx, cancel := context.WithCancel(ctx)
defer func() {
// shutdown termstatus
cancel()
@ -72,7 +73,7 @@ Exit status is 3 if some source data could not be read (incomplete snapshot crea
term.Run(cancelCtx)
}()
return runBackup(backupOptions, globalOptions, term, args)
return runBackup(ctx, backupOptions, globalOptions, term, args)
},
}
@ -527,7 +528,7 @@ func findParentSnapshot(ctx context.Context, repo restic.Repository, opts Backup
return parentID, nil
}
func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Terminal, args []string) error {
func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, term *termstatus.Terminal, args []string) error {
err := opts.Check(gopts, args)
if err != nil {
return err
@ -550,7 +551,7 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
Verbosef("open repository\n")
}
repo, err := OpenRepository(gopts)
repo, err := OpenRepository(ctx, gopts)
if err != nil {
return err
}
@ -577,7 +578,7 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
progressReporter.SetMinUpdatePause(calculateProgressInterval(!gopts.Quiet, gopts.JSON))
wg, wgCtx := errgroup.WithContext(gopts.ctx)
wg, wgCtx := errgroup.WithContext(ctx)
cancelCtx, cancel := context.WithCancel(wgCtx)
defer cancel()
wg.Go(func() error { return progressReporter.Run(cancelCtx) })
@ -585,7 +586,7 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
if !gopts.JSON {
progressPrinter.V("lock repository")
}
lock, err := lockRepo(gopts.ctx, repo)
lock, ctx, err := lockRepo(ctx, repo)
defer unlockRepo(lock)
if err != nil {
return err
@ -605,7 +606,7 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
var parentSnapshotID *restic.ID
if !opts.Stdin {
parentSnapshotID, err = findParentSnapshot(gopts.ctx, repo, opts, targets, timeStamp)
parentSnapshotID, err = findParentSnapshot(ctx, repo, opts, targets, timeStamp)
if err != nil {
return err
}
@ -622,7 +623,7 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
if !gopts.JSON {
progressPrinter.V("load index files")
}
err = repo.LoadIndex(gopts.ctx)
err = repo.LoadIndex(ctx)
if err != nil {
return err
}
@ -727,7 +728,7 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
if !gopts.JSON {
progressPrinter.V("start backup on %v", targets)
}
_, id, err := arch.Snapshot(gopts.ctx, targets, snapshotOpts)
_, id, err := arch.Snapshot(ctx, targets, snapshotOpts)
// cleanly shutdown all running goroutines
cancel()

29
cmd/restic/cmd_cat.go
View File

@ -1,6 +1,7 @@
package main
import (
"context"
"encoding/json"
"github.com/spf13/cobra"
@ -24,7 +25,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
return runCat(globalOptions, args)
return runCat(cmd.Context(), globalOptions, args)
},
}
@ -32,23 +33,23 @@ func init() {
cmdRoot.AddCommand(cmdCat)
}
func runCat(gopts GlobalOptions, args []string) error {
func runCat(ctx context.Context, gopts GlobalOptions, args []string) error {
if len(args) < 1 || (args[0] != "masterkey" && args[0] != "config" && len(args) != 2) {
return errors.Fatal("type or ID not specified")
}
repo, err := OpenRepository(gopts)
repo, err := OpenRepository(ctx, gopts)
if err != nil {
return err
}
if !gopts.NoLock {
lock, err := lockRepo(gopts.ctx, repo)
var lock *restic.Lock
lock, ctx, err = lockRepo(ctx, repo)
defer unlockRepo(lock)
if err != nil {
return err
}
defer unlockRepo(lock)
}
tpe := args[0]
@ -62,7 +63,7 @@ func runCat(gopts GlobalOptions, args []string) error {
}
// find snapshot id with prefix
id, err = restic.FindSnapshot(gopts.ctx, repo.Backend(), args[1])
id, err = restic.FindSnapshot(ctx, repo.Backend(), args[1])
if err != nil {
return errors.Fatalf("could not find snapshot: %v\n", err)
}
@ -79,7 +80,7 @@ func runCat(gopts GlobalOptions, args []string) error {
Println(string(buf))
return nil
case "index":
buf, err := repo.LoadUnpacked(gopts.ctx, restic.IndexFile, id, nil)
buf, err := repo.LoadUnpacked(ctx, restic.IndexFile, id, nil)
if err != nil {
return err
}
@ -87,7 +88,7 @@ func runCat(gopts GlobalOptions, args []string) error {
Println(string(buf))
return nil
case "snapshot":
sn, err := restic.LoadSnapshot(gopts.ctx, repo, id)
sn, err := restic.LoadSnapshot(ctx, repo, id)
if err != nil {
return err
}
@ -100,7 +101,7 @@ func runCat(gopts GlobalOptions, args []string) error {
Println(string(buf))
return nil
case "key":
key, err := repository.LoadKey(gopts.ctx, repo, id.String())
key, err := repository.LoadKey(ctx, repo, id.String())
if err != nil {
return err
}
@ -121,7 +122,7 @@ func runCat(gopts GlobalOptions, args []string) error {
Println(string(buf))
return nil
case "lock":
lock, err := restic.LoadLock(gopts.ctx, repo, id)
lock, err := restic.LoadLock(ctx, repo, id)
if err != nil {
return err
}
@ -136,7 +137,7 @@ func runCat(gopts GlobalOptions, args []string) error {
case "pack":
h := restic.Handle{Type: restic.PackFile, Name: id.String()}
buf, err := backend.LoadAll(gopts.ctx, nil, repo.Backend(), h)
buf, err := backend.LoadAll(ctx, nil, repo.Backend(), h)
if err != nil {
return err
}
@ -150,7 +151,7 @@ func runCat(gopts GlobalOptions, args []string) error {
return err
case "blob":
err = repo.LoadIndex(gopts.ctx)
err = repo.LoadIndex(ctx)
if err != nil {
return err
}
@ -161,7 +162,7 @@ func runCat(gopts GlobalOptions, args []string) error {
continue
}
buf, err := repo.LoadBlob(gopts.ctx, t, id, nil)
buf, err := repo.LoadBlob(ctx, t, id, nil)
if err != nil {
return err
}

22
cmd/restic/cmd_check.go
View File

@ -1,6 +1,7 @@
package main
import (
"context"
"io/ioutil"
"math/rand"
"strconv"
@ -34,7 +35,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
return runCheck(checkOptions, globalOptions, args)
return runCheck(cmd.Context(), checkOptions, globalOptions, args)
},
PreRunE: func(cmd *cobra.Command, args []string) error {
return checkFlags(checkOptions)
@ -191,7 +192,7 @@ func prepareCheckCache(opts CheckOptions, gopts *GlobalOptions) (cleanup func())
return cleanup
}
func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args []string) error {
if len(args) != 0 {
return errors.Fatal("the check command expects no arguments, only options - please see `restic help check` for usage and flags")
}
@ -202,14 +203,15 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
return code, nil
})
repo, err := OpenRepository(gopts)
repo, err := OpenRepository(ctx, gopts)
if err != nil {
return err
}
if !gopts.NoLock {
Verbosef("create exclusive lock for repository\n")
lock, err := lockRepoExclusive(gopts.ctx, repo)
var lock *restic.Lock
lock, ctx, err = lockRepoExclusive(ctx, repo)
defer unlockRepo(lock)
if err != nil {
return err
@ -217,13 +219,13 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
}
chkr := checker.New(repo, opts.CheckUnused)
err = chkr.LoadSnapshots(gopts.ctx)
err = chkr.LoadSnapshots(ctx)
if err != nil {
return err
}
Verbosef("load indexes\n")
hints, errs := chkr.LoadIndex(gopts.ctx)
hints, errs := chkr.LoadIndex(ctx)
errorsFound := false
suggestIndexRebuild := false
@ -260,7 +262,7 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
errChan := make(chan error)
Verbosef("check all packs\n")
go chkr.Packs(gopts.ctx, errChan)
go chkr.Packs(ctx, errChan)
for err := range errChan {
if checker.IsOrphanedPack(err) {
@ -287,7 +289,7 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
defer wg.Done()
bar := newProgressMax(!gopts.Quiet, 0, "snapshots")
defer bar.Done()
chkr.Structure(gopts.ctx, bar, errChan)
chkr.Structure(ctx, bar, errChan)
}()
for err := range errChan {
@ -308,7 +310,7 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
wg.Wait()
if opts.CheckUnused {
for _, id := range chkr.UnusedBlobs(gopts.ctx) {
for _, id := range chkr.UnusedBlobs(ctx) {
Verbosef("unused blob %v\n", id)
errorsFound = true
}
@ -320,7 +322,7 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
p := newProgressMax(!gopts.Quiet, packCount, "packs")
errChan := make(chan error)
go chkr.ReadPacks(gopts.ctx, packs, p, errChan)
go chkr.ReadPacks(ctx, packs, p, errChan)
for err := range errChan {
errorsFound = true

16
cmd/restic/cmd_copy.go
View File

@ -32,7 +32,7 @@ This can be mitigated by the "--copy-chunker-params" option when initializing a
new destination repository using the "init" command.
`,
RunE: func(cmd *cobra.Command, args []string) error {
return runCopy(copyOptions, globalOptions, args)
return runCopy(cmd.Context(), copyOptions, globalOptions, args)
},
}
@ -52,7 +52,7 @@ func init() {
initMultiSnapshotFilterOptions(f, &copyOptions.snapshotFilterOptions, true)
}
func runCopy(opts CopyOptions, gopts GlobalOptions, args []string) error {
func runCopy(ctx context.Context, opts CopyOptions, gopts GlobalOptions, args []string) error {
secondaryGopts, isFromRepo, err := fillSecondaryGlobalOpts(opts.secondaryRepoOptions, gopts, "destination")
if err != nil {
return err
@ -62,28 +62,26 @@ func runCopy(opts CopyOptions, gopts GlobalOptions, args []string) error {
gopts, secondaryGopts = secondaryGopts, gopts
}
ctx, cancel := context.WithCancel(gopts.ctx)
defer cancel()
srcRepo, err := OpenRepository(gopts)
srcRepo, err := OpenRepository(ctx, gopts)
if err != nil {
return err
}
dstRepo, err := OpenRepository(secondaryGopts)
dstRepo, err := OpenRepository(ctx, secondaryGopts)
if err != nil {
return err
}
if !gopts.NoLock {
srcLock, err := lockRepo(ctx, srcRepo)
var srcLock *restic.Lock
srcLock, ctx, err = lockRepo(ctx, srcRepo)
defer unlockRepo(srcLock)
if err != nil {
return err
}
}
dstLock, err := lockRepo(ctx, dstRepo)
dstLock, ctx, err := lockRepo(ctx, dstRepo)
defer unlockRepo(dstLock)
if err != nil {
return err

34
cmd/restic/cmd_debug.go
View File

@ -46,7 +46,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
return runDebugDump(globalOptions, args)
return runDebugDump(cmd.Context(), globalOptions, args)
},
}
@ -141,18 +141,19 @@ func dumpIndexes(ctx context.Context, repo restic.Repository, wr io.Writer) erro
})
}
func runDebugDump(gopts GlobalOptions, args []string) error {
func runDebugDump(ctx context.Context, gopts GlobalOptions, args []string) error {
if len(args) != 1 {
return errors.Fatal("type not specified")
}
repo, err := OpenRepository(gopts)
repo, err := OpenRepository(ctx, gopts)
if err != nil {
return err
}
if !gopts.NoLock {
lock, err := lockRepo(gopts.ctx, repo)
var lock *restic.Lock
lock, ctx, err = lockRepo(ctx, repo)
defer unlockRepo(lock)
if err != nil {
return err
@ -163,20 +164,20 @@ func runDebugDump(gopts GlobalOptions, args []string) error {
switch tpe {
case "indexes":
return dumpIndexes(gopts.ctx, repo, gopts.stdout)
return dumpIndexes(ctx, repo, gopts.stdout)
case "snapshots":
return debugPrintSnapshots(gopts.ctx, repo, gopts.stdout)
return debugPrintSnapshots(ctx, repo, gopts.stdout)
case "packs":
return printPacks(gopts.ctx, repo, gopts.stdout)
return printPacks(ctx, repo, gopts.stdout)
case "all":
Printf("snapshots:\n")
err := debugPrintSnapshots(gopts.ctx, repo, gopts.stdout)
err := debugPrintSnapshots(ctx, repo, gopts.stdout)
if err != nil {
return err
}
Printf("\nindexes:\n")
err = dumpIndexes(gopts.ctx, repo, gopts.stdout)
err = dumpIndexes(ctx, repo, gopts.stdout)
if err != nil {
return err
}
@ -192,7 +193,7 @@ var cmdDebugExamine = &cobra.Command{
Short: "Examine a pack file",
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
return runDebugExamine(globalOptions, args)
return runDebugExamine(cmd.Context(), globalOptions, args)
},
}
@ -426,8 +427,8 @@ func storePlainBlob(id restic.ID, prefix string, plain []byte) error {
return nil
}
func runDebugExamine(gopts GlobalOptions, args []string) error {
repo, err := OpenRepository(gopts)
func runDebugExamine(ctx context.Context, gopts GlobalOptions, args []string) error {
repo, err := OpenRepository(ctx, gopts)
if err != nil {
return err
}
@ -436,7 +437,7 @@ func runDebugExamine(gopts GlobalOptions, args []string) error {
for _, name := range args {
id, err := restic.ParseID(name)
if err != nil {
name, err = restic.Find(gopts.ctx, repo.Backend(), restic.PackFile, name)
name, err = restic.Find(ctx, repo.Backend(), restic.PackFile, name)
if err == nil {
id, err = restic.ParseID(name)
}
@ -453,20 +454,21 @@ func runDebugExamine(gopts GlobalOptions, args []string) error {
}
if !gopts.NoLock {
lock, err := lockRepo(gopts.ctx, repo)
var lock *restic.Lock
lock, ctx, err = lockRepo(ctx, repo)
defer unlockRepo(lock)
if err != nil {
return err
}
}
err = repo.LoadIndex(gopts.ctx)
err = repo.LoadIndex(ctx)
if err != nil {
return err
}
for _, id := range ids {
err := examinePack(gopts.ctx, repo, id)
err := examinePack(ctx, repo, id)
if err != nil {
Warnf("error: %v\n", err)
}

12
cmd/restic/cmd_diff.go
View File

@ -35,7 +35,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
return runDiff(diffOptions, globalOptions, args)
return runDiff(cmd.Context(), diffOptions, globalOptions, args)
},
}
@ -321,21 +321,19 @@ func (c *Comparer) diffTree(ctx context.Context, stats *DiffStatsContainer, pref
return nil
}
func runDiff(opts DiffOptions, gopts GlobalOptions, args []string) error {
func runDiff(ctx context.Context, opts DiffOptions, gopts GlobalOptions, args []string) error {
if len(args) != 2 {
return errors.Fatalf("specify two snapshot IDs")
}
ctx, cancel := context.WithCancel(gopts.ctx)
defer cancel()
repo, err := OpenRepository(gopts)
repo, err := OpenRepository(ctx, gopts)
if err != nil {
return err
}
if !gopts.NoLock {
lock, err := lockRepo(ctx, repo)
var lock *restic.Lock
lock, ctx, err = lockRepo(ctx, repo)
defer unlockRepo(lock)
if err != nil {
return err

13
cmd/restic/cmd_dump.go
View File

@ -34,7 +34,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
return runDump(dumpOptions, globalOptions, args)
return runDump(cmd.Context(), dumpOptions, globalOptions, args)
},
}
@ -107,9 +107,7 @@ func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.Repositor
return fmt.Errorf("path %q not found in snapshot", item)
}
func runDump(opts DumpOptions, gopts GlobalOptions, args []string) error {
ctx := gopts.ctx
func runDump(ctx context.Context, opts DumpOptions, gopts GlobalOptions, args []string) error {
if len(args) != 2 {
return errors.Fatal("no file and no snapshot ID specified")
}
@ -127,13 +125,14 @@ func runDump(opts DumpOptions, gopts GlobalOptions, args []string) error {
splittedPath := splitPath(path.Clean(pathToPrint))
repo, err := OpenRepository(gopts)
repo, err := OpenRepository(ctx, gopts)
if err != nil {
return err
}
if !gopts.NoLock {
lock, err := lockRepo(ctx, repo)
var lock *restic.Lock
lock, ctx, err = lockRepo(ctx, repo)
defer unlockRepo(lock)
if err != nil {
return err
@ -154,7 +153,7 @@ func runDump(opts DumpOptions, gopts GlobalOptions, args []string) error {
}
}
sn, err := restic.LoadSnapshot(gopts.ctx, repo, id)
sn, err := restic.LoadSnapshot(ctx, repo, id)
if err != nil {
Exitf(2, "loading snapshot %q failed: %v", snapshotIDString, err)
}

16
cmd/restic/cmd_find.go
View File

@ -38,7 +38,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
return runFind(findOptions, globalOptions, args)
return runFind(cmd.Context(), findOptions, globalOptions, args)
},
}
@ -534,7 +534,7 @@ func (f *Finder) findObjectsPacks(ctx context.Context) {
}
}
func runFind(opts FindOptions, gopts GlobalOptions, args []string) error {
func runFind(ctx context.Context, opts FindOptions, gopts GlobalOptions, args []string) error {
if len(args) == 0 {
return errors.Fatal("wrong number of arguments")
}
@ -568,31 +568,29 @@ func runFind(opts FindOptions, gopts GlobalOptions, args []string) error {
return errors.Fatal("cannot have several ID types")
}
repo, err := OpenRepository(gopts)
repo, err := OpenRepository(ctx, gopts)
if err != nil {
return err
}
if !gopts.NoLock {
lock, err := lockRepo(gopts.ctx, repo)
var lock *restic.Lock
lock, ctx, err = lockRepo(ctx, repo)
defer unlockRepo(lock)
if err != nil {
return err
}
}
snapshotLister, err := backend.MemorizeList(gopts.ctx, repo.Backend(), restic.SnapshotFile)
snapshotLister, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile)
if err != nil {
return err
}
if err = repo.LoadIndex(gopts.ctx); err != nil {
if err = repo.LoadIndex(ctx); err != nil {
return err
}
ctx, cancel := context.WithCancel(gopts.ctx)
defer cancel()
f := &Finder{
repo: repo,
pat: pat,

16
cmd/restic/cmd_forget.go
View File

@ -32,7 +32,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
return runForget(forgetOptions, globalOptions, args)
return runForget(cmd.Context(), forgetOptions, globalOptions, args)
},
}
@ -99,13 +99,13 @@ func init() {
addPruneOptions(cmdForget)
}
func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error {
func runForget(ctx context.Context, opts ForgetOptions, gopts GlobalOptions, args []string) error {
err := verifyPruneOptions(&pruneOptions)
if err != nil {
return err
}
repo, err := OpenRepository(gopts)
repo, err := OpenRepository(ctx, gopts)
if err != nil {
return err
}
@ -115,16 +115,14 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error {
}
if !opts.DryRun || !gopts.NoLock {
lock, err := lockRepoExclusive(gopts.ctx, repo)
var lock *restic.Lock
lock, ctx, err = lockRepoExclusive(ctx, repo)
defer unlockRepo(lock)
if err != nil {
return err
}
}
ctx, cancel := context.WithCancel(gopts.ctx)
defer cancel()
var snapshots restic.Snapshots
removeSnIDs := restic.NewIDSet()
@ -219,7 +217,7 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error {
if len(removeSnIDs) > 0 {
if !opts.DryRun {
err := DeleteFilesChecked(gopts, repo, removeSnIDs, restic.SnapshotFile)
err := DeleteFilesChecked(ctx, gopts, repo, removeSnIDs, restic.SnapshotFile)
if err != nil {
return err
}
@ -242,7 +240,7 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error {
Verbosef("%d snapshots have been removed, running prune\n", len(removeSnIDs))
}
pruneOptions.DryRun = opts.DryRun
return runPruneWithRepo(pruneOptions, gopts, repo, removeSnIDs)
return runPruneWithRepo(ctx, pruneOptions, gopts, repo, removeSnIDs)
}
return nil

15
cmd/restic/cmd_init.go
View File

@ -1,6 +1,7 @@
package main
import (
"context"
"strconv"
"github.com/restic/chunker"
@ -25,7 +26,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
return runInit(initOptions, globalOptions, args)
return runInit(cmd.Context(), initOptions, globalOptions, args)
},
}
@ -47,7 +48,7 @@ func init() {
f.StringVar(&initOptions.RepositoryVersion, "repository-version", "stable", "repository format version to use, allowed values are a format version, 'latest' and 'stable'")
}
func runInit(opts InitOptions, gopts GlobalOptions, args []string) error {
func runInit(ctx context.Context, opts InitOptions, gopts GlobalOptions, args []string) error {
var version uint
if opts.RepositoryVersion == "latest" || opts.RepositoryVersion == "" {
version = restic.MaxRepoVersion
@ -64,7 +65,7 @@ func runInit(opts InitOptions, gopts GlobalOptions, args []string) error {
return errors.Fatalf("only repository versions between %v and %v are allowed", restic.MinRepoVersion, restic.MaxRepoVersion)
}
chunkerPolynomial, err := maybeReadChunkerPolynomial(opts, gopts)
chunkerPolynomial, err := maybeReadChunkerPolynomial(ctx, opts, gopts)
if err != nil {
return err
}
@ -81,7 +82,7 @@ func runInit(opts InitOptions, gopts GlobalOptions, args []string) error {
return err
}
be, err := create(repo, gopts.extended)
be, err := create(ctx, repo, gopts.extended)
if err != nil {
return errors.Fatalf("create repository at %s failed: %v\n", location.StripPassword(gopts.Repo), err)
}
@ -94,7 +95,7 @@ func runInit(opts InitOptions, gopts GlobalOptions, args []string) error {
return err
}
err = s.Init(gopts.ctx, version, gopts.password, chunkerPolynomial)
err = s.Init(ctx, version, gopts.password, chunkerPolynomial)
if err != nil {
return errors.Fatalf("create key in repository at %s failed: %v\n", location.StripPassword(gopts.Repo), err)
}
@ -108,14 +109,14 @@ func runInit(opts InitOptions, gopts GlobalOptions, args []string) error {
return nil
}
func maybeReadChunkerPolynomial(opts InitOptions, gopts GlobalOptions) (*chunker.Pol, error) {
func maybeReadChunkerPolynomial(ctx context.Context, opts InitOptions, gopts GlobalOptions) (*chunker.Pol, error) {
if opts.CopyChunkerParameters {
otherGopts, _, err := fillSecondaryGlobalOpts(opts.secondaryRepoOptions, gopts, "secondary")
if err != nil {
return nil, err
}
otherRepo, err := OpenRepository(otherGopts)
otherRepo, err := OpenRepository(ctx, otherGopts)
if err != nil {
return nil, err
}

37
cmd/restic/cmd_key.go
View File

@ -28,7 +28,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
return runKey(globalOptions, args)
return runKey(cmd.Context(), globalOptions, args)
},
}
@ -120,18 +120,18 @@ func getNewPassword(gopts GlobalOptions) (string, error) {
"enter password again: ")
}
func addKey(gopts GlobalOptions, repo *repository.Repository) error {
func addKey(ctx context.Context, repo *repository.Repository, gopts GlobalOptions) error {
pw, err := getNewPassword(gopts)
if err != nil {
return err
}
id, err := repository.AddKey(gopts.ctx, repo, pw, keyUsername, keyHostname, repo.Key())
id, err := repository.AddKey(ctx, repo, pw, keyUsername, keyHostname, repo.Key())
if err != nil {
return errors.Fatalf("creating new key failed: %v\n", err)
}
err = switchToNewKeyAndRemoveIfBroken(gopts.ctx, repo, id, pw)
err = switchToNewKeyAndRemoveIfBroken(ctx, repo, id, pw)
if err != nil {
return err
}
@ -156,25 +156,25 @@ func deleteKey(ctx context.Context, repo *repository.Repository, name string) er
return nil
}
func changePassword(gopts GlobalOptions, repo *repository.Repository) error {
func changePassword(ctx context.Context, repo *repository.Repository, gopts GlobalOptions) error {
pw, err := getNewPassword(gopts)
if err != nil {
return err
}
id, err := repository.AddKey(gopts.ctx, repo, pw, "", "", repo.Key())
id, err := repository.AddKey(ctx, repo, pw, "", "", repo.Key())
if err != nil {
return errors.Fatalf("creating new key failed: %v\n", err)
}
oldID := repo.KeyName()
err = switchToNewKeyAndRemoveIfBroken(gopts.ctx, repo, id, pw)
err = switchToNewKeyAndRemoveIfBroken(ctx, repo, id, pw)
if err != nil {
return err
}
h := restic.Handle{Type: restic.KeyFile, Name: oldID}
err = repo.Backend().Remove(gopts.ctx, h)
err = repo.Backend().Remove(ctx, h)
if err != nil {
return err
}
@ -197,22 +197,19 @@ func switchToNewKeyAndRemoveIfBroken(ctx context.Context, repo *repository.Repos
return nil
}
func runKey(gopts GlobalOptions, args []string) error {
func runKey(ctx context.Context, gopts GlobalOptions, args []string) error {
if len(args) < 1 || (args[0] == "remove" && len(args) != 2) || (args[0] != "remove" && len(args) != 1) {
return errors.Fatal("wrong number of arguments")
}
ctx, cancel := context.WithCancel(gopts.ctx)
defer cancel()
repo, err := OpenRepository(gopts)
repo, err := OpenRepository(ctx, gopts)
if err != nil {
return err
}
switch args[0] {
case "list":
lock, err := lockRepo(ctx, repo)
lock, ctx, err := lockRepo(ctx, repo)
defer unlockRepo(lock)
if err != nil {
return err
@ -220,15 +217,15 @@ func runKey(gopts GlobalOptions, args []string) error {
return listKeys(ctx, repo, gopts)
case "add":
lock, err := lockRepo(ctx, repo)
lock, ctx, err := lockRepo(ctx, repo)
defer unlockRepo(lock)
if err != nil {
return err
}
return addKey(gopts, repo)
return addKey(ctx, repo, gopts)
case "remove":
lock, err := lockRepoExclusive(ctx, repo)
lock, ctx, err := lockRepoExclusive(ctx, repo)
defer unlockRepo(lock)
if err != nil {
return err
@ -239,15 +236,15 @@ func runKey(gopts GlobalOptions, args []string) error {
return err
}
return deleteKey(gopts.ctx, repo, id)
return deleteKey(ctx, repo, id)
case "passwd":
lock, err := lockRepoExclusive(ctx, repo)
lock, ctx, err := lockRepoExclusive(ctx, repo)
defer unlockRepo(lock)
if err != nil {
return err
}
return changePassword(gopts, repo)
return changePassword(ctx, repo, gopts)
}
return nil

17
cmd/restic/cmd_list.go
View File

@ -1,6 +1,8 @@
package main
import (
"context"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
@ -21,7 +23,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
return runList(cmd, globalOptions, args)
return runList(cmd.Context(), cmd, globalOptions, args)
},
}
@ -29,18 +31,19 @@ func init() {
cmdRoot.AddCommand(cmdList)
}
func runList(cmd *cobra.Command, opts GlobalOptions, args []string) error {
func runList(ctx context.Context, cmd *cobra.Command, opts GlobalOptions, args []string) error {
if len(args) != 1 {
return errors.Fatal("type not specified, usage: " + cmd.Use)
}
repo, err := OpenRepository(opts)
repo, err := OpenRepository(ctx, opts)
if err != nil {
return err
}
if !opts.NoLock && args[0] != "locks" {
lock, err := lockRepo(opts.ctx, repo)
var lock *restic.Lock
lock, ctx, err = lockRepo(ctx, repo)
defer unlockRepo(lock)
if err != nil {
return err
@ -60,11 +63,11 @@ func runList(cmd *cobra.Command, opts GlobalOptions, args []string) error {
case "locks":
t = restic.LockFile
case "blobs":
return repository.ForAllIndexes(opts.ctx, repo, func(id restic.ID, idx *repository.Index, oldFormat bool, err error) error {
return repository.ForAllIndexes(ctx, repo, func(id restic.ID, idx *repository.Index, oldFormat bool, err error) error {
if err != nil {
return err
}
idx.Each(opts.ctx, func(blobs restic.PackedBlob) {
idx.Each(ctx, func(blobs restic.PackedBlob) {
Printf("%v %v\n", blobs.Type, blobs.ID)
})
return nil
@ -73,7 +76,7 @@ func runList(cmd *cobra.Command, opts GlobalOptions, args []string) error {
return errors.Fatal("invalid type")
}
return repo.List(opts.ctx, t, func(id restic.ID, size int64) error {
return repo.List(ctx, t, func(id restic.ID, size int64) error {
Printf("%s\n", id)
return nil
})

13
cmd/restic/cmd_ls.go
View File

@ -42,7 +42,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
return runLs(lsOptions, globalOptions, args)
return runLs(cmd.Context(), lsOptions, globalOptions, args)
},
}
@ -111,7 +111,7 @@ func lsNodeJSON(enc *json.Encoder, path string, node *restic.Node) error {
return enc.Encode(n)
}
func runLs(opts LsOptions, gopts GlobalOptions, args []string) error {
func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []string) error {
if len(args) == 0 {
return errors.Fatal("no snapshot ID specified, specify snapshot ID or use special ID 'latest'")
}
@ -161,23 +161,20 @@ func runLs(opts LsOptions, gopts GlobalOptions, args []string) error {
return false
}
repo, err := OpenRepository(gopts)
repo, err := OpenRepository(ctx, gopts)
if err != nil {
return err
}
snapshotLister, err := backend.MemorizeList(gopts.ctx, repo.Backend(), restic.SnapshotFile)
snapshotLister, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile)
if err != nil {
return err
}
if err = repo.LoadIndex(gopts.ctx); err != nil {
if err = repo.LoadIndex(ctx); err != nil {
return err
}
ctx, cancel := context.WithCancel(gopts.ctx)
defer cancel()
var (
printSnapshot func(sn *restic.Snapshot)
printNode func(path string, node *restic.Node)

23
cmd/restic/cmd_migrate.go
View File

@ -1,6 +1,8 @@
package main
import (
"context"
"github.com/restic/restic/internal/migrations"
"github.com/restic/restic/internal/restic"
@ -22,7 +24,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
return runMigrate(migrateOptions, globalOptions, args)
return runMigrate(cmd.Context(), migrateOptions, globalOptions, args)
},
}
@ -39,8 +41,7 @@ func init() {
f.BoolVarP(&migrateOptions.Force, "force", "f", false, `apply a migration a second time`)
}
func checkMigrations(opts MigrateOptions, gopts GlobalOptions, repo restic.Repository) error {
ctx := gopts.ctx
func checkMigrations(ctx context.Context, repo restic.Repository) error {
Printf("available migrations:\n")
found := false
@ -63,9 +64,7 @@ func checkMigrations(opts MigrateOptions, gopts GlobalOptions, repo restic.Repos
return nil
}
func applyMigrations(opts MigrateOptions, gopts GlobalOptions, repo restic.Repository, args []string) error {
ctx := gopts.ctx
func applyMigrations(ctx context.Context, opts MigrateOptions, gopts GlobalOptions, repo restic.Repository, args []string) error {
var firsterr error
for _, name := range args {
for _, m := range migrations.All {
@ -94,7 +93,7 @@ func applyMigrations(opts MigrateOptions, gopts GlobalOptions, repo restic.Repos
checkGopts := gopts
// the repository is already locked
checkGopts.NoLock = true
err = runCheck(checkOptions, checkGopts, []string{})
err = runCheck(ctx, checkOptions, checkGopts, []string{})
if err != nil {
return err
}
@ -117,21 +116,21 @@ func applyMigrations(opts MigrateOptions, gopts GlobalOptions, repo restic.Repos
return firsterr
}
func runMigrate(opts MigrateOptions, gopts GlobalOptions, args []string) error {
repo, err := OpenRepository(gopts)
func runMigrate(ctx context.Context, opts MigrateOptions, gopts GlobalOptions, args []string) error {
repo, err := OpenRepository(ctx, gopts)
if err != nil {
return err
}
lock, err := lockRepoExclusive(gopts.ctx, repo)
lock, ctx, err := lockRepoExclusive(ctx, repo)
defer unlockRepo(lock)
if err != nil {
return err
}
if len(args) == 0 {
return checkMigrations(opts, gopts, repo)
return checkMigrations(ctx, repo)
}
return applyMigrations(opts, gopts, repo, args)
return applyMigrations(ctx, opts, gopts, repo, args)
}

13
cmd/restic/cmd_mount.go
View File

@ -4,6 +4,7 @@
package main
import (
"context"
"os"
"strings"
"time"
@ -12,6 +13,7 @@ import (
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
resticfs "github.com/restic/restic/internal/fs"
"github.com/restic/restic/internal/fuse"
@ -66,7 +68,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
return runMount(mountOptions, globalOptions, args)
return runMount(cmd.Context(), mountOptions, globalOptions, args)
},
}
@ -98,7 +100,7 @@ func init() {
_ = mountFlags.MarkDeprecated("snapshot-template", "use --time-template")
}
func runMount(opts MountOptions, gopts GlobalOptions, args []string) error {
func runMount(ctx context.Context, opts MountOptions, gopts GlobalOptions, args []string) error {
if opts.TimeTemplate == "" {
return errors.Fatal("time template string cannot be empty")
}
@ -114,20 +116,21 @@ func runMount(opts MountOptions, gopts GlobalOptions, args []string) error {
debug.Log("start mount")
defer debug.Log("finish mount")
repo, err := OpenRepository(gopts)
repo, err := OpenRepository(ctx, gopts)
if err != nil {
return err
}
if !gopts.NoLock {
lock, err := lockRepo(gopts.ctx, repo)
var lock *restic.Lock
lock, ctx, err = lockRepo(ctx, repo)
defer unlockRepo(lock)
if err != nil {
return err
}
}
err = repo.LoadIndex(gopts.ctx)
err = repo.LoadIndex(ctx)
if err != nil {
return err
}

53
cmd/restic/cmd_prune.go
View File

@ -34,7 +34,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
return runPrune(pruneOptions, globalOptions)
return runPrune(cmd.Context(), pruneOptions, globalOptions)
},
}
@ -134,7 +134,7 @@ func verifyPruneOptions(opts *PruneOptions) error {
return nil
}
func runPrune(opts PruneOptions, gopts GlobalOptions) error {
func runPrune(ctx context.Context, opts PruneOptions, gopts GlobalOptions) error {
err := verifyPruneOptions(&opts)
if err != nil {
return err
@ -144,7 +144,7 @@ func runPrune(opts PruneOptions, gopts GlobalOptions) error {
return errors.Fatal("disabled compression and `--repack-uncompressed` are mutually exclusive")
}
repo, err := OpenRepository(gopts)
repo, err := OpenRepository(ctx, gopts)
if err != nil {
return err
}
@ -165,16 +165,16 @@ func runPrune(opts PruneOptions, gopts GlobalOptions) error {
opts.unsafeRecovery = true
}
lock, err := lockRepoExclusive(gopts.ctx, repo)
lock, ctx, err := lockRepoExclusive(ctx, repo)
defer unlockRepo(lock)
if err != nil {
return err
}
return runPruneWithRepo(opts, gopts, repo, restic.NewIDSet())
return runPruneWithRepo(ctx, opts, gopts, repo, restic.NewIDSet())
}
func runPruneWithRepo(opts PruneOptions, gopts GlobalOptions, repo *repository.Repository, ignoreSnapshots restic.IDSet) error {
func runPruneWithRepo(ctx context.Context, opts PruneOptions, gopts GlobalOptions, repo *repository.Repository, ignoreSnapshots restic.IDSet) error {
// we do not need index updates while pruning!
repo.DisableAutoIndexUpdate()
@ -184,12 +184,12 @@ func runPruneWithRepo(opts PruneOptions, gopts GlobalOptions, repo *repository.R
Verbosef("loading indexes...\n")
// loading the index before the snapshots is ok, as we use an exclusive lock here
err := repo.LoadIndex(gopts.ctx)
err := repo.LoadIndex(ctx)
if err != nil {
return err
}
plan, stats, err := planPrune(opts, gopts, repo, ignoreSnapshots)
plan, stats, err := planPrune(ctx, opts, gopts, repo, ignoreSnapshots)
if err != nil {
return err
}
@ -199,7 +199,7 @@ func runPruneWithRepo(opts PruneOptions, gopts GlobalOptions, repo *repository.R
return err
}
return doPrune(opts, gopts, repo, plan)
return doPrune(ctx, opts, gopts, repo, plan)
}
type pruneStats struct {
@ -255,11 +255,10 @@ type packInfoWithID struct {
// planPrune selects which files to rewrite and which to delete and which blobs to keep.
// Also some summary statistics are returned.
func planPrune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, ignoreSnapshots restic.IDSet) (prunePlan, pruneStats, error) {
ctx := gopts.ctx
func planPrune(ctx context.Context, opts PruneOptions, gopts GlobalOptions, repo restic.Repository, ignoreSnapshots restic.IDSet) (prunePlan, pruneStats, error) {
var stats pruneStats
usedBlobs, err := getUsedBlobs(gopts, repo, ignoreSnapshots)
usedBlobs, err := getUsedBlobs(ctx, gopts, repo, ignoreSnapshots)
if err != nil {
return prunePlan{}, stats, err
}
@ -652,9 +651,7 @@ func printPruneStats(gopts GlobalOptions, stats pruneStats) error {
// - rebuild the index while ignoring all files that will be deleted
// - delete the files
// plan.removePacks and plan.ignorePacks are modified in this function.
func doPrune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, plan prunePlan) (err error) {
ctx := gopts.ctx
func doPrune(ctx context.Context, opts PruneOptions, gopts GlobalOptions, repo restic.Repository, plan prunePlan) (err error) {
if opts.DryRun {
if !gopts.JSON && gopts.verbosity >= 2 {
if len(plan.removePacksFirst) > 0 {
@ -670,7 +667,7 @@ func doPrune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, pla
// unreferenced packs can be safely deleted first
if len(plan.removePacksFirst) != 0 {
Verbosef("deleting unreferenced packs\n")
DeleteFiles(gopts, repo, plan.removePacksFirst, restic.PackFile)
DeleteFiles(ctx, gopts, repo, plan.removePacksFirst, restic.PackFile)
}
if len(plan.repackPacks) != 0 {
@ -703,12 +700,12 @@ func doPrune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, pla
if opts.unsafeRecovery {
Verbosef("deleting index files\n")
indexFiles := repo.Index().(*repository.MasterIndex).IDs()
err = DeleteFilesChecked(gopts, repo, indexFiles, restic.IndexFile)
err = DeleteFilesChecked(ctx, gopts, repo, indexFiles, restic.IndexFile)
if err != nil {
return errors.Fatalf("%s", err)
}
} else if len(plan.ignorePacks) != 0 {
err = rebuildIndexFiles(gopts, repo, plan.ignorePacks, nil)
err = rebuildIndexFiles(ctx, gopts, repo, plan.ignorePacks, nil)
if err != nil {
return errors.Fatalf("%s", err)
}
@ -716,11 +713,11 @@ func doPrune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, pla
if len(plan.removePacks) != 0 {
Verbosef("removing %d old packs\n", len(plan.removePacks))
DeleteFiles(gopts, repo, plan.removePacks, restic.PackFile)
DeleteFiles(ctx, gopts, repo, plan.removePacks, restic.PackFile)
}
if opts.unsafeRecovery {
_, err = writeIndexFiles(gopts, repo, plan.ignorePacks, nil)
_, err = writeIndexFiles(ctx, gopts, repo, plan.ignorePacks, nil)
if err != nil {
return errors.Fatalf("%s", err)
}
@ -730,31 +727,29 @@ func doPrune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, pla
return nil
}
func writeIndexFiles(gopts GlobalOptions, repo restic.Repository, removePacks restic.IDSet, extraObsolete restic.IDs) (restic.IDSet, error) {
func writeIndexFiles(ctx context.Context, gopts GlobalOptions, repo restic.Repository, removePacks restic.IDSet, extraObsolete restic.IDs) (restic.IDSet, error) {
Verbosef("rebuilding index\n")
bar := newProgressMax(!gopts.Quiet, 0, "packs processed")
obsoleteIndexes, err := repo.Index().Save(gopts.ctx, repo, removePacks, extraObsolete, bar)
obsoleteIndexes, err := repo.Index().Save(ctx, repo, removePacks, extraObsolete, bar)
bar.Done()
return obsoleteIndexes, err
}
func rebuildIndexFiles(gopts GlobalOptions, repo restic.Repository, removePacks restic.IDSet, extraObsolete restic.IDs) error {
obsoleteIndexes, err := writeIndexFiles(gopts, repo, removePacks, extraObsolete)
func rebuildIndexFiles(ctx context.Context, gopts GlobalOptions, repo restic.Repository, removePacks restic.IDSet, extraObsolete restic.IDs) error {
obsoleteIndexes, err := writeIndexFiles(ctx, gopts, repo, removePacks, extraObsolete)
if err != nil {
return err
}
Verbosef("deleting obsolete index files\n")
return DeleteFilesChecked(gopts, repo, obsoleteIndexes, restic.IndexFile)
return DeleteFilesChecked(ctx, gopts, repo, obsoleteIndexes, restic.IndexFile)
}
func getUsedBlobs(gopts GlobalOptions, repo restic.Repository, ignoreSnapshots restic.IDSet) (usedBlobs restic.BlobSet, err error) {
ctx := gopts.ctx
func getUsedBlobs(ctx context.Context, gopts GlobalOptions, repo restic.Repository, ignoreSnapshots restic.IDSet) (usedBlobs restic.BlobSet, err error) {
var snapshotTrees restic.IDs
Verbosef("loading all snapshots...\n")
err = restic.ForAllSnapshots(gopts.ctx, repo.Backend(), repo, ignoreSnapshots,
err = restic.ForAllSnapshots(ctx, repo.Backend(), repo, ignoreSnapshots,
func(id restic.ID, sn *restic.Snapshot, err error) error {
if err != nil {
debug.Log("failed to load snapshot %v (error %v)", id, err)

18
cmd/restic/cmd_rebuild_index.go
View File

@ -1,6 +1,8 @@
package main
import (
"context"
"github.com/restic/restic/internal/pack"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
@ -22,7 +24,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
return runRebuildIndex(rebuildIndexOptions, globalOptions)
return runRebuildIndex(cmd.Context(), rebuildIndexOptions, globalOptions)
},
}
@ -40,24 +42,22 @@ func init() {
}
func runRebuildIndex(opts RebuildIndexOptions, gopts GlobalOptions) error {
repo, err := OpenRepository(gopts)
func runRebuildIndex(ctx context.Context, opts RebuildIndexOptions, gopts GlobalOptions) error {
repo, err := OpenRepository(ctx, gopts)
if err != nil {
return err
}
lock, err := lockRepoExclusive(gopts.ctx, repo)
lock, ctx, err := lockRepoExclusive(ctx, repo)
defer unlockRepo(lock)
if err != nil {
return err
}
return rebuildIndex(opts, gopts, repo, restic.NewIDSet())
return rebuildIndex(ctx, opts, gopts, repo, restic.NewIDSet())
}
func rebuildIndex(opts RebuildIndexOptions, gopts GlobalOptions, repo *repository.Repository, ignorePacks restic.IDSet) error {
ctx := gopts.ctx
func rebuildIndex(ctx context.Context, opts RebuildIndexOptions, gopts GlobalOptions, repo *repository.Repository, ignorePacks restic.IDSet) error {
var obsoleteIndexes restic.IDs
packSizeFromList := make(map[restic.ID]int64)
packSizeFromIndex := make(map[restic.ID]int64)
@ -141,7 +141,7 @@ func rebuildIndex(opts RebuildIndexOptions, gopts GlobalOptions, repo *repositor
}
}
err = rebuildIndexFiles(gopts, repo, removePacks, obsoleteIndexes)
err = rebuildIndexFiles(ctx, gopts, repo, removePacks, obsoleteIndexes)
if err != nil {
return err
}

28
cmd/restic/cmd_recover.go
View File

@ -27,7 +27,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
return runRecover(globalOptions)
return runRecover(cmd.Context(), globalOptions)
},
}
@ -35,30 +35,30 @@ func init() {
cmdRoot.AddCommand(cmdRecover)
}
func runRecover(gopts GlobalOptions) error {
func runRecover(ctx context.Context, gopts GlobalOptions) error {
hostname, err := os.Hostname()
if err != nil {
return err
}
repo, err := OpenRepository(gopts)
repo, err := OpenRepository(ctx, gopts)
if err != nil {
return err
}
lock, err := lockRepo(gopts.ctx, repo)
lock, ctx, err := lockRepo(ctx, repo)
defer unlockRepo(lock)
if err != nil {
return err
}
snapshotLister, err := backend.MemorizeList(gopts.ctx, repo.Backend(), restic.SnapshotFile)
snapshotLister, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile)
if err != nil {
return err
}
Verbosef("load index files\n")
if err = repo.LoadIndex(gopts.ctx); err != nil {
if err = repo.LoadIndex(ctx); err != nil {
return err
}
@ -66,7 +66,7 @@ func runRecover(gopts GlobalOptions) error {
// tree. If it is not referenced, we have a root tree.
trees := make(map[restic.ID]bool)
repo.Index().Each(gopts.ctx, func(blob restic.PackedBlob) {
repo.Index().Each(ctx, func(blob restic.PackedBlob) {
if blob.Type == restic.TreeBlob {
trees[blob.Blob.ID] = false
}
@ -75,7 +75,7 @@ func runRecover(gopts GlobalOptions) error {
Verbosef("load %d trees\n", len(trees))
bar := newProgressMax(!gopts.Quiet, uint64(len(trees)), "trees loaded")
for id := range trees {
tree, err := restic.LoadTree(gopts.ctx, repo, id)
tree, err := restic.LoadTree(ctx, repo, id)
if err != nil {
Warnf("unable to load tree %v: %v\n", id.Str(), err)
continue
@ -91,7 +91,7 @@ func runRecover(gopts GlobalOptions) error {
bar.Done()
Verbosef("load snapshots\n")
err = restic.ForAllSnapshots(gopts.ctx, snapshotLister, repo, nil, func(id restic.ID, sn *restic.Snapshot, err error) error {
err = restic.ForAllSnapshots(ctx, snapshotLister, repo, nil, func(id restic.ID, sn *restic.Snapshot, err error) error {
trees[*sn.Tree] = true
return nil
})
@ -132,18 +132,18 @@ func runRecover(gopts GlobalOptions) error {
}
}
wg, ctx := errgroup.WithContext(gopts.ctx)
repo.StartPackUploader(ctx, wg)
wg, wgCtx := errgroup.WithContext(ctx)
repo.StartPackUploader(wgCtx, wg)
var treeID restic.ID
wg.Go(func() error {
var err error
treeID, err = restic.SaveTree(ctx, repo, tree)
treeID, err = restic.SaveTree(wgCtx, repo, tree)
if err != nil {
return errors.Fatalf("unable to save new tree to the repository: %v", err)
}
err = repo.Flush(ctx)
err = repo.Flush(wgCtx)
if err != nil {
return errors.Fatalf("unable to save blobs to the repository: %v", err)
}
@ -154,7 +154,7 @@ func runRecover(gopts GlobalOptions) error {
return err
}
return createSnapshot(gopts.ctx, "/recover", hostname, []string{"recovered"}, repo, &treeID)
return createSnapshot(ctx, "/recover", hostname, []string{"recovered"}, repo, &treeID)
}

11
cmd/restic/cmd_restore.go
View File

@ -1,6 +1,7 @@
package main
import (
"context"
"strings"
"time"
@ -30,7 +31,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
return runRestore(restoreOptions, globalOptions, args)
return runRestore(cmd.Context(), restoreOptions, globalOptions, args)
},
}
@ -63,8 +64,7 @@ func init() {
flags.BoolVar(&restoreOptions.Verify, "verify", false, "verify restored files content")
}
func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error {
ctx := gopts.ctx
func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions, args []string) error {
hasExcludes := len(opts.Exclude) > 0 || len(opts.InsensitiveExclude) > 0
hasIncludes := len(opts.Include) > 0 || len(opts.InsensitiveInclude) > 0
@ -117,13 +117,14 @@ func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error {
debug.Log("restore %v to %v", snapshotIDString, opts.Target)
repo, err := OpenRepository(gopts)
repo, err := OpenRepository(ctx, gopts)
if err != nil {
return err
}
if !gopts.NoLock {
lock, err := lockRepo(ctx, repo)
var lock *restic.Lock
lock, ctx, err = lockRepo(ctx, repo)
defer unlockRepo(lock)
if err != nil {
return err

7
cmd/restic/cmd_self_update.go
View File

@ -3,6 +3,7 @@
package main
import (
"context"
"os"
"path/filepath"
@ -27,7 +28,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
return runSelfUpdate(selfUpdateOptions, globalOptions, args)
return runSelfUpdate(cmd.Context(), selfUpdateOptions, globalOptions, args)
},
}
@ -45,7 +46,7 @@ func init() {
flags.StringVar(&selfUpdateOptions.Output, "output", "", "Save the downloaded file as `filename` (default: running binary itself)")
}
func runSelfUpdate(opts SelfUpdateOptions, gopts GlobalOptions, args []string) error {
func runSelfUpdate(ctx context.Context, opts SelfUpdateOptions, gopts GlobalOptions, args []string) error {
if opts.Output == "" {
file, err := os.Executable()
if err != nil {
@ -73,7 +74,7 @@ func runSelfUpdate(opts SelfUpdateOptions, gopts GlobalOptions, args []string) e
Verbosef("writing restic to %v\n", opts.Output)
v, err := selfupdate.DownloadLatestStableRelease(gopts.ctx, opts.Output, version, Verbosef)
v, err := selfupdate.DownloadLatestStableRelease(ctx, opts.Output, version, Verbosef)
if err != nil {
return errors.Fatalf("unable to update restic: %v", err)
}

12
cmd/restic/cmd_snapshots.go
View File

@ -26,7 +26,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
return runSnapshots(snapshotOptions, globalOptions, args)
return runSnapshots(cmd.Context(), snapshotOptions, globalOptions, args)
},
}
@ -57,23 +57,21 @@ func init() {
f.StringVarP(&snapshotOptions.GroupBy, "group-by", "g", "", "`group` snapshots by host, paths and/or tags, separated by comma")
}
func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) error {
repo, err := OpenRepository(gopts)
func runSnapshots(ctx context.Context, opts SnapshotOptions, gopts GlobalOptions, args []string) error {
repo, err := OpenRepository(ctx, gopts)
if err != nil {
return err
}
if !gopts.NoLock {
lock, err := lockRepo(gopts.ctx, repo)
var lock *restic.Lock
lock, ctx, err = lockRepo(ctx, repo)
defer unlockRepo(lock)
if err != nil {
return err
}
}
ctx, cancel := context.WithCancel(gopts.ctx)
defer cancel()
var snapshots restic.Snapshots
for sn := range FindFilteredSnapshots(ctx, repo.Backend(), repo, opts.Hosts, opts.Tags, opts.Paths, args) {
snapshots = append(snapshots, sn)

14
cmd/restic/cmd_stats.go
View File

@ -47,7 +47,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
return runStats(globalOptions, args)
return runStats(cmd.Context(), globalOptions, args)
},
}
@ -68,29 +68,27 @@ func init() {
initMultiSnapshotFilterOptions(f, &statsOptions.snapshotFilterOptions, true)
}
func runStats(gopts GlobalOptions, args []string) error {
func runStats(ctx context.Context, gopts GlobalOptions, args []string) error {
err := verifyStatsInput(gopts, args)
if err != nil {
return err
}
ctx, cancel := context.WithCancel(gopts.ctx)
defer cancel()
repo, err := OpenRepository(gopts)
repo, err := OpenRepository(ctx, gopts)
if err != nil {
return err
}
if !gopts.NoLock {
lock, err := lockRepo(ctx, repo)
var lock *restic.Lock
lock, ctx, err = lockRepo(ctx, repo)
defer unlockRepo(lock)
if err != nil {
return err
}
}
snapshotLister, err := backend.MemorizeList(gopts.ctx, repo.Backend(), restic.SnapshotFile)
snapshotLister, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile)
if err != nil {
return err
}

11
cmd/restic/cmd_tag.go
View File

@ -29,7 +29,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
return runTag(tagOptions, globalOptions, args)
return runTag(cmd.Context(), tagOptions, globalOptions, args)
},
}
@ -95,7 +95,7 @@ func changeTags(ctx context.Context, repo *repository.Repository, sn *restic.Sna
return changed, nil
}
func runTag(opts TagOptions, gopts GlobalOptions, args []string) error {
func runTag(ctx context.Context, opts TagOptions, gopts GlobalOptions, args []string) error {
if len(opts.SetTags) == 0 && len(opts.AddTags) == 0 && len(opts.RemoveTags) == 0 {
return errors.Fatal("nothing to do!")
}
@ -103,14 +103,15 @@ func runTag(opts TagOptions, gopts GlobalOptions, args []string) error {
return errors.Fatal("--set and --add/--remove cannot be given at the same time")
}
repo, err := OpenRepository(gopts)
repo, err := OpenRepository(ctx, gopts)
if err != nil {
return err
}
if !gopts.NoLock {
Verbosef("create exclusive lock for repository\n")
lock, err := lockRepoExclusive(gopts.ctx, repo)
var lock *restic.Lock
lock, ctx, err = lockRepoExclusive(ctx, repo)
defer unlockRepo(lock)
if err != nil {
return err
@ -118,8 +119,6 @@ func runTag(opts TagOptions, gopts GlobalOptions, args []string) error {
}
changeCnt := 0
ctx, cancel := context.WithCancel(gopts.ctx)
defer cancel()
for sn := range FindFilteredSnapshots(ctx, repo.Backend(), repo, opts.Hosts, opts.Tags, opts.Paths, args) {
changed, err := changeTags(ctx, repo, sn, opts.SetTags.Flatten(), opts.AddTags.Flatten(), opts.RemoveTags.Flatten())
if err != nil {

10
cmd/restic/cmd_unlock.go
View File

@ -1,6 +1,8 @@
package main
import (
"context"
"github.com/restic/restic/internal/restic"
"github.com/spf13/cobra"
)
@ -18,7 +20,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
return runUnlock(unlockOptions, globalOptions)
return runUnlock(cmd.Context(), unlockOptions, globalOptions)
},
}
@ -35,8 +37,8 @@ func init() {
unlockCmd.Flags().BoolVar(&unlockOptions.RemoveAll, "remove-all", false, "remove all locks, even non-stale ones")
}
func runUnlock(opts UnlockOptions, gopts GlobalOptions) error {
repo, err := OpenRepository(gopts)
func runUnlock(ctx context.Context, opts UnlockOptions, gopts GlobalOptions) error {
repo, err := OpenRepository(ctx, gopts)
if err != nil {
return err
}
@ -46,7 +48,7 @@ func runUnlock(opts UnlockOptions, gopts GlobalOptions) error {
fn = restic.RemoveAllLocks
}
processed, err := fn(gopts.ctx, repo)
processed, err := fn(ctx, repo)
if err != nil {
return err
}

14
cmd/restic/delete.go
View File

@ -1,6 +1,8 @@
package main
import (
"context"
"golang.org/x/sync/errgroup"
"github.com/restic/restic/internal/restic"
@ -8,22 +10,22 @@ import (
// DeleteFiles deletes the given fileList of fileType in parallel
// it will print a warning if there is an error, but continue deleting the remaining files
func DeleteFiles(gopts GlobalOptions, repo restic.Repository, fileList restic.IDSet, fileType restic.FileType) {
_ = deleteFiles(gopts, true, repo, fileList, fileType)
func DeleteFiles(ctx context.Context, gopts GlobalOptions, repo restic.Repository, fileList restic.IDSet, fileType restic.FileType) {
_ = deleteFiles(ctx, gopts, true, repo, fileList, fileType)
}
// DeleteFilesChecked deletes the given fileList of fileType in parallel
// if an error occurs, it will cancel and return this error
func DeleteFilesChecked(gopts GlobalOptions, repo restic.Repository, fileList restic.IDSet, fileType restic.FileType) error {
return deleteFiles(gopts, false, repo, fileList, fileType)
func DeleteFilesChecked(ctx context.Context, gopts GlobalOptions, repo restic.Repository, fileList restic.IDSet, fileType restic.FileType) error {
return deleteFiles(ctx, gopts, false, repo, fileList, fileType)
}
// deleteFiles deletes the given fileList of fileType in parallel
// if ignoreError=true, it will print a warning if there was an error, else it will abort.
func deleteFiles(gopts GlobalOptions, ignoreError bool, repo restic.Repository, fileList restic.IDSet, fileType restic.FileType) error {
func deleteFiles(ctx context.Context, gopts GlobalOptions, ignoreError bool, repo restic.Repository, fileList restic.IDSet, fileType restic.FileType) error {
totalCount := len(fileList)
fileChan := make(chan restic.ID)
wg, ctx := errgroup.WithContext(gopts.ctx)
wg, ctx := errgroup.WithContext(ctx)
wg.Go(func() error {
defer close(fileChan)
for id := range fileList {

40
cmd/restic/global.go
View File

@ -68,7 +68,6 @@ type GlobalOptions struct {
backend.TransportOptions
limiter.Limits
ctx context.Context
password string
stdout io.Writer
stderr io.Writer
@ -93,10 +92,11 @@ var globalOptions = GlobalOptions{
}
var isReadingPassword bool
var internalGlobalCtx context.Context
func init() {
var cancel context.CancelFunc
globalOptions.ctx, cancel = context.WithCancel(context.Background())
internalGlobalCtx, cancel = context.WithCancel(context.Background())
AddCleanupHandler(func(code int) (int, error) {
// Must be called before the unlock cleanup handler to ensure that the latter is
// not blocked due to limited number of backend connections, see #1434
@ -428,13 +428,13 @@ func ReadRepo(opts GlobalOptions) (string, error) {
const maxKeys = 20
// OpenRepository reads the password and opens the repository.
func OpenRepository(opts GlobalOptions) (*repository.Repository, error) {
func OpenRepository(ctx context.Context, opts GlobalOptions) (*repository.Repository, error) {
repo, err := ReadRepo(opts)
if err != nil {
return nil, err
}
be, err := open(repo, opts, opts.extended)
be, err := open(ctx, repo, opts, opts.extended)
if err != nil {
return nil, err
}
@ -478,7 +478,7 @@ func OpenRepository(opts GlobalOptions) (*repository.Repository, error) {
continue
}
err = s.SearchKey(opts.ctx, opts.password, maxKeys, opts.KeyHint)
err = s.SearchKey(ctx, opts.password, maxKeys, opts.KeyHint)
if err != nil && passwordTriesLeft > 1 {
opts.password = ""
fmt.Fprintf(os.Stderr, "%s. Try again\n", err)
@ -695,7 +695,7 @@ func parseConfig(loc location.Location, opts options.Options) (interface{}, erro
}
// Open the backend specified by a location config.
func open(s string, gopts GlobalOptions, opts options.Options) (restic.Backend, error) {
func open(ctx context.Context, s string, gopts GlobalOptions, opts options.Options) (restic.Backend, error) {
debug.Log("parsing location %v", location.StripPassword(s))
loc, err := location.Parse(s)
if err != nil {
@ -720,19 +720,19 @@ func open(s string, gopts GlobalOptions, opts options.Options) (restic.Backend,
switch loc.Scheme {
case "local":
be, err = local.Open(globalOptions.ctx, cfg.(local.Config))
be, err = local.Open(ctx, cfg.(local.Config))
case "sftp":
be, err = sftp.Open(globalOptions.ctx, cfg.(sftp.Config))
be, err = sftp.Open(ctx, cfg.(sftp.Config))
case "s3":
be, err = s3.Open(globalOptions.ctx, cfg.(s3.Config), rt)
be, err = s3.Open(ctx, cfg.(s3.Config), rt)
case "gs":
be, err = gs.Open(cfg.(gs.Config), rt)
case "azure":
be, err = azure.Open(cfg.(azure.Config), rt)
case "swift":
be, err = swift.Open(globalOptions.ctx, cfg.(swift.Config), rt)
be, err = swift.Open(ctx, cfg.(swift.Config), rt)
case "b2":
be, err = b2.Open(globalOptions.ctx, cfg.(b2.Config), rt)
be, err = b2.Open(ctx, cfg.(b2.Config), rt)
case "rest":
be, err = rest.Open(cfg.(rest.Config), rt)
case "rclone":
@ -760,7 +760,7 @@ func open(s string, gopts GlobalOptions, opts options.Options) (restic.Backend,
}
// check if config is there
fi, err := be.Stat(globalOptions.ctx, restic.Handle{Type: restic.ConfigFile})
fi, err := be.Stat(ctx, restic.Handle{Type: restic.ConfigFile})
if err != nil {
return nil, errors.Fatalf("unable to open config file: %v\nIs there a repository at the following location?\n%v", err, location.StripPassword(s))
}
@ -773,7 +773,7 @@ func open(s string, gopts GlobalOptions, opts options.Options) (restic.Backend,
}
// Create the backend specified by URI.
func create(s string, opts options.Options) (restic.Backend, error) {
func create(ctx context.Context, s string, opts options.Options) (restic.Backend, error) {
debug.Log("parsing location %v", s)
loc, err := location.Parse(s)
if err != nil {
@ -792,23 +792,23 @@ func create(s string, opts options.Options) (restic.Backend, error) {
switch loc.Scheme {
case "local":
return local.Create(globalOptions.ctx, cfg.(local.Config))
return local.Create(ctx, cfg.(local.Config))
case "sftp":
return sftp.Create(globalOptions.ctx, cfg.(sftp.Config))
return sftp.Create(ctx, cfg.(sftp.Config))
case "s3":
return s3.Create(globalOptions.ctx, cfg.(s3.Config), rt)
return s3.Create(ctx, cfg.(s3.Config), rt)
case "gs":
return gs.Create(cfg.(gs.Config), rt)
case "azure":
return azure.Create(cfg.(azure.Config), rt)
case "swift":
return swift.Open(globalOptions.ctx, cfg.(swift.Config), rt)
return swift.Open(ctx, cfg.(swift.Config), rt)
case "b2":
return b2.Create(globalOptions.ctx, cfg.(b2.Config), rt)
return b2.Create(ctx, cfg.(b2.Config), rt)
case "rest":
return rest.Create(globalOptions.ctx, cfg.(rest.Config), rt)
return rest.Create(ctx, cfg.(rest.Config), rt)
case "rclone":
return rclone.Create(globalOptions.ctx, cfg.(rclone.Config))
return rclone.Create(ctx, cfg.(rclone.Config))
}
debug.Log("invalid repository scheme: %v", s)

9
cmd/restic/integration_fuse_test.go
View File

@ -4,6 +4,7 @@
package main
import (
"context"
"fmt"
"os"
"path/filepath"
@ -57,7 +58,7 @@ func testRunMount(t testing.TB, gopts GlobalOptions, dir string) {
opts := MountOptions{
TimeTemplate: time.RFC3339,
}
rtest.OK(t, runMount(opts, gopts, []string{dir}))
rtest.OK(t, runMount(context.TODO(), opts, gopts, []string{dir}))
}
func testRunUmount(t testing.TB, gopts GlobalOptions, dir string) {
@ -119,7 +120,7 @@ func checkSnapshots(t testing.TB, global GlobalOptions, repo *repository.Reposit
}
for _, id := range snapshotIDs {
snapshot, err := restic.LoadSnapshot(global.ctx, repo, id)
snapshot, err := restic.LoadSnapshot(context.TODO(), repo, id)
rtest.OK(t, err)
ts := snapshot.Time.Format(time.RFC3339)
@ -160,7 +161,7 @@ func TestMount(t *testing.T) {
testRunInit(t, env.gopts)
repo, err := OpenRepository(env.gopts)
repo, err := OpenRepository(context.TODO(), env.gopts)
rtest.OK(t, err)
checkSnapshots(t, env.gopts, repo, env.mountpoint, env.repo, []restic.ID{}, 0)
@ -205,7 +206,7 @@ func TestMountSameTimestamps(t *testing.T) {
rtest.SetupTarTestFixture(t, env.base, filepath.Join("testdata", "repo-same-timestamps.tar.gz"))
repo, err := OpenRepository(env.gopts)
repo, err := OpenRepository(context.TODO(), env.gopts)
rtest.OK(t, err)
ids := []restic.ID{

2
cmd/restic/integration_helpers_test.go
View File

@ -2,7 +2,6 @@ package main
import (
"bytes"
"context"
"fmt"
"io/ioutil"
"os"
@ -193,7 +192,6 @@ func withTestEnvironment(t testing.TB) (env *testEnvironment, cleanup func()) {
Repo: env.repo,
Quiet: true,
CacheDir: env.cache,
ctx: context.Background(),
password: rtest.TestPassword,
stdout: os.Stdout,
stderr: os.Stderr,

112
cmd/restic/integration_test.go
View File

@ -52,12 +52,12 @@ func testRunInit(t testing.TB, opts GlobalOptions) {
restic.TestDisableCheckPolynomial(t)
restic.TestSetLockTimeout(t, 0)
rtest.OK(t, runInit(InitOptions{}, opts, nil))
rtest.OK(t, runInit(context.TODO(), InitOptions{}, opts, nil))
t.Logf("repository initialized at %v", opts.Repo)
}
func testRunBackupAssumeFailure(t testing.TB, dir string, target []string, opts BackupOptions, gopts GlobalOptions) error {
ctx, cancel := context.WithCancel(gopts.ctx)
ctx, cancel := context.WithCancel(context.TODO())
defer cancel()
var wg errgroup.Group
@ -71,7 +71,7 @@ func testRunBackupAssumeFailure(t testing.TB, dir string, target []string, opts
defer cleanup()
}
backupErr := runBackup(opts, gopts, term, target)
backupErr := runBackup(ctx, opts, gopts, term, target)
cancel()
@ -95,7 +95,7 @@ func testRunList(t testing.TB, tpe string, opts GlobalOptions) restic.IDs {
globalOptions.stdout = os.Stdout
}()
rtest.OK(t, runList(cmdList, opts, []string{tpe}))
rtest.OK(t, runList(context.TODO(), cmdList, opts, []string{tpe}))
return parseIDsFromReader(t, buf)
}
@ -112,7 +112,7 @@ func testRunRestoreLatest(t testing.TB, gopts GlobalOptions, dir string, paths [
},
}
rtest.OK(t, runRestore(opts, gopts, []string{"latest"}))
rtest.OK(t, runRestore(context.TODO(), opts, gopts, []string{"latest"}))
}
func testRunRestoreExcludes(t testing.TB, gopts GlobalOptions, dir string, snapshotID restic.ID, excludes []string) {
@ -121,7 +121,7 @@ func testRunRestoreExcludes(t testing.TB, gopts GlobalOptions, dir string, snaps
Exclude: excludes,
}
rtest.OK(t, runRestore(opts, gopts, []string{snapshotID.String()}))
rtest.OK(t, runRestore(context.TODO(), opts, gopts, []string{snapshotID.String()}))
}
func testRunRestoreIncludes(t testing.TB, gopts GlobalOptions, dir string, snapshotID restic.ID, includes []string) {
@ -130,11 +130,11 @@ func testRunRestoreIncludes(t testing.TB, gopts GlobalOptions, dir string, snaps
Include: includes,
}
rtest.OK(t, runRestore(opts, gopts, []string{snapshotID.String()}))
rtest.OK(t, runRestore(context.TODO(), opts, gopts, []string{snapshotID.String()}))
}
func testRunRestoreAssumeFailure(t testing.TB, snapshotID string, opts RestoreOptions, gopts GlobalOptions) error {
err := runRestore(opts, gopts, []string{snapshotID})
err := runRestore(context.TODO(), opts, gopts, []string{snapshotID})
return err
}
@ -144,7 +144,7 @@ func testRunCheck(t testing.TB, gopts GlobalOptions) {
ReadData: true,
CheckUnused: true,
}
rtest.OK(t, runCheck(opts, gopts, nil))
rtest.OK(t, runCheck(context.TODO(), opts, gopts, nil))
}
func testRunCheckOutput(gopts GlobalOptions) (string, error) {
@ -159,7 +159,7 @@ func testRunCheckOutput(gopts GlobalOptions) (string, error) {
ReadData: true,
}
err := runCheck(opts, gopts, nil)
err := runCheck(context.TODO(), opts, gopts, nil)
return buf.String(), err
}
@ -177,7 +177,7 @@ func testRunDiffOutput(gopts GlobalOptions, firstSnapshotID string, secondSnapsh
opts := DiffOptions{
ShowMetadata: false,
}
err := runDiff(opts, gopts, []string{firstSnapshotID, secondSnapshotID})
err := runDiff(context.TODO(), opts, gopts, []string{firstSnapshotID, secondSnapshotID})
return buf.String(), err
}
@ -187,7 +187,7 @@ func testRunRebuildIndex(t testing.TB, gopts GlobalOptions) {
globalOptions.stdout = os.Stdout
}()
rtest.OK(t, runRebuildIndex(RebuildIndexOptions{}, gopts))
rtest.OK(t, runRebuildIndex(context.TODO(), RebuildIndexOptions{}, gopts))
}
func testRunLs(t testing.TB, gopts GlobalOptions, snapshotID string) []string {
@ -202,7 +202,7 @@ func testRunLs(t testing.TB, gopts GlobalOptions, snapshotID string) []string {
opts := LsOptions{}
rtest.OK(t, runLs(opts, gopts, []string{snapshotID}))
rtest.OK(t, runLs(context.TODO(), opts, gopts, []string{snapshotID}))
return strings.Split(buf.String(), "\n")
}
@ -218,7 +218,7 @@ func testRunFind(t testing.TB, wantJSON bool, gopts GlobalOptions, pattern strin
opts := FindOptions{}
rtest.OK(t, runFind(opts, gopts, []string{pattern}))
rtest.OK(t, runFind(context.TODO(), opts, gopts, []string{pattern}))
return buf.Bytes()
}
@ -234,7 +234,7 @@ func testRunSnapshots(t testing.TB, gopts GlobalOptions) (newest *Snapshot, snap
opts := SnapshotOptions{}
rtest.OK(t, runSnapshots(opts, globalOptions, []string{}))
rtest.OK(t, runSnapshots(context.TODO(), opts, globalOptions, []string{}))
snapshots := []Snapshot{}
rtest.OK(t, json.Unmarshal(buf.Bytes(), &snapshots))
@ -251,7 +251,7 @@ func testRunSnapshots(t testing.TB, gopts GlobalOptions) (newest *Snapshot, snap
func testRunForget(t testing.TB, gopts GlobalOptions, args ...string) {
opts := ForgetOptions{}
rtest.OK(t, runForget(opts, gopts, args))
rtest.OK(t, runForget(context.TODO(), opts, gopts, args))
}
func testRunForgetJSON(t testing.TB, gopts GlobalOptions, args ...string) {
@ -269,7 +269,7 @@ func testRunForgetJSON(t testing.TB, gopts GlobalOptions, args ...string) {
Last: 1,
}
rtest.OK(t, runForget(opts, gopts, args))
rtest.OK(t, runForget(context.TODO(), opts, gopts, args))
var forgets []*ForgetGroup
rtest.OK(t, json.Unmarshal(buf.Bytes(), &forgets))
@ -288,7 +288,7 @@ func testRunPrune(t testing.TB, gopts GlobalOptions, opts PruneOptions) {
defer func() {
gopts.backendTestHook = oldHook
}()
rtest.OK(t, runPrune(opts, gopts))
rtest.OK(t, runPrune(context.TODO(), opts, gopts))
}
func testSetupBackupData(t testing.TB, env *testEnvironment) string {
@ -437,11 +437,11 @@ func TestBackupNonExistingFile(t *testing.T) {
}
func removePacksExcept(gopts GlobalOptions, t *testing.T, keep restic.IDSet, removeTreePacks bool) {
r, err := OpenRepository(gopts)
r, err := OpenRepository(context.TODO(), gopts)
rtest.OK(t, err)
// Get all tree packs
rtest.OK(t, r.LoadIndex(gopts.ctx))
rtest.OK(t, r.LoadIndex(context.TODO()))
treePacks := restic.NewIDSet()
r.Index().Each(context.TODO(), func(pb restic.PackedBlob) {
@ -451,11 +451,11 @@ func removePacksExcept(gopts GlobalOptions, t *testing.T, keep restic.IDSet, rem
})
// remove all packs containing data blobs
rtest.OK(t, r.List(gopts.ctx, restic.PackFile, func(id restic.ID, size int64) error {
rtest.OK(t, r.List(context.TODO(), restic.PackFile, func(id restic.ID, size int64) error {
if treePacks.Has(id) != removeTreePacks || keep.Has(id) {
return nil
}
return r.Backend().Remove(gopts.ctx, restic.Handle{Type: restic.PackFile, Name: id.String()})
return r.Backend().Remove(context.TODO(), restic.Handle{Type: restic.PackFile, Name: id.String()})
}))
}
@ -479,7 +479,7 @@ func TestBackupSelfHealing(t *testing.T) {
testRunRebuildIndex(t, env.gopts)
// now the repo is also missing the data blob in the index; check should report this
rtest.Assert(t, runCheck(CheckOptions{}, env.gopts, nil) != nil,
rtest.Assert(t, runCheck(context.TODO(), CheckOptions{}, env.gopts, nil) != nil,
"check should have reported an error")
// second backup should report an error but "heal" this situation
@ -502,9 +502,9 @@ func TestBackupTreeLoadError(t *testing.T) {
// Backup a subdirectory first, such that we can remove the tree pack for the subdirectory
testRunBackup(t, env.testdata, []string{"test"}, opts, env.gopts)
r, err := OpenRepository(env.gopts)
r, err := OpenRepository(context.TODO(), env.gopts)
rtest.OK(t, err)
rtest.OK(t, r.LoadIndex(env.gopts.ctx))
rtest.OK(t, r.LoadIndex(context.TODO()))
treePacks := restic.NewIDSet()
r.Index().Each(context.TODO(), func(pb restic.PackedBlob) {
if pb.Type == restic.TreeBlob {
@ -517,11 +517,11 @@ func TestBackupTreeLoadError(t *testing.T) {
// delete the subdirectory pack first
for id := range treePacks {
rtest.OK(t, r.Backend().Remove(env.gopts.ctx, restic.Handle{Type: restic.PackFile, Name: id.String()}))
rtest.OK(t, r.Backend().Remove(context.TODO(), restic.Handle{Type: restic.PackFile, Name: id.String()}))
}
testRunRebuildIndex(t, env.gopts)
// now the repo is missing the tree blob in the index; check should report this
rtest.Assert(t, runCheck(CheckOptions{}, env.gopts, nil) != nil, "check should have reported an error")
rtest.Assert(t, runCheck(context.TODO(), CheckOptions{}, env.gopts, nil) != nil, "check should have reported an error")
// second backup should report an error but "heal" this situation
err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{filepath.Base(env.testdata)}, opts, env.gopts)
rtest.Assert(t, err != nil, "backup should have reported an error for the subdirectory")
@ -531,7 +531,7 @@ func TestBackupTreeLoadError(t *testing.T) {
removePacksExcept(env.gopts, t, restic.NewIDSet(), true)
testRunRebuildIndex(t, env.gopts)
// now the repo is also missing the data blob in the index; check should report this
rtest.Assert(t, runCheck(CheckOptions{}, env.gopts, nil) != nil, "check should have reported an error")
rtest.Assert(t, runCheck(context.TODO(), CheckOptions{}, env.gopts, nil) != nil, "check should have reported an error")
// second backup should report an error but "heal" this situation
err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{filepath.Base(env.testdata)}, opts, env.gopts)
rtest.Assert(t, err != nil, "backup should have reported an error")
@ -761,7 +761,7 @@ func testRunCopy(t testing.TB, srcGopts GlobalOptions, dstGopts GlobalOptions) {
},
}
rtest.OK(t, runCopy(copyOpts, gopts, nil))
rtest.OK(t, runCopy(context.TODO(), copyOpts, gopts, nil))
}
func TestCopy(t *testing.T) {
@ -903,15 +903,15 @@ func TestInitCopyChunkerParams(t *testing.T) {
password: env2.gopts.password,
},
}
rtest.Assert(t, runInit(initOpts, env.gopts, nil) != nil, "expected invalid init options to fail")
rtest.Assert(t, runInit(context.TODO(), initOpts, env.gopts, nil) != nil, "expected invalid init options to fail")
initOpts.CopyChunkerParameters = true
rtest.OK(t, runInit(initOpts, env.gopts, nil))
rtest.OK(t, runInit(context.TODO(), initOpts, env.gopts, nil))
repo, err := OpenRepository(env.gopts)
repo, err := OpenRepository(context.TODO(), env.gopts)
rtest.OK(t, err)
otherRepo, err := OpenRepository(env2.gopts)
otherRepo, err := OpenRepository(context.TODO(), env2.gopts)
rtest.OK(t, err)
rtest.Assert(t, repo.Config().ChunkerPolynomial == otherRepo.Config().ChunkerPolynomial,
@ -920,7 +920,7 @@ func TestInitCopyChunkerParams(t *testing.T) {
}
func testRunTag(t testing.TB, opts TagOptions, gopts GlobalOptions) {
rtest.OK(t, runTag(opts, gopts, []string{}))
rtest.OK(t, runTag(context.TODO(), opts, gopts, []string{}))
}
func TestTag(t *testing.T) {
@ -1012,7 +1012,7 @@ func testRunKeyListOtherIDs(t testing.TB, gopts GlobalOptions) []string {
globalOptions.stdout = os.Stdout
}()
rtest.OK(t, runKey(gopts, []string{"list"}))
rtest.OK(t, runKey(context.TODO(), gopts, []string{"list"}))
scanner := bufio.NewScanner(buf)
exp := regexp.MustCompile(`^ ([a-f0-9]+) `)
@ -1033,7 +1033,7 @@ func testRunKeyAddNewKey(t testing.TB, newPassword string, gopts GlobalOptions)
testKeyNewPassword = ""
}()
rtest.OK(t, runKey(gopts, []string{"add"}))
rtest.OK(t, runKey(context.TODO(), gopts, []string{"add"}))
}
func testRunKeyAddNewKeyUserHost(t testing.TB, gopts GlobalOptions) {
@ -1047,11 +1047,11 @@ func testRunKeyAddNewKeyUserHost(t testing.TB, gopts GlobalOptions) {
rtest.OK(t, cmdKey.Flags().Parse([]string{"--user=john", "--host=example.com"}))
t.Log("adding key for john@example.com")
rtest.OK(t, runKey(gopts, []string{"add"}))
rtest.OK(t, runKey(context.TODO(), gopts, []string{"add"}))
repo, err := OpenRepository(gopts)
repo, err := OpenRepository(context.TODO(), gopts)
rtest.OK(t, err)
key, err := repository.SearchKey(gopts.ctx, repo, testKeyNewPassword, 2, "")
key, err := repository.SearchKey(context.TODO(), repo, testKeyNewPassword, 2, "")
rtest.OK(t, err)
rtest.Equals(t, "john", key.Username)
@ -1064,13 +1064,13 @@ func testRunKeyPasswd(t testing.TB, newPassword string, gopts GlobalOptions) {
testKeyNewPassword = ""
}()
rtest.OK(t, runKey(gopts, []string{"passwd"}))
rtest.OK(t, runKey(context.TODO(), gopts, []string{"passwd"}))
}
func testRunKeyRemove(t testing.TB, gopts GlobalOptions, IDs []string) {
t.Logf("remove %d keys: %q\n", len(IDs), IDs)
for _, id := range IDs {
rtest.OK(t, runKey(gopts, []string{"remove", id}))
rtest.OK(t, runKey(context.TODO(), gopts, []string{"remove", id}))
}
}
@ -1100,7 +1100,7 @@ func TestKeyAddRemove(t *testing.T) {
env.gopts.password = passwordList[len(passwordList)-1]
t.Logf("testing access with last password %q\n", env.gopts.password)
rtest.OK(t, runKey(env.gopts, []string{"list"}))
rtest.OK(t, runKey(context.TODO(), env.gopts, []string{"list"}))
testRunCheck(t, env.gopts)
testRunKeyAddNewKeyUserHost(t, env.gopts)
@ -1128,16 +1128,16 @@ func TestKeyProblems(t *testing.T) {
testKeyNewPassword = ""
}()
err := runKey(env.gopts, []string{"passwd"})
err := runKey(context.TODO(), env.gopts, []string{"passwd"})
t.Log(err)
rtest.Assert(t, err != nil, "expected passwd change to fail")
err = runKey(env.gopts, []string{"add"})
err = runKey(context.TODO(), env.gopts, []string{"add"})
t.Log(err)
rtest.Assert(t, err != nil, "expected key adding to fail")
t.Logf("testing access with initial password %q\n", env.gopts.password)
rtest.OK(t, runKey(env.gopts, []string{"list"}))
rtest.OK(t, runKey(context.TODO(), env.gopts, []string{"list"}))
testRunCheck(t, env.gopts)
}
@ -1549,7 +1549,7 @@ func TestRebuildIndexFailsOnAppendOnly(t *testing.T) {
env.gopts.backendTestHook = func(r restic.Backend) (restic.Backend, error) {
return &appendOnlyBackend{r}, nil
}
err := runRebuildIndex(RebuildIndexOptions{}, env.gopts)
err := runRebuildIndex(context.TODO(), RebuildIndexOptions{}, env.gopts)
if err == nil {
t.Error("expected rebuildIndex to fail")
}
@ -1645,18 +1645,18 @@ func testPrune(t *testing.T, pruneOpts PruneOptions, checkOpts CheckOptions) {
testRunForgetJSON(t, env.gopts)
testRunForget(t, env.gopts, firstSnapshot[0].String())
testRunPrune(t, env.gopts, pruneOpts)
rtest.OK(t, runCheck(checkOpts, env.gopts, nil))
rtest.OK(t, runCheck(context.TODO(), checkOpts, env.gopts, nil))
}
var pruneDefaultOptions = PruneOptions{MaxUnused: "5%"}
func listPacks(gopts GlobalOptions, t *testing.T) restic.IDSet {
r, err := OpenRepository(gopts)
r, err := OpenRepository(context.TODO(), gopts)
rtest.OK(t, err)
packs := restic.NewIDSet()
rtest.OK(t, r.List(gopts.ctx, restic.PackFile, func(id restic.ID, size int64) error {
rtest.OK(t, r.List(context.TODO(), restic.PackFile, func(id restic.ID, size int64) error {
packs.Insert(id)
return nil
}))
@ -1697,7 +1697,7 @@ func TestPruneWithDamagedRepository(t *testing.T) {
env.gopts.backendTestHook = oldHook
}()
// prune should fail
rtest.Assert(t, runPrune(pruneDefaultOptions, env.gopts) == errorPacksMissing,
rtest.Assert(t, runPrune(context.TODO(), pruneDefaultOptions, env.gopts) == errorPacksMissing,
"prune should have reported index not complete error")
}
@ -1769,7 +1769,7 @@ func testEdgeCaseRepo(t *testing.T, tarfile string, optionsCheck CheckOptions, o
if checkOK {
testRunCheck(t, env.gopts)
} else {
rtest.Assert(t, runCheck(optionsCheck, env.gopts, nil) != nil,
rtest.Assert(t, runCheck(context.TODO(), optionsCheck, env.gopts, nil) != nil,
"check should have reported an error")
}
@ -1777,7 +1777,7 @@ func testEdgeCaseRepo(t *testing.T, tarfile string, optionsCheck CheckOptions, o
testRunPrune(t, env.gopts, optionsPrune)
testRunCheck(t, env.gopts)
} else {
rtest.Assert(t, runPrune(optionsPrune, env.gopts) != nil,
rtest.Assert(t, runPrune(context.TODO(), optionsPrune, env.gopts) != nil,
"prune should have reported an error")
}
}
@ -1848,10 +1848,10 @@ func TestListOnce(t *testing.T) {
testRunForgetJSON(t, env.gopts)
testRunForget(t, env.gopts, firstSnapshot[0].String())
testRunPrune(t, env.gopts, pruneOpts)
rtest.OK(t, runCheck(checkOpts, env.gopts, nil))
rtest.OK(t, runCheck(context.TODO(), checkOpts, env.gopts, nil))
rtest.OK(t, runRebuildIndex(RebuildIndexOptions{}, env.gopts))
rtest.OK(t, runRebuildIndex(RebuildIndexOptions{ReadAllPacks: true}, env.gopts))
rtest.OK(t, runRebuildIndex(context.TODO(), RebuildIndexOptions{}, env.gopts))
rtest.OK(t, runRebuildIndex(context.TODO(), RebuildIndexOptions{ReadAllPacks: true}, env.gopts))
}
func TestHardLink(t *testing.T) {
@ -2204,7 +2204,7 @@ func TestFindListOnce(t *testing.T) {
testRunBackup(t, "", []string{filepath.Join(env.testdata, "0", "0", "9", "3")}, opts, env.gopts)
thirdSnapshot := restic.NewIDSet(testRunList(t, "snapshots", env.gopts)...)
repo, err := OpenRepository(env.gopts)
repo, err := OpenRepository(context.TODO(), env.gopts)
rtest.OK(t, err)
snapshotIDs := restic.NewIDSet()

176
cmd/restic/lock.go
View File

@ -7,27 +7,31 @@ import (
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
)
type lockContext struct {
cancel context.CancelFunc
refreshWG sync.WaitGroup
}
var globalLocks struct {
locks []*restic.Lock
cancelRefresh chan struct{}
refreshWG sync.WaitGroup
locks map[*restic.Lock]*lockContext
sync.Mutex
sync.Once
}
func lockRepo(ctx context.Context, repo *repository.Repository) (*restic.Lock, error) {
func lockRepo(ctx context.Context, repo restic.Repository) (*restic.Lock, context.Context, error) {
return lockRepository(ctx, repo, false)
}
func lockRepoExclusive(ctx context.Context, repo *repository.Repository) (*restic.Lock, error) {
func lockRepoExclusive(ctx context.Context, repo restic.Repository) (*restic.Lock, context.Context, error) {
return lockRepository(ctx, repo, true)
}
func lockRepository(ctx context.Context, repo *repository.Repository, exclusive bool) (*restic.Lock, error) {
// lockRepository wraps the ctx such that it is cancelled when the repository is unlocked
// cancelling the original context also stops the lock refresh
func lockRepository(ctx context.Context, repo restic.Repository, exclusive bool) (*restic.Lock, context.Context, error) {
// make sure that a repository is unlocked properly and after cancel() was
// called by the cleanup handler in global.go
globalLocks.Do(func() {
@ -41,53 +45,114 @@ func lockRepository(ctx context.Context, repo *repository.Repository, exclusive
lock, err := lockFn(ctx, repo)
if err != nil {
return nil, errors.WithMessage(err, "unable to create lock in backend")
return nil, ctx, errors.WithMessage(err, "unable to create lock in backend")
}
debug.Log("create lock %p (exclusive %v)", lock, exclusive)
globalLocks.Lock()
if globalLocks.cancelRefresh == nil {
debug.Log("start goroutine for lock refresh")
globalLocks.cancelRefresh = make(chan struct{})
globalLocks.refreshWG = sync.WaitGroup{}
globalLocks.refreshWG.Add(1)
go refreshLocks(&globalLocks.refreshWG, globalLocks.cancelRefresh)
ctx, cancel := context.WithCancel(ctx)
lockInfo := &lockContext{
cancel: cancel,
}
lockInfo.refreshWG.Add(2)
refreshChan := make(chan struct{})
globalLocks.locks = append(globalLocks.locks, lock)
globalLocks.Lock()
globalLocks.locks[lock] = lockInfo
go refreshLocks(ctx, lock, lockInfo, refreshChan)
go monitorLockRefresh(ctx, lock, lockInfo, refreshChan)
globalLocks.Unlock()
return lock, err
return lock, ctx, err
}
var refreshInterval = 5 * time.Minute
func refreshLocks(wg *sync.WaitGroup, done <-chan struct{}) {
debug.Log("start")
defer func() {
wg.Done()
globalLocks.Lock()
globalLocks.cancelRefresh = nil
globalLocks.Unlock()
}()
// consider a lock refresh failed a bit before the lock actually becomes stale
// the difference allows to compensate for a small time drift between clients.
var refreshabilityTimeout = restic.StaleLockTimeout - refreshInterval*3/2
func refreshLocks(ctx context.Context, lock *restic.Lock, lockInfo *lockContext, refreshed chan<- struct{}) {
debug.Log("start")
ticker := time.NewTicker(refreshInterval)
lastRefresh := lock.Time
defer func() {
ticker.Stop()
// ensure that the context was cancelled before removing the lock
lockInfo.cancel()
// remove the lock from the repo
debug.Log("unlocking repository with lock %v", lock)
if err := lock.Unlock(); err != nil {
debug.Log("error while unlocking: %v", err)
Warnf("error while unlocking: %v", err)
}
lockInfo.refreshWG.Done()
}()
for {
select {
case <-done:
case <-ctx.Done():
debug.Log("terminate")
return
case <-ticker.C:
if time.Since(lastRefresh) > refreshabilityTimeout {
// the lock is too old, wait until the expiry monitor cancels the context
continue
}
debug.Log("refreshing locks")
globalLocks.Lock()
for _, lock := range globalLocks.locks {
err := lock.Refresh(context.TODO())
if err != nil {
Warnf("unable to refresh lock: %v\n", err)
err := lock.Refresh(context.TODO())
if err != nil {
Warnf("unable to refresh lock: %v\n", err)
} else {
lastRefresh = lock.Time
// inform monitor gorountine about successful refresh
select {
case <-ctx.Done():
case refreshed <- struct{}{}:
}
}
globalLocks.Unlock()
}
}
}
func monitorLockRefresh(ctx context.Context, lock *restic.Lock, lockInfo *lockContext, refreshed <-chan struct{}) {
// time.Now() might use a monotonic timer which is paused during standby
// convert to unix time to ensure we compare real time values
lastRefresh := time.Now().Unix()
pollDuration := 1 * time.Second
if refreshInterval < pollDuration {
// require for TestLockFailedRefresh
pollDuration = refreshInterval / 5
}
// timers are paused during standby, which is a problem as the refresh timeout
// _must_ expire if the host was too long in standby. Thus fall back to periodic checks
// https://github.com/golang/go/issues/35012
timer := time.NewTimer(pollDuration)
defer func() {
timer.Stop()
lockInfo.cancel()
lockInfo.refreshWG.Done()
}()
for {
select {
case <-ctx.Done():
debug.Log("terminate expiry monitoring")
return
case <-refreshed:
lastRefresh = time.Now().Unix()
case <-timer.C:
if float64(time.Now().Unix()-lastRefresh) < refreshInterval.Seconds() {
// restart timer
timer.Reset(pollDuration)
continue
}
Warnf("Fatal: failed to refresh lock in time\n")
return
}
}
}
@ -98,40 +163,35 @@ func unlockRepo(lock *restic.Lock) {
}
globalLocks.Lock()
defer globalLocks.Unlock()
lockInfo, exists := globalLocks.locks[lock]
delete(globalLocks.locks, lock)
globalLocks.Unlock()
for i := 0; i < len(globalLocks.locks); i++ {
if lock == globalLocks.locks[i] {
// remove the lock from the repo
debug.Log("unlocking repository with lock %v", lock)
if err := lock.Unlock(); err != nil {
debug.Log("error while unlocking: %v", err)
Warnf("error while unlocking: %v", err)
return
}
// remove the lock from the list of locks
globalLocks.locks = append(globalLocks.locks[:i], globalLocks.locks[i+1:]...)
return
}
if !exists {
debug.Log("unable to find lock %v in the global list of locks, ignoring", lock)
return
}
debug.Log("unable to find lock %v in the global list of locks, ignoring", lock)
lockInfo.cancel()
lockInfo.refreshWG.Wait()
}
func unlockAll(code int) (int, error) {
globalLocks.Lock()
defer globalLocks.Unlock()
locks := globalLocks.locks
debug.Log("unlocking %d locks", len(globalLocks.locks))
for _, lock := range globalLocks.locks {
if err := lock.Unlock(); err != nil {
debug.Log("error while unlocking: %v", err)
return code, err
}
debug.Log("successfully removed lock")
for _, lockInfo := range globalLocks.locks {
lockInfo.cancel()
}
globalLocks.locks = make(map[*restic.Lock]*lockContext)
globalLocks.Unlock()
for _, lockInfo := range locks {
lockInfo.refreshWG.Wait()
}
globalLocks.locks = globalLocks.locks[:0]
return code, nil
}
func init() {
globalLocks.locks = make(map[*restic.Lock]*lockContext)
}

130
cmd/restic/lock_test.go Normal file
View File

@ -0,0 +1,130 @@
package main
import (
"context"
"fmt"
"testing"
"time"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
)
func openTestRepo(t *testing.T, wrapper backendWrapper) (*repository.Repository, func(), *testEnvironment) {
env, cleanup := withTestEnvironment(t)
if wrapper != nil {
env.gopts.backendTestHook = wrapper
}
testRunInit(t, env.gopts)
repo, err := OpenRepository(context.TODO(), env.gopts)
rtest.OK(t, err)
return repo, cleanup, env
}
func checkedLockRepo(ctx context.Context, t *testing.T, repo restic.Repository) (*restic.Lock, context.Context) {
lock, wrappedCtx, err := lockRepo(ctx, repo)
rtest.OK(t, err)
rtest.OK(t, wrappedCtx.Err())
if lock.Stale() {
t.Fatal("lock returned stale lock")
}
return lock, wrappedCtx
}
func TestLock(t *testing.T) {
repo, cleanup, _ := openTestRepo(t, nil)
defer cleanup()
lock, wrappedCtx := checkedLockRepo(context.Background(), t, repo)
unlockRepo(lock)
if wrappedCtx.Err() == nil {
t.Fatal("unlock did not cancel context")
}
}
func TestLockCancel(t *testing.T) {
repo, cleanup, _ := openTestRepo(t, nil)
defer cleanup()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
lock, wrappedCtx := checkedLockRepo(ctx, t, repo)
cancel()
if wrappedCtx.Err() == nil {
t.Fatal("canceled parent context did not cancel context")
}
// unlockRepo should not crash
unlockRepo(lock)
}
func TestLockUnlockAll(t *testing.T) {
repo, cleanup, _ := openTestRepo(t, nil)
defer cleanup()
lock, wrappedCtx := checkedLockRepo(context.Background(), t, repo)
_, err := unlockAll(0)
rtest.OK(t, err)
if wrappedCtx.Err() == nil {
t.Fatal("canceled parent context did not cancel context")
}
// unlockRepo should not crash
unlockRepo(lock)
}
func TestLockConflict(t *testing.T) {
repo, cleanup, env := openTestRepo(t, nil)
defer cleanup()
repo2, err := OpenRepository(context.TODO(), env.gopts)
rtest.OK(t, err)
lock, _, err := lockRepoExclusive(context.Background(), repo)
rtest.OK(t, err)
defer unlockRepo(lock)
_, _, err = lockRepo(context.Background(), repo2)
if err == nil {
t.Fatal("second lock should have failed")
}
}
type writeOnceBackend struct {
restic.Backend
written bool
}
func (b *writeOnceBackend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
if b.written {
return fmt.Errorf("fail after first write")
}
b.written = true
return b.Backend.Save(ctx, h, rd)
}
func TestLockFailedRefresh(t *testing.T) {
repo, cleanup, _ := openTestRepo(t, func(r restic.Backend) (restic.Backend, error) {
return &writeOnceBackend{Backend: r}, nil
})
defer cleanup()
// reduce locking intervals to be suitable for testing
ri, rt := refreshInterval, refreshabilityTimeout
refreshInterval = 20 * time.Millisecond
refreshabilityTimeout = 100 * time.Millisecond
defer func() {
refreshInterval, refreshabilityTimeout = ri, rt
}()
lock, wrappedCtx := checkedLockRepo(context.Background(), t, repo)
select {
case <-wrappedCtx.Done():
// expected lock refresh failure
case <-time.After(time.Second):
t.Fatal("failed lock refresh did not cause context cancellation")
}
// unlockRepo should not crash
unlockRepo(lock)
}

2
cmd/restic/main.go
View File

@ -95,7 +95,7 @@ func main() {
debug.Log("main %#v", os.Args)
debug.Log("restic %s compiled with %v on %v/%v",
version, runtime.Version(), runtime.GOOS, runtime.GOARCH)
err := cmdRoot.Execute()
err := cmdRoot.ExecuteContext(internalGlobalCtx)
switch {
case restic.IsAlreadyLocked(err):

46
internal/restic/lock.go
View File

@ -137,23 +137,37 @@ func (l *Lock) fillUserInfo() error {
// non-exclusive lock is to be created, an error is only returned when an
// exclusive lock is found.
func (l *Lock) checkForOtherLocks(ctx context.Context) error {
return ForAllLocks(ctx, l.repo, l.lockID, func(id ID, lock *Lock, err error) error {
if err != nil {
// ignore locks that cannot be loaded
debug.Log("ignore lock %v: %v", id, err)
var err error
// retry locking a few times
for i := 0; i < 3; i++ {
err = ForAllLocks(ctx, l.repo, l.lockID, func(id ID, lock *Lock, err error) error {
if err != nil {
// if we cannot load a lock then it is unclear whether it can be ignored
// it could either be invalid or just unreadable due to network/permission problems
debug.Log("ignore lock %v: %v", id, err)
return errors.Fatal(err.Error())
}
if l.Exclusive {
return &alreadyLockedError{otherLock: lock}
}
if !l.Exclusive && lock.Exclusive {
return &alreadyLockedError{otherLock: lock}
}
return nil
})
// no lock detected
if err == nil {
return nil
}
if l.Exclusive {
return &alreadyLockedError{otherLock: lock}
// lock conflicts are permanent
if _, ok := err.(*alreadyLockedError); ok {
return err
}
if !l.Exclusive && lock.Exclusive {
return &alreadyLockedError{otherLock: lock}
}
return nil
})
}
return err
}
// createLock acquires the lock by creating a file in the repository.
@ -175,14 +189,14 @@ func (l *Lock) Unlock() error {
return l.repo.Backend().Remove(context.TODO(), Handle{Type: LockFile, Name: l.lockID.String()})
}
var staleTimeout = 30 * time.Minute
var StaleLockTimeout = 30 * time.Minute
// Stale returns true if the lock is stale. A lock is stale if the timestamp is
// older than 30 minutes or if it was created on the current machine and the
// process isn't alive any more.
func (l *Lock) Stale() bool {
debug.Log("testing if lock %v for process %d is stale", l, l.PID)
if time.Since(l.Time) > staleTimeout {
if time.Since(l.Time) > StaleLockTimeout {
debug.Log("lock is stale, timestamp is too old: %v\n", l.Time)
return true
}

28
internal/restic/lock_test.go
View File

@ -2,10 +2,13 @@ package restic_test
import (
"context"
"fmt"
"io"
"os"
"testing"
"time"
"github.com/restic/restic/internal/backend/mem"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
@ -49,6 +52,31 @@ func TestMultipleLock(t *testing.T) {
rtest.OK(t, lock2.Unlock())
}
type failLockLoadingBackend struct {
restic.Backend
}
func (be *failLockLoadingBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
if h.Type == restic.LockFile {
return fmt.Errorf("error loading lock")
}
return be.Backend.Load(ctx, h, length, offset, fn)
}
func TestMultipleLockFailure(t *testing.T) {
be := &failLockLoadingBackend{Backend: mem.New()}
repo, cleanup := repository.TestRepositoryWithBackend(t, be, 0)
defer cleanup()
lock1, err := restic.NewLock(context.TODO(), repo)
rtest.OK(t, err)
_, err = restic.NewLock(context.TODO(), repo)
rtest.Assert(t, err != nil, "unreadable lock file did not result in an error")
rtest.OK(t, lock1.Unlock())
}
func TestLockExclusive(t *testing.T) {
repo, cleanup := repository.TestRepository(t)
defer cleanup()