diff --git a/changelog/unreleased/issue-1542 b/changelog/unreleased/issue-1542 new file mode 100644 index 000000000..7e16c44aa --- /dev/null +++ b/changelog/unreleased/issue-1542 @@ -0,0 +1,17 @@ +Enhancement: Add --dry-run/-n option to backup command + +Testing exclude filters and other configuration options required running a +normal backup. Wrong filters could then cause files to be uploaded +unexpectedly. It was also not possible to approximately determine beforehand +how much data has to be uploaded. + +We added a new --dry-run/-n option to the backup command, which performs +all the normal steps of a backup without actually writing any changes to +the repository. Passing -vv will log information about files that would +be added, allowing verification of source and exclusion backup options +without committing changes to the repository. + +https://github.com/restic/restic/issues/1542 +https://github.com/restic/restic/pull/2308 +https://github.com/restic/restic/pull/3210 +https://github.com/restic/restic/pull/3300 diff --git a/cmd/restic/cmd_backup.go b/cmd/restic/cmd_backup.go index a285f851d..ef229ee8a 100644 --- a/cmd/restic/cmd_backup.go +++ b/cmd/restic/cmd_backup.go @@ -92,6 +92,7 @@ type BackupOptions struct { IgnoreInode bool IgnoreCtime bool UseFsSnapshot bool + DryRun bool } var backupOptions BackupOptions @@ -132,6 +133,7 @@ func init() { f.BoolVar(&backupOptions.WithAtime, "with-atime", false, "store the atime for all files and directories") f.BoolVar(&backupOptions.IgnoreInode, "ignore-inode", false, "ignore inode number changes when checking for modified files") f.BoolVar(&backupOptions.IgnoreCtime, "ignore-ctime", false, "ignore ctime changes when checking for modified files") + f.BoolVarP(&backupOptions.DryRun, "dry-run", "n", false, "do not upload or write any data, just show what would be done") if runtime.GOOS == "windows" { f.BoolVar(&backupOptions.UseFsSnapshot, "use-fs-snapshot", false, "use filesystem snapshot where possible (currently only Windows VSS)") } @@ -535,6 +537,7 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina Run(ctx context.Context) error Error(item string, fi os.FileInfo, err error) error Finish(snapshotID restic.ID) + SetDryRun() // ui.StdioWrapper Stdout() io.WriteCloser @@ -554,6 +557,11 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina p = ui.NewBackup(term, gopts.verbosity) } + if opts.DryRun { + repo.SetDryRun() + p.SetDryRun() + } + // use the terminal for stdout/stderr prevStdout, prevStderr := gopts.stdout, gopts.stderr defer func() { @@ -722,7 +730,7 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina // Report finished execution p.Finish(id) - if !gopts.JSON { + if !gopts.JSON && !opts.DryRun { p.P("snapshot %s saved\n", id.Str()) } if !success { diff --git a/cmd/restic/integration_test.go b/cmd/restic/integration_test.go index ad1abcb91..28c32f716 100644 --- a/cmd/restic/integration_test.go +++ b/cmd/restic/integration_test.go @@ -345,6 +345,57 @@ func testBackup(t *testing.T, useFsSnapshot bool) { testRunCheck(t, env.gopts) } +func TestDryRunBackup(t *testing.T) { + env, cleanup := withTestEnvironment(t) + defer cleanup() + + testSetupBackupData(t, env) + opts := BackupOptions{} + dryOpts := BackupOptions{DryRun: true} + + // dry run before first backup + testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, dryOpts, env.gopts) + snapshotIDs := testRunList(t, "snapshots", env.gopts) + rtest.Assert(t, len(snapshotIDs) == 0, + "expected no snapshot, got %v", snapshotIDs) + packIDs := testRunList(t, "packs", env.gopts) + rtest.Assert(t, len(packIDs) == 0, + "expected no data, got %v", snapshotIDs) + indexIDs := testRunList(t, "index", env.gopts) + rtest.Assert(t, len(indexIDs) == 0, + "expected no index, got %v", snapshotIDs) + + // first backup + testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, env.gopts) + snapshotIDs = testRunList(t, "snapshots", env.gopts) + packIDs = testRunList(t, "packs", env.gopts) + indexIDs = testRunList(t, "index", env.gopts) + + // dry run between backups + testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, dryOpts, env.gopts) + snapshotIDsAfter := testRunList(t, "snapshots", env.gopts) + rtest.Equals(t, snapshotIDs, snapshotIDsAfter) + dataIDsAfter := testRunList(t, "packs", env.gopts) + rtest.Equals(t, packIDs, dataIDsAfter) + indexIDsAfter := testRunList(t, "index", env.gopts) + rtest.Equals(t, indexIDs, indexIDsAfter) + + // second backup, implicit incremental + testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, env.gopts) + snapshotIDs = testRunList(t, "snapshots", env.gopts) + packIDs = testRunList(t, "packs", env.gopts) + indexIDs = testRunList(t, "index", env.gopts) + + // another dry run + testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, dryOpts, env.gopts) + snapshotIDsAfter = testRunList(t, "snapshots", env.gopts) + rtest.Equals(t, snapshotIDs, snapshotIDsAfter) + dataIDsAfter = testRunList(t, "packs", env.gopts) + rtest.Equals(t, packIDs, dataIDsAfter) + indexIDsAfter = testRunList(t, "index", env.gopts) + rtest.Equals(t, indexIDs, indexIDsAfter) +} + func TestBackupNonExistingFile(t *testing.T) { env, cleanup := withTestEnvironment(t) defer cleanup() diff --git a/doc/040_backup.rst b/doc/040_backup.rst index 56ea97e97..850952f07 100644 --- a/doc/040_backup.rst +++ b/doc/040_backup.rst @@ -187,6 +187,23 @@ On **Windows**, a file is considered unchanged when its path, size and modification time match, and only ``--force`` has any effect. The other options are recognized but ignored. +Dry Runs +******** + +You can perform a backup in dry run mode to see what would happen without +modifying the repo. + +- ``--dry-run``/``-n`` Report what would be done, without writing to the repository + +Combined with ``--verbose``, you can see a list of changes: + +.. code-block:: console + + $ restic -r /srv/restic-repo backup ~/work --dry-run -vv | grep "added" + modified /plan.txt, saved in 0.000s (9.110 KiB added) + modified /archive.tar.gz, saved in 0.140s (25.542 MiB added) + Would be added to the repo: 25.551 MiB + Excluding Files *************** diff --git a/internal/backend/dryrun/dry_backend.go b/internal/backend/dryrun/dry_backend.go new file mode 100644 index 000000000..2b0735d66 --- /dev/null +++ b/internal/backend/dryrun/dry_backend.go @@ -0,0 +1,79 @@ +package dryrun + +import ( + "context" + "io" + + "github.com/restic/restic/internal/debug" + "github.com/restic/restic/internal/restic" +) + +// Backend passes reads through to an underlying layer and accepts writes, but +// doesn't do anything. Also removes are ignored. +// So in fact, this backend silently ignores all operations that would modify +// the repo and does normal operations else. +// This is used for `backup --dry-run`. +type Backend struct { + b restic.Backend +} + +// statically ensure that RetryBackend implements restic.Backend. +var _ restic.Backend = &Backend{} + +// New returns a new backend that saves all data in a map in memory. +func New(be restic.Backend) *Backend { + b := &Backend{b: be} + debug.Log("created new dry backend") + return b +} + +// Save adds new Data to the backend. +func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { + if err := h.Valid(); err != nil { + return err + } + + debug.Log("faked saving %v bytes at %v", rd.Length(), h) + + // don't save anything, just return ok + return nil +} + +// Remove deletes a file from the backend. +func (be *Backend) Remove(ctx context.Context, h restic.Handle) error { + return nil +} + +// Location returns the location of the backend. +func (be *Backend) Location() string { + return "DRY:" + be.b.Location() +} + +// Delete removes all data in the backend. +func (be *Backend) Delete(ctx context.Context) error { + return nil +} + +func (be *Backend) Close() error { + return be.b.Close() +} + +func (be *Backend) IsNotExist(err error) bool { + return be.b.IsNotExist(err) +} + +func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { + return be.b.List(ctx, t, fn) +} + +func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(io.Reader) error) error { + return be.b.Load(ctx, h, length, offset, fn) +} + +func (be *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) { + return be.b.Stat(ctx, h) +} + +func (be *Backend) Test(ctx context.Context, h restic.Handle) (bool, error) { + return be.b.Test(ctx, h) +} diff --git a/internal/backend/dryrun/dry_backend_test.go b/internal/backend/dryrun/dry_backend_test.go new file mode 100644 index 000000000..c3cabf801 --- /dev/null +++ b/internal/backend/dryrun/dry_backend_test.go @@ -0,0 +1,137 @@ +package dryrun_test + +import ( + "context" + "fmt" + "io" + "io/ioutil" + "sort" + "strings" + "testing" + + "github.com/restic/restic/internal/restic" + + "github.com/restic/restic/internal/backend/dryrun" + "github.com/restic/restic/internal/backend/mem" +) + +// make sure that Backend implements backend.Backend +var _ restic.Backend = &dryrun.Backend{} + +func newBackends() (*dryrun.Backend, restic.Backend) { + m := mem.New() + return dryrun.New(m), m +} + +func TestDry(t *testing.T) { + ctx := context.TODO() + + d, m := newBackends() + // Since the dry backend is a mostly write-only overlay, the standard backend test suite + // won't pass. Instead, perform a series of operations over the backend, testing the state + // at each step. + steps := []struct { + be restic.Backend + op string + fname string + content string + wantErr string + }{ + {d, "loc", "", "DRY:RAM", ""}, + {d, "delete", "", "", ""}, + {d, "stat", "a", "", "not found"}, + {d, "list", "", "", ""}, + {d, "save", "", "", "invalid"}, + {d, "test", "a", "", ""}, + {m, "save", "a", "baz", ""}, // save a directly to the mem backend + {d, "save", "b", "foob", ""}, // b is not saved + {d, "save", "b", "xxx", ""}, // no error as b is not saved + {d, "test", "a", "1", ""}, + {d, "test", "b", "", ""}, + {d, "stat", "", "", "invalid"}, + {d, "stat", "a", "a 3", ""}, + {d, "load", "a", "baz", ""}, + {d, "load", "b", "", "not found"}, + {d, "list", "", "a", ""}, + {d, "remove", "c", "", ""}, + {d, "stat", "b", "", "not found"}, + {d, "list", "", "a", ""}, + {d, "remove", "a", "", ""}, // a is in fact not removed + {d, "list", "", "a", ""}, + {m, "remove", "a", "", ""}, // remove a from the mem backend + {d, "list", "", "", ""}, + {d, "close", "", "", ""}, + {d, "close", "", "", ""}, + } + + for i, step := range steps { + var err error + var boolRes bool + + handle := restic.Handle{Type: restic.PackFile, Name: step.fname} + switch step.op { + case "save": + err = step.be.Save(ctx, handle, restic.NewByteReader([]byte(step.content))) + case "test": + boolRes, err = step.be.Test(ctx, handle) + if boolRes != (step.content != "") { + t.Errorf("%d. Test(%q) = %v, want %v", i, step.fname, boolRes, step.content != "") + } + case "list": + fileList := []string{} + err = step.be.List(ctx, restic.PackFile, func(fi restic.FileInfo) error { + fileList = append(fileList, fi.Name) + return nil + }) + sort.Strings(fileList) + files := strings.Join(fileList, " ") + if files != step.content { + t.Errorf("%d. List = %q, want %q", i, files, step.content) + } + case "loc": + loc := step.be.Location() + if loc != step.content { + t.Errorf("%d. Location = %q, want %q", i, loc, step.content) + } + case "delete": + err = step.be.Delete(ctx) + case "remove": + err = step.be.Remove(ctx, handle) + case "stat": + var fi restic.FileInfo + fi, err = step.be.Stat(ctx, handle) + if err == nil { + fis := fmt.Sprintf("%s %d", fi.Name, fi.Size) + if fis != step.content { + t.Errorf("%d. Stat = %q, want %q", i, fis, step.content) + } + } + case "load": + data := "" + err = step.be.Load(ctx, handle, 100, 0, func(rd io.Reader) error { + buf, err := ioutil.ReadAll(rd) + data = string(buf) + return err + }) + if data != step.content { + t.Errorf("%d. Load = %q, want %q", i, data, step.content) + } + case "close": + err = step.be.Close() + default: + t.Fatalf("%d. unknown step operation %q", i, step.op) + } + if step.wantErr != "" { + if err == nil { + t.Errorf("%d. %s error = nil, want %q", i, step.op, step.wantErr) + } else if !strings.Contains(err.Error(), step.wantErr) { + t.Errorf("%d. %s error = %q, doesn't contain %q", i, step.op, err, step.wantErr) + } else if step.wantErr == "not found" && !step.be.IsNotExist(err) { + t.Errorf("%d. IsNotExist(%s error) = false, want true", i, step.op) + } + + } else if err != nil { + t.Errorf("%d. %s error = %q, want nil", i, step.op, err) + } + } +} diff --git a/internal/repository/repository.go b/internal/repository/repository.go index a13a0d784..4344fff2d 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -10,6 +10,7 @@ import ( "sync" "github.com/restic/chunker" + "github.com/restic/restic/internal/backend/dryrun" "github.com/restic/restic/internal/cache" "github.com/restic/restic/internal/crypto" "github.com/restic/restic/internal/debug" @@ -72,6 +73,11 @@ func (r *Repository) UseCache(c *cache.Cache) { r.be = c.Wrap(r.be) } +// SetDryRun sets the repo backend into dry-run mode. +func (r *Repository) SetDryRun() { + r.be = dryrun.New(r.be) +} + // PrefixLength returns the number of bytes required so that all prefixes of // all IDs of type t are unique. func (r *Repository) PrefixLength(ctx context.Context, t restic.FileType) (int, error) { diff --git a/internal/ui/backup.go b/internal/ui/backup.go index 313eebf7b..59f85487f 100644 --- a/internal/ui/backup.go +++ b/internal/ui/backup.go @@ -35,6 +35,7 @@ type Backup struct { start time.Time totalBytes uint64 + dry bool // true if writes are faked totalCh chan counter processedCh chan counter @@ -385,7 +386,11 @@ func (b *Backup) Finish(snapshotID restic.ID) { b.P("Dirs: %5d new, %5d changed, %5d unmodified\n", b.summary.Dirs.New, b.summary.Dirs.Changed, b.summary.Dirs.Unchanged) b.V("Data Blobs: %5d new\n", b.summary.ItemStats.DataBlobs) b.V("Tree Blobs: %5d new\n", b.summary.ItemStats.TreeBlobs) - b.P("Added to the repo: %-5s\n", formatBytes(b.summary.ItemStats.DataSize+b.summary.ItemStats.TreeSize)) + verb := "Added" + if b.dry { + verb = "Would add" + } + b.P("%s to the repo: %-5s\n", verb, formatBytes(b.summary.ItemStats.DataSize+b.summary.ItemStats.TreeSize)) b.P("\n") b.P("processed %v files, %v in %s", b.summary.Files.New+b.summary.Files.Changed+b.summary.Files.Unchanged, @@ -399,3 +404,7 @@ func (b *Backup) Finish(snapshotID restic.ID) { func (b *Backup) SetMinUpdatePause(d time.Duration) { b.MinUpdatePause = d } + +func (b *Backup) SetDryRun() { + b.dry = true +} diff --git a/internal/ui/json/backup.go b/internal/ui/json/backup.go index b6a2ed03d..70bc23801 100644 --- a/internal/ui/json/backup.go +++ b/internal/ui/json/backup.go @@ -34,6 +34,7 @@ type Backup struct { term *termstatus.Terminal v uint start time.Time + dry bool totalBytes uint64 @@ -403,6 +404,7 @@ func (b *Backup) Finish(snapshotID restic.ID) { TotalBytesProcessed: b.summary.ProcessedBytes, TotalDuration: time.Since(b.start).Seconds(), SnapshotID: snapshotID.Str(), + DryRun: b.dry, }) } @@ -412,6 +414,11 @@ func (b *Backup) SetMinUpdatePause(d time.Duration) { b.MinUpdatePause = d } +// SetDryRun marks the backup as a "dry run". +func (b *Backup) SetDryRun() { + b.dry = true +} + type statusUpdate struct { MessageType string `json:"message_type"` // "status" SecondsElapsed uint64 `json:"seconds_elapsed,omitempty"` @@ -457,4 +464,5 @@ type summaryOutput struct { TotalBytesProcessed uint64 `json:"total_bytes_processed"` TotalDuration float64 `json:"total_duration"` // in seconds SnapshotID string `json:"snapshot_id"` + DryRun bool `json:"dry_run,omitempty"` }