From 3026baea0762a2245a4d5c1743188ca9c6b1ba95 Mon Sep 17 00:00:00 2001 From: Gabriel Kabbe Date: Mon, 27 Nov 2023 21:04:24 +0100 Subject: [PATCH 01/11] rewrite: Add structs for tracking metadata changes Adds * snapshotMetadataArgs, which holds the new metadata as strings parsed from the command line * snapshotMetadata, which holds the new metadata converted to the correct types --- cmd/restic/cmd_rewrite.go | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/cmd/restic/cmd_rewrite.go b/cmd/restic/cmd_rewrite.go index 2d5c5716d..183df9a09 100644 --- a/cmd/restic/cmd_rewrite.go +++ b/cmd/restic/cmd_rewrite.go @@ -3,6 +3,7 @@ package main import ( "context" "fmt" + "time" "github.com/spf13/cobra" "golang.org/x/sync/errgroup" @@ -46,16 +47,28 @@ Exit status is 0 if the command was successful, and non-zero if there was any er }, } +type snapshotMetadata struct { + Hostname string + Time *time.Time +} + +type SnapshotMetadataArgs struct { + Hostname string + Time string +} + // RewriteOptions collects all options for the rewrite command. type RewriteOptions struct { - Forget bool - DryRun bool + Forget bool + DryRun bool + Metadata *SnapshotMetadataArgs restic.SnapshotFilter excludePatternOptions } var rewriteOptions RewriteOptions +var metadataOptions SnapshotMetadataArgs func init() { cmdRoot.AddCommand(cmdRewrite) From da1704b2d516db28b974d5bd066ac445d541aada Mon Sep 17 00:00:00 2001 From: Gabriel Kabbe Date: Mon, 27 Nov 2023 21:18:29 +0100 Subject: [PATCH 02/11] rewrite: Add tests Pass nil instead of metadata to existing tests --- cmd/restic/cmd_rewrite_integration_test.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/cmd/restic/cmd_rewrite_integration_test.go b/cmd/restic/cmd_rewrite_integration_test.go index e6007973b..7d4226691 100644 --- a/cmd/restic/cmd_rewrite_integration_test.go +++ b/cmd/restic/cmd_rewrite_integration_test.go @@ -9,12 +9,13 @@ import ( rtest "github.com/restic/restic/internal/test" ) -func testRunRewriteExclude(t testing.TB, gopts GlobalOptions, excludes []string, forget bool) { +func testRunRewriteExclude(t testing.TB, gopts GlobalOptions, excludes []string, forget bool, metadata *SnapshotMetadataArgs) { opts := RewriteOptions{ excludePatternOptions: excludePatternOptions{ Excludes: excludes, }, - Forget: forget, + Forget: forget, + Metadata: metadata, } rtest.OK(t, runRewrite(context.TODO(), opts, gopts, nil)) @@ -38,7 +39,7 @@ func TestRewrite(t *testing.T) { createBasicRewriteRepo(t, env) // exclude some data - testRunRewriteExclude(t, env.gopts, []string{"3"}, false) + testRunRewriteExclude(t, env.gopts, []string{"3"}, false, nil) snapshotIDs := testRunList(t, "snapshots", env.gopts) rtest.Assert(t, len(snapshotIDs) == 2, "expected two snapshots, got %v", snapshotIDs) testRunCheck(t, env.gopts) @@ -50,7 +51,7 @@ func TestRewriteUnchanged(t *testing.T) { snapshotID := createBasicRewriteRepo(t, env) // use an exclude that will not exclude anything - testRunRewriteExclude(t, env.gopts, []string{"3dflkhjgdflhkjetrlkhjgfdlhkj"}, false) + testRunRewriteExclude(t, env.gopts, []string{"3dflkhjgdflhkjetrlkhjgfdlhkj"}, false, nil) newSnapshotIDs := testRunList(t, "snapshots", env.gopts) rtest.Assert(t, len(newSnapshotIDs) == 1, "expected one snapshot, got %v", newSnapshotIDs) rtest.Assert(t, snapshotID == newSnapshotIDs[0], "snapshot id changed unexpectedly") @@ -63,7 +64,7 @@ func TestRewriteReplace(t *testing.T) { snapshotID := createBasicRewriteRepo(t, env) // exclude some data - testRunRewriteExclude(t, env.gopts, []string{"3"}, true) + testRunRewriteExclude(t, env.gopts, []string{"3"}, true, nil) newSnapshotIDs := testRunList(t, "snapshots", env.gopts) rtest.Assert(t, len(newSnapshotIDs) == 1, "expected one snapshot, got %v", newSnapshotIDs) rtest.Assert(t, snapshotID != newSnapshotIDs[0], "snapshot id should have changed") From 7bf38b6c50b641bfaaf913ff0732e84663401656 Mon Sep 17 00:00:00 2001 From: Gabriel Kabbe Date: Mon, 27 Nov 2023 21:29:20 +0100 Subject: [PATCH 03/11] rewrite: Add test TestRewriteMetadata --- cmd/restic/cmd_rewrite_integration_test.go | 35 ++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/cmd/restic/cmd_rewrite_integration_test.go b/cmd/restic/cmd_rewrite_integration_test.go index 7d4226691..5cb205262 100644 --- a/cmd/restic/cmd_rewrite_integration_test.go +++ b/cmd/restic/cmd_rewrite_integration_test.go @@ -72,3 +72,38 @@ func TestRewriteReplace(t *testing.T) { testRunPrune(t, env.gopts, PruneOptions{MaxUnused: "0"}) testRunCheck(t, env.gopts) } + +func testRewriteMetadata(t *testing.T, metadata SnapshotMetadataArgs) { + env, cleanup := withTestEnvironment(t) + env.gopts.backendTestHook = nil + defer cleanup() + createBasicRewriteRepo(t, env) + repo, _ := OpenRepository(context.TODO(), env.gopts) + + testRunRewriteExclude(t, env.gopts, []string{}, true, &metadata) + + snapshots := FindFilteredSnapshots(context.TODO(), repo, repo, &restic.SnapshotFilter{}, []string{}) + + newSnapshot := <-snapshots + + if metadata.Time != "" { + rtest.Assert(t, newSnapshot.Time.Format(TimeFormat) == metadata.Time, "New snapshot should have time %s", metadata.Time) + } + + if metadata.Hostname != "" { + rtest.Assert(t, newSnapshot.Hostname == metadata.Hostname, "New snapshot should have host %s", metadata.Hostname) + } +} + +func TestRewriteMetadata(t *testing.T) { + newHost := "new host" + newTime := "1999-01-01 11:11:11" + + for _, metadata := range []SnapshotMetadataArgs{ + {Hostname: "", Time: newTime}, + {Hostname: newHost, Time: ""}, + {Hostname: newHost, Time: newTime}, + } { + testRewriteMetadata(t, metadata) + } +} From a02d8d75c26148498a618802fa620d648fdbe33f Mon Sep 17 00:00:00 2001 From: Gabriel Kabbe Date: Mon, 27 Nov 2023 21:34:57 +0100 Subject: [PATCH 04/11] rewrite: Implement rewriting metadata --- cmd/restic/cmd_repair_snapshots.go | 2 +- cmd/restic/cmd_rewrite.go | 58 ++++++++++++++++++++-- cmd/restic/cmd_rewrite_integration_test.go | 6 +-- 3 files changed, 57 insertions(+), 9 deletions(-) diff --git a/cmd/restic/cmd_repair_snapshots.go b/cmd/restic/cmd_repair_snapshots.go index 82231518b..786097132 100644 --- a/cmd/restic/cmd_repair_snapshots.go +++ b/cmd/restic/cmd_repair_snapshots.go @@ -148,7 +148,7 @@ func runRepairSnapshots(ctx context.Context, gopts GlobalOptions, opts RepairOpt changed, err := filterAndReplaceSnapshot(ctx, repo, sn, func(ctx context.Context, sn *restic.Snapshot) (restic.ID, error) { return rewriter.RewriteTree(ctx, repo, "/", *sn.Tree) - }, opts.DryRun, opts.Forget, "repaired") + }, opts.DryRun, opts.Forget, nil, "repaired") if err != nil { return errors.Fatalf("unable to rewrite snapshot ID %q: %v", sn.ID().Str(), err) } diff --git a/cmd/restic/cmd_rewrite.go b/cmd/restic/cmd_rewrite.go index 183df9a09..1a6f39d04 100644 --- a/cmd/restic/cmd_rewrite.go +++ b/cmd/restic/cmd_rewrite.go @@ -57,6 +57,25 @@ type SnapshotMetadataArgs struct { Time string } +func (sma SnapshotMetadataArgs) convert() (*snapshotMetadata, error) { + if sma.Time == "" && sma.Hostname == "" { + return nil, nil + } + + var timeStamp *time.Time + if sma.Time != "" { + t, err := time.ParseInLocation(TimeFormat, sma.Time, time.Local) + if err != nil { + return nil, errors.Fatalf("error in time option: %v\n", err) + } + timeStamp = &t + } else { + timeStamp = nil + } + return &snapshotMetadata{Hostname: sma.Hostname, Time: timeStamp}, nil + +} + // RewriteOptions collects all options for the rewrite command. type RewriteOptions struct { Forget bool @@ -76,6 +95,10 @@ func init() { f := cmdRewrite.Flags() f.BoolVarP(&rewriteOptions.Forget, "forget", "", false, "remove original snapshots after creating new ones") f.BoolVarP(&rewriteOptions.DryRun, "dry-run", "n", false, "do not do anything, just print what would be done") + f.StringVar(&metadataOptions.Hostname, "new-host", "", "rewrite hostname") + f.StringVar(&metadataOptions.Time, "new-time", "", "rewrite time of the backup") + + rewriteOptions.Metadata = &metadataOptions initMultiSnapshotFilter(f, &rewriteOptions.SnapshotFilter, true) initExcludePatternOptions(f, &rewriteOptions.excludePatternOptions) @@ -91,6 +114,12 @@ func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *resti return false, err } + metadata, err := opts.Metadata.convert() + + if err != nil { + return false, err + } + selectByName := func(nodepath string) bool { for _, reject := range rejectByNameFuncs { if reject(nodepath) { @@ -114,10 +143,10 @@ func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *resti return filterAndReplaceSnapshot(ctx, repo, sn, func(ctx context.Context, sn *restic.Snapshot) (restic.ID, error) { return rewriter.RewriteTree(ctx, repo, "/", *sn.Tree) - }, opts.DryRun, opts.Forget, "rewrite") + }, opts.DryRun, opts.Forget, metadata, "rewrite") } -func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *restic.Snapshot, filter func(ctx context.Context, sn *restic.Snapshot) (restic.ID, error), dryRun bool, forget bool, addTag string) (bool, error) { +func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *restic.Snapshot, filter func(ctx context.Context, sn *restic.Snapshot) (restic.ID, error), dryRun bool, forget bool, metadata *snapshotMetadata, addTag string) (bool, error) { wg, wgCtx := errgroup.WithContext(ctx) repo.StartPackUploader(wgCtx, wg) @@ -151,7 +180,7 @@ func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *r return true, nil } - if filteredTree == *sn.Tree { + if filteredTree == *sn.Tree && metadata == nil { debug.Log("Snapshot %v not modified", sn) return false, nil } @@ -164,6 +193,14 @@ func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *r Verbosef("would remove old snapshot\n") } + if metadata != nil && metadata.Time != nil { + Verbosef("would set time to %s\n", metadata.Time) + } + + if metadata != nil && metadata.Hostname != "" { + Verbosef("would set time to %s\n", metadata.Hostname) + } + return true, nil } @@ -175,6 +212,17 @@ func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *r sn.AddTags([]string{addTag}) } + if metadata != nil && metadata.Time != nil { + Verbosef("Setting time to %s\n", *metadata.Time) + sn.Time = *metadata.Time + } + + if metadata != nil && metadata.Hostname != "" { + Verbosef("Setting host to %s\n", metadata.Hostname) + sn.Hostname = metadata.Hostname + + } + // Save the new snapshot. id, err := restic.SaveSnapshot(ctx, repo, sn) if err != nil { @@ -194,8 +242,8 @@ func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *r } func runRewrite(ctx context.Context, opts RewriteOptions, gopts GlobalOptions, args []string) error { - if opts.excludePatternOptions.Empty() { - return errors.Fatal("Nothing to do: no excludes provided") + if opts.excludePatternOptions.Empty() && opts.Metadata == nil { + return errors.Fatal("Nothing to do: no excludes provided and no new metadata provided") } repo, err := OpenRepository(ctx, gopts) diff --git a/cmd/restic/cmd_rewrite_integration_test.go b/cmd/restic/cmd_rewrite_integration_test.go index 5cb205262..d52679f86 100644 --- a/cmd/restic/cmd_rewrite_integration_test.go +++ b/cmd/restic/cmd_rewrite_integration_test.go @@ -39,7 +39,7 @@ func TestRewrite(t *testing.T) { createBasicRewriteRepo(t, env) // exclude some data - testRunRewriteExclude(t, env.gopts, []string{"3"}, false, nil) + testRunRewriteExclude(t, env.gopts, []string{"3"}, false, &SnapshotMetadataArgs{Hostname: "", Time: ""}) snapshotIDs := testRunList(t, "snapshots", env.gopts) rtest.Assert(t, len(snapshotIDs) == 2, "expected two snapshots, got %v", snapshotIDs) testRunCheck(t, env.gopts) @@ -51,7 +51,7 @@ func TestRewriteUnchanged(t *testing.T) { snapshotID := createBasicRewriteRepo(t, env) // use an exclude that will not exclude anything - testRunRewriteExclude(t, env.gopts, []string{"3dflkhjgdflhkjetrlkhjgfdlhkj"}, false, nil) + testRunRewriteExclude(t, env.gopts, []string{"3dflkhjgdflhkjetrlkhjgfdlhkj"}, false, &SnapshotMetadataArgs{Hostname: "", Time: ""}) newSnapshotIDs := testRunList(t, "snapshots", env.gopts) rtest.Assert(t, len(newSnapshotIDs) == 1, "expected one snapshot, got %v", newSnapshotIDs) rtest.Assert(t, snapshotID == newSnapshotIDs[0], "snapshot id changed unexpectedly") @@ -64,7 +64,7 @@ func TestRewriteReplace(t *testing.T) { snapshotID := createBasicRewriteRepo(t, env) // exclude some data - testRunRewriteExclude(t, env.gopts, []string{"3"}, true, nil) + testRunRewriteExclude(t, env.gopts, []string{"3"}, true, &SnapshotMetadataArgs{Hostname: "", Time: ""}) newSnapshotIDs := testRunList(t, "snapshots", env.gopts) rtest.Assert(t, len(newSnapshotIDs) == 1, "expected one snapshot, got %v", newSnapshotIDs) rtest.Assert(t, snapshotID != newSnapshotIDs[0], "snapshot id should have changed") From 004520a238925e1bb30e417f694911042b520b6d Mon Sep 17 00:00:00 2001 From: Gabriel Kabbe Date: Mon, 27 Nov 2023 21:35:13 +0100 Subject: [PATCH 05/11] rewrite: Add changelog --- changelog/unreleased/pull-4573 | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 changelog/unreleased/pull-4573 diff --git a/changelog/unreleased/pull-4573 b/changelog/unreleased/pull-4573 new file mode 100644 index 000000000..bd5c2c423 --- /dev/null +++ b/changelog/unreleased/pull-4573 @@ -0,0 +1,5 @@ +Enhancement: Add `--new-host` and `--new-time` options to `rewrite` command + +`restic rewrite` now allows rewriting the host and / or time metadata of a snapshot. + +https://github.com/restic/restic/pull/4573 From 7de97d7480596fb52e3047e5ee3056c4904b9030 Mon Sep 17 00:00:00 2001 From: Gabriel Kabbe Date: Mon, 27 Nov 2023 22:13:02 +0100 Subject: [PATCH 06/11] rewrite: Add documentation --- doc/045_working_with_repos.rst | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/doc/045_working_with_repos.rst b/doc/045_working_with_repos.rst index 77c7a15b5..ce1cb8754 100644 --- a/doc/045_working_with_repos.rst +++ b/doc/045_working_with_repos.rst @@ -234,6 +234,27 @@ modifying the repository. Instead restic will only print the actions it would perform. +Modifying metadata of snapshots +=============================== + +Sometimes it may be desirable to change the metadata of an existing snapshot. +Currently, rewriting the hostname and the time of the backup is supported. +This is possible using the ``rewrite`` command with the option ``--new-host`` followed by the desired new hostname or the option ``--new-time`` followed by the desired new timestamp. + +.. code-block:: console + $ restic rewrite --new-host newhost --new-time "1999-01-01 11:11:11" + + repository b7dbade3 opened (version 2, compression level auto) + [0:00] 100.00% 1 / 1 index files loaded + + snapshot 8ed674f4 of [/path/to/abc.txt] at 2023-11-27 21:57:52.439139291 +0100 CET) + Setting time to 1999-01-01 11:11:11 +0100 CET + Setting host to newhost + saved new snapshot c05da643 + + modified 1 snapshots + + .. _checking-integrity: Checking integrity and consistency From 893d0d6325ec8b4edb64d168c771c8256dcfaf80 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 24 Dec 2023 14:36:27 +0100 Subject: [PATCH 07/11] rewrite: cleanup new metadata options and fix no parameters check --- cmd/restic/cmd_rewrite.go | 53 ++++++++++++---------- cmd/restic/cmd_rewrite_integration_test.go | 16 +++---- 2 files changed, 36 insertions(+), 33 deletions(-) diff --git a/cmd/restic/cmd_rewrite.go b/cmd/restic/cmd_rewrite.go index 1a6f39d04..553a9da91 100644 --- a/cmd/restic/cmd_rewrite.go +++ b/cmd/restic/cmd_rewrite.go @@ -52,12 +52,16 @@ type snapshotMetadata struct { Time *time.Time } -type SnapshotMetadataArgs struct { +type snapshotMetadataArgs struct { Hostname string Time string } -func (sma SnapshotMetadataArgs) convert() (*snapshotMetadata, error) { +func (sma snapshotMetadataArgs) empty() bool { + return sma.Hostname == "" && sma.Time == "" +} + +func (sma snapshotMetadataArgs) convert() (*snapshotMetadata, error) { if sma.Time == "" && sma.Hostname == "" { return nil, nil } @@ -78,16 +82,15 @@ func (sma SnapshotMetadataArgs) convert() (*snapshotMetadata, error) { // RewriteOptions collects all options for the rewrite command. type RewriteOptions struct { - Forget bool - DryRun bool - Metadata *SnapshotMetadataArgs + Forget bool + DryRun bool + metadata snapshotMetadataArgs restic.SnapshotFilter excludePatternOptions } var rewriteOptions RewriteOptions -var metadataOptions SnapshotMetadataArgs func init() { cmdRoot.AddCommand(cmdRewrite) @@ -95,15 +98,15 @@ func init() { f := cmdRewrite.Flags() f.BoolVarP(&rewriteOptions.Forget, "forget", "", false, "remove original snapshots after creating new ones") f.BoolVarP(&rewriteOptions.DryRun, "dry-run", "n", false, "do not do anything, just print what would be done") - f.StringVar(&metadataOptions.Hostname, "new-host", "", "rewrite hostname") - f.StringVar(&metadataOptions.Time, "new-time", "", "rewrite time of the backup") - - rewriteOptions.Metadata = &metadataOptions + f.StringVar(&rewriteOptions.metadata.Hostname, "new-host", "", "replace hostname") + f.StringVar(&rewriteOptions.metadata.Time, "new-time", "", "replace time of the backup") initMultiSnapshotFilter(f, &rewriteOptions.SnapshotFilter, true) initExcludePatternOptions(f, &rewriteOptions.excludePatternOptions) } +type rewriteFilterFunc func(ctx context.Context, sn *restic.Snapshot) (restic.ID, error) + func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *restic.Snapshot, opts RewriteOptions) (bool, error) { if sn.Tree == nil { return false, errors.Errorf("snapshot %v has nil tree", sn.ID().Str()) @@ -114,7 +117,7 @@ func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *resti return false, err } - metadata, err := opts.Metadata.convert() + metadata, err := opts.metadata.convert() if err != nil { return false, err @@ -146,7 +149,8 @@ func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *resti }, opts.DryRun, opts.Forget, metadata, "rewrite") } -func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *restic.Snapshot, filter func(ctx context.Context, sn *restic.Snapshot) (restic.ID, error), dryRun bool, forget bool, metadata *snapshotMetadata, addTag string) (bool, error) { +func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *restic.Snapshot, + filter rewriteFilterFunc, dryRun bool, forget bool, newMetadata *snapshotMetadata, addTag string) (bool, error) { wg, wgCtx := errgroup.WithContext(ctx) repo.StartPackUploader(wgCtx, wg) @@ -180,7 +184,7 @@ func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *r return true, nil } - if filteredTree == *sn.Tree && metadata == nil { + if filteredTree == *sn.Tree && newMetadata == nil { debug.Log("Snapshot %v not modified", sn) return false, nil } @@ -193,12 +197,12 @@ func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *r Verbosef("would remove old snapshot\n") } - if metadata != nil && metadata.Time != nil { - Verbosef("would set time to %s\n", metadata.Time) + if newMetadata != nil && newMetadata.Time != nil { + Verbosef("would set time to %s\n", newMetadata.Time) } - if metadata != nil && metadata.Hostname != "" { - Verbosef("would set time to %s\n", metadata.Hostname) + if newMetadata != nil && newMetadata.Hostname != "" { + Verbosef("would set time to %s\n", newMetadata.Hostname) } return true, nil @@ -212,15 +216,14 @@ func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *r sn.AddTags([]string{addTag}) } - if metadata != nil && metadata.Time != nil { - Verbosef("Setting time to %s\n", *metadata.Time) - sn.Time = *metadata.Time + if newMetadata != nil && newMetadata.Time != nil { + Verbosef("setting time to %s\n", *newMetadata.Time) + sn.Time = *newMetadata.Time } - if metadata != nil && metadata.Hostname != "" { - Verbosef("Setting host to %s\n", metadata.Hostname) - sn.Hostname = metadata.Hostname - + if newMetadata != nil && newMetadata.Hostname != "" { + Verbosef("setting host to %s\n", newMetadata.Hostname) + sn.Hostname = newMetadata.Hostname } // Save the new snapshot. @@ -242,7 +245,7 @@ func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *r } func runRewrite(ctx context.Context, opts RewriteOptions, gopts GlobalOptions, args []string) error { - if opts.excludePatternOptions.Empty() && opts.Metadata == nil { + if opts.excludePatternOptions.Empty() && opts.metadata.empty() { return errors.Fatal("Nothing to do: no excludes provided and no new metadata provided") } diff --git a/cmd/restic/cmd_rewrite_integration_test.go b/cmd/restic/cmd_rewrite_integration_test.go index d52679f86..b61ae0bb1 100644 --- a/cmd/restic/cmd_rewrite_integration_test.go +++ b/cmd/restic/cmd_rewrite_integration_test.go @@ -9,13 +9,13 @@ import ( rtest "github.com/restic/restic/internal/test" ) -func testRunRewriteExclude(t testing.TB, gopts GlobalOptions, excludes []string, forget bool, metadata *SnapshotMetadataArgs) { +func testRunRewriteExclude(t testing.TB, gopts GlobalOptions, excludes []string, forget bool, metadata snapshotMetadataArgs) { opts := RewriteOptions{ excludePatternOptions: excludePatternOptions{ Excludes: excludes, }, Forget: forget, - Metadata: metadata, + metadata: metadata, } rtest.OK(t, runRewrite(context.TODO(), opts, gopts, nil)) @@ -39,7 +39,7 @@ func TestRewrite(t *testing.T) { createBasicRewriteRepo(t, env) // exclude some data - testRunRewriteExclude(t, env.gopts, []string{"3"}, false, &SnapshotMetadataArgs{Hostname: "", Time: ""}) + testRunRewriteExclude(t, env.gopts, []string{"3"}, false, snapshotMetadataArgs{Hostname: "", Time: ""}) snapshotIDs := testRunList(t, "snapshots", env.gopts) rtest.Assert(t, len(snapshotIDs) == 2, "expected two snapshots, got %v", snapshotIDs) testRunCheck(t, env.gopts) @@ -51,7 +51,7 @@ func TestRewriteUnchanged(t *testing.T) { snapshotID := createBasicRewriteRepo(t, env) // use an exclude that will not exclude anything - testRunRewriteExclude(t, env.gopts, []string{"3dflkhjgdflhkjetrlkhjgfdlhkj"}, false, &SnapshotMetadataArgs{Hostname: "", Time: ""}) + testRunRewriteExclude(t, env.gopts, []string{"3dflkhjgdflhkjetrlkhjgfdlhkj"}, false, snapshotMetadataArgs{Hostname: "", Time: ""}) newSnapshotIDs := testRunList(t, "snapshots", env.gopts) rtest.Assert(t, len(newSnapshotIDs) == 1, "expected one snapshot, got %v", newSnapshotIDs) rtest.Assert(t, snapshotID == newSnapshotIDs[0], "snapshot id changed unexpectedly") @@ -64,7 +64,7 @@ func TestRewriteReplace(t *testing.T) { snapshotID := createBasicRewriteRepo(t, env) // exclude some data - testRunRewriteExclude(t, env.gopts, []string{"3"}, true, &SnapshotMetadataArgs{Hostname: "", Time: ""}) + testRunRewriteExclude(t, env.gopts, []string{"3"}, true, snapshotMetadataArgs{Hostname: "", Time: ""}) newSnapshotIDs := testRunList(t, "snapshots", env.gopts) rtest.Assert(t, len(newSnapshotIDs) == 1, "expected one snapshot, got %v", newSnapshotIDs) rtest.Assert(t, snapshotID != newSnapshotIDs[0], "snapshot id should have changed") @@ -73,14 +73,14 @@ func TestRewriteReplace(t *testing.T) { testRunCheck(t, env.gopts) } -func testRewriteMetadata(t *testing.T, metadata SnapshotMetadataArgs) { +func testRewriteMetadata(t *testing.T, metadata snapshotMetadataArgs) { env, cleanup := withTestEnvironment(t) env.gopts.backendTestHook = nil defer cleanup() createBasicRewriteRepo(t, env) repo, _ := OpenRepository(context.TODO(), env.gopts) - testRunRewriteExclude(t, env.gopts, []string{}, true, &metadata) + testRunRewriteExclude(t, env.gopts, []string{}, true, metadata) snapshots := FindFilteredSnapshots(context.TODO(), repo, repo, &restic.SnapshotFilter{}, []string{}) @@ -99,7 +99,7 @@ func TestRewriteMetadata(t *testing.T) { newHost := "new host" newTime := "1999-01-01 11:11:11" - for _, metadata := range []SnapshotMetadataArgs{ + for _, metadata := range []snapshotMetadataArgs{ {Hostname: "", Time: newTime}, {Hostname: newHost, Time: ""}, {Hostname: newHost, Time: newTime}, From 2730d05fcef9abf61a17c2016408b98fd591835d Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 24 Dec 2023 14:40:12 +0100 Subject: [PATCH 08/11] rewrite: Don't walk snapshot content if only metadata is modified --- cmd/restic/cmd_rewrite.go | 48 +++++++++++++++++++++++---------------- 1 file changed, 29 insertions(+), 19 deletions(-) diff --git a/cmd/restic/cmd_rewrite.go b/cmd/restic/cmd_rewrite.go index 553a9da91..c83a2ba87 100644 --- a/cmd/restic/cmd_rewrite.go +++ b/cmd/restic/cmd_rewrite.go @@ -123,30 +123,40 @@ func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *resti return false, err } - selectByName := func(nodepath string) bool { - for _, reject := range rejectByNameFuncs { - if reject(nodepath) { - return false + var filter rewriteFilterFunc + + if len(rejectByNameFuncs) > 0 { + selectByName := func(nodepath string) bool { + for _, reject := range rejectByNameFuncs { + if reject(nodepath) { + return false + } } + return true + } + + rewriter := walker.NewTreeRewriter(walker.RewriteOpts{ + RewriteNode: func(node *restic.Node, path string) *restic.Node { + if selectByName(path) { + return node + } + Verbosef(fmt.Sprintf("excluding %s\n", path)) + return nil + }, + DisableNodeCache: true, + }) + + filter = func(ctx context.Context, sn *restic.Snapshot) (restic.ID, error) { + return rewriter.RewriteTree(ctx, repo, "/", *sn.Tree) + } + } else { + filter = func(ctx context.Context, sn *restic.Snapshot) (restic.ID, error) { + return *sn.Tree, nil } - return true } - rewriter := walker.NewTreeRewriter(walker.RewriteOpts{ - RewriteNode: func(node *restic.Node, path string) *restic.Node { - if selectByName(path) { - return node - } - Verbosef(fmt.Sprintf("excluding %s\n", path)) - return nil - }, - DisableNodeCache: true, - }) - return filterAndReplaceSnapshot(ctx, repo, sn, - func(ctx context.Context, sn *restic.Snapshot) (restic.ID, error) { - return rewriter.RewriteTree(ctx, repo, "/", *sn.Tree) - }, opts.DryRun, opts.Forget, metadata, "rewrite") + filter, opts.DryRun, opts.Forget, metadata, "rewrite") } func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *restic.Snapshot, From c31f5f986ca3e78dd940c9f89a811a5f982a42e4 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 24 Dec 2023 15:03:45 +0100 Subject: [PATCH 09/11] rewrite: Minor cleanups --- cmd/restic/cmd_rewrite.go | 15 ++++++--------- cmd/restic/cmd_rewrite_integration_test.go | 2 +- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/cmd/restic/cmd_rewrite.go b/cmd/restic/cmd_rewrite.go index c83a2ba87..afd80aca1 100644 --- a/cmd/restic/cmd_rewrite.go +++ b/cmd/restic/cmd_rewrite.go @@ -62,7 +62,7 @@ func (sma snapshotMetadataArgs) empty() bool { } func (sma snapshotMetadataArgs) convert() (*snapshotMetadata, error) { - if sma.Time == "" && sma.Hostname == "" { + if sma.empty() { return nil, nil } @@ -73,11 +73,8 @@ func (sma snapshotMetadataArgs) convert() (*snapshotMetadata, error) { return nil, errors.Fatalf("error in time option: %v\n", err) } timeStamp = &t - } else { - timeStamp = nil } return &snapshotMetadata{Hostname: sma.Hostname, Time: timeStamp}, nil - } // RewriteOptions collects all options for the rewrite command. @@ -85,7 +82,7 @@ type RewriteOptions struct { Forget bool DryRun bool - metadata snapshotMetadataArgs + Metadata snapshotMetadataArgs restic.SnapshotFilter excludePatternOptions } @@ -98,8 +95,8 @@ func init() { f := cmdRewrite.Flags() f.BoolVarP(&rewriteOptions.Forget, "forget", "", false, "remove original snapshots after creating new ones") f.BoolVarP(&rewriteOptions.DryRun, "dry-run", "n", false, "do not do anything, just print what would be done") - f.StringVar(&rewriteOptions.metadata.Hostname, "new-host", "", "replace hostname") - f.StringVar(&rewriteOptions.metadata.Time, "new-time", "", "replace time of the backup") + f.StringVar(&rewriteOptions.Metadata.Hostname, "new-host", "", "replace hostname") + f.StringVar(&rewriteOptions.Metadata.Time, "new-time", "", "replace time of the backup") initMultiSnapshotFilter(f, &rewriteOptions.SnapshotFilter, true) initExcludePatternOptions(f, &rewriteOptions.excludePatternOptions) @@ -117,7 +114,7 @@ func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *resti return false, err } - metadata, err := opts.metadata.convert() + metadata, err := opts.Metadata.convert() if err != nil { return false, err @@ -255,7 +252,7 @@ func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *r } func runRewrite(ctx context.Context, opts RewriteOptions, gopts GlobalOptions, args []string) error { - if opts.excludePatternOptions.Empty() && opts.metadata.empty() { + if opts.excludePatternOptions.Empty() && opts.Metadata.empty() { return errors.Fatal("Nothing to do: no excludes provided and no new metadata provided") } diff --git a/cmd/restic/cmd_rewrite_integration_test.go b/cmd/restic/cmd_rewrite_integration_test.go index b61ae0bb1..3f8e29c94 100644 --- a/cmd/restic/cmd_rewrite_integration_test.go +++ b/cmd/restic/cmd_rewrite_integration_test.go @@ -15,7 +15,7 @@ func testRunRewriteExclude(t testing.TB, gopts GlobalOptions, excludes []string, Excludes: excludes, }, Forget: forget, - metadata: metadata, + Metadata: metadata, } rtest.OK(t, runRewrite(context.TODO(), opts, gopts, nil)) From 649a6409eeaac45b7def956434b0e24916a3fca6 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 24 Dec 2023 15:04:05 +0100 Subject: [PATCH 10/11] rewrite: cleanup tests --- cmd/restic/cmd_rewrite_integration_test.go | 14 ++++++-------- internal/restic/testing.go | 19 +++++++++++++++++++ internal/restic/testing_test.go | 21 +-------------------- 3 files changed, 26 insertions(+), 28 deletions(-) diff --git a/cmd/restic/cmd_rewrite_integration_test.go b/cmd/restic/cmd_rewrite_integration_test.go index 3f8e29c94..532855f57 100644 --- a/cmd/restic/cmd_rewrite_integration_test.go +++ b/cmd/restic/cmd_rewrite_integration_test.go @@ -65,8 +65,7 @@ func TestRewriteReplace(t *testing.T) { // exclude some data testRunRewriteExclude(t, env.gopts, []string{"3"}, true, snapshotMetadataArgs{Hostname: "", Time: ""}) - newSnapshotIDs := testRunList(t, "snapshots", env.gopts) - rtest.Assert(t, len(newSnapshotIDs) == 1, "expected one snapshot, got %v", newSnapshotIDs) + newSnapshotIDs := testListSnapshots(t, env.gopts, 1) rtest.Assert(t, snapshotID != newSnapshotIDs[0], "snapshot id should have changed") // check forbids unused blobs, thus remove them first testRunPrune(t, env.gopts, PruneOptions{MaxUnused: "0"}) @@ -75,16 +74,15 @@ func TestRewriteReplace(t *testing.T) { func testRewriteMetadata(t *testing.T, metadata snapshotMetadataArgs) { env, cleanup := withTestEnvironment(t) - env.gopts.backendTestHook = nil defer cleanup() createBasicRewriteRepo(t, env) - repo, _ := OpenRepository(context.TODO(), env.gopts) - testRunRewriteExclude(t, env.gopts, []string{}, true, metadata) - snapshots := FindFilteredSnapshots(context.TODO(), repo, repo, &restic.SnapshotFilter{}, []string{}) - - newSnapshot := <-snapshots + repo, _ := OpenRepository(context.TODO(), env.gopts) + snapshots, err := restic.TestLoadAllSnapshots(context.TODO(), repo, nil) + rtest.OK(t, err) + rtest.Assert(t, len(snapshots) == 1, "expected one snapshot, got %v", len(snapshots)) + newSnapshot := snapshots[0] if metadata.Time != "" { rtest.Assert(t, newSnapshot.Time.Format(TimeFormat) == metadata.Time, "New snapshot should have time %s", metadata.Time) diff --git a/internal/restic/testing.go b/internal/restic/testing.go index 004df627c..d2acd3ee9 100644 --- a/internal/restic/testing.go +++ b/internal/restic/testing.go @@ -187,3 +187,22 @@ func ParseDurationOrPanic(s string) Duration { return d } + +// TestLoadAllSnapshots returns a list of all snapshots in the repo. +// If a snapshot ID is in excludeIDs, it will not be included in the result. +func TestLoadAllSnapshots(ctx context.Context, repo Repository, excludeIDs IDSet) (snapshots Snapshots, err error) { + err = ForAllSnapshots(ctx, repo, repo, excludeIDs, func(id ID, sn *Snapshot, err error) error { + if err != nil { + return err + } + + snapshots = append(snapshots, sn) + return nil + }) + + if err != nil { + return nil, err + } + + return snapshots, nil +} diff --git a/internal/restic/testing_test.go b/internal/restic/testing_test.go index bc3ad2e87..ae8f8dd34 100644 --- a/internal/restic/testing_test.go +++ b/internal/restic/testing_test.go @@ -17,32 +17,13 @@ const ( testDepth = 2 ) -// LoadAllSnapshots returns a list of all snapshots in the repo. -// If a snapshot ID is in excludeIDs, it will not be included in the result. -func loadAllSnapshots(ctx context.Context, repo restic.Repository, excludeIDs restic.IDSet) (snapshots restic.Snapshots, err error) { - err = restic.ForAllSnapshots(ctx, repo, repo, excludeIDs, func(id restic.ID, sn *restic.Snapshot, err error) error { - if err != nil { - return err - } - - snapshots = append(snapshots, sn) - return nil - }) - - if err != nil { - return nil, err - } - - return snapshots, nil -} - func TestCreateSnapshot(t *testing.T) { repo := repository.TestRepository(t) for i := 0; i < testCreateSnapshots; i++ { restic.TestCreateSnapshot(t, repo, testSnapshotTime.Add(time.Duration(i)*time.Second), testDepth) } - snapshots, err := loadAllSnapshots(context.TODO(), repo, restic.NewIDSet()) + snapshots, err := restic.TestLoadAllSnapshots(context.TODO(), repo, restic.NewIDSet()) if err != nil { t.Fatal(err) } From 01b33734ab1ad244a44c29965a928f8028bf4755 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 24 Dec 2023 15:09:46 +0100 Subject: [PATCH 11/11] rewrite: update command output in docs --- doc/045_working_with_repos.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/045_working_with_repos.rst b/doc/045_working_with_repos.rst index ce1cb8754..68c181fa2 100644 --- a/doc/045_working_with_repos.rst +++ b/doc/045_working_with_repos.rst @@ -248,8 +248,8 @@ This is possible using the ``rewrite`` command with the option ``--new-host`` fo [0:00] 100.00% 1 / 1 index files loaded snapshot 8ed674f4 of [/path/to/abc.txt] at 2023-11-27 21:57:52.439139291 +0100 CET) - Setting time to 1999-01-01 11:11:11 +0100 CET - Setting host to newhost + setting time to 1999-01-01 11:11:11 +0100 CET + setting host to newhost saved new snapshot c05da643 modified 1 snapshots