octoleo/restic
2
2
Fork 0
mirror of https://github.com/octoleo/restic.git synced 2024-11-21 20:35:12 +00:00
Code Issues Releases Wiki Activity

Merge pull request #4573 from gab50000/rewrite_time

Browse Source 
Rewrite metadata
This commit is contained in:
Michael Eischer 2023-12-24 14:42:00 +00:00 committed by GitHub
commit 23e1b4bbb1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 182 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
changelog/unreleased/pull-4573 Normal file
View File

@ -0,0 +1,5 @@
Enhancement: Add `--new-host` and `--new-time` options to `rewrite` command
`restic rewrite` now allows rewriting the host and / or time metadata of a snapshot.
https://github.com/restic/restic/pull/4573

2
cmd/restic/cmd_repair_snapshots.go
View File

@ -148,7 +148,7 @@ func runRepairSnapshots(ctx context.Context, gopts GlobalOptions, opts RepairOpt
changed, err := filterAndReplaceSnapshot(ctx, repo, sn,
func(ctx context.Context, sn *restic.Snapshot) (restic.ID, error) {
return rewriter.RewriteTree(ctx, repo, "/", *sn.Tree)
}, opts.DryRun, opts.Forget, "repaired")
}, opts.DryRun, opts.Forget, nil, "repaired")
if err != nil {
return errors.Fatalf("unable to rewrite snapshot ID %q: %v", sn.ID().Str(), err)
}

117
cmd/restic/cmd_rewrite.go
View File

@ -3,6 +3,7 @@ package main
import (
"context"
"fmt"
"time"
"github.com/spf13/cobra"
"golang.org/x/sync/errgroup"
@ -46,11 +47,42 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
},
}
type snapshotMetadata struct {
Hostname string
Time *time.Time
}
type snapshotMetadataArgs struct {
Hostname string
Time string
}
func (sma snapshotMetadataArgs) empty() bool {
return sma.Hostname == "" && sma.Time == ""
}
func (sma snapshotMetadataArgs) convert() (*snapshotMetadata, error) {
if sma.empty() {
return nil, nil
}
var timeStamp *time.Time
if sma.Time != "" {
t, err := time.ParseInLocation(TimeFormat, sma.Time, time.Local)
if err != nil {
return nil, errors.Fatalf("error in time option: %v\n", err)
}
timeStamp = &t
}
return &snapshotMetadata{Hostname: sma.Hostname, Time: timeStamp}, nil
}
// RewriteOptions collects all options for the rewrite command.
type RewriteOptions struct {
Forget bool
DryRun bool
Metadata snapshotMetadataArgs
restic.SnapshotFilter
excludePatternOptions
}
@ -63,11 +95,15 @@ func init() {
f := cmdRewrite.Flags()
f.BoolVarP(&rewriteOptions.Forget, "forget", "", false, "remove original snapshots after creating new ones")
f.BoolVarP(&rewriteOptions.DryRun, "dry-run", "n", false, "do not do anything, just print what would be done")
f.StringVar(&rewriteOptions.Metadata.Hostname, "new-host", "", "replace hostname")
f.StringVar(&rewriteOptions.Metadata.Time, "new-time", "", "replace time of the backup")
initMultiSnapshotFilter(f, &rewriteOptions.SnapshotFilter, true)
initExcludePatternOptions(f, &rewriteOptions.excludePatternOptions)
}
type rewriteFilterFunc func(ctx context.Context, sn *restic.Snapshot) (restic.ID, error)
func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *restic.Snapshot, opts RewriteOptions) (bool, error) {
if sn.Tree == nil {
return false, errors.Errorf("snapshot %v has nil tree", sn.ID().Str())
@ -78,33 +114,50 @@ func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *resti
return false, err
}
selectByName := func(nodepath string) bool {
for _, reject := range rejectByNameFuncs {
if reject(nodepath) {
return false
}
}
return true
metadata, err := opts.Metadata.convert()
if err != nil {
return false, err
}
rewriter := walker.NewTreeRewriter(walker.RewriteOpts{
RewriteNode: func(node *restic.Node, path string) *restic.Node {
if selectByName(path) {
return node
var filter rewriteFilterFunc
if len(rejectByNameFuncs) > 0 {
selectByName := func(nodepath string) bool {
for _, reject := range rejectByNameFuncs {
if reject(nodepath) {
return false
}
}
Verbosef(fmt.Sprintf("excluding %s\n", path))
return nil
},
DisableNodeCache: true,
})
return true
}
rewriter := walker.NewTreeRewriter(walker.RewriteOpts{
RewriteNode: func(node *restic.Node, path string) *restic.Node {
if selectByName(path) {
return node
}
Verbosef(fmt.Sprintf("excluding %s\n", path))
return nil
},
DisableNodeCache: true,
})
filter = func(ctx context.Context, sn *restic.Snapshot) (restic.ID, error) {
return rewriter.RewriteTree(ctx, repo, "/", *sn.Tree)
}
} else {
filter = func(ctx context.Context, sn *restic.Snapshot) (restic.ID, error) {
return *sn.Tree, nil
}
}
return filterAndReplaceSnapshot(ctx, repo, sn,
func(ctx context.Context, sn *restic.Snapshot) (restic.ID, error) {
return rewriter.RewriteTree(ctx, repo, "/", *sn.Tree)
}, opts.DryRun, opts.Forget, "rewrite")
filter, opts.DryRun, opts.Forget, metadata, "rewrite")
}
func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *restic.Snapshot, filter func(ctx context.Context, sn *restic.Snapshot) (restic.ID, error), dryRun bool, forget bool, addTag string) (bool, error) {
func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *restic.Snapshot,
filter rewriteFilterFunc, dryRun bool, forget bool, newMetadata *snapshotMetadata, addTag string) (bool, error) {
wg, wgCtx := errgroup.WithContext(ctx)
repo.StartPackUploader(wgCtx, wg)
@ -138,7 +191,7 @@ func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *r
return true, nil
}
if filteredTree == *sn.Tree {
if filteredTree == *sn.Tree && newMetadata == nil {
debug.Log("Snapshot %v not modified", sn)
return false, nil
}
@ -151,6 +204,14 @@ func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *r
Verbosef("would remove old snapshot\n")
}
if newMetadata != nil && newMetadata.Time != nil {
Verbosef("would set time to %s\n", newMetadata.Time)
}
if newMetadata != nil && newMetadata.Hostname != "" {
Verbosef("would set time to %s\n", newMetadata.Hostname)
}
return true, nil
}
@ -162,6 +223,16 @@ func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *r
sn.AddTags([]string{addTag})
}
if newMetadata != nil && newMetadata.Time != nil {
Verbosef("setting time to %s\n", *newMetadata.Time)
sn.Time = *newMetadata.Time
}
if newMetadata != nil && newMetadata.Hostname != "" {
Verbosef("setting host to %s\n", newMetadata.Hostname)
sn.Hostname = newMetadata.Hostname
}
// Save the new snapshot.
id, err := restic.SaveSnapshot(ctx, repo, sn)
if err != nil {
@ -181,8 +252,8 @@ func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *r
}
func runRewrite(ctx context.Context, opts RewriteOptions, gopts GlobalOptions, args []string) error {
if opts.excludePatternOptions.Empty() {
return errors.Fatal("Nothing to do: no excludes provided")
if opts.excludePatternOptions.Empty() && opts.Metadata.empty() {
return errors.Fatal("Nothing to do: no excludes provided and no new metadata provided")
}
repo, err := OpenRepository(ctx, gopts)

48
cmd/restic/cmd_rewrite_integration_test.go
View File

@ -9,12 +9,13 @@ import (
rtest "github.com/restic/restic/internal/test"
)
func testRunRewriteExclude(t testing.TB, gopts GlobalOptions, excludes []string, forget bool) {
func testRunRewriteExclude(t testing.TB, gopts GlobalOptions, excludes []string, forget bool, metadata snapshotMetadataArgs) {
opts := RewriteOptions{
excludePatternOptions: excludePatternOptions{
Excludes: excludes,
},
Forget: forget,
Forget: forget,
Metadata: metadata,
}
rtest.OK(t, runRewrite(context.TODO(), opts, gopts, nil))
@ -38,7 +39,7 @@ func TestRewrite(t *testing.T) {
createBasicRewriteRepo(t, env)
// exclude some data
testRunRewriteExclude(t, env.gopts, []string{"3"}, false)
testRunRewriteExclude(t, env.gopts, []string{"3"}, false, snapshotMetadataArgs{Hostname: "", Time: ""})
snapshotIDs := testRunList(t, "snapshots", env.gopts)
rtest.Assert(t, len(snapshotIDs) == 2, "expected two snapshots, got %v", snapshotIDs)
testRunCheck(t, env.gopts)
@ -50,7 +51,7 @@ func TestRewriteUnchanged(t *testing.T) {
snapshotID := createBasicRewriteRepo(t, env)
// use an exclude that will not exclude anything
testRunRewriteExclude(t, env.gopts, []string{"3dflkhjgdflhkjetrlkhjgfdlhkj"}, false)
testRunRewriteExclude(t, env.gopts, []string{"3dflkhjgdflhkjetrlkhjgfdlhkj"}, false, snapshotMetadataArgs{Hostname: "", Time: ""})
newSnapshotIDs := testRunList(t, "snapshots", env.gopts)
rtest.Assert(t, len(newSnapshotIDs) == 1, "expected one snapshot, got %v", newSnapshotIDs)
rtest.Assert(t, snapshotID == newSnapshotIDs[0], "snapshot id changed unexpectedly")
@ -63,11 +64,44 @@ func TestRewriteReplace(t *testing.T) {
snapshotID := createBasicRewriteRepo(t, env)
// exclude some data
testRunRewriteExclude(t, env.gopts, []string{"3"}, true)
newSnapshotIDs := testRunList(t, "snapshots", env.gopts)
rtest.Assert(t, len(newSnapshotIDs) == 1, "expected one snapshot, got %v", newSnapshotIDs)
testRunRewriteExclude(t, env.gopts, []string{"3"}, true, snapshotMetadataArgs{Hostname: "", Time: ""})
newSnapshotIDs := testListSnapshots(t, env.gopts, 1)
rtest.Assert(t, snapshotID != newSnapshotIDs[0], "snapshot id should have changed")
// check forbids unused blobs, thus remove them first
testRunPrune(t, env.gopts, PruneOptions{MaxUnused: "0"})
testRunCheck(t, env.gopts)
}
func testRewriteMetadata(t *testing.T, metadata snapshotMetadataArgs) {
env, cleanup := withTestEnvironment(t)
defer cleanup()
createBasicRewriteRepo(t, env)
testRunRewriteExclude(t, env.gopts, []string{}, true, metadata)
repo, _ := OpenRepository(context.TODO(), env.gopts)
snapshots, err := restic.TestLoadAllSnapshots(context.TODO(), repo, nil)
rtest.OK(t, err)
rtest.Assert(t, len(snapshots) == 1, "expected one snapshot, got %v", len(snapshots))
newSnapshot := snapshots[0]
if metadata.Time != "" {
rtest.Assert(t, newSnapshot.Time.Format(TimeFormat) == metadata.Time, "New snapshot should have time %s", metadata.Time)
}
if metadata.Hostname != "" {
rtest.Assert(t, newSnapshot.Hostname == metadata.Hostname, "New snapshot should have host %s", metadata.Hostname)
}
}
func TestRewriteMetadata(t *testing.T) {
newHost := "new host"
newTime := "1999-01-01 11:11:11"
for _, metadata := range []snapshotMetadataArgs{
{Hostname: "", Time: newTime},
{Hostname: newHost, Time: ""},
{Hostname: newHost, Time: newTime},
} {
testRewriteMetadata(t, metadata)
}
}

21
doc/045_working_with_repos.rst
View File

@ -234,6 +234,27 @@ modifying the repository. Instead restic will only print the actions it would
perform.
Modifying metadata of snapshots
===============================
Sometimes it may be desirable to change the metadata of an existing snapshot.
Currently, rewriting the hostname and the time of the backup is supported.
This is possible using the ``rewrite`` command with the option ``--new-host`` followed by the desired new hostname or the option ``--new-time`` followed by the desired new timestamp.
.. code-block:: console
$ restic rewrite --new-host newhost --new-time "1999-01-01 11:11:11"
repository b7dbade3 opened (version 2, compression level auto)
[0:00] 100.00% 1 / 1 index files loaded
snapshot 8ed674f4 of [/path/to/abc.txt] at 2023-11-27 21:57:52.439139291 +0100 CET)
setting time to 1999-01-01 11:11:11 +0100 CET
setting host to newhost
saved new snapshot c05da643
modified 1 snapshots
.. _checking-integrity:
Checking integrity and consistency

19
internal/restic/testing.go
View File

@ -187,3 +187,22 @@ func ParseDurationOrPanic(s string) Duration {
return d
}
// TestLoadAllSnapshots returns a list of all snapshots in the repo.
// If a snapshot ID is in excludeIDs, it will not be included in the result.
func TestLoadAllSnapshots(ctx context.Context, repo Repository, excludeIDs IDSet) (snapshots Snapshots, err error) {
err = ForAllSnapshots(ctx, repo, repo, excludeIDs, func(id ID, sn *Snapshot, err error) error {
if err != nil {
return err
}
snapshots = append(snapshots, sn)
return nil
})
if err != nil {
return nil, err
}
return snapshots, nil
}

21
internal/restic/testing_test.go
View File

@ -17,32 +17,13 @@ const (
testDepth = 2
)
// LoadAllSnapshots returns a list of all snapshots in the repo.
// If a snapshot ID is in excludeIDs, it will not be included in the result.
func loadAllSnapshots(ctx context.Context, repo restic.Repository, excludeIDs restic.IDSet) (snapshots restic.Snapshots, err error) {
err = restic.ForAllSnapshots(ctx, repo, repo, excludeIDs, func(id restic.ID, sn *restic.Snapshot, err error) error {
if err != nil {
return err
}
snapshots = append(snapshots, sn)
return nil
})
if err != nil {
return nil, err
}
return snapshots, nil
}
func TestCreateSnapshot(t *testing.T) {
repo := repository.TestRepository(t)
for i := 0; i < testCreateSnapshots; i++ {
restic.TestCreateSnapshot(t, repo, testSnapshotTime.Add(time.Duration(i)*time.Second), testDepth)
}
snapshots, err := loadAllSnapshots(context.TODO(), repo, restic.NewIDSet())
snapshots, err := restic.TestLoadAllSnapshots(context.TODO(), repo, restic.NewIDSet())
if err != nil {
t.Fatal(err)
}