From 9cef6b4c69d6151f50067b06a67dc383f5e79fec Mon Sep 17 00:00:00 2001 From: Alexander Weiss Date: Thu, 6 Aug 2020 17:24:00 +0200 Subject: [PATCH 01/25] Add troubleshooting documentation --- doc/077_troubleshooting.rst | 105 ++++++++++++++++++++++++++++++++++++ doc/index.rst | 1 + 2 files changed, 106 insertions(+) create mode 100644 doc/077_troubleshooting.rst diff --git a/doc/077_troubleshooting.rst b/doc/077_troubleshooting.rst new file mode 100644 index 000000000..504853954 --- /dev/null +++ b/doc/077_troubleshooting.rst @@ -0,0 +1,105 @@ +.. + Normally, there are no heading levels assigned to certain characters as the structure is + determined from the succession of headings. However, this convention is used in Python’s + Style Guide for documenting which you may follow: + + # with overline, for parts + * for chapters + = for sections + - for subsections + ^ for subsubsections + " for paragraphs + +######################### +Troubleshooting +######################### + +Being a backup software, the repository format ensures that the data saved in the repository +is verifiable and error-restistant. Restic even implements some self-healing functionalities. + +However, situations might occur where your repository gets in an incorrect state and measurements +need to be done to get you out of this situation. These situations might be due to hardware failure, +accidentially removing files directly from the repository or bugs in the restic implementation. + +This document is meant to give you some hints about how to recover from such situations. + +1. Stay calm and don't over-react +******************************************** + +The most important thing if you find yourself in the situation of a damaged repository is to +stay calm and don't do anything you might regret later. + +The following point should be always considered: + +- Make a copy of you repository and try to recover from that copy. If you suspect a storage failure, + it may be even better, to make *two* copies: one to get all data out of the possibly failing storage + and another one to try the recovery process. +- Pause your regular operations on the repository or let them run on a copy. You will especially make + sure that no `forget` or `prune` is run as these command are supposed to remove data and may result + in data loss. +- Search if your issue is already known and solved. Good starting points are the restic forum and the + github issues. +- Get you some help if you are unsure what to do. Find a colleage or friend to discuss what should be done. + Also feel free to consult the restic forum. +- When using the commands below, make sure you read and understand the documentation. Some of the commands + may not be your every-day commands, so make sure you really understand what they are doing. + + +2. `check` is your friend +******************************************** + +Run `restic check` to find out what type of error you have. The results may be technical but can give you +a good hint what's really wrong. + +Moreover, you can always run a `check` to ensure that your repair really was sucessful and your repository +is in a sane state again. +But make sure that your needed data is also still contained in your repository ;-) + +Note that `check` also prints out warning in some cases. These warnings point out that the repo may be +optimized but is still in perfect shape and does not need any troubleshooting. + +3. Index trouble -> `rebuild-index` +******************************************** + +A common problem with broken repostories is that the index does no longer correctly represent the contents +of your pack files. This is especially the case if some pack files got lost. +`rebuild-index` recovers this situation and ensures that the index exactly represents the pack files. + +You might even need to manually remove corrupted pack files. In this case make sure, you run +`restic rebuild-index` after. + +Also if you encounter problems with the index files itselves, `rebuild-index` will solve these problems +immediately. + +However, rebuilding the index does not solve every problem, e.g. lost pack files. + +4. Delete unneeded defect snapshots -> `forget` +******************************************** + +If you encounter defect snapshots but realize you can spare them, it is often a good idea to simply +delete them using `forget`. In case that your repository remains with just sane snapshots (including +all trees and files) the next `prune` run will put your repository in a sane state. + +This can be also used if you manage to create new snapshots which can replace the defect ones, see +below. + +5. No fear to `backup` again +******************************************** + +There are quite some self-healing mechanisms withing the `backup` command. So it is always a good idea to +backup again and check if this did heal your repository. +If you realize that a specific file is broken in your repository and you have this file, any run of +`backup` which includes that file will be able to heal the situation. + +Note that `backup` relies on a correct index state, so make sure your index is fine or run `rebuild-index` +before running `backup`. + +6. Unreferenced tree -> `recover` +******************************************** + +If for some reason you have unreferenced trees in your repository but you actually need them, run +`recover` it will generate a new snapshot which allows access to all trees that you have in your +repository. + +Note that `recover` relies on a correct index state, so make sure your index is fine or run `rebuild-index` +before running `recover`. diff --git a/doc/index.rst b/doc/index.rst index 034dbda23..8b72dcf58 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -14,6 +14,7 @@ Restic Documentation 060_forget 070_encryption 075_scripting + 077_troubleshooting 080_examples 090_participating 100_references From 5f58797ba7d54c718caea3a14957151aa7f4a873 Mon Sep 17 00:00:00 2001 From: Alexander Weiss Date: Wed, 5 Aug 2020 21:32:15 +0200 Subject: [PATCH 02/25] Add repair command --- cmd/restic/cmd_repair.go | 269 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 269 insertions(+) create mode 100644 cmd/restic/cmd_repair.go diff --git a/cmd/restic/cmd_repair.go b/cmd/restic/cmd_repair.go new file mode 100644 index 000000000..aa3a9caca --- /dev/null +++ b/cmd/restic/cmd_repair.go @@ -0,0 +1,269 @@ +package main + +import ( + "github.com/restic/restic/internal/debug" + "github.com/restic/restic/internal/restic" + + "github.com/spf13/cobra" +) + +var cmdRepair = &cobra.Command{ + Use: "repair [flags] [snapshot ID] [...]", + Short: "Repair snapshots", + Long: ` +The "repair" command allows to repair broken snapshots. +It scans the given snapshots and generates new ones where +damaged tress and file contents are removed. +If the broken snapshots are deleted, a prune run will +be able to refit the repository. + +The command depends on a good state of the index, so if +there are inaccurancies in the index, make sure to run +"rebuild-index" before! + + +WARNING: +======== +Repairing and deleting broken snapshots causes data loss! +It will remove broken dirs and modify broken files in +the modified snapshots. + +If the contents of directories and files are still available, +the better option is to redo a backup which in that case is +able to "heal" already present snapshots. +Only use this command if you need to recover an old and +broken snapshot! + +EXIT STATUS +=========== + +Exit status is 0 if the command was successful, and non-zero if there was any error. +`, + DisableAutoGenTag: true, + RunE: func(cmd *cobra.Command, args []string) error { + return runRepair(repairOptions, args) + }, +} + +// RestoreOptions collects all options for the restore command. +type RepairOptions struct { + Hosts []string + Paths []string + Tags restic.TagLists + AddTag string + Append string + DryRun bool + DeleteSnapshots bool +} + +var repairOptions RepairOptions + +func init() { + cmdRoot.AddCommand(cmdRepair) + flags := cmdRepair.Flags() + flags.StringArrayVarP(&repairOptions.Hosts, "host", "H", nil, `only consider snapshots for this host (can be specified multiple times)`) + flags.Var(&repairOptions.Tags, "tag", "only consider snapshots which include this `taglist`") + flags.StringArrayVar(&repairOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`") + flags.StringVar(&repairOptions.AddTag, "add-tag", "repaired", "tag to add to repaired snapshots") + flags.StringVar(&repairOptions.Append, "append", ".repaired", "string to append to repaired dirs/files; remove files if emtpy or impossible to repair") + flags.BoolVarP(&repairOptions.DryRun, "dry-run", "n", true, "don't do anything, only show what would be done") + flags.BoolVar(&repairOptions.DeleteSnapshots, "delete-snapshots", false, "delete original snapshots") +} + +func runRepair(opts RepairOptions, args []string) error { + switch { + case opts.DryRun: + Printf("\n note: --dry-run is set\n-> repair will only show what it would do.\n\n") + case opts.DeleteSnapshots: + Printf("\n note: --dry-run is not set and --delete is set\n-> this may result in data loss!\n\n") + } + + repo, err := OpenRepository(globalOptions) + if err != nil { + return err + } + + lock, err := lockRepoExclusive(globalOptions.ctx, repo) + defer unlockRepo(lock) + if err != nil { + return err + } + + if err := repo.LoadIndex(globalOptions.ctx); err != nil { + return err + } + + // get snapshots to check & repair + var snapshots []*restic.Snapshot + for sn := range FindFilteredSnapshots(globalOptions.ctx, repo.Backend(), repo, opts.Hosts, opts.Tags, opts.Paths, args) { + snapshots = append(snapshots, sn) + } + + return repairSnapshots(opts, repo, snapshots) +} + +func repairSnapshots(opts RepairOptions, repo restic.Repository, snapshots []*restic.Snapshot) error { + ctx := globalOptions.ctx + + replaces := make(idMap) + seen := restic.NewIDSet() + deleteSn := restic.NewIDSet() + + Verbosef("check and repair %d snapshots\n", len(snapshots)) + bar := newProgressMax(!globalOptions.Quiet, uint64(len(snapshots)), "snapshots") + for _, sn := range snapshots { + debug.Log("process snapshot %v", sn.ID()) + Printf("%v:\n", sn) + newID, changed, err := repairTree(opts, repo, "/", *sn.Tree, replaces, seen) + switch { + case err != nil: + Printf("the root tree is damaged -> delete snapshot.\n") + deleteSn.Insert(*sn.ID()) + case changed: + err = changeSnapshot(opts, repo, sn, newID) + if err != nil { + return err + } + deleteSn.Insert(*sn.ID()) + default: + Printf("is ok.\n") + } + debug.Log("processed snapshot %v", sn.ID()) + bar.Add(1) + } + bar.Done() + + err := repo.Flush(ctx) + if err != nil { + return err + } + + if len(deleteSn) > 0 && opts.DeleteSnapshots { + Verbosef("delete %d snapshots...\n", len(deleteSn)) + if !opts.DryRun { + DeleteFiles(globalOptions, repo, deleteSn, restic.SnapshotFile) + } + } + return nil +} + +// changeSnapshot creates a modified snapshot: +// - set the tree to newID +// - add the rag opts.AddTag +// - preserve original ID +// if opts.DryRun is set, it doesn't change anything but only +func changeSnapshot(opts RepairOptions, repo restic.Repository, sn *restic.Snapshot, newID restic.ID) error { + sn.AddTags([]string{opts.AddTag}) + // Retain the original snapshot id over all tag changes. + if sn.Original == nil { + sn.Original = sn.ID() + } + sn.Tree = &newID + if !opts.DryRun { + newID, err := repo.SaveJSONUnpacked(globalOptions.ctx, restic.SnapshotFile, sn) + if err != nil { + return err + } + Printf("snapshot repaired -> %v created.\n", newID.Str()) + } else { + Printf("would have repaired snpshot %v.\n", sn.ID().Str()) + } + return nil +} + +type idMap map[restic.ID]restic.ID + +// repairTree checks and repairs a tree and all its subtrees +// Two error cases are checked: +// - trees which cannot be loaded (-> the tree contents will be removed) +// - files whose contents are not fully available (-> file will be modified) +// In case of an error, the changes made depends on: +// - opts.Append: string to append to "repared" names; if empty files will not repaired but deleted +// - opts.DryRun: if set to true, only print out what to but don't change anything +func repairTree(opts RepairOptions, repo restic.Repository, path string, treeID restic.ID, replaces idMap, seen restic.IDSet) (restic.ID, bool, error) { + ctx := globalOptions.ctx + + // check if tree was already changed + newID, ok := replaces[treeID] + if ok { + return newID, true, nil + } + + // check if tree was seen but not changed + if seen.Has(treeID) { + return treeID, false, nil + } + + tree, err := repo.LoadTree(ctx, treeID) + if err != nil { + return newID, false, err + } + + var newNodes []*restic.Node + changed := false + for _, node := range tree.Nodes { + switch node.Type { + case "file": + ok := true + var newContent restic.IDs + var newSize uint64 + // check all contents and remove if not available + for _, id := range node.Content { + if size, found := repo.LookupBlobSize(id, restic.DataBlob); !found { + ok = false + } else { + newContent = append(newContent, id) + newSize += uint64(size) + } + } + if !ok { + changed = true + if opts.Append == "" || newSize == 0 { + Printf("removed defect file '%v'\n", path+node.Name) + continue + } + Printf("repaired defect file '%v'", path+node.Name) + node.Name = node.Name + opts.Append + Printf(" to '%v'\n", node.Name) + node.Content = newContent + node.Size = newSize + } + case "dir": + // rewrite if necessary + newID, c, err := repairTree(opts, repo, path+node.Name+"/", *node.Subtree, replaces, seen) + switch { + case err != nil: + // If we get an error, we remove this subtree + changed = true + Printf("removed defect dir '%v'", path+node.Name) + node.Name = node.Name + opts.Append + Printf("(now emtpy '%v')\n", node.Name) + node.Subtree = nil + case c: + node.Subtree = &newID + changed = true + } + } + newNodes = append(newNodes, node) + } + + if !changed { + seen.Insert(treeID) + return treeID, false, nil + } + + tree.Nodes = newNodes + + if !opts.DryRun { + newID, err = repo.SaveTree(ctx, tree) + if err != nil { + return newID, false, err + } + Printf("modified tree %v, new id: %v\n", treeID.Str(), newID.Str()) + } else { + Printf("would have modified tree %v\n", treeID.Str()) + } + + replaces[treeID] = newID + return newID, true, nil +} From 6557f36f615c58bf98772bf6ee49b289d9670ef1 Mon Sep 17 00:00:00 2001 From: Alexander Weiss Date: Thu, 6 Aug 2020 12:31:37 +0200 Subject: [PATCH 03/25] Add changelog and docu for #2876 --- changelog/unreleased/issue-1759 | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 changelog/unreleased/issue-1759 diff --git a/changelog/unreleased/issue-1759 b/changelog/unreleased/issue-1759 new file mode 100644 index 000000000..0b0d28aa5 --- /dev/null +++ b/changelog/unreleased/issue-1759 @@ -0,0 +1,16 @@ +Enhancement: Add new command repair + +We've added a new command repair which allows to repair snapshots even if needed +parts of it are not accessable in the repository. Note that using this command +can lead to data loss! + +Some corrupted repositories were reported in several issues and so far restic +lacked a possibility to accept data loss but clean those up such that the +repository returns to a sane state. This possibility was now added. + +https://github.com/restic/restic/issues/1759 +https://github.com/restic/restic/issues/1798 +https://github.com/restic/restic/issues/2334 +https://github.com/restic/restic/pull/2876 +https://forum.restic.net/t/corrupted-repo-how-to-repair/799 +https://forum.restic.net/t/recovery-options-for-damaged-repositories/1571 From 99a05d5ab23f16a6d6eed08d0315deea563e3f50 Mon Sep 17 00:00:00 2001 From: Alexander Weiss Date: Thu, 6 Aug 2020 17:24:00 +0200 Subject: [PATCH 04/25] Update troubleshooting documentation --- doc/077_troubleshooting.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/doc/077_troubleshooting.rst b/doc/077_troubleshooting.rst index 504853954..50c19565e 100644 --- a/doc/077_troubleshooting.rst +++ b/doc/077_troubleshooting.rst @@ -103,3 +103,9 @@ repository. Note that `recover` relies on a correct index state, so make sure your index is fine or run `rebuild-index` before running `recover`. + +7. Repair defect snapshots using `repair` +******************************************** + +If all other things did not help, you can repair defect snapshots with `repair`. Note that the repaired +snapshots will miss data which was referenced in the defect snapshot. From 08ae708b3b2eaa2c9f6b614737b2589adc1a562d Mon Sep 17 00:00:00 2001 From: Alexander Weiss Date: Sun, 22 Nov 2020 22:33:02 +0100 Subject: [PATCH 05/25] make linter happy --- cmd/restic/cmd_repair.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/restic/cmd_repair.go b/cmd/restic/cmd_repair.go index aa3a9caca..e159da261 100644 --- a/cmd/restic/cmd_repair.go +++ b/cmd/restic/cmd_repair.go @@ -84,7 +84,8 @@ func runRepair(opts RepairOptions, args []string) error { } lock, err := lockRepoExclusive(globalOptions.ctx, repo) - defer unlockRepo(lock) + // to make linter happy, as unlockRepo returns an error (which is ignored) + defer func() { _ = unlockRepo(lock) }() if err != nil { return err } From d23a2e192576da40f8da86ae31182fab8115b189 Mon Sep 17 00:00:00 2001 From: Alexander Weiss Date: Sat, 20 Feb 2021 20:16:05 +0100 Subject: [PATCH 06/25] better error handling and correct nil tree behavior --- cmd/restic/cmd_repair.go | 72 ++++++++++++++++++++++++++++------------ 1 file changed, 51 insertions(+), 21 deletions(-) diff --git a/cmd/restic/cmd_repair.go b/cmd/restic/cmd_repair.go index e159da261..47beaf3cf 100644 --- a/cmd/restic/cmd_repair.go +++ b/cmd/restic/cmd_repair.go @@ -84,8 +84,7 @@ func runRepair(opts RepairOptions, args []string) error { } lock, err := lockRepoExclusive(globalOptions.ctx, repo) - // to make linter happy, as unlockRepo returns an error (which is ignored) - defer func() { _ = unlockRepo(lock) }() + defer unlockRepo(lock) if err != nil { return err } @@ -115,9 +114,11 @@ func repairSnapshots(opts RepairOptions, repo restic.Repository, snapshots []*re for _, sn := range snapshots { debug.Log("process snapshot %v", sn.ID()) Printf("%v:\n", sn) - newID, changed, err := repairTree(opts, repo, "/", *sn.Tree, replaces, seen) + newID, changed, lErr, err := repairTree(opts, repo, "/", sn.Tree, replaces, seen) switch { case err != nil: + return err + case lErr: Printf("the root tree is damaged -> delete snapshot.\n") deleteSn.Insert(*sn.ID()) case changed: @@ -153,13 +154,13 @@ func repairSnapshots(opts RepairOptions, repo restic.Repository, snapshots []*re // - add the rag opts.AddTag // - preserve original ID // if opts.DryRun is set, it doesn't change anything but only -func changeSnapshot(opts RepairOptions, repo restic.Repository, sn *restic.Snapshot, newID restic.ID) error { +func changeSnapshot(opts RepairOptions, repo restic.Repository, sn *restic.Snapshot, newID *restic.ID) error { sn.AddTags([]string{opts.AddTag}) // Retain the original snapshot id over all tag changes. if sn.Original == nil { sn.Original = sn.ID() } - sn.Tree = &newID + sn.Tree = newID if !opts.DryRun { newID, err := repo.SaveJSONUnpacked(globalOptions.ctx, restic.SnapshotFile, sn) if err != nil { @@ -175,29 +176,43 @@ func changeSnapshot(opts RepairOptions, repo restic.Repository, sn *restic.Snaps type idMap map[restic.ID]restic.ID // repairTree checks and repairs a tree and all its subtrees -// Two error cases are checked: +// Three error cases are checked: +// - tree is a nil tree (-> will be replaced by an empty tree) // - trees which cannot be loaded (-> the tree contents will be removed) // - files whose contents are not fully available (-> file will be modified) // In case of an error, the changes made depends on: // - opts.Append: string to append to "repared" names; if empty files will not repaired but deleted // - opts.DryRun: if set to true, only print out what to but don't change anything -func repairTree(opts RepairOptions, repo restic.Repository, path string, treeID restic.ID, replaces idMap, seen restic.IDSet) (restic.ID, bool, error) { +// Returns: +// - the new ID +// - whether the ID changed +// - whether there was a load error when loading this tre +// - error for other errors (these are errors when saving a tree) +func repairTree(opts RepairOptions, repo restic.Repository, path string, treeID *restic.ID, replaces idMap, seen restic.IDSet) (*restic.ID, bool, bool, error) { ctx := globalOptions.ctx + // handle and repair nil trees + if treeID == nil { + empty, err := emptyTree(opts.DryRun, repo) + Printf("repaired nil tree '%v'\n", path) + return &empty, true, false, err + } + // check if tree was already changed - newID, ok := replaces[treeID] + newID, ok := replaces[*treeID] if ok { - return newID, true, nil + return &newID, true, false, nil } // check if tree was seen but not changed - if seen.Has(treeID) { - return treeID, false, nil + if seen.Has(*treeID) { + return treeID, false, false, nil } - tree, err := repo.LoadTree(ctx, treeID) + tree, err := repo.LoadTree(ctx, *treeID) if err != nil { - return newID, false, err + // mark as load error + return &newID, false, true, nil } var newNodes []*restic.Node @@ -231,17 +246,23 @@ func repairTree(opts RepairOptions, repo restic.Repository, path string, treeID } case "dir": // rewrite if necessary - newID, c, err := repairTree(opts, repo, path+node.Name+"/", *node.Subtree, replaces, seen) + newID, c, lErr, err := repairTree(opts, repo, path+node.Name+"/", node.Subtree, replaces, seen) switch { case err != nil: + return newID, true, false, err + case lErr: // If we get an error, we remove this subtree changed = true Printf("removed defect dir '%v'", path+node.Name) node.Name = node.Name + opts.Append Printf("(now emtpy '%v')\n", node.Name) - node.Subtree = nil + empty, err := emptyTree(opts.DryRun, repo) + if err != nil { + return newID, true, false, err + } + node.Subtree = &empty case c: - node.Subtree = &newID + node.Subtree = newID changed = true } } @@ -249,8 +270,8 @@ func repairTree(opts RepairOptions, repo restic.Repository, path string, treeID } if !changed { - seen.Insert(treeID) - return treeID, false, nil + seen.Insert(*treeID) + return treeID, false, false, nil } tree.Nodes = newNodes @@ -258,13 +279,22 @@ func repairTree(opts RepairOptions, repo restic.Repository, path string, treeID if !opts.DryRun { newID, err = repo.SaveTree(ctx, tree) if err != nil { - return newID, false, err + return &newID, true, false, err } Printf("modified tree %v, new id: %v\n", treeID.Str(), newID.Str()) } else { Printf("would have modified tree %v\n", treeID.Str()) } - replaces[treeID] = newID - return newID, true, nil + replaces[*treeID] = newID + return &newID, true, false, nil +} + +func emptyTree(dryRun bool, repo restic.Repository) (restic.ID, error) { + ctx := globalOptions.ctx + var tree restic.Tree + if !dryRun { + return repo.SaveTree(ctx, &tree) + } + return restic.ID{}, nil } From 947f0c345e3f298f4971e6acedbea9eec28e2ac0 Mon Sep 17 00:00:00 2001 From: Alexander Weiss Date: Sat, 20 Feb 2021 20:56:03 +0100 Subject: [PATCH 07/25] correct typos --- cmd/restic/cmd_repair.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/cmd/restic/cmd_repair.go b/cmd/restic/cmd_repair.go index 47beaf3cf..19574c43d 100644 --- a/cmd/restic/cmd_repair.go +++ b/cmd/restic/cmd_repair.go @@ -45,7 +45,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er }, } -// RestoreOptions collects all options for the restore command. +// RepairOptions collects all options for the repair command. type RepairOptions struct { Hosts []string Paths []string @@ -65,7 +65,7 @@ func init() { flags.Var(&repairOptions.Tags, "tag", "only consider snapshots which include this `taglist`") flags.StringArrayVar(&repairOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`") flags.StringVar(&repairOptions.AddTag, "add-tag", "repaired", "tag to add to repaired snapshots") - flags.StringVar(&repairOptions.Append, "append", ".repaired", "string to append to repaired dirs/files; remove files if emtpy or impossible to repair") + flags.StringVar(&repairOptions.Append, "append", ".repaired", "string to append to repaired dirs/files; remove files if empty or impossible to repair") flags.BoolVarP(&repairOptions.DryRun, "dry-run", "n", true, "don't do anything, only show what would be done") flags.BoolVar(&repairOptions.DeleteSnapshots, "delete-snapshots", false, "delete original snapshots") } @@ -75,7 +75,7 @@ func runRepair(opts RepairOptions, args []string) error { case opts.DryRun: Printf("\n note: --dry-run is set\n-> repair will only show what it would do.\n\n") case opts.DeleteSnapshots: - Printf("\n note: --dry-run is not set and --delete is set\n-> this may result in data loss!\n\n") + Printf("\n note: --dry-run is not set and --delete-snapshots is set\n-> this may result in data loss!\n\n") } repo, err := OpenRepository(globalOptions) @@ -168,7 +168,7 @@ func changeSnapshot(opts RepairOptions, repo restic.Repository, sn *restic.Snaps } Printf("snapshot repaired -> %v created.\n", newID.Str()) } else { - Printf("would have repaired snpshot %v.\n", sn.ID().Str()) + Printf("would have repaired snapshot %v.\n", sn.ID().Str()) } return nil } @@ -235,10 +235,10 @@ func repairTree(opts RepairOptions, repo restic.Repository, path string, treeID if !ok { changed = true if opts.Append == "" || newSize == 0 { - Printf("removed defect file '%v'\n", path+node.Name) + Printf("removed defective file '%v'\n", path+node.Name) continue } - Printf("repaired defect file '%v'", path+node.Name) + Printf("repaired defective file '%v'", path+node.Name) node.Name = node.Name + opts.Append Printf(" to '%v'\n", node.Name) node.Content = newContent @@ -253,9 +253,9 @@ func repairTree(opts RepairOptions, repo restic.Repository, path string, treeID case lErr: // If we get an error, we remove this subtree changed = true - Printf("removed defect dir '%v'", path+node.Name) + Printf("removed defective dir '%v'", path+node.Name) node.Name = node.Name + opts.Append - Printf("(now emtpy '%v')\n", node.Name) + Printf("(now empty '%v')\n", node.Name) empty, err := emptyTree(opts.DryRun, repo) if err != nil { return newID, true, false, err From a14a63cd29204908c5a9a37a5cca93a1cea12b7c Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 10 Dec 2022 17:18:04 +0100 Subject: [PATCH 08/25] modernize code --- cmd/restic/cmd_repair.go | 105 ++++++++++++++++++++------------------- 1 file changed, 53 insertions(+), 52 deletions(-) diff --git a/cmd/restic/cmd_repair.go b/cmd/restic/cmd_repair.go index 19574c43d..30fe8be1c 100644 --- a/cmd/restic/cmd_repair.go +++ b/cmd/restic/cmd_repair.go @@ -1,8 +1,11 @@ package main import ( + "context" + "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/restic" + "golang.org/x/sync/errgroup" "github.com/spf13/cobra" ) @@ -41,15 +44,14 @@ Exit status is 0 if the command was successful, and non-zero if there was any er `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - return runRepair(repairOptions, args) + return runRepair(cmd.Context(), globalOptions, repairOptions, args) }, } // RepairOptions collects all options for the repair command. type RepairOptions struct { - Hosts []string - Paths []string - Tags restic.TagLists + restic.SnapshotFilter + AddTag string Append string DryRun bool @@ -61,16 +63,16 @@ var repairOptions RepairOptions func init() { cmdRoot.AddCommand(cmdRepair) flags := cmdRepair.Flags() - flags.StringArrayVarP(&repairOptions.Hosts, "host", "H", nil, `only consider snapshots for this host (can be specified multiple times)`) - flags.Var(&repairOptions.Tags, "tag", "only consider snapshots which include this `taglist`") - flags.StringArrayVar(&repairOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`") + + initMultiSnapshotFilter(flags, &repairOptions.SnapshotFilter, true) + flags.StringVar(&repairOptions.AddTag, "add-tag", "repaired", "tag to add to repaired snapshots") flags.StringVar(&repairOptions.Append, "append", ".repaired", "string to append to repaired dirs/files; remove files if empty or impossible to repair") flags.BoolVarP(&repairOptions.DryRun, "dry-run", "n", true, "don't do anything, only show what would be done") flags.BoolVar(&repairOptions.DeleteSnapshots, "delete-snapshots", false, "delete original snapshots") } -func runRepair(opts RepairOptions, args []string) error { +func runRepair(ctx context.Context, gopts GlobalOptions, opts RepairOptions, args []string) error { switch { case opts.DryRun: Printf("\n note: --dry-run is set\n-> repair will only show what it would do.\n\n") @@ -78,64 +80,67 @@ func runRepair(opts RepairOptions, args []string) error { Printf("\n note: --dry-run is not set and --delete-snapshots is set\n-> this may result in data loss!\n\n") } - repo, err := OpenRepository(globalOptions) + repo, err := OpenRepository(ctx, globalOptions) if err != nil { return err } - lock, err := lockRepoExclusive(globalOptions.ctx, repo) + lock, ctx, err := lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON) defer unlockRepo(lock) if err != nil { return err } - if err := repo.LoadIndex(globalOptions.ctx); err != nil { + if err := repo.LoadIndex(ctx); err != nil { return err } // get snapshots to check & repair var snapshots []*restic.Snapshot - for sn := range FindFilteredSnapshots(globalOptions.ctx, repo.Backend(), repo, opts.Hosts, opts.Tags, opts.Paths, args) { + for sn := range FindFilteredSnapshots(ctx, repo.Backend(), repo, &opts.SnapshotFilter, args) { snapshots = append(snapshots, sn) } - return repairSnapshots(opts, repo, snapshots) + return repairSnapshots(ctx, opts, repo, snapshots) } -func repairSnapshots(opts RepairOptions, repo restic.Repository, snapshots []*restic.Snapshot) error { - ctx := globalOptions.ctx - +func repairSnapshots(ctx context.Context, opts RepairOptions, repo restic.Repository, snapshots []*restic.Snapshot) error { replaces := make(idMap) seen := restic.NewIDSet() deleteSn := restic.NewIDSet() Verbosef("check and repair %d snapshots\n", len(snapshots)) bar := newProgressMax(!globalOptions.Quiet, uint64(len(snapshots)), "snapshots") - for _, sn := range snapshots { - debug.Log("process snapshot %v", sn.ID()) - Printf("%v:\n", sn) - newID, changed, lErr, err := repairTree(opts, repo, "/", sn.Tree, replaces, seen) - switch { - case err != nil: - return err - case lErr: - Printf("the root tree is damaged -> delete snapshot.\n") - deleteSn.Insert(*sn.ID()) - case changed: - err = changeSnapshot(opts, repo, sn, newID) - if err != nil { + wg, ctx := errgroup.WithContext(ctx) + repo.StartPackUploader(ctx, wg) + wg.Go(func() error { + for _, sn := range snapshots { + debug.Log("process snapshot %v", sn.ID()) + Printf("%v:\n", sn) + newID, changed, lErr, err := repairTree(ctx, opts, repo, "/", sn.Tree, replaces, seen) + switch { + case err != nil: return err + case lErr: + Printf("the root tree is damaged -> delete snapshot.\n") + deleteSn.Insert(*sn.ID()) + case changed: + err = changeSnapshot(ctx, opts, repo, sn, newID) + if err != nil { + return err + } + deleteSn.Insert(*sn.ID()) + default: + Printf("is ok.\n") } - deleteSn.Insert(*sn.ID()) - default: - Printf("is ok.\n") + debug.Log("processed snapshot %v", sn.ID()) + bar.Add(1) } - debug.Log("processed snapshot %v", sn.ID()) - bar.Add(1) - } - bar.Done() + bar.Done() + return repo.Flush(ctx) + }) - err := repo.Flush(ctx) + err := wg.Wait() if err != nil { return err } @@ -143,7 +148,7 @@ func repairSnapshots(opts RepairOptions, repo restic.Repository, snapshots []*re if len(deleteSn) > 0 && opts.DeleteSnapshots { Verbosef("delete %d snapshots...\n", len(deleteSn)) if !opts.DryRun { - DeleteFiles(globalOptions, repo, deleteSn, restic.SnapshotFile) + DeleteFiles(ctx, globalOptions, repo, deleteSn, restic.SnapshotFile) } } return nil @@ -154,7 +159,7 @@ func repairSnapshots(opts RepairOptions, repo restic.Repository, snapshots []*re // - add the rag opts.AddTag // - preserve original ID // if opts.DryRun is set, it doesn't change anything but only -func changeSnapshot(opts RepairOptions, repo restic.Repository, sn *restic.Snapshot, newID *restic.ID) error { +func changeSnapshot(ctx context.Context, opts RepairOptions, repo restic.Repository, sn *restic.Snapshot, newID *restic.ID) error { sn.AddTags([]string{opts.AddTag}) // Retain the original snapshot id over all tag changes. if sn.Original == nil { @@ -162,7 +167,7 @@ func changeSnapshot(opts RepairOptions, repo restic.Repository, sn *restic.Snaps } sn.Tree = newID if !opts.DryRun { - newID, err := repo.SaveJSONUnpacked(globalOptions.ctx, restic.SnapshotFile, sn) + newID, err := restic.SaveSnapshot(ctx, repo, sn) if err != nil { return err } @@ -188,12 +193,10 @@ type idMap map[restic.ID]restic.ID // - whether the ID changed // - whether there was a load error when loading this tre // - error for other errors (these are errors when saving a tree) -func repairTree(opts RepairOptions, repo restic.Repository, path string, treeID *restic.ID, replaces idMap, seen restic.IDSet) (*restic.ID, bool, bool, error) { - ctx := globalOptions.ctx - +func repairTree(ctx context.Context, opts RepairOptions, repo restic.Repository, path string, treeID *restic.ID, replaces idMap, seen restic.IDSet) (*restic.ID, bool, bool, error) { // handle and repair nil trees if treeID == nil { - empty, err := emptyTree(opts.DryRun, repo) + empty, err := emptyTree(ctx, repo, opts.DryRun) Printf("repaired nil tree '%v'\n", path) return &empty, true, false, err } @@ -209,7 +212,7 @@ func repairTree(opts RepairOptions, repo restic.Repository, path string, treeID return treeID, false, false, nil } - tree, err := repo.LoadTree(ctx, *treeID) + tree, err := restic.LoadTree(ctx, repo, *treeID) if err != nil { // mark as load error return &newID, false, true, nil @@ -246,7 +249,7 @@ func repairTree(opts RepairOptions, repo restic.Repository, path string, treeID } case "dir": // rewrite if necessary - newID, c, lErr, err := repairTree(opts, repo, path+node.Name+"/", node.Subtree, replaces, seen) + newID, c, lErr, err := repairTree(ctx, opts, repo, path+node.Name+"/", node.Subtree, replaces, seen) switch { case err != nil: return newID, true, false, err @@ -256,7 +259,7 @@ func repairTree(opts RepairOptions, repo restic.Repository, path string, treeID Printf("removed defective dir '%v'", path+node.Name) node.Name = node.Name + opts.Append Printf("(now empty '%v')\n", node.Name) - empty, err := emptyTree(opts.DryRun, repo) + empty, err := emptyTree(ctx, repo, opts.DryRun) if err != nil { return newID, true, false, err } @@ -277,7 +280,7 @@ func repairTree(opts RepairOptions, repo restic.Repository, path string, treeID tree.Nodes = newNodes if !opts.DryRun { - newID, err = repo.SaveTree(ctx, tree) + newID, err = restic.SaveTree(ctx, repo, tree) if err != nil { return &newID, true, false, err } @@ -290,11 +293,9 @@ func repairTree(opts RepairOptions, repo restic.Repository, path string, treeID return &newID, true, false, nil } -func emptyTree(dryRun bool, repo restic.Repository) (restic.ID, error) { - ctx := globalOptions.ctx - var tree restic.Tree +func emptyTree(ctx context.Context, repo restic.Repository, dryRun bool) (restic.ID, error) { if !dryRun { - return repo.SaveTree(ctx, &tree) + return restic.SaveTree(ctx, repo, &restic.Tree{}) } return restic.ID{}, nil } From db459eda21016aa4f60c2f14c0babd11f5a0f8c4 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 10 Dec 2022 17:25:38 +0100 Subject: [PATCH 09/25] move to subcommand --- ...{cmd_repair.go => cmd_repair_snapshots.go} | 28 +++++++++++-------- 1 file changed, 17 insertions(+), 11 deletions(-) rename cmd/restic/{cmd_repair.go => cmd_repair_snapshots.go} (87%) diff --git a/cmd/restic/cmd_repair.go b/cmd/restic/cmd_repair_snapshots.go similarity index 87% rename from cmd/restic/cmd_repair.go rename to cmd/restic/cmd_repair_snapshots.go index 30fe8be1c..a1e6b7f61 100644 --- a/cmd/restic/cmd_repair.go +++ b/cmd/restic/cmd_repair_snapshots.go @@ -11,10 +11,15 @@ import ( ) var cmdRepair = &cobra.Command{ - Use: "repair [flags] [snapshot ID] [...]", + Use: "repair", + Short: "Repair commands", +} + +var cmdRepairSnapshots = &cobra.Command{ + Use: "snapshots [flags] [snapshot ID] [...]", Short: "Repair snapshots", Long: ` -The "repair" command allows to repair broken snapshots. +The "repair snapshots" command allows to repair broken snapshots. It scans the given snapshots and generates new ones where damaged tress and file contents are removed. If the broken snapshots are deleted, a prune run will @@ -44,7 +49,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - return runRepair(cmd.Context(), globalOptions, repairOptions, args) + return runRepairSnapshots(cmd.Context(), globalOptions, repairSnapshotOptions, args) }, } @@ -58,21 +63,22 @@ type RepairOptions struct { DeleteSnapshots bool } -var repairOptions RepairOptions +var repairSnapshotOptions RepairOptions func init() { cmdRoot.AddCommand(cmdRepair) - flags := cmdRepair.Flags() + cmdRepair.AddCommand(cmdRepairSnapshots) + flags := cmdRepairSnapshots.Flags() - initMultiSnapshotFilter(flags, &repairOptions.SnapshotFilter, true) + initMultiSnapshotFilter(flags, &repairSnapshotOptions.SnapshotFilter, true) - flags.StringVar(&repairOptions.AddTag, "add-tag", "repaired", "tag to add to repaired snapshots") - flags.StringVar(&repairOptions.Append, "append", ".repaired", "string to append to repaired dirs/files; remove files if empty or impossible to repair") - flags.BoolVarP(&repairOptions.DryRun, "dry-run", "n", true, "don't do anything, only show what would be done") - flags.BoolVar(&repairOptions.DeleteSnapshots, "delete-snapshots", false, "delete original snapshots") + flags.StringVar(&repairSnapshotOptions.AddTag, "add-tag", "repaired", "tag to add to repaired snapshots") + flags.StringVar(&repairSnapshotOptions.Append, "append", ".repaired", "string to append to repaired dirs/files; remove files if empty or impossible to repair") + flags.BoolVarP(&repairSnapshotOptions.DryRun, "dry-run", "n", true, "don't do anything, only show what would be done") + flags.BoolVar(&repairSnapshotOptions.DeleteSnapshots, "delete-snapshots", false, "delete original snapshots") } -func runRepair(ctx context.Context, gopts GlobalOptions, opts RepairOptions, args []string) error { +func runRepairSnapshots(ctx context.Context, gopts GlobalOptions, opts RepairOptions, args []string) error { switch { case opts.DryRun: Printf("\n note: --dry-run is set\n-> repair will only show what it would do.\n\n") From 118d599d0ac0053560262e17af3dda8eb92d770a Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Tue, 27 Dec 2022 18:25:39 +0100 Subject: [PATCH 10/25] Rename 'rebuild-index' to 'repair index' The old name still works, but is deprecated. --- cmd/restic/cmd_check.go | 2 +- cmd/restic/cmd_prune.go | 2 +- cmd/restic/cmd_repair.go | 14 ++++++++ ...d_rebuild_index.go => cmd_repair_index.go} | 36 ++++++++++++------- cmd/restic/cmd_repair_snapshots.go | 9 +---- cmd/restic/integration_test.go | 14 ++++---- doc/060_forget.rst | 2 +- doc/077_troubleshooting.rst | 12 +++---- doc/manual_rest.rst | 2 +- internal/archiver/archiver.go | 2 +- 10 files changed, 57 insertions(+), 38 deletions(-) create mode 100644 cmd/restic/cmd_repair.go rename cmd/restic/{cmd_rebuild_index.go => cmd_repair_index.go} (75%) diff --git a/cmd/restic/cmd_check.go b/cmd/restic/cmd_check.go index e5f29a7e5..b9f3199b2 100644 --- a/cmd/restic/cmd_check.go +++ b/cmd/restic/cmd_check.go @@ -245,7 +245,7 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args } if suggestIndexRebuild { - Printf("Duplicate packs/old indexes are non-critical, you can run `restic rebuild-index' to correct this.\n") + Printf("Duplicate packs/old indexes are non-critical, you can run `restic repair index' to correct this.\n") } if mixedFound { Printf("Mixed packs with tree and data blobs are non-critical, you can run `restic prune` to correct this.\n") diff --git a/cmd/restic/cmd_prune.go b/cmd/restic/cmd_prune.go index 6104002b0..1138bb55b 100644 --- a/cmd/restic/cmd_prune.go +++ b/cmd/restic/cmd_prune.go @@ -488,7 +488,7 @@ func decidePackAction(ctx context.Context, opts PruneOptions, repo restic.Reposi // Pack size does not fit and pack is needed => error // If the pack is not needed, this is no error, the pack can // and will be simply removed, see below. - Warnf("pack %s: calculated size %d does not match real size %d\nRun 'restic rebuild-index'.\n", + Warnf("pack %s: calculated size %d does not match real size %d\nRun 'restic repair index'.\n", id.Str(), p.unusedSize+p.usedSize, packSize) return errorSizeNotMatching } diff --git a/cmd/restic/cmd_repair.go b/cmd/restic/cmd_repair.go new file mode 100644 index 000000000..aefe02f3c --- /dev/null +++ b/cmd/restic/cmd_repair.go @@ -0,0 +1,14 @@ +package main + +import ( + "github.com/spf13/cobra" +) + +var cmdRepair = &cobra.Command{ + Use: "repair", + Short: "Repair the repository", +} + +func init() { + cmdRoot.AddCommand(cmdRepair) +} diff --git a/cmd/restic/cmd_rebuild_index.go b/cmd/restic/cmd_repair_index.go similarity index 75% rename from cmd/restic/cmd_rebuild_index.go rename to cmd/restic/cmd_repair_index.go index 5d70a9e12..25d6b1cab 100644 --- a/cmd/restic/cmd_rebuild_index.go +++ b/cmd/restic/cmd_repair_index.go @@ -7,15 +7,15 @@ import ( "github.com/restic/restic/internal/pack" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" - "github.com/spf13/cobra" + "github.com/spf13/pflag" ) -var cmdRebuildIndex = &cobra.Command{ - Use: "rebuild-index [flags]", +var cmdRepairIndex = &cobra.Command{ + Use: "index [flags]", Short: "Build a new index", Long: ` -The "rebuild-index" command creates a new index based on the pack files in the +The "repair index" command creates a new index based on the pack files in the repository. EXIT STATUS @@ -25,25 +25,37 @@ Exit status is 0 if the command was successful, and non-zero if there was any er `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - return runRebuildIndex(cmd.Context(), rebuildIndexOptions, globalOptions) + return runRebuildIndex(cmd.Context(), repairIndexOptions, globalOptions) }, } -// RebuildIndexOptions collects all options for the rebuild-index command. -type RebuildIndexOptions struct { +var cmdRebuildIndex = &cobra.Command{ + Use: "rebuild-index [flags]", + Short: cmdRepairIndex.Short, + Long: cmdRepairIndex.Long, + Deprecated: `Use "repair index" instead`, + DisableAutoGenTag: true, + RunE: cmdRepairIndex.RunE, +} + +// RepairIndexOptions collects all options for the repair index command. +type RepairIndexOptions struct { ReadAllPacks bool } -var rebuildIndexOptions RebuildIndexOptions +var repairIndexOptions RepairIndexOptions func init() { + cmdRepair.AddCommand(cmdRepairIndex) + // add alias for old name cmdRoot.AddCommand(cmdRebuildIndex) - f := cmdRebuildIndex.Flags() - f.BoolVar(&rebuildIndexOptions.ReadAllPacks, "read-all-packs", false, "read all pack files to generate new index from scratch") + for _, f := range []*pflag.FlagSet{cmdRepairIndex.Flags(), cmdRebuildIndex.Flags()} { + f.BoolVar(&repairIndexOptions.ReadAllPacks, "read-all-packs", false, "read all pack files to generate new index from scratch") + } } -func runRebuildIndex(ctx context.Context, opts RebuildIndexOptions, gopts GlobalOptions) error { +func runRebuildIndex(ctx context.Context, opts RepairIndexOptions, gopts GlobalOptions) error { repo, err := OpenRepository(ctx, gopts) if err != nil { return err @@ -58,7 +70,7 @@ func runRebuildIndex(ctx context.Context, opts RebuildIndexOptions, gopts Global return rebuildIndex(ctx, opts, gopts, repo, restic.NewIDSet()) } -func rebuildIndex(ctx context.Context, opts RebuildIndexOptions, gopts GlobalOptions, repo *repository.Repository, ignorePacks restic.IDSet) error { +func rebuildIndex(ctx context.Context, opts RepairIndexOptions, gopts GlobalOptions, repo *repository.Repository, ignorePacks restic.IDSet) error { var obsoleteIndexes restic.IDs packSizeFromList := make(map[restic.ID]int64) packSizeFromIndex := make(map[restic.ID]int64) diff --git a/cmd/restic/cmd_repair_snapshots.go b/cmd/restic/cmd_repair_snapshots.go index a1e6b7f61..8b9005900 100644 --- a/cmd/restic/cmd_repair_snapshots.go +++ b/cmd/restic/cmd_repair_snapshots.go @@ -10,11 +10,6 @@ import ( "github.com/spf13/cobra" ) -var cmdRepair = &cobra.Command{ - Use: "repair", - Short: "Repair commands", -} - var cmdRepairSnapshots = &cobra.Command{ Use: "snapshots [flags] [snapshot ID] [...]", Short: "Repair snapshots", @@ -27,7 +22,7 @@ be able to refit the repository. The command depends on a good state of the index, so if there are inaccurancies in the index, make sure to run -"rebuild-index" before! +"repair index" before! WARNING: @@ -66,12 +61,10 @@ type RepairOptions struct { var repairSnapshotOptions RepairOptions func init() { - cmdRoot.AddCommand(cmdRepair) cmdRepair.AddCommand(cmdRepairSnapshots) flags := cmdRepairSnapshots.Flags() initMultiSnapshotFilter(flags, &repairSnapshotOptions.SnapshotFilter, true) - flags.StringVar(&repairSnapshotOptions.AddTag, "add-tag", "repaired", "tag to add to repaired snapshots") flags.StringVar(&repairSnapshotOptions.Append, "append", ".repaired", "string to append to repaired dirs/files; remove files if empty or impossible to repair") flags.BoolVarP(&repairSnapshotOptions.DryRun, "dry-run", "n", true, "don't do anything, only show what would be done") diff --git a/cmd/restic/integration_test.go b/cmd/restic/integration_test.go index 10ebbaf13..42fd26d6b 100644 --- a/cmd/restic/integration_test.go +++ b/cmd/restic/integration_test.go @@ -188,7 +188,7 @@ func testRunRebuildIndex(t testing.TB, gopts GlobalOptions) { globalOptions.stdout = os.Stdout }() - rtest.OK(t, runRebuildIndex(context.TODO(), RebuildIndexOptions{}, gopts)) + rtest.OK(t, runRebuildIndex(context.TODO(), RepairIndexOptions{}, gopts)) } func testRunLs(t testing.TB, gopts GlobalOptions, snapshotID string) []string { @@ -1504,8 +1504,8 @@ func testRebuildIndex(t *testing.T, backendTestHook backendWrapper) { t.Fatalf("expected no error from checker for test repository, got %v", err) } - if !strings.Contains(out, "restic rebuild-index") { - t.Fatalf("did not find hint for rebuild-index command") + if !strings.Contains(out, "restic repair index") { + t.Fatalf("did not find hint for repair index command") } env.gopts.backendTestHook = backendTestHook @@ -1518,7 +1518,7 @@ func testRebuildIndex(t *testing.T, backendTestHook backendWrapper) { } if err != nil { - t.Fatalf("expected no error from checker after rebuild-index, got: %v", err) + t.Fatalf("expected no error from checker after repair index, got: %v", err) } } @@ -1599,7 +1599,7 @@ func TestRebuildIndexFailsOnAppendOnly(t *testing.T) { env.gopts.backendTestHook = func(r restic.Backend) (restic.Backend, error) { return &appendOnlyBackend{r}, nil } - err := runRebuildIndex(context.TODO(), RebuildIndexOptions{}, env.gopts) + err := runRebuildIndex(context.TODO(), RepairIndexOptions{}, env.gopts) if err == nil { t.Error("expected rebuildIndex to fail") } @@ -1887,8 +1887,8 @@ func TestListOnce(t *testing.T) { testRunPrune(t, env.gopts, pruneOpts) rtest.OK(t, runCheck(context.TODO(), checkOpts, env.gopts, nil)) - rtest.OK(t, runRebuildIndex(context.TODO(), RebuildIndexOptions{}, env.gopts)) - rtest.OK(t, runRebuildIndex(context.TODO(), RebuildIndexOptions{ReadAllPacks: true}, env.gopts)) + rtest.OK(t, runRebuildIndex(context.TODO(), RepairIndexOptions{}, env.gopts)) + rtest.OK(t, runRebuildIndex(context.TODO(), RepairIndexOptions{ReadAllPacks: true}, env.gopts)) } func TestHardLink(t *testing.T) { diff --git a/doc/060_forget.rst b/doc/060_forget.rst index 2353ef6a0..72c7ae97f 100644 --- a/doc/060_forget.rst +++ b/doc/060_forget.rst @@ -472,7 +472,7 @@ space. However, a **failed** ``prune`` run can cause the repository to become **temporarily unusable**. Therefore, make sure that you have a stable connection to the repository storage, before running this command. In case the command fails, it may become necessary to manually remove all files from the `index/` folder of the repository and -run `rebuild-index` afterwards. +run `repair index` afterwards. To prevent accidental usages of the ``--unsafe-recover-no-free-space`` option it is necessary to first run ``prune --unsafe-recover-no-free-space SOME-ID`` and then replace diff --git a/doc/077_troubleshooting.rst b/doc/077_troubleshooting.rst index 50c19565e..5b86ffd87 100644 --- a/doc/077_troubleshooting.rst +++ b/doc/077_troubleshooting.rst @@ -58,17 +58,17 @@ But make sure that your needed data is also still contained in your repository ; Note that `check` also prints out warning in some cases. These warnings point out that the repo may be optimized but is still in perfect shape and does not need any troubleshooting. -3. Index trouble -> `rebuild-index` +3. Index trouble -> `repair index` ******************************************** A common problem with broken repostories is that the index does no longer correctly represent the contents of your pack files. This is especially the case if some pack files got lost. -`rebuild-index` recovers this situation and ensures that the index exactly represents the pack files. +`repair index` recovers this situation and ensures that the index exactly represents the pack files. You might even need to manually remove corrupted pack files. In this case make sure, you run -`restic rebuild-index` after. +`restic repair index` after. -Also if you encounter problems with the index files itselves, `rebuild-index` will solve these problems +Also if you encounter problems with the index files itselves, `repair index` will solve these problems immediately. However, rebuilding the index does not solve every problem, e.g. lost pack files. @@ -91,7 +91,7 @@ backup again and check if this did heal your repository. If you realize that a specific file is broken in your repository and you have this file, any run of `backup` which includes that file will be able to heal the situation. -Note that `backup` relies on a correct index state, so make sure your index is fine or run `rebuild-index` +Note that `backup` relies on a correct index state, so make sure your index is fine or run `repair index` before running `backup`. 6. Unreferenced tree -> `recover` @@ -101,7 +101,7 @@ If for some reason you have unreferenced trees in your repository but you actual `recover` it will generate a new snapshot which allows access to all trees that you have in your repository. -Note that `recover` relies on a correct index state, so make sure your index is fine or run `rebuild-index` +Note that `recover` relies on a correct index state, so make sure your index is fine or run `repair index` before running `recover`. 7. Repair defect snapshots using `repair` diff --git a/doc/manual_rest.rst b/doc/manual_rest.rst index f812e3a70..093144722 100644 --- a/doc/manual_rest.rst +++ b/doc/manual_rest.rst @@ -35,8 +35,8 @@ Usage help is available: migrate Apply migrations mount Mount the repository prune Remove unneeded data from the repository - rebuild-index Build a new index recover Recover data from the repository not referenced by snapshots + repair Repair the repository restore Extract the data from a snapshot rewrite Rewrite snapshots to exclude unwanted files self-update Update the restic binary diff --git a/internal/archiver/archiver.go b/internal/archiver/archiver.go index a56965d63..3c1cc33d0 100644 --- a/internal/archiver/archiver.go +++ b/internal/archiver/archiver.go @@ -207,7 +207,7 @@ func (arch *Archiver) wrapLoadTreeError(id restic.ID, err error) error { if arch.Repo.Index().Has(restic.BlobHandle{ID: id, Type: restic.TreeBlob}) { err = errors.Errorf("tree %v could not be loaded; the repository could be damaged: %v", id, err) } else { - err = errors.Errorf("tree %v is not known; the repository could be damaged, run `rebuild-index` to try to repair it", id) + err = errors.Errorf("tree %v is not known; the repository could be damaged, run `repair index` to try to repair it", id) } return err } From 903651c719ebae82b8c25fe29f7bf78a7400ab2f Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Tue, 27 Dec 2022 20:24:49 +0100 Subject: [PATCH 11/25] repair snapshots: partially synchronize code with rewrite command Simplify CLI options: * Rename "DeleteSnapshots" to "Forget" * Replace "AddTag" and "Append" with hardcoded values Change output and snapshot modifications to be more in line with the "rewrite" command. --- cmd/restic/cmd_repair_snapshots.go | 62 +++++++++++++++--------------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/cmd/restic/cmd_repair_snapshots.go b/cmd/restic/cmd_repair_snapshots.go index 8b9005900..8a51d72be 100644 --- a/cmd/restic/cmd_repair_snapshots.go +++ b/cmd/restic/cmd_repair_snapshots.go @@ -3,6 +3,7 @@ package main import ( "context" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/restic" "golang.org/x/sync/errgroup" @@ -50,12 +51,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er // RepairOptions collects all options for the repair command. type RepairOptions struct { - restic.SnapshotFilter + DryRun bool + Forget bool - AddTag string - Append string - DryRun bool - DeleteSnapshots bool + restic.SnapshotFilter } var repairSnapshotOptions RepairOptions @@ -64,28 +63,31 @@ func init() { cmdRepair.AddCommand(cmdRepairSnapshots) flags := cmdRepairSnapshots.Flags() + flags.BoolVarP(&repairSnapshotOptions.DryRun, "dry-run", "n", false, "do not do anything, just print what would be done") + flags.BoolVarP(&repairSnapshotOptions.Forget, "forget", "", false, "remove original snapshots after creating new ones") + initMultiSnapshotFilter(flags, &repairSnapshotOptions.SnapshotFilter, true) - flags.StringVar(&repairSnapshotOptions.AddTag, "add-tag", "repaired", "tag to add to repaired snapshots") - flags.StringVar(&repairSnapshotOptions.Append, "append", ".repaired", "string to append to repaired dirs/files; remove files if empty or impossible to repair") - flags.BoolVarP(&repairSnapshotOptions.DryRun, "dry-run", "n", true, "don't do anything, only show what would be done") - flags.BoolVar(&repairSnapshotOptions.DeleteSnapshots, "delete-snapshots", false, "delete original snapshots") } func runRepairSnapshots(ctx context.Context, gopts GlobalOptions, opts RepairOptions, args []string) error { - switch { - case opts.DryRun: - Printf("\n note: --dry-run is set\n-> repair will only show what it would do.\n\n") - case opts.DeleteSnapshots: - Printf("\n note: --dry-run is not set and --delete-snapshots is set\n-> this may result in data loss!\n\n") - } - repo, err := OpenRepository(ctx, globalOptions) if err != nil { return err } - lock, ctx, err := lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON) - defer unlockRepo(lock) + if !opts.DryRun { + var lock *restic.Lock + var err error + lock, ctx, err = lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON) + defer unlockRepo(lock) + if err != nil { + return err + } + } else { + repo.SetDryRun() + } + + snapshotLister, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile) if err != nil { return err } @@ -96,7 +98,7 @@ func runRepairSnapshots(ctx context.Context, gopts GlobalOptions, opts RepairOpt // get snapshots to check & repair var snapshots []*restic.Snapshot - for sn := range FindFilteredSnapshots(ctx, repo.Backend(), repo, &opts.SnapshotFilter, args) { + for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, &opts.SnapshotFilter, args) { snapshots = append(snapshots, sn) } @@ -124,7 +126,7 @@ func repairSnapshots(ctx context.Context, opts RepairOptions, repo restic.Reposi Printf("the root tree is damaged -> delete snapshot.\n") deleteSn.Insert(*sn.ID()) case changed: - err = changeSnapshot(ctx, opts, repo, sn, newID) + err = changeSnapshot(ctx, opts.DryRun, repo, sn, newID) if err != nil { return err } @@ -144,7 +146,7 @@ func repairSnapshots(ctx context.Context, opts RepairOptions, repo restic.Reposi return err } - if len(deleteSn) > 0 && opts.DeleteSnapshots { + if len(deleteSn) > 0 && opts.Forget { Verbosef("delete %d snapshots...\n", len(deleteSn)) if !opts.DryRun { DeleteFiles(ctx, globalOptions, repo, deleteSn, restic.SnapshotFile) @@ -158,14 +160,12 @@ func repairSnapshots(ctx context.Context, opts RepairOptions, repo restic.Reposi // - add the rag opts.AddTag // - preserve original ID // if opts.DryRun is set, it doesn't change anything but only -func changeSnapshot(ctx context.Context, opts RepairOptions, repo restic.Repository, sn *restic.Snapshot, newID *restic.ID) error { - sn.AddTags([]string{opts.AddTag}) - // Retain the original snapshot id over all tag changes. - if sn.Original == nil { - sn.Original = sn.ID() - } +func changeSnapshot(ctx context.Context, dryRun bool, repo restic.Repository, sn *restic.Snapshot, newID *restic.ID) error { + sn.AddTags([]string{"repaired"}) + // Always set the original snapshot id as this essentially a new snapshot. + sn.Original = sn.ID() sn.Tree = newID - if !opts.DryRun { + if !dryRun { newID, err := restic.SaveSnapshot(ctx, repo, sn) if err != nil { return err @@ -236,12 +236,12 @@ func repairTree(ctx context.Context, opts RepairOptions, repo restic.Repository, } if !ok { changed = true - if opts.Append == "" || newSize == 0 { + if newSize == 0 { Printf("removed defective file '%v'\n", path+node.Name) continue } Printf("repaired defective file '%v'", path+node.Name) - node.Name = node.Name + opts.Append + node.Name = node.Name + ".repaired" Printf(" to '%v'\n", node.Name) node.Content = newContent node.Size = newSize @@ -256,7 +256,7 @@ func repairTree(ctx context.Context, opts RepairOptions, repo restic.Repository, // If we get an error, we remove this subtree changed = true Printf("removed defective dir '%v'", path+node.Name) - node.Name = node.Name + opts.Append + node.Name = node.Name + ".repaired" Printf("(now empty '%v')\n", node.Name) empty, err := emptyTree(ctx, repo, opts.DryRun) if err != nil { From 375189488cf5e391b9aba1405803398dd3d7daa3 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Tue, 27 Dec 2022 21:05:21 +0100 Subject: [PATCH 12/25] rewrite: prepare for code sharing with rewrite snapshots --- cmd/restic/cmd_rewrite.go | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/cmd/restic/cmd_rewrite.go b/cmd/restic/cmd_rewrite.go index 744686390..4019d9264 100644 --- a/cmd/restic/cmd_rewrite.go +++ b/cmd/restic/cmd_rewrite.go @@ -87,22 +87,31 @@ func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *resti return true } + return filterAndReplaceSnapshot(ctx, repo, sn, + func(ctx context.Context, sn *restic.Snapshot) (restic.ID, error) { + return walker.FilterTree(ctx, repo, "/", *sn.Tree, &walker.TreeFilterVisitor{ + SelectByName: selectByName, + PrintExclude: func(path string) { Verbosef(fmt.Sprintf("excluding %s\n", path)) }, + }) + }, opts.DryRun, opts.Forget, "rewrite") +} + +func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *restic.Snapshot, filter func(ctx context.Context, sn *restic.Snapshot) (restic.ID, error), dryRun bool, forget bool, addTag string) (bool, error) { + wg, wgCtx := errgroup.WithContext(ctx) repo.StartPackUploader(wgCtx, wg) var filteredTree restic.ID wg.Go(func() error { - filteredTree, err = walker.FilterTree(wgCtx, repo, "/", *sn.Tree, &walker.TreeFilterVisitor{ - SelectByName: selectByName, - PrintExclude: func(path string) { Verbosef(fmt.Sprintf("excluding %s\n", path)) }, - }) + var err error + filteredTree, err = filter(ctx, sn) if err != nil { return err } return repo.Flush(wgCtx) }) - err = wg.Wait() + err := wg.Wait() if err != nil { return false, err } @@ -113,10 +122,10 @@ func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *resti } debug.Log("Snapshot %v modified", sn) - if opts.DryRun { + if dryRun { Verbosef("would save new snapshot\n") - if opts.Forget { + if forget { Verbosef("would remove old snapshot\n") } @@ -125,10 +134,10 @@ func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *resti // Always set the original snapshot id as this essentially a new snapshot. sn.Original = sn.ID() - *sn.Tree = filteredTree + sn.Tree = &filteredTree - if !opts.Forget { - sn.AddTags([]string{"rewrite"}) + if !forget { + sn.AddTags([]string{addTag}) } // Save the new snapshot. @@ -138,7 +147,7 @@ func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *resti } Verbosef("saved new snapshot %v\n", id.Str()) - if opts.Forget { + if forget { h := restic.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()} if err = repo.Backend().Remove(ctx, h); err != nil { return false, err From 8c4caf09a82a5b8adc540d92abc68194051af679 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Tue, 27 Dec 2022 21:34:07 +0100 Subject: [PATCH 13/25] repair snapshots: Do not rename repaired files The files in a tree must be sorted in lexical order. However, this cannot be guaranteed when appending a filename suffix. For two files file, file.rep where "file" is broken, this would result in file.repaired, file.rep which is no longer sorted. In addition, adding a filename suffix is also prone to filename collisions which would require a rather complex search for a collision-free name in order to work reliably. --- cmd/restic/cmd_repair_snapshots.go | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/cmd/restic/cmd_repair_snapshots.go b/cmd/restic/cmd_repair_snapshots.go index 8a51d72be..0413c43bf 100644 --- a/cmd/restic/cmd_repair_snapshots.go +++ b/cmd/restic/cmd_repair_snapshots.go @@ -240,9 +240,7 @@ func repairTree(ctx context.Context, opts RepairOptions, repo restic.Repository, Printf("removed defective file '%v'\n", path+node.Name) continue } - Printf("repaired defective file '%v'", path+node.Name) - node.Name = node.Name + ".repaired" - Printf(" to '%v'\n", node.Name) + Printf("repaired defective file '%v'\n", path+node.Name) node.Content = newContent node.Size = newSize } @@ -255,9 +253,7 @@ func repairTree(ctx context.Context, opts RepairOptions, repo restic.Repository, case lErr: // If we get an error, we remove this subtree changed = true - Printf("removed defective dir '%v'", path+node.Name) - node.Name = node.Name + ".repaired" - Printf("(now empty '%v')\n", node.Name) + Printf("replaced defective dir '%v'", path+node.Name) empty, err := emptyTree(ctx, repo, opts.DryRun) if err != nil { return newID, true, false, err From 1a9705fc957fe1fedd709da5bbf016a5b2c4d98f Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Wed, 28 Dec 2022 10:38:40 +0100 Subject: [PATCH 14/25] walker: Simplify change detection in FilterTree Now the rewritten tree is always serialized which makes sure that we don't accidentally miss any relevant changes. --- internal/walker/rewriter.go | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/internal/walker/rewriter.go b/internal/walker/rewriter.go index cd05f69f5..96afbb07e 100644 --- a/internal/walker/rewriter.go +++ b/internal/walker/rewriter.go @@ -42,7 +42,6 @@ func FilterTree(ctx context.Context, repo BlobLoadSaver, nodepath string, nodeID debug.Log("filterTree: %s, nodeId: %s\n", nodepath, nodeID.Str()) - changed := false tb := restic.NewTreeJSONBuilder() for _, node := range curTree.Nodes { path := path.Join(nodepath, node.Name) @@ -50,7 +49,6 @@ func FilterTree(ctx context.Context, repo BlobLoadSaver, nodepath string, nodeID if visitor.PrintExclude != nil { visitor.PrintExclude(path) } - changed = true continue } @@ -65,9 +63,6 @@ func FilterTree(ctx context.Context, repo BlobLoadSaver, nodepath string, nodeID if err != nil { return restic.ID{}, err } - if !node.Subtree.Equal(newID) { - changed = true - } node.Subtree = &newID err = tb.AddNode(node) if err != nil { @@ -75,17 +70,15 @@ func FilterTree(ctx context.Context, repo BlobLoadSaver, nodepath string, nodeID } } - if changed { - tree, err := tb.Finalize() - if err != nil { - return restic.ID{}, err - } - - // Save new tree - newTreeID, _, _, err := repo.SaveBlob(ctx, restic.TreeBlob, tree, restic.ID{}, false) - debug.Log("filterTree: save new tree for %s as %v\n", nodepath, newTreeID) - return newTreeID, err + tree, err := tb.Finalize() + if err != nil { + return restic.ID{}, err } - return nodeID, nil + // Save new tree + newTreeID, _, _, err := repo.SaveBlob(ctx, restic.TreeBlob, tree, restic.ID{}, false) + if !newTreeID.Equal(nodeID) { + debug.Log("filterTree: save new tree for %s as %v\n", nodepath, newTreeID) + } + return newTreeID, err } From bc2399fbd9c9951602ae4ed6be9c346625d8face Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Wed, 28 Dec 2022 10:42:21 +0100 Subject: [PATCH 15/25] walker: recurse into directory based on node type A broken directory might also not have a subtree. --- internal/walker/rewriter.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/walker/rewriter.go b/internal/walker/rewriter.go index 96afbb07e..bef3bd688 100644 --- a/internal/walker/rewriter.go +++ b/internal/walker/rewriter.go @@ -52,7 +52,7 @@ func FilterTree(ctx context.Context, repo BlobLoadSaver, nodepath string, nodeID continue } - if node.Subtree == nil { + if node.Type != "dir" { err = tb.AddNode(node) if err != nil { return restic.ID{}, err From 38dac78180db7ae4d49e04cd16d4fb94ddae8c0a Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Wed, 28 Dec 2022 11:04:28 +0100 Subject: [PATCH 16/25] walker: restructure FilterTree into TreeRewriter The more generic RewriteNode callback replaces the SelectByName and PrintExclude functions. The main part of this change is a preparation to allow using the TreeRewriter for the `repair snapshots` command. --- cmd/restic/cmd_rewrite.go | 15 +++-- internal/walker/rewriter.go | 37 +++++++++---- internal/walker/rewriter_test.go | 94 +++++++++++++++++++++----------- internal/walker/walker_test.go | 5 +- 4 files changed, 103 insertions(+), 48 deletions(-) diff --git a/cmd/restic/cmd_rewrite.go b/cmd/restic/cmd_rewrite.go index 4019d9264..a60fdc8fc 100644 --- a/cmd/restic/cmd_rewrite.go +++ b/cmd/restic/cmd_rewrite.go @@ -87,12 +87,19 @@ func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *resti return true } + rewriter := walker.NewTreeRewriter(walker.RewriteOpts{ + RewriteNode: func(node *restic.Node, path string) *restic.Node { + if selectByName(path) { + return node + } + Verbosef(fmt.Sprintf("excluding %s\n", path)) + return nil + }, + }) + return filterAndReplaceSnapshot(ctx, repo, sn, func(ctx context.Context, sn *restic.Snapshot) (restic.ID, error) { - return walker.FilterTree(ctx, repo, "/", *sn.Tree, &walker.TreeFilterVisitor{ - SelectByName: selectByName, - PrintExclude: func(path string) { Verbosef(fmt.Sprintf("excluding %s\n", path)) }, - }) + return rewriter.RewriteTree(ctx, repo, "/", *sn.Tree) }, opts.DryRun, opts.Forget, "rewrite") } diff --git a/internal/walker/rewriter.go b/internal/walker/rewriter.go index bef3bd688..48f16a53a 100644 --- a/internal/walker/rewriter.go +++ b/internal/walker/rewriter.go @@ -9,13 +9,28 @@ import ( "github.com/restic/restic/internal/restic" ) -// SelectByNameFunc returns true for all items that should be included (files and -// dirs). If false is returned, files are ignored and dirs are not even walked. -type SelectByNameFunc func(item string) bool +type NodeRewriteFunc func(node *restic.Node, path string) *restic.Node -type TreeFilterVisitor struct { - SelectByName SelectByNameFunc - PrintExclude func(string) +type RewriteOpts struct { + // return nil to remove the node + RewriteNode NodeRewriteFunc +} + +type TreeRewriter struct { + opts RewriteOpts +} + +func NewTreeRewriter(opts RewriteOpts) *TreeRewriter { + rw := &TreeRewriter{ + opts: opts, + } + // setup default implementations + if rw.opts.RewriteNode == nil { + rw.opts.RewriteNode = func(node *restic.Node, path string) *restic.Node { + return node + } + } + return rw } type BlobLoadSaver interface { @@ -23,7 +38,7 @@ type BlobLoadSaver interface { restic.BlobLoader } -func FilterTree(ctx context.Context, repo BlobLoadSaver, nodepath string, nodeID restic.ID, visitor *TreeFilterVisitor) (newNodeID restic.ID, err error) { +func (t *TreeRewriter) RewriteTree(ctx context.Context, repo BlobLoadSaver, nodepath string, nodeID restic.ID) (newNodeID restic.ID, err error) { curTree, err := restic.LoadTree(ctx, repo, nodeID) if err != nil { return restic.ID{}, err @@ -45,10 +60,8 @@ func FilterTree(ctx context.Context, repo BlobLoadSaver, nodepath string, nodeID tb := restic.NewTreeJSONBuilder() for _, node := range curTree.Nodes { path := path.Join(nodepath, node.Name) - if !visitor.SelectByName(path) { - if visitor.PrintExclude != nil { - visitor.PrintExclude(path) - } + node = t.opts.RewriteNode(node, path) + if node == nil { continue } @@ -59,7 +72,7 @@ func FilterTree(ctx context.Context, repo BlobLoadSaver, nodepath string, nodeID } continue } - newID, err := FilterTree(ctx, repo, path, *node.Subtree, visitor) + newID, err := t.RewriteTree(ctx, repo, path, *node.Subtree) if err != nil { return restic.ID{}, err } diff --git a/internal/walker/rewriter_test.go b/internal/walker/rewriter_test.go index 3dcf0ac9e..8f99fe9bd 100644 --- a/internal/walker/rewriter_test.go +++ b/internal/walker/rewriter_test.go @@ -5,7 +5,6 @@ import ( "fmt" "testing" - "github.com/google/go-cmp/cmp" "github.com/pkg/errors" "github.com/restic/restic/internal/restic" ) @@ -38,26 +37,26 @@ func (t WritableTreeMap) Dump() { } } -type checkRewriteFunc func(t testing.TB) (visitor TreeFilterVisitor, final func(testing.TB)) +type checkRewriteFunc func(t testing.TB) (rewriter *TreeRewriter, final func(testing.TB)) // checkRewriteItemOrder ensures that the order of the 'path' arguments is the one passed in as 'want'. func checkRewriteItemOrder(want []string) checkRewriteFunc { pos := 0 - return func(t testing.TB) (visitor TreeFilterVisitor, final func(testing.TB)) { - vis := TreeFilterVisitor{ - SelectByName: func(path string) bool { + return func(t testing.TB) (rewriter *TreeRewriter, final func(testing.TB)) { + rewriter = NewTreeRewriter(RewriteOpts{ + RewriteNode: func(node *restic.Node, path string) *restic.Node { if pos >= len(want) { t.Errorf("additional unexpected path found: %v", path) - return false + return nil } if path != want[pos] { t.Errorf("wrong path found, want %q, got %q", want[pos], path) } pos++ - return true + return node }, - } + }) final = func(t testing.TB) { if pos != len(want) { @@ -65,21 +64,20 @@ func checkRewriteItemOrder(want []string) checkRewriteFunc { } } - return vis, final + return rewriter, final } } -// checkRewriteSkips excludes nodes if path is in skipFor, it checks that all excluded entries are printed. +// checkRewriteSkips excludes nodes if path is in skipFor, it checks that rewriting proceedes in the correct order. func checkRewriteSkips(skipFor map[string]struct{}, want []string) checkRewriteFunc { var pos int - printed := make(map[string]struct{}) - return func(t testing.TB) (visitor TreeFilterVisitor, final func(testing.TB)) { - vis := TreeFilterVisitor{ - SelectByName: func(path string) bool { + return func(t testing.TB) (rewriter *TreeRewriter, final func(testing.TB)) { + rewriter = NewTreeRewriter(RewriteOpts{ + RewriteNode: func(node *restic.Node, path string) *restic.Node { if pos >= len(want) { t.Errorf("additional unexpected path found: %v", path) - return false + return nil } if path != want[pos] { @@ -87,27 +85,39 @@ func checkRewriteSkips(skipFor map[string]struct{}, want []string) checkRewriteF } pos++ - _, ok := skipFor[path] - return !ok - }, - PrintExclude: func(s string) { - if _, ok := printed[s]; ok { - t.Errorf("path was already printed %v", s) + _, skip := skipFor[path] + if skip { + return nil } - printed[s] = struct{}{} + return node }, - } + }) final = func(t testing.TB) { - if !cmp.Equal(skipFor, printed) { - t.Errorf("unexpected paths skipped: %s", cmp.Diff(skipFor, printed)) - } if pos != len(want) { t.Errorf("not enough items returned, want %d, got %d", len(want), pos) } } - return vis, final + return rewriter, final + } +} + +// checkIncreaseNodeSize modifies each node by changing its size. +func checkIncreaseNodeSize(increase uint64) checkRewriteFunc { + return func(t testing.TB) (rewriter *TreeRewriter, final func(testing.TB)) { + rewriter = NewTreeRewriter(RewriteOpts{ + RewriteNode: func(node *restic.Node, path string) *restic.Node { + if node.Type == "file" { + node.Size += increase + } + return node + }, + }) + + final = func(t testing.TB) {} + + return rewriter, final } } @@ -172,6 +182,21 @@ func TestRewriter(t *testing.T) { }, ), }, + { // modify node + tree: TestTree{ + "foo": TestFile{Size: 21}, + "subdir": TestTree{ + "subfile": TestFile{Size: 21}, + }, + }, + newTree: TestTree{ + "foo": TestFile{Size: 42}, + "subdir": TestTree{ + "subfile": TestFile{Size: 42}, + }, + }, + check: checkIncreaseNodeSize(21), + }, } for _, test := range tests { @@ -186,8 +211,8 @@ func TestRewriter(t *testing.T) { ctx, cancel := context.WithCancel(context.TODO()) defer cancel() - vis, last := test.check(t) - newRoot, err := FilterTree(ctx, modrepo, "/", root, &vis) + rewriter, last := test.check(t) + newRoot, err := rewriter.RewriteTree(ctx, modrepo, "/", root) if err != nil { t.Error(err) } @@ -213,8 +238,15 @@ func TestRewriterFailOnUnknownFields(t *testing.T) { ctx, cancel := context.WithCancel(context.TODO()) defer cancel() - // use nil visitor to crash if the tree loading works unexpectedly - _, err := FilterTree(ctx, tm, "/", id, nil) + + rewriter := NewTreeRewriter(RewriteOpts{ + RewriteNode: func(node *restic.Node, path string) *restic.Node { + // tree loading must not succeed + t.Fail() + return node + }, + }) + _, err := rewriter.RewriteTree(ctx, tm, "/", id) if err == nil { t.Error("missing error on unknown field") diff --git a/internal/walker/walker_test.go b/internal/walker/walker_test.go index 6c4fd3436..8de1a9dc4 100644 --- a/internal/walker/walker_test.go +++ b/internal/walker/walker_test.go @@ -14,7 +14,9 @@ import ( type TestTree map[string]interface{} // TestNode is used to test the walker. -type TestFile struct{} +type TestFile struct { + Size uint64 +} func BuildTreeMap(tree TestTree) (m TreeMap, root restic.ID) { m = TreeMap{} @@ -37,6 +39,7 @@ func buildTreeMap(tree TestTree, m TreeMap) restic.ID { err := tb.AddNode(&restic.Node{ Name: name, Type: "file", + Size: elem.Size, }) if err != nil { panic(err) From 1bd1f3008ddf74f6f702c1c9c2b7c86d938e83f6 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Wed, 28 Dec 2022 11:34:55 +0100 Subject: [PATCH 17/25] walker: extend TreeRewriter to support snapshot repairing This adds support for caching already rewritten trees, handling of load errors and disabling the check that the serialization doesn't lead to data loss. --- cmd/restic/cmd_rewrite.go | 1 + internal/walker/rewriter.go | 58 +++++++++++++--- internal/walker/rewriter_test.go | 114 ++++++++++++++++++++++++++++++- 3 files changed, 161 insertions(+), 12 deletions(-) diff --git a/cmd/restic/cmd_rewrite.go b/cmd/restic/cmd_rewrite.go index a60fdc8fc..e5c65850d 100644 --- a/cmd/restic/cmd_rewrite.go +++ b/cmd/restic/cmd_rewrite.go @@ -95,6 +95,7 @@ func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *resti Verbosef(fmt.Sprintf("excluding %s\n", path)) return nil }, + DisableNodeCache: true, }) return filterAndReplaceSnapshot(ctx, repo, sn, diff --git a/internal/walker/rewriter.go b/internal/walker/rewriter.go index 48f16a53a..649857032 100644 --- a/internal/walker/rewriter.go +++ b/internal/walker/rewriter.go @@ -10,26 +10,45 @@ import ( ) type NodeRewriteFunc func(node *restic.Node, path string) *restic.Node +type FailedTreeRewriteFunc func(nodeID restic.ID, path string, err error) (restic.ID, error) type RewriteOpts struct { // return nil to remove the node RewriteNode NodeRewriteFunc + // decide what to do with a tree that could not be loaded. Return nil to remove the node. By default the load error is returned which causes the operation to fail. + RewriteFailedTree FailedTreeRewriteFunc + + AllowUnstableSerialization bool + DisableNodeCache bool } +type idMap map[restic.ID]restic.ID + type TreeRewriter struct { opts RewriteOpts + + replaces idMap } func NewTreeRewriter(opts RewriteOpts) *TreeRewriter { rw := &TreeRewriter{ opts: opts, } + if !opts.DisableNodeCache { + rw.replaces = make(idMap) + } // setup default implementations if rw.opts.RewriteNode == nil { rw.opts.RewriteNode = func(node *restic.Node, path string) *restic.Node { return node } } + if rw.opts.RewriteFailedTree == nil { + // fail with error by default + rw.opts.RewriteFailedTree = func(nodeID restic.ID, path string, err error) (restic.ID, error) { + return restic.ID{}, err + } + } return rw } @@ -39,20 +58,29 @@ type BlobLoadSaver interface { } func (t *TreeRewriter) RewriteTree(ctx context.Context, repo BlobLoadSaver, nodepath string, nodeID restic.ID) (newNodeID restic.ID, err error) { - curTree, err := restic.LoadTree(ctx, repo, nodeID) - if err != nil { - return restic.ID{}, err + // check if tree was already changed + newID, ok := t.replaces[nodeID] + if ok { + return newID, nil } - // check that we can properly encode this tree without losing information - // The alternative of using json/Decoder.DisallowUnknownFields() doesn't work as we use - // a custom UnmarshalJSON to decode trees, see also https://github.com/golang/go/issues/41144 - testID, err := restic.SaveTree(ctx, repo, curTree) + // a nil nodeID will lead to a load error + curTree, err := restic.LoadTree(ctx, repo, nodeID) if err != nil { - return restic.ID{}, err + return t.opts.RewriteFailedTree(nodeID, nodepath, err) } - if nodeID != testID { - return restic.ID{}, fmt.Errorf("cannot encode tree at %q without losing information", nodepath) + + if !t.opts.AllowUnstableSerialization { + // check that we can properly encode this tree without losing information + // The alternative of using json/Decoder.DisallowUnknownFields() doesn't work as we use + // a custom UnmarshalJSON to decode trees, see also https://github.com/golang/go/issues/41144 + testID, err := restic.SaveTree(ctx, repo, curTree) + if err != nil { + return restic.ID{}, err + } + if nodeID != testID { + return restic.ID{}, fmt.Errorf("cannot encode tree at %q without losing information", nodepath) + } } debug.Log("filterTree: %s, nodeId: %s\n", nodepath, nodeID.Str()) @@ -72,7 +100,12 @@ func (t *TreeRewriter) RewriteTree(ctx context.Context, repo BlobLoadSaver, node } continue } - newID, err := t.RewriteTree(ctx, repo, path, *node.Subtree) + // treat nil as null id + var subtree restic.ID + if node.Subtree != nil { + subtree = *node.Subtree + } + newID, err := t.RewriteTree(ctx, repo, path, subtree) if err != nil { return restic.ID{}, err } @@ -90,6 +123,9 @@ func (t *TreeRewriter) RewriteTree(ctx context.Context, repo BlobLoadSaver, node // Save new tree newTreeID, _, _, err := repo.SaveBlob(ctx, restic.TreeBlob, tree, restic.ID{}, false) + if t.replaces != nil { + t.replaces[nodeID] = newTreeID + } if !newTreeID.Equal(nodeID) { debug.Log("filterTree: save new tree for %s as %v\n", nodepath, newTreeID) } diff --git a/internal/walker/rewriter_test.go b/internal/walker/rewriter_test.go index 8f99fe9bd..07ce5f72f 100644 --- a/internal/walker/rewriter_test.go +++ b/internal/walker/rewriter_test.go @@ -7,6 +7,7 @@ import ( "github.com/pkg/errors" "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/test" ) // WritableTreeMap also support saving @@ -69,7 +70,7 @@ func checkRewriteItemOrder(want []string) checkRewriteFunc { } // checkRewriteSkips excludes nodes if path is in skipFor, it checks that rewriting proceedes in the correct order. -func checkRewriteSkips(skipFor map[string]struct{}, want []string) checkRewriteFunc { +func checkRewriteSkips(skipFor map[string]struct{}, want []string, disableCache bool) checkRewriteFunc { var pos int return func(t testing.TB) (rewriter *TreeRewriter, final func(testing.TB)) { @@ -91,6 +92,7 @@ func checkRewriteSkips(skipFor map[string]struct{}, want []string) checkRewriteF } return node }, + DisableNodeCache: disableCache, }) final = func(t testing.TB) { @@ -160,6 +162,7 @@ func TestRewriter(t *testing.T) { "/subdir", "/subdir/subfile", }, + false, ), }, { // exclude dir @@ -180,6 +183,7 @@ func TestRewriter(t *testing.T) { "/foo", "/subdir", }, + false, ), }, { // modify node @@ -197,6 +201,75 @@ func TestRewriter(t *testing.T) { }, check: checkIncreaseNodeSize(21), }, + { // test cache + tree: TestTree{ + // both subdirs are identical + "subdir1": TestTree{ + "subfile": TestFile{}, + "subfile2": TestFile{}, + }, + "subdir2": TestTree{ + "subfile": TestFile{}, + "subfile2": TestFile{}, + }, + }, + newTree: TestTree{ + "subdir1": TestTree{ + "subfile2": TestFile{}, + }, + "subdir2": TestTree{ + "subfile2": TestFile{}, + }, + }, + check: checkRewriteSkips( + map[string]struct{}{ + "/subdir1/subfile": {}, + }, + []string{ + "/subdir1", + "/subdir1/subfile", + "/subdir1/subfile2", + "/subdir2", + }, + false, + ), + }, + { // test disabled cache + tree: TestTree{ + // both subdirs are identical + "subdir1": TestTree{ + "subfile": TestFile{}, + "subfile2": TestFile{}, + }, + "subdir2": TestTree{ + "subfile": TestFile{}, + "subfile2": TestFile{}, + }, + }, + newTree: TestTree{ + "subdir1": TestTree{ + "subfile2": TestFile{}, + }, + "subdir2": TestTree{ + "subfile": TestFile{}, + "subfile2": TestFile{}, + }, + }, + check: checkRewriteSkips( + map[string]struct{}{ + "/subdir1/subfile": {}, + }, + []string{ + "/subdir1", + "/subdir1/subfile", + "/subdir1/subfile2", + "/subdir2", + "/subdir2/subfile", + "/subdir2/subfile2", + }, + true, + ), + }, } for _, test := range tests { @@ -251,4 +324,43 @@ func TestRewriterFailOnUnknownFields(t *testing.T) { if err == nil { t.Error("missing error on unknown field") } + + // check that the serialization check can be disabled + rewriter = NewTreeRewriter(RewriteOpts{ + AllowUnstableSerialization: true, + }) + root, err := rewriter.RewriteTree(ctx, tm, "/", id) + test.OK(t, err) + _, expRoot := BuildTreeMap(TestTree{ + "subfile": TestFile{}, + }) + test.Assert(t, root == expRoot, "mismatched trees") +} + +func TestRewriterTreeLoadError(t *testing.T) { + tm := WritableTreeMap{TreeMap{}} + id := restic.NewRandomID() + + ctx, cancel := context.WithCancel(context.TODO()) + defer cancel() + + // also check that load error by default cause the operation to fail + rewriter := NewTreeRewriter(RewriteOpts{}) + _, err := rewriter.RewriteTree(ctx, tm, "/", id) + if err == nil { + t.Fatal("missing error on unloadable tree") + } + + replacementID := restic.NewRandomID() + rewriter = NewTreeRewriter(RewriteOpts{ + RewriteFailedTree: func(nodeID restic.ID, path string, err error) (restic.ID, error) { + if nodeID != id || path != "/" { + t.Fail() + } + return replacementID, nil + }, + }) + newRoot, err := rewriter.RewriteTree(ctx, tm, "/", id) + test.OK(t, err) + test.Equals(t, replacementID, newRoot) } From e17ee40a31e8eb9004c300b346fe4d054cb45318 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Wed, 28 Dec 2022 11:50:02 +0100 Subject: [PATCH 18/25] repair snapshots: Port to use walker.TreeRewriter --- cmd/restic/cmd_repair_snapshots.go | 180 +++++++++-------------------- 1 file changed, 56 insertions(+), 124 deletions(-) diff --git a/cmd/restic/cmd_repair_snapshots.go b/cmd/restic/cmd_repair_snapshots.go index 0413c43bf..ac75637fd 100644 --- a/cmd/restic/cmd_repair_snapshots.go +++ b/cmd/restic/cmd_repair_snapshots.go @@ -6,6 +6,7 @@ import ( "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/walker" "golang.org/x/sync/errgroup" "github.com/spf13/cobra" @@ -106,8 +107,56 @@ func runRepairSnapshots(ctx context.Context, gopts GlobalOptions, opts RepairOpt } func repairSnapshots(ctx context.Context, opts RepairOptions, repo restic.Repository, snapshots []*restic.Snapshot) error { - replaces := make(idMap) - seen := restic.NewIDSet() + // Three error cases are checked: + // - tree is a nil tree (-> will be replaced by an empty tree) + // - trees which cannot be loaded (-> the tree contents will be removed) + // - files whose contents are not fully available (-> file will be modified) + rewriter := walker.NewTreeRewriter(walker.RewriteOpts{ + RewriteNode: func(node *restic.Node, path string) *restic.Node { + if node.Type != "file" { + return node + } + + ok := true + var newContent restic.IDs + var newSize uint64 + // check all contents and remove if not available + for _, id := range node.Content { + if size, found := repo.LookupBlobSize(id, restic.DataBlob); !found { + ok = false + } else { + newContent = append(newContent, id) + newSize += uint64(size) + } + } + if !ok { + if newSize == 0 { + Printf("removed defective file '%v'\n", path+node.Name) + node = nil + } else { + Printf("repaired defective file '%v'\n", path+node.Name) + node.Content = newContent + node.Size = newSize + } + } + return node + }, + RewriteFailedTree: func(nodeID restic.ID, path string, _ error) (restic.ID, error) { + if path == "/" { + // remove snapshots with invalid root node + return restic.ID{}, nil + } + // If a subtree fails to load, remove it + Printf("removed defective dir '%v'", path) + emptyID, err := restic.SaveTree(ctx, repo, &restic.Tree{}) + if err != nil { + return restic.ID{}, err + } + return emptyID, nil + }, + AllowUnstableSerialization: true, + }) + deleteSn := restic.NewIDSet() Verbosef("check and repair %d snapshots\n", len(snapshots)) @@ -118,15 +167,16 @@ func repairSnapshots(ctx context.Context, opts RepairOptions, repo restic.Reposi for _, sn := range snapshots { debug.Log("process snapshot %v", sn.ID()) Printf("%v:\n", sn) - newID, changed, lErr, err := repairTree(ctx, opts, repo, "/", sn.Tree, replaces, seen) + newID, err := rewriter.RewriteTree(ctx, repo, "/", *sn.Tree) + switch { case err != nil: return err - case lErr: + case newID.IsNull(): Printf("the root tree is damaged -> delete snapshot.\n") deleteSn.Insert(*sn.ID()) - case changed: - err = changeSnapshot(ctx, opts.DryRun, repo, sn, newID) + case !newID.Equal(*sn.Tree): + err = changeSnapshot(ctx, opts.DryRun, repo, sn, &newID) if err != nil { return err } @@ -176,121 +226,3 @@ func changeSnapshot(ctx context.Context, dryRun bool, repo restic.Repository, sn } return nil } - -type idMap map[restic.ID]restic.ID - -// repairTree checks and repairs a tree and all its subtrees -// Three error cases are checked: -// - tree is a nil tree (-> will be replaced by an empty tree) -// - trees which cannot be loaded (-> the tree contents will be removed) -// - files whose contents are not fully available (-> file will be modified) -// In case of an error, the changes made depends on: -// - opts.Append: string to append to "repared" names; if empty files will not repaired but deleted -// - opts.DryRun: if set to true, only print out what to but don't change anything -// Returns: -// - the new ID -// - whether the ID changed -// - whether there was a load error when loading this tre -// - error for other errors (these are errors when saving a tree) -func repairTree(ctx context.Context, opts RepairOptions, repo restic.Repository, path string, treeID *restic.ID, replaces idMap, seen restic.IDSet) (*restic.ID, bool, bool, error) { - // handle and repair nil trees - if treeID == nil { - empty, err := emptyTree(ctx, repo, opts.DryRun) - Printf("repaired nil tree '%v'\n", path) - return &empty, true, false, err - } - - // check if tree was already changed - newID, ok := replaces[*treeID] - if ok { - return &newID, true, false, nil - } - - // check if tree was seen but not changed - if seen.Has(*treeID) { - return treeID, false, false, nil - } - - tree, err := restic.LoadTree(ctx, repo, *treeID) - if err != nil { - // mark as load error - return &newID, false, true, nil - } - - var newNodes []*restic.Node - changed := false - for _, node := range tree.Nodes { - switch node.Type { - case "file": - ok := true - var newContent restic.IDs - var newSize uint64 - // check all contents and remove if not available - for _, id := range node.Content { - if size, found := repo.LookupBlobSize(id, restic.DataBlob); !found { - ok = false - } else { - newContent = append(newContent, id) - newSize += uint64(size) - } - } - if !ok { - changed = true - if newSize == 0 { - Printf("removed defective file '%v'\n", path+node.Name) - continue - } - Printf("repaired defective file '%v'\n", path+node.Name) - node.Content = newContent - node.Size = newSize - } - case "dir": - // rewrite if necessary - newID, c, lErr, err := repairTree(ctx, opts, repo, path+node.Name+"/", node.Subtree, replaces, seen) - switch { - case err != nil: - return newID, true, false, err - case lErr: - // If we get an error, we remove this subtree - changed = true - Printf("replaced defective dir '%v'", path+node.Name) - empty, err := emptyTree(ctx, repo, opts.DryRun) - if err != nil { - return newID, true, false, err - } - node.Subtree = &empty - case c: - node.Subtree = newID - changed = true - } - } - newNodes = append(newNodes, node) - } - - if !changed { - seen.Insert(*treeID) - return treeID, false, false, nil - } - - tree.Nodes = newNodes - - if !opts.DryRun { - newID, err = restic.SaveTree(ctx, repo, tree) - if err != nil { - return &newID, true, false, err - } - Printf("modified tree %v, new id: %v\n", treeID.Str(), newID.Str()) - } else { - Printf("would have modified tree %v\n", treeID.Str()) - } - - replaces[*treeID] = newID - return &newID, true, false, nil -} - -func emptyTree(ctx context.Context, repo restic.Repository, dryRun bool) (restic.ID, error) { - if !dryRun { - return restic.SaveTree(ctx, repo, &restic.Tree{}) - } - return restic.ID{}, nil -} From 4ce87a7f6487034cfdf113965801a2b02c447d43 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Tue, 27 Dec 2022 21:31:04 +0100 Subject: [PATCH 19/25] repair snapshots: port to filterAndReplaceSnapshot The previous approach of rewriting all snapshots first, then flushing the repository data and finally removing old snapshots has the downside that an interrupted command execution leaves behind broken snapshots as not all new data is already flushed. --- cmd/restic/cmd_repair_snapshots.go | 104 ++++++++--------------------- cmd/restic/cmd_rewrite.go | 14 ++++ 2 files changed, 42 insertions(+), 76 deletions(-) diff --git a/cmd/restic/cmd_repair_snapshots.go b/cmd/restic/cmd_repair_snapshots.go index ac75637fd..036338161 100644 --- a/cmd/restic/cmd_repair_snapshots.go +++ b/cmd/restic/cmd_repair_snapshots.go @@ -4,10 +4,9 @@ import ( "context" "github.com/restic/restic/internal/backend" - "github.com/restic/restic/internal/debug" + "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/walker" - "golang.org/x/sync/errgroup" "github.com/spf13/cobra" ) @@ -97,16 +96,6 @@ func runRepairSnapshots(ctx context.Context, gopts GlobalOptions, opts RepairOpt return err } - // get snapshots to check & repair - var snapshots []*restic.Snapshot - for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, &opts.SnapshotFilter, args) { - snapshots = append(snapshots, sn) - } - - return repairSnapshots(ctx, opts, repo, snapshots) -} - -func repairSnapshots(ctx context.Context, opts RepairOptions, repo restic.Repository, snapshots []*restic.Snapshot) error { // Three error cases are checked: // - tree is a nil tree (-> will be replaced by an empty tree) // - trees which cannot be loaded (-> the tree contents will be removed) @@ -157,72 +146,35 @@ func repairSnapshots(ctx context.Context, opts RepairOptions, repo restic.Reposi AllowUnstableSerialization: true, }) - deleteSn := restic.NewIDSet() - - Verbosef("check and repair %d snapshots\n", len(snapshots)) - bar := newProgressMax(!globalOptions.Quiet, uint64(len(snapshots)), "snapshots") - wg, ctx := errgroup.WithContext(ctx) - repo.StartPackUploader(ctx, wg) - wg.Go(func() error { - for _, sn := range snapshots { - debug.Log("process snapshot %v", sn.ID()) - Printf("%v:\n", sn) - newID, err := rewriter.RewriteTree(ctx, repo, "/", *sn.Tree) - - switch { - case err != nil: - return err - case newID.IsNull(): - Printf("the root tree is damaged -> delete snapshot.\n") - deleteSn.Insert(*sn.ID()) - case !newID.Equal(*sn.Tree): - err = changeSnapshot(ctx, opts.DryRun, repo, sn, &newID) - if err != nil { - return err - } - deleteSn.Insert(*sn.ID()) - default: - Printf("is ok.\n") - } - debug.Log("processed snapshot %v", sn.ID()) - bar.Add(1) - } - bar.Done() - return repo.Flush(ctx) - }) - - err := wg.Wait() - if err != nil { - return err - } - - if len(deleteSn) > 0 && opts.Forget { - Verbosef("delete %d snapshots...\n", len(deleteSn)) - if !opts.DryRun { - DeleteFiles(ctx, globalOptions, repo, deleteSn, restic.SnapshotFile) - } - } - return nil -} - -// changeSnapshot creates a modified snapshot: -// - set the tree to newID -// - add the rag opts.AddTag -// - preserve original ID -// if opts.DryRun is set, it doesn't change anything but only -func changeSnapshot(ctx context.Context, dryRun bool, repo restic.Repository, sn *restic.Snapshot, newID *restic.ID) error { - sn.AddTags([]string{"repaired"}) - // Always set the original snapshot id as this essentially a new snapshot. - sn.Original = sn.ID() - sn.Tree = newID - if !dryRun { - newID, err := restic.SaveSnapshot(ctx, repo, sn) + changedCount := 0 + for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, &opts.SnapshotFilter, args) { + Verbosef("\nsnapshot %s of %v at %s)\n", sn.ID().Str(), sn.Paths, sn.Time) + changed, err := filterAndReplaceSnapshot(ctx, repo, sn, + func(ctx context.Context, sn *restic.Snapshot) (restic.ID, error) { + return rewriter.RewriteTree(ctx, repo, "/", *sn.Tree) + }, opts.DryRun, opts.Forget, "repaired") if err != nil { - return err + return errors.Fatalf("unable to rewrite snapshot ID %q: %v", sn.ID().Str(), err) + } + if changed { + changedCount++ } - Printf("snapshot repaired -> %v created.\n", newID.Str()) - } else { - Printf("would have repaired snapshot %v.\n", sn.ID().Str()) } + + Verbosef("\n") + if changedCount == 0 { + if !opts.DryRun { + Verbosef("no snapshots were modified\n") + } else { + Verbosef("no snapshots would be modified\n") + } + } else { + if !opts.DryRun { + Verbosef("modified %v snapshots\n", changedCount) + } else { + Verbosef("would modify %v snapshots\n", changedCount) + } + } + return nil } diff --git a/cmd/restic/cmd_rewrite.go b/cmd/restic/cmd_rewrite.go index e5c65850d..c08797c48 100644 --- a/cmd/restic/cmd_rewrite.go +++ b/cmd/restic/cmd_rewrite.go @@ -124,6 +124,20 @@ func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *r return false, err } + if filteredTree.IsNull() { + if dryRun { + Verbosef("would delete empty snapshot\n") + } else { + h := restic.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()} + if err = repo.Backend().Remove(ctx, h); err != nil { + return false, err + } + debug.Log("removed empty snapshot %v", sn.ID()) + Verbosef("removed empty snapshot %v\n", sn.ID().Str()) + } + return true, nil + } + if filteredTree == *sn.Tree { debug.Log("Snapshot %v not modified", sn) return false, nil From f6cc10578ddd9bc15b4c0130477210362c6526a9 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Wed, 28 Dec 2022 12:14:33 +0100 Subject: [PATCH 20/25] repair snapshots: Always sanitize file nodes If the node for a file is intact, this is a no-op. --- cmd/restic/cmd_repair_snapshots.go | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/cmd/restic/cmd_repair_snapshots.go b/cmd/restic/cmd_repair_snapshots.go index 036338161..58da8132a 100644 --- a/cmd/restic/cmd_repair_snapshots.go +++ b/cmd/restic/cmd_repair_snapshots.go @@ -107,7 +107,7 @@ func runRepairSnapshots(ctx context.Context, gopts GlobalOptions, opts RepairOpt } ok := true - var newContent restic.IDs + var newContent restic.IDs = restic.IDs{} var newSize uint64 // check all contents and remove if not available for _, id := range node.Content { @@ -119,15 +119,13 @@ func runRepairSnapshots(ctx context.Context, gopts GlobalOptions, opts RepairOpt } } if !ok { - if newSize == 0 { - Printf("removed defective file '%v'\n", path+node.Name) - node = nil - } else { - Printf("repaired defective file '%v'\n", path+node.Name) - node.Content = newContent - node.Size = newSize - } + Verbosef(" file %q: removed missing content\n", path) + } else if newSize != node.Size { + Verbosef(" file %q: fixed incorrect size\n", path) } + // no-ops if already correct + node.Content = newContent + node.Size = newSize return node }, RewriteFailedTree: func(nodeID restic.ID, path string, _ error) (restic.ID, error) { From 7c8dd61e8c815c44d1ac13fad65214539e47b4f0 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Wed, 28 Dec 2022 12:15:39 +0100 Subject: [PATCH 21/25] repair snapshots: cleanup warnings --- cmd/restic/cmd_repair_snapshots.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/restic/cmd_repair_snapshots.go b/cmd/restic/cmd_repair_snapshots.go index 58da8132a..e50d34b9d 100644 --- a/cmd/restic/cmd_repair_snapshots.go +++ b/cmd/restic/cmd_repair_snapshots.go @@ -130,11 +130,12 @@ func runRepairSnapshots(ctx context.Context, gopts GlobalOptions, opts RepairOpt }, RewriteFailedTree: func(nodeID restic.ID, path string, _ error) (restic.ID, error) { if path == "/" { + Verbosef(" dir %q: not readable\n", path) // remove snapshots with invalid root node return restic.ID{}, nil } // If a subtree fails to load, remove it - Printf("removed defective dir '%v'", path) + Verbosef(" dir %q: replaced with empty directory\n", path) emptyID, err := restic.SaveTree(ctx, repo, &restic.Tree{}) if err != nil { return restic.ID{}, err From 9c64a95df80d7f1dfb8f048630e2f432246ab544 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Wed, 28 Dec 2022 15:12:39 +0100 Subject: [PATCH 22/25] doc: rewrite troubleshooting section --- doc/077_troubleshooting.rst | 225 ++++++++++++++++++++++++------------ 1 file changed, 154 insertions(+), 71 deletions(-) diff --git a/doc/077_troubleshooting.rst b/doc/077_troubleshooting.rst index 5b86ffd87..fe317acfc 100644 --- a/doc/077_troubleshooting.rst +++ b/doc/077_troubleshooting.rst @@ -14,98 +14,181 @@ Troubleshooting ######################### -Being a backup software, the repository format ensures that the data saved in the repository -is verifiable and error-restistant. Restic even implements some self-healing functionalities. +The repository format used by restic is designed to be error resistant. In +particular, commands like, for example, ``backup`` or ``prune`` can be interrupted +at *any* point in time without damaging the repository. You might have to run +``unlock`` manually though, but that's it. -However, situations might occur where your repository gets in an incorrect state and measurements -need to be done to get you out of this situation. These situations might be due to hardware failure, -accidentially removing files directly from the repository or bugs in the restic implementation. +However, a repository might be damaged if some of its files are damaged or lost. +This can occur due to hardware failures, accidentally removing files from the +repository or bugs in the implementation of restic. -This document is meant to give you some hints about how to recover from such situations. +The following steps will help you recover a repository. This guide does not cover +all possible types of repository damages. Thus, if the steps do not work for you +or you are unsure how to proceed, then ask for help. Please always include the +check output discussed in the next section and what steps you've taken to repair +the repository so far. -1. Stay calm and don't over-react -******************************************** +* `Forum `_ +* Our IRC channel ``#restic`` on ``irc.libera.chat`` -The most important thing if you find yourself in the situation of a damaged repository is to -stay calm and don't do anything you might regret later. - -The following point should be always considered: - -- Make a copy of you repository and try to recover from that copy. If you suspect a storage failure, - it may be even better, to make *two* copies: one to get all data out of the possibly failing storage - and another one to try the recovery process. -- Pause your regular operations on the repository or let them run on a copy. You will especially make - sure that no `forget` or `prune` is run as these command are supposed to remove data and may result - in data loss. -- Search if your issue is already known and solved. Good starting points are the restic forum and the - github issues. -- Get you some help if you are unsure what to do. Find a colleage or friend to discuss what should be done. - Also feel free to consult the restic forum. -- When using the commands below, make sure you read and understand the documentation. Some of the commands - may not be your every-day commands, so make sure you really understand what they are doing. +Make sure that you **use the latest available restic version**. It can contain +bugfixes, and improvements to simplify the repair of a repository. It might also +contain a fix for your repository problems! -2. `check` is your friend -******************************************** +1. Find out what is damaged +*************************** -Run `restic check` to find out what type of error you have. The results may be technical but can give you -a good hint what's really wrong. +The first step is always to check the repository. -Moreover, you can always run a `check` to ensure that your repair really was sucessful and your repository -is in a sane state again. -But make sure that your needed data is also still contained in your repository ;-) - -Note that `check` also prints out warning in some cases. These warnings point out that the repo may be -optimized but is still in perfect shape and does not need any troubleshooting. +.. code-block:: console -3. Index trouble -> `repair index` -******************************************** + $ restic check --read-data -A common problem with broken repostories is that the index does no longer correctly represent the contents -of your pack files. This is especially the case if some pack files got lost. -`repair index` recovers this situation and ensures that the index exactly represents the pack files. + using temporary cache in /tmp/restic-check-cache-1418935501 + repository 12345678 opened (version 2, compression level auto) + created new cache in /tmp/restic-check-cache-1418935501 + create exclusive lock for repository + load indexes + check all packs + check snapshots, trees and blobs + error for tree 7ef8ebab: + id 7ef8ebabc59aadda1a237d23ca7abac487b627a9b86508aa0194690446ff71f6 not found in repository + [0:02] 100.00% 7 / 7 snapshots + read all data + [0:05] 100.00% 25 / 25 packs + Fatal: repository contains errors -You might even need to manually remove corrupted pack files. In this case make sure, you run -`restic repair index` after. +.. note:: -Also if you encounter problems with the index files itselves, `repair index` will solve these problems -immediately. + This will download the whole repository. If retrieving data from the backend is + expensive, then omit the ``--read-data`` option. Keep a copy of the check output + as it might be necessary later on! -However, rebuilding the index does not solve every problem, e.g. lost pack files. +If the output contains warnings that the ``ciphertext verification failed`` for +some blobs in the repository, then please ask for help in the forum or our IRC +channel. These errors are often caused by hardware problems which **must** be +investigated and fixed. Otherwise, the backup will be damaged again and again. -4. Delete unneeded defect snapshots -> `forget` -******************************************** +Similarly, if a repository is repeatedly damaged, please open an `issue on Github +`_ as this could indicate a bug +somewhere. Please include the check output and additional information that might +help locate the problem. -If you encounter defect snapshots but realize you can spare them, it is often a good idea to simply -delete them using `forget`. In case that your repository remains with just sane snapshots (including -all trees and files) the next `prune` run will put your repository in a sane state. -This can be also used if you manage to create new snapshots which can replace the defect ones, see -below. +2. Backup the repository +************************ -5. No fear to `backup` again -******************************************** +Create a full copy of the repository if possible. Or at the very least make a +copy of the ``index`` and ``snapshots`` folders. This will allow you to roll back +the repository if the repair procedure fails. If your repository resides in a +cloud storage, then you can for example use `rclone `_ to +make such a copy. -There are quite some self-healing mechanisms withing the `backup` command. So it is always a good idea to -backup again and check if this did heal your repository. -If you realize that a specific file is broken in your repository and you have this file, any run of -`backup` which includes that file will be able to heal the situation. +Please disable all regular operations on the repository to prevent unexpected +changes. Especially, ``forget`` or ``prune`` must be disabled as they could +remove data unexpectedly. -Note that `backup` relies on a correct index state, so make sure your index is fine or run `repair index` -before running `backup`. +.. warning:: -6. Unreferenced tree -> `recover` -******************************************** + If you suspect hardware problems, then you *must* investigate those first. + Otherwise, the repository will soon be damaged again. -If for some reason you have unreferenced trees in your repository but you actually need them, run -`recover` it will generate a new snapshot which allows access to all trees that you have in your -repository. +Please take the time to understand what the commands described in the following +do. If you are unsure, then ask for help in the forum or our IRC channel. Search +whether your issue is already known and solved. Please take a look at the +`forum`_ and `Github issues `_. -Note that `recover` relies on a correct index state, so make sure your index is fine or run `repair index` -before running `recover`. -7. Repair defect snapshots using `repair` -******************************************** +3. Repair the index +******************* -If all other things did not help, you can repair defect snapshots with `repair`. Note that the repaired -snapshots will miss data which was referenced in the defect snapshot. +Restic relies on its index to contain correct information about what data is +stored in the repository. Thus, the first step to repair a repository is to +repair the index: + +.. code-block:: console + + $ restic repair index + + repository a14e5863 opened (version 2, compression level auto) + loading indexes... + getting pack files to read... + removing not found pack file 83ad44f59b05f6bce13376b022ac3194f24ca19e7a74926000b6e316ec6ea5a4 + rebuilding index + [0:00] 100.00% 27 / 27 packs processed + deleting obsolete index files + [0:00] 100.00% 3 / 3 files deleted + done + +This ensures that no longer existing files are removed from the index. All later +steps to repair the repository rely on a correct index. That is, you must always +repair the index first! + +Please note that it is not recommended to repair the index unless the repository +is actually damaged. + + +4. Run all backups (optional) +***************************** + +With a correct index, the ``backup`` command guarantees that newly created +snapshots can be restored successfully. It can also heal older snapshots, +if the missing data is also contained in the new snapshot. + +Therefore, it is recommended to run all your ``backup`` tasks again. In some +cases, this is enough to fully repair the repository. + + +5. Remove missing data from snapshots +************************************* + +If your repository is still missing data, then you can use the ``repair snapshots`` +command to remove all inaccessible data from the snapshots. That is, this will +result in a limited amount of data loss. Using the ``--forget`` option, the +command will automatically remove the original, damaged snapshots. + +.. code-block:: console + + $ restic repair snapshots --forget + + snapshot 6979421e of [/home/user/restic/restic] at 2022-11-02 20:59:18.617503315 +0100 CET) + file "/restic/internal/fuse/snapshots_dir.go": removed missing content + file "/restic/internal/restorer/restorer_unix_test.go": removed missing content + file "/restic/internal/walker/walker.go": removed missing content + saved new snapshot 7b094cea + removed old snapshot 6979421e + + modified 1 snapshots + +If you did not add the ``--forget`` option, then you have to manually delete all +modified snapshots using the ``forget`` command. In the example above, you'd have +to run ``restic forget 6979421e``. + + +6. Check the repository again +***************************** + +Phew, we're almost done now. To make sure that the repository has been successfully +repaired please run ``check`` again. + +.. code-block:: console + + $ restic check --read-data + + using temporary cache in /tmp/restic-check-cache-2569290785 + repository a14e5863 opened (version 2, compression level auto) + created new cache in /tmp/restic-check-cache-2569290785 + create exclusive lock for repository + load indexes + check all packs + check snapshots, trees and blobs + [0:00] 100.00% 7 / 7 snapshots + read all data + [0:00] 100.00% 25 / 25 packs + no errors were found + +If the ``check`` command did not complete with ``no errors were found``, then +the repository is still damaged. At this point, please ask for help at the +`forum`_ or our IRC channel ``#restic`` on ``irc.libera.chat``. From 5aa37acdaa39f03b28dfda6a5cb13339b41eddf7 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Wed, 28 Dec 2022 15:32:00 +0100 Subject: [PATCH 23/25] repair snapshots: cleanup command help --- cmd/restic/cmd_repair_snapshots.go | 33 ++++++++++++++---------------- 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/cmd/restic/cmd_repair_snapshots.go b/cmd/restic/cmd_repair_snapshots.go index e50d34b9d..5e9ec4130 100644 --- a/cmd/restic/cmd_repair_snapshots.go +++ b/cmd/restic/cmd_repair_snapshots.go @@ -15,28 +15,25 @@ var cmdRepairSnapshots = &cobra.Command{ Use: "snapshots [flags] [snapshot ID] [...]", Short: "Repair snapshots", Long: ` -The "repair snapshots" command allows to repair broken snapshots. -It scans the given snapshots and generates new ones where -damaged tress and file contents are removed. -If the broken snapshots are deleted, a prune run will -be able to refit the repository. +The "repair snapshots" command repairs broken snapshots. It scans the given +snapshots and generates new ones with damaged directories and file contents +removed. If the broken snapshots are deleted, a prune run will be able to +clean up the repository. -The command depends on a good state of the index, so if -there are inaccurancies in the index, make sure to run -"repair index" before! +The command depends on a correct index, thus make sure to run "repair index" +first! -WARNING: -======== -Repairing and deleting broken snapshots causes data loss! -It will remove broken dirs and modify broken files in -the modified snapshots. +WARNING +======= -If the contents of directories and files are still available, -the better option is to redo a backup which in that case is -able to "heal" already present snapshots. -Only use this command if you need to recover an old and -broken snapshot! +Repairing and deleting broken snapshots causes data loss! It will remove broken +directories and modify broken files in the modified snapshots. + +If the contents of directories and files are still available, the better option +is to run "backup" which in that case is able to heal existing snapshots. Only +use the "repair snapshots" command if you need to recover an old and broken +snapshot! EXIT STATUS =========== From e71367e6b90ae44477dbb53bbbf4698746376375 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Wed, 28 Dec 2022 15:40:59 +0100 Subject: [PATCH 24/25] repair snapshots: update changelog --- changelog/unreleased/issue-1759 | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/changelog/unreleased/issue-1759 b/changelog/unreleased/issue-1759 index 0b0d28aa5..1b698f845 100644 --- a/changelog/unreleased/issue-1759 +++ b/changelog/unreleased/issue-1759 @@ -1,14 +1,18 @@ -Enhancement: Add new command repair +Enhancement: Add `repair index` and `repair snapshots` commands -We've added a new command repair which allows to repair snapshots even if needed -parts of it are not accessable in the repository. Note that using this command -can lead to data loss! +The `rebuild-index` command has been renamed to `repair index`. The old name +will still work, but is deprecated. -Some corrupted repositories were reported in several issues and so far restic -lacked a possibility to accept data loss but clean those up such that the -repository returns to a sane state. This possibility was now added. +When a snapshot was damaged, the only option up to now was to completely forget +the snapshot, even if only some unimportant file was damaged. + +We've added a `repair snapshots` command, which can repair snapshots by removing +damaged directories and missing files contents. Note that using this command +can lead to data loss! Please see the "Troubleshooting" section in the documentation +for more details. https://github.com/restic/restic/issues/1759 +https://github.com/restic/restic/issues/1714 https://github.com/restic/restic/issues/1798 https://github.com/restic/restic/issues/2334 https://github.com/restic/restic/pull/2876 From 78e5aa6d30402a6bead64515dd224963adc38dff Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Thu, 4 May 2023 23:00:46 +0200 Subject: [PATCH 25/25] repair snapshots: add basic tests --- .../integration_repair_snapshots_test.go | 135 ++++++++++++++++++ cmd/restic/integration_test.go | 23 ++- 2 files changed, 157 insertions(+), 1 deletion(-) create mode 100644 cmd/restic/integration_repair_snapshots_test.go diff --git a/cmd/restic/integration_repair_snapshots_test.go b/cmd/restic/integration_repair_snapshots_test.go new file mode 100644 index 000000000..04ef6ad1d --- /dev/null +++ b/cmd/restic/integration_repair_snapshots_test.go @@ -0,0 +1,135 @@ +package main + +import ( + "context" + "hash/fnv" + "io" + "math/rand" + "os" + "path/filepath" + "reflect" + "testing" + + "github.com/restic/restic/internal/restic" + rtest "github.com/restic/restic/internal/test" +) + +func testRunRepairSnapshot(t testing.TB, gopts GlobalOptions, forget bool) { + opts := RepairOptions{ + Forget: forget, + } + + rtest.OK(t, runRepairSnapshots(context.TODO(), gopts, opts, nil)) +} + +func createRandomFile(t testing.TB, env *testEnvironment, path string, size int) { + fn := filepath.Join(env.testdata, path) + rtest.OK(t, os.MkdirAll(filepath.Dir(fn), 0o755)) + + h := fnv.New64() + _, err := h.Write([]byte(path)) + rtest.OK(t, err) + r := rand.New(rand.NewSource(int64(h.Sum64()))) + + f, err := os.OpenFile(fn, os.O_CREATE|os.O_RDWR, 0o644) + rtest.OK(t, err) + _, err = io.Copy(f, io.LimitReader(r, int64(size))) + rtest.OK(t, err) + rtest.OK(t, f.Close()) +} + +func TestRepairSnapshotsWithLostData(t *testing.T) { + env, cleanup := withTestEnvironment(t) + defer cleanup() + + testRunInit(t, env.gopts) + + createRandomFile(t, env, "foo/bar/file", 512*1024) + testRunBackup(t, "", []string{env.testdata}, BackupOptions{}, env.gopts) + testListSnapshots(t, env.gopts, 1) + // damage repository + removePacksExcept(env.gopts, t, restic.NewIDSet(), false) + + createRandomFile(t, env, "foo/bar/file2", 256*1024) + testRunBackup(t, "", []string{env.testdata}, BackupOptions{}, env.gopts) + snapshotIDs := testListSnapshots(t, env.gopts, 2) + testRunCheckMustFail(t, env.gopts) + + // repair but keep broken snapshots + testRunRebuildIndex(t, env.gopts) + testRunRepairSnapshot(t, env.gopts, false) + testListSnapshots(t, env.gopts, 4) + testRunCheckMustFail(t, env.gopts) + + // repository must be ok after removing the broken snapshots + testRunForget(t, env.gopts, snapshotIDs[0].String(), snapshotIDs[1].String()) + testListSnapshots(t, env.gopts, 2) + _, err := testRunCheckOutput(env.gopts) + rtest.OK(t, err) +} + +func TestRepairSnapshotsWithLostTree(t *testing.T) { + env, cleanup := withTestEnvironment(t) + defer cleanup() + + testRunInit(t, env.gopts) + + createRandomFile(t, env, "foo/bar/file", 12345) + testRunBackup(t, "", []string{env.testdata}, BackupOptions{}, env.gopts) + oldSnapshot := testListSnapshots(t, env.gopts, 1) + oldPacks := testRunList(t, "packs", env.gopts) + + // keep foo/bar unchanged + createRandomFile(t, env, "foo/bar2", 1024) + testRunBackup(t, "", []string{env.testdata}, BackupOptions{}, env.gopts) + testListSnapshots(t, env.gopts, 2) + + // remove tree for foo/bar and the now completely broken first snapshot + removePacks(env.gopts, t, restic.NewIDSet(oldPacks...)) + testRunForget(t, env.gopts, oldSnapshot[0].String()) + testRunCheckMustFail(t, env.gopts) + + // repair + testRunRebuildIndex(t, env.gopts) + testRunRepairSnapshot(t, env.gopts, true) + testListSnapshots(t, env.gopts, 1) + _, err := testRunCheckOutput(env.gopts) + rtest.OK(t, err) +} + +func TestRepairSnapshotsWithLostRootTree(t *testing.T) { + env, cleanup := withTestEnvironment(t) + defer cleanup() + + testRunInit(t, env.gopts) + + createRandomFile(t, env, "foo/bar/file", 12345) + testRunBackup(t, "", []string{env.testdata}, BackupOptions{}, env.gopts) + testListSnapshots(t, env.gopts, 1) + oldPacks := testRunList(t, "packs", env.gopts) + + // remove all trees + removePacks(env.gopts, t, restic.NewIDSet(oldPacks...)) + testRunCheckMustFail(t, env.gopts) + + // repair + testRunRebuildIndex(t, env.gopts) + testRunRepairSnapshot(t, env.gopts, true) + testListSnapshots(t, env.gopts, 0) + _, err := testRunCheckOutput(env.gopts) + rtest.OK(t, err) +} + +func TestRepairSnapshotsIntact(t *testing.T) { + env, cleanup := withTestEnvironment(t) + defer cleanup() + testSetupBackupData(t, env) + testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{}, env.gopts) + oldSnapshotIDs := testListSnapshots(t, env.gopts, 1) + + // use an exclude that will not exclude anything + testRunRepairSnapshot(t, env.gopts, false) + snapshotIDs := testListSnapshots(t, env.gopts, 1) + rtest.Assert(t, reflect.DeepEqual(oldSnapshotIDs, snapshotIDs), "unexpected snapshot id mismatch %v vs. %v", oldSnapshotIDs, snapshotIDs) + testRunCheck(t, env.gopts) +} diff --git a/cmd/restic/integration_test.go b/cmd/restic/integration_test.go index 42fd26d6b..211089253 100644 --- a/cmd/restic/integration_test.go +++ b/cmd/restic/integration_test.go @@ -100,6 +100,13 @@ func testRunList(t testing.TB, tpe string, opts GlobalOptions) restic.IDs { return parseIDsFromReader(t, buf) } +func testListSnapshots(t testing.TB, opts GlobalOptions, expected int) restic.IDs { + t.Helper() + snapshotIDs := testRunList(t, "snapshots", opts) + rtest.Assert(t, len(snapshotIDs) == expected, "expected %v snapshot, got %v", expected, snapshotIDs) + return snapshotIDs +} + func testRunRestore(t testing.TB, opts GlobalOptions, dir string, snapshotID restic.ID) { testRunRestoreExcludes(t, opts, dir, snapshotID, nil) } @@ -164,6 +171,11 @@ func testRunCheckOutput(gopts GlobalOptions) (string, error) { return buf.String(), err } +func testRunCheckMustFail(t testing.TB, gopts GlobalOptions) { + _, err := testRunCheckOutput(gopts) + rtest.Assert(t, err != nil, "expected non nil error after check of damaged repository") +} + func testRunDiffOutput(gopts GlobalOptions, firstSnapshotID string, secondSnapshotID string) (string, error) { buf := bytes.NewBuffer(nil) @@ -486,7 +498,16 @@ func TestBackupNonExistingFile(t *testing.T) { testRunBackup(t, "", dirs, opts, env.gopts) } -func removePacksExcept(gopts GlobalOptions, t *testing.T, keep restic.IDSet, removeTreePacks bool) { +func removePacks(gopts GlobalOptions, t testing.TB, remove restic.IDSet) { + r, err := OpenRepository(context.TODO(), gopts) + rtest.OK(t, err) + + for id := range remove { + rtest.OK(t, r.Backend().Remove(context.TODO(), restic.Handle{Type: restic.PackFile, Name: id.String()})) + } +} + +func removePacksExcept(gopts GlobalOptions, t testing.TB, keep restic.IDSet, removeTreePacks bool) { r, err := OpenRepository(context.TODO(), gopts) rtest.OK(t, err)