From db26dc75e1cffc0edd85a5725e4c7f1d95b3a54f Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 22 Oct 2023 16:27:08 +0200 Subject: [PATCH 1/3] repair packs: add experimental command This allows recovering a repository from several cases of damaged blobs. --- cmd/restic/cmd_repair_packs.go | 153 +++++++++++++++++++++++++++++++++ 1 file changed, 153 insertions(+) create mode 100644 cmd/restic/cmd_repair_packs.go diff --git a/cmd/restic/cmd_repair_packs.go b/cmd/restic/cmd_repair_packs.go new file mode 100644 index 000000000..503e92d69 --- /dev/null +++ b/cmd/restic/cmd_repair_packs.go @@ -0,0 +1,153 @@ +package main + +import ( + "context" + "io" + "os" + + "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/repository" + "github.com/restic/restic/internal/restic" + "github.com/spf13/cobra" + "golang.org/x/sync/errgroup" +) + +var cmdRepairPacks = &cobra.Command{ + Use: "packs [packIDs...]", + Short: "Salvage damaged pack files", + Long: ` +WARNING: The CLI for this command is experimental and will likely change in the future! + +The "repair packs" command extracts intact blobs from the specified pack files, rebuilds +the index to remove the damaged pack files and removes the pack files from the repository. + +EXIT STATUS +=========== + +Exit status is 0 if the command was successful, and non-zero if there was any error. +`, + DisableAutoGenTag: true, + RunE: func(cmd *cobra.Command, args []string) error { + return runRepairPacks(cmd.Context(), globalOptions, args) + }, +} + +func init() { + cmdRepair.AddCommand(cmdRepairPacks) +} + +// FIXME feature flag + +func runRepairPacks(ctx context.Context, gopts GlobalOptions, args []string) error { + ids := restic.NewIDSet() + for _, arg := range args { + id, err := restic.ParseID(arg) + if err != nil { + return err + } + ids.Insert(id) + } + if len(ids) == 0 { + return errors.Fatal("no ids specified") + } + + repo, err := OpenRepository(ctx, gopts) + if err != nil { + return err + } + + lock, ctx, err := lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON) + defer unlockRepo(lock) + if err != nil { + return err + } + + return repairPacks(ctx, gopts, repo, ids) +} + +func repairPacks(ctx context.Context, gopts GlobalOptions, repo *repository.Repository, ids restic.IDSet) error { + bar := newIndexProgress(gopts.Quiet, gopts.JSON) + err := repo.LoadIndex(ctx, bar) + if err != nil { + return errors.Fatalf("%s", err) + } + + Warnf("saving backup copies of pack files in current folder\n") + for id := range ids { + f, err := os.OpenFile("pack-"+id.String(), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o666) + if err != nil { + return errors.Fatalf("%s", err) + } + + err = repo.Backend().Load(ctx, restic.Handle{Type: restic.PackFile, Name: id.String()}, 0, 0, func(rd io.Reader) error { + _, err := f.Seek(0, 0) + if err != nil { + return err + } + _, err = io.Copy(f, rd) + return err + }) + if err != nil { + return errors.Fatalf("%s", err) + } + } + + wg, wgCtx := errgroup.WithContext(ctx) + repo.StartPackUploader(wgCtx, wg) + repo.DisableAutoIndexUpdate() + + Warnf("salvaging intact data from specified pack files\n") + bar = newProgressMax(!gopts.Quiet, uint64(len(ids)), "pack files") + defer bar.Done() + + wg.Go(func() error { + // examine all data the indexes have for the pack file + for b := range repo.Index().ListPacks(wgCtx, ids) { + blobs := b.Blobs + if len(blobs) == 0 { + Warnf("no blobs found for pack %v\n", b.PackID) + bar.Add(1) + continue + } + + err = repository.StreamPack(wgCtx, repo.Backend().Load, repo.Key(), b.PackID, blobs, func(blob restic.BlobHandle, buf []byte, err error) error { + if err != nil { + // Fallback path + buf, err = repo.LoadBlob(wgCtx, blob.Type, blob.ID, nil) + if err != nil { + Warnf("failed to load blob %v: %v\n", blob.ID, err) + return nil + } + } + id, _, _, err := repo.SaveBlob(wgCtx, blob.Type, buf, restic.ID{}, true) + if !id.Equal(blob.ID) { + panic("pack id mismatch during upload") + } + return err + }) + if err != nil { + return err + } + bar.Add(1) + } + return repo.Flush(wgCtx) + }) + + if err := wg.Wait(); err != nil { + return errors.Fatalf("%s", err) + } + bar.Done() + + // remove salvaged packs from index + err = rebuildIndexFiles(ctx, gopts, repo, ids, nil) + if err != nil { + return errors.Fatalf("%s", err) + } + + // cleanup + Warnf("removing salvaged pack files\n") + DeleteFiles(ctx, gopts, repo, ids, restic.PackFile) + + Warnf("\nUse `restic repair snapshots --forget` to remove the corrupted data blobs from all snapshots\n") + return nil +} From a28940ea2970d504e2f083b0c827ef917a3bbf2b Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 22 Oct 2023 16:28:48 +0200 Subject: [PATCH 2/3] check: Suggest usage of `restic repair packs` for corrupted blobs For now, the guide is only shown if the blob content does not match its hash. The main intended usage is to handle data corruption errors when using maximum compression in restic 0.16.0 --- cmd/restic/cmd_check.go | 17 +++++++++++++++++ internal/checker/checker.go | 12 +++++++++++- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/cmd/restic/cmd_check.go b/cmd/restic/cmd_check.go index 5f03c446b..c637ce89c 100644 --- a/cmd/restic/cmd_check.go +++ b/cmd/restic/cmd_check.go @@ -330,11 +330,28 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args go chkr.ReadPacks(ctx, packs, p, errChan) + var salvagePacks restic.IDs + for err := range errChan { errorsFound = true Warnf("%v\n", err) + if err, ok := err.(*checker.ErrPackData); ok { + if strings.Contains(err.Error(), "wrong data returned, hash is") { + salvagePacks = append(salvagePacks, err.PackID) + } + } } p.Done() + + if len(salvagePacks) > 0 { + Warnf("\nThe repository contains pack files with damaged blobs. These blobs must be removed to repair the repository. This can be done using the following commands:\n\n") + var strIds []string + for _, id := range salvagePacks { + strIds = append(strIds, id.String()) + } + Warnf("restic repair packs %v\nrestic repair snapshots --forget\n\n", strings.Join(strIds, " ")) + Warnf("Corrupted blobs are either caused by hardware problems or bugs in restic. Please open an issue at https://github.com/restic/restic/issues/new/choose for further troubleshooting!\n") + } } switch { diff --git a/internal/checker/checker.go b/internal/checker/checker.go index a5bb43731..59bc20daf 100644 --- a/internal/checker/checker.go +++ b/internal/checker/checker.go @@ -90,6 +90,16 @@ func (err *ErrOldIndexFormat) Error() string { return fmt.Sprintf("index %v has old format", err.ID) } +// ErrPackData is returned if errors are discovered while verifying a packfile +type ErrPackData struct { + PackID restic.ID + errs []error +} + +func (e *ErrPackData) Error() string { + return fmt.Sprintf("pack %v contains %v errors: %v", e.PackID, len(e.errs), e.errs) +} + func (c *Checker) LoadSnapshots(ctx context.Context) error { var err error c.snapshots, err = backend.MemorizeList(ctx, c.repo.Backend(), restic.SnapshotFile) @@ -635,7 +645,7 @@ func checkPack(ctx context.Context, r restic.Repository, id restic.ID, blobs []r } if len(errs) > 0 { - return errors.Errorf("pack %v contains %v errors: %v", id, len(errs), errs) + return &ErrPackData{PackID: id, errs: errs} } return nil From d1d45109744d26b0641977c0def31c55d5a05ef0 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 22 Oct 2023 16:40:27 +0200 Subject: [PATCH 3/3] repair packs: Add stub feature flag implementation --- cmd/restic/cmd_check.go | 2 +- cmd/restic/cmd_repair_packs.go | 9 +++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/cmd/restic/cmd_check.go b/cmd/restic/cmd_check.go index c637ce89c..fd512c7e7 100644 --- a/cmd/restic/cmd_check.go +++ b/cmd/restic/cmd_check.go @@ -349,7 +349,7 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args for _, id := range salvagePacks { strIds = append(strIds, id.String()) } - Warnf("restic repair packs %v\nrestic repair snapshots --forget\n\n", strings.Join(strIds, " ")) + Warnf("RESTIC_FEATURES=repair-packs-v1 restic repair packs %v\nrestic repair snapshots --forget\n\n", strings.Join(strIds, " ")) Warnf("Corrupted blobs are either caused by hardware problems or bugs in restic. Please open an issue at https://github.com/restic/restic/issues/new/choose for further troubleshooting!\n") } } diff --git a/cmd/restic/cmd_repair_packs.go b/cmd/restic/cmd_repair_packs.go index 503e92d69..aadfe73be 100644 --- a/cmd/restic/cmd_repair_packs.go +++ b/cmd/restic/cmd_repair_packs.go @@ -36,9 +36,14 @@ func init() { cmdRepair.AddCommand(cmdRepairPacks) } -// FIXME feature flag - func runRepairPacks(ctx context.Context, gopts GlobalOptions, args []string) error { + // FIXME discuss and add proper feature flag mechanism + flag, _ := os.LookupEnv("RESTIC_FEATURES") + if flag != "repair-packs-v1" { + return errors.Fatal("This command is experimental and may change/be removed without notice between restic versions. " + + "Set the environment variable 'RESTIC_FEATURES=repair-packs-v1' to enable it.") + } + ids := restic.NewIDSet() for _, arg := range args { id, err := restic.ParseID(arg)