From bedff1ed6d7402759ba57d4de4b8c5fc80889dfb Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 20 Jan 2024 15:40:09 +0100 Subject: [PATCH 01/11] split deleteFiles into UI and logic parts --- cmd/restic/delete.go | 56 ++++++++++--------------------------- internal/restic/parallel.go | 42 ++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 42 deletions(-) diff --git a/cmd/restic/delete.go b/cmd/restic/delete.go index 1b7937bd3..c3a7e039d 100644 --- a/cmd/restic/delete.go +++ b/cmd/restic/delete.go @@ -3,9 +3,6 @@ package main import ( "context" - "golang.org/x/sync/errgroup" - - "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/restic" ) @@ -24,46 +21,21 @@ func DeleteFilesChecked(ctx context.Context, gopts GlobalOptions, repo restic.Re // deleteFiles deletes the given fileList of fileType in parallel // if ignoreError=true, it will print a warning if there was an error, else it will abort. func deleteFiles(ctx context.Context, gopts GlobalOptions, ignoreError bool, repo restic.Repository, fileList restic.IDSet, fileType restic.FileType) error { - totalCount := len(fileList) - fileChan := make(chan restic.ID) - wg, ctx := errgroup.WithContext(ctx) - wg.Go(func() error { - defer close(fileChan) - for id := range fileList { - select { - case fileChan <- id: - case <-ctx.Done(): - return ctx.Err() + bar := newProgressMax(!gopts.JSON && !gopts.Quiet, 0, "files deleted") + defer bar.Done() + + return restic.ParallelRemove(ctx, repo, fileList, fileType, func(id restic.ID, err error) error { + if err != nil { + if !gopts.JSON { + Warnf("unable to remove %v/%v from the repository\n", fileType, id) + } + if !ignoreError { + return err } } + if !gopts.JSON && gopts.verbosity > 2 { + Verbosef("removed %v/%v\n", fileType, id) + } return nil - }) - - bar := newProgressMax(!gopts.JSON && !gopts.Quiet, uint64(totalCount), "files deleted") - defer bar.Done() - // deleting files is IO-bound - workerCount := repo.Connections() - for i := 0; i < int(workerCount); i++ { - wg.Go(func() error { - for id := range fileChan { - h := backend.Handle{Type: fileType, Name: id.String()} - err := repo.Backend().Remove(ctx, h) - if err != nil { - if !gopts.JSON { - Warnf("unable to remove %v from the repository\n", h) - } - if !ignoreError { - return err - } - } - if !gopts.JSON && gopts.verbosity > 2 { - Verbosef("removed %v\n", h) - } - bar.Add(1) - } - return nil - }) - } - err := wg.Wait() - return err + }, bar) } diff --git a/internal/restic/parallel.go b/internal/restic/parallel.go index b22a249fe..cefbf0358 100644 --- a/internal/restic/parallel.go +++ b/internal/restic/parallel.go @@ -3,7 +3,9 @@ package restic import ( "context" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/debug" + "github.com/restic/restic/internal/ui/progress" "golang.org/x/sync/errgroup" ) @@ -50,3 +52,43 @@ func ParallelList(ctx context.Context, r Lister, t FileType, parallelism uint, f return wg.Wait() } + +// ParallelRemove deletes the given fileList of fileType in parallel +// if callback returns an error, then it will abort. +func ParallelRemove(ctx context.Context, repo Repository, fileList IDSet, fileType FileType, report func(id ID, err error) error, bar *progress.Counter) error { + fileChan := make(chan ID) + wg, ctx := errgroup.WithContext(ctx) + wg.Go(func() error { + defer close(fileChan) + for id := range fileList { + select { + case fileChan <- id: + case <-ctx.Done(): + return ctx.Err() + } + } + return nil + }) + + bar.SetMax(uint64(len(fileList))) + + // deleting files is IO-bound + workerCount := repo.Connections() + for i := 0; i < int(workerCount); i++ { + wg.Go(func() error { + for id := range fileChan { + h := backend.Handle{Type: fileType, Name: id.String()} + err := repo.Backend().Remove(ctx, h) + if report != nil { + err = report(id, err) + } + if err != nil { + return err + } + bar.Add(1) + } + return nil + }) + } + return wg.Wait() +} From cb50832d508906be8a5845519147f01b921154df Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 20 Jan 2024 15:58:06 +0100 Subject: [PATCH 02/11] index: let MasterIndex.Save also delete obsolete indexes --- cmd/restic/cmd_prune.go | 32 +++++++++--------- cmd/restic/cmd_repair_index.go | 2 +- cmd/restic/cmd_repair_packs.go | 2 +- internal/index/master_index.go | 50 +++++++++++++++++++---------- internal/index/master_index_test.go | 12 +------ internal/repository/repack_test.go | 30 ++++++----------- internal/restic/repository.go | 9 +++++- 7 files changed, 69 insertions(+), 68 deletions(-) diff --git a/cmd/restic/cmd_prune.go b/cmd/restic/cmd_prune.go index 10abbf9f0..efd8f6e3a 100644 --- a/cmd/restic/cmd_prune.go +++ b/cmd/restic/cmd_prune.go @@ -15,6 +15,7 @@ import ( "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/ui" + "github.com/restic/restic/internal/ui/progress" "github.com/spf13/cobra" ) @@ -766,7 +767,7 @@ func doPrune(ctx context.Context, opts PruneOptions, gopts GlobalOptions, repo r return errors.Fatalf("%s", err) } } else if len(plan.ignorePacks) != 0 { - err = rebuildIndexFiles(ctx, gopts, repo, plan.ignorePacks, nil) + err = rebuildIndexFiles(ctx, gopts, repo, plan.ignorePacks, nil, false) if err != nil { return errors.Fatalf("%s", err) } @@ -778,7 +779,7 @@ func doPrune(ctx context.Context, opts PruneOptions, gopts GlobalOptions, repo r } if opts.unsafeRecovery { - _, err = writeIndexFiles(ctx, gopts, repo, plan.ignorePacks, nil) + err = rebuildIndexFiles(ctx, gopts, repo, plan.ignorePacks, nil, true) if err != nil { return errors.Fatalf("%s", err) } @@ -788,23 +789,22 @@ func doPrune(ctx context.Context, opts PruneOptions, gopts GlobalOptions, repo r return nil } -func writeIndexFiles(ctx context.Context, gopts GlobalOptions, repo restic.Repository, removePacks restic.IDSet, extraObsolete restic.IDs) (restic.IDSet, error) { +func rebuildIndexFiles(ctx context.Context, gopts GlobalOptions, repo restic.Repository, removePacks restic.IDSet, extraObsolete restic.IDs, skipDeletion bool) error { Verbosef("rebuilding index\n") bar := newProgressMax(!gopts.Quiet, 0, "packs processed") - obsoleteIndexes, err := repo.Index().Save(ctx, repo, removePacks, extraObsolete, bar) - bar.Done() - return obsoleteIndexes, err -} - -func rebuildIndexFiles(ctx context.Context, gopts GlobalOptions, repo restic.Repository, removePacks restic.IDSet, extraObsolete restic.IDs) error { - obsoleteIndexes, err := writeIndexFiles(ctx, gopts, repo, removePacks, extraObsolete) - if err != nil { - return err - } - - Verbosef("deleting obsolete index files\n") - return DeleteFilesChecked(ctx, gopts, repo, obsoleteIndexes, restic.IndexFile) + return repo.Index().Save(ctx, repo, removePacks, extraObsolete, restic.MasterIndexSaveOpts{ + SaveProgress: bar, + DeleteProgress: func() *progress.Counter { + return newProgressMax(!gopts.Quiet, 0, "old indexes deleted") + }, + DeleteReport: func(id restic.ID, err error) { + if gopts.verbosity > 2 { + Verbosef("removed index %v\n", id.String()) + } + }, + SkipDeletion: skipDeletion, + }) } func getUsedBlobs(ctx context.Context, repo restic.Repository, ignoreSnapshots restic.IDSet, quiet bool) (usedBlobs restic.CountedBlobSet, err error) { diff --git a/cmd/restic/cmd_repair_index.go b/cmd/restic/cmd_repair_index.go index c8a94b470..fc5506b34 100644 --- a/cmd/restic/cmd_repair_index.go +++ b/cmd/restic/cmd_repair_index.go @@ -154,7 +154,7 @@ func rebuildIndex(ctx context.Context, opts RepairIndexOptions, gopts GlobalOpti } } - err = rebuildIndexFiles(ctx, gopts, repo, removePacks, obsoleteIndexes) + err = rebuildIndexFiles(ctx, gopts, repo, removePacks, obsoleteIndexes, false) if err != nil { return err } diff --git a/cmd/restic/cmd_repair_packs.go b/cmd/restic/cmd_repair_packs.go index 723bdbccb..c572e02c5 100644 --- a/cmd/restic/cmd_repair_packs.go +++ b/cmd/restic/cmd_repair_packs.go @@ -145,7 +145,7 @@ func repairPacks(ctx context.Context, gopts GlobalOptions, repo *repository.Repo bar.Done() // remove salvaged packs from index - err = rebuildIndexFiles(ctx, gopts, repo, ids, nil) + err = rebuildIndexFiles(ctx, gopts, repo, ids, nil, false) if err != nil { return errors.Fatalf("%s", err) } diff --git a/internal/index/master_index.go b/internal/index/master_index.go index 073c9ace4..4c114b955 100644 --- a/internal/index/master_index.go +++ b/internal/index/master_index.go @@ -9,7 +9,6 @@ import ( "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/restic" - "github.com/restic/restic/internal/ui/progress" "golang.org/x/sync/errgroup" ) @@ -267,23 +266,22 @@ func (mi *MasterIndex) MergeFinalIndexes() error { // Save saves all known indexes to index files, leaving out any // packs whose ID is contained in packBlacklist from finalized indexes. -// The new index contains the IDs of all known indexes in the "supersedes" -// field. The IDs are also returned in the IDSet obsolete. -// After calling this function, you should remove the obsolete index files. -func (mi *MasterIndex) Save(ctx context.Context, repo restic.SaverUnpacked, packBlacklist restic.IDSet, extraObsolete restic.IDs, p *progress.Counter) (obsolete restic.IDSet, err error) { - p.SetMax(uint64(len(mi.Packs(packBlacklist)))) +// It also removes the old index files and those listed in extraObsolete. +func (mi *MasterIndex) Save(ctx context.Context, repo restic.Repository, excludePacks restic.IDSet, extraObsolete restic.IDs, opts restic.MasterIndexSaveOpts) error { + p := opts.SaveProgress + p.SetMax(uint64(len(mi.Packs(excludePacks)))) mi.idxMutex.Lock() defer mi.idxMutex.Unlock() - debug.Log("start rebuilding index of %d indexes, pack blacklist: %v", len(mi.idx), packBlacklist) + debug.Log("start rebuilding index of %d indexes, excludePacks: %v", len(mi.idx), excludePacks) newIndex := NewIndex() - obsolete = restic.NewIDSet() + obsolete := restic.NewIDSet() // track spawned goroutines using wg, create a new context which is // cancelled as soon as an error occurs. - wg, ctx := errgroup.WithContext(ctx) + wg, wgCtx := errgroup.WithContext(ctx) ch := make(chan *Index) @@ -310,21 +308,21 @@ func (mi *MasterIndex) Save(ctx context.Context, repo restic.SaverUnpacked, pack debug.Log("adding index %d", i) - for pbs := range idx.EachByPack(ctx, packBlacklist) { + for pbs := range idx.EachByPack(wgCtx, excludePacks) { newIndex.StorePack(pbs.PackID, pbs.Blobs) p.Add(1) if IndexFull(newIndex, mi.compress) { select { case ch <- newIndex: - case <-ctx.Done(): - return ctx.Err() + case <-wgCtx.Done(): + return wgCtx.Err() } newIndex = NewIndex() } } } - err = newIndex.AddToSupersedes(extraObsolete...) + err := newIndex.AddToSupersedes(extraObsolete...) if err != nil { return err } @@ -332,7 +330,7 @@ func (mi *MasterIndex) Save(ctx context.Context, repo restic.SaverUnpacked, pack select { case ch <- newIndex: - case <-ctx.Done(): + case <-wgCtx.Done(): } return nil }) @@ -341,7 +339,7 @@ func (mi *MasterIndex) Save(ctx context.Context, repo restic.SaverUnpacked, pack worker := func() error { for idx := range ch { idx.Finalize() - if _, err := SaveIndex(ctx, repo, idx); err != nil { + if _, err := SaveIndex(wgCtx, repo, idx); err != nil { return err } } @@ -354,9 +352,27 @@ func (mi *MasterIndex) Save(ctx context.Context, repo restic.SaverUnpacked, pack for i := 0; i < workerCount; i++ { wg.Go(worker) } - err = wg.Wait() + err := wg.Wait() + p.Done() + if err != nil { + return err + } - return obsolete, err + if opts.SkipDeletion { + return nil + } + + p = nil + if opts.DeleteProgress != nil { + p = opts.DeleteProgress() + } + defer p.Done() + return restic.ParallelRemove(ctx, repo, obsolete, restic.IndexFile, func(id restic.ID, err error) error { + if opts.DeleteReport != nil { + opts.DeleteReport(id, err) + } + return err + }, p) } // SaveIndex saves an index in the repository. diff --git a/internal/index/master_index_test.go b/internal/index/master_index_test.go index f76feb5fa..dcf6a94f6 100644 --- a/internal/index/master_index_test.go +++ b/internal/index/master_index_test.go @@ -8,7 +8,6 @@ import ( "testing" "time" - "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/checker" "github.com/restic/restic/internal/crypto" "github.com/restic/restic/internal/index" @@ -363,20 +362,11 @@ func testIndexSave(t *testing.T, version uint) { t.Fatal(err) } - obsoletes, err := repo.Index().Save(context.TODO(), repo, nil, nil, nil) + err = repo.Index().Save(context.TODO(), repo, nil, nil, restic.MasterIndexSaveOpts{}) if err != nil { t.Fatalf("unable to save new index: %v", err) } - for id := range obsoletes { - t.Logf("remove index %v", id.Str()) - h := backend.Handle{Type: restic.IndexFile, Name: id.String()} - err = repo.Backend().Remove(context.TODO(), h) - if err != nil { - t.Errorf("error removing index %v: %v", id, err) - } - } - checker := checker.New(repo, false) err = checker.LoadSnapshots(context.TODO()) if err != nil { diff --git a/internal/repository/repack_test.go b/internal/repository/repack_test.go index 20f0f2685..bab04f6b7 100644 --- a/internal/repository/repack_test.go +++ b/internal/repository/repack_test.go @@ -173,39 +173,27 @@ func flush(t *testing.T, repo restic.Repository) { func rebuildIndex(t *testing.T, repo restic.Repository) { err := repo.SetIndex(index.NewMasterIndex()) - if err != nil { - t.Fatal(err) - } + rtest.OK(t, err) packs := make(map[restic.ID]int64) err = repo.List(context.TODO(), restic.PackFile, func(id restic.ID, size int64) error { packs[id] = size return nil }) - if err != nil { - t.Fatal(err) - } + rtest.OK(t, err) _, err = repo.(*repository.Repository).CreateIndexFromPacks(context.TODO(), packs, nil) - if err != nil { - t.Fatal(err) - } + rtest.OK(t, err) + var obsoleteIndexes restic.IDs err = repo.List(context.TODO(), restic.IndexFile, func(id restic.ID, size int64) error { - h := backend.Handle{ - Type: restic.IndexFile, - Name: id.String(), - } - return repo.Backend().Remove(context.TODO(), h) + obsoleteIndexes = append(obsoleteIndexes, id) + return nil }) - if err != nil { - t.Fatal(err) - } + rtest.OK(t, err) - _, err = repo.Index().Save(context.TODO(), repo, restic.NewIDSet(), nil, nil) - if err != nil { - t.Fatal(err) - } + err = repo.Index().Save(context.TODO(), repo, restic.NewIDSet(), obsoleteIndexes, restic.MasterIndexSaveOpts{}) + rtest.OK(t, err) } func reloadIndex(t *testing.T, repo restic.Repository) { diff --git a/internal/restic/repository.go b/internal/restic/repository.go index 6818847c0..1c6c8d39d 100644 --- a/internal/restic/repository.go +++ b/internal/restic/repository.go @@ -89,6 +89,13 @@ type PackBlobs struct { Blobs []Blob } +type MasterIndexSaveOpts struct { + SaveProgress *progress.Counter + DeleteProgress func() *progress.Counter + DeleteReport func(id ID, err error) + SkipDeletion bool +} + // MasterIndex keeps track of the blobs are stored within files. type MasterIndex interface { Has(BlobHandle) bool @@ -99,7 +106,7 @@ type MasterIndex interface { Each(ctx context.Context, fn func(PackedBlob)) ListPacks(ctx context.Context, packs IDSet) <-chan PackBlobs - Save(ctx context.Context, repo SaverUnpacked, packBlacklist IDSet, extraObsolete IDs, p *progress.Counter) (obsolete IDSet, err error) + Save(ctx context.Context, repo Repository, excludePacks IDSet, extraObsolete IDs, opts MasterIndexSaveOpts) error } // Lister allows listing files in a backend. From d26d2d41f82eb35e263d16d1ad0241e12b4ccb8f Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 20 Jan 2024 18:10:11 +0100 Subject: [PATCH 03/11] backup/restore: extract termstatus initialization --- cmd/restic/cmd_backup.go | 30 +++------------------------- cmd/restic/cmd_restore.go | 29 +++------------------------ cmd/restic/termstatus.go | 42 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 53 deletions(-) create mode 100644 cmd/restic/termstatus.go diff --git a/cmd/restic/cmd_backup.go b/cmd/restic/cmd_backup.go index 348050895..0596ee918 100644 --- a/cmd/restic/cmd_backup.go +++ b/cmd/restic/cmd_backup.go @@ -12,7 +12,6 @@ import ( "runtime" "strconv" "strings" - "sync" "time" "github.com/spf13/cobra" @@ -25,7 +24,6 @@ import ( "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/textfile" - "github.com/restic/restic/internal/ui" "github.com/restic/restic/internal/ui/backup" "github.com/restic/restic/internal/ui/termstatus" ) @@ -56,31 +54,9 @@ Exit status is 3 if some source data could not be read (incomplete snapshot crea }, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - ctx := cmd.Context() - var wg sync.WaitGroup - cancelCtx, cancel := context.WithCancel(ctx) - defer func() { - // shutdown termstatus - cancel() - wg.Wait() - }() - - term := termstatus.New(globalOptions.stdout, globalOptions.stderr, globalOptions.Quiet) - wg.Add(1) - go func() { - defer wg.Done() - term.Run(cancelCtx) - }() - - // use the terminal for stdout/stderr - prevStdout, prevStderr := globalOptions.stdout, globalOptions.stderr - defer func() { - globalOptions.stdout, globalOptions.stderr = prevStdout, prevStderr - }() - stdioWrapper := ui.NewStdioWrapper(term) - globalOptions.stdout, globalOptions.stderr = stdioWrapper.Stdout(), stdioWrapper.Stderr() - - return runBackup(ctx, backupOptions, globalOptions, term, args) + term, cancel := setupTermstatus(cmd.Context()) + defer cancel() + return runBackup(cmd.Context(), backupOptions, globalOptions, term, args) }, } diff --git a/cmd/restic/cmd_restore.go b/cmd/restic/cmd_restore.go index 6045a5d41..b5c62fdea 100644 --- a/cmd/restic/cmd_restore.go +++ b/cmd/restic/cmd_restore.go @@ -3,7 +3,6 @@ package main import ( "context" "strings" - "sync" "time" "github.com/restic/restic/internal/debug" @@ -38,31 +37,9 @@ Exit status is 0 if the command was successful, and non-zero if there was any er `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - ctx := cmd.Context() - var wg sync.WaitGroup - cancelCtx, cancel := context.WithCancel(ctx) - defer func() { - // shutdown termstatus - cancel() - wg.Wait() - }() - - term := termstatus.New(globalOptions.stdout, globalOptions.stderr, globalOptions.Quiet) - wg.Add(1) - go func() { - defer wg.Done() - term.Run(cancelCtx) - }() - - // allow usage of warnf / verbosef - prevStdout, prevStderr := globalOptions.stdout, globalOptions.stderr - defer func() { - globalOptions.stdout, globalOptions.stderr = prevStdout, prevStderr - }() - stdioWrapper := ui.NewStdioWrapper(term) - globalOptions.stdout, globalOptions.stderr = stdioWrapper.Stdout(), stdioWrapper.Stderr() - - return runRestore(ctx, restoreOptions, globalOptions, term, args) + term, cancel := setupTermstatus(cmd.Context()) + defer cancel() + return runRestore(cmd.Context(), restoreOptions, globalOptions, term, args) }, } diff --git a/cmd/restic/termstatus.go b/cmd/restic/termstatus.go new file mode 100644 index 000000000..e39054427 --- /dev/null +++ b/cmd/restic/termstatus.go @@ -0,0 +1,42 @@ +package main + +import ( + "context" + "sync" + + "github.com/restic/restic/internal/ui" + "github.com/restic/restic/internal/ui/termstatus" +) + +// setupTermstatus creates a new termstatus and reroutes globalOptions.{stdout,stderr} to it +// The returned function must be called to shut down the termstatus, +// +// Expected usage: +// ``` +// term, cancel := setupTermstatus(ctx) +// defer cancel() +// // do stuff +// ``` +func setupTermstatus(ctx context.Context) (*termstatus.Terminal, func()) { + var wg sync.WaitGroup + cancelCtx, cancel := context.WithCancel(ctx) + + term := termstatus.New(globalOptions.stdout, globalOptions.stderr, globalOptions.Quiet) + wg.Add(1) + go func() { + defer wg.Done() + term.Run(cancelCtx) + }() + + // use the termstatus for stdout/stderr + prevStdout, prevStderr := globalOptions.stdout, globalOptions.stderr + stdioWrapper := ui.NewStdioWrapper(term) + globalOptions.stdout, globalOptions.stderr = stdioWrapper.Stdout(), stdioWrapper.Stderr() + + return term, func() { + // shutdown termstatus + globalOptions.stdout, globalOptions.stderr = prevStdout, prevStderr + cancel() + wg.Wait() + } +} From 6b65a495b1ca9b3ce7fbad9074b4084eb19c9393 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 20 Jan 2024 18:12:36 +0100 Subject: [PATCH 04/11] backup/restore: fix termstatus initialization The termstatus must only be canceled once the command has returned. Otherwise output may be lost when the context gets canceled. --- cmd/restic/cmd_backup.go | 2 +- cmd/restic/cmd_restore.go | 2 +- cmd/restic/termstatus.go | 7 ++++--- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/cmd/restic/cmd_backup.go b/cmd/restic/cmd_backup.go index 0596ee918..2ea2a4ec5 100644 --- a/cmd/restic/cmd_backup.go +++ b/cmd/restic/cmd_backup.go @@ -54,7 +54,7 @@ Exit status is 3 if some source data could not be read (incomplete snapshot crea }, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - term, cancel := setupTermstatus(cmd.Context()) + term, cancel := setupTermstatus() defer cancel() return runBackup(cmd.Context(), backupOptions, globalOptions, term, args) }, diff --git a/cmd/restic/cmd_restore.go b/cmd/restic/cmd_restore.go index b5c62fdea..1208d30eb 100644 --- a/cmd/restic/cmd_restore.go +++ b/cmd/restic/cmd_restore.go @@ -37,7 +37,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - term, cancel := setupTermstatus(cmd.Context()) + term, cancel := setupTermstatus() defer cancel() return runRestore(cmd.Context(), restoreOptions, globalOptions, term, args) }, diff --git a/cmd/restic/termstatus.go b/cmd/restic/termstatus.go index e39054427..cf3cd82ee 100644 --- a/cmd/restic/termstatus.go +++ b/cmd/restic/termstatus.go @@ -13,13 +13,14 @@ import ( // // Expected usage: // ``` -// term, cancel := setupTermstatus(ctx) +// term, cancel := setupTermstatus() // defer cancel() // // do stuff // ``` -func setupTermstatus(ctx context.Context) (*termstatus.Terminal, func()) { +func setupTermstatus() (*termstatus.Terminal, func()) { var wg sync.WaitGroup - cancelCtx, cancel := context.WithCancel(ctx) + // only shutdown once cancel is called to ensure that no output is lost + cancelCtx, cancel := context.WithCancel(context.Background()) term := termstatus.New(globalOptions.stdout, globalOptions.stderr, globalOptions.Quiet) wg.Add(1) From d7a50fe739e3e252ab8089b7b369aa889142ca4c Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 20 Jan 2024 18:34:11 +0100 Subject: [PATCH 05/11] properly show termstatus progress bar if visible less than one frame If a progress bar using termstatus was only visible for less than one frame, then its output could be lost. --- cmd/restic/progress.go | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/cmd/restic/progress.go b/cmd/restic/progress.go index 8b33f94c9..9d93863ad 100644 --- a/cmd/restic/progress.go +++ b/cmd/restic/progress.go @@ -30,7 +30,7 @@ func calculateProgressInterval(show bool, json bool) time.Duration { } // newTerminalProgressMax returns a progress.Counter that prints to stdout or terminal if provided. -func newGenericProgressMax(show bool, max uint64, description string, print func(status string)) *progress.Counter { +func newGenericProgressMax(show bool, max uint64, description string, print func(status string, final bool)) *progress.Counter { if !show { return nil } @@ -46,16 +46,18 @@ func newGenericProgressMax(show bool, max uint64, description string, print func ui.FormatDuration(d), ui.FormatPercent(v, max), v, max, description) } - print(status) - if final { - fmt.Print("\n") - } + print(status, final) }) } func newTerminalProgressMax(show bool, max uint64, description string, term *termstatus.Terminal) *progress.Counter { - return newGenericProgressMax(show, max, description, func(status string) { - term.SetStatus([]string{status}) + return newGenericProgressMax(show, max, description, func(status string, final bool) { + if final { + term.SetStatus([]string{}) + term.Print(status) + } else { + term.SetStatus([]string{status}) + } }) } @@ -64,7 +66,7 @@ func newProgressMax(show bool, max uint64, description string) *progress.Counter return newGenericProgressMax(show, max, description, printProgress) } -func printProgress(status string) { +func printProgress(status string, final bool) { canUpdateStatus := stdoutCanUpdateStatus() @@ -95,6 +97,9 @@ func printProgress(status string) { } _, _ = os.Stdout.Write([]byte(clear + status + carriageControl)) + if final { + _, _ = os.Stdout.Write([]byte("\n")) + } } func newIndexProgress(quiet bool, json bool) *progress.Counter { From feeab84204fab8964793cc13c9264cad255ded80 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 20 Jan 2024 18:40:22 +0100 Subject: [PATCH 06/11] repair pack: extract the repair logic into the repository package Currently, the cmd/restic package contains a significant amount of code that modifies repository internals. This code should in the mid-term move into the repository package. --- cmd/restic/cmd_repair_packs.go | 75 ++++--------------------- cmd/restic/progress.go | 18 ++++++ internal/repository/repair_pack.go | 88 ++++++++++++++++++++++++++++++ internal/ui/progress/printer.go | 30 ++++++++++ 4 files changed, 148 insertions(+), 63 deletions(-) create mode 100644 internal/repository/repair_pack.go create mode 100644 internal/ui/progress/printer.go diff --git a/cmd/restic/cmd_repair_packs.go b/cmd/restic/cmd_repair_packs.go index c572e02c5..04b06c33b 100644 --- a/cmd/restic/cmd_repair_packs.go +++ b/cmd/restic/cmd_repair_packs.go @@ -9,8 +9,8 @@ import ( "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/ui/termstatus" "github.com/spf13/cobra" - "golang.org/x/sync/errgroup" ) var cmdRepairPacks = &cobra.Command{ @@ -29,7 +29,9 @@ Exit status is 0 if the command was successful, and non-zero if there was any er `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - return runRepairPacks(cmd.Context(), globalOptions, args) + term, cancel := setupTermstatus() + defer cancel() + return runRepairPacks(cmd.Context(), globalOptions, term, args) }, } @@ -37,7 +39,7 @@ func init() { cmdRepair.AddCommand(cmdRepairPacks) } -func runRepairPacks(ctx context.Context, gopts GlobalOptions, args []string) error { +func runRepairPacks(ctx context.Context, gopts GlobalOptions, term *termstatus.Terminal, args []string) error { // FIXME discuss and add proper feature flag mechanism flag, _ := os.LookupEnv("RESTIC_FEATURES") if flag != "repair-packs-v1" { @@ -68,21 +70,19 @@ func runRepairPacks(ctx context.Context, gopts GlobalOptions, args []string) err return err } - return repairPacks(ctx, gopts, repo, ids) -} - -func repairPacks(ctx context.Context, gopts GlobalOptions, repo *repository.Repository, ids restic.IDSet) error { bar := newIndexProgress(gopts.Quiet, gopts.JSON) - err := repo.LoadIndex(ctx, bar) + err = repo.LoadIndex(ctx, bar) if err != nil { return errors.Fatalf("%s", err) } - Warnf("saving backup copies of pack files in current folder\n") + printer := newTerminalProgressPrinter(gopts.verbosity, term) + + printer.P("saving backup copies of pack files to current folder") for id := range ids { f, err := os.OpenFile("pack-"+id.String(), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o666) if err != nil { - return errors.Fatalf("%s", err) + return err } err = repo.Backend().Load(ctx, backend.Handle{Type: restic.PackFile, Name: id.String()}, 0, 0, func(rd io.Reader) error { @@ -94,66 +94,15 @@ func repairPacks(ctx context.Context, gopts GlobalOptions, repo *repository.Repo return err }) if err != nil { - return errors.Fatalf("%s", err) + return err } } - wg, wgCtx := errgroup.WithContext(ctx) - repo.StartPackUploader(wgCtx, wg) - repo.DisableAutoIndexUpdate() - - Warnf("salvaging intact data from specified pack files\n") - bar = newProgressMax(!gopts.Quiet, uint64(len(ids)), "pack files") - defer bar.Done() - - wg.Go(func() error { - // examine all data the indexes have for the pack file - for b := range repo.Index().ListPacks(wgCtx, ids) { - blobs := b.Blobs - if len(blobs) == 0 { - Warnf("no blobs found for pack %v\n", b.PackID) - bar.Add(1) - continue - } - - err = repo.LoadBlobsFromPack(wgCtx, b.PackID, blobs, func(blob restic.BlobHandle, buf []byte, err error) error { - if err != nil { - // Fallback path - buf, err = repo.LoadBlob(wgCtx, blob.Type, blob.ID, nil) - if err != nil { - Warnf("failed to load blob %v: %v\n", blob.ID, err) - return nil - } - } - id, _, _, err := repo.SaveBlob(wgCtx, blob.Type, buf, restic.ID{}, true) - if !id.Equal(blob.ID) { - panic("pack id mismatch during upload") - } - return err - }) - if err != nil { - return err - } - bar.Add(1) - } - return repo.Flush(wgCtx) - }) - - if err := wg.Wait(); err != nil { - return errors.Fatalf("%s", err) - } - bar.Done() - - // remove salvaged packs from index - err = rebuildIndexFiles(ctx, gopts, repo, ids, nil, false) + err = repository.RepairPacks(ctx, repo, ids, printer) if err != nil { return errors.Fatalf("%s", err) } - // cleanup - Warnf("removing salvaged pack files\n") - DeleteFiles(ctx, gopts, repo, ids, restic.PackFile) - Warnf("\nUse `restic repair snapshots --forget` to remove the corrupted data blobs from all snapshots\n") return nil } diff --git a/cmd/restic/progress.go b/cmd/restic/progress.go index 9d93863ad..48aa209a6 100644 --- a/cmd/restic/progress.go +++ b/cmd/restic/progress.go @@ -109,3 +109,21 @@ func newIndexProgress(quiet bool, json bool) *progress.Counter { func newIndexTerminalProgress(quiet bool, json bool, term *termstatus.Terminal) *progress.Counter { return newTerminalProgressMax(!quiet && !json && stdoutIsTerminal(), 0, "index files loaded", term) } + +type terminalProgressPrinter struct { + term *termstatus.Terminal + ui.Message + show bool +} + +func (t *terminalProgressPrinter) NewCounter(description string) *progress.Counter { + return newTerminalProgressMax(t.show, 0, description, t.term) +} + +func newTerminalProgressPrinter(verbosity uint, term *termstatus.Terminal) progress.Printer { + return &terminalProgressPrinter{ + term: term, + Message: *ui.NewMessage(term, verbosity), + show: verbosity > 0, + } +} diff --git a/internal/repository/repair_pack.go b/internal/repository/repair_pack.go new file mode 100644 index 000000000..5f3d43dc3 --- /dev/null +++ b/internal/repository/repair_pack.go @@ -0,0 +1,88 @@ +package repository + +import ( + "context" + "errors" + "io" + + "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/ui/progress" + "golang.org/x/sync/errgroup" +) + +func RepairPacks(ctx context.Context, repo *Repository, ids restic.IDSet, printer progress.Printer) error { + wg, wgCtx := errgroup.WithContext(ctx) + repo.StartPackUploader(wgCtx, wg) + repo.DisableAutoIndexUpdate() + + printer.P("salvaging intact data from specified pack files") + bar := printer.NewCounter("pack files") + bar.SetMax(uint64(len(ids))) + defer bar.Done() + + wg.Go(func() error { + // examine all data the indexes have for the pack file + for b := range repo.Index().ListPacks(wgCtx, ids) { + blobs := b.Blobs + if len(blobs) == 0 { + printer.E("no blobs found for pack %v", b.PackID) + bar.Add(1) + continue + } + + err := repo.LoadBlobsFromPack(wgCtx, b.PackID, blobs, func(blob restic.BlobHandle, buf []byte, err error) error { + if err != nil { + // Fallback path + buf, err = repo.LoadBlob(wgCtx, blob.Type, blob.ID, nil) + if err != nil { + printer.E("failed to load blob %v: %v", blob.ID, err) + return nil + } + } + id, _, _, err := repo.SaveBlob(wgCtx, blob.Type, buf, restic.ID{}, true) + if !id.Equal(blob.ID) { + panic("pack id mismatch during upload") + } + return err + }) + if err != nil { + return err + } + bar.Add(1) + } + return repo.Flush(wgCtx) + }) + + err := wg.Wait() + bar.Done() + if err != nil { + return err + } + + // remove salvaged packs from index + printer.P("rebuilding index") + + bar = printer.NewCounter("packs processed") + err = repo.Index().Save(ctx, repo, ids, nil, restic.MasterIndexSaveOpts{ + SaveProgress: bar, + DeleteProgress: func() *progress.Counter { + return printer.NewCounter("old indexes deleted") + }, + DeleteReport: func(id restic.ID, err error) { + printer.VV("removed index %v", id.String()) + }, + }) + + if err != nil { + return err + } + + // cleanup + printer.P("removing salvaged pack files") + // if we fail to delete the damaged pack files, then prune will remove them later on + bar = printer.NewCounter("files deleted") + _ = restic.ParallelRemove(ctx, repo, ids, restic.PackFile, nil, bar) + bar.Done() + + return nil +} diff --git a/internal/ui/progress/printer.go b/internal/ui/progress/printer.go new file mode 100644 index 000000000..c95383d3e --- /dev/null +++ b/internal/ui/progress/printer.go @@ -0,0 +1,30 @@ +package progress + +// A Printer can can return a new counter or print messages +// at different log levels. +// It must be safe to call its methods from concurrent goroutines. +type Printer interface { + NewCounter(description string) *Counter + + E(msg string, args ...interface{}) + P(msg string, args ...interface{}) + V(msg string, args ...interface{}) + VV(msg string, args ...interface{}) +} + +// NoopPrinter discards all messages +type NoopPrinter struct{} + +var _ Printer = (*NoopPrinter)(nil) + +func (*NoopPrinter) NewCounter(description string) *Counter { + return nil +} + +func (*NoopPrinter) E(msg string, args ...interface{}) {} + +func (*NoopPrinter) P(msg string, args ...interface{}) {} + +func (*NoopPrinter) V(msg string, args ...interface{}) {} + +func (*NoopPrinter) VV(msg string, args ...interface{}) {} From 7c351bc53c51337e9fd615f5db082ff6c593d7bd Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 20 Jan 2024 21:52:27 +0100 Subject: [PATCH 07/11] repair pack: reenable auto index updates The method is not available on the restic.Repository interface that is used for testing. Drop the call as a small amount of additional index writes is not a problem. --- internal/repository/repair_pack.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/internal/repository/repair_pack.go b/internal/repository/repair_pack.go index 5f3d43dc3..bedbc31df 100644 --- a/internal/repository/repair_pack.go +++ b/internal/repository/repair_pack.go @@ -10,10 +10,9 @@ import ( "golang.org/x/sync/errgroup" ) -func RepairPacks(ctx context.Context, repo *Repository, ids restic.IDSet, printer progress.Printer) error { +func RepairPacks(ctx context.Context, repo restic.Repository, ids restic.IDSet, printer progress.Printer) error { wg, wgCtx := errgroup.WithContext(ctx) repo.StartPackUploader(wgCtx, wg) - repo.DisableAutoIndexUpdate() printer.P("salvaging intact data from specified pack files") bar := printer.NewCounter("pack files") From 764b0bacd613ea4a84db16c96d6eb24c727d1e55 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 20 Jan 2024 21:54:27 +0100 Subject: [PATCH 08/11] repair pack: add support for truncated files --- internal/repository/repair_pack.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/repository/repair_pack.go b/internal/repository/repair_pack.go index bedbc31df..64279e727 100644 --- a/internal/repository/repair_pack.go +++ b/internal/repository/repair_pack.go @@ -44,7 +44,8 @@ func RepairPacks(ctx context.Context, repo restic.Repository, ids restic.IDSet, } return err }) - if err != nil { + // ignore truncated file parts + if err != nil && !errors.Is(err, io.ErrUnexpectedEOF) { return err } bar.Add(1) From 42c9318b9c50957bd7bb68e0d13dfd18bdc3e9e0 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 20 Jan 2024 21:54:47 +0100 Subject: [PATCH 09/11] repair pack: add tests --- internal/repository/repack_test.go | 3 +- internal/repository/repair_pack_test.go | 130 ++++++++++++++++++++++++ 2 files changed, 132 insertions(+), 1 deletion(-) create mode 100644 internal/repository/repair_pack_test.go diff --git a/internal/repository/repack_test.go b/internal/repository/repack_test.go index bab04f6b7..3de077a7d 100644 --- a/internal/repository/repack_test.go +++ b/internal/repository/repack_test.go @@ -62,7 +62,7 @@ func createRandomBlobs(t testing.TB, repo restic.Repository, blobs int, pData fl } } -func createRandomWrongBlob(t testing.TB, repo restic.Repository) { +func createRandomWrongBlob(t testing.TB, repo restic.Repository) restic.BlobHandle { length := randomSize(10*1024, 1024*1024) // 10KiB to 1MiB of data buf := make([]byte, length) rand.Read(buf) @@ -80,6 +80,7 @@ func createRandomWrongBlob(t testing.TB, repo restic.Repository) { if err := repo.Flush(context.Background()); err != nil { t.Fatalf("repo.Flush() returned error %v", err) } + return restic.BlobHandle{ID: id, Type: restic.DataBlob} } // selectBlobs splits the list of all blobs randomly into two lists. A blob diff --git a/internal/repository/repair_pack_test.go b/internal/repository/repair_pack_test.go new file mode 100644 index 000000000..e37c42eb7 --- /dev/null +++ b/internal/repository/repair_pack_test.go @@ -0,0 +1,130 @@ +package repository_test + +import ( + "context" + "math/rand" + "testing" + "time" + + "github.com/restic/restic/internal/backend" + "github.com/restic/restic/internal/index" + "github.com/restic/restic/internal/repository" + "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/test" + rtest "github.com/restic/restic/internal/test" + "github.com/restic/restic/internal/ui/progress" +) + +func listBlobs(repo restic.Repository) restic.BlobSet { + blobs := restic.NewBlobSet() + repo.Index().Each(context.TODO(), func(pb restic.PackedBlob) { + blobs.Insert(pb.BlobHandle) + }) + return blobs +} + +func replaceFile(t *testing.T, repo restic.Repository, h backend.Handle, damage func([]byte) []byte) { + buf, err := backend.LoadAll(context.TODO(), nil, repo.Backend(), h) + test.OK(t, err) + buf = damage(buf) + test.OK(t, repo.Backend().Remove(context.TODO(), h)) + test.OK(t, repo.Backend().Save(context.TODO(), h, backend.NewByteReader(buf, repo.Backend().Hasher()))) +} + +func TestRepairBrokenPack(t *testing.T) { + repository.TestAllVersions(t, testRepairBrokenPack) +} + +func testRepairBrokenPack(t *testing.T, version uint) { + tests := []struct { + name string + damage func(repo restic.Repository, packsBefore restic.IDSet) (restic.IDSet, restic.BlobSet) + }{ + { + "valid pack", + func(repo restic.Repository, packsBefore restic.IDSet) (restic.IDSet, restic.BlobSet) { + return packsBefore, restic.NewBlobSet() + }, + }, + { + "broken pack", + func(repo restic.Repository, packsBefore restic.IDSet) (restic.IDSet, restic.BlobSet) { + wrongBlob := createRandomWrongBlob(t, repo) + damagedPacks := findPacksForBlobs(t, repo, restic.NewBlobSet(wrongBlob)) + return damagedPacks, restic.NewBlobSet(wrongBlob) + }, + }, + { + "partially broken pack", + func(repo restic.Repository, packsBefore restic.IDSet) (restic.IDSet, restic.BlobSet) { + // damage one of the pack files + damagedID := packsBefore.List()[0] + replaceFile(t, repo, backend.Handle{Type: backend.PackFile, Name: damagedID.String()}, + func(buf []byte) []byte { + buf[0] ^= 0xff + return buf + }) + + // find blob that starts at offset 0 + var damagedBlob restic.BlobHandle + for blobs := range repo.Index().ListPacks(context.TODO(), restic.NewIDSet(damagedID)) { + for _, blob := range blobs.Blobs { + if blob.Offset == 0 { + damagedBlob = blob.BlobHandle + } + } + } + + return restic.NewIDSet(damagedID), restic.NewBlobSet(damagedBlob) + }, + }, { + "truncated pack", + func(repo restic.Repository, packsBefore restic.IDSet) (restic.IDSet, restic.BlobSet) { + // damage one of the pack files + damagedID := packsBefore.List()[0] + replaceFile(t, repo, backend.Handle{Type: backend.PackFile, Name: damagedID.String()}, + func(buf []byte) []byte { + buf = buf[0:10] + return buf + }) + + // all blobs in the file are broken + damagedBlobs := restic.NewBlobSet() + for blobs := range repo.Index().ListPacks(context.TODO(), restic.NewIDSet(damagedID)) { + for _, blob := range blobs.Blobs { + damagedBlobs.Insert(blob.BlobHandle) + } + } + return restic.NewIDSet(damagedID), damagedBlobs + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + repo := repository.TestRepositoryWithVersion(t, version) + + seed := time.Now().UnixNano() + rand.Seed(seed) + t.Logf("rand seed is %v", seed) + + createRandomBlobs(t, repo, 5, 0.7) + packsBefore := listPacks(t, repo) + blobsBefore := listBlobs(repo) + + toRepair, damagedBlobs := test.damage(repo, packsBefore) + + rtest.OK(t, repository.RepairPacks(context.TODO(), repo, toRepair, &progress.NoopPrinter{})) + // reload index + rtest.OK(t, repo.SetIndex(index.NewMasterIndex())) + repo.LoadIndex(context.TODO(), nil) + + packsAfter := listPacks(t, repo) + blobsAfter := listBlobs(repo) + + rtest.Assert(t, len(packsAfter.Intersect(toRepair)) == 0, "some damaged packs were not removed") + rtest.Assert(t, len(packsBefore.Sub(toRepair).Sub(packsAfter)) == 0, "not-damaged packs were removed") + rtest.Assert(t, blobsBefore.Sub(damagedBlobs).Equals(blobsAfter), "diverging blob lists") + }) + } +} From fd579421dd8c9e1bdd501fe96ea7a926bd432f78 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 20 Jan 2024 21:50:23 +0100 Subject: [PATCH 10/11] repository: deduplicate test --- internal/repository/repository_test.go | 60 +++++++------------------- 1 file changed, 16 insertions(+), 44 deletions(-) diff --git a/internal/repository/repository_test.go b/internal/repository/repository_test.go index 272ea94ac..886486136 100644 --- a/internal/repository/repository_test.go +++ b/internal/repository/repository_test.go @@ -28,10 +28,19 @@ var testSizes = []int{5, 23, 2<<18 + 23, 1 << 20} var rnd = rand.New(rand.NewSource(time.Now().UnixNano())) func TestSave(t *testing.T) { - repository.TestAllVersions(t, testSave) + repository.TestAllVersions(t, testSavePassID) + repository.TestAllVersions(t, testSaveCalculateID) } -func testSave(t *testing.T, version uint) { +func testSavePassID(t *testing.T, version uint) { + testSave(t, version, false) +} + +func testSaveCalculateID(t *testing.T, version uint) { + testSave(t, version, true) +} + +func testSave(t *testing.T, version uint, calculateID bool) { repo := repository.TestRepositoryWithVersion(t, version) for _, size := range testSizes { @@ -45,51 +54,14 @@ func testSave(t *testing.T, version uint) { repo.StartPackUploader(context.TODO(), &wg) // save - sid, _, _, err := repo.SaveBlob(context.TODO(), restic.DataBlob, data, restic.ID{}, false) + inputID := restic.ID{} + if !calculateID { + inputID = id + } + sid, _, _, err := repo.SaveBlob(context.TODO(), restic.DataBlob, data, inputID, false) rtest.OK(t, err) - rtest.Equals(t, id, sid) - rtest.OK(t, repo.Flush(context.Background())) - // rtest.OK(t, repo.SaveIndex()) - - // read back - buf, err := repo.LoadBlob(context.TODO(), restic.DataBlob, id, nil) - rtest.OK(t, err) - rtest.Equals(t, size, len(buf)) - - rtest.Assert(t, len(buf) == len(data), - "number of bytes read back does not match: expected %d, got %d", - len(data), len(buf)) - - rtest.Assert(t, bytes.Equal(buf, data), - "data does not match: expected %02x, got %02x", - data, buf) - } -} - -func TestSaveFrom(t *testing.T) { - repository.TestAllVersions(t, testSaveFrom) -} - -func testSaveFrom(t *testing.T, version uint) { - repo := repository.TestRepositoryWithVersion(t, version) - - for _, size := range testSizes { - data := make([]byte, size) - _, err := io.ReadFull(rnd, data) - rtest.OK(t, err) - - id := restic.Hash(data) - - var wg errgroup.Group - repo.StartPackUploader(context.TODO(), &wg) - - // save - id2, _, _, err := repo.SaveBlob(context.TODO(), restic.DataBlob, data, id, false) - rtest.OK(t, err) - rtest.Equals(t, id, id2) - rtest.OK(t, repo.Flush(context.Background())) // read back From f0e1ad2285a455ccb5503ad0363f2ca8ea0fc299 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 20 Jan 2024 21:58:28 +0100 Subject: [PATCH 11/11] fix linter warning --- internal/repository/repair_pack_test.go | 2 +- internal/ui/progress/printer.go | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/internal/repository/repair_pack_test.go b/internal/repository/repair_pack_test.go index e37c42eb7..6b20dbffb 100644 --- a/internal/repository/repair_pack_test.go +++ b/internal/repository/repair_pack_test.go @@ -117,7 +117,7 @@ func testRepairBrokenPack(t *testing.T, version uint) { rtest.OK(t, repository.RepairPacks(context.TODO(), repo, toRepair, &progress.NoopPrinter{})) // reload index rtest.OK(t, repo.SetIndex(index.NewMasterIndex())) - repo.LoadIndex(context.TODO(), nil) + rtest.OK(t, repo.LoadIndex(context.TODO(), nil)) packsAfter := listPacks(t, repo) blobsAfter := listBlobs(repo) diff --git a/internal/ui/progress/printer.go b/internal/ui/progress/printer.go index c95383d3e..a671621e9 100644 --- a/internal/ui/progress/printer.go +++ b/internal/ui/progress/printer.go @@ -17,14 +17,14 @@ type NoopPrinter struct{} var _ Printer = (*NoopPrinter)(nil) -func (*NoopPrinter) NewCounter(description string) *Counter { +func (*NoopPrinter) NewCounter(_ string) *Counter { return nil } -func (*NoopPrinter) E(msg string, args ...interface{}) {} +func (*NoopPrinter) E(_ string, _ ...interface{}) {} -func (*NoopPrinter) P(msg string, args ...interface{}) {} +func (*NoopPrinter) P(_ string, _ ...interface{}) {} -func (*NoopPrinter) V(msg string, args ...interface{}) {} +func (*NoopPrinter) V(_ string, _ ...interface{}) {} -func (*NoopPrinter) VV(msg string, args ...interface{}) {} +func (*NoopPrinter) VV(_ string, _ ...interface{}) {}