From 3fd0ad744807a1b75c1f5caba69c05c044a84a45 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 13 Aug 2023 20:33:13 +0200 Subject: [PATCH] repository: list index files only once --- cmd/restic/cmd_debug.go | 2 +- cmd/restic/cmd_list.go | 2 +- cmd/restic/cmd_repair_index.go | 2 +- internal/checker/checker.go | 2 +- internal/index/index_parallel.go | 4 ++-- internal/index/index_parallel_test.go | 4 ++-- internal/repository/repository.go | 15 +++++++++++++-- 7 files changed, 21 insertions(+), 10 deletions(-) diff --git a/cmd/restic/cmd_debug.go b/cmd/restic/cmd_debug.go index c7aa82449..5890b4a38 100644 --- a/cmd/restic/cmd_debug.go +++ b/cmd/restic/cmd_debug.go @@ -134,7 +134,7 @@ func printPacks(ctx context.Context, repo *repository.Repository, wr io.Writer) } func dumpIndexes(ctx context.Context, repo restic.Repository, wr io.Writer) error { - return index.ForAllIndexes(ctx, repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error { + return index.ForAllIndexes(ctx, repo.Backend(), repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error { Printf("index_id: %v\n", id) if err != nil { return err diff --git a/cmd/restic/cmd_list.go b/cmd/restic/cmd_list.go index bd02cedc7..5974da9ac 100644 --- a/cmd/restic/cmd_list.go +++ b/cmd/restic/cmd_list.go @@ -63,7 +63,7 @@ func runList(ctx context.Context, cmd *cobra.Command, gopts GlobalOptions, args case "locks": t = restic.LockFile case "blobs": - return index.ForAllIndexes(ctx, repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error { + return index.ForAllIndexes(ctx, repo.Backend(), repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error { if err != nil { return err } diff --git a/cmd/restic/cmd_repair_index.go b/cmd/restic/cmd_repair_index.go index b1905836a..622c77801 100644 --- a/cmd/restic/cmd_repair_index.go +++ b/cmd/restic/cmd_repair_index.go @@ -88,7 +88,7 @@ func rebuildIndex(ctx context.Context, opts RepairIndexOptions, gopts GlobalOpti } else { Verbosef("loading indexes...\n") mi := index.NewMasterIndex() - err := index.ForAllIndexes(ctx, repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error { + err := index.ForAllIndexes(ctx, repo.Backend(), repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error { if err != nil { Warnf("removing invalid index %v: %v\n", id, err) obsoleteIndexes = append(obsoleteIndexes, id) diff --git a/internal/checker/checker.go b/internal/checker/checker.go index b2512969e..85e078a08 100644 --- a/internal/checker/checker.go +++ b/internal/checker/checker.go @@ -117,7 +117,7 @@ func (c *Checker) LoadIndex(ctx context.Context) (hints []error, errs []error) { debug.Log("Start") packToIndex := make(map[restic.ID]restic.IDSet) - err := index.ForAllIndexes(ctx, c.repo, func(id restic.ID, index *index.Index, oldFormat bool, err error) error { + err := index.ForAllIndexes(ctx, c.repo.Backend(), c.repo, func(id restic.ID, index *index.Index, oldFormat bool, err error) error { debug.Log("process index %v, err %v", id, err) if oldFormat { diff --git a/internal/index/index_parallel.go b/internal/index/index_parallel.go index e7e46e88a..d505d756e 100644 --- a/internal/index/index_parallel.go +++ b/internal/index/index_parallel.go @@ -11,7 +11,7 @@ import ( // ForAllIndexes loads all index files in parallel and calls the given callback. // It is guaranteed that the function is not run concurrently. If the callback // returns an error, this function is cancelled and also returns that error. -func ForAllIndexes(ctx context.Context, repo restic.Repository, +func ForAllIndexes(ctx context.Context, lister restic.Lister, repo restic.Repository, fn func(id restic.ID, index *Index, oldFormat bool, err error) error) error { // decoding an index can take quite some time such that this can be both CPU- or IO-bound @@ -19,7 +19,7 @@ func ForAllIndexes(ctx context.Context, repo restic.Repository, workerCount := repo.Connections() + uint(runtime.GOMAXPROCS(0)) var m sync.Mutex - return restic.ParallelList(ctx, repo.Backend(), restic.IndexFile, workerCount, func(ctx context.Context, id restic.ID, size int64) error { + return restic.ParallelList(ctx, lister, restic.IndexFile, workerCount, func(ctx context.Context, id restic.ID, size int64) error { var err error var idx *Index oldFormat := false diff --git a/internal/index/index_parallel_test.go b/internal/index/index_parallel_test.go index 760374510..86be46473 100644 --- a/internal/index/index_parallel_test.go +++ b/internal/index/index_parallel_test.go @@ -29,7 +29,7 @@ func TestRepositoryForAllIndexes(t *testing.T) { // check that all expected indexes are loaded without errors indexIDs := restic.NewIDSet() var indexErr error - rtest.OK(t, index.ForAllIndexes(context.TODO(), repo, func(id restic.ID, index *index.Index, oldFormat bool, err error) error { + rtest.OK(t, index.ForAllIndexes(context.TODO(), repo.Backend(), repo, func(id restic.ID, index *index.Index, oldFormat bool, err error) error { if err != nil { indexErr = err } @@ -42,7 +42,7 @@ func TestRepositoryForAllIndexes(t *testing.T) { // must failed with the returned error iterErr := errors.New("error to pass upwards") - err := index.ForAllIndexes(context.TODO(), repo, func(id restic.ID, index *index.Index, oldFormat bool, err error) error { + err := index.ForAllIndexes(context.TODO(), repo.Backend(), repo, func(id restic.ID, index *index.Index, oldFormat bool, err error) error { return iterErr }) diff --git a/internal/repository/repository.go b/internal/repository/repository.go index 7c3c86d2b..8ec3c598e 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -584,9 +584,20 @@ func (r *Repository) SetIndex(i restic.MasterIndex) error { func (r *Repository) LoadIndex(ctx context.Context, p *progress.Counter) error { debug.Log("Loading index") + indexList, err := backend.MemorizeList(ctx, r.Backend(), restic.IndexFile) + if err != nil { + return err + } + if p != nil { var numIndexFiles uint64 - err := r.List(ctx, restic.IndexFile, func(id restic.ID, size int64) error { + err := indexList.List(ctx, restic.IndexFile, func(fi restic.FileInfo) error { + _, err := restic.ParseID(fi.Name) + if err != nil { + debug.Log("unable to parse %v as an ID", fi.Name) + return nil + } + numIndexFiles++ return nil }) @@ -597,7 +608,7 @@ func (r *Repository) LoadIndex(ctx context.Context, p *progress.Counter) error { defer p.Done() } - err := index.ForAllIndexes(ctx, r, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error { + err = index.ForAllIndexes(ctx, indexList, r, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error { if err != nil { return err }