octoleo
/
restic
mirror of https://github.com/octoleo/restic.git
2
2
Fork 0
Code Issues Releases Wiki Activity

repository: list index files only once

This commit is contained in:
Michael Eischer 2023-08-13 20:33:13 +02:00
parent b6593ad7df
commit 3fd0ad7448
7 changed files with 21 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/restic/cmd_debug.go
View File

@ -134,7 +134,7 @@ func printPacks(ctx context.Context, repo *repository.Repository, wr io.Writer)
}
func dumpIndexes(ctx context.Context, repo restic.Repository, wr io.Writer) error {
return index.ForAllIndexes(ctx, repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error {
return index.ForAllIndexes(ctx, repo.Backend(), repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error {
Printf("index_id: %v\n", id)
if err != nil {
return err

2
cmd/restic/cmd_list.go
View File

@ -63,7 +63,7 @@ func runList(ctx context.Context, cmd *cobra.Command, gopts GlobalOptions, args
case "locks":
t = restic.LockFile
case "blobs":
return index.ForAllIndexes(ctx, repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error {
return index.ForAllIndexes(ctx, repo.Backend(), repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error {
if err != nil {
return err
}

2
cmd/restic/cmd_repair_index.go
View File

@ -88,7 +88,7 @@ func rebuildIndex(ctx context.Context, opts RepairIndexOptions, gopts GlobalOpti
} else {
Verbosef("loading indexes...\n")
mi := index.NewMasterIndex()
err := index.ForAllIndexes(ctx, repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error {
err := index.ForAllIndexes(ctx, repo.Backend(), repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error {
if err != nil {
Warnf("removing invalid index %v: %v\n", id, err)
obsoleteIndexes = append(obsoleteIndexes, id)

2
internal/checker/checker.go
View File

@ -117,7 +117,7 @@ func (c *Checker) LoadIndex(ctx context.Context) (hints []error, errs []error) {
debug.Log("Start")
packToIndex := make(map[restic.ID]restic.IDSet)
err := index.ForAllIndexes(ctx, c.repo, func(id restic.ID, index *index.Index, oldFormat bool, err error) error {
err := index.ForAllIndexes(ctx, c.repo.Backend(), c.repo, func(id restic.ID, index *index.Index, oldFormat bool, err error) error {
debug.Log("process index %v, err %v", id, err)
if oldFormat {

4
internal/index/index_parallel.go
View File

@ -11,7 +11,7 @@ import (
// ForAllIndexes loads all index files in parallel and calls the given callback.
// It is guaranteed that the function is not run concurrently. If the callback
// returns an error, this function is cancelled and also returns that error.
func ForAllIndexes(ctx context.Context, repo restic.Repository,
func ForAllIndexes(ctx context.Context, lister restic.Lister, repo restic.Repository,
fn func(id restic.ID, index *Index, oldFormat bool, err error) error) error {
// decoding an index can take quite some time such that this can be both CPU- or IO-bound
@ -19,7 +19,7 @@ func ForAllIndexes(ctx context.Context, repo restic.Repository,
workerCount := repo.Connections() + uint(runtime.GOMAXPROCS(0))
var m sync.Mutex
return restic.ParallelList(ctx, repo.Backend(), restic.IndexFile, workerCount, func(ctx context.Context, id restic.ID, size int64) error {
return restic.ParallelList(ctx, lister, restic.IndexFile, workerCount, func(ctx context.Context, id restic.ID, size int64) error {
var err error
var idx *Index
oldFormat := false

4
internal/index/index_parallel_test.go
View File

@ -29,7 +29,7 @@ func TestRepositoryForAllIndexes(t *testing.T) {
// check that all expected indexes are loaded without errors
indexIDs := restic.NewIDSet()
var indexErr error
rtest.OK(t, index.ForAllIndexes(context.TODO(), repo, func(id restic.ID, index *index.Index, oldFormat bool, err error) error {
rtest.OK(t, index.ForAllIndexes(context.TODO(), repo.Backend(), repo, func(id restic.ID, index *index.Index, oldFormat bool, err error) error {
if err != nil {
indexErr = err
}
@ -42,7 +42,7 @@ func TestRepositoryForAllIndexes(t *testing.T) {
// must failed with the returned error
iterErr := errors.New("error to pass upwards")
err := index.ForAllIndexes(context.TODO(), repo, func(id restic.ID, index *index.Index, oldFormat bool, err error) error {
err := index.ForAllIndexes(context.TODO(), repo.Backend(), repo, func(id restic.ID, index *index.Index, oldFormat bool, err error) error {
return iterErr
})

15
internal/repository/repository.go
View File

@ -584,9 +584,20 @@ func (r *Repository) SetIndex(i restic.MasterIndex) error {
func (r *Repository) LoadIndex(ctx context.Context, p *progress.Counter) error {
debug.Log("Loading index")
indexList, err := backend.MemorizeList(ctx, r.Backend(), restic.IndexFile)
if err != nil {
return err
}
if p != nil {
var numIndexFiles uint64
err := r.List(ctx, restic.IndexFile, func(id restic.ID, size int64) error {
err := indexList.List(ctx, restic.IndexFile, func(fi restic.FileInfo) error {
_, err := restic.ParseID(fi.Name)
if err != nil {
debug.Log("unable to parse %v as an ID", fi.Name)
return nil
}
numIndexFiles++
return nil
})
@ -597,7 +608,7 @@ func (r *Repository) LoadIndex(ctx context.Context, p *progress.Counter) error {
defer p.Done()
}
err := index.ForAllIndexes(ctx, r, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error {
err = index.ForAllIndexes(ctx, indexList, r, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error {
if err != nil {
return err
}