octoleo/restic
2
2
Fork 0
mirror of https://github.com/octoleo/restic.git synced 2024-12-22 10:58:55 +00:00
Code Issues Releases Wiki Activity

repository: list index files only once

Browse Source
This commit is contained in:
Michael Eischer 2023-08-13 20:33:13 +02:00
parent b6593ad7df
commit 3fd0ad7448
7 changed files with 21 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/restic/cmd_debug.go
View File

@ -134,7 +134,7 @@ func printPacks(ctx context.Context, repo *repository.Repository, wr io.Writer)
} }
func dumpIndexes(ctx context.Context, repo restic.Repository, wr io.Writer) error { func dumpIndexes(ctx context.Context, repo restic.Repository, wr io.Writer) error {
return index.ForAllIndexes(ctx, repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error { return index.ForAllIndexes(ctx, repo.Backend(), repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error {
Printf("index_id: %v\n", id) Printf("index_id: %v\n", id)
if err != nil { if err != nil {
return err return err

2
cmd/restic/cmd_list.go
View File

@ -63,7 +63,7 @@ func runList(ctx context.Context, cmd *cobra.Command, gopts GlobalOptions, args
case "locks": case "locks":
t = restic.LockFile t = restic.LockFile
case "blobs": case "blobs":
return index.ForAllIndexes(ctx, repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error { return index.ForAllIndexes(ctx, repo.Backend(), repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error {
if err != nil { if err != nil {
return err return err
} }

2
cmd/restic/cmd_repair_index.go
View File

@ -88,7 +88,7 @@ func rebuildIndex(ctx context.Context, opts RepairIndexOptions, gopts GlobalOpti
} else { } else {
Verbosef("loading indexes...\n") Verbosef("loading indexes...\n")
mi := index.NewMasterIndex() mi := index.NewMasterIndex()
err := index.ForAllIndexes(ctx, repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error { err := index.ForAllIndexes(ctx, repo.Backend(), repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error {
if err != nil { if err != nil {
Warnf("removing invalid index %v: %v\n", id, err) Warnf("removing invalid index %v: %v\n", id, err)
obsoleteIndexes = append(obsoleteIndexes, id) obsoleteIndexes = append(obsoleteIndexes, id)

2
internal/checker/checker.go
View File

@ -117,7 +117,7 @@ func (c *Checker) LoadIndex(ctx context.Context) (hints []error, errs []error) {
debug.Log("Start") debug.Log("Start")
packToIndex := make(map[restic.ID]restic.IDSet) packToIndex := make(map[restic.ID]restic.IDSet)
err := index.ForAllIndexes(ctx, c.repo, func(id restic.ID, index *index.Index, oldFormat bool, err error) error { err := index.ForAllIndexes(ctx, c.repo.Backend(), c.repo, func(id restic.ID, index *index.Index, oldFormat bool, err error) error {
debug.Log("process index %v, err %v", id, err) debug.Log("process index %v, err %v", id, err)
if oldFormat { if oldFormat {

4
internal/index/index_parallel.go
View File

@ -11,7 +11,7 @@ import (
// ForAllIndexes loads all index files in parallel and calls the given callback. // ForAllIndexes loads all index files in parallel and calls the given callback.
// It is guaranteed that the function is not run concurrently. If the callback // It is guaranteed that the function is not run concurrently. If the callback
// returns an error, this function is cancelled and also returns that error. // returns an error, this function is cancelled and also returns that error.
func ForAllIndexes(ctx context.Context, repo restic.Repository, func ForAllIndexes(ctx context.Context, lister restic.Lister, repo restic.Repository,
fn func(id restic.ID, index *Index, oldFormat bool, err error) error) error { fn func(id restic.ID, index *Index, oldFormat bool, err error) error) error {
// decoding an index can take quite some time such that this can be both CPU- or IO-bound // decoding an index can take quite some time such that this can be both CPU- or IO-bound
@ -19,7 +19,7 @@ func ForAllIndexes(ctx context.Context, repo restic.Repository,
workerCount := repo.Connections() + uint(runtime.GOMAXPROCS(0)) workerCount := repo.Connections() + uint(runtime.GOMAXPROCS(0))
var m sync.Mutex var m sync.Mutex
return restic.ParallelList(ctx, repo.Backend(), restic.IndexFile, workerCount, func(ctx context.Context, id restic.ID, size int64) error { return restic.ParallelList(ctx, lister, restic.IndexFile, workerCount, func(ctx context.Context, id restic.ID, size int64) error {
var err error var err error
var idx *Index var idx *Index
oldFormat := false oldFormat := false

4
internal/index/index_parallel_test.go
View File

@ -29,7 +29,7 @@ func TestRepositoryForAllIndexes(t *testing.T) {
// check that all expected indexes are loaded without errors // check that all expected indexes are loaded without errors
indexIDs := restic.NewIDSet() indexIDs := restic.NewIDSet()
var indexErr error var indexErr error
rtest.OK(t, index.ForAllIndexes(context.TODO(), repo, func(id restic.ID, index *index.Index, oldFormat bool, err error) error { rtest.OK(t, index.ForAllIndexes(context.TODO(), repo.Backend(), repo, func(id restic.ID, index *index.Index, oldFormat bool, err error) error {
if err != nil { if err != nil {
indexErr = err indexErr = err
} }
@ -42,7 +42,7 @@ func TestRepositoryForAllIndexes(t *testing.T) {
// must failed with the returned error // must failed with the returned error
iterErr := errors.New("error to pass upwards") iterErr := errors.New("error to pass upwards")
err := index.ForAllIndexes(context.TODO(), repo, func(id restic.ID, index *index.Index, oldFormat bool, err error) error { err := index.ForAllIndexes(context.TODO(), repo.Backend(), repo, func(id restic.ID, index *index.Index, oldFormat bool, err error) error {
return iterErr return iterErr
}) })

15
internal/repository/repository.go
View File

@ -584,9 +584,20 @@ func (r *Repository) SetIndex(i restic.MasterIndex) error {
func (r *Repository) LoadIndex(ctx context.Context, p *progress.Counter) error { func (r *Repository) LoadIndex(ctx context.Context, p *progress.Counter) error {
debug.Log("Loading index") debug.Log("Loading index")
indexList, err := backend.MemorizeList(ctx, r.Backend(), restic.IndexFile)
if err != nil {
return err
}
if p != nil { if p != nil {
var numIndexFiles uint64 var numIndexFiles uint64
err := r.List(ctx, restic.IndexFile, func(id restic.ID, size int64) error { err := indexList.List(ctx, restic.IndexFile, func(fi restic.FileInfo) error {
_, err := restic.ParseID(fi.Name)
if err != nil {
debug.Log("unable to parse %v as an ID", fi.Name)
return nil
}
numIndexFiles++ numIndexFiles++
return nil return nil
}) })
@ -597,7 +608,7 @@ func (r *Repository) LoadIndex(ctx context.Context, p *progress.Counter) error {
defer p.Done() defer p.Done()
} }
err := index.ForAllIndexes(ctx, r, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error { err = index.ForAllIndexes(ctx, indexList, r, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error {
if err != nil { if err != nil {
return err return err
} }