octoleo/restic
2
2
Fork 0
mirror of https://github.com/octoleo/restic.git synced 2025-01-22 22:58:26 +00:00
Code Issues Releases Wiki Activity

restic: let FindUsedBlobs handle multiple snapshots at once

Browse Source
This commit is contained in:
Michael Eischer 2020-11-07 14:16:04 +01:00 committed by Alexander Neumann
parent 258ce0c1e5
commit eda8c67616
4 changed files with 14 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
cmd/restic/cmd_prune.go
View File

@ -574,20 +574,14 @@ func getUsedBlobs(gopts GlobalOptions, repo restic.Repository, ignoreSnapshots r
bar := newProgressMax(!gopts.Quiet, uint64(len(snapshotTrees)), "snapshots") bar := newProgressMax(!gopts.Quiet, uint64(len(snapshotTrees)), "snapshots")
defer bar.Done() defer bar.Done()
for _, tree := range snapshotTrees {
debug.Log("process tree %v", tree)
err = restic.FindUsedBlobs(ctx, repo, tree, usedBlobs) err = restic.FindUsedBlobs(ctx, repo, snapshotTrees, usedBlobs, bar)
if err != nil { if err != nil {
if repo.Backend().IsNotExist(err) { if repo.Backend().IsNotExist(err) {
return nil, errors.Fatal("unable to load a tree from the repo: " + err.Error()) return nil, errors.Fatal("unable to load a tree from the repo: " + err.Error())
}
return nil, err
} }
debug.Log("processed tree %v", tree) return nil, err
bar.Add(1)
} }
return usedBlobs, nil return usedBlobs, nil
} }

2
cmd/restic/cmd_stats.go
View File

@ -166,7 +166,7 @@ func statsWalkSnapshot(ctx context.Context, snapshot *restic.Snapshot, repo rest
if statsOptions.countMode == countModeRawData { if statsOptions.countMode == countModeRawData {
// count just the sizes of unique blobs; we don't need to walk the tree // count just the sizes of unique blobs; we don't need to walk the tree
// ourselves in this case, since a nifty function does it for us // ourselves in this case, since a nifty function does it for us
return restic.FindUsedBlobs(ctx, repo, *snapshot.Tree, stats.blobs) return restic.FindUsedBlobs(ctx, repo, restic.IDs{*snapshot.Tree}, stats.blobs, nil)
} }
err := walker.Walk(ctx, repo, *snapshot.Tree, restic.NewIDSet(), statsWalkTree(repo, stats)) err := walker.Walk(ctx, repo, *snapshot.Tree, restic.NewIDSet(), statsWalkTree(repo, stats))

7
internal/restic/find.go
View File

@ -4,6 +4,7 @@ import (
"context" "context"
"sync" "sync"
"github.com/restic/restic/internal/ui/progress"
"golang.org/x/sync/errgroup" "golang.org/x/sync/errgroup"
) )
@ -14,11 +15,11 @@ type TreeLoader interface {
// FindUsedBlobs traverses the tree ID and adds all seen blobs (trees and data // FindUsedBlobs traverses the tree ID and adds all seen blobs (trees and data
// blobs) to the set blobs. Already seen tree blobs will not be visited again. // blobs) to the set blobs. Already seen tree blobs will not be visited again.
func FindUsedBlobs(ctx context.Context, repo TreeLoader, treeID ID, blobs BlobSet) error { func FindUsedBlobs(ctx context.Context, repo TreeLoader, treeIDs IDs, blobs BlobSet, p *progress.Counter) error {
var lock sync.Mutex var lock sync.Mutex
wg, ctx := errgroup.WithContext(ctx) wg, ctx := errgroup.WithContext(ctx)
treeStream := StreamTrees(ctx, wg, repo, IDs{treeID}, func(treeID ID) bool { treeStream := StreamTrees(ctx, wg, repo, treeIDs, func(treeID ID) bool {
// locking is necessary the goroutine below concurrently adds data blobs // locking is necessary the goroutine below concurrently adds data blobs
lock.Lock() lock.Lock()
h := BlobHandle{ID: treeID, Type: TreeBlob} h := BlobHandle{ID: treeID, Type: TreeBlob}
@ -27,7 +28,7 @@ func FindUsedBlobs(ctx context.Context, repo TreeLoader, treeID ID, blobs BlobSe
blobs.Insert(h) blobs.Insert(h)
lock.Unlock() lock.Unlock()
return blobReferenced return blobReferenced
}, nil) }, p)
wg.Go(func() error { wg.Go(func() error {
for tree := range treeStream { for tree := range treeStream {

8
internal/restic/find_test.go
View File

@ -94,7 +94,7 @@ func TestFindUsedBlobs(t *testing.T) {
for i, sn := range snapshots { for i, sn := range snapshots {
usedBlobs := restic.NewBlobSet() usedBlobs := restic.NewBlobSet()
err := restic.FindUsedBlobs(context.TODO(), repo, *sn.Tree, usedBlobs) err := restic.FindUsedBlobs(context.TODO(), repo, restic.IDs{*sn.Tree}, usedBlobs, nil)
if err != nil { if err != nil {
t.Errorf("FindUsedBlobs returned error: %v", err) t.Errorf("FindUsedBlobs returned error: %v", err)
continue continue
@ -133,12 +133,12 @@ func TestFindUsedBlobsSkipsSeenBlobs(t *testing.T) {
t.Logf("snapshot %v saved, tree %v", snapshot.ID().Str(), snapshot.Tree.Str()) t.Logf("snapshot %v saved, tree %v", snapshot.ID().Str(), snapshot.Tree.Str())
usedBlobs := restic.NewBlobSet() usedBlobs := restic.NewBlobSet()
err := restic.FindUsedBlobs(context.TODO(), repo, *snapshot.Tree, usedBlobs) err := restic.FindUsedBlobs(context.TODO(), repo, restic.IDs{*snapshot.Tree}, usedBlobs, nil)
if err != nil { if err != nil {
t.Fatalf("FindUsedBlobs returned error: %v", err) t.Fatalf("FindUsedBlobs returned error: %v", err)
} }
err = restic.FindUsedBlobs(context.TODO(), ForbiddenRepo{}, *snapshot.Tree, usedBlobs) err = restic.FindUsedBlobs(context.TODO(), ForbiddenRepo{}, restic.IDs{*snapshot.Tree}, usedBlobs, nil)
if err != nil { if err != nil {
t.Fatalf("FindUsedBlobs returned error: %v", err) t.Fatalf("FindUsedBlobs returned error: %v", err)
} }
@ -154,7 +154,7 @@ func BenchmarkFindUsedBlobs(b *testing.B) {
for i := 0; i < b.N; i++ { for i := 0; i < b.N; i++ {
blobs := restic.NewBlobSet() blobs := restic.NewBlobSet()
err := restic.FindUsedBlobs(context.TODO(), repo, *sn.Tree, blobs) err := restic.FindUsedBlobs(context.TODO(), repo, restic.IDs{*sn.Tree}, blobs, nil)
if err != nil { if err != nil {
b.Error(err) b.Error(err)
} }