From 6ff9517e454b05d188cec1d19292f13293ddf394 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Fri, 19 Aug 2022 21:06:33 +0200 Subject: [PATCH 1/3] repository: MasterIndex.ListPacks / Index.EachByPack allow earlier GC Allow earlier garbage collection of some of the intermediate data structures. --- internal/repository/index.go | 2 ++ internal/repository/master_index.go | 2 ++ 2 files changed, 4 insertions(+) diff --git a/internal/repository/index.go b/internal/repository/index.go index 8978b60d5..767357eb3 100644 --- a/internal/repository/index.go +++ b/internal/repository/index.go @@ -294,6 +294,8 @@ func (idx *Index) EachByPack(ctx context.Context, packBlacklist restic.IDSet) <- result.Blobs = append(result.Blobs, idx.toPackedBlob(e, restic.BlobType(typ)).Blob) } } + // allow GC once entry is no longer necessary + byPack[packID] = nil select { case <-ctx.Done(): return diff --git a/internal/repository/master_index.go b/internal/repository/master_index.go index 955080e82..058a3a662 100644 --- a/internal/repository/master_index.go +++ b/internal/repository/master_index.go @@ -458,6 +458,8 @@ func (mi *MasterIndex) ListPacks(ctx context.Context, packs restic.IDSet) <-chan // pass on packs for packID, pbs := range packBlob { + // allow GC + packBlob[packID] = nil select { case out <- restic.PackBlobs{PackID: packID, Blobs: pbs}: case <-ctx.Done(): From 77b1980d8e77889d4b49a2912b9397bcd29588cc Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Fri, 19 Aug 2022 21:10:43 +0200 Subject: [PATCH 2/3] repository: MasterIndex.Packs: reduce allocations --- internal/repository/master_index.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/repository/master_index.go b/internal/repository/master_index.go index 058a3a662..05d0fdce2 100644 --- a/internal/repository/master_index.go +++ b/internal/repository/master_index.go @@ -150,7 +150,7 @@ func (mi *MasterIndex) Packs(packBlacklist restic.IDSet) restic.IDSet { packs := restic.NewIDSet() for _, idx := range mi.idx { idxPacks := idx.Packs() - if idx.final { + if idx.final && len(packBlacklist) > 0 { idxPacks = idxPacks.Sub(packBlacklist) } packs.Merge(idxPacks) From 7a992fc794f2cac9c063b8af31caf4d5fcc5532e Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Fri, 19 Aug 2022 21:11:02 +0200 Subject: [PATCH 3/3] repository: Reduce buffer reallocations in ForAllIndexes Previously the buffer was grown incrementally inside `repo.LoadUnpacked`. But we can do better as we already know how large the index will be. Allocate a bit more memory to increase the chance that the buffer can be reused in the future. --- internal/repository/index_parallel.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/internal/repository/index_parallel.go b/internal/repository/index_parallel.go index dcf33113e..4fe0797af 100644 --- a/internal/repository/index_parallel.go +++ b/internal/repository/index_parallel.go @@ -52,6 +52,10 @@ func ForAllIndexes(ctx context.Context, repo restic.Repository, var idx *Index oldFormat := false + if cap(buf) < int(fi.Size) { + // overallocate a bit + buf = make([]byte, fi.Size+128*1024) + } buf, err = repo.LoadUnpacked(ctx, restic.IndexFile, fi.ID, buf[:0]) if err == nil { idx, oldFormat, err = DecodeIndex(buf, fi.ID)