From ddcf549ebadc82fcf7e26713f69f97becde08bf9 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 12 Jun 2022 14:45:42 +0200 Subject: [PATCH] repository: remove IsMixedPack and add replacement for checker Repositories with mixed packs are probably quite rare by now. When loading data blobs from a mixed pack file, this will no longer trigger caching that file. However, usually tree blobs are accessed first such that this shouldn't make much of a difference. The checker gets a simpler replacement. --- internal/checker/checker.go | 19 ++++++++++++- internal/repository/index.go | 40 +++------------------------ internal/repository/master_index.go | 12 -------- internal/repository/packer_manager.go | 3 +- internal/repository/repository.go | 7 +---- 5 files changed, 24 insertions(+), 57 deletions(-) diff --git a/internal/checker/checker.go b/internal/checker/checker.go index 9209c2f15..dba401390 100644 --- a/internal/checker/checker.go +++ b/internal/checker/checker.go @@ -100,6 +100,22 @@ func (c *Checker) LoadSnapshots(ctx context.Context) error { return err } +func computePackTypes(ctx context.Context, idx restic.MasterIndex) map[restic.ID]restic.BlobType { + packs := make(map[restic.ID]restic.BlobType) + idx.Each(ctx, func(pb restic.PackedBlob) { + tpe, exists := packs[pb.PackID] + if exists { + if pb.Type != tpe { + tpe = restic.InvalidBlob + } + } else { + tpe = pb.Type + } + packs[pb.PackID] = tpe + }) + return packs +} + // LoadIndex loads all index files. func (c *Checker) LoadIndex(ctx context.Context) (hints []error, errs []error) { debug.Log("Start") @@ -149,6 +165,7 @@ func (c *Checker) LoadIndex(ctx context.Context) (hints []error, errs []error) { // compute pack size using index entries c.packs = pack.Size(ctx, c.masterIndex, false) + packTypes := computePackTypes(ctx, c.masterIndex) debug.Log("checking for duplicate packs") for packID := range c.packs { @@ -159,7 +176,7 @@ func (c *Checker) LoadIndex(ctx context.Context) (hints []error, errs []error) { Indexes: packToIndex[packID], }) } - if c.masterIndex.IsMixedPack(packID) { + if packTypes[packID] == restic.InvalidBlob { hints = append(hints, &ErrMixedPack{ PackID: packID, }) diff --git a/internal/repository/index.go b/internal/repository/index.go index 69903c872..70a1de4fe 100644 --- a/internal/repository/index.go +++ b/internal/repository/index.go @@ -43,10 +43,9 @@ import ( // Index holds lookup tables for id -> pack. type Index struct { - m sync.Mutex - byType [restic.NumBlobTypes]indexMap - packs restic.IDs - mixedPacks restic.IDSet + m sync.Mutex + byType [restic.NumBlobTypes]indexMap + packs restic.IDs final bool // set to true for all indexes read from the backend ("finalized") ids restic.IDs // set to the IDs of the contained finalized indexes @@ -57,8 +56,7 @@ type Index struct { // NewIndex returns a new index. func NewIndex() *Index { return &Index{ - mixedPacks: restic.NewIDSet(), - created: time.Now(), + created: time.Now(), } } @@ -462,11 +460,6 @@ func (idx *Index) Dump(w io.Writer) error { return nil } -// MixedPacks returns an IDSet that contain packs which have mixed blobs. -func (idx *Index) MixedPacks() restic.IDSet { - return idx.mixedPacks -} - // merge() merges indexes, i.e. idx.merge(idx2) merges the contents of idx2 into idx. // During merging exact duplicates are removed; idx2 is not changed by this method. func (idx *Index) merge(idx2 *Index) error { @@ -509,7 +502,6 @@ func (idx *Index) merge(idx2 *Index) error { }) } - idx.mixedPacks.Merge(idx2.mixedPacks) idx.ids = append(idx.ids, idx2.ids...) idx.supersedes = append(idx.supersedes, idx2.supersedes...) @@ -543,7 +535,6 @@ func DecodeIndex(buf []byte, id restic.ID) (idx *Index, oldFormat bool, err erro idx = NewIndex() for _, pack := range idxJSON.Packs { - var data, tree bool packID := idx.addToPacks(pack.ID) for _, blob := range pack.Blobs { @@ -555,17 +546,6 @@ func DecodeIndex(buf []byte, id restic.ID) (idx *Index, oldFormat bool, err erro Length: blob.Length, UncompressedLength: blob.UncompressedLength, }) - - switch blob.Type { - case restic.DataBlob: - data = true - case restic.TreeBlob: - tree = true - } - } - - if data && tree { - idx.mixedPacks.Insert(pack.ID) } } idx.supersedes = idxJSON.Supersedes @@ -589,7 +569,6 @@ func decodeOldIndex(buf []byte) (idx *Index, err error) { idx = NewIndex() for _, pack := range list { - var data, tree bool packID := idx.addToPacks(pack.ID) for _, blob := range pack.Blobs { @@ -601,17 +580,6 @@ func decodeOldIndex(buf []byte) (idx *Index, err error) { Length: blob.Length, // no compressed length in the old index format }) - - switch blob.Type { - case restic.DataBlob: - data = true - case restic.TreeBlob: - tree = true - } - } - - if data && tree { - idx.mixedPacks.Insert(pack.ID) } } idx.final = true diff --git a/internal/repository/master_index.go b/internal/repository/master_index.go index 424e60046..93e83dca0 100644 --- a/internal/repository/master_index.go +++ b/internal/repository/master_index.go @@ -106,18 +106,6 @@ func (mi *MasterIndex) Has(bh restic.BlobHandle) bool { return false } -func (mi *MasterIndex) IsMixedPack(packID restic.ID) bool { - mi.idxMutex.RLock() - defer mi.idxMutex.RUnlock() - - for _, idx := range mi.idx { - if idx.MixedPacks().Has(packID) { - return true - } - } - return false -} - // IDs returns the IDs of all indexes contained in the index. func (mi *MasterIndex) IDs() restic.IDSet { mi.idxMutex.RLock() diff --git a/internal/repository/packer_manager.go b/internal/repository/packer_manager.go index e83bf8769..ba4c5e18e 100644 --- a/internal/repository/packer_manager.go +++ b/internal/repository/packer_manager.go @@ -157,8 +157,7 @@ func (r *Repository) savePacker(ctx context.Context, t restic.BlobType, p *Packe } id := restic.IDFromHash(hr.Sum(nil)) - h := restic.Handle{Type: restic.PackFile, Name: id.String(), - ContainedBlobType: t} + h := restic.Handle{Type: restic.PackFile, Name: id.String(), ContainedBlobType: t} var beHash []byte if beHr != nil { beHash = beHr.Sum(nil) diff --git a/internal/repository/repository.go b/internal/repository/repository.go index 7a2546bfd..929634df9 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -277,12 +277,7 @@ func (r *Repository) LoadBlob(ctx context.Context, t restic.BlobType, id restic. } // load blob from pack - bt := t - if r.idx.IsMixedPack(blob.PackID) { - bt = restic.InvalidBlob - } - h := restic.Handle{Type: restic.PackFile, - Name: blob.PackID.String(), ContainedBlobType: bt} + h := restic.Handle{Type: restic.PackFile, Name: blob.PackID.String(), ContainedBlobType: t} switch { case cap(buf) < int(blob.Length):