Merge pull request #3949 from MichaelEischer/simplify-mixedpacks

repository: remove IsMixedPack and add replacement for checker

internal/checker/checker.go

@@ -100,6 +100,22 @@ func (c *Checker) LoadSnapshots(ctx context.Context) error {
 	return err
 }
 
+func computePackTypes(ctx context.Context, idx restic.MasterIndex) map[restic.ID]restic.BlobType {
+	packs := make(map[restic.ID]restic.BlobType)
+	idx.Each(ctx, func(pb restic.PackedBlob) {
+		tpe, exists := packs[pb.PackID]
+		if exists {
+			if pb.Type != tpe {
+				tpe = restic.InvalidBlob
+			}
+		} else {
+			tpe = pb.Type
+		}
+		packs[pb.PackID] = tpe
+	})
+	return packs
+}
+
 // LoadIndex loads all index files.
 func (c *Checker) LoadIndex(ctx context.Context) (hints []error, errs []error) {
 	debug.Log("Start")
@@ -149,6 +165,7 @@ func (c *Checker) LoadIndex(ctx context.Context) (hints []error, errs []error) {
 
 	// compute pack size using index entries
 	c.packs = pack.Size(ctx, c.masterIndex, false)
+	packTypes := computePackTypes(ctx, c.masterIndex)
 
 	debug.Log("checking for duplicate packs")
 	for packID := range c.packs {
@@ -159,7 +176,7 @@ func (c *Checker) LoadIndex(ctx context.Context) (hints []error, errs []error) {
 				Indexes: packToIndex[packID],
 			})
 		}
-		if c.masterIndex.IsMixedPack(packID) {
+		if packTypes[packID] == restic.InvalidBlob {
 			hints = append(hints, &ErrMixedPack{
 				PackID: packID,
 			})

internal/repository/index.go

@@ -43,10 +43,9 @@ import (
 
 // Index holds lookup tables for id -> pack.
 type Index struct {
-	m          sync.Mutex
-	byType     [restic.NumBlobTypes]indexMap
-	packs      restic.IDs
-	mixedPacks restic.IDSet
+	m      sync.Mutex
+	byType [restic.NumBlobTypes]indexMap
+	packs  restic.IDs
 
 	final      bool       // set to true for all indexes read from the backend ("finalized")
 	ids        restic.IDs // set to the IDs of the contained finalized indexes
@@ -57,8 +56,7 @@ type Index struct {
 // NewIndex returns a new index.
 func NewIndex() *Index {
 	return &Index{
-		mixedPacks: restic.NewIDSet(),
-		created:    time.Now(),
+		created: time.Now(),
 	}
 }
 
@@ -462,11 +460,6 @@ func (idx *Index) Dump(w io.Writer) error {
 	return nil
 }
 
-// MixedPacks returns an IDSet that contain packs which have mixed blobs.
-func (idx *Index) MixedPacks() restic.IDSet {
-	return idx.mixedPacks
-}
-
 // merge() merges indexes, i.e. idx.merge(idx2) merges the contents of idx2 into idx.
 // During merging exact duplicates are removed;  idx2 is not changed by this method.
 func (idx *Index) merge(idx2 *Index) error {
@@ -509,7 +502,6 @@ func (idx *Index) merge(idx2 *Index) error {
 		})
 	}
 
-	idx.mixedPacks.Merge(idx2.mixedPacks)
 	idx.ids = append(idx.ids, idx2.ids...)
 	idx.supersedes = append(idx.supersedes, idx2.supersedes...)
 
@@ -543,7 +535,6 @@ func DecodeIndex(buf []byte, id restic.ID) (idx *Index, oldFormat bool, err erro
 
 	idx = NewIndex()
 	for _, pack := range idxJSON.Packs {
-		var data, tree bool
 		packID := idx.addToPacks(pack.ID)
 
 		for _, blob := range pack.Blobs {
@@ -555,17 +546,6 @@ func DecodeIndex(buf []byte, id restic.ID) (idx *Index, oldFormat bool, err erro
 				Length:             blob.Length,
 				UncompressedLength: blob.UncompressedLength,
 			})
-
-			switch blob.Type {
-			case restic.DataBlob:
-				data = true
-			case restic.TreeBlob:
-				tree = true
-			}
-		}
-
-		if data && tree {
-			idx.mixedPacks.Insert(pack.ID)
 		}
 	}
 	idx.supersedes = idxJSON.Supersedes
@@ -589,7 +569,6 @@ func decodeOldIndex(buf []byte) (idx *Index, err error) {
 
 	idx = NewIndex()
 	for _, pack := range list {
-		var data, tree bool
 		packID := idx.addToPacks(pack.ID)
 
 		for _, blob := range pack.Blobs {
@@ -601,17 +580,6 @@ func decodeOldIndex(buf []byte) (idx *Index, err error) {
 				Length: blob.Length,
 				// no compressed length in the old index format
 			})
-
-			switch blob.Type {
-			case restic.DataBlob:
-				data = true
-			case restic.TreeBlob:
-				tree = true
-			}
-		}
-
-		if data && tree {
-			idx.mixedPacks.Insert(pack.ID)
 		}
 	}
 	idx.final = true

internal/repository/master_index.go

@@ -106,18 +106,6 @@ func (mi *MasterIndex) Has(bh restic.BlobHandle) bool {
 	return false
 }
 
-func (mi *MasterIndex) IsMixedPack(packID restic.ID) bool {
-	mi.idxMutex.RLock()
-	defer mi.idxMutex.RUnlock()
-
-	for _, idx := range mi.idx {
-		if idx.MixedPacks().Has(packID) {
-			return true
-		}
-	}
-	return false
-}
-
 // IDs returns the IDs of all indexes contained in the index.
 func (mi *MasterIndex) IDs() restic.IDSet {
 	mi.idxMutex.RLock()

internal/repository/packer_manager.go

@@ -157,8 +157,7 @@ func (r *Repository) savePacker(ctx context.Context, t restic.BlobType, p *Packe
 	}
 
 	id := restic.IDFromHash(hr.Sum(nil))
-	h := restic.Handle{Type: restic.PackFile, Name: id.String(),
-		ContainedBlobType: t}
+	h := restic.Handle{Type: restic.PackFile, Name: id.String(), ContainedBlobType: t}
 	var beHash []byte
 	if beHr != nil {
 		beHash = beHr.Sum(nil)

internal/repository/repository.go

@@ -271,12 +271,7 @@ func (r *Repository) LoadBlob(ctx context.Context, t restic.BlobType, id restic.
 		}
 
 		// load blob from pack
-		bt := t
-		if r.idx.IsMixedPack(blob.PackID) {
-			bt = restic.InvalidBlob
-		}
-		h := restic.Handle{Type: restic.PackFile,
-			Name: blob.PackID.String(), ContainedBlobType: bt}
+		h := restic.Handle{Type: restic.PackFile, Name: blob.PackID.String(), ContainedBlobType: t}
 
 		switch {
 		case cap(buf) < int(blob.Length):