diff --git a/cmd/restic/integration_test.go b/cmd/restic/integration_test.go index 49121bb1d..ebf63e930 100644 --- a/cmd/restic/integration_test.go +++ b/cmd/restic/integration_test.go @@ -1470,7 +1470,7 @@ func TestRebuildIndexAlwaysFull(t *testing.T) { defer func() { repository.IndexFull = indexFull }() - repository.IndexFull = func(*repository.Index) bool { return true } + repository.IndexFull = func(*repository.Index, bool) bool { return true } testRebuildIndex(t, nil) } diff --git a/internal/repository/index.go b/internal/repository/index.go index 5f6fe1997..520fcbd8e 100644 --- a/internal/repository/index.go +++ b/internal/repository/index.go @@ -93,12 +93,13 @@ func (idx *Index) Final() bool { } const ( - indexMaxBlobs = 50000 - indexMaxAge = 10 * time.Minute + indexMaxBlobs = 50000 + indexMaxBlobsCompressed = 3 * indexMaxBlobs + indexMaxAge = 10 * time.Minute ) // IndexFull returns true iff the index is "full enough" to be saved as a preliminary index. -var IndexFull = func(idx *Index) bool { +var IndexFull = func(idx *Index, compress bool) bool { idx.m.Lock() defer idx.m.Unlock() @@ -109,12 +110,18 @@ var IndexFull = func(idx *Index) bool { blobs += idx.byType[typ].len() } age := time.Since(idx.created) + var maxBlobs uint + if compress { + maxBlobs = indexMaxBlobsCompressed + } else { + maxBlobs = indexMaxBlobs + } switch { case age >= indexMaxAge: debug.Log("index %p is old enough", idx, age) return true - case blobs >= indexMaxBlobs: + case blobs >= maxBlobs: debug.Log("index %p has %d blobs", idx, blobs) return true } diff --git a/internal/repository/master_index.go b/internal/repository/master_index.go index 9056528a2..96462d4a4 100644 --- a/internal/repository/master_index.go +++ b/internal/repository/master_index.go @@ -16,6 +16,7 @@ type MasterIndex struct { idx []*Index pendingBlobs restic.BlobSet idxMutex sync.RWMutex + compress bool } // NewMasterIndex creates a new master index. @@ -28,6 +29,10 @@ func NewMasterIndex() *MasterIndex { return &MasterIndex{idx: idx, pendingBlobs: restic.NewBlobSet()} } +func (mi *MasterIndex) markCompressed() { + mi.compress = true +} + // Lookup queries all known Indexes for the ID and returns all matches. func (mi *MasterIndex) Lookup(bh restic.BlobHandle) (pbs []restic.PackedBlob) { mi.idxMutex.RLock() @@ -206,7 +211,7 @@ func (mi *MasterIndex) FinalizeFullIndexes() []*Index { continue } - if IndexFull(idx) { + if IndexFull(idx, mi.compress) { debug.Log("index %p is full", idx) idx.Finalize() list = append(list, idx) @@ -334,7 +339,7 @@ func (mi *MasterIndex) Save(ctx context.Context, repo restic.Repository, packBla for pbs := range idx.EachByPack(ctx, packBlacklist) { newIndex.StorePack(pbs.packID, pbs.blobs) p.Add(1) - if IndexFull(newIndex) { + if IndexFull(newIndex, mi.compress) { select { case ch <- newIndex: case <-ctx.Done(): diff --git a/internal/repository/repository.go b/internal/repository/repository.go index f028b57ad..bbbc1af68 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -698,6 +698,9 @@ func (r *Repository) SearchKey(ctx context.Context, password string, maxKeys int } else if err != nil { return errors.Fatalf("config cannot be loaded: %v", err) } + if r.Config().Version >= 2 { + r.idx.markCompressed() + } return nil } diff --git a/internal/repository/repository_test.go b/internal/repository/repository_test.go index 7cc593e04..6da685ff3 100644 --- a/internal/repository/repository_test.go +++ b/internal/repository/repository_test.go @@ -367,7 +367,7 @@ func TestRepositoryIncrementalIndex(t *testing.T) { repo := r.(*repository.Repository) - repository.IndexFull = func(*repository.Index) bool { return true } + repository.IndexFull = func(*repository.Index, bool) bool { return true } // add 15 packs for j := 0; j < 5; j++ {