repository: Increase index size for repo version 2

A compressed index is only about one third the size of an uncompressed
one. Thus increase the number of entries in an index to avoid cluttering
the repository with small indexes.

cmd/restic/integration_test.go

@@ -1470,7 +1470,7 @@ func TestRebuildIndexAlwaysFull(t *testing.T) {
 	defer func() {
 		repository.IndexFull = indexFull
 	}()
-	repository.IndexFull = func(*repository.Index) bool { return true }
+	repository.IndexFull = func(*repository.Index, bool) bool { return true }
 	testRebuildIndex(t, nil)
 }
 

internal/repository/index.go

@@ -94,11 +94,12 @@ func (idx *Index) Final() bool {
 
 const (
 	indexMaxBlobs           = 50000
+	indexMaxBlobsCompressed = 3 * indexMaxBlobs
 	indexMaxAge             = 10 * time.Minute
 )
 
 // IndexFull returns true iff the index is "full enough" to be saved as a preliminary index.
-var IndexFull = func(idx *Index) bool {
+var IndexFull = func(idx *Index, compress bool) bool {
 	idx.m.Lock()
 	defer idx.m.Unlock()
 
@@ -109,12 +110,18 @@ var IndexFull = func(idx *Index) bool {
 		blobs += idx.byType[typ].len()
 	}
 	age := time.Since(idx.created)
+	var maxBlobs uint
+	if compress {
+		maxBlobs = indexMaxBlobsCompressed
+	} else {
+		maxBlobs = indexMaxBlobs
+	}
 
 	switch {
 	case age >= indexMaxAge:
 		debug.Log("index %p is old enough", idx, age)
 		return true
-	case blobs >= indexMaxBlobs:
+	case blobs >= maxBlobs:
 		debug.Log("index %p has %d blobs", idx, blobs)
 		return true
 	}

internal/repository/master_index.go

@@ -16,6 +16,7 @@ type MasterIndex struct {
 	idx          []*Index
 	pendingBlobs restic.BlobSet
 	idxMutex     sync.RWMutex
+	compress     bool
 }
 
 // NewMasterIndex creates a new master index.
@@ -28,6 +29,10 @@ func NewMasterIndex() *MasterIndex {
 	return &MasterIndex{idx: idx, pendingBlobs: restic.NewBlobSet()}
 }
 
+func (mi *MasterIndex) markCompressed() {
+	mi.compress = true
+}
+
 // Lookup queries all known Indexes for the ID and returns all matches.
 func (mi *MasterIndex) Lookup(bh restic.BlobHandle) (pbs []restic.PackedBlob) {
 	mi.idxMutex.RLock()
@@ -206,7 +211,7 @@ func (mi *MasterIndex) FinalizeFullIndexes() []*Index {
 			continue
 		}
 
-		if IndexFull(idx) {
+		if IndexFull(idx, mi.compress) {
 			debug.Log("index %p is full", idx)
 			idx.Finalize()
 			list = append(list, idx)
@@ -334,7 +339,7 @@ func (mi *MasterIndex) Save(ctx context.Context, repo restic.Repository, packBla
 			for pbs := range idx.EachByPack(ctx, packBlacklist) {
 				newIndex.StorePack(pbs.packID, pbs.blobs)
 				p.Add(1)
-				if IndexFull(newIndex) {
+				if IndexFull(newIndex, mi.compress) {
 					select {
 					case ch <- newIndex:
 					case <-ctx.Done():

internal/repository/repository.go

@@ -698,6 +698,9 @@ func (r *Repository) SearchKey(ctx context.Context, password string, maxKeys int
 	} else if err != nil {
 		return errors.Fatalf("config cannot be loaded: %v", err)
 	}
+	if r.Config().Version >= 2 {
+		r.idx.markCompressed()
+	}
 	return nil
 }
 

internal/repository/repository_test.go

@@ -367,7 +367,7 @@ func TestRepositoryIncrementalIndex(t *testing.T) {
 
 	repo := r.(*repository.Repository)
 
-	repository.IndexFull = func(*repository.Index) bool { return true }
+	repository.IndexFull = func(*repository.Index, bool) bool { return true }
 
 	// add 15 packs
 	for j := 0; j < 5; j++ {