Use in-memory index to rebuild index in prune

changelog/unreleased/pull-2718

@@ -3,6 +3,8 @@ Enhancement: Improve pruning performance and make pruning more customizable
 The `prune` command is now much faster. This is especially the case for remote
 repositories or repositories with not much data to remove.
 Also the memory usage of the `prune` command is now reduced.
+Restic used to rebuild the index from scratch after pruning. This is now
+changed and the index rebuilding uses the information already known by `prune`.
 
 By default, the `prune` command no longer removes all unused data. This
 behavior can be fine-tuned by new options, like the acceptable amount of unused space or
@@ -14,9 +16,11 @@ also shows what `prune` would do.
 
 Fixes several open issues, e.g.:
 https://github.com/restic/restic/issues/1140
+https://github.com/restic/restic/issues/1599
 https://github.com/restic/restic/issues/1985
 https://github.com/restic/restic/issues/2112
 https://github.com/restic/restic/issues/2227
 https://github.com/restic/restic/issues/2305
 
 https://github.com/restic/restic/pull/2718
+https://github.com/restic/restic/pull/2842

cmd/restic/cmd_prune.go

@@ -471,19 +471,26 @@ func prune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, usedB
 		DeleteFiles(gopts, repo, removePacksFirst, restic.PackFile)
 	}
 
+	packsAddedByRepack := 0
 	if len(repackPacks) != 0 {
+		packsAddedByRepack -= len(repo.Index().Packs())
 		Verbosef("repacking packs\n")
 		bar := newProgressMax(!gopts.Quiet, uint64(len(repackPacks)), "packs repacked")
 		_, err := repository.Repack(ctx, repo, repackPacks, keepBlobs, bar)
 		if err != nil {
 			return err
 		}
+		packsAddedByRepack += len(repo.Index().Packs())
+
 		// Also remove repacked packs
 		removePacks.Merge(repackPacks)
 	}
 
 	if len(removePacks) != 0 {
-		if err = rebuildIndex(ctx, repo, removePacks); err != nil {
+		totalpacks := int(stats.packs.used+stats.packs.partlyUsed+stats.packs.unused) -
+			len(removePacks) + packsAddedByRepack
+		err = rebuildIndexFiles(gopts, repo, removePacks, uint64(totalpacks))
+		if err != nil {
 			return err
 		}
 
@@ -495,6 +502,20 @@ func prune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, usedB
 	return nil
 }
 
+func rebuildIndexFiles(gopts GlobalOptions, repo restic.Repository, removePacks restic.IDSet, packcount uint64) error {
+	Verbosef("rebuilding index\n")
+
+	bar := newProgressMax(!gopts.Quiet, packcount, "packs processed")
+	obsoleteIndexes, err := (repo.Index()).(*repository.MasterIndex).
+		Save(gopts.ctx, repo, removePacks, bar)
+	if err != nil {
+		return err
+	}
+
+	Verbosef("deleting obsolete index files\n")
+	return DeleteFilesChecked(gopts, repo, obsoleteIndexes, restic.IndexFile)
+}
+
 func getUsedBlobs(gopts GlobalOptions, repo restic.Repository, snapshots []*restic.Snapshot) (usedBlobs restic.BlobSet, err error) {
 	ctx := gopts.ctx
 

doc/060_forget.rst

@@ -99,12 +99,10 @@ command must be run:
     
     repacking packs
     [0:00] 100.00%  2 / 2 packs repacked
-    counting files in repo
+    rebuilding index
-    [0:00] 100.00%  3 / 3 packs
+    [0:00] 100.00%  3 / 3 packs processed
-    finding old index files
+    deleting obsolete index files
-    saved new indexes as [59270b3a]
+    [0:00] 100.00%  3 / 3 files deleted
-    remove 4 old index files
-    [0:00] 100.00%  4 / 4 files deleted
     removing 3 old packs
     [0:00] 100.00%  3 / 3 files deleted
     done
@@ -147,12 +145,10 @@ to ``forget``:
     
     repacking packs
     [0:00] 100.00%  2 / 2 packs repacked
-    counting files in repo
+    rebuilding index
-    [0:00] 100.00%  3 / 3 packs
+    [0:00] 100.00%  3 / 3 packs processed
-    finding old index files
+    deleting obsolete index files
-    saved new indexes as [59270b3a]
+    [0:00] 100.00%  3 / 3 files deleted
-    remove 4 old index files
-    [0:00] 100.00%  4 / 4 files deleted
     removing 3 old packs
     [0:00] 100.00%  3 / 3 files deleted
     done

internal/repository/repository.go

@@ -322,7 +322,10 @@ func (r *Repository) Flush(ctx context.Context) error {
 		return err
 	}
 
-	// Save index after flushing
+	// Save index after flushing only if noAutoIndexUpdate is not set
+	if r.noAutoIndexUpdate {
+		return nil
+	}
 	return r.SaveIndex(ctx)
 }