diff --git a/cmd/restic/cmd_prune.go b/cmd/restic/cmd_prune.go
index cfd22b1cd..46d62460b 100644
--- a/cmd/restic/cmd_prune.go
+++ b/cmd/restic/cmd_prune.go
@@ -277,6 +277,7 @@ func planPrune(ctx context.Context, opts PruneOptions, gopts GlobalOptions, repo
 	}
 
 	if len(plan.repackPacks) != 0 {
+		blobCount := keepBlobs.Len()
 		// when repacking, we do not want to keep blobs which are
 		// already contained in kept packs, so delete them from keepBlobs
 		repo.Index().Each(ctx, func(blob restic.PackedBlob) {
@@ -285,6 +286,11 @@ func planPrune(ctx context.Context, opts PruneOptions, gopts GlobalOptions, repo
 			}
 			keepBlobs.Delete(blob.BlobHandle)
 		})
+
+		if keepBlobs.Len() < blobCount/2 {
+			// replace with copy to shrink map to necessary size if there's a chance to benefit
+			keepBlobs = keepBlobs.Copy()
+		}
 	} else {
 		// keepBlobs is only needed if packs are repacked
 		keepBlobs = nil
diff --git a/internal/restic/counted_blob_set.go b/internal/restic/counted_blob_set.go
index 29644215b..f965d3129 100644
--- a/internal/restic/counted_blob_set.go
+++ b/internal/restic/counted_blob_set.go
@@ -57,3 +57,12 @@ func (s CountedBlobSet) String() string {
 
 	return "{" + str[1:len(str)-1] + "}"
 }
+
+// Copy returns a copy of the CountedBlobSet.
+func (s CountedBlobSet) Copy() CountedBlobSet {
+	cp := make(CountedBlobSet, len(s))
+	for k, v := range s {
+		cp[k] = v
+	}
+	return cp
+}
diff --git a/internal/restic/counted_blob_set_test.go b/internal/restic/counted_blob_set_test.go
index c5c321503..681751e91 100644
--- a/internal/restic/counted_blob_set_test.go
+++ b/internal/restic/counted_blob_set_test.go
@@ -35,3 +35,11 @@ func TestCountedBlobSet(t *testing.T) {
 	s := bs.String()
 	test.Assert(t, len(s) > 10, "invalid string: %v", s)
 }
+
+func TestCountedBlobSetCopy(t *testing.T) {
+	bs := restic.NewCountedBlobSet(restic.NewRandomBlobHandle(), restic.NewRandomBlobHandle(), restic.NewRandomBlobHandle())
+	test.Equals(t, bs.Len(), 3)
+	cp := bs.Copy()
+	test.Equals(t, cp.Len(), 3)
+	test.Equals(t, bs.List(), cp.List())
+}