From 2ee654763b398b7070a2c58e5820d42ec46fe95c Mon Sep 17 00:00:00 2001
From: Alexander Weiss <alex@weissfam.de>
Date: Fri, 17 Jul 2020 22:14:46 +0200
Subject: [PATCH] Delete files in parallel

---
 cmd/restic/cmd_prune.go | 13 ++-------
 cmd/restic/delete.go    | 62 +++++++++++++++++++++++++++++++++++++++++
 2 files changed, 64 insertions(+), 11 deletions(-)
 create mode 100644 cmd/restic/delete.go

diff --git a/cmd/restic/cmd_prune.go b/cmd/restic/cmd_prune.go
index 7d374b1b8..dad96868b 100644
--- a/cmd/restic/cmd_prune.go
+++ b/cmd/restic/cmd_prune.go
@@ -296,17 +296,8 @@ func pruneRepository(gopts GlobalOptions, repo restic.Repository) error {
 	}
 
 	if len(removePacks) != 0 {
-		bar = newProgressMax(!gopts.Quiet, uint64(len(removePacks)), "packs deleted")
-		bar.Start()
-		for packID := range removePacks {
-			h := restic.Handle{Type: restic.DataFile, Name: packID.String()}
-			err = repo.Backend().Remove(ctx, h)
-			if err != nil {
-				Warnf("unable to remove file %v from the repository\n", packID.Str())
-			}
-			bar.Report(restic.Stat{Blobs: 1})
-		}
-		bar.Done()
+		Verbosef("remove %d old packs\n", len(removePacks))
+		DeleteFiles(gopts, repo, removePacks, restic.DataFile)
 	}
 
 	Verbosef("done\n")
diff --git a/cmd/restic/delete.go b/cmd/restic/delete.go
new file mode 100644
index 000000000..0d92f3375
--- /dev/null
+++ b/cmd/restic/delete.go
@@ -0,0 +1,62 @@
+package main
+
+import (
+	"golang.org/x/sync/errgroup"
+
+	"github.com/restic/restic/internal/restic"
+)
+
+// DeleteFiles deletes the given fileList of fileType in parallel
+// it will print a warning if there is an error, but continue deleting the remaining files
+func DeleteFiles(gopts GlobalOptions, repo restic.Repository, fileList restic.IDSet, fileType restic.FileType) {
+	deleteFiles(gopts, true, repo, fileList, fileType)
+}
+
+// DeleteFiles deletes the given fileList of fileType in parallel
+// if an error occurs, it will cancel and return this error
+func DeleteFilesChecked(gopts GlobalOptions, repo restic.Repository, fileList restic.IDSet, fileType restic.FileType) error {
+	return deleteFiles(gopts, false, repo, fileList, fileType)
+}
+
+const numDeleteWorkers = 8
+
+// deleteFiles deletes the given fileList of fileType in parallel
+// if ignoreError=true, it will print a warning if there was an error, else it will abort.
+func deleteFiles(gopts GlobalOptions, ignoreError bool, repo restic.Repository, fileList restic.IDSet, fileType restic.FileType) error {
+	totalCount := len(fileList)
+	fileChan := make(chan restic.ID)
+	go func() {
+		for id := range fileList {
+			fileChan <- id
+		}
+		close(fileChan)
+	}()
+
+	bar := newProgressMax(!gopts.JSON && !gopts.Quiet, uint64(totalCount), "files deleted")
+	wg, ctx := errgroup.WithContext(gopts.ctx)
+	bar.Start()
+	for i := 0; i < numDeleteWorkers; i++ {
+		wg.Go(func() error {
+			for id := range fileChan {
+				h := restic.Handle{Type: fileType, Name: id.String()}
+				err := repo.Backend().Remove(ctx, h)
+				if err != nil {
+					if !gopts.JSON {
+						Warnf("unable to remove %v from the repository\n", h)
+					}
+					if !ignoreError {
+						return err
+					}
+				}
+				if !gopts.JSON && gopts.verbosity >= 2 {
+					Verbosef("removed %v\n", h)
+				}
+				bar.Report(restic.Stat{Blobs: 1})
+			}
+			return nil
+		})
+	}
+	err := wg.Wait()
+	bar.Done()
+	return err
+}