Delete files in parallel

cmd/restic/cmd_prune.go

@@ -296,17 +296,8 @@ func pruneRepository(gopts GlobalOptions, repo restic.Repository) error {
 	}
 
 	if len(removePacks) != 0 {
-		bar = newProgressMax(!gopts.Quiet, uint64(len(removePacks)), "packs deleted")
-		bar.Start()
-		for packID := range removePacks {
-			h := restic.Handle{Type: restic.DataFile, Name: packID.String()}
-			err = repo.Backend().Remove(ctx, h)
-			if err != nil {
-				Warnf("unable to remove file %v from the repository\n", packID.Str())
-			}
-			bar.Report(restic.Stat{Blobs: 1})
-		}
-		bar.Done()
+		Verbosef("remove %d old packs\n", len(removePacks))
+		DeleteFiles(gopts, repo, removePacks, restic.DataFile)
 	}
 
 	Verbosef("done\n")

cmd/restic/delete.go

@@ -0,0 +1,62 @@
+package main
+
+import (
+	"golang.org/x/sync/errgroup"
+
+	"github.com/restic/restic/internal/restic"
+)
+
+// DeleteFiles deletes the given fileList of fileType in parallel
+// it will print a warning if there is an error, but continue deleting the remaining files
+func DeleteFiles(gopts GlobalOptions, repo restic.Repository, fileList restic.IDSet, fileType restic.FileType) {
+	deleteFiles(gopts, true, repo, fileList, fileType)
+}
+
+// DeleteFiles deletes the given fileList of fileType in parallel
+// if an error occurs, it will cancel and return this error
+func DeleteFilesChecked(gopts GlobalOptions, repo restic.Repository, fileList restic.IDSet, fileType restic.FileType) error {
+	return deleteFiles(gopts, false, repo, fileList, fileType)
+}
+
+const numDeleteWorkers = 8
+
+// deleteFiles deletes the given fileList of fileType in parallel
+// if ignoreError=true, it will print a warning if there was an error, else it will abort.
+func deleteFiles(gopts GlobalOptions, ignoreError bool, repo restic.Repository, fileList restic.IDSet, fileType restic.FileType) error {
+	totalCount := len(fileList)
+	fileChan := make(chan restic.ID)
+	go func() {
+		for id := range fileList {
+			fileChan <- id
+		}
+		close(fileChan)
+	}()
+
+	bar := newProgressMax(!gopts.JSON && !gopts.Quiet, uint64(totalCount), "files deleted")
+	wg, ctx := errgroup.WithContext(gopts.ctx)
+	bar.Start()
+	for i := 0; i < numDeleteWorkers; i++ {
+		wg.Go(func() error {
+			for id := range fileChan {
+				h := restic.Handle{Type: fileType, Name: id.String()}
+				err := repo.Backend().Remove(ctx, h)
+				if err != nil {
+					if !gopts.JSON {
+						Warnf("unable to remove %v from the repository\n", h)
+					}
+					if !ignoreError {
+						return err
+					}
+				}
+				if !gopts.JSON && gopts.verbosity >= 2 {
+					Verbosef("removed %v\n", h)
+				}
+				bar.Report(restic.Stat{Blobs: 1})
+			}
+			return nil
+		})
+	}
+	err := wg.Wait()
+	bar.Done()
+	return err
+}