diff --git a/internal/backend/b2/b2.go b/internal/backend/b2/b2.go
index 150d396d5..26e23627a 100644
--- a/internal/backend/b2/b2.go
+++ b/internal/backend/b2/b2.go
@@ -274,14 +274,22 @@ func (be *b2Backend) Remove(ctx context.Context, h restic.Handle) error {
 	be.sem.GetToken()
 	defer be.sem.ReleaseToken()
 
-	obj := be.bucket.Object(be.Filename(h))
-	err := obj.Delete(ctx)
-	// consider a file as removed if b2 informs us that it does not exist
-	if b2.IsNotExist(err) {
-		return nil
+	// the retry backend will also repeat the remove method up to 10 times
+	for i := 0; i < 3; i++ {
+		obj := be.bucket.Object(be.Filename(h))
+		err := obj.Delete(ctx)
+		if err == nil {
+			// keep deleting until we are sure that no leftover file versions exist
+			continue
+		}
+		// consider a file as removed if b2 informs us that it does not exist
+		if b2.IsNotExist(err) {
+			return nil
+		}
+		return errors.Wrap(err, "Delete")
 	}
 
-	return errors.Wrap(err, "Delete")
+	return errors.New("failed to delete all file versions")
 }
 
 type semLocker struct {