Cancel current command if cache becomes unusable

If the cache suddenly disappears, the current command will now fail.

changelog/unreleased/pull-4166

@@ -0,0 +1,7 @@
+Enhancement: Cancel current command if cache becomes unusable
+
+If the cache directory was removed or ran out of space while restic was
+running, this caused further caching attempts to fail and drastically slow down
+the command execution. Now, the currently running command is canceled instead.
+
+https://github.com/restic/restic/pull/4166

internal/cache/backend.go

@@ -83,7 +83,7 @@ func (b *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRea
 	if err != nil {
 		debug.Log("unable to save %v to cache: %v", h, err)
 		_ = b.Cache.remove(h)
-		return nil
+		return err
 	}
 
 	return nil
@@ -106,20 +106,7 @@ func (b *Backend) cacheFile(ctx context.Context, h restic.Handle) error {
 		return nil
 	}
 
-	// test again, maybe the file was cached in the meantime
+	defer func() {
-	if !b.Cache.Has(h) {
-
-		// nope, it's still not in the cache, pull it from the repo and save it
-
-		err := b.Backend.Load(ctx, h, 0, 0, func(rd io.Reader) error {
-			return b.Cache.Save(h, rd)
-		})
-		if err != nil {
-			// try to remove from the cache, ignore errors
-			_ = b.Cache.remove(h)
-		}
-	}
-
 		// signal other waiting goroutines that the file may now be cached
 		close(finish)
 
@@ -127,6 +114,20 @@ func (b *Backend) cacheFile(ctx context.Context, h restic.Handle) error {
 		b.inProgressMutex.Lock()
 		delete(b.inProgress, h)
 		b.inProgressMutex.Unlock()
+	}()
+
+	// test again, maybe the file was cached in the meantime
+	if !b.Cache.Has(h) {
+		// nope, it's still not in the cache, pull it from the repo and save it
+		err := b.Backend.Load(ctx, h, 0, 0, func(rd io.Reader) error {
+			return b.Cache.Save(h, rd)
+		})
+		if err != nil {
+			// try to remove from the cache, ignore errors
+			_ = b.Cache.remove(h)
+		}
+		return err
+	}
 
 	return nil
 }
@@ -178,12 +179,14 @@ func (b *Backend) Load(ctx context.Context, h restic.Handle, length int, offset
 
 	debug.Log("auto-store %v in the cache", h)
 	err = b.cacheFile(ctx, h)
-	if err == nil {
+	if err != nil {
+		return err
+	}
+
 	inCache, err = b.loadFromCache(ctx, h, length, offset, consumer)
 	if inCache {
 		return err
 	}
-	}
 
 	debug.Log("error caching %v: %v, falling back to backend", h, err)
 	return b.Backend.Load(ctx, h, length, offset, consumer)

internal/cache/file_test.go

@@ -11,8 +11,10 @@ import (
 	"time"
 
 	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/fs"
 	"github.com/restic/restic/internal/restic"
 	"github.com/restic/restic/internal/test"
+	rtest "github.com/restic/restic/internal/test"
 
 	"golang.org/x/sync/errgroup"
 )
@@ -266,3 +268,19 @@ func TestFileSaveConcurrent(t *testing.T) {
 	saved := load(t, c, h)
 	test.Equals(t, data, saved)
 }
+
+func TestFileSaveAfterDamage(t *testing.T) {
+	c := TestNewCache(t)
+	rtest.OK(t, fs.RemoveAll(c.path))
+
+	// save a few bytes of data in the cache
+	data := test.Random(123456789, 42)
+	id := restic.Hash(data)
+	h := restic.Handle{
+		Type: restic.PackFile,
+		Name: id.String(),
+	}
+	if err := c.Save(h, bytes.NewReader(data)); err == nil {
+		t.Fatal("Missing error when saving to deleted cache directory")
+	}
+}