From 772a907533f60fb943c8255c84752b4605fe8325 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 30 Dec 2023 21:40:41 +0100 Subject: [PATCH] repository: StreamPack delivers blobs at most once If an error occurred while streaming a pack file, this could result in passing some of the blobs multiple times to the callback function. This significantly complicates using StreamPack correctly and is unnecessary. Retries do not change the content of a blob and thus only deliver the same result over and over again. --- internal/repository/repository.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/internal/repository/repository.go b/internal/repository/repository.go index d034911c5..0b50382b8 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -887,9 +887,9 @@ type BackendLoadFn func(ctx context.Context, h restic.Handle, length int, offset const maxUnusedRange = 4 * 1024 * 1024 // StreamPack loads the listed blobs from the specified pack file. The plaintext blob is passed to -// the handleBlobFn callback or an error if decryption failed or the blob hash does not match. In -// case of download errors handleBlobFn might be called multiple times for the same blob. If the -// callback returns an error, then StreamPack will abort and not retry it. +// the handleBlobFn callback or an error if decryption failed or the blob hash does not match. +// handleBlobFn is never called multiple times for the same blob. If the callback returns an error, +// then StreamPack will abort and not retry it. func StreamPack(ctx context.Context, beLoad BackendLoadFn, key *crypto.Key, packID restic.ID, blobs []restic.Blob, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error { if len(blobs) == 0 { // nothing to do @@ -951,7 +951,9 @@ func streamPackPart(ctx context.Context, beLoad BackendLoadFn, key *crypto.Key, currentBlobEnd := dataStart var buf []byte var decode []byte - for _, entry := range blobs { + for len(blobs) > 0 { + entry := blobs[0] + skipBytes := int(entry.Offset - currentBlobEnd) if skipBytes < 0 { return errors.Errorf("overlapping blobs in pack %v", packID) @@ -1014,6 +1016,8 @@ func streamPackPart(ctx context.Context, beLoad BackendLoadFn, key *crypto.Key, cancel() return backoff.Permanent(err) } + // ensure that each blob is only passed once to handleBlobFn + blobs = blobs[1:] } return nil })