Add tests for StreamPack

internal/repository/repository_test.go

@@ -4,17 +4,22 @@ import (
 	"bytes"
 	"context"
 	"crypto/sha256"
+	"encoding/json"
 	"fmt"
 	"io"
 	"math/rand"
 	"os"
 	"path/filepath"
+	"strings"
 	"testing"
 	"time"
 
+	"github.com/google/go-cmp/cmp"
 	"github.com/restic/restic/internal/archiver"
+	"github.com/restic/restic/internal/crypto"
 	"github.com/restic/restic/internal/repository"
 	"github.com/restic/restic/internal/restic"
+	"github.com/restic/restic/internal/test"
 	rtest "github.com/restic/restic/internal/test"
 )
 
@@ -408,4 +413,202 @@ func TestRepositoryIncrementalIndex(t *testing.T) {
 			t.Errorf("pack %v listed in %d indexes\n", packID, len(ids))
 		}
 	}
+
+}
+
+// buildPackfileWithoutHeader returns a manually built pack file without a header.
+func buildPackfileWithoutHeader(t testing.TB, blobSizes []int, key *crypto.Key) (blobs []restic.Blob, packfile []byte) {
+	var offset uint
+	for i, size := range blobSizes {
+		plaintext := test.Random(800+i, size)
+
+		// we use a deterministic nonce here so the whole process is
+		// deterministic, last byte is the blob index
+		var nonce = []byte{
+			0x15, 0x98, 0xc0, 0xf7, 0xb9, 0x65, 0x97, 0x74,
+			0x12, 0xdc, 0xd3, 0x62, 0xa9, 0x6e, 0x20, byte(i),
+		}
+
+		before := len(packfile)
+		packfile = append(packfile, nonce...)
+		packfile = key.Seal(packfile, nonce, plaintext, nil)
+		after := len(packfile)
+
+		ciphertextLength := after - before
+
+		blobs = append(blobs, restic.Blob{
+			BlobHandle: restic.BlobHandle{
+				ID:   restic.Hash(plaintext),
+				Type: restic.DataBlob,
+			},
+			Length: uint(ciphertextLength),
+			Offset: offset,
+		})
+
+		offset = uint(len(packfile))
+	}
+
+	return blobs, packfile
+}
+
+func TestStreamPack(t *testing.T) {
+	// always use the same key for deterministic output
+	const jsonKey = `{"mac":{"k":"eQenuI8adktfzZMuC8rwdA==","r":"k8cfAly2qQSky48CQK7SBA=="},"encrypt":"MKO9gZnRiQFl8mDUurSDa9NMjiu9MUifUrODTHS05wo="}`
+
+	var key crypto.Key
+	err := json.Unmarshal([]byte(jsonKey), &key)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	blobSizes := []int{
+		10,
+		5231,
+		18812,
+		123123,
+		12301,
+		892242,
+		28616,
+		13351,
+		252287,
+		188883,
+		2522811,
+		18883,
+	}
+
+	packfileBlobs, packfile := buildPackfileWithoutHeader(t, blobSizes, &key)
+
+	load := func(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
+		data := packfile
+
+		if offset > int64(len(data)) {
+			offset = 0
+			length = 0
+		}
+		data = data[offset:]
+
+		if length > len(data) {
+			length = len(data)
+		}
+
+		data = data[:length]
+
+		return fn(bytes.NewReader(data))
+
+	}
+
+	// first, test regular usage
+	t.Run("regular", func(t *testing.T) {
+		tests := []struct {
+			blobs []restic.Blob
+		}{
+			{packfileBlobs[1:2]},
+			{packfileBlobs[2:5]},
+			{packfileBlobs[2:8]},
+			{[]restic.Blob{
+				packfileBlobs[0],
+				packfileBlobs[8],
+				packfileBlobs[4],
+			}},
+			{[]restic.Blob{
+				packfileBlobs[0],
+				packfileBlobs[len(packfileBlobs)-1],
+			}},
+		}
+
+		for _, test := range tests {
+			t.Run("", func(t *testing.T) {
+				ctx, cancel := context.WithCancel(context.Background())
+				defer cancel()
+
+				gotBlobs := make(map[restic.ID]int)
+
+				handleBlob := func(blob restic.BlobHandle, buf []byte, err error) error {
+					gotBlobs[blob.ID]++
+
+					id := restic.Hash(buf)
+					if !id.Equal(blob.ID) {
+						t.Fatalf("wrong id %v for blob %s returned", id, blob.ID)
+					}
+
+					return err
+				}
+
+				wantBlobs := make(map[restic.ID]int)
+				for _, blob := range test.blobs {
+					wantBlobs[blob.ID] = 1
+				}
+
+				err = repository.StreamPack(ctx, load, &key, restic.ID{}, test.blobs, handleBlob)
+				if err != nil {
+					t.Fatal(err)
+				}
+
+				if !cmp.Equal(wantBlobs, gotBlobs) {
+					t.Fatal(cmp.Diff(wantBlobs, gotBlobs))
+				}
+			})
+		}
+	})
+
+	// next, test invalid uses, which should return an error
+	t.Run("invalid", func(t *testing.T) {
+		tests := []struct {
+			blobs []restic.Blob
+			err   string
+		}{
+			{
+				// pass one blob several times
+				blobs: []restic.Blob{
+					packfileBlobs[3],
+					packfileBlobs[8],
+					packfileBlobs[3],
+					packfileBlobs[4],
+				},
+				err: "overlapping blobs in pack",
+			},
+
+			{
+				// pass something that's not a valid blob in the current pack file
+				blobs: []restic.Blob{
+					{
+						Offset: 123,
+						Length: 20000,
+					},
+				},
+				err: "ciphertext verification failed",
+			},
+
+			{
+				// pass a blob that's too small
+				blobs: []restic.Blob{
+					{
+						Offset: 123,
+						Length: 10,
+					},
+				},
+				err: "invalid blob length",
+			},
+		}
+
+		for _, test := range tests {
+			t.Run("", func(t *testing.T) {
+				ctx, cancel := context.WithCancel(context.Background())
+				defer cancel()
+
+				handleBlob := func(blob restic.BlobHandle, buf []byte, err error) error {
+					return err
+				}
+
+				err = repository.StreamPack(ctx, load, &key, restic.ID{}, test.blobs, handleBlob)
+				if err == nil {
+					t.Fatalf("wanted error %v, got nil", test.err)
+				}
+
+				if !strings.Contains(err.Error(), test.err) {
+					t.Fatalf("wrong error returned, it should contain %q but was %q", test.err, err)
+				}
+			})
+		}
+	})
 }