restic/internal/restorer/packqueue_test.go

package restorer

import (
	"testing"

	"github.com/restic/restic/internal/restic"
	rtest "github.com/restic/restic/internal/test"
)

func processPack(t *testing.T, data *TestRepo, pack *packInfo, files []*fileInfo) {
	for _, file := range files {
		data.idx.forEachFilePack(file, func(packIdx int, packID restic.ID, packBlobs []restic.Blob) bool {
			// assert file's head pack
			rtest.Equals(t, pack.id, packID)
			file.blobs = file.blobs[len(packBlobs):]
			return false // only interested in the head pack
		})
	}
}

func TestPackQueueBasic(t *testing.T) {
	data := newTestRepo([]TestFile{
		TestFile{
			name: "file",
			blobs: []TestBlob{
				TestBlob{"data1", "pack1"},
				TestBlob{"data2", "pack2"},
			},
		},
	})

	queue, err := newPackQueue(data.idx, data.files, func(_ map[*fileInfo]struct{}) bool { return false })
	rtest.OK(t, err)

	// assert initial queue state
	rtest.Equals(t, false, queue.isEmpty())
	rtest.Equals(t, 0, queue.packs[data.packID("pack1")].cost)
	rtest.Equals(t, 1, queue.packs[data.packID("pack2")].cost)

	// get first pack
	pack, files := queue.nextPack()
	rtest.Equals(t, "pack1", data.packName(pack))
	rtest.Equals(t, 1, len(files))
	rtest.Equals(t, false, queue.isEmpty())
	// TODO assert pack is inprogress

	// can't process the second pack until the first one is processed
	{
		pack, files := queue.nextPack()
		rtest.Equals(t, true, pack == nil)
		rtest.Equals(t, true, files == nil)
		rtest.Equals(t, false, queue.isEmpty())
	}

	// requeue the pack without processing
	rtest.Equals(t, true, queue.requeuePack(pack, []*fileInfo{}, []*fileInfo{}))
	rtest.Equals(t, false, queue.isEmpty())
	rtest.Equals(t, 0, queue.packs[data.packID("pack1")].cost)
	rtest.Equals(t, 1, queue.packs[data.packID("pack2")].cost)

	// get the first pack again
	pack, files = queue.nextPack()
	rtest.Equals(t, "pack1", data.packName(pack))
	rtest.Equals(t, 1, len(files))
	rtest.Equals(t, false, queue.isEmpty())

	// process the first pack and return it to the queue
	processPack(t, data, pack, files)
	rtest.Equals(t, false, queue.requeuePack(pack, files, []*fileInfo{}))
	rtest.Equals(t, 0, queue.packs[data.packID("pack2")].cost)

	// get the second pack
	pack, files = queue.nextPack()
	rtest.Equals(t, "pack2", data.packName(pack))
	rtest.Equals(t, 1, len(files))
	rtest.Equals(t, false, queue.isEmpty())

	// process the second pack and return it to the queue
	processPack(t, data, pack, files)
	rtest.Equals(t, false, queue.requeuePack(pack, files, []*fileInfo{}))

	// all packs processed
	rtest.Equals(t, true, queue.isEmpty())
}

func TestPackQueueFailedFile(t *testing.T) {
	// point of this test is to assert that enqueuePack removes
	// all references to files that failed restore

	data := newTestRepo([]TestFile{
		TestFile{
			name: "file",
			blobs: []TestBlob{
				TestBlob{"data1", "pack1"},
				TestBlob{"data2", "pack2"},
			},
		},
	})

	queue, err := newPackQueue(data.idx, data.files, func(_ map[*fileInfo]struct{}) bool { return false })
	rtest.OK(t, err)

	pack, files := queue.nextPack()
	rtest.Equals(t, false, queue.requeuePack(pack, []*fileInfo{}, files /*failed*/))
	rtest.Equals(t, true, queue.isEmpty())
}

func TestPackQueueOrderingCost(t *testing.T) {
	// assert pack1 is selected before pack2:
	// pack1 is ready to restore file1, pack2 is ready to restore file2
	// but pack2 cannot be immediately used to restore file1

	data := newTestRepo([]TestFile{
		TestFile{
			name: "file1",
			blobs: []TestBlob{
				TestBlob{"data1", "pack1"},
				TestBlob{"data2", "pack2"},
			},
		},
		TestFile{
			name: "file2",
			blobs: []TestBlob{
				TestBlob{"data2", "pack2"},
			},
		},
	})

	queue, err := newPackQueue(data.idx, data.files, func(_ map[*fileInfo]struct{}) bool { return false })
	rtest.OK(t, err)

	// assert initial pack costs
	rtest.Equals(t, 0, data.pack(queue, "pack1").cost)
	rtest.Equals(t, 0, data.pack(queue, "pack1").index) // head of the heap
	rtest.Equals(t, 1, data.pack(queue, "pack2").cost)
	rtest.Equals(t, 1, data.pack(queue, "pack2").index)

	pack, files := queue.nextPack()
	// assert selected pack and queue state
	rtest.Equals(t, "pack1", data.packName(pack))
	// process the pack
	processPack(t, data, pack, files)
	rtest.Equals(t, false, queue.requeuePack(pack, files, []*fileInfo{}))
}

func TestPackQueueOrderingInprogress(t *testing.T) {
	// finish restoring one file before starting another

	data := newTestRepo([]TestFile{
		TestFile{
			name: "file1",
			blobs: []TestBlob{
				TestBlob{"data1-1", "pack1-1"},
				TestBlob{"data1-2", "pack1-2"},
			},
		},
		TestFile{
			name: "file2",
			blobs: []TestBlob{
				TestBlob{"data2-1", "pack2-1"},
				TestBlob{"data2-2", "pack2-2"},
			},
		},
	})

	var inprogress *fileInfo
	queue, err := newPackQueue(data.idx, data.files, func(files map[*fileInfo]struct{}) bool {
		_, found := files[inprogress]
		return found
	})
	rtest.OK(t, err)

	// first pack of a file
	pack, files := queue.nextPack()
	rtest.Equals(t, 1, len(files))
	file := files[0]
	processPack(t, data, pack, files)
	inprogress = files[0]
	queue.requeuePack(pack, files, []*fileInfo{})

	// second pack of the same file
	pack, files = queue.nextPack()
	rtest.Equals(t, 1, len(files))
	rtest.Equals(t, true, file == files[0]) // same file as before
	processPack(t, data, pack, files)
	inprogress = nil
	queue.requeuePack(pack, files, []*fileInfo{})

	// first pack of the second file
	pack, files = queue.nextPack()
	rtest.Equals(t, 1, len(files))
	rtest.Equals(t, false, file == files[0]) // different file as before
}

func TestPackQueuePackMultiuse(t *testing.T) {
	// the same pack is required multiple times to restore the same file

	data := newTestRepo([]TestFile{
		TestFile{
			name: "file",
			blobs: []TestBlob{
				TestBlob{"data1", "pack1"},
				TestBlob{"data2", "pack2"},
				TestBlob{"data3", "pack1"}, // pack1 reuse, new blob
				TestBlob{"data2", "pack2"}, // pack2 reuse, same blob
			},
		},
	})

	queue, err := newPackQueue(data.idx, data.files, func(_ map[*fileInfo]struct{}) bool { return false })
	rtest.OK(t, err)

	pack, files := queue.nextPack()
	rtest.Equals(t, "pack1", data.packName(pack))
	rtest.Equals(t, 1, len(pack.files))
	processPack(t, data, pack, files)
	rtest.Equals(t, true, queue.requeuePack(pack, files, []*fileInfo{}))

	pack, files = queue.nextPack()
	rtest.Equals(t, "pack2", data.packName(pack))
	rtest.Equals(t, 1, len(pack.files))
	processPack(t, data, pack, files)
	rtest.Equals(t, true, queue.requeuePack(pack, files, []*fileInfo{}))

	pack, files = queue.nextPack()
	rtest.Equals(t, "pack1", data.packName(pack))
	processPack(t, data, pack, files)
	rtest.Equals(t, false, queue.requeuePack(pack, files, []*fileInfo{}))

	pack, files = queue.nextPack()
	rtest.Equals(t, "pack2", data.packName(pack))
	processPack(t, data, pack, files)
	rtest.Equals(t, false, queue.requeuePack(pack, files, []*fileInfo{}))

	rtest.Equals(t, true, queue.isEmpty())
}