restic/cmd/restic/cmd_prune_integration_test.go

package main

import (
	"context"
	"encoding/json"
	"path/filepath"
	"testing"

	"github.com/restic/restic/internal/backend"
	rtest "github.com/restic/restic/internal/test"
)

func testRunPrune(t testing.TB, gopts GlobalOptions, opts PruneOptions) {
	oldHook := gopts.backendTestHook
	gopts.backendTestHook = func(r backend.Backend) (backend.Backend, error) { return newListOnceBackend(r), nil }
	defer func() {
		gopts.backendTestHook = oldHook
	}()
	rtest.OK(t, runPrune(context.TODO(), opts, gopts))
}

func TestPrune(t *testing.T) {
	testPruneVariants(t, false)
	testPruneVariants(t, true)
}

func testPruneVariants(t *testing.T, unsafeNoSpaceRecovery bool) {
	suffix := ""
	if unsafeNoSpaceRecovery {
		suffix = "-recovery"
	}
	t.Run("0"+suffix, func(t *testing.T) {
		opts := PruneOptions{MaxUnused: "0%", unsafeRecovery: unsafeNoSpaceRecovery}
		checkOpts := CheckOptions{ReadData: true, CheckUnused: true}
		testPrune(t, opts, checkOpts)
	})

	t.Run("50"+suffix, func(t *testing.T) {
		opts := PruneOptions{MaxUnused: "50%", unsafeRecovery: unsafeNoSpaceRecovery}
		checkOpts := CheckOptions{ReadData: true}
		testPrune(t, opts, checkOpts)
	})

	t.Run("unlimited"+suffix, func(t *testing.T) {
		opts := PruneOptions{MaxUnused: "unlimited", unsafeRecovery: unsafeNoSpaceRecovery}
		checkOpts := CheckOptions{ReadData: true}
		testPrune(t, opts, checkOpts)
	})

	t.Run("CachableOnly"+suffix, func(t *testing.T) {
		opts := PruneOptions{MaxUnused: "5%", RepackCachableOnly: true, unsafeRecovery: unsafeNoSpaceRecovery}
		checkOpts := CheckOptions{ReadData: true}
		testPrune(t, opts, checkOpts)
	})
	t.Run("Small", func(t *testing.T) {
		opts := PruneOptions{MaxUnused: "unlimited", RepackSmall: true}
		checkOpts := CheckOptions{ReadData: true, CheckUnused: true}
		testPrune(t, opts, checkOpts)
	})
}

func createPrunableRepo(t *testing.T, env *testEnvironment) {
	testSetupBackupData(t, env)
	opts := BackupOptions{}

	testRunBackup(t, "", []string{filepath.Join(env.testdata, "0", "0", "9")}, opts, env.gopts)
	firstSnapshot := testListSnapshots(t, env.gopts, 1)[0]

	testRunBackup(t, "", []string{filepath.Join(env.testdata, "0", "0", "9", "2")}, opts, env.gopts)
	testRunBackup(t, "", []string{filepath.Join(env.testdata, "0", "0", "9", "3")}, opts, env.gopts)
	testListSnapshots(t, env.gopts, 3)

	testRunForgetJSON(t, env.gopts)
	testRunForget(t, env.gopts, firstSnapshot.String())
}

func testRunForgetJSON(t testing.TB, gopts GlobalOptions, args ...string) {
	buf, err := withCaptureStdout(func() error {
		gopts.JSON = true
		opts := ForgetOptions{
			DryRun: true,
			Last:   1,
		}
		return runForget(context.TODO(), opts, gopts, args)
	})
	rtest.OK(t, err)

	var forgets []*ForgetGroup
	rtest.OK(t, json.Unmarshal(buf.Bytes(), &forgets))

	rtest.Assert(t, len(forgets) == 1,
		"Expected 1 snapshot group, got %v", len(forgets))
	rtest.Assert(t, len(forgets[0].Keep) == 1,
		"Expected 1 snapshot to be kept, got %v", len(forgets[0].Keep))
	rtest.Assert(t, len(forgets[0].Remove) == 2,
		"Expected 2 snapshots to be removed, got %v", len(forgets[0].Remove))
}

func testPrune(t *testing.T, pruneOpts PruneOptions, checkOpts CheckOptions) {
	env, cleanup := withTestEnvironment(t)
	defer cleanup()

	createPrunableRepo(t, env)
	testRunPrune(t, env.gopts, pruneOpts)
	rtest.OK(t, runCheck(context.TODO(), checkOpts, env.gopts, nil))
}

var pruneDefaultOptions = PruneOptions{MaxUnused: "5%"}

func TestPruneWithDamagedRepository(t *testing.T) {
	env, cleanup := withTestEnvironment(t)
	defer cleanup()

	datafile := filepath.Join("testdata", "backup-data.tar.gz")
	testRunInit(t, env.gopts)

	rtest.SetupTarTestFixture(t, env.testdata, datafile)
	opts := BackupOptions{}

	// create and delete snapshot to create unused blobs
	testRunBackup(t, "", []string{filepath.Join(env.testdata, "0", "0", "9", "2")}, opts, env.gopts)
	firstSnapshot := testListSnapshots(t, env.gopts, 1)[0]
	testRunForget(t, env.gopts, firstSnapshot.String())

	oldPacks := listPacks(env.gopts, t)

	// create new snapshot, but lose all data
	testRunBackup(t, "", []string{filepath.Join(env.testdata, "0", "0", "9", "3")}, opts, env.gopts)
	testListSnapshots(t, env.gopts, 1)
	removePacksExcept(env.gopts, t, oldPacks, false)

	oldHook := env.gopts.backendTestHook
	env.gopts.backendTestHook = func(r backend.Backend) (backend.Backend, error) { return newListOnceBackend(r), nil }
	defer func() {
		env.gopts.backendTestHook = oldHook
	}()
	// prune should fail
	rtest.Assert(t, runPrune(context.TODO(), pruneDefaultOptions, env.gopts) == errorPacksMissing,
		"prune should have reported index not complete error")
}

// Test repos for edge cases
func TestEdgeCaseRepos(t *testing.T) {
	opts := CheckOptions{}

	// repo where index is completely missing
	// => check and prune should fail
	t.Run("no-index", func(t *testing.T) {
		testEdgeCaseRepo(t, "repo-index-missing.tar.gz", opts, pruneDefaultOptions, false, false)
	})

	// repo where an existing and used blob is missing from the index
	// => check and prune should fail
	t.Run("index-missing-blob", func(t *testing.T) {
		testEdgeCaseRepo(t, "repo-index-missing-blob.tar.gz", opts, pruneDefaultOptions, false, false)
	})

	// repo where a blob is missing
	// => check and prune should fail
	t.Run("missing-data", func(t *testing.T) {
		testEdgeCaseRepo(t, "repo-data-missing.tar.gz", opts, pruneDefaultOptions, false, false)
	})

	// repo where blobs which are not needed are missing or in invalid pack files
	// => check should fail and prune should repair this
	t.Run("missing-unused-data", func(t *testing.T) {
		testEdgeCaseRepo(t, "repo-unused-data-missing.tar.gz", opts, pruneDefaultOptions, false, true)
	})

	// repo where data exists that is not referenced
	// => check and prune should fully work
	t.Run("unreferenced-data", func(t *testing.T) {
		testEdgeCaseRepo(t, "repo-unreferenced-data.tar.gz", opts, pruneDefaultOptions, true, true)
	})

	// repo where an obsolete index still exists
	// => check and prune should fully work
	t.Run("obsolete-index", func(t *testing.T) {
		testEdgeCaseRepo(t, "repo-obsolete-index.tar.gz", opts, pruneDefaultOptions, true, true)
	})

	// repo which contains mixed (data/tree) packs
	// => check and prune should fully work
	t.Run("mixed-packs", func(t *testing.T) {
		testEdgeCaseRepo(t, "repo-mixed.tar.gz", opts, pruneDefaultOptions, true, true)
	})

	// repo which contains duplicate blobs
	// => checking for unused data should report an error and prune resolves the
	// situation
	opts = CheckOptions{
		ReadData:    true,
		CheckUnused: true,
	}
	t.Run("duplicates", func(t *testing.T) {
		testEdgeCaseRepo(t, "repo-duplicates.tar.gz", opts, pruneDefaultOptions, false, true)
	})
}

func testEdgeCaseRepo(t *testing.T, tarfile string, optionsCheck CheckOptions, optionsPrune PruneOptions, checkOK, pruneOK bool) {
	env, cleanup := withTestEnvironment(t)
	defer cleanup()

	datafile := filepath.Join("testdata", tarfile)
	rtest.SetupTarTestFixture(t, env.base, datafile)

	if checkOK {
		testRunCheck(t, env.gopts)
	} else {
		rtest.Assert(t, runCheck(context.TODO(), optionsCheck, env.gopts, nil) != nil,
			"check should have reported an error")
	}

	if pruneOK {
		testRunPrune(t, env.gopts, optionsPrune)
		testRunCheck(t, env.gopts)
	} else {
		rtest.Assert(t, runPrune(context.TODO(), optionsPrune, env.gopts) != nil,
			"prune should have reported an error")
	}
}