mirror of
https://github.com/octoleo/restic.git
synced 2024-11-27 23:36:40 +00:00
73c9780321
Files were not included in the backup if the extended metadata for the file could not be read. This is rather drastic. Instead settle on returning a warning but still including the file in the backup.
2367 lines
57 KiB
Go
2367 lines
57 KiB
Go
package archiver
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"path/filepath"
|
|
"runtime"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"syscall"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/google/go-cmp/cmp"
|
|
"github.com/restic/restic/internal/backend"
|
|
"github.com/restic/restic/internal/backend/mem"
|
|
"github.com/restic/restic/internal/checker"
|
|
"github.com/restic/restic/internal/errors"
|
|
"github.com/restic/restic/internal/feature"
|
|
"github.com/restic/restic/internal/fs"
|
|
"github.com/restic/restic/internal/repository"
|
|
"github.com/restic/restic/internal/restic"
|
|
rtest "github.com/restic/restic/internal/test"
|
|
"golang.org/x/sync/errgroup"
|
|
)
|
|
|
|
func prepareTempdirRepoSrc(t testing.TB, src TestDir) (string, restic.Repository) {
|
|
tempdir := rtest.TempDir(t)
|
|
repo := repository.TestRepository(t)
|
|
|
|
TestCreateFiles(t, tempdir, src)
|
|
|
|
return tempdir, repo
|
|
}
|
|
|
|
func saveFile(t testing.TB, repo archiverRepo, filename string, filesystem fs.FS) (*restic.Node, ItemStats) {
|
|
wg, ctx := errgroup.WithContext(context.TODO())
|
|
repo.StartPackUploader(ctx, wg)
|
|
|
|
arch := New(repo, filesystem, Options{})
|
|
arch.runWorkers(ctx, wg)
|
|
|
|
arch.Error = func(item string, err error) error {
|
|
t.Errorf("archiver error for %v: %v", item, err)
|
|
return err
|
|
}
|
|
|
|
var (
|
|
completeReadingCallback bool
|
|
|
|
completeCallbackNode *restic.Node
|
|
completeCallbackStats ItemStats
|
|
completeCallback bool
|
|
|
|
startCallback bool
|
|
)
|
|
|
|
completeReading := func() {
|
|
completeReadingCallback = true
|
|
if completeCallback {
|
|
t.Error("callbacks called in wrong order")
|
|
}
|
|
}
|
|
|
|
complete := func(node *restic.Node, stats ItemStats) {
|
|
completeCallback = true
|
|
completeCallbackNode = node
|
|
completeCallbackStats = stats
|
|
}
|
|
|
|
start := func() {
|
|
startCallback = true
|
|
}
|
|
|
|
file, err := arch.FS.OpenFile(filename, fs.O_RDONLY|fs.O_NOFOLLOW, 0)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
fi, err := file.Stat()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
res := arch.fileSaver.Save(ctx, "/", filename, file, fi, start, completeReading, complete)
|
|
|
|
fnr := res.take(ctx)
|
|
if fnr.err != nil {
|
|
t.Fatal(fnr.err)
|
|
}
|
|
|
|
arch.stopWorkers()
|
|
err = repo.Flush(context.Background())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
if err := wg.Wait(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
if !startCallback {
|
|
t.Errorf("start callback did not happen")
|
|
}
|
|
|
|
if !completeReadingCallback {
|
|
t.Errorf("completeReading callback did not happen")
|
|
}
|
|
|
|
if !completeCallback {
|
|
t.Errorf("complete callback did not happen")
|
|
}
|
|
|
|
if completeCallbackNode == nil {
|
|
t.Errorf("no node returned for complete callback")
|
|
}
|
|
|
|
if completeCallbackNode != nil && !fnr.node.Equals(*completeCallbackNode) {
|
|
t.Errorf("different node returned for complete callback")
|
|
}
|
|
|
|
if completeCallbackStats != fnr.stats {
|
|
t.Errorf("different stats return for complete callback, want:\n %v\ngot:\n %v", fnr.stats, completeCallbackStats)
|
|
}
|
|
|
|
return fnr.node, fnr.stats
|
|
}
|
|
|
|
func TestArchiverSaveFile(t *testing.T) {
|
|
var tests = []TestFile{
|
|
{Content: ""},
|
|
{Content: "foo"},
|
|
{Content: string(rtest.Random(23, 12*1024*1024+1287898))},
|
|
}
|
|
|
|
for _, testfile := range tests {
|
|
t.Run("", func(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
|
|
tempdir, repo := prepareTempdirRepoSrc(t, TestDir{"file": testfile})
|
|
node, stats := saveFile(t, repo, filepath.Join(tempdir, "file"), fs.Track{FS: fs.Local{}})
|
|
|
|
TestEnsureFileContent(ctx, t, repo, "file", node, testfile)
|
|
if stats.DataSize != uint64(len(testfile.Content)) {
|
|
t.Errorf("wrong stats returned in DataSize, want %d, got %d", len(testfile.Content), stats.DataSize)
|
|
}
|
|
if stats.DataBlobs <= 0 && len(testfile.Content) > 0 {
|
|
t.Errorf("wrong stats returned in DataBlobs, want > 0, got %d", stats.DataBlobs)
|
|
}
|
|
if stats.TreeSize != 0 {
|
|
t.Errorf("wrong stats returned in TreeSize, want 0, got %d", stats.TreeSize)
|
|
}
|
|
if stats.TreeBlobs != 0 {
|
|
t.Errorf("wrong stats returned in DataBlobs, want 0, got %d", stats.DataBlobs)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestArchiverSaveFileReaderFS(t *testing.T) {
|
|
var tests = []struct {
|
|
Data string
|
|
}{
|
|
{Data: "foo"},
|
|
{Data: string(rtest.Random(23, 12*1024*1024+1287898))},
|
|
}
|
|
|
|
for _, test := range tests {
|
|
t.Run("", func(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
|
|
repo := repository.TestRepository(t)
|
|
|
|
ts := time.Now()
|
|
filename := "xx"
|
|
readerFs := &fs.Reader{
|
|
ModTime: ts,
|
|
Mode: 0123,
|
|
Name: filename,
|
|
ReadCloser: io.NopCloser(strings.NewReader(test.Data)),
|
|
}
|
|
|
|
node, stats := saveFile(t, repo, filename, readerFs)
|
|
|
|
TestEnsureFileContent(ctx, t, repo, "file", node, TestFile{Content: test.Data})
|
|
if stats.DataSize != uint64(len(test.Data)) {
|
|
t.Errorf("wrong stats returned in DataSize, want %d, got %d", len(test.Data), stats.DataSize)
|
|
}
|
|
if stats.DataBlobs <= 0 && len(test.Data) > 0 {
|
|
t.Errorf("wrong stats returned in DataBlobs, want > 0, got %d", stats.DataBlobs)
|
|
}
|
|
if stats.TreeSize != 0 {
|
|
t.Errorf("wrong stats returned in TreeSize, want 0, got %d", stats.TreeSize)
|
|
}
|
|
if stats.TreeBlobs != 0 {
|
|
t.Errorf("wrong stats returned in DataBlobs, want 0, got %d", stats.DataBlobs)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestArchiverSave(t *testing.T) {
|
|
var tests = []TestFile{
|
|
{Content: ""},
|
|
{Content: "foo"},
|
|
{Content: string(rtest.Random(23, 12*1024*1024+1287898))},
|
|
}
|
|
|
|
for _, testfile := range tests {
|
|
t.Run("", func(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
|
|
tempdir, repo := prepareTempdirRepoSrc(t, TestDir{"file": testfile})
|
|
|
|
wg, ctx := errgroup.WithContext(ctx)
|
|
repo.StartPackUploader(ctx, wg)
|
|
|
|
arch := New(repo, fs.Track{FS: fs.Local{}}, Options{})
|
|
arch.Error = func(item string, err error) error {
|
|
t.Errorf("archiver error for %v: %v", item, err)
|
|
return err
|
|
}
|
|
arch.runWorkers(ctx, wg)
|
|
arch.summary = &Summary{}
|
|
|
|
node, excluded, err := arch.save(ctx, "/", filepath.Join(tempdir, "file"), nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
if excluded {
|
|
t.Errorf("Save() excluded the node, that's unexpected")
|
|
}
|
|
|
|
fnr := node.take(ctx)
|
|
if fnr.err != nil {
|
|
t.Fatal(fnr.err)
|
|
}
|
|
|
|
if fnr.node == nil {
|
|
t.Fatalf("returned node is nil")
|
|
}
|
|
|
|
stats := fnr.stats
|
|
|
|
arch.stopWorkers()
|
|
err = repo.Flush(ctx)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
TestEnsureFileContent(ctx, t, repo, "file", fnr.node, testfile)
|
|
if stats.DataSize != uint64(len(testfile.Content)) {
|
|
t.Errorf("wrong stats returned in DataSize, want %d, got %d", len(testfile.Content), stats.DataSize)
|
|
}
|
|
if stats.DataBlobs <= 0 && len(testfile.Content) > 0 {
|
|
t.Errorf("wrong stats returned in DataBlobs, want > 0, got %d", stats.DataBlobs)
|
|
}
|
|
if stats.TreeSize != 0 {
|
|
t.Errorf("wrong stats returned in TreeSize, want 0, got %d", stats.TreeSize)
|
|
}
|
|
if stats.TreeBlobs != 0 {
|
|
t.Errorf("wrong stats returned in DataBlobs, want 0, got %d", stats.DataBlobs)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestArchiverSaveReaderFS(t *testing.T) {
|
|
var tests = []struct {
|
|
Data string
|
|
}{
|
|
{Data: "foo"},
|
|
{Data: string(rtest.Random(23, 12*1024*1024+1287898))},
|
|
}
|
|
|
|
for _, test := range tests {
|
|
t.Run("", func(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
|
|
repo := repository.TestRepository(t)
|
|
|
|
wg, ctx := errgroup.WithContext(ctx)
|
|
repo.StartPackUploader(ctx, wg)
|
|
|
|
ts := time.Now()
|
|
filename := "xx"
|
|
readerFs := &fs.Reader{
|
|
ModTime: ts,
|
|
Mode: 0123,
|
|
Name: filename,
|
|
ReadCloser: io.NopCloser(strings.NewReader(test.Data)),
|
|
}
|
|
|
|
arch := New(repo, readerFs, Options{})
|
|
arch.Error = func(item string, err error) error {
|
|
t.Errorf("archiver error for %v: %v", item, err)
|
|
return err
|
|
}
|
|
arch.runWorkers(ctx, wg)
|
|
arch.summary = &Summary{}
|
|
|
|
node, excluded, err := arch.save(ctx, "/", filename, nil)
|
|
t.Logf("Save returned %v %v", node, err)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
if excluded {
|
|
t.Errorf("Save() excluded the node, that's unexpected")
|
|
}
|
|
|
|
fnr := node.take(ctx)
|
|
if fnr.err != nil {
|
|
t.Fatal(fnr.err)
|
|
}
|
|
|
|
if fnr.node == nil {
|
|
t.Fatalf("returned node is nil")
|
|
}
|
|
|
|
stats := fnr.stats
|
|
|
|
arch.stopWorkers()
|
|
err = repo.Flush(ctx)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
TestEnsureFileContent(ctx, t, repo, "file", fnr.node, TestFile{Content: test.Data})
|
|
if stats.DataSize != uint64(len(test.Data)) {
|
|
t.Errorf("wrong stats returned in DataSize, want %d, got %d", len(test.Data), stats.DataSize)
|
|
}
|
|
if stats.DataBlobs <= 0 && len(test.Data) > 0 {
|
|
t.Errorf("wrong stats returned in DataBlobs, want > 0, got %d", stats.DataBlobs)
|
|
}
|
|
if stats.TreeSize != 0 {
|
|
t.Errorf("wrong stats returned in TreeSize, want 0, got %d", stats.TreeSize)
|
|
}
|
|
if stats.TreeBlobs != 0 {
|
|
t.Errorf("wrong stats returned in DataBlobs, want 0, got %d", stats.DataBlobs)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func BenchmarkArchiverSaveFileSmall(b *testing.B) {
|
|
const fileSize = 4 * 1024
|
|
d := TestDir{"file": TestFile{
|
|
Content: string(rtest.Random(23, fileSize)),
|
|
}}
|
|
|
|
b.SetBytes(fileSize)
|
|
|
|
for i := 0; i < b.N; i++ {
|
|
b.StopTimer()
|
|
tempdir, repo := prepareTempdirRepoSrc(b, d)
|
|
b.StartTimer()
|
|
|
|
_, stats := saveFile(b, repo, filepath.Join(tempdir, "file"), fs.Track{FS: fs.Local{}})
|
|
|
|
b.StopTimer()
|
|
if stats.DataSize != fileSize {
|
|
b.Errorf("wrong stats returned in DataSize, want %d, got %d", fileSize, stats.DataSize)
|
|
}
|
|
if stats.DataBlobs <= 0 {
|
|
b.Errorf("wrong stats returned in DataBlobs, want > 0, got %d", stats.DataBlobs)
|
|
}
|
|
if stats.TreeSize != 0 {
|
|
b.Errorf("wrong stats returned in TreeSize, want 0, got %d", stats.TreeSize)
|
|
}
|
|
if stats.TreeBlobs != 0 {
|
|
b.Errorf("wrong stats returned in DataBlobs, want 0, got %d", stats.DataBlobs)
|
|
}
|
|
b.StartTimer()
|
|
}
|
|
}
|
|
|
|
func BenchmarkArchiverSaveFileLarge(b *testing.B) {
|
|
const fileSize = 40*1024*1024 + 1287898
|
|
d := TestDir{"file": TestFile{
|
|
Content: string(rtest.Random(23, fileSize)),
|
|
}}
|
|
|
|
b.SetBytes(fileSize)
|
|
|
|
for i := 0; i < b.N; i++ {
|
|
b.StopTimer()
|
|
tempdir, repo := prepareTempdirRepoSrc(b, d)
|
|
b.StartTimer()
|
|
|
|
_, stats := saveFile(b, repo, filepath.Join(tempdir, "file"), fs.Track{FS: fs.Local{}})
|
|
|
|
b.StopTimer()
|
|
if stats.DataSize != fileSize {
|
|
b.Errorf("wrong stats returned in DataSize, want %d, got %d", fileSize, stats.DataSize)
|
|
}
|
|
if stats.DataBlobs <= 0 {
|
|
b.Errorf("wrong stats returned in DataBlobs, want > 0, got %d", stats.DataBlobs)
|
|
}
|
|
if stats.TreeSize != 0 {
|
|
b.Errorf("wrong stats returned in TreeSize, want 0, got %d", stats.TreeSize)
|
|
}
|
|
if stats.TreeBlobs != 0 {
|
|
b.Errorf("wrong stats returned in DataBlobs, want 0, got %d", stats.DataBlobs)
|
|
}
|
|
b.StartTimer()
|
|
}
|
|
}
|
|
|
|
type blobCountingRepo struct {
|
|
archiverRepo
|
|
|
|
m sync.Mutex
|
|
saved map[restic.BlobHandle]uint
|
|
}
|
|
|
|
func (repo *blobCountingRepo) SaveBlob(ctx context.Context, t restic.BlobType, buf []byte, id restic.ID, storeDuplicate bool) (restic.ID, bool, int, error) {
|
|
id, exists, size, err := repo.archiverRepo.SaveBlob(ctx, t, buf, id, storeDuplicate)
|
|
if exists {
|
|
return id, exists, size, err
|
|
}
|
|
h := restic.BlobHandle{ID: id, Type: t}
|
|
repo.m.Lock()
|
|
repo.saved[h]++
|
|
repo.m.Unlock()
|
|
return id, exists, size, err
|
|
}
|
|
|
|
func (repo *blobCountingRepo) SaveTree(ctx context.Context, t *restic.Tree) (restic.ID, error) {
|
|
id, err := restic.SaveTree(ctx, repo.archiverRepo, t)
|
|
h := restic.BlobHandle{ID: id, Type: restic.TreeBlob}
|
|
repo.m.Lock()
|
|
repo.saved[h]++
|
|
repo.m.Unlock()
|
|
return id, err
|
|
}
|
|
|
|
func appendToFile(t testing.TB, filename string, data []byte) {
|
|
f, err := os.OpenFile(filename, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
_, err = f.Write(data)
|
|
if err != nil {
|
|
_ = f.Close()
|
|
t.Fatal(err)
|
|
}
|
|
|
|
err = f.Close()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func TestArchiverSaveFileIncremental(t *testing.T) {
|
|
tempdir := rtest.TempDir(t)
|
|
|
|
repo := &blobCountingRepo{
|
|
archiverRepo: repository.TestRepository(t),
|
|
saved: make(map[restic.BlobHandle]uint),
|
|
}
|
|
|
|
data := rtest.Random(23, 512*1024+887898)
|
|
testfile := filepath.Join(tempdir, "testfile")
|
|
|
|
for i := 0; i < 3; i++ {
|
|
appendToFile(t, testfile, data)
|
|
node, _ := saveFile(t, repo, testfile, fs.Track{FS: fs.Local{}})
|
|
|
|
t.Logf("node blobs: %v", node.Content)
|
|
|
|
for h, n := range repo.saved {
|
|
if n > 1 {
|
|
t.Errorf("iteration %v: blob %v saved more than once (%d times)", i, h, n)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func save(t testing.TB, filename string, data []byte) {
|
|
f, err := os.Create(filename)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
_, err = f.Write(data)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
err = f.Sync()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
err = f.Close()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func chmodTwice(t testing.TB, name string) {
|
|
// POSIX says that ctime is updated "even if the file status does not
|
|
// change", but let's make sure it does change, just in case.
|
|
err := os.Chmod(name, 0700)
|
|
rtest.OK(t, err)
|
|
|
|
sleep()
|
|
|
|
err = os.Chmod(name, 0600)
|
|
rtest.OK(t, err)
|
|
}
|
|
|
|
func lstat(t testing.TB, name string) os.FileInfo {
|
|
fi, err := os.Lstat(name)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
return fi
|
|
}
|
|
|
|
func setTimestamp(t testing.TB, filename string, atime, mtime time.Time) {
|
|
var utimes = [...]syscall.Timespec{
|
|
syscall.NsecToTimespec(atime.UnixNano()),
|
|
syscall.NsecToTimespec(mtime.UnixNano()),
|
|
}
|
|
|
|
err := syscall.UtimesNano(filename, utimes[:])
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func remove(t testing.TB, filename string) {
|
|
err := os.Remove(filename)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func rename(t testing.TB, oldname, newname string) {
|
|
err := os.Rename(oldname, newname)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func nodeFromFI(t testing.TB, filename string, fi os.FileInfo) *restic.Node {
|
|
node, err := restic.NodeFromFileInfo(filename, fi, false)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
return node
|
|
}
|
|
|
|
// sleep sleeps long enough to ensure a timestamp change.
|
|
func sleep() {
|
|
d := 50 * time.Millisecond
|
|
if runtime.GOOS == "darwin" {
|
|
// On older Darwin instances, the file system only supports one second
|
|
// granularity.
|
|
d = 1500 * time.Millisecond
|
|
}
|
|
time.Sleep(d)
|
|
}
|
|
|
|
func TestFileChanged(t *testing.T) {
|
|
var defaultContent = []byte("foobar")
|
|
|
|
var tests = []struct {
|
|
Name string
|
|
SkipForWindows bool
|
|
Content []byte
|
|
Modify func(t testing.TB, filename string)
|
|
ChangeIgnore uint
|
|
SameFile bool
|
|
}{
|
|
{
|
|
Name: "same-content-new-file",
|
|
Modify: func(t testing.TB, filename string) {
|
|
remove(t, filename)
|
|
sleep()
|
|
save(t, filename, defaultContent)
|
|
},
|
|
},
|
|
{
|
|
Name: "same-content-new-timestamp",
|
|
Modify: func(t testing.TB, filename string) {
|
|
sleep()
|
|
save(t, filename, defaultContent)
|
|
},
|
|
},
|
|
{
|
|
Name: "new-content-same-timestamp",
|
|
// on Windows, there's no "create time" field users cannot modify,
|
|
// so we're unable to detect if a file has been modified when the
|
|
// timestamps are reset, so we skip this test for Windows
|
|
SkipForWindows: true,
|
|
Modify: func(t testing.TB, filename string) {
|
|
fi, err := os.Stat(filename)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
extFI := fs.ExtendedStat(fi)
|
|
save(t, filename, bytes.ToUpper(defaultContent))
|
|
sleep()
|
|
setTimestamp(t, filename, extFI.AccessTime, extFI.ModTime)
|
|
},
|
|
},
|
|
{
|
|
Name: "other-content",
|
|
Modify: func(t testing.TB, filename string) {
|
|
remove(t, filename)
|
|
sleep()
|
|
save(t, filename, []byte("xxxxxx"))
|
|
},
|
|
},
|
|
{
|
|
Name: "longer-content",
|
|
Modify: func(t testing.TB, filename string) {
|
|
save(t, filename, []byte("xxxxxxxxxxxxxxxxxxxxxx"))
|
|
},
|
|
},
|
|
{
|
|
Name: "new-file",
|
|
Modify: func(t testing.TB, filename string) {
|
|
remove(t, filename)
|
|
sleep()
|
|
save(t, filename, defaultContent)
|
|
},
|
|
},
|
|
{
|
|
Name: "ctime-change",
|
|
Modify: chmodTwice,
|
|
SameFile: false,
|
|
SkipForWindows: true, // No ctime on Windows, so this test would fail.
|
|
},
|
|
{
|
|
Name: "ignore-ctime-change",
|
|
Modify: chmodTwice,
|
|
ChangeIgnore: ChangeIgnoreCtime,
|
|
SameFile: true,
|
|
SkipForWindows: true, // No ctime on Windows, so this test is meaningless.
|
|
},
|
|
{
|
|
Name: "ignore-inode",
|
|
Modify: func(t testing.TB, filename string) {
|
|
fi := lstat(t, filename)
|
|
// First create the new file, then remove the old one,
|
|
// so that the old file retains its inode number.
|
|
tempname := filename + ".old"
|
|
rename(t, filename, tempname)
|
|
save(t, filename, defaultContent)
|
|
remove(t, tempname)
|
|
setTimestamp(t, filename, fi.ModTime(), fi.ModTime())
|
|
},
|
|
ChangeIgnore: ChangeIgnoreCtime | ChangeIgnoreInode,
|
|
SameFile: true,
|
|
},
|
|
}
|
|
|
|
for _, test := range tests {
|
|
t.Run(test.Name, func(t *testing.T) {
|
|
if runtime.GOOS == "windows" && test.SkipForWindows {
|
|
t.Skip("don't run test on Windows")
|
|
}
|
|
|
|
tempdir := rtest.TempDir(t)
|
|
|
|
filename := filepath.Join(tempdir, "file")
|
|
content := defaultContent
|
|
if test.Content != nil {
|
|
content = test.Content
|
|
}
|
|
save(t, filename, content)
|
|
|
|
fiBefore := lstat(t, filename)
|
|
node := nodeFromFI(t, filename, fiBefore)
|
|
|
|
if fileChanged(fiBefore, node, 0) {
|
|
t.Fatalf("unchanged file detected as changed")
|
|
}
|
|
|
|
test.Modify(t, filename)
|
|
|
|
fiAfter := lstat(t, filename)
|
|
|
|
if test.SameFile {
|
|
// file should be detected as unchanged
|
|
if fileChanged(fiAfter, node, test.ChangeIgnore) {
|
|
t.Fatalf("unmodified file detected as changed")
|
|
}
|
|
} else {
|
|
// file should be detected as changed
|
|
if !fileChanged(fiAfter, node, test.ChangeIgnore) && !test.SameFile {
|
|
t.Fatalf("modified file detected as unchanged")
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestFilChangedSpecialCases(t *testing.T) {
|
|
tempdir := rtest.TempDir(t)
|
|
|
|
filename := filepath.Join(tempdir, "file")
|
|
content := []byte("foobar")
|
|
save(t, filename, content)
|
|
|
|
t.Run("nil-node", func(t *testing.T) {
|
|
fi := lstat(t, filename)
|
|
if !fileChanged(fi, nil, 0) {
|
|
t.Fatal("nil node detected as unchanged")
|
|
}
|
|
})
|
|
|
|
t.Run("type-change", func(t *testing.T) {
|
|
fi := lstat(t, filename)
|
|
node := nodeFromFI(t, filename, fi)
|
|
node.Type = "symlink"
|
|
if !fileChanged(fi, node, 0) {
|
|
t.Fatal("node with changed type detected as unchanged")
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestArchiverSaveDir(t *testing.T) {
|
|
const targetNodeName = "targetdir"
|
|
|
|
var tests = []struct {
|
|
src TestDir
|
|
chdir string
|
|
target string
|
|
want TestDir
|
|
}{
|
|
{
|
|
src: TestDir{
|
|
"targetfile": TestFile{Content: string(rtest.Random(888, 2*1024*1024+5000))},
|
|
},
|
|
target: ".",
|
|
want: TestDir{
|
|
"targetdir": TestDir{
|
|
"targetfile": TestFile{Content: string(rtest.Random(888, 2*1024*1024+5000))},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
src: TestDir{
|
|
"targetdir": TestDir{
|
|
"foo": TestFile{Content: "foo"},
|
|
"emptyfile": TestFile{Content: ""},
|
|
"bar": TestFile{Content: "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"},
|
|
"largefile": TestFile{Content: string(rtest.Random(888, 2*1024*1024+5000))},
|
|
"largerfile": TestFile{Content: string(rtest.Random(234, 5*1024*1024+5000))},
|
|
},
|
|
},
|
|
target: "targetdir",
|
|
},
|
|
{
|
|
src: TestDir{
|
|
"foo": TestFile{Content: "foo"},
|
|
"emptyfile": TestFile{Content: ""},
|
|
"bar": TestFile{Content: "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"},
|
|
},
|
|
target: ".",
|
|
want: TestDir{
|
|
"targetdir": TestDir{
|
|
"foo": TestFile{Content: "foo"},
|
|
"emptyfile": TestFile{Content: ""},
|
|
"bar": TestFile{Content: "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
src: TestDir{
|
|
"foo": TestDir{
|
|
"subdir": TestDir{
|
|
"x": TestFile{Content: "xxx"},
|
|
"y": TestFile{Content: "yyyyyyyyyyyyyyyy"},
|
|
"z": TestFile{Content: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz"},
|
|
},
|
|
"file": TestFile{Content: "just a test"},
|
|
},
|
|
},
|
|
chdir: "foo/subdir",
|
|
target: "../../",
|
|
want: TestDir{
|
|
"targetdir": TestDir{
|
|
"foo": TestDir{
|
|
"subdir": TestDir{
|
|
"x": TestFile{Content: "xxx"},
|
|
"y": TestFile{Content: "yyyyyyyyyyyyyyyy"},
|
|
"z": TestFile{Content: "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz"},
|
|
},
|
|
"file": TestFile{Content: "just a test"},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
src: TestDir{
|
|
"foo": TestDir{
|
|
"file": TestFile{Content: "just a test"},
|
|
"file2": TestFile{Content: "again"},
|
|
},
|
|
},
|
|
target: "./foo",
|
|
want: TestDir{
|
|
"targetdir": TestDir{
|
|
"file": TestFile{Content: "just a test"},
|
|
"file2": TestFile{Content: "again"},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
for _, test := range tests {
|
|
t.Run("", func(t *testing.T) {
|
|
tempdir, repo := prepareTempdirRepoSrc(t, test.src)
|
|
|
|
wg, ctx := errgroup.WithContext(context.Background())
|
|
repo.StartPackUploader(ctx, wg)
|
|
|
|
arch := New(repo, fs.Track{FS: fs.Local{}}, Options{})
|
|
arch.runWorkers(ctx, wg)
|
|
arch.summary = &Summary{}
|
|
|
|
chdir := tempdir
|
|
if test.chdir != "" {
|
|
chdir = filepath.Join(chdir, test.chdir)
|
|
}
|
|
|
|
back := rtest.Chdir(t, chdir)
|
|
defer back()
|
|
|
|
fi, err := fs.Lstat(test.target)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
ft, err := arch.saveDir(ctx, "/", test.target, fi, nil, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
fnr := ft.take(ctx)
|
|
node, stats := fnr.node, fnr.stats
|
|
|
|
t.Logf("stats: %v", stats)
|
|
if stats.DataSize != 0 {
|
|
t.Errorf("wrong stats returned in DataSize, want 0, got %d", stats.DataSize)
|
|
}
|
|
if stats.DataBlobs != 0 {
|
|
t.Errorf("wrong stats returned in DataBlobs, want 0, got %d", stats.DataBlobs)
|
|
}
|
|
if stats.TreeSize == 0 {
|
|
t.Errorf("wrong stats returned in TreeSize, want > 0, got %d", stats.TreeSize)
|
|
}
|
|
if stats.TreeBlobs <= 0 {
|
|
t.Errorf("wrong stats returned in TreeBlobs, want > 0, got %d", stats.TreeBlobs)
|
|
}
|
|
|
|
node.Name = targetNodeName
|
|
tree := &restic.Tree{Nodes: []*restic.Node{node}}
|
|
treeID, err := restic.SaveTree(ctx, repo, tree)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
arch.stopWorkers()
|
|
|
|
err = repo.Flush(ctx)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
err = wg.Wait()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
want := test.want
|
|
if want == nil {
|
|
want = test.src
|
|
}
|
|
TestEnsureTree(context.TODO(), t, "/", repo, treeID, want)
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestArchiverSaveDirIncremental(t *testing.T) {
|
|
tempdir := rtest.TempDir(t)
|
|
|
|
repo := &blobCountingRepo{
|
|
archiverRepo: repository.TestRepository(t),
|
|
saved: make(map[restic.BlobHandle]uint),
|
|
}
|
|
|
|
appendToFile(t, filepath.Join(tempdir, "testfile"), []byte("foobar"))
|
|
|
|
// save the empty directory several times in a row, then have a look if the
|
|
// archiver did save the same tree several times
|
|
for i := 0; i < 5; i++ {
|
|
wg, ctx := errgroup.WithContext(context.TODO())
|
|
repo.StartPackUploader(ctx, wg)
|
|
|
|
arch := New(repo, fs.Track{FS: fs.Local{}}, Options{})
|
|
arch.runWorkers(ctx, wg)
|
|
arch.summary = &Summary{}
|
|
|
|
fi, err := fs.Lstat(tempdir)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
ft, err := arch.saveDir(ctx, "/", tempdir, fi, nil, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
fnr := ft.take(ctx)
|
|
node, stats := fnr.node, fnr.stats
|
|
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
if i == 0 {
|
|
// operation must have added new tree data
|
|
if stats.DataSize != 0 {
|
|
t.Errorf("wrong stats returned in DataSize, want 0, got %d", stats.DataSize)
|
|
}
|
|
if stats.DataBlobs != 0 {
|
|
t.Errorf("wrong stats returned in DataBlobs, want 0, got %d", stats.DataBlobs)
|
|
}
|
|
if stats.TreeSize == 0 {
|
|
t.Errorf("wrong stats returned in TreeSize, want > 0, got %d", stats.TreeSize)
|
|
}
|
|
if stats.TreeBlobs <= 0 {
|
|
t.Errorf("wrong stats returned in TreeBlobs, want > 0, got %d", stats.TreeBlobs)
|
|
}
|
|
} else {
|
|
// operation must not have added any new data
|
|
if stats.DataSize != 0 {
|
|
t.Errorf("wrong stats returned in DataSize, want 0, got %d", stats.DataSize)
|
|
}
|
|
if stats.DataBlobs != 0 {
|
|
t.Errorf("wrong stats returned in DataBlobs, want 0, got %d", stats.DataBlobs)
|
|
}
|
|
if stats.TreeSize != 0 {
|
|
t.Errorf("wrong stats returned in TreeSize, want 0, got %d", stats.TreeSize)
|
|
}
|
|
if stats.TreeBlobs != 0 {
|
|
t.Errorf("wrong stats returned in TreeBlobs, want 0, got %d", stats.TreeBlobs)
|
|
}
|
|
}
|
|
|
|
t.Logf("node subtree %v", node.Subtree)
|
|
|
|
arch.stopWorkers()
|
|
err = repo.Flush(ctx)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
err = wg.Wait()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
for h, n := range repo.saved {
|
|
if n > 1 {
|
|
t.Errorf("iteration %v: blob %v saved more than once (%d times)", i, h, n)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// bothZeroOrNeither fails the test if only one of exp, act is zero.
|
|
func bothZeroOrNeither(tb testing.TB, exp, act uint64) {
|
|
tb.Helper()
|
|
if (exp == 0 && act != 0) || (exp != 0 && act == 0) {
|
|
rtest.Equals(tb, exp, act)
|
|
}
|
|
}
|
|
|
|
func TestArchiverSaveTree(t *testing.T) {
|
|
symlink := func(from, to string) func(t testing.TB) {
|
|
return func(t testing.TB) {
|
|
err := os.Symlink(from, to)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// The toplevel directory is not counted in the ItemStats
|
|
var tests = []struct {
|
|
src TestDir
|
|
prepare func(t testing.TB)
|
|
targets []string
|
|
want TestDir
|
|
stat Summary
|
|
}{
|
|
{
|
|
src: TestDir{
|
|
"targetfile": TestFile{Content: string("foobar")},
|
|
},
|
|
targets: []string{"targetfile"},
|
|
want: TestDir{
|
|
"targetfile": TestFile{Content: string("foobar")},
|
|
},
|
|
stat: Summary{
|
|
ItemStats: ItemStats{1, 6, 32 + 6, 0, 0, 0},
|
|
ProcessedBytes: 6,
|
|
Files: ChangeStats{1, 0, 0},
|
|
Dirs: ChangeStats{0, 0, 0},
|
|
},
|
|
},
|
|
{
|
|
src: TestDir{
|
|
"targetfile": TestFile{Content: string("foobar")},
|
|
},
|
|
prepare: symlink("targetfile", "filesymlink"),
|
|
targets: []string{"targetfile", "filesymlink"},
|
|
want: TestDir{
|
|
"targetfile": TestFile{Content: string("foobar")},
|
|
"filesymlink": TestSymlink{Target: "targetfile"},
|
|
},
|
|
stat: Summary{
|
|
ItemStats: ItemStats{1, 6, 32 + 6, 0, 0, 0},
|
|
ProcessedBytes: 6,
|
|
Files: ChangeStats{1, 0, 0},
|
|
Dirs: ChangeStats{0, 0, 0},
|
|
},
|
|
},
|
|
{
|
|
src: TestDir{
|
|
"dir": TestDir{
|
|
"subdir": TestDir{
|
|
"subsubdir": TestDir{
|
|
"targetfile": TestFile{Content: string("foobar")},
|
|
},
|
|
},
|
|
"otherfile": TestFile{Content: string("xxx")},
|
|
},
|
|
},
|
|
prepare: symlink("subdir", filepath.FromSlash("dir/symlink")),
|
|
targets: []string{filepath.FromSlash("dir/symlink")},
|
|
want: TestDir{
|
|
"dir": TestDir{
|
|
"symlink": TestSymlink{Target: "subdir"},
|
|
},
|
|
},
|
|
stat: Summary{
|
|
ItemStats: ItemStats{0, 0, 0, 1, 0x154, 0x16a},
|
|
ProcessedBytes: 0,
|
|
Files: ChangeStats{0, 0, 0},
|
|
Dirs: ChangeStats{1, 0, 0},
|
|
},
|
|
},
|
|
{
|
|
src: TestDir{
|
|
"dir": TestDir{
|
|
"subdir": TestDir{
|
|
"subsubdir": TestDir{
|
|
"targetfile": TestFile{Content: string("foobar")},
|
|
},
|
|
},
|
|
"otherfile": TestFile{Content: string("xxx")},
|
|
},
|
|
},
|
|
prepare: symlink("subdir", filepath.FromSlash("dir/symlink")),
|
|
targets: []string{filepath.FromSlash("dir/symlink/subsubdir")},
|
|
want: TestDir{
|
|
"dir": TestDir{
|
|
"symlink": TestDir{
|
|
"subsubdir": TestDir{
|
|
"targetfile": TestFile{Content: string("foobar")},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
stat: Summary{
|
|
ItemStats: ItemStats{1, 6, 32 + 6, 3, 0x47f, 0x4c1},
|
|
ProcessedBytes: 6,
|
|
Files: ChangeStats{1, 0, 0},
|
|
Dirs: ChangeStats{3, 0, 0},
|
|
},
|
|
},
|
|
}
|
|
|
|
for _, test := range tests {
|
|
t.Run("", func(t *testing.T) {
|
|
tempdir, repo := prepareTempdirRepoSrc(t, test.src)
|
|
|
|
testFS := fs.Track{FS: fs.Local{}}
|
|
|
|
arch := New(repo, testFS, Options{})
|
|
|
|
wg, ctx := errgroup.WithContext(context.TODO())
|
|
repo.StartPackUploader(ctx, wg)
|
|
|
|
arch.runWorkers(ctx, wg)
|
|
arch.summary = &Summary{}
|
|
|
|
back := rtest.Chdir(t, tempdir)
|
|
defer back()
|
|
|
|
if test.prepare != nil {
|
|
test.prepare(t)
|
|
}
|
|
|
|
atree, err := NewTree(testFS, test.targets)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
fn, _, err := arch.saveTree(ctx, "/", atree, nil, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
fnr := fn.take(context.TODO())
|
|
if fnr.err != nil {
|
|
t.Fatal(fnr.err)
|
|
}
|
|
|
|
treeID := *fnr.node.Subtree
|
|
|
|
arch.stopWorkers()
|
|
err = repo.Flush(ctx)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
err = wg.Wait()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
want := test.want
|
|
if want == nil {
|
|
want = test.src
|
|
}
|
|
TestEnsureTree(context.TODO(), t, "/", repo, treeID, want)
|
|
stat := arch.summary
|
|
bothZeroOrNeither(t, uint64(test.stat.DataBlobs), uint64(stat.DataBlobs))
|
|
bothZeroOrNeither(t, uint64(test.stat.TreeBlobs), uint64(stat.TreeBlobs))
|
|
bothZeroOrNeither(t, test.stat.DataSize, stat.DataSize)
|
|
bothZeroOrNeither(t, test.stat.DataSizeInRepo, stat.DataSizeInRepo)
|
|
bothZeroOrNeither(t, test.stat.TreeSizeInRepo, stat.TreeSizeInRepo)
|
|
rtest.Equals(t, test.stat.ProcessedBytes, stat.ProcessedBytes)
|
|
rtest.Equals(t, test.stat.Files, stat.Files)
|
|
rtest.Equals(t, test.stat.Dirs, stat.Dirs)
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestArchiverSnapshot(t *testing.T) {
|
|
var tests = []struct {
|
|
name string
|
|
src TestDir
|
|
want TestDir
|
|
chdir string
|
|
targets []string
|
|
}{
|
|
{
|
|
name: "single-file",
|
|
src: TestDir{
|
|
"foo": TestFile{Content: "foo"},
|
|
},
|
|
targets: []string{"foo"},
|
|
},
|
|
{
|
|
name: "file-current-dir",
|
|
src: TestDir{
|
|
"foo": TestFile{Content: "foo"},
|
|
},
|
|
targets: []string{"./foo"},
|
|
},
|
|
{
|
|
name: "dir",
|
|
src: TestDir{
|
|
"target": TestDir{
|
|
"foo": TestFile{Content: "foo"},
|
|
},
|
|
},
|
|
targets: []string{"target"},
|
|
},
|
|
{
|
|
name: "dir-current-dir",
|
|
src: TestDir{
|
|
"target": TestDir{
|
|
"foo": TestFile{Content: "foo"},
|
|
},
|
|
},
|
|
targets: []string{"./target"},
|
|
},
|
|
{
|
|
name: "content-dir-current-dir",
|
|
src: TestDir{
|
|
"target": TestDir{
|
|
"foo": TestFile{Content: "foo"},
|
|
},
|
|
},
|
|
targets: []string{"./target/."},
|
|
},
|
|
{
|
|
name: "current-dir",
|
|
src: TestDir{
|
|
"target": TestDir{
|
|
"foo": TestFile{Content: "foo"},
|
|
},
|
|
},
|
|
targets: []string{"."},
|
|
},
|
|
{
|
|
name: "subdir",
|
|
src: TestDir{
|
|
"subdir": TestDir{
|
|
"foo": TestFile{Content: "foo"},
|
|
"subsubdir": TestDir{
|
|
"foo": TestFile{Content: "foo in subsubdir"},
|
|
},
|
|
},
|
|
"other": TestFile{Content: "another file"},
|
|
},
|
|
targets: []string{"subdir"},
|
|
want: TestDir{
|
|
"subdir": TestDir{
|
|
"foo": TestFile{Content: "foo"},
|
|
"subsubdir": TestDir{
|
|
"foo": TestFile{Content: "foo in subsubdir"},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
name: "subsubdir",
|
|
src: TestDir{
|
|
"subdir": TestDir{
|
|
"foo": TestFile{Content: "foo"},
|
|
"subsubdir": TestDir{
|
|
"foo": TestFile{Content: "foo in subsubdir"},
|
|
},
|
|
},
|
|
"other": TestFile{Content: "another file"},
|
|
},
|
|
targets: []string{"subdir/subsubdir"},
|
|
want: TestDir{
|
|
"subdir": TestDir{
|
|
"subsubdir": TestDir{
|
|
"foo": TestFile{Content: "foo in subsubdir"},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
name: "parent-dir",
|
|
src: TestDir{
|
|
"subdir": TestDir{
|
|
"foo": TestFile{Content: "foo"},
|
|
},
|
|
"other": TestFile{Content: "another file"},
|
|
},
|
|
chdir: "subdir",
|
|
targets: []string{".."},
|
|
},
|
|
{
|
|
name: "parent-parent-dir",
|
|
src: TestDir{
|
|
"subdir": TestDir{
|
|
"foo": TestFile{Content: "foo"},
|
|
"subsubdir": TestDir{
|
|
"empty": TestFile{Content: ""},
|
|
},
|
|
},
|
|
"other": TestFile{Content: "another file"},
|
|
},
|
|
chdir: "subdir/subsubdir",
|
|
targets: []string{"../.."},
|
|
},
|
|
{
|
|
name: "parent-parent-dir-slash",
|
|
src: TestDir{
|
|
"subdir": TestDir{
|
|
"subsubdir": TestDir{
|
|
"foo": TestFile{Content: "foo"},
|
|
},
|
|
},
|
|
"other": TestFile{Content: "another file"},
|
|
},
|
|
chdir: "subdir/subsubdir",
|
|
targets: []string{"../../"},
|
|
want: TestDir{
|
|
"subdir": TestDir{
|
|
"subsubdir": TestDir{
|
|
"foo": TestFile{Content: "foo"},
|
|
},
|
|
},
|
|
"other": TestFile{Content: "another file"},
|
|
},
|
|
},
|
|
{
|
|
name: "parent-subdir",
|
|
src: TestDir{
|
|
"subdir": TestDir{
|
|
"foo": TestFile{Content: "foo"},
|
|
},
|
|
"other": TestFile{Content: "another file"},
|
|
},
|
|
chdir: "subdir",
|
|
targets: []string{"../subdir"},
|
|
want: TestDir{
|
|
"subdir": TestDir{
|
|
"foo": TestFile{Content: "foo"},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
name: "parent-parent-dir-subdir",
|
|
src: TestDir{
|
|
"subdir": TestDir{
|
|
"subsubdir": TestDir{
|
|
"foo": TestFile{Content: "foo"},
|
|
},
|
|
},
|
|
"other": TestFile{Content: "another file"},
|
|
},
|
|
chdir: "subdir/subsubdir",
|
|
targets: []string{"../../subdir/subsubdir"},
|
|
want: TestDir{
|
|
"subdir": TestDir{
|
|
"subsubdir": TestDir{
|
|
"foo": TestFile{Content: "foo"},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
name: "included-multiple1",
|
|
src: TestDir{
|
|
"subdir": TestDir{
|
|
"subsubdir": TestDir{
|
|
"foo": TestFile{Content: "foo"},
|
|
},
|
|
"other": TestFile{Content: "another file"},
|
|
},
|
|
},
|
|
targets: []string{"subdir", "subdir/subsubdir"},
|
|
},
|
|
{
|
|
name: "included-multiple2",
|
|
src: TestDir{
|
|
"subdir": TestDir{
|
|
"subsubdir": TestDir{
|
|
"foo": TestFile{Content: "foo"},
|
|
},
|
|
"other": TestFile{Content: "another file"},
|
|
},
|
|
},
|
|
targets: []string{"subdir/subsubdir", "subdir"},
|
|
},
|
|
{
|
|
name: "collision",
|
|
src: TestDir{
|
|
"subdir": TestDir{
|
|
"foo": TestFile{Content: "foo in subdir"},
|
|
"subsubdir": TestDir{
|
|
"foo": TestFile{Content: "foo in subsubdir"},
|
|
},
|
|
},
|
|
"foo": TestFile{Content: "another file"},
|
|
},
|
|
chdir: "subdir",
|
|
targets: []string{".", "../foo"},
|
|
want: TestDir{
|
|
|
|
"foo": TestFile{Content: "foo in subdir"},
|
|
"subsubdir": TestDir{
|
|
"foo": TestFile{Content: "foo in subsubdir"},
|
|
},
|
|
"foo-1": TestFile{Content: "another file"},
|
|
},
|
|
},
|
|
}
|
|
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
|
|
tempdir, repo := prepareTempdirRepoSrc(t, test.src)
|
|
|
|
arch := New(repo, fs.Track{FS: fs.Local{}}, Options{})
|
|
|
|
chdir := tempdir
|
|
if test.chdir != "" {
|
|
chdir = filepath.Join(chdir, filepath.FromSlash(test.chdir))
|
|
}
|
|
|
|
back := rtest.Chdir(t, chdir)
|
|
defer back()
|
|
|
|
var targets []string
|
|
for _, target := range test.targets {
|
|
targets = append(targets, os.ExpandEnv(target))
|
|
}
|
|
|
|
t.Logf("targets: %v", targets)
|
|
sn, snapshotID, _, err := arch.Snapshot(ctx, targets, SnapshotOptions{Time: time.Now()})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
t.Logf("saved as %v", snapshotID.Str())
|
|
|
|
want := test.want
|
|
if want == nil {
|
|
want = test.src
|
|
}
|
|
TestEnsureSnapshot(t, repo, snapshotID, want)
|
|
|
|
checker.TestCheckRepo(t, repo, false)
|
|
|
|
// check that the snapshot contains the targets with absolute paths
|
|
for i, target := range sn.Paths {
|
|
atarget, err := filepath.Abs(test.targets[i])
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
if target != atarget {
|
|
t.Errorf("wrong path in snapshot: want %v, got %v", atarget, target)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestArchiverSnapshotSelect(t *testing.T) {
|
|
var tests = []struct {
|
|
name string
|
|
src TestDir
|
|
want TestDir
|
|
selFn SelectFunc
|
|
err string
|
|
}{
|
|
{
|
|
name: "include-all",
|
|
src: TestDir{
|
|
"work": TestDir{
|
|
"foo": TestFile{Content: "foo"},
|
|
"foo.txt": TestFile{Content: "foo text file"},
|
|
"subdir": TestDir{
|
|
"other": TestFile{Content: "other in subdir"},
|
|
"bar.txt": TestFile{Content: "bar.txt in subdir"},
|
|
},
|
|
},
|
|
"other": TestFile{Content: "another file"},
|
|
},
|
|
selFn: func(item string, fi os.FileInfo) bool {
|
|
return true
|
|
},
|
|
},
|
|
{
|
|
name: "exclude-all",
|
|
src: TestDir{
|
|
"work": TestDir{
|
|
"foo": TestFile{Content: "foo"},
|
|
"foo.txt": TestFile{Content: "foo text file"},
|
|
"subdir": TestDir{
|
|
"other": TestFile{Content: "other in subdir"},
|
|
"bar.txt": TestFile{Content: "bar.txt in subdir"},
|
|
},
|
|
},
|
|
"other": TestFile{Content: "another file"},
|
|
},
|
|
selFn: func(item string, fi os.FileInfo) bool {
|
|
return false
|
|
},
|
|
err: "snapshot is empty",
|
|
},
|
|
{
|
|
name: "exclude-txt-files",
|
|
src: TestDir{
|
|
"work": TestDir{
|
|
"foo": TestFile{Content: "foo"},
|
|
"foo.txt": TestFile{Content: "foo text file"},
|
|
"subdir": TestDir{
|
|
"other": TestFile{Content: "other in subdir"},
|
|
"bar.txt": TestFile{Content: "bar.txt in subdir"},
|
|
},
|
|
},
|
|
"other": TestFile{Content: "another file"},
|
|
},
|
|
want: TestDir{
|
|
"work": TestDir{
|
|
"foo": TestFile{Content: "foo"},
|
|
"subdir": TestDir{
|
|
"other": TestFile{Content: "other in subdir"},
|
|
},
|
|
},
|
|
"other": TestFile{Content: "another file"},
|
|
},
|
|
selFn: func(item string, fi os.FileInfo) bool {
|
|
return filepath.Ext(item) != ".txt"
|
|
},
|
|
},
|
|
{
|
|
name: "exclude-dir",
|
|
src: TestDir{
|
|
"work": TestDir{
|
|
"foo": TestFile{Content: "foo"},
|
|
"foo.txt": TestFile{Content: "foo text file"},
|
|
"subdir": TestDir{
|
|
"other": TestFile{Content: "other in subdir"},
|
|
"bar.txt": TestFile{Content: "bar.txt in subdir"},
|
|
},
|
|
},
|
|
"other": TestFile{Content: "another file"},
|
|
},
|
|
want: TestDir{
|
|
"work": TestDir{
|
|
"foo": TestFile{Content: "foo"},
|
|
"foo.txt": TestFile{Content: "foo text file"},
|
|
},
|
|
"other": TestFile{Content: "another file"},
|
|
},
|
|
selFn: func(item string, fi os.FileInfo) bool {
|
|
return filepath.Base(item) != "subdir"
|
|
},
|
|
},
|
|
{
|
|
name: "select-absolute-paths",
|
|
src: TestDir{
|
|
"foo": TestFile{Content: "foo"},
|
|
},
|
|
selFn: func(item string, fi os.FileInfo) bool {
|
|
return filepath.IsAbs(item)
|
|
},
|
|
},
|
|
}
|
|
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
|
|
tempdir, repo := prepareTempdirRepoSrc(t, test.src)
|
|
|
|
arch := New(repo, fs.Track{FS: fs.Local{}}, Options{})
|
|
arch.Select = test.selFn
|
|
|
|
back := rtest.Chdir(t, tempdir)
|
|
defer back()
|
|
|
|
targets := []string{"."}
|
|
_, snapshotID, _, err := arch.Snapshot(ctx, targets, SnapshotOptions{Time: time.Now()})
|
|
if test.err != "" {
|
|
if err == nil {
|
|
t.Fatalf("expected error not found, got %v, wanted %q", err, test.err)
|
|
}
|
|
|
|
if err.Error() != test.err {
|
|
t.Fatalf("unexpected error, want %q, got %q", test.err, err)
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
t.Logf("saved as %v", snapshotID.Str())
|
|
|
|
want := test.want
|
|
if want == nil {
|
|
want = test.src
|
|
}
|
|
TestEnsureSnapshot(t, repo, snapshotID, want)
|
|
|
|
checker.TestCheckRepo(t, repo, false)
|
|
})
|
|
}
|
|
}
|
|
|
|
// MockFS keeps track which files are read.
|
|
type MockFS struct {
|
|
fs.FS
|
|
|
|
m sync.Mutex
|
|
bytesRead map[string]int // tracks bytes read from all opened files
|
|
}
|
|
|
|
func (m *MockFS) Open(name string) (fs.File, error) {
|
|
f, err := m.FS.Open(name)
|
|
if err != nil {
|
|
return f, err
|
|
}
|
|
|
|
return MockFile{File: f, fs: m, filename: name}, nil
|
|
}
|
|
|
|
func (m *MockFS) OpenFile(name string, flag int, perm os.FileMode) (fs.File, error) {
|
|
f, err := m.FS.OpenFile(name, flag, perm)
|
|
if err != nil {
|
|
return f, err
|
|
}
|
|
|
|
return MockFile{File: f, fs: m, filename: name}, nil
|
|
}
|
|
|
|
type MockFile struct {
|
|
fs.File
|
|
filename string
|
|
|
|
fs *MockFS
|
|
}
|
|
|
|
func (f MockFile) Read(p []byte) (int, error) {
|
|
n, err := f.File.Read(p)
|
|
if n > 0 {
|
|
f.fs.m.Lock()
|
|
f.fs.bytesRead[f.filename] += n
|
|
f.fs.m.Unlock()
|
|
}
|
|
return n, err
|
|
}
|
|
|
|
func checkSnapshotStats(t *testing.T, sn *restic.Snapshot, stat Summary) {
|
|
rtest.Equals(t, stat.Files.New, sn.Summary.FilesNew)
|
|
rtest.Equals(t, stat.Files.Changed, sn.Summary.FilesChanged)
|
|
rtest.Equals(t, stat.Files.Unchanged, sn.Summary.FilesUnmodified)
|
|
rtest.Equals(t, stat.Dirs.New, sn.Summary.DirsNew)
|
|
rtest.Equals(t, stat.Dirs.Changed, sn.Summary.DirsChanged)
|
|
rtest.Equals(t, stat.Dirs.Unchanged, sn.Summary.DirsUnmodified)
|
|
rtest.Equals(t, stat.ProcessedBytes, sn.Summary.TotalBytesProcessed)
|
|
rtest.Equals(t, stat.Files.New+stat.Files.Changed+stat.Files.Unchanged, sn.Summary.TotalFilesProcessed)
|
|
bothZeroOrNeither(t, uint64(stat.DataBlobs), uint64(sn.Summary.DataBlobs))
|
|
bothZeroOrNeither(t, uint64(stat.TreeBlobs), uint64(sn.Summary.TreeBlobs))
|
|
bothZeroOrNeither(t, uint64(stat.DataSize+stat.TreeSize), uint64(sn.Summary.DataAdded))
|
|
bothZeroOrNeither(t, uint64(stat.DataSizeInRepo+stat.TreeSizeInRepo), uint64(sn.Summary.DataAddedPacked))
|
|
}
|
|
|
|
func TestArchiverParent(t *testing.T) {
|
|
var tests = []struct {
|
|
src TestDir
|
|
modify func(path string)
|
|
statInitial Summary
|
|
statSecond Summary
|
|
}{
|
|
{
|
|
src: TestDir{
|
|
"targetfile": TestFile{Content: string(rtest.Random(888, 2*1024*1024+5000))},
|
|
},
|
|
statInitial: Summary{
|
|
Files: ChangeStats{1, 0, 0},
|
|
Dirs: ChangeStats{0, 0, 0},
|
|
ProcessedBytes: 2102152,
|
|
ItemStats: ItemStats{3, 0x201593, 0x201632, 1, 0, 0},
|
|
},
|
|
statSecond: Summary{
|
|
Files: ChangeStats{0, 0, 1},
|
|
Dirs: ChangeStats{0, 0, 0},
|
|
ProcessedBytes: 2102152,
|
|
},
|
|
},
|
|
{
|
|
src: TestDir{
|
|
"targetDir": TestDir{
|
|
"targetfile": TestFile{Content: string(rtest.Random(888, 1234))},
|
|
"targetfile2": TestFile{Content: string(rtest.Random(888, 1235))},
|
|
},
|
|
},
|
|
statInitial: Summary{
|
|
Files: ChangeStats{2, 0, 0},
|
|
Dirs: ChangeStats{1, 0, 0},
|
|
ProcessedBytes: 2469,
|
|
ItemStats: ItemStats{2, 0xe1c, 0xcd9, 2, 0, 0},
|
|
},
|
|
statSecond: Summary{
|
|
Files: ChangeStats{0, 0, 2},
|
|
Dirs: ChangeStats{0, 0, 1},
|
|
ProcessedBytes: 2469,
|
|
},
|
|
},
|
|
{
|
|
src: TestDir{
|
|
"targetDir": TestDir{
|
|
"targetfile": TestFile{Content: string(rtest.Random(888, 1234))},
|
|
},
|
|
"targetfile2": TestFile{Content: string(rtest.Random(888, 1235))},
|
|
},
|
|
modify: func(path string) {
|
|
remove(t, filepath.Join(path, "targetDir", "targetfile"))
|
|
save(t, filepath.Join(path, "targetfile2"), []byte("foobar"))
|
|
},
|
|
statInitial: Summary{
|
|
Files: ChangeStats{2, 0, 0},
|
|
Dirs: ChangeStats{1, 0, 0},
|
|
ProcessedBytes: 2469,
|
|
ItemStats: ItemStats{2, 0xe13, 0xcf8, 2, 0, 0},
|
|
},
|
|
statSecond: Summary{
|
|
Files: ChangeStats{0, 1, 0},
|
|
Dirs: ChangeStats{0, 1, 0},
|
|
ProcessedBytes: 6,
|
|
ItemStats: ItemStats{1, 0x305, 0x233, 2, 0, 0},
|
|
},
|
|
},
|
|
}
|
|
|
|
for _, test := range tests {
|
|
t.Run("", func(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
|
|
tempdir, repo := prepareTempdirRepoSrc(t, test.src)
|
|
|
|
testFS := &MockFS{
|
|
FS: fs.Track{FS: fs.Local{}},
|
|
bytesRead: make(map[string]int),
|
|
}
|
|
|
|
arch := New(repo, testFS, Options{})
|
|
|
|
back := rtest.Chdir(t, tempdir)
|
|
defer back()
|
|
|
|
firstSnapshot, firstSnapshotID, summary, err := arch.Snapshot(ctx, []string{"."}, SnapshotOptions{Time: time.Now()})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
t.Logf("first backup saved as %v", firstSnapshotID.Str())
|
|
t.Logf("testfs: %v", testFS)
|
|
|
|
// check that all files have been read exactly once
|
|
TestWalkFiles(t, ".", test.src, func(filename string, item interface{}) error {
|
|
file, ok := item.(TestFile)
|
|
if !ok {
|
|
return nil
|
|
}
|
|
|
|
n, ok := testFS.bytesRead[filename]
|
|
if !ok {
|
|
t.Fatalf("file %v was not read at all", filename)
|
|
}
|
|
|
|
if n != len(file.Content) {
|
|
t.Fatalf("file %v: read %v bytes, wanted %v bytes", filename, n, len(file.Content))
|
|
}
|
|
return nil
|
|
})
|
|
rtest.Equals(t, test.statInitial.Files, summary.Files)
|
|
rtest.Equals(t, test.statInitial.Dirs, summary.Dirs)
|
|
rtest.Equals(t, test.statInitial.ProcessedBytes, summary.ProcessedBytes)
|
|
checkSnapshotStats(t, firstSnapshot, test.statInitial)
|
|
|
|
if test.modify != nil {
|
|
test.modify(tempdir)
|
|
}
|
|
|
|
opts := SnapshotOptions{
|
|
Time: time.Now(),
|
|
ParentSnapshot: firstSnapshot,
|
|
}
|
|
testFS.bytesRead = map[string]int{}
|
|
secondSnapshot, secondSnapshotID, summary, err := arch.Snapshot(ctx, []string{"."}, opts)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
if test.modify == nil {
|
|
// check that no files were read this time
|
|
rtest.Equals(t, map[string]int{}, testFS.bytesRead)
|
|
}
|
|
rtest.Equals(t, test.statSecond.Files, summary.Files)
|
|
rtest.Equals(t, test.statSecond.Dirs, summary.Dirs)
|
|
rtest.Equals(t, test.statSecond.ProcessedBytes, summary.ProcessedBytes)
|
|
checkSnapshotStats(t, secondSnapshot, test.statSecond)
|
|
|
|
t.Logf("second backup saved as %v", secondSnapshotID.Str())
|
|
t.Logf("testfs: %v", testFS)
|
|
|
|
checker.TestCheckRepo(t, repo, false)
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestArchiverErrorReporting(t *testing.T) {
|
|
ignoreErrorForBasename := func(basename string) ErrorFunc {
|
|
return func(item string, err error) error {
|
|
if filepath.Base(item) == "targetfile" {
|
|
t.Logf("ignoring error for targetfile: %v", err)
|
|
return nil
|
|
}
|
|
|
|
t.Errorf("error handler called for unexpected file %v: %v", item, err)
|
|
return err
|
|
}
|
|
}
|
|
|
|
chmodUnreadable := func(filename string) func(testing.TB) {
|
|
return func(t testing.TB) {
|
|
if runtime.GOOS == "windows" {
|
|
t.Skip("Skipping this test for windows")
|
|
}
|
|
|
|
err := os.Chmod(filepath.FromSlash(filename), 0004)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
}
|
|
|
|
var tests = []struct {
|
|
name string
|
|
src TestDir
|
|
want TestDir
|
|
prepare func(t testing.TB)
|
|
errFn ErrorFunc
|
|
mustError bool
|
|
}{
|
|
{
|
|
name: "no-error",
|
|
src: TestDir{
|
|
"targetfile": TestFile{Content: "foobar"},
|
|
},
|
|
},
|
|
{
|
|
name: "file-unreadable",
|
|
src: TestDir{
|
|
"targetfile": TestFile{Content: "foobar"},
|
|
},
|
|
prepare: chmodUnreadable("targetfile"),
|
|
mustError: true,
|
|
},
|
|
{
|
|
name: "file-unreadable-ignore-error",
|
|
src: TestDir{
|
|
"targetfile": TestFile{Content: "foobar"},
|
|
"other": TestFile{Content: "xxx"},
|
|
},
|
|
want: TestDir{
|
|
"other": TestFile{Content: "xxx"},
|
|
},
|
|
prepare: chmodUnreadable("targetfile"),
|
|
errFn: ignoreErrorForBasename("targetfile"),
|
|
},
|
|
{
|
|
name: "file-subdir-unreadable",
|
|
src: TestDir{
|
|
"subdir": TestDir{
|
|
"targetfile": TestFile{Content: "foobar"},
|
|
},
|
|
},
|
|
prepare: chmodUnreadable("subdir/targetfile"),
|
|
mustError: true,
|
|
},
|
|
{
|
|
name: "file-subdir-unreadable-ignore-error",
|
|
src: TestDir{
|
|
"subdir": TestDir{
|
|
"targetfile": TestFile{Content: "foobar"},
|
|
"other": TestFile{Content: "xxx"},
|
|
},
|
|
},
|
|
want: TestDir{
|
|
"subdir": TestDir{
|
|
"other": TestFile{Content: "xxx"},
|
|
},
|
|
},
|
|
prepare: chmodUnreadable("subdir/targetfile"),
|
|
errFn: ignoreErrorForBasename("targetfile"),
|
|
},
|
|
}
|
|
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
|
|
tempdir, repo := prepareTempdirRepoSrc(t, test.src)
|
|
|
|
back := rtest.Chdir(t, tempdir)
|
|
defer back()
|
|
|
|
if test.prepare != nil {
|
|
test.prepare(t)
|
|
}
|
|
|
|
arch := New(repo, fs.Track{FS: fs.Local{}}, Options{})
|
|
arch.Error = test.errFn
|
|
|
|
_, snapshotID, _, err := arch.Snapshot(ctx, []string{"."}, SnapshotOptions{Time: time.Now()})
|
|
if test.mustError {
|
|
if err != nil {
|
|
t.Logf("found expected error (%v), skipping further checks", err)
|
|
return
|
|
}
|
|
|
|
t.Fatalf("expected error not returned by archiver")
|
|
return
|
|
}
|
|
|
|
if err != nil {
|
|
t.Fatalf("unexpected error of type %T found: %v", err, err)
|
|
}
|
|
|
|
t.Logf("saved as %v", snapshotID.Str())
|
|
|
|
want := test.want
|
|
if want == nil {
|
|
want = test.src
|
|
}
|
|
TestEnsureSnapshot(t, repo, snapshotID, want)
|
|
|
|
checker.TestCheckRepo(t, repo, false)
|
|
})
|
|
}
|
|
}
|
|
|
|
type noCancelBackend struct {
|
|
backend.Backend
|
|
}
|
|
|
|
func (c *noCancelBackend) Remove(_ context.Context, h backend.Handle) error {
|
|
return c.Backend.Remove(context.Background(), h)
|
|
}
|
|
|
|
func (c *noCancelBackend) Save(_ context.Context, h backend.Handle, rd backend.RewindReader) error {
|
|
return c.Backend.Save(context.Background(), h, rd)
|
|
}
|
|
|
|
func (c *noCancelBackend) Load(_ context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
|
|
return c.Backend.Load(context.Background(), h, length, offset, fn)
|
|
}
|
|
|
|
func (c *noCancelBackend) Stat(_ context.Context, h backend.Handle) (backend.FileInfo, error) {
|
|
return c.Backend.Stat(context.Background(), h)
|
|
}
|
|
|
|
func (c *noCancelBackend) List(_ context.Context, t backend.FileType, fn func(backend.FileInfo) error) error {
|
|
return c.Backend.List(context.Background(), t, fn)
|
|
}
|
|
|
|
func (c *noCancelBackend) Delete(_ context.Context) error {
|
|
return c.Backend.Delete(context.Background())
|
|
}
|
|
|
|
func TestArchiverContextCanceled(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
cancel()
|
|
|
|
tempdir := rtest.TempDir(t)
|
|
TestCreateFiles(t, tempdir, TestDir{
|
|
"targetfile": TestFile{Content: "foobar"},
|
|
})
|
|
|
|
// Ensure that the archiver itself reports the canceled context and not just the backend
|
|
repo, _ := repository.TestRepositoryWithBackend(t, &noCancelBackend{mem.New()}, 0, repository.Options{})
|
|
|
|
back := rtest.Chdir(t, tempdir)
|
|
defer back()
|
|
|
|
arch := New(repo, fs.Track{FS: fs.Local{}}, Options{})
|
|
|
|
_, snapshotID, _, err := arch.Snapshot(ctx, []string{"."}, SnapshotOptions{Time: time.Now()})
|
|
|
|
if err != nil {
|
|
t.Logf("found expected error (%v)", err)
|
|
return
|
|
}
|
|
if snapshotID.IsNull() {
|
|
t.Fatalf("no error returned but found null id")
|
|
}
|
|
|
|
t.Fatalf("expected error not returned by archiver")
|
|
}
|
|
|
|
// TrackFS keeps track which files are opened. For some files, an error is injected.
|
|
type TrackFS struct {
|
|
fs.FS
|
|
|
|
errorOn map[string]error
|
|
|
|
opened map[string]uint
|
|
m sync.Mutex
|
|
}
|
|
|
|
func (m *TrackFS) Open(name string) (fs.File, error) {
|
|
m.m.Lock()
|
|
m.opened[name]++
|
|
m.m.Unlock()
|
|
|
|
return m.FS.Open(name)
|
|
}
|
|
|
|
func (m *TrackFS) OpenFile(name string, flag int, perm os.FileMode) (fs.File, error) {
|
|
m.m.Lock()
|
|
m.opened[name]++
|
|
m.m.Unlock()
|
|
|
|
return m.FS.OpenFile(name, flag, perm)
|
|
}
|
|
|
|
type failSaveRepo struct {
|
|
archiverRepo
|
|
failAfter int32
|
|
cnt int32
|
|
err error
|
|
}
|
|
|
|
func (f *failSaveRepo) SaveBlob(ctx context.Context, t restic.BlobType, buf []byte, id restic.ID, storeDuplicate bool) (restic.ID, bool, int, error) {
|
|
val := atomic.AddInt32(&f.cnt, 1)
|
|
if val >= f.failAfter {
|
|
return restic.Hash(buf), false, 0, f.err
|
|
}
|
|
|
|
return f.archiverRepo.SaveBlob(ctx, t, buf, id, storeDuplicate)
|
|
}
|
|
|
|
func TestArchiverAbortEarlyOnError(t *testing.T) {
|
|
var testErr = errors.New("test error")
|
|
|
|
var tests = []struct {
|
|
src TestDir
|
|
wantOpen map[string]uint
|
|
failAfter uint // error after so many blobs have been saved to the repo
|
|
err error
|
|
}{
|
|
{
|
|
src: TestDir{
|
|
"dir": TestDir{
|
|
"bar": TestFile{Content: "foobar"},
|
|
"baz": TestFile{Content: "foobar"},
|
|
"foo": TestFile{Content: "foobar"},
|
|
},
|
|
},
|
|
wantOpen: map[string]uint{
|
|
filepath.FromSlash("dir/bar"): 1,
|
|
filepath.FromSlash("dir/baz"): 1,
|
|
filepath.FromSlash("dir/foo"): 1,
|
|
},
|
|
},
|
|
{
|
|
src: TestDir{
|
|
"dir": TestDir{
|
|
"file0": TestFile{Content: string(rtest.Random(0, 1024))},
|
|
"file1": TestFile{Content: string(rtest.Random(1, 1024))},
|
|
"file2": TestFile{Content: string(rtest.Random(2, 1024))},
|
|
"file3": TestFile{Content: string(rtest.Random(3, 1024))},
|
|
"file4": TestFile{Content: string(rtest.Random(4, 1024))},
|
|
"file5": TestFile{Content: string(rtest.Random(5, 1024))},
|
|
"file6": TestFile{Content: string(rtest.Random(6, 1024))},
|
|
"file7": TestFile{Content: string(rtest.Random(7, 1024))},
|
|
"file8": TestFile{Content: string(rtest.Random(8, 1024))},
|
|
"file9": TestFile{Content: string(rtest.Random(9, 1024))},
|
|
},
|
|
},
|
|
wantOpen: map[string]uint{
|
|
filepath.FromSlash("dir/file0"): 1,
|
|
filepath.FromSlash("dir/file1"): 1,
|
|
filepath.FromSlash("dir/file2"): 1,
|
|
filepath.FromSlash("dir/file3"): 1,
|
|
filepath.FromSlash("dir/file8"): 0,
|
|
filepath.FromSlash("dir/file9"): 0,
|
|
},
|
|
// fails after four to seven files were opened, as the ReadConcurrency allows for
|
|
// two queued files and SaveBlobConcurrency for one blob queued for saving.
|
|
failAfter: 4,
|
|
err: testErr,
|
|
},
|
|
}
|
|
|
|
for _, test := range tests {
|
|
t.Run("", func(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
|
|
tempdir, repo := prepareTempdirRepoSrc(t, test.src)
|
|
|
|
back := rtest.Chdir(t, tempdir)
|
|
defer back()
|
|
|
|
testFS := &TrackFS{
|
|
FS: fs.Track{FS: fs.Local{}},
|
|
opened: make(map[string]uint),
|
|
}
|
|
|
|
if testFS.errorOn == nil {
|
|
testFS.errorOn = make(map[string]error)
|
|
}
|
|
|
|
testRepo := &failSaveRepo{
|
|
archiverRepo: repo,
|
|
failAfter: int32(test.failAfter),
|
|
err: test.err,
|
|
}
|
|
|
|
// at most two files may be queued
|
|
arch := New(testRepo, testFS, Options{
|
|
ReadConcurrency: 2,
|
|
SaveBlobConcurrency: 1,
|
|
})
|
|
|
|
_, _, _, err := arch.Snapshot(ctx, []string{"."}, SnapshotOptions{Time: time.Now()})
|
|
if !errors.Is(err, test.err) {
|
|
t.Errorf("expected error (%v) not found, got %v", test.err, err)
|
|
}
|
|
|
|
t.Logf("Snapshot return error: %v", err)
|
|
|
|
t.Logf("track fs: %v", testFS.opened)
|
|
|
|
for k, v := range test.wantOpen {
|
|
if testFS.opened[k] != v {
|
|
t.Errorf("opened %v %d times, want %d", k, testFS.opened[k], v)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func snapshot(t testing.TB, repo archiverRepo, fs fs.FS, parent *restic.Snapshot, filename string) (*restic.Snapshot, *restic.Node) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
|
|
arch := New(repo, fs, Options{})
|
|
|
|
sopts := SnapshotOptions{
|
|
Time: time.Now(),
|
|
ParentSnapshot: parent,
|
|
}
|
|
snapshot, _, _, err := arch.Snapshot(ctx, []string{filename}, sopts)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
tree, err := restic.LoadTree(ctx, repo, *snapshot.Tree)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
node := tree.Find(filename)
|
|
if node == nil {
|
|
t.Fatalf("unable to find node for testfile in snapshot")
|
|
}
|
|
|
|
return snapshot, node
|
|
}
|
|
|
|
// StatFS allows overwriting what is returned by the Lstat function.
|
|
type StatFS struct {
|
|
fs.FS
|
|
|
|
OverrideLstat map[string]os.FileInfo
|
|
OnlyOverrideStat bool
|
|
}
|
|
|
|
func (fs *StatFS) Lstat(name string) (os.FileInfo, error) {
|
|
if !fs.OnlyOverrideStat {
|
|
if fi, ok := fs.OverrideLstat[fixpath(name)]; ok {
|
|
return fi, nil
|
|
}
|
|
}
|
|
|
|
return fs.FS.Lstat(name)
|
|
}
|
|
|
|
func (fs *StatFS) OpenFile(name string, flags int, perm os.FileMode) (fs.File, error) {
|
|
if fi, ok := fs.OverrideLstat[fixpath(name)]; ok {
|
|
f, err := fs.FS.OpenFile(name, flags, perm)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
wrappedFile := fileStat{
|
|
File: f,
|
|
fi: fi,
|
|
}
|
|
return wrappedFile, nil
|
|
}
|
|
|
|
return fs.FS.OpenFile(name, flags, perm)
|
|
}
|
|
|
|
type fileStat struct {
|
|
fs.File
|
|
fi os.FileInfo
|
|
}
|
|
|
|
func (f fileStat) Stat() (os.FileInfo, error) {
|
|
return f.fi, nil
|
|
}
|
|
|
|
// used by wrapFileInfo, use untyped const in order to avoid having a version
|
|
// of wrapFileInfo for each OS
|
|
const (
|
|
mockFileInfoMode = 0400
|
|
mockFileInfoUID = 51234
|
|
mockFileInfoGID = 51235
|
|
)
|
|
|
|
func TestMetadataChanged(t *testing.T) {
|
|
defer feature.TestSetFlag(t, feature.Flag, feature.DeviceIDForHardlinks, true)()
|
|
|
|
files := TestDir{
|
|
"testfile": TestFile{
|
|
Content: "foo bar test file",
|
|
},
|
|
}
|
|
|
|
tempdir, repo := prepareTempdirRepoSrc(t, files)
|
|
|
|
back := rtest.Chdir(t, tempdir)
|
|
defer back()
|
|
|
|
// get metadata
|
|
fi := lstat(t, "testfile")
|
|
want, err := restic.NodeFromFileInfo("testfile", fi, false)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
fs := &StatFS{
|
|
FS: fs.Local{},
|
|
OverrideLstat: map[string]os.FileInfo{
|
|
"testfile": fi,
|
|
},
|
|
}
|
|
|
|
sn, node2 := snapshot(t, repo, fs, nil, "testfile")
|
|
|
|
// set some values so we can then compare the nodes
|
|
want.DeviceID = 0
|
|
want.Content = node2.Content
|
|
want.Path = ""
|
|
if len(want.ExtendedAttributes) == 0 {
|
|
want.ExtendedAttributes = nil
|
|
}
|
|
|
|
want.AccessTime = want.ModTime
|
|
|
|
// make sure that metadata was recorded successfully
|
|
if !cmp.Equal(want, node2) {
|
|
t.Fatalf("metadata does not match:\n%v", cmp.Diff(want, node2))
|
|
}
|
|
|
|
// modify the mode by wrapping it in a new struct, uses the consts defined above
|
|
fs.OverrideLstat["testfile"] = wrapFileInfo(fi)
|
|
|
|
// set the override values in the 'want' node which
|
|
want.Mode = 0400
|
|
// ignore UID and GID on Windows
|
|
if runtime.GOOS != "windows" {
|
|
want.UID = 51234
|
|
want.GID = 51235
|
|
}
|
|
// no user and group name
|
|
want.User = ""
|
|
want.Group = ""
|
|
|
|
// make another snapshot
|
|
_, node3 := snapshot(t, repo, fs, sn, "testfile")
|
|
// Override username and group to empty string - in case underlying system has user with UID 51234
|
|
// See https://github.com/restic/restic/issues/2372
|
|
node3.User = ""
|
|
node3.Group = ""
|
|
|
|
// make sure that metadata was recorded successfully
|
|
if !cmp.Equal(want, node3) {
|
|
t.Fatalf("metadata does not match:\n%v", cmp.Diff(want, node3))
|
|
}
|
|
|
|
// make sure the content matches
|
|
TestEnsureFileContent(context.Background(), t, repo, "testfile", node3, files["testfile"].(TestFile))
|
|
|
|
checker.TestCheckRepo(t, repo, false)
|
|
}
|
|
|
|
func TestRacyFileSwap(t *testing.T) {
|
|
files := TestDir{
|
|
"file": TestFile{
|
|
Content: "foo bar test file",
|
|
},
|
|
}
|
|
|
|
tempdir, repo := prepareTempdirRepoSrc(t, files)
|
|
|
|
back := rtest.Chdir(t, tempdir)
|
|
defer back()
|
|
|
|
// get metadata of current folder
|
|
fi := lstat(t, ".")
|
|
tempfile := filepath.Join(tempdir, "file")
|
|
|
|
statfs := &StatFS{
|
|
FS: fs.Local{},
|
|
OverrideLstat: map[string]os.FileInfo{
|
|
tempfile: fi,
|
|
},
|
|
OnlyOverrideStat: true,
|
|
}
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
|
|
wg, ctx := errgroup.WithContext(ctx)
|
|
repo.StartPackUploader(ctx, wg)
|
|
|
|
arch := New(repo, fs.Track{FS: statfs}, Options{})
|
|
arch.Error = func(item string, err error) error {
|
|
t.Logf("archiver error as expected for %v: %v", item, err)
|
|
return err
|
|
}
|
|
arch.runWorkers(ctx, wg)
|
|
|
|
// fs.Track will panic if the file was not closed
|
|
_, excluded, err := arch.save(ctx, "/", tempfile, nil)
|
|
if err == nil {
|
|
t.Errorf("Save() should have failed")
|
|
}
|
|
|
|
if excluded {
|
|
t.Errorf("Save() excluded the node, that's unexpected")
|
|
}
|
|
}
|
|
|
|
func TestMetadataBackupErrorFiltering(t *testing.T) {
|
|
tempdir := t.TempDir()
|
|
repo := repository.TestRepository(t)
|
|
|
|
filename := filepath.Join(tempdir, "file")
|
|
rtest.OK(t, os.WriteFile(filename, []byte("example"), 0o600))
|
|
fi, err := os.Stat(filename)
|
|
rtest.OK(t, err)
|
|
|
|
arch := New(repo, fs.Local{}, Options{})
|
|
|
|
var filteredErr error
|
|
replacementErr := fmt.Errorf("replacement")
|
|
arch.Error = func(item string, err error) error {
|
|
filteredErr = err
|
|
return replacementErr
|
|
}
|
|
|
|
// check that errors from reading extended metadata are properly filtered
|
|
node, err := arch.nodeFromFileInfo("file", filename+"invalid", fi, false)
|
|
rtest.Assert(t, node != nil, "node is missing")
|
|
rtest.Assert(t, err == replacementErr, "expected %v got %v", replacementErr, err)
|
|
rtest.Assert(t, filteredErr != nil, "missing inner error")
|
|
}
|