diff --git a/changelog/unreleased/issue-79 b/changelog/unreleased/issue-79 index c5f8ffaed..b69ea519c 100644 --- a/changelog/unreleased/issue-79 +++ b/changelog/unreleased/issue-79 @@ -1,8 +1,8 @@ Enhancement: Restore files with many zeros as sparse files -On all platforms except Windows, the restorer may now write files containing -long runs of zeros as sparse files (also called files with holes): the zeros -are not actually written to disk. +When using `restore --sparse`, the restorer may now write files containing long +runs of zeros as sparse files (also called files with holes): the zeros are not +actually written to disk. How much space is saved by writing sparse files depends on the operating system, file system and the distribution of zeros in the file. diff --git a/cmd/restic/cmd_restore.go b/cmd/restic/cmd_restore.go index faf8c851a..1da8407a4 100644 --- a/cmd/restic/cmd_restore.go +++ b/cmd/restic/cmd_restore.go @@ -1,7 +1,6 @@ package main import ( - "runtime" "strings" "time" @@ -60,9 +59,7 @@ func init() { flags.StringVarP(&restoreOptions.Target, "target", "t", "", "directory to extract data to") initSingleSnapshotFilterOptions(flags, &restoreOptions.snapshotFilterOptions) - if runtime.GOOS != "windows" { - flags.BoolVar(&restoreOptions.Sparse, "sparse", false, "restore files as sparse (not supported on windows)") - } + flags.BoolVar(&restoreOptions.Sparse, "sparse", false, "restore files as sparse") flags.BoolVar(&restoreOptions.Verify, "verify", false, "verify restored files content") } diff --git a/internal/restorer/fileswriter.go b/internal/restorer/fileswriter.go index 47fb5572c..0a26101f4 100644 --- a/internal/restorer/fileswriter.go +++ b/internal/restorer/fileswriter.go @@ -67,7 +67,7 @@ func (w *filesWriter) writeToFile(path string, blob []byte, offset int64, create if createSize >= 0 { if sparse { - err = f.Truncate(createSize) + err = truncateSparse(f, createSize) if err != nil { return nil, err } diff --git a/internal/restorer/restorer_test.go b/internal/restorer/restorer_test.go index 7113ca100..f57868b4f 100644 --- a/internal/restorer/restorer_test.go +++ b/internal/restorer/restorer_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "io/ioutil" + "math" "os" "path/filepath" "runtime" @@ -11,6 +12,7 @@ import ( "testing" "time" + "github.com/restic/restic/internal/archiver" "github.com/restic/restic/internal/fs" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" @@ -849,3 +851,58 @@ func TestVerifyCancel(t *testing.T) { rtest.Equals(t, 1, len(errs)) rtest.Assert(t, strings.Contains(errs[0].Error(), "Invalid file size for"), "wrong error %q", errs[0].Error()) } + +func TestRestorerSparseFiles(t *testing.T) { + repo, cleanup := repository.TestRepository(t) + defer cleanup() + + var zeros [1<<20 + 13]byte + + target := &fs.Reader{ + Mode: 0600, + Name: "/zeros", + ReadCloser: ioutil.NopCloser(bytes.NewReader(zeros[:])), + } + sc := archiver.NewScanner(target) + err := sc.Scan(context.TODO(), []string{"/zeros"}) + rtest.OK(t, err) + + arch := archiver.New(repo, target, archiver.Options{}) + _, id, err := arch.Snapshot(context.Background(), []string{"/zeros"}, + archiver.SnapshotOptions{}) + rtest.OK(t, err) + + res, err := NewRestorer(context.TODO(), repo, id, true) + rtest.OK(t, err) + + tempdir, cleanup := rtest.TempDir(t) + defer cleanup() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + err = res.RestoreTo(ctx, tempdir) + rtest.OK(t, err) + + filename := filepath.Join(tempdir, "zeros") + content, err := ioutil.ReadFile(filename) + rtest.OK(t, err) + + rtest.Equals(t, len(zeros[:]), len(content)) + rtest.Equals(t, zeros[:], content) + + blocks := getBlockCount(t, filename) + if blocks < 0 { + return + } + + // st.Blocks is the size in 512-byte blocks. + denseBlocks := math.Ceil(float64(len(zeros)) / 512) + sparsity := 1 - float64(blocks)/denseBlocks + + // This should report 100% sparse. We don't assert that, + // as the behavior of sparse writes depends on the underlying + // file system as well as the OS. + t.Logf("wrote %d zeros as %d blocks, %.1f%% sparse", + len(zeros), blocks, 100*sparsity) +} diff --git a/internal/restorer/restorer_unix_test.go b/internal/restorer/restorer_unix_test.go index dc8d6adeb..76f86c60b 100644 --- a/internal/restorer/restorer_unix_test.go +++ b/internal/restorer/restorer_unix_test.go @@ -4,17 +4,12 @@ package restorer import ( - "bytes" "context" - "io/ioutil" - "math" "os" "path/filepath" "syscall" "testing" - "github.com/restic/restic/internal/archiver" - "github.com/restic/restic/internal/fs" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" @@ -66,59 +61,12 @@ func TestRestorerRestoreEmptyHardlinkedFileds(t *testing.T) { } } -func TestRestorerSparseFiles(t *testing.T) { - repo, cleanup := repository.TestRepository(t) - defer cleanup() - - var zeros [1<<20 + 13]byte - - target := &fs.Reader{ - Mode: 0600, - Name: "/zeros", - ReadCloser: ioutil.NopCloser(bytes.NewReader(zeros[:])), - } - sc := archiver.NewScanner(target) - err := sc.Scan(context.TODO(), []string{"/zeros"}) - rtest.OK(t, err) - - arch := archiver.New(repo, target, archiver.Options{}) - _, id, err := arch.Snapshot(context.Background(), []string{"/zeros"}, - archiver.SnapshotOptions{}) - rtest.OK(t, err) - - res, err := NewRestorer(context.TODO(), repo, id, true) - rtest.OK(t, err) - - tempdir, cleanup := rtest.TempDir(t) - defer cleanup() - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - err = res.RestoreTo(ctx, tempdir) - rtest.OK(t, err) - - filename := filepath.Join(tempdir, "zeros") - content, err := ioutil.ReadFile(filename) - rtest.OK(t, err) - - rtest.Equals(t, len(zeros[:]), len(content)) - rtest.Equals(t, zeros[:], content) - +func getBlockCount(t *testing.T, filename string) int64 { fi, err := os.Stat(filename) rtest.OK(t, err) st := fi.Sys().(*syscall.Stat_t) if st == nil { - return + return -1 } - - // st.Blocks is the size in 512-byte blocks. - denseBlocks := math.Ceil(float64(len(zeros)) / 512) - sparsity := 1 - float64(st.Blocks)/denseBlocks - - // This should report 100% sparse. We don't assert that, - // as the behavior of sparse writes depends on the underlying - // file system as well as the OS. - t.Logf("wrote %d zeros as %d blocks, %.1f%% sparse", - len(zeros), st.Blocks, 100*sparsity) + return st.Blocks } diff --git a/internal/restorer/restorer_windows_test.go b/internal/restorer/restorer_windows_test.go new file mode 100644 index 000000000..3ec4b1f11 --- /dev/null +++ b/internal/restorer/restorer_windows_test.go @@ -0,0 +1,35 @@ +//go:build windows +// +build windows + +package restorer + +import ( + "math" + "syscall" + "testing" + "unsafe" + + rtest "github.com/restic/restic/internal/test" + "golang.org/x/sys/windows" +) + +func getBlockCount(t *testing.T, filename string) int64 { + libkernel32 := windows.NewLazySystemDLL("kernel32.dll") + err := libkernel32.Load() + rtest.OK(t, err) + proc := libkernel32.NewProc("GetCompressedFileSizeW") + err = proc.Find() + rtest.OK(t, err) + + namePtr, err := syscall.UTF16PtrFromString(filename) + rtest.OK(t, err) + + result, _, _ := proc.Call(uintptr(unsafe.Pointer(namePtr)), 0) + + const invalidFileSize = uintptr(4294967295) + if result == invalidFileSize { + return -1 + } + + return int64(math.Ceil(float64(result) / 512)) +} diff --git a/internal/restorer/truncate_other.go b/internal/restorer/truncate_other.go new file mode 100644 index 000000000..ed7ab04c5 --- /dev/null +++ b/internal/restorer/truncate_other.go @@ -0,0 +1,10 @@ +//go:build !windows +// +build !windows + +package restorer + +import "os" + +func truncateSparse(f *os.File, size int64) error { + return f.Truncate(size) +} diff --git a/internal/restorer/truncate_windows.go b/internal/restorer/truncate_windows.go new file mode 100644 index 000000000..831a117d1 --- /dev/null +++ b/internal/restorer/truncate_windows.go @@ -0,0 +1,19 @@ +package restorer + +import ( + "os" + + "github.com/restic/restic/internal/debug" + "golang.org/x/sys/windows" +) + +func truncateSparse(f *os.File, size int64) error { + // try setting the sparse file attribute, but ignore the error if it fails + var t uint32 + err := windows.DeviceIoControl(windows.Handle(f.Fd()), windows.FSCTL_SET_SPARSE, nil, 0, nil, 0, &t, nil) + if err != nil { + debug.Log("failed to set sparse attribute for %v: %v", f.Name(), err) + } + + return f.Truncate(size) +}