restore: support sparse restores also on windows

changelog/unreleased/issue-79

@@ -1,8 +1,8 @@
 Enhancement: Restore files with many zeros as sparse files
 
-On all platforms except Windows, the restorer may now write files containing
+When using `restore --sparse`, the restorer may now write files containing long
-long runs of zeros as sparse files (also called files with holes): the zeros
+runs of zeros as sparse files (also called files with holes): the zeros are not
-are not actually written to disk.
+actually written to disk.
 
 How much space is saved by writing sparse files depends on the operating
 system, file system and the distribution of zeros in the file.

cmd/restic/cmd_restore.go

@@ -1,7 +1,6 @@
 package main
 
 import (
-	"runtime"
 	"strings"
 	"time"
 
@@ -60,9 +59,7 @@ func init() {
 	flags.StringVarP(&restoreOptions.Target, "target", "t", "", "directory to extract data to")
 
 	initSingleSnapshotFilterOptions(flags, &restoreOptions.snapshotFilterOptions)
-	if runtime.GOOS != "windows" {
+	flags.BoolVar(&restoreOptions.Sparse, "sparse", false, "restore files as sparse")
-		flags.BoolVar(&restoreOptions.Sparse, "sparse", false, "restore files as sparse (not supported on windows)")
-	}
 	flags.BoolVar(&restoreOptions.Verify, "verify", false, "verify restored files content")
 }
 

internal/restorer/fileswriter.go

@@ -67,7 +67,7 @@ func (w *filesWriter) writeToFile(path string, blob []byte, offset int64, create
 
 		if createSize >= 0 {
 			if sparse {
-				err = f.Truncate(createSize)
+				err = truncateSparse(f, createSize)
 				if err != nil {
 					return nil, err
 				}

internal/restorer/restorer_test.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"io/ioutil"
+	"math"
 	"os"
 	"path/filepath"
 	"runtime"
@@ -11,6 +12,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/restic/restic/internal/archiver"
 	"github.com/restic/restic/internal/fs"
 	"github.com/restic/restic/internal/repository"
 	"github.com/restic/restic/internal/restic"
@@ -849,3 +851,58 @@ func TestVerifyCancel(t *testing.T) {
 	rtest.Equals(t, 1, len(errs))
 	rtest.Assert(t, strings.Contains(errs[0].Error(), "Invalid file size for"), "wrong error %q", errs[0].Error())
 }
+
+func TestRestorerSparseFiles(t *testing.T) {
+	repo, cleanup := repository.TestRepository(t)
+	defer cleanup()
+
+	var zeros [1<<20 + 13]byte
+
+	target := &fs.Reader{
+		Mode:       0600,
+		Name:       "/zeros",
+		ReadCloser: ioutil.NopCloser(bytes.NewReader(zeros[:])),
+	}
+	sc := archiver.NewScanner(target)
+	err := sc.Scan(context.TODO(), []string{"/zeros"})
+	rtest.OK(t, err)
+
+	arch := archiver.New(repo, target, archiver.Options{})
+	_, id, err := arch.Snapshot(context.Background(), []string{"/zeros"},
+		archiver.SnapshotOptions{})
+	rtest.OK(t, err)
+
+	res, err := NewRestorer(context.TODO(), repo, id, true)
+	rtest.OK(t, err)
+
+	tempdir, cleanup := rtest.TempDir(t)
+	defer cleanup()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	err = res.RestoreTo(ctx, tempdir)
+	rtest.OK(t, err)
+
+	filename := filepath.Join(tempdir, "zeros")
+	content, err := ioutil.ReadFile(filename)
+	rtest.OK(t, err)
+
+	rtest.Equals(t, len(zeros[:]), len(content))
+	rtest.Equals(t, zeros[:], content)
+
+	blocks := getBlockCount(t, filename)
+	if blocks < 0 {
+		return
+	}
+
+	// st.Blocks is the size in 512-byte blocks.
+	denseBlocks := math.Ceil(float64(len(zeros)) / 512)
+	sparsity := 1 - float64(blocks)/denseBlocks
+
+	// This should report 100% sparse. We don't assert that,
+	// as the behavior of sparse writes depends on the underlying
+	// file system as well as the OS.
+	t.Logf("wrote %d zeros as %d blocks, %.1f%% sparse",
+		len(zeros), blocks, 100*sparsity)
+}

internal/restorer/restorer_unix_test.go

@@ -4,17 +4,12 @@
 package restorer
 
 import (
-	"bytes"
 	"context"
-	"io/ioutil"
-	"math"
 	"os"
 	"path/filepath"
 	"syscall"
 	"testing"
 
-	"github.com/restic/restic/internal/archiver"
-	"github.com/restic/restic/internal/fs"
 	"github.com/restic/restic/internal/repository"
 	"github.com/restic/restic/internal/restic"
 	rtest "github.com/restic/restic/internal/test"
@@ -66,59 +61,12 @@ func TestRestorerRestoreEmptyHardlinkedFileds(t *testing.T) {
 	}
 }
 
-func TestRestorerSparseFiles(t *testing.T) {
+func getBlockCount(t *testing.T, filename string) int64 {
-	repo, cleanup := repository.TestRepository(t)
-	defer cleanup()
-
-	var zeros [1<<20 + 13]byte
-
-	target := &fs.Reader{
-		Mode:       0600,
-		Name:       "/zeros",
-		ReadCloser: ioutil.NopCloser(bytes.NewReader(zeros[:])),
-	}
-	sc := archiver.NewScanner(target)
-	err := sc.Scan(context.TODO(), []string{"/zeros"})
-	rtest.OK(t, err)
-
-	arch := archiver.New(repo, target, archiver.Options{})
-	_, id, err := arch.Snapshot(context.Background(), []string{"/zeros"},
-		archiver.SnapshotOptions{})
-	rtest.OK(t, err)
-
-	res, err := NewRestorer(context.TODO(), repo, id, true)
-	rtest.OK(t, err)
-
-	tempdir, cleanup := rtest.TempDir(t)
-	defer cleanup()
-
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-
-	err = res.RestoreTo(ctx, tempdir)
-	rtest.OK(t, err)
-
-	filename := filepath.Join(tempdir, "zeros")
-	content, err := ioutil.ReadFile(filename)
-	rtest.OK(t, err)
-
-	rtest.Equals(t, len(zeros[:]), len(content))
-	rtest.Equals(t, zeros[:], content)
-
 	fi, err := os.Stat(filename)
 	rtest.OK(t, err)
 	st := fi.Sys().(*syscall.Stat_t)
 	if st == nil {
-		return
+		return -1
 	}
-
+	return st.Blocks
-	// st.Blocks is the size in 512-byte blocks.
-	denseBlocks := math.Ceil(float64(len(zeros)) / 512)
-	sparsity := 1 - float64(st.Blocks)/denseBlocks
-
-	// This should report 100% sparse. We don't assert that,
-	// as the behavior of sparse writes depends on the underlying
-	// file system as well as the OS.
-	t.Logf("wrote %d zeros as %d blocks, %.1f%% sparse",
-		len(zeros), st.Blocks, 100*sparsity)
 }

internal/restorer/restorer_windows_test.go

@@ -0,0 +1,35 @@
+//go:build windows
+// +build windows
+
+package restorer
+
+import (
+	"math"
+	"syscall"
+	"testing"
+	"unsafe"
+
+	rtest "github.com/restic/restic/internal/test"
+	"golang.org/x/sys/windows"
+)
+
+func getBlockCount(t *testing.T, filename string) int64 {
+	libkernel32 := windows.NewLazySystemDLL("kernel32.dll")
+	err := libkernel32.Load()
+	rtest.OK(t, err)
+	proc := libkernel32.NewProc("GetCompressedFileSizeW")
+	err = proc.Find()
+	rtest.OK(t, err)
+
+	namePtr, err := syscall.UTF16PtrFromString(filename)
+	rtest.OK(t, err)
+
+	result, _, _ := proc.Call(uintptr(unsafe.Pointer(namePtr)), 0)
+
+	const invalidFileSize = uintptr(4294967295)
+	if result == invalidFileSize {
+		return -1
+	}
+
+	return int64(math.Ceil(float64(result) / 512))
+}

internal/restorer/truncate_other.go

@@ -0,0 +1,10 @@
+//go:build !windows
+// +build !windows
+
+package restorer
+
+import "os"
+
+func truncateSparse(f *os.File, size int64) error {
+	return f.Truncate(size)
+}

internal/restorer/truncate_windows.go

@@ -0,0 +1,19 @@
+package restorer
+
+import (
+	"os"
+
+	"github.com/restic/restic/internal/debug"
+	"golang.org/x/sys/windows"
+)
+
+func truncateSparse(f *os.File, size int64) error {
+	// try setting the sparse file attribute, but ignore the error if it fails
+	var t uint32
+	err := windows.DeviceIoControl(windows.Handle(f.Fd()), windows.FSCTL_SET_SPARSE, nil, 0, nil, 0, &t, nil)
+	if err != nil {
+		debug.Log("failed to set sparse attribute for %v: %v", f.Name(), err)
+	}
+
+	return f.Truncate(size)
+}