octoleo/restic
2
2
Fork 0
mirror of https://github.com/octoleo/restic.git synced 2024-11-23 05:12:10 +00:00
Code Issues Releases Wiki Activity

Merge pull request #4837 from MichaelEischer/restore-options

Browse Source 
Make restore overwrite behavior configurable
This commit is contained in:
Michael Eischer 2024-06-12 22:52:55 +02:00 committed by GitHub
commit 663151db57
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
17 changed files with 486 additions and 228 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
changelog/unreleased/issue-4817 Normal file
View File

@ -0,0 +1,11 @@
Enhancement: Make overwrite behavior of `restore` customizable
The `restore` command now supports an `--overwrite` option to configure whether
already existing files are overwritten. The default is `--overwrite always`,
which overwrites existing files. `--overwrite if-newer` only restores files
from the snapshot that are newer than the local state. And `--overwrite never`
does not modify existing files.
https://github.com/restic/restic/issues/4817
https://github.com/restic/restic/issues/200
https://github.com/restic/restic/pull/4837

12
cmd/restic/cmd_restore.go
View File

@ -47,8 +47,9 @@ type RestoreOptions struct {
includePatternOptions
Target string
restic.SnapshotFilter
Sparse bool
Verify bool
Sparse bool
Verify bool
Overwrite restorer.OverwriteBehavior
}
var restoreOptions RestoreOptions
@ -65,6 +66,7 @@ func init() {
initSingleSnapshotFilter(flags, &restoreOptions.SnapshotFilter)
flags.BoolVar(&restoreOptions.Sparse, "sparse", false, "restore files as sparse")
flags.BoolVar(&restoreOptions.Verify, "verify", false, "verify restored files content")
flags.Var(&restoreOptions.Overwrite, "overwrite", "overwrite behavior, one of (always|if-newer|never) (default: always)")
}
func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions,
@ -137,7 +139,11 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions,
}
progress := restoreui.NewProgress(printer, calculateProgressInterval(!gopts.Quiet, gopts.JSON))
res := restorer.NewRestorer(repo, sn, opts.Sparse, progress)
res := restorer.NewRestorer(repo, sn, restorer.Options{
Sparse: opts.Sparse,
Progress: progress,
Overwrite: opts.Overwrite,
})
totalErrors := 0
res.Error = func(location string, err error) error {

9
doc/050_restore.rst
View File

@ -88,6 +88,15 @@ disk space. Note that the exact location of the holes can differ from those in
the original file, as their location is determined while restoring and is not
stored explicitly.
Restoring in-place
------------------
By default, the ``restore`` command overwrites already existing files in the target
directory. This behavior can be configured via the ``--overwrite`` option. The
default is ``--overwrite always``. To only overwrite existing files if the file in
the snapshot is newer, use ``--overwrite if-newer``. To never overwrite existing files,
use ``--overwrite never``.
Restore using mount
===================

9
doc/075_scripting.rst
View File

@ -502,11 +502,14 @@ Status
+----------------------+------------------------------------------------------------+
|``files_restored`` | Files restored |
+----------------------+------------------------------------------------------------+
|``files_skipped`` | Files skipped due to overwrite setting |
+----------------------+------------------------------------------------------------+
|``total_bytes`` | Total number of bytes in restore set |
+----------------------+------------------------------------------------------------+
|``bytes_restored`` | Number of bytes restored |
+----------------------+------------------------------------------------------------+
|``bytes_skipped`` | Total size of skipped files |
+----------------------+------------------------------------------------------------+
Summary
^^^^^^^
@ -520,10 +523,14 @@ Summary
+----------------------+------------------------------------------------------------+
|``files_restored`` | Files restored |
+----------------------+------------------------------------------------------------+
|``files_skipped`` | Files skipped due to overwrite setting |
+----------------------+------------------------------------------------------------+
|``total_bytes`` | Total number of bytes in restore set |
+----------------------+------------------------------------------------------------+
|``bytes_restored`` | Number of bytes restored |
+----------------------+------------------------------------------------------------+
|``bytes_skipped`` | Total size of skipped files |
+----------------------+------------------------------------------------------------+
snapshots

59
internal/restorer/filerestorer.go
View File

@ -14,11 +14,6 @@ import (
"github.com/restic/restic/internal/ui/restore"
)
// TODO if a blob is corrupt, there may be good blob copies in other packs
// TODO evaluate if it makes sense to split download and processing workers
// pro: can (slowly) read network and decrypt/write files concurrently
// con: each worker needs to keep one pack in memory
const (
largeFileBlobCount = 25
)
@ -120,6 +115,13 @@ func (r *fileRestorer) restoreFiles(ctx context.Context) error {
// create packInfo from fileInfo
for _, file := range r.files {
fileBlobs := file.blobs.(restic.IDs)
if len(fileBlobs) == 0 {
err := r.restoreEmptyFileAt(file.location)
if errFile := r.sanitizeError(file, err); errFile != nil {
return errFile
}
}
largeFile := len(fileBlobs) > largeFileBlobCount
var packsMap map[restic.ID][]fileBlobInfo
if largeFile {
@ -159,6 +161,8 @@ func (r *fileRestorer) restoreFiles(ctx context.Context) error {
file.blobs = packsMap
}
}
// drop no longer necessary file list
r.files = nil
wg, ctx := errgroup.WithContext(ctx)
downloadCh := make(chan *packInfo)
@ -195,6 +199,19 @@ func (r *fileRestorer) restoreFiles(ctx context.Context) error {
return wg.Wait()
}
func (r *fileRestorer) restoreEmptyFileAt(location string) error {
f, err := createFile(r.targetPath(location), 0, false)
if err != nil {
return err
}
if err = f.Close(); err != nil {
return err
}
r.progress.AddProgress(location, 0, 0)
return nil
}
type blobToFileOffsetsMapping map[restic.ID]struct {
files map[*fileInfo][]int64 // file -> offsets (plural!) of the blob in the file
blob restic.Blob
@ -240,32 +257,6 @@ func (r *fileRestorer) downloadPack(ctx context.Context, pack *packInfo) error {
// track already processed blobs for precise error reporting
processedBlobs := restic.NewBlobSet()
for _, entry := range blobs {
occurrences := 0
for _, offsets := range entry.files {
occurrences += len(offsets)
}
// With a maximum blob size of 8MB, the normal blob streaming has to write
// at most 800MB for a single blob. This should be short enough to avoid
// network connection timeouts. Based on a quick test, a limit of 100 only
// selects a very small number of blobs (the number of references per blob
// - aka. `count` - seem to follow a expontential distribution)
if occurrences > 100 {
// process frequently referenced blobs first as these can take a long time to write
// which can cause backend connections to time out
delete(blobs, entry.blob.ID)
partialBlobs := blobToFileOffsetsMapping{entry.blob.ID: entry}
err := r.downloadBlobs(ctx, pack.id, partialBlobs, processedBlobs)
if err := r.reportError(blobs, processedBlobs, err); err != nil {
return err
}
}
}
if len(blobs) == 0 {
return nil
}
err := r.downloadBlobs(ctx, pack.id, blobs, processedBlobs)
return r.reportError(blobs, processedBlobs, err)
}
@ -339,11 +330,7 @@ func (r *fileRestorer) downloadBlobs(ctx context.Context, packID restic.ID,
createSize = file.size
}
writeErr := r.filesWriter.writeToFile(r.targetPath(file.location), blobData, offset, createSize, file.sparse)
if r.progress != nil {
r.progress.AddProgress(file.location, uint64(len(blobData)), uint64(file.size))
}
r.progress.AddProgress(file.location, uint64(len(blobData)), uint64(file.size))
return writeErr
}
err := r.sanitizeError(file, writeToFile())

4
internal/restorer/filerestorer_test.go
View File

@ -206,6 +206,10 @@ func TestFileRestorerBasic(t *testing.T) {
{"data3-1", "pack3-1"},
},
},
{
name: "empty",
blobs: []TestBlob{},
},
}, nil, sparse)
}
}

79
internal/restorer/fileswriter.go
View File

@ -39,6 +39,48 @@ func newFilesWriter(count int) *filesWriter {
}
}
func createFile(path string, createSize int64, sparse bool) (*os.File, error) {
var f *os.File
var err error
if f, err = os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600); err != nil {
if !fs.IsAccessDenied(err) {
return nil, err
}
// If file is readonly, clear the readonly flag by resetting the
// permissions of the file and try again
// as the metadata will be set again in the second pass and the
// readonly flag will be applied again if needed.
if err = fs.ResetPermissions(path); err != nil {
return nil, err
}
if f, err = os.OpenFile(path, os.O_TRUNC|os.O_WRONLY, 0600); err != nil {
return nil, err
}
}
if createSize > 0 {
if sparse {
err = truncateSparse(f, createSize)
if err != nil {
_ = f.Close()
return nil, err
}
} else {
err := fs.PreallocateFile(f, createSize)
if err != nil {
// Just log the preallocate error but don't let it cause the restore process to fail.
// Preallocate might return an error if the filesystem (implementation) does not
// support preallocation or our parameters combination to the preallocate call
// This should yield a syscall.ENOTSUP error, but some other errors might also
// show up.
debug.Log("Failed to preallocate %v with size %v: %v", path, createSize, err)
}
}
}
return f, err
}
func (w *filesWriter) writeToFile(path string, blob []byte, offset int64, createSize int64, sparse bool) error {
bucket := &w.buckets[uint(xxhash.Sum64String(path))%uint(len(w.buckets))]
@ -53,21 +95,9 @@ func (w *filesWriter) writeToFile(path string, blob []byte, offset int64, create
var f *os.File
var err error
if createSize >= 0 {
if f, err = os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600); err != nil {
if fs.IsAccessDenied(err) {
// If file is readonly, clear the readonly flag by resetting the
// permissions of the file and try again
// as the metadata will be set again in the second pass and the
// readonly flag will be applied again if needed.
if err = fs.ResetPermissions(path); err != nil {
return nil, err
}
if f, err = os.OpenFile(path, os.O_TRUNC|os.O_WRONLY, 0600); err != nil {
return nil, err
}
} else {
return nil, err
}
f, err = createFile(path, createSize, sparse)
if err != nil {
return nil, err
}
} else if f, err = os.OpenFile(path, os.O_WRONLY, 0600); err != nil {
return nil, err
@ -76,25 +106,6 @@ func (w *filesWriter) writeToFile(path string, blob []byte, offset int64, create
wr := &partialFile{File: f, users: 1, sparse: sparse}
bucket.files[path] = wr
if createSize >= 0 {
if sparse {
err = truncateSparse(f, createSize)
if err != nil {
return nil, err
}
} else {
err := fs.PreallocateFile(wr.File, createSize)
if err != nil {
// Just log the preallocate error but don't let it cause the restore process to fail.
// Preallocate might return an error if the filesystem (implementation) does not
// support preallocation or our parameters combination to the preallocate call
// This should yield a syscall.ENOTSUP error, but some other errors might also
// show up.
debug.Log("Failed to preallocate %v with size %v: %v", path, createSize, err)
}
}
}
return wr, nil
}

211
internal/restorer/restorer.go
View File

@ -2,6 +2,7 @@ package restorer
import (
"context"
"fmt"
"os"
"path/filepath"
"sync/atomic"
@ -17,11 +18,13 @@ import (
// Restorer is used to restore a snapshot to a directory.
type Restorer struct {
repo restic.Repository
sn *restic.Snapshot
sparse bool
repo restic.Repository
sn *restic.Snapshot
sparse bool
progress *restoreui.Progress
overwrite OverwriteBehavior
progress *restoreui.Progress
fileList map[string]struct{}
Error func(location string, err error) error
Warn func(message string)
@ -30,15 +33,66 @@ type Restorer struct {
var restorerAbortOnAllErrors = func(_ string, err error) error { return err }
type Options struct {
Sparse bool
Progress *restoreui.Progress
Overwrite OverwriteBehavior
}
type OverwriteBehavior int
// Constants for different overwrite behavior
const (
OverwriteAlways OverwriteBehavior = 0
OverwriteIfNewer OverwriteBehavior = 1
OverwriteNever OverwriteBehavior = 2
OverwriteInvalid OverwriteBehavior = 3
)
// Set implements the method needed for pflag command flag parsing.
func (c *OverwriteBehavior) Set(s string) error {
switch s {
case "always":
*c = OverwriteAlways
case "if-newer":
*c = OverwriteIfNewer
case "never":
*c = OverwriteNever
default:
*c = OverwriteInvalid
return fmt.Errorf("invalid overwrite behavior %q, must be one of (always|if-newer|never)", s)
}
return nil
}
func (c *OverwriteBehavior) String() string {
switch *c {
case OverwriteAlways:
return "always"
case OverwriteIfNewer:
return "if-newer"
case OverwriteNever:
return "never"
default:
return "invalid"
}
}
func (c *OverwriteBehavior) Type() string {
return "behavior"
}
// NewRestorer creates a restorer preloaded with the content from the snapshot id.
func NewRestorer(repo restic.Repository, sn *restic.Snapshot, sparse bool,
progress *restoreui.Progress) *Restorer {
func NewRestorer(repo restic.Repository, sn *restic.Snapshot, opts Options) *Restorer {
r := &Restorer{
repo: repo,
sparse: sparse,
sparse: opts.Sparse,
progress: opts.Progress,
overwrite: opts.Overwrite,
fileList: make(map[string]struct{}),
Error: restorerAbortOnAllErrors,
SelectFilter: func(string, string, *restic.Node) (bool, bool) { return true, true },
progress: progress,
sn: sn,
}
@ -170,10 +224,7 @@ func (res *Restorer) restoreNodeTo(ctx context.Context, node *restic.Node, targe
return err
}
if res.progress != nil {
res.progress.AddProgress(location, 0, 0)
}
res.progress.AddProgress(location, 0, 0)
return res.restoreNodeMetadataTo(node, target, location)
}
@ -195,39 +246,12 @@ func (res *Restorer) restoreHardlinkAt(node *restic.Node, target, path, location
return errors.WithStack(err)
}
if res.progress != nil {
res.progress.AddProgress(location, 0, 0)
}
res.progress.AddProgress(location, 0, 0)
// TODO investigate if hardlinks have separate metadata on any supported system
return res.restoreNodeMetadataTo(node, path, location)
}
func (res *Restorer) restoreEmptyFileAt(node *restic.Node, target, location string) error {
wr, err := os.OpenFile(target, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600)
if fs.IsAccessDenied(err) {
// If file is readonly, clear the readonly flag by resetting the
// permissions of the file and try again
// as the metadata will be set again in the second pass and the
// readonly flag will be applied again if needed.
if err = fs.ResetPermissions(target); err != nil {
return err
}
if wr, err = os.OpenFile(target, os.O_TRUNC|os.O_WRONLY, 0600); err != nil {
return err
}
}
if err = wr.Close(); err != nil {
return err
}
if res.progress != nil {
res.progress.AddProgress(location, 0, 0)
}
return res.restoreNodeMetadataTo(node, target, location)
}
// RestoreTo creates the directories and files in the snapshot below dst.
// Before an item is created, res.Filter is called.
func (res *Restorer) RestoreTo(ctx context.Context, dst string) error {
@ -250,9 +274,7 @@ func (res *Restorer) RestoreTo(ctx context.Context, dst string) error {
_, err = res.traverseTree(ctx, dst, string(filepath.Separator), *res.sn.Tree, treeVisitor{
enterDir: func(_ *restic.Node, target, location string) error {
debug.Log("first pass, enterDir: mkdir %q, leaveDir should restore metadata", location)
if res.progress != nil {
res.progress.AddFile(0)
}
res.progress.AddFile(0)
// create dir with default permissions
// #leaveDir restores dir metadata after visiting all children
return fs.MkdirAll(target, 0700)
@ -268,37 +290,25 @@ func (res *Restorer) RestoreTo(ctx context.Context, dst string) error {
}
if node.Type != "file" {
if res.progress != nil {
res.progress.AddFile(0)
}
res.progress.AddFile(0)
return nil
}
if node.Size == 0 {
if res.progress != nil {
res.progress.AddFile(node.Size)
}
return nil // deal with empty files later
}
if node.Links > 1 {
if idx.Has(node.Inode, node.DeviceID) {
if res.progress != nil {
// a hardlinked file does not increase the restore size
res.progress.AddFile(0)
}
// a hardlinked file does not increase the restore size
res.progress.AddFile(0)
return nil
}
idx.Add(node.Inode, node.DeviceID, location)
}
if res.progress != nil {
return res.withOverwriteCheck(node, target, false, func() error {
res.progress.AddFile(node.Size)
}
filerestorer.addFile(location, node.Content, int64(node.Size))
return nil
filerestorer.addFile(location, node.Content, int64(node.Size))
res.trackFile(location)
return nil
})
},
})
if err != nil {
@ -317,26 +327,26 @@ func (res *Restorer) RestoreTo(ctx context.Context, dst string) error {
visitNode: func(node *restic.Node, target, location string) error {
debug.Log("second pass, visitNode: restore node %q", location)
if node.Type != "file" {
return res.restoreNodeTo(ctx, node, target, location)
}
// create empty files, but not hardlinks to empty files
if node.Size == 0 && (node.Links < 2 || !idx.Has(node.Inode, node.DeviceID)) {
if node.Links > 1 {
idx.Add(node.Inode, node.DeviceID, location)
}
return res.restoreEmptyFileAt(node, target, location)
return res.withOverwriteCheck(node, target, false, func() error {
return res.restoreNodeTo(ctx, node, target, location)
})
}
if idx.Has(node.Inode, node.DeviceID) && idx.Value(node.Inode, node.DeviceID) != location {
return res.restoreHardlinkAt(node, filerestorer.targetPath(idx.Value(node.Inode, node.DeviceID)), target, location)
return res.withOverwriteCheck(node, target, true, func() error {
return res.restoreHardlinkAt(node, filerestorer.targetPath(idx.Value(node.Inode, node.DeviceID)), target, location)
})
}
return res.restoreNodeMetadataTo(node, target, location)
if res.hasRestoredFile(location) {
return res.restoreNodeMetadataTo(node, target, location)
}
// don't touch skipped files
return nil
},
leaveDir: func(node *restic.Node, target, location string) error {
err := res.restoreNodeMetadataTo(node, target, location)
if err == nil && res.progress != nil {
if err == nil {
res.progress.AddProgress(location, 0, 0)
}
return err
@ -345,6 +355,53 @@ func (res *Restorer) RestoreTo(ctx context.Context, dst string) error {
return err
}
func (res *Restorer) trackFile(location string) {
res.fileList[location] = struct{}{}
}
func (res *Restorer) hasRestoredFile(location string) bool {
_, ok := res.fileList[location]
return ok
}
func (res *Restorer) withOverwriteCheck(node *restic.Node, target string, isHardlink bool, cb func() error) error {
overwrite, err := shouldOverwrite(res.overwrite, node, target)
if err != nil {
return err
} else if !overwrite {
size := node.Size
if isHardlink {
size = 0
}
res.progress.AddSkippedFile(size)
return nil
}
return cb()
}
func shouldOverwrite(overwrite OverwriteBehavior, node *restic.Node, destination string) (bool, error) {
if overwrite == OverwriteAlways {
return true, nil
}
fi, err := fs.Lstat(destination)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
return true, nil
}
return false, err
}
if overwrite == OverwriteIfNewer {
// return if node is newer
return node.ModTime.After(fi.ModTime()), nil
} else if overwrite == OverwriteNever {
// file exists
return false, nil
}
panic("unknown overwrite behavior")
}
// Snapshot returns the snapshot this restorer is configured to use.
func (res *Restorer) Snapshot() *restic.Snapshot {
return res.sn
@ -375,8 +432,8 @@ func (res *Restorer) VerifyFiles(ctx context.Context, dst string) (int, error) {
defer close(work)
_, err := res.traverseTree(ctx, dst, string(filepath.Separator), *res.sn.Tree, treeVisitor{
visitNode: func(node *restic.Node, target, _ string) error {
if node.Type != "file" {
visitNode: func(node *restic.Node, target, location string) error {
if node.Type != "file" || !res.hasRestoredFile(location) {
return nil
}
select {

101
internal/restorer/restorer_test.go
View File

@ -343,7 +343,7 @@ func TestRestorer(t *testing.T) {
sn, id := saveSnapshot(t, repo, test.Snapshot, noopGetGenericAttributes)
t.Logf("snapshot saved as %v", id.Str())
res := NewRestorer(repo, sn, false, nil)
res := NewRestorer(repo, sn, Options{})
tempdir := rtest.TempDir(t)
// make sure we're creating a new subdir of the tempdir
@ -460,7 +460,7 @@ func TestRestorerRelative(t *testing.T) {
sn, id := saveSnapshot(t, repo, test.Snapshot, noopGetGenericAttributes)
t.Logf("snapshot saved as %v", id.Str())
res := NewRestorer(repo, sn, false, nil)
res := NewRestorer(repo, sn, Options{})
tempdir := rtest.TempDir(t)
cleanup := rtest.Chdir(t, tempdir)
@ -689,7 +689,7 @@ func TestRestorerTraverseTree(t *testing.T) {
repo := repository.TestRepository(t)
sn, _ := saveSnapshot(t, repo, test.Snapshot, noopGetGenericAttributes)
res := NewRestorer(repo, sn, false, nil)
res := NewRestorer(repo, sn, Options{})
res.SelectFilter = test.Select
@ -765,7 +765,7 @@ func TestRestorerConsistentTimestampsAndPermissions(t *testing.T) {
},
}, noopGetGenericAttributes)
res := NewRestorer(repo, sn, false, nil)
res := NewRestorer(repo, sn, Options{})
res.SelectFilter = func(item string, dstpath string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool) {
switch filepath.ToSlash(item) {
@ -820,7 +820,7 @@ func TestVerifyCancel(t *testing.T) {
repo := repository.TestRepository(t)
sn, _ := saveSnapshot(t, repo, snapshot, noopGetGenericAttributes)
res := NewRestorer(repo, sn, false, nil)
res := NewRestorer(repo, sn, Options{})
tempdir := rtest.TempDir(t)
ctx, cancel := context.WithCancel(context.Background())
@ -862,7 +862,7 @@ func TestRestorerSparseFiles(t *testing.T) {
archiver.SnapshotOptions{})
rtest.OK(t, err)
res := NewRestorer(repo, sn, true, nil)
res := NewRestorer(repo, sn, Options{Sparse: true})
tempdir := rtest.TempDir(t)
ctx, cancel := context.WithCancel(context.Background())
@ -893,3 +893,92 @@ func TestRestorerSparseFiles(t *testing.T) {
t.Logf("wrote %d zeros as %d blocks, %.1f%% sparse",
len(zeros), blocks, 100*sparsity)
}
func TestRestorerOverwriteBehavior(t *testing.T) {
baseTime := time.Now()
baseSnapshot := Snapshot{
Nodes: map[string]Node{
"foo": File{Data: "content: foo\n", ModTime: baseTime},
"dirtest": Dir{
Nodes: map[string]Node{
"file": File{Data: "content: file\n", ModTime: baseTime},
},
ModTime: baseTime,
},
},
}
overwriteSnapshot := Snapshot{
Nodes: map[string]Node{
"foo": File{Data: "content: new\n", ModTime: baseTime.Add(time.Second)},
"dirtest": Dir{
Nodes: map[string]Node{
"file": File{Data: "content: file2\n", ModTime: baseTime.Add(-time.Second)},
},
},
},
}
var tests = []struct {
Overwrite OverwriteBehavior
Files map[string]string
}{
{
Overwrite: OverwriteAlways,
Files: map[string]string{
"foo": "content: new\n",
"dirtest/file": "content: file2\n",
},
},
{
Overwrite: OverwriteIfNewer,
Files: map[string]string{
"foo": "content: new\n",
"dirtest/file": "content: file\n",
},
},
{
Overwrite: OverwriteNever,
Files: map[string]string{
"foo": "content: foo\n",
"dirtest/file": "content: file\n",
},
},
}
for _, test := range tests {
t.Run("", func(t *testing.T) {
repo := repository.TestRepository(t)
tempdir := filepath.Join(rtest.TempDir(t), "target")
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
// base snapshot
sn, id := saveSnapshot(t, repo, baseSnapshot, noopGetGenericAttributes)
t.Logf("base snapshot saved as %v", id.Str())
res := NewRestorer(repo, sn, Options{})
rtest.OK(t, res.RestoreTo(ctx, tempdir))
// overwrite snapshot
sn, id = saveSnapshot(t, repo, overwriteSnapshot, noopGetGenericAttributes)
t.Logf("overwrite snapshot saved as %v", id.Str())
res = NewRestorer(repo, sn, Options{Overwrite: test.Overwrite})
rtest.OK(t, res.RestoreTo(ctx, tempdir))
_, err := res.VerifyFiles(ctx, tempdir)
rtest.OK(t, err)
for filename, content := range test.Files {
data, err := os.ReadFile(filepath.Join(tempdir, filepath.FromSlash(filename)))
if err != nil {
t.Errorf("unable to read file %v: %v", filename, err)
continue
}
if !bytes.Equal(data, []byte(content)) {
t.Errorf("file %v has wrong content: want %q, got %q", filename, content, data)
}
}
})
}
}

31
internal/restorer/restorer_unix_test.go
View File

@ -31,7 +31,7 @@ func TestRestorerRestoreEmptyHardlinkedFileds(t *testing.T) {
},
}, noopGetGenericAttributes)
res := NewRestorer(repo, sn, false, nil)
res := NewRestorer(repo, sn, Options{})
res.SelectFilter = func(item string, dstpath string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool) {
return true, true
@ -70,16 +70,13 @@ func getBlockCount(t *testing.T, filename string) int64 {
}
type printerMock struct {
filesFinished, filesTotal, allBytesWritten, allBytesTotal uint64
s restoreui.State
}
func (p *printerMock) Update(_, _, _, _ uint64, _ time.Duration) {
func (p *printerMock) Update(_ restoreui.State, _ time.Duration) {
}
func (p *printerMock) Finish(filesFinished, filesTotal, allBytesWritten, allBytesTotal uint64, _ time.Duration) {
p.filesFinished = filesFinished
p.filesTotal = filesTotal
p.allBytesWritten = allBytesWritten
p.allBytesTotal = allBytesTotal
func (p *printerMock) Finish(s restoreui.State, _ time.Duration) {
p.s = s
}
func TestRestorerProgressBar(t *testing.T) {
@ -99,7 +96,7 @@ func TestRestorerProgressBar(t *testing.T) {
mock := &printerMock{}
progress := restoreui.NewProgress(mock, 0)
res := NewRestorer(repo, sn, false, progress)
res := NewRestorer(repo, sn, Options{Progress: progress})
res.SelectFilter = func(item string, dstpath string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool) {
return true, true
}
@ -112,12 +109,12 @@ func TestRestorerProgressBar(t *testing.T) {
rtest.OK(t, err)
progress.Finish()
const filesFinished = 4
const filesTotal = filesFinished
const allBytesWritten = 10
const allBytesTotal = allBytesWritten
rtest.Assert(t, mock.filesFinished == filesFinished, "filesFinished: expected %v, got %v", filesFinished, mock.filesFinished)
rtest.Assert(t, mock.filesTotal == filesTotal, "filesTotal: expected %v, got %v", filesTotal, mock.filesTotal)
rtest.Assert(t, mock.allBytesWritten == allBytesWritten, "allBytesWritten: expected %v, got %v", allBytesWritten, mock.allBytesWritten)
rtest.Assert(t, mock.allBytesTotal == allBytesTotal, "allBytesTotal: expected %v, got %v", allBytesTotal, mock.allBytesTotal)
rtest.Equals(t, restoreui.State{
FilesFinished: 4,
FilesTotal: 4,
FilesSkipped: 0,
AllBytesWritten: 10,
AllBytesTotal: 10,
AllBytesSkipped: 0,
}, mock.s)
}

2
internal/restorer/restorer_windows_test.go
View File

@ -269,7 +269,7 @@ func setup(t *testing.T, nodesMap map[string]Node) *Restorer {
sn, _ := saveSnapshot(t, repo, Snapshot{
Nodes: nodesMap,
}, getFileAttributes)
res := NewRestorer(repo, sn, false, nil)
res := NewRestorer(repo, sn, Options{})
return res
}

32
internal/ui/restore/json.go
View File

@ -20,31 +20,35 @@ func (t *jsonPrinter) print(status interface{}) {
t.terminal.Print(ui.ToJSONString(status))
}
func (t *jsonPrinter) Update(filesFinished, filesTotal, allBytesWritten, allBytesTotal uint64, duration time.Duration) {
func (t *jsonPrinter) Update(p State, duration time.Duration) {
status := statusUpdate{
MessageType: "status",
SecondsElapsed: uint64(duration / time.Second),
TotalFiles: filesTotal,
FilesRestored: filesFinished,
TotalBytes: allBytesTotal,
BytesRestored: allBytesWritten,
TotalFiles: p.FilesTotal,
FilesRestored: p.FilesFinished,
FilesSkipped: p.FilesSkipped,
TotalBytes: p.AllBytesTotal,
BytesRestored: p.AllBytesWritten,
BytesSkipped: p.AllBytesSkipped,
}
if allBytesTotal > 0 {
status.PercentDone = float64(allBytesWritten) / float64(allBytesTotal)
if p.AllBytesTotal > 0 {
status.PercentDone = float64(p.AllBytesWritten) / float64(p.AllBytesTotal)
}
t.print(status)
}
func (t *jsonPrinter) Finish(filesFinished, filesTotal, allBytesWritten, allBytesTotal uint64, duration time.Duration) {
func (t *jsonPrinter) Finish(p State, duration time.Duration) {
status := summaryOutput{
MessageType: "summary",
SecondsElapsed: uint64(duration / time.Second),
TotalFiles: filesTotal,
FilesRestored: filesFinished,
TotalBytes: allBytesTotal,
BytesRestored: allBytesWritten,
TotalFiles: p.FilesTotal,
FilesRestored: p.FilesFinished,
FilesSkipped: p.FilesSkipped,
TotalBytes: p.AllBytesTotal,
BytesRestored: p.AllBytesWritten,
BytesSkipped: p.AllBytesSkipped,
}
t.print(status)
}
@ -55,8 +59,10 @@ type statusUpdate struct {
PercentDone float64 `json:"percent_done"`
TotalFiles uint64 `json:"total_files,omitempty"`
FilesRestored uint64 `json:"files_restored,omitempty"`
FilesSkipped uint64 `json:"files_skipped,omitempty"`
TotalBytes uint64 `json:"total_bytes,omitempty"`
BytesRestored uint64 `json:"bytes_restored,omitempty"`
BytesSkipped uint64 `json:"bytes_skipped,omitempty"`
}
type summaryOutput struct {
@ -64,6 +70,8 @@ type summaryOutput struct {
SecondsElapsed uint64 `json:"seconds_elapsed,omitempty"`
TotalFiles uint64 `json:"total_files,omitempty"`
FilesRestored uint64 `json:"files_restored,omitempty"`
FilesSkipped uint64 `json:"files_skipped,omitempty"`
TotalBytes uint64 `json:"total_bytes,omitempty"`
BytesRestored uint64 `json:"bytes_restored,omitempty"`
BytesSkipped uint64 `json:"bytes_skipped,omitempty"`
}

20
internal/ui/restore/json_test.go
View File

@ -10,20 +10,34 @@ import (
func TestJSONPrintUpdate(t *testing.T) {
term := &mockTerm{}
printer := NewJSONProgress(term)
printer.Update(3, 11, 29, 47, 5*time.Second)
printer.Update(State{3, 11, 0, 29, 47, 0}, 5*time.Second)
test.Equals(t, []string{"{\"message_type\":\"status\",\"seconds_elapsed\":5,\"percent_done\":0.6170212765957447,\"total_files\":11,\"files_restored\":3,\"total_bytes\":47,\"bytes_restored\":29}\n"}, term.output)
}
func TestJSONPrintUpdateWithSkipped(t *testing.T) {
term := &mockTerm{}
printer := NewJSONProgress(term)
printer.Update(State{3, 11, 2, 29, 47, 59}, 5*time.Second)
test.Equals(t, []string{"{\"message_type\":\"status\",\"seconds_elapsed\":5,\"percent_done\":0.6170212765957447,\"total_files\":11,\"files_restored\":3,\"files_skipped\":2,\"total_bytes\":47,\"bytes_restored\":29,\"bytes_skipped\":59}\n"}, term.output)
}
func TestJSONPrintSummaryOnSuccess(t *testing.T) {
term := &mockTerm{}
printer := NewJSONProgress(term)
printer.Finish(11, 11, 47, 47, 5*time.Second)
printer.Finish(State{11, 11, 0, 47, 47, 0}, 5*time.Second)
test.Equals(t, []string{"{\"message_type\":\"summary\",\"seconds_elapsed\":5,\"total_files\":11,\"files_restored\":11,\"total_bytes\":47,\"bytes_restored\":47}\n"}, term.output)
}
func TestJSONPrintSummaryOnErrors(t *testing.T) {
term := &mockTerm{}
printer := NewJSONProgress(term)
printer.Finish(3, 11, 29, 47, 5*time.Second)
printer.Finish(State{3, 11, 0, 29, 47, 0}, 5*time.Second)
test.Equals(t, []string{"{\"message_type\":\"summary\",\"seconds_elapsed\":5,\"total_files\":11,\"files_restored\":3,\"total_bytes\":47,\"bytes_restored\":29}\n"}, term.output)
}
func TestJSONPrintSummaryOnSuccessWithSkipped(t *testing.T) {
term := &mockTerm{}
printer := NewJSONProgress(term)
printer.Finish(State{11, 11, 2, 47, 47, 59}, 5*time.Second)
test.Equals(t, []string{"{\"message_type\":\"summary\",\"seconds_elapsed\":5,\"total_files\":11,\"files_restored\":11,\"files_skipped\":2,\"total_bytes\":47,\"bytes_restored\":47,\"bytes_skipped\":59}\n"}, term.output)
}

50
internal/ui/restore/progress.go
View File

@ -7,15 +7,21 @@ import (
"github.com/restic/restic/internal/ui/progress"
)
type State struct {
FilesFinished uint64
FilesTotal uint64
FilesSkipped uint64
AllBytesWritten uint64
AllBytesTotal uint64
AllBytesSkipped uint64
}
type Progress struct {
updater progress.Updater
m sync.Mutex
progressInfoMap map[string]progressInfoEntry
filesFinished uint64
filesTotal uint64
allBytesWritten uint64
allBytesTotal uint64
s State
started time.Time
printer ProgressPrinter
@ -32,8 +38,8 @@ type term interface {
}
type ProgressPrinter interface {
Update(filesFinished, filesTotal, allBytesWritten, allBytesTotal uint64, duration time.Duration)
Finish(filesFinished, filesTotal, allBytesWritten, allBytesTotal uint64, duration time.Duration)
Update(progress State, duration time.Duration)
Finish(progress State, duration time.Duration)
}
func NewProgress(printer ProgressPrinter, interval time.Duration) *Progress {
@ -51,23 +57,31 @@ func (p *Progress) update(runtime time.Duration, final bool) {
defer p.m.Unlock()
if !final {
p.printer.Update(p.filesFinished, p.filesTotal, p.allBytesWritten, p.allBytesTotal, runtime)
p.printer.Update(p.s, runtime)
} else {
p.printer.Finish(p.filesFinished, p.filesTotal, p.allBytesWritten, p.allBytesTotal, runtime)
p.printer.Finish(p.s, runtime)
}
}
// AddFile starts tracking a new file with the given size
func (p *Progress) AddFile(size uint64) {
if p == nil {
return
}
p.m.Lock()
defer p.m.Unlock()
p.filesTotal++
p.allBytesTotal += size
p.s.FilesTotal++
p.s.AllBytesTotal += size
}
// AddProgress accumulates the number of bytes written for a file
func (p *Progress) AddProgress(name string, bytesWrittenPortion uint64, bytesTotal uint64) {
if p == nil {
return
}
p.m.Lock()
defer p.m.Unlock()
@ -78,13 +92,25 @@ func (p *Progress) AddProgress(name string, bytesWrittenPortion uint64, bytesTot
entry.bytesWritten += bytesWrittenPortion
p.progressInfoMap[name] = entry
p.allBytesWritten += bytesWrittenPortion
p.s.AllBytesWritten += bytesWrittenPortion
if entry.bytesWritten == entry.bytesTotal {
delete(p.progressInfoMap, name)
p.filesFinished++
p.s.FilesFinished++
}
}
func (p *Progress) AddSkippedFile(size uint64) {
if p == nil {
return
}
p.m.Lock()
defer p.m.Unlock()
p.s.FilesSkipped++
p.s.AllBytesSkipped += size
}
func (p *Progress) Finish() {
p.updater.Done()
}

36
internal/ui/restore/progress_test.go
View File

@ -8,7 +8,7 @@ import (
)
type printerTraceEntry struct {
filesFinished, filesTotal, allBytesWritten, allBytesTotal uint64
progress State
duration time.Duration
isFinished bool
@ -22,11 +22,11 @@ type mockPrinter struct {
const mockFinishDuration = 42 * time.Second
func (p *mockPrinter) Update(filesFinished, filesTotal, allBytesWritten, allBytesTotal uint64, duration time.Duration) {
p.trace = append(p.trace, printerTraceEntry{filesFinished, filesTotal, allBytesWritten, allBytesTotal, duration, false})
func (p *mockPrinter) Update(progress State, duration time.Duration) {
p.trace = append(p.trace, printerTraceEntry{progress, duration, false})
}
func (p *mockPrinter) Finish(filesFinished, filesTotal, allBytesWritten, allBytesTotal uint64, _ time.Duration) {
p.trace = append(p.trace, printerTraceEntry{filesFinished, filesTotal, allBytesWritten, allBytesTotal, mockFinishDuration, true})
func (p *mockPrinter) Finish(progress State, _ time.Duration) {
p.trace = append(p.trace, printerTraceEntry{progress, mockFinishDuration, true})
}
func testProgress(fn func(progress *Progress) bool) printerTrace {
@ -45,7 +45,7 @@ func TestNew(t *testing.T) {
return false
})
test.Equals(t, printerTrace{
printerTraceEntry{0, 0, 0, 0, 0, false},
printerTraceEntry{State{0, 0, 0, 0, 0, 0}, 0, false},
}, result)
}
@ -57,7 +57,7 @@ func TestAddFile(t *testing.T) {
return false
})
test.Equals(t, printerTrace{
printerTraceEntry{0, 1, 0, fileSize, 0, false},
printerTraceEntry{State{0, 1, 0, 0, fileSize, 0}, 0, false},
}, result)
}
@ -71,7 +71,7 @@ func TestFirstProgressOnAFile(t *testing.T) {
return false
})
test.Equals(t, printerTrace{
printerTraceEntry{0, 1, expectedBytesWritten, expectedBytesTotal, 0, false},
printerTraceEntry{State{0, 1, 0, expectedBytesWritten, expectedBytesTotal, 0}, 0, false},
}, result)
}
@ -86,7 +86,7 @@ func TestLastProgressOnAFile(t *testing.T) {
return false
})
test.Equals(t, printerTrace{
printerTraceEntry{1, 1, fileSize, fileSize, 0, false},
printerTraceEntry{State{1, 1, 0, fileSize, fileSize, 0}, 0, false},
}, result)
}
@ -102,7 +102,7 @@ func TestLastProgressOnLastFile(t *testing.T) {
return false
})
test.Equals(t, printerTrace{
printerTraceEntry{2, 2, 50 + fileSize, 50 + fileSize, 0, false},
printerTraceEntry{State{2, 2, 0, 50 + fileSize, 50 + fileSize, 0}, 0, false},
}, result)
}
@ -117,7 +117,7 @@ func TestSummaryOnSuccess(t *testing.T) {
return true
})
test.Equals(t, printerTrace{
printerTraceEntry{2, 2, 50 + fileSize, 50 + fileSize, mockFinishDuration, true},
printerTraceEntry{State{2, 2, 0, 50 + fileSize, 50 + fileSize, 0}, mockFinishDuration, true},
}, result)
}
@ -132,6 +132,18 @@ func TestSummaryOnErrors(t *testing.T) {
return true
})
test.Equals(t, printerTrace{
printerTraceEntry{1, 2, 50 + fileSize/2, 50 + fileSize, mockFinishDuration, true},
printerTraceEntry{State{1, 2, 0, 50 + fileSize/2, 50 + fileSize, 0}, mockFinishDuration, true},
}, result)
}
func TestSkipFile(t *testing.T) {
fileSize := uint64(100)
result := testProgress(func(progress *Progress) bool {
progress.AddSkippedFile(fileSize)
return true
})
test.Equals(t, printerTrace{
printerTraceEntry{State{0, 0, 1, 0, 0, fileSize}, mockFinishDuration, true},
}, result)
}

28
internal/ui/restore/text.go
View File

@ -17,30 +17,36 @@ func NewTextProgress(terminal term) ProgressPrinter {
}
}
func (t *textPrinter) Update(filesFinished, filesTotal, allBytesWritten, allBytesTotal uint64, duration time.Duration) {
func (t *textPrinter) Update(p State, duration time.Duration) {
timeLeft := ui.FormatDuration(duration)
formattedAllBytesWritten := ui.FormatBytes(allBytesWritten)
formattedAllBytesTotal := ui.FormatBytes(allBytesTotal)
allPercent := ui.FormatPercent(allBytesWritten, allBytesTotal)
formattedAllBytesWritten := ui.FormatBytes(p.AllBytesWritten)
formattedAllBytesTotal := ui.FormatBytes(p.AllBytesTotal)
allPercent := ui.FormatPercent(p.AllBytesWritten, p.AllBytesTotal)
progress := fmt.Sprintf("[%s] %s %v files/dirs %s, total %v files/dirs %v",
timeLeft, allPercent, filesFinished, formattedAllBytesWritten, filesTotal, formattedAllBytesTotal)
timeLeft, allPercent, p.FilesFinished, formattedAllBytesWritten, p.FilesTotal, formattedAllBytesTotal)
if p.FilesSkipped > 0 {
progress += fmt.Sprintf(", skipped %v files/dirs %v", p.FilesSkipped, ui.FormatBytes(p.AllBytesSkipped))
}
t.terminal.SetStatus([]string{progress})
}
func (t *textPrinter) Finish(filesFinished, filesTotal, allBytesWritten, allBytesTotal uint64, duration time.Duration) {
func (t *textPrinter) Finish(p State, duration time.Duration) {
t.terminal.SetStatus([]string{})
timeLeft := ui.FormatDuration(duration)
formattedAllBytesTotal := ui.FormatBytes(allBytesTotal)
formattedAllBytesTotal := ui.FormatBytes(p.AllBytesTotal)
var summary string
if filesFinished == filesTotal && allBytesWritten == allBytesTotal {
summary = fmt.Sprintf("Summary: Restored %d files/dirs (%s) in %s", filesTotal, formattedAllBytesTotal, timeLeft)
if p.FilesFinished == p.FilesTotal && p.AllBytesWritten == p.AllBytesTotal {
summary = fmt.Sprintf("Summary: Restored %d files/dirs (%s) in %s", p.FilesTotal, formattedAllBytesTotal, timeLeft)
} else {
formattedAllBytesWritten := ui.FormatBytes(allBytesWritten)
formattedAllBytesWritten := ui.FormatBytes(p.AllBytesWritten)
summary = fmt.Sprintf("Summary: Restored %d / %d files/dirs (%s / %s) in %s",
filesFinished, filesTotal, formattedAllBytesWritten, formattedAllBytesTotal, timeLeft)
p.FilesFinished, p.FilesTotal, formattedAllBytesWritten, formattedAllBytesTotal, timeLeft)
}
if p.FilesSkipped > 0 {
summary += fmt.Sprintf(", skipped %v files/dirs %v", p.FilesSkipped, ui.FormatBytes(p.AllBytesSkipped))
}
t.terminal.Print(summary)

20
internal/ui/restore/text_test.go
View File

@ -22,20 +22,34 @@ func (m *mockTerm) SetStatus(lines []string) {
func TestPrintUpdate(t *testing.T) {
term := &mockTerm{}
printer := NewTextProgress(term)
printer.Update(3, 11, 29, 47, 5*time.Second)
printer.Update(State{3, 11, 0, 29, 47, 0}, 5*time.Second)
test.Equals(t, []string{"[0:05] 61.70% 3 files/dirs 29 B, total 11 files/dirs 47 B"}, term.output)
}
func TestPrintUpdateWithSkipped(t *testing.T) {
term := &mockTerm{}
printer := NewTextProgress(term)
printer.Update(State{3, 11, 2, 29, 47, 59}, 5*time.Second)
test.Equals(t, []string{"[0:05] 61.70% 3 files/dirs 29 B, total 11 files/dirs 47 B, skipped 2 files/dirs 59 B"}, term.output)
}
func TestPrintSummaryOnSuccess(t *testing.T) {
term := &mockTerm{}
printer := NewTextProgress(term)
printer.Finish(11, 11, 47, 47, 5*time.Second)
printer.Finish(State{11, 11, 0, 47, 47, 0}, 5*time.Second)
test.Equals(t, []string{"Summary: Restored 11 files/dirs (47 B) in 0:05"}, term.output)
}
func TestPrintSummaryOnErrors(t *testing.T) {
term := &mockTerm{}
printer := NewTextProgress(term)
printer.Finish(3, 11, 29, 47, 5*time.Second)
printer.Finish(State{3, 11, 0, 29, 47, 0}, 5*time.Second)
test.Equals(t, []string{"Summary: Restored 3 / 11 files/dirs (29 B / 47 B) in 0:05"}, term.output)
}
func TestPrintSummaryOnSuccessWithSkipped(t *testing.T) {
term := &mockTerm{}
printer := NewTextProgress(term)
printer.Finish(State{11, 11, 2, 47, 47, 59}, 5*time.Second)
test.Equals(t, []string{"Summary: Restored 11 files/dirs (47 B) in 0:05, skipped 2 files/dirs 59 B"}, term.output)
}