octoleo/restic
2
2
Fork 0
mirror of https://github.com/octoleo/restic.git synced 2024-12-22 10:58:55 +00:00
Code Issues Releases Wiki Activity

Merge pull request #1772 from ifedorenko/restore-verify

Browse Source 
Restore verify
This commit is contained in:
Alexander Neumann 2018-06-23 14:50:29 +02:00
commit 7ddf91b65c
8 changed files with 301 additions and 187 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
changelog/unreleased/pull-1772 Normal file
View File

@ -0,0 +1,6 @@
Enhancement: Add restore --verify to verify restored file content
Restore will print error message if restored file content does not match
expected SHA256 checksum
https://github.com/restic/restic/pull/1772

11
cmd/restic/cmd_restore.go
View File

@ -5,6 +5,7 @@ import (
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/filter"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/restorer"
"github.com/spf13/cobra"
)
@ -33,6 +34,7 @@ type RestoreOptions struct {
Host string
Paths []string
Tags restic.TagLists
Verify bool
}
var restoreOptions RestoreOptions
@ -48,6 +50,7 @@ func init() {
flags.StringVarP(&restoreOptions.Host, "host", "H", "", `only consider snapshots for this host when the snapshot ID is "latest"`)
flags.Var(&restoreOptions.Tags, "tag", "only consider snapshots which include this `taglist` for snapshot ID \"latest\"")
flags.StringArrayVar(&restoreOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path` for snapshot ID \"latest\"")
flags.BoolVar(&restoreOptions.Verify, "verify", false, "verify restored files content")
}
func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error {
@ -104,7 +107,7 @@ func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error {
}
}
res, err := restic.NewRestorer(repo, id)
res, err := restorer.NewRestorer(repo, id)
if err != nil {
Exitf(2, "creating restorer failed: %v\n", err)
}
@ -153,6 +156,12 @@ func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error {
Verbosef("restoring %s to %s\n", res.Snapshot(), opts.Target)
err = res.RestoreTo(ctx, opts.Target)
if err == nil && opts.Verify {
Verbosef("verifying files in %s\n", opts.Target)
var count int
count, err = res.VerifyFiles(ctx, opts.Target)
Verbosef("finished verifying %d files in %s\n", count, opts.Target)
}
if totalErrors > 0 {
Printf("There were %d errors\n", totalErrors)
}

3
cmd/restic/integration_test.go
View File

@ -875,9 +875,6 @@ func TestRestoreNoMetadataOnIgnoredIntermediateDirs(t *testing.T) {
fi, err := os.Stat(f1)
rtest.OK(t, err)
rtest.Assert(t, fi.ModTime() != time.Unix(0, 0),
"meta data of intermediate directory has been restore although it was ignored")
// restore with filter "*", this should restore meta data on everything.
testRunRestoreIncludes(t, env.gopts, filepath.Join(env.base, "restore1"), snapshotID, []string{"*"})

17
internal/restic/node.go
View File

@ -134,7 +134,7 @@ func (node Node) GetExtendedAttribute(a string) []byte {
return nil
}
// CreateAt creates the node at the given path and restores all the meta data.
// CreateAt creates the node at the given path but does NOT restore node meta data.
func (node *Node) CreateAt(ctx context.Context, path string, repo Repository, idx *HardlinkIndex) error {
debug.Log("create node %v at %v", node.Name, path)
@ -169,6 +169,11 @@ func (node *Node) CreateAt(ctx context.Context, path string, repo Repository, id
return errors.Errorf("filetype %q not implemented!\n", node.Type)
}
return nil
}
// RestoreMetadata restores node metadata
func (node Node) RestoreMetadata(path string) error {
err := node.restoreMetadata(path)
if err != nil {
debug.Log("restoreMetadata(%s) error %v", path, err)
@ -192,12 +197,10 @@ func (node Node) restoreMetadata(path string) error {
}
}
if node.Type != "dir" {
if err := node.RestoreTimestamps(path); err != nil {
debug.Log("error restoring timestamps for dir %v: %v", path, err)
if firsterr != nil {
firsterr = err
}
if err := node.RestoreTimestamps(path); err != nil {
debug.Log("error restoring timestamps for dir %v: %v", path, err)
if firsterr != nil {
firsterr = err
}
}

1
internal/restic/node_test.go
View File

@ -182,6 +182,7 @@ func TestNodeRestoreAt(t *testing.T) {
for _, test := range nodeTests {
nodePath := filepath.Join(tempdir, test.Name)
rtest.OK(t, test.CreateAt(context.TODO(), nodePath, nil, idx))
rtest.OK(t, test.RestoreMetadata(nodePath))
if test.Type == "symlink" && runtime.GOOS == "windows" {
continue

173
internal/restic/restorer.go
View File

@ -1,173 +0,0 @@
package restic
import (
"context"
"os"
"path/filepath"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/fs"
)
// Restorer is used to restore a snapshot to a directory.
type Restorer struct {
repo Repository
sn *Snapshot
Error func(dir string, node *Node, err error) error
SelectFilter func(item string, dstpath string, node *Node) (selectedForRestore bool, childMayBeSelected bool)
}
var restorerAbortOnAllErrors = func(str string, node *Node, err error) error { return err }
// NewRestorer creates a restorer preloaded with the content from the snapshot id.
func NewRestorer(repo Repository, id ID) (*Restorer, error) {
r := &Restorer{
repo: repo, Error: restorerAbortOnAllErrors,
SelectFilter: func(string, string, *Node) (bool, bool) { return true, true },
}
var err error
r.sn, err = LoadSnapshot(context.TODO(), repo, id)
if err != nil {
return nil, err
}
return r, nil
}
// restoreTo restores a tree from the repo to a destination. target is the path in
// the file system, location within the snapshot.
func (res *Restorer) restoreTo(ctx context.Context, target, location string, treeID ID, idx *HardlinkIndex) error {
debug.Log("%v %v %v", target, location, treeID)
tree, err := res.repo.LoadTree(ctx, treeID)
if err != nil {
debug.Log("error loading tree %v: %v", treeID, err)
return res.Error(location, nil, err)
}
for _, node := range tree.Nodes {
// ensure that the node name does not contain anything that refers to a
// top-level directory.
nodeName := filepath.Base(filepath.Join(string(filepath.Separator), node.Name))
if nodeName != node.Name {
debug.Log("node %q has invalid name %q", node.Name, nodeName)
err := res.Error(location, node, errors.New("node has invalid name"))
if err != nil {
return err
}
continue
}
nodeTarget := filepath.Join(target, nodeName)
nodeLocation := filepath.Join(location, nodeName)
if target == nodeTarget || !fs.HasPathPrefix(target, nodeTarget) {
debug.Log("target: %v %v", target, nodeTarget)
debug.Log("node %q has invalid target path %q", node.Name, nodeTarget)
err := res.Error(nodeLocation, node, errors.New("node has invalid path"))
if err != nil {
return err
}
continue
}
// sockets cannot be restored
if node.Type == "socket" {
continue
}
selectedForRestore, childMayBeSelected := res.SelectFilter(nodeLocation, nodeTarget, node)
debug.Log("SelectFilter returned %v %v", selectedForRestore, childMayBeSelected)
if node.Type == "dir" && childMayBeSelected {
if node.Subtree == nil {
return errors.Errorf("Dir without subtree in tree %v", treeID.Str())
}
err = res.restoreTo(ctx, nodeTarget, nodeLocation, *node.Subtree, idx)
if err != nil {
err = res.Error(nodeLocation, node, err)
if err != nil {
return err
}
}
}
if selectedForRestore {
err = res.restoreNodeTo(ctx, node, nodeTarget, nodeLocation, idx)
if err != nil {
err = res.Error(nodeLocation, node, errors.Wrap(err, "restoreNodeTo"))
if err != nil {
return err
}
}
// Restore directory timestamp at the end. If we would do it earlier, restoring files within
// the directory would overwrite the timestamp of the directory they are in.
err = node.RestoreTimestamps(nodeTarget)
if err != nil {
err = res.Error(nodeLocation, node, errors.Wrap(err, "RestoreTimestamps"))
if err != nil {
return err
}
}
}
}
return nil
}
func (res *Restorer) restoreNodeTo(ctx context.Context, node *Node, target, location string, idx *HardlinkIndex) error {
debug.Log("%v %v %v", node.Name, target, location)
err := node.CreateAt(ctx, target, res.repo, idx)
if err != nil {
debug.Log("node.CreateAt(%s) error %v", target, err)
}
// Did it fail because of ENOENT?
if err != nil && os.IsNotExist(errors.Cause(err)) {
debug.Log("create intermediate paths")
// Create parent directories and retry
err = fs.MkdirAll(filepath.Dir(target), 0700)
if err == nil || os.IsExist(errors.Cause(err)) {
err = node.CreateAt(ctx, target, res.repo, idx)
}
}
if err != nil {
debug.Log("error %v", err)
err = res.Error(location, node, err)
if err != nil {
return err
}
}
return nil
}
// RestoreTo creates the directories and files in the snapshot below dst.
// Before an item is created, res.Filter is called.
func (res *Restorer) RestoreTo(ctx context.Context, dst string) error {
var err error
if !filepath.IsAbs(dst) {
dst, err = filepath.Abs(dst)
if err != nil {
return errors.Wrap(err, "Abs")
}
}
idx := NewHardlinkIndex()
return res.restoreTo(ctx, dst, string(filepath.Separator), *res.sn.Tree, idx)
}
// Snapshot returns the snapshot this restorer is configured to use.
func (res *Restorer) Snapshot() *Snapshot {
return res.sn
}

270
internal/restorer/restorer.go Normal file
View File

@ -0,0 +1,270 @@
package restorer
import (
"context"
"os"
"path/filepath"
"github.com/restic/restic/internal/crypto"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/fs"
"github.com/restic/restic/internal/restic"
)
// Restorer is used to restore a snapshot to a directory.
type Restorer struct {
repo restic.Repository
sn *restic.Snapshot
Error func(dir string, node *restic.Node, err error) error
SelectFilter func(item string, dstpath string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool)
}
var restorerAbortOnAllErrors = func(str string, node *restic.Node, err error) error { return err }
// NewRestorer creates a restorer preloaded with the content from the snapshot id.
func NewRestorer(repo restic.Repository, id restic.ID) (*Restorer, error) {
r := &Restorer{
repo: repo, Error: restorerAbortOnAllErrors,
SelectFilter: func(string, string, *restic.Node) (bool, bool) { return true, true },
}
var err error
r.sn, err = restic.LoadSnapshot(context.TODO(), repo, id)
if err != nil {
return nil, err
}
return r, nil
}
type treeVisitor struct {
enterDir func(node *restic.Node, target, location string) error
visitNode func(node *restic.Node, target, location string) error
leaveDir func(node *restic.Node, target, location string) error
}
// traverseTree traverses a tree from the repo and calls treeVisitor.
// target is the path in the file system, location within the snapshot.
func (res *Restorer) traverseTree(ctx context.Context, target, location string, treeID restic.ID, visitor treeVisitor) error {
debug.Log("%v %v %v", target, location, treeID)
tree, err := res.repo.LoadTree(ctx, treeID)
if err != nil {
debug.Log("error loading tree %v: %v", treeID, err)
return res.Error(location, nil, err)
}
for _, node := range tree.Nodes {
// ensure that the node name does not contain anything that refers to a
// top-level directory.
nodeName := filepath.Base(filepath.Join(string(filepath.Separator), node.Name))
if nodeName != node.Name {
debug.Log("node %q has invalid name %q", node.Name, nodeName)
err := res.Error(location, node, errors.New("node has invalid name"))
if err != nil {
return err
}
continue
}
nodeTarget := filepath.Join(target, nodeName)
nodeLocation := filepath.Join(location, nodeName)
if target == nodeTarget || !fs.HasPathPrefix(target, nodeTarget) {
debug.Log("target: %v %v", target, nodeTarget)
debug.Log("node %q has invalid target path %q", node.Name, nodeTarget)
err := res.Error(nodeLocation, node, errors.New("node has invalid path"))
if err != nil {
return err
}
continue
}
// sockets cannot be restored
if node.Type == "socket" {
continue
}
selectedForRestore, childMayBeSelected := res.SelectFilter(nodeLocation, nodeTarget, node)
debug.Log("SelectFilter returned %v %v", selectedForRestore, childMayBeSelected)
sanitizeError := func(err error) error {
if err != nil {
err = res.Error(nodeTarget, node, err)
}
return err
}
enteredDir := false
if node.Type == "dir" {
if node.Subtree == nil {
return errors.Errorf("Dir without subtree in tree %v", treeID.Str())
}
// ifedorenko: apparently a dir can be selected explicitly or implicitly when a child is selected
// to support implicit selection, visit the directory from within visitor#visitNode
if selectedForRestore {
enteredDir = true
err = sanitizeError(visitor.enterDir(node, nodeTarget, nodeLocation))
if err != nil {
return err
}
} else {
_visitor := visitor
visitor = treeVisitor{
enterDir: _visitor.enterDir,
visitNode: func(node *restic.Node, nodeTarget, nodeLocation string) error {
if !enteredDir {
enteredDir = true
derr := sanitizeError(_visitor.enterDir(node, nodeTarget, nodeLocation))
if derr != nil {
return derr
}
}
return _visitor.visitNode(node, nodeTarget, nodeLocation)
},
leaveDir: _visitor.leaveDir,
}
}
if childMayBeSelected {
err = sanitizeError(res.traverseTree(ctx, nodeTarget, nodeLocation, *node.Subtree, visitor))
if err != nil {
return err
}
}
}
if selectedForRestore && node.Type != "dir" {
err = sanitizeError(visitor.visitNode(node, nodeTarget, nodeLocation))
if err != nil {
err = res.Error(nodeLocation, node, errors.Wrap(err, "restoreNodeTo"))
if err != nil {
return err
}
}
}
if enteredDir {
err = sanitizeError(visitor.leaveDir(node, nodeTarget, nodeLocation))
if err != nil {
err = res.Error(nodeLocation, node, errors.Wrap(err, "RestoreTimestamps"))
if err != nil {
return err
}
}
}
}
return nil
}
func (res *Restorer) restoreNodeTo(ctx context.Context, node *restic.Node, target, location string, idx *restic.HardlinkIndex) error {
debug.Log("restoreNode %v %v %v", node.Name, target, location)
err := node.CreateAt(ctx, target, res.repo, idx)
if err != nil {
debug.Log("node.CreateAt(%s) error %v", target, err)
}
if err == nil {
err = res.restoreNodeMetadataTo(node, target, location)
}
return err
}
func (res *Restorer) restoreNodeMetadataTo(node *restic.Node, target, location string) error {
debug.Log("restoreNodeMetadata %v %v %v", node.Name, target, location)
err := node.RestoreMetadata(target)
if err != nil {
debug.Log("node.RestoreMetadata(%s) error %v", target, err)
}
return err
}
// RestoreTo creates the directories and files in the snapshot below dst.
// Before an item is created, res.Filter is called.
func (res *Restorer) RestoreTo(ctx context.Context, dst string) error {
var err error
if !filepath.IsAbs(dst) {
dst, err = filepath.Abs(dst)
if err != nil {
return errors.Wrap(err, "Abs")
}
}
idx := restic.NewHardlinkIndex()
return res.traverseTree(ctx, dst, string(filepath.Separator), *res.sn.Tree, treeVisitor{
enterDir: func(node *restic.Node, target, location string) error {
// create dir with default permissions
// #leaveDir restores dir metadata after visiting all children
return fs.MkdirAll(target, 0700)
},
visitNode: func(node *restic.Node, target, location string) error {
return res.restoreNodeTo(ctx, node, target, location, idx)
},
leaveDir: func(node *restic.Node, target, location string) error {
// Restore directory permissions and timestamp at the end. If we did it earlier
// - children restore could fail because of restictive directory permission
// - children restore could overwrite the timestamp of the directory they are in
return res.restoreNodeMetadataTo(node, target, location)
},
})
}
// Snapshot returns the snapshot this restorer is configured to use.
func (res *Restorer) Snapshot() *restic.Snapshot {
return res.sn
}
// VerifyFiles reads all snapshot files and verifies their contents
func (res *Restorer) VerifyFiles(ctx context.Context, dst string) (int, error) {
// TODO multithreaded?
count := 0
err := res.traverseTree(ctx, dst, string(filepath.Separator), *res.sn.Tree, treeVisitor{
enterDir: func(node *restic.Node, target, location string) error { return nil },
visitNode: func(node *restic.Node, target, location string) error {
if node.Type != "file" {
return nil
}
count++
stat, err := os.Stat(target)
if err != nil {
return err
}
if int64(node.Size) != stat.Size() {
return errors.Errorf("Invalid file size: expected %d got %d", node.Size, stat.Size())
}
offset := int64(0)
for _, blobID := range node.Content {
rd, err := os.Open(target)
if err != nil {
return err
}
blobs, _ := res.repo.Index().Lookup(blobID, restic.DataBlob)
length := blobs[0].Length - uint(crypto.Extension)
buf := make([]byte, length) // TODO do I want to reuse the buffer somehow?
_, err = rd.ReadAt(buf, offset)
if err != nil {
return err
}
if !blobID.Equal(restic.Hash(buf)) {
return errors.Errorf("Unexpected contents starting at offset %d", offset)
}
offset += int64(length)
}
return nil
},
leaveDir: func(node *restic.Node, target, location string) error { return nil },
})
return count, err
}

7
internal/restic/restorer_test.go → internal/restorer/restorer_test.go
View File

@ -1,4 +1,4 @@
package restic_test
package restorer_test
import (
"bytes"
@ -13,6 +13,7 @@ import (
"github.com/restic/restic/internal/fs"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/restorer"
rtest "github.com/restic/restic/internal/test"
)
@ -264,7 +265,7 @@ func TestRestorer(t *testing.T) {
_, id := saveSnapshot(t, repo, test.Snapshot)
t.Logf("snapshot saved as %v", id.Str())
res, err := restic.NewRestorer(repo, id)
res, err := restorer.NewRestorer(repo, id)
if err != nil {
t.Fatal(err)
}
@ -377,7 +378,7 @@ func TestRestorerRelative(t *testing.T) {
_, id := saveSnapshot(t, repo, test.Snapshot)
t.Logf("snapshot saved as %v", id.Str())
res, err := restic.NewRestorer(repo, id)
res, err := restorer.NewRestorer(repo, id)
if err != nil {
t.Fatal(err)
}