octoleo/restic
2
2
Fork 0
mirror of https://github.com/octoleo/restic.git synced 2024-11-16 01:57:10 +00:00
Code Issues Releases Wiki Activity

restore: significantly reduce memory footprint

Browse Source 
reworked restore error callback to use file location
path instead of much heavier Node. this reduced restore
memory usage by as much as 50% in some of my tests.

Signed-off-by: Igor Fedorenko <igor@ifedorenko.com>
This commit is contained in:
Igor Fedorenko 2018-09-14 20:55:30 -04:00 committed by Alexander Neumann
parent 9f3ca97ee8
commit 9e24154ec9
3 changed files with 45 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
cmd/restic/cmd_restore.go
View File

@ -113,8 +113,8 @@ func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error {
} }
totalErrors := 0 totalErrors := 0
res.Error = func(dir string, node *restic.Node, err error) error { res.Error = func(location string, err error) error {
Warnf("ignoring error for %s: %s\n", dir, err) Warnf("ignoring error for %s: %s\n", location, err)
totalErrors++ totalErrors++
return nil return nil
} }

18
internal/restorer/restorer.go
View File

@ -18,11 +18,11 @@ type Restorer struct {
repo restic.Repository repo restic.Repository
sn *restic.Snapshot sn *restic.Snapshot
Error func(dir string, node *restic.Node, err error) error Error func(location string, err error) error
SelectFilter func(item string, dstpath string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool) SelectFilter func(item string, dstpath string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool)
} }
var restorerAbortOnAllErrors = func(str string, node *restic.Node, err error) error { return err } var restorerAbortOnAllErrors = func(location string, err error) error { return err }
// NewRestorer creates a restorer preloaded with the content from the snapshot id. // NewRestorer creates a restorer preloaded with the content from the snapshot id.
func NewRestorer(repo restic.Repository, id restic.ID) (*Restorer, error) { func NewRestorer(repo restic.Repository, id restic.ID) (*Restorer, error) {
@ -55,7 +55,7 @@ func (res *Restorer) traverseTree(ctx context.Context, target, location string,
tree, err := res.repo.LoadTree(ctx, treeID) tree, err := res.repo.LoadTree(ctx, treeID)
if err != nil { if err != nil {
debug.Log("error loading tree %v: %v", treeID, err) debug.Log("error loading tree %v: %v", treeID, err)
return res.Error(location, nil, err) return res.Error(location, err)
} }
for _, node := range tree.Nodes { for _, node := range tree.Nodes {
@ -65,7 +65,7 @@ func (res *Restorer) traverseTree(ctx context.Context, target, location string,
nodeName := filepath.Base(filepath.Join(string(filepath.Separator), node.Name)) nodeName := filepath.Base(filepath.Join(string(filepath.Separator), node.Name))
if nodeName != node.Name { if nodeName != node.Name {
debug.Log("node %q has invalid name %q", node.Name, nodeName) debug.Log("node %q has invalid name %q", node.Name, nodeName)
err := res.Error(location, node, errors.New("node has invalid name")) err := res.Error(location, errors.Errorf("invalid child node name %s", node.Name))
if err != nil { if err != nil {
return err return err
} }
@ -78,7 +78,7 @@ func (res *Restorer) traverseTree(ctx context.Context, target, location string,
if target == nodeTarget || !fs.HasPathPrefix(target, nodeTarget) { if target == nodeTarget || !fs.HasPathPrefix(target, nodeTarget) {
debug.Log("target: %v %v", target, nodeTarget) debug.Log("target: %v %v", target, nodeTarget)
debug.Log("node %q has invalid target path %q", node.Name, nodeTarget) debug.Log("node %q has invalid target path %q", node.Name, nodeTarget)
err := res.Error(nodeLocation, node, errors.New("node has invalid path")) err := res.Error(nodeLocation, errors.New("node has invalid path"))
if err != nil { if err != nil {
return err return err
} }
@ -95,7 +95,7 @@ func (res *Restorer) traverseTree(ctx context.Context, target, location string,
sanitizeError := func(err error) error { sanitizeError := func(err error) error {
if err != nil { if err != nil {
err = res.Error(nodeTarget, node, err) err = res.Error(nodeLocation, err)
} }
return err return err
} }
@ -194,9 +194,6 @@ func (res *Restorer) RestoreTo(ctx context.Context, dst string) error {
filerestorer := newFileRestorer(dst, res.repo.Backend().Load, res.repo.Key(), filePackTraverser{lookup: res.repo.Index().Lookup}) filerestorer := newFileRestorer(dst, res.repo.Backend().Load, res.repo.Key(), filePackTraverser{lookup: res.repo.Index().Lookup})
// path->node map, only needed to call res.Error, which uses the node during tests
nodes := make(map[string]*restic.Node)
// first tree pass: create directories and collect all files to restore // first tree pass: create directories and collect all files to restore
err = res.traverseTree(ctx, dst, string(filepath.Separator), *res.sn.Tree, treeVisitor{ err = res.traverseTree(ctx, dst, string(filepath.Separator), *res.sn.Tree, treeVisitor{
enterDir: func(node *restic.Node, target, location string) error { enterDir: func(node *restic.Node, target, location string) error {
@ -224,7 +221,6 @@ func (res *Restorer) RestoreTo(ctx context.Context, dst string) error {
idx.Add(node.Inode, node.DeviceID, location) idx.Add(node.Inode, node.DeviceID, location)
} }
nodes[target] = node
filerestorer.addFile(location, node.Content) filerestorer.addFile(location, node.Content)
return nil return nil
@ -235,7 +231,7 @@ func (res *Restorer) RestoreTo(ctx context.Context, dst string) error {
return err return err
} }
err = filerestorer.restoreFiles(ctx, func(path string, err error) { res.Error(path, nodes[path], err) }) err = filerestorer.restoreFiles(ctx, func(location string, err error) { res.Error(location, err) })
if err != nil { if err != nil {
return err return err
} }

72
internal/restorer/restorer_test.go
View File

@ -134,8 +134,8 @@ func TestRestorer(t *testing.T) {
var tests = []struct { var tests = []struct {
Snapshot Snapshot
Files map[string]string Files map[string]string
ErrorsMust map[string]string ErrorsMust map[string]map[string]struct{}
ErrorsMay map[string]string ErrorsMay map[string]map[string]struct{}
Select func(item string, dstpath string, node *restic.Node) (selectForRestore bool, childMayBeSelected bool) Select func(item string, dstpath string, node *restic.Node) (selectForRestore bool, childMayBeSelected bool)
}{ }{
// valid test cases // valid test cases
@ -249,9 +249,11 @@ func TestRestorer(t *testing.T) {
`..\..\foo\..\bar\..\xx\test2`: File{"test2\n"}, `..\..\foo\..\bar\..\xx\test2`: File{"test2\n"},
}, },
}, },
ErrorsMay: map[string]string{ ErrorsMay: map[string]map[string]struct{}{
`/#..\test`: "node has invalid name", `/`: {
`/#..\..\foo\..\bar\..\xx\test2`: "node has invalid name", `invalid child node name ..\test`: struct{}{},
`invalid child node name ..\..\foo\..\bar\..\xx\test2`: struct{}{},
},
}, },
}, },
{ {
@ -261,9 +263,11 @@ func TestRestorer(t *testing.T) {
`../../foo/../bar/../xx/test2`: File{"test2\n"}, `../../foo/../bar/../xx/test2`: File{"test2\n"},
}, },
}, },
ErrorsMay: map[string]string{ ErrorsMay: map[string]map[string]struct{}{
`/#../test`: "node has invalid name", `/`: {
`/#../../foo/../bar/../xx/test2`: "node has invalid name", `invalid child node name ../test`: struct{}{},
`invalid child node name ../../foo/../bar/../xx/test2`: struct{}{},
},
}, },
}, },
{ {
@ -290,8 +294,10 @@ func TestRestorer(t *testing.T) {
Files: map[string]string{ Files: map[string]string{
"top": "toplevel file", "top": "toplevel file",
}, },
ErrorsMust: map[string]string{ ErrorsMust: map[string]map[string]struct{}{
`/x#..`: "node has invalid name", `/x`: {
`invalid child node name ..`: struct{}{},
},
}, },
}, },
} }
@ -329,11 +335,14 @@ func TestRestorer(t *testing.T) {
return true, true return true, true
} }
errors := make(map[string]string) errors := make(map[string]map[string]struct{})
res.Error = func(dir string, node *restic.Node, err error) error { res.Error = func(location string, err error) error {
t.Logf("restore returned error for %q in dir %v: %v", node.Name, dir, err) location = toSlash(location)
dir = toSlash(dir) t.Logf("restore returned error for %q: %v", location, err)
errors[dir+"#"+node.Name] = err.Error() if errors[location] == nil {
errors[location] = make(map[string]struct{})
}
errors[location][err.Error()] = struct{}{}
return nil return nil
} }
@ -345,33 +354,27 @@ func TestRestorer(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
for filename, errorMessage := range test.ErrorsMust { for location, expectedErrors := range test.ErrorsMust {
msg, ok := errors[filename] actualErrors, ok := errors[location]
if !ok { if !ok {
t.Errorf("expected error for %v, found none", filename) t.Errorf("expected error(s) for %v, found none", location)
continue continue
} }
if msg != "" && msg != errorMessage { rtest.Equals(t, expectedErrors, actualErrors)
t.Errorf("wrong error message for %v: got %q, want %q",
filename, msg, errorMessage) delete(errors, location)
} }
delete(errors, filename) for location, expectedErrors := range test.ErrorsMay {
} actualErrors, ok := errors[location]
for filename, errorMessage := range test.ErrorsMay {
msg, ok := errors[filename]
if !ok { if !ok {
continue continue
} }
if msg != "" && msg != errorMessage { rtest.Equals(t, expectedErrors, actualErrors)
t.Errorf("wrong error message for %v: got %q, want %q",
filename, msg, errorMessage)
}
delete(errors, filename) delete(errors, location)
} }
for filename, err := range errors { for filename, err := range errors {
@ -436,10 +439,9 @@ func TestRestorerRelative(t *testing.T) {
defer cleanup() defer cleanup()
errors := make(map[string]string) errors := make(map[string]string)
res.Error = func(dir string, node *restic.Node, err error) error { res.Error = func(location string, err error) error {
t.Logf("restore returned error for %q in dir %v: %v", node.Name, dir, err) t.Logf("restore returned error for %q: %v", location, err)
dir = toSlash(dir) errors[location] = err.Error()
errors[dir+"#"+node.Name] = err.Error()
return nil return nil
} }