octoleo
/
restic
mirror of https://github.com/octoleo/restic.git
2
2
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #1839 from restic/fix-find 

Fix find, do not skip some snapshots
This commit is contained in:
Alexander Neumann 2018-06-10 10:08:47 +02:00
parent c204382ea9 321efec60c
commit 63779c1eb4
11 changed files with 712 additions and 114 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
changelog/unreleased/issue-1825 Normal file
View File

@ -0,0 +1,12 @@
Bugfix: Correct `find` to not skip snapshots
Under certain circumstances, the `find` command was found to skip snapshots
containing directories with files to look for when the directories haven't been
modified at all, and were already printed as part of a different snapshot. This
is now corrected.
In addition, we've switched to our own matching/pattern implementation, so now
things like `restic find "/home/user/foo/**/main.go"` are possible.
https://github.com/restic/restic/issues/1825
https://github.com/restic/restic/issues/1823

16
cmd/restic/cmd_backup.go
View File

@ -336,6 +336,14 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
return err
}
timeStamp := time.Now()
if opts.TimeStamp != "" {
timeStamp, err = time.Parse(TimeFormat, opts.TimeStamp)
if err != nil {
return errors.Fatalf("error in time option: %v\n", err)
}
}
var t tomb.Tomb
p := ui.NewBackup(term, gopts.verbosity)
@ -402,14 +410,6 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
return true
}
timeStamp := time.Now()
if opts.TimeStamp != "" {
timeStamp, err = time.Parse(TimeFormat, opts.TimeStamp)
if err != nil {
return errors.Fatalf("error in time option: %v\n", err)
}
}
var targetFS fs.FS = fs.Local{}
if opts.Stdin {
p.V("read data from stdin")

122
cmd/restic/cmd_find.go
View File

@ -3,7 +3,6 @@ package main
import (
"context"
"encoding/json"
"path/filepath"
"strings"
"time"
@ -11,7 +10,9 @@ import (
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/filter"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/walker"
)
var cmdFind = &cobra.Command{
@ -94,7 +95,7 @@ type statefulOutput struct {
hits int
}
func (s *statefulOutput) PrintJSON(prefix string, node *restic.Node) {
func (s *statefulOutput) PrintJSON(path string, node *restic.Node) {
type findNode restic.Node
b, err := json.Marshal(struct {
// Add these attributes
@ -111,7 +112,7 @@ func (s *statefulOutput) PrintJSON(prefix string, node *restic.Node) {
Content byte `json:"content,omitempty"`
Subtree byte `json:"subtree,omitempty"`
}{
Path: filepath.Join(prefix, node.Name),
Path: path,
Permissions: node.Mode.String(),
findNode: (*findNode)(node),
})
@ -138,22 +139,22 @@ func (s *statefulOutput) PrintJSON(prefix string, node *restic.Node) {
s.hits++
}
func (s *statefulOutput) PrintNormal(prefix string, node *restic.Node) {
func (s *statefulOutput) PrintNormal(path string, node *restic.Node) {
if s.newsn != s.oldsn {
if s.oldsn != nil {
Verbosef("\n")
}
s.oldsn = s.newsn
Verbosef("Found matching entries in snapshot %s\n", s.oldsn.ID())
Verbosef("Found matching entries in snapshot %s\n", s.oldsn.ID().Str())
}
Printf(formatNode(prefix, node, s.ListLong) + "\n")
Printf(formatNode(path, node, s.ListLong) + "\n")
}
func (s *statefulOutput) Print(prefix string, node *restic.Node) {
func (s *statefulOutput) Print(path string, node *restic.Node) {
if s.JSON {
s.PrintJSON(prefix, node)
s.PrintJSON(path, node)
} else {
s.PrintNormal(prefix, node)
s.PrintNormal(path, node)
}
}
@ -174,74 +175,75 @@ func (s *statefulOutput) Finish() {
// Finder bundles information needed to find a file or directory.
type Finder struct {
repo restic.Repository
pat findPattern
out statefulOutput
notfound restic.IDSet
repo restic.Repository
pat findPattern
out statefulOutput
ignoreTrees restic.IDSet
}
func (f *Finder) findInTree(ctx context.Context, treeID restic.ID, prefix string) error {
if f.notfound.Has(treeID) {
debug.Log("%v skipping tree %v, has already been checked", prefix, treeID)
return nil
func (f *Finder) findInSnapshot(ctx context.Context, sn *restic.Snapshot) error {
debug.Log("searching in snapshot %s\n for entries within [%s %s]", sn.ID(), f.pat.oldest, f.pat.newest)
if sn.Tree == nil {
return errors.Errorf("snapshot %v has no tree", sn.ID().Str())
}
debug.Log("%v checking tree %v\n", prefix, treeID)
f.out.newsn = sn
return walker.Walk(ctx, f.repo, *sn.Tree, f.ignoreTrees, func(nodepath string, node *restic.Node, err error) (bool, error) {
if err != nil {
return false, err
}
tree, err := f.repo.LoadTree(ctx, treeID)
if err != nil {
return err
}
var found bool
for _, node := range tree.Nodes {
debug.Log(" testing entry %q\n", node.Name)
if node == nil {
return false, nil
}
name := node.Name
if f.pat.ignoreCase {
name = strings.ToLower(name)
}
m, err := filepath.Match(f.pat.pattern, name)
foundMatch, err := filter.Match(f.pat.pattern, nodepath)
if err != nil {
return err
}
if m {
if !f.pat.oldest.IsZero() && node.ModTime.Before(f.pat.oldest) {
debug.Log(" ModTime is older than %s\n", f.pat.oldest)
continue
}
if !f.pat.newest.IsZero() && node.ModTime.After(f.pat.newest) {
debug.Log(" ModTime is newer than %s\n", f.pat.newest)
continue
}
debug.Log(" found match\n")
found = true
f.out.Print(prefix, node)
return false, err
}
var (
ignoreIfNoMatch = true
errIfNoMatch error
)
if node.Type == "dir" {
if err := f.findInTree(ctx, *node.Subtree, filepath.Join(prefix, node.Name)); err != nil {
return err
childMayMatch, err := filter.ChildMatch(f.pat.pattern, nodepath)
if err != nil {
return false, err
}
if !childMayMatch {
ignoreIfNoMatch = true
errIfNoMatch = walker.SkipNode
} else {
ignoreIfNoMatch = false
}
}
}
if !found {
f.notfound.Insert(treeID)
}
if !foundMatch {
return ignoreIfNoMatch, errIfNoMatch
}
return nil
}
if !f.pat.oldest.IsZero() && node.ModTime.Before(f.pat.oldest) {
debug.Log(" ModTime is older than %s\n", f.pat.oldest)
return ignoreIfNoMatch, errIfNoMatch
}
func (f *Finder) findInSnapshot(ctx context.Context, sn *restic.Snapshot) error {
debug.Log("searching in snapshot %s\n for entries within [%s %s]", sn.ID(), f.pat.oldest, f.pat.newest)
if !f.pat.newest.IsZero() && node.ModTime.After(f.pat.newest) {
debug.Log(" ModTime is newer than %s\n", f.pat.newest)
return ignoreIfNoMatch, errIfNoMatch
}
f.out.newsn = sn
return f.findInTree(ctx, *sn.Tree, string(filepath.Separator))
debug.Log(" found match\n")
f.out.Print(nodepath, node)
return false, nil
})
}
func runFind(opts FindOptions, gopts GlobalOptions, args []string) error {
@ -289,10 +291,10 @@ func runFind(opts FindOptions, gopts GlobalOptions, args []string) error {
defer cancel()
f := &Finder{
repo: repo,
pat: pat,
out: statefulOutput{ListLong: opts.ListLong, JSON: globalOptions.JSON},
notfound: restic.NewIDSet(),
repo: repo,
pat: pat,
out: statefulOutput{ListLong: opts.ListLong, JSON: globalOptions.JSON},
ignoreTrees: restic.NewIDSet(),
}
for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, opts.Snapshots) {
if err = f.findInSnapshot(ctx, sn); err != nil {

36
cmd/restic/cmd_ls.go
View File

@ -2,13 +2,12 @@ package main
import (
"context"
"path/filepath"
"github.com/spf13/cobra"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/walker"
)
var cmdLs = &cobra.Command{
@ -46,26 +45,6 @@ func init() {
flags.StringArrayVar(&lsOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`, when no snapshot ID is given")
}
func printTree(ctx context.Context, repo *repository.Repository, id *restic.ID, prefix string) error {
tree, err := repo.LoadTree(ctx, *id)
if err != nil {
return err
}
for _, entry := range tree.Nodes {
Printf("%s\n", formatNode(prefix, entry, lsOptions.ListLong))
if entry.Type == "dir" && entry.Subtree != nil {
entryPath := prefix + string(filepath.Separator) + entry.Name
if err = printTree(ctx, repo, entry.Subtree, entryPath); err != nil {
return err
}
}
}
return nil
}
func runLs(opts LsOptions, gopts GlobalOptions, args []string) error {
if len(args) == 0 && opts.Host == "" && len(opts.Tags) == 0 && len(opts.Paths) == 0 {
return errors.Fatal("Invalid arguments, either give one or more snapshot IDs or set filters.")
@ -85,7 +64,18 @@ func runLs(opts LsOptions, gopts GlobalOptions, args []string) error {
for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, args) {
Verbosef("snapshot %s of %v at %s):\n", sn.ID().Str(), sn.Paths, sn.Time)
if err = printTree(gopts.ctx, repo, sn.Tree, ""); err != nil {
err := walker.Walk(ctx, repo, *sn.Tree, nil, func(nodepath string, node *restic.Node, err error) (bool, error) {
if err != nil {
return false, err
}
if node == nil {
return false, nil
}
Printf("%s\n", formatNode(nodepath, node, lsOptions.ListLong))
return false, nil
})
if err != nil {
return err
}
}

8
cmd/restic/format.go
View File

@ -3,7 +3,6 @@ package main
import (
"fmt"
"os"
"path/filepath"
"time"
"github.com/restic/restic/internal/restic"
@ -63,10 +62,9 @@ func formatDuration(d time.Duration) string {
return formatSeconds(sec)
}
func formatNode(prefix string, n *restic.Node, long bool) string {
nodepath := prefix + string(filepath.Separator) + n.Name
func formatNode(path string, n *restic.Node, long bool) string {
if !long {
return nodepath
return path
}
var mode os.FileMode
@ -92,6 +90,6 @@ func formatNode(prefix string, n *restic.Node, long bool) string {
return fmt.Sprintf("%s %5d %5d %6d %s %s%s",
mode|n.Mode, n.UID, n.GID, n.Size,
n.ModTime.Format(TimeFormat), nodepath,
n.ModTime.Format(TimeFormat), path,
target)
}

8
cmd/restic/integration_test.go
View File

@ -387,23 +387,23 @@ func TestBackupExclude(t *testing.T) {
testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, env.gopts)
snapshots, snapshotID := lastSnapshot(snapshots, loadSnapshotMap(t, env.gopts))
files := testRunLs(t, env.gopts, snapshotID)
rtest.Assert(t, includes(files, filepath.Join(string(filepath.Separator), "testdata", "foo.tar.gz")),
rtest.Assert(t, includes(files, "/testdata/foo.tar.gz"),
"expected file %q in first snapshot, but it's not included", "foo.tar.gz")
opts.Excludes = []string{"*.tar.gz"}
testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, env.gopts)
snapshots, snapshotID = lastSnapshot(snapshots, loadSnapshotMap(t, env.gopts))
files = testRunLs(t, env.gopts, snapshotID)
rtest.Assert(t, !includes(files, filepath.Join(string(filepath.Separator), "testdata", "foo.tar.gz")),
rtest.Assert(t, !includes(files, "/testdata/foo.tar.gz"),
"expected file %q not in first snapshot, but it's included", "foo.tar.gz")
opts.Excludes = []string{"*.tar.gz", "private/secret"}
testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, env.gopts)
_, snapshotID = lastSnapshot(snapshots, loadSnapshotMap(t, env.gopts))
files = testRunLs(t, env.gopts, snapshotID)
rtest.Assert(t, !includes(files, filepath.Join(string(filepath.Separator), "testdata", "foo.tar.gz")),
rtest.Assert(t, !includes(files, "/testdata/foo.tar.gz"),
"expected file %q not in first snapshot, but it's included", "foo.tar.gz")
rtest.Assert(t, !includes(files, filepath.Join(string(filepath.Separator), "testdata", "private", "secret", "passwords.txt")),
rtest.Assert(t, !includes(files, "/testdata/private/secret/passwords.txt"),
"expected file %q not in first snapshot, but it's included", "passwords.txt")
}

6
internal/filter/filter.go
View File

@ -83,6 +83,12 @@ func childMatch(patterns, strs []string) (matched bool, err error) {
return true, nil
}
ok, pos := hasDoubleWildcard(patterns)
if ok && len(strs) >= pos {
// cut off at the double wildcard
strs = strs[:pos]
}
// match path against absolute pattern prefix
l := 0
if len(strs) > len(patterns) {

56
internal/filter/filter_test.go
View File

@ -83,6 +83,8 @@ var matchTests = []struct {
{"foo/**/bar/*.go", "bar/main.go", false},
{"foo/**/bar", "/home/user/foo/x/y/bar", true},
{"foo/**/bar", "/home/user/foo/x/y/bar/main.go", true},
{"foo/**/bar/**/x", "/home/user/foo/bar/x", true},
{"foo/**/bar/**/x", "/home/user/foo/blaaa/blaz/bar/shared/work/x", true},
{"user/**/important*", "/home/user/work/x/y/hidden/x", false},
{"user/**/hidden*/**/c", "/home/user/work/x/y/hidden/z/a/b/c", true},
{"c:/foo/*test.*", "c:/foo/bar/test.go", false},
@ -107,20 +109,28 @@ func testpattern(t *testing.T, pattern, path string, shouldMatch bool) {
func TestMatch(t *testing.T) {
for _, test := range matchTests {
testpattern(t, test.pattern, test.path, test.match)
t.Run("", func(t *testing.T) {
testpattern(t, test.pattern, test.path, test.match)
})
// Test with native path separator
if filepath.Separator != '/' {
// Test with pattern as native
pattern := strings.Replace(test.pattern, "/", string(filepath.Separator), -1)
testpattern(t, pattern, test.path, test.match)
// Test with pattern as native
t.Run("pattern-native", func(t *testing.T) {
testpattern(t, pattern, test.path, test.match)
})
// Test with path as native
path := strings.Replace(test.path, "/", string(filepath.Separator), -1)
testpattern(t, test.pattern, path, test.match)
t.Run("path-native", func(t *testing.T) {
// Test with path as native
testpattern(t, test.pattern, path, test.match)
})
// Test with both pattern and path as native
testpattern(t, pattern, path, test.match)
t.Run("both-native", func(t *testing.T) {
// Test with both pattern and path as native
testpattern(t, pattern, path, test.match)
})
}
}
}
@ -147,6 +157,16 @@ var childMatchTests = []struct {
{"/foo/**/baz", "/foo/bar/baz", true},
{"/foo/**/baz", "/foo/bar/baz/blah", true},
{"/foo/**/qux", "/foo/bar/baz/qux", true},
{"/foo/**/qux", "/foo/bar/baz", true},
{"/foo/**/qux", "/foo/bar/baz/boo", true},
{"/foo/**", "/foo/bar/baz", true},
{"/foo/**", "/foo/bar", true},
{"foo/**/bar/**/x", "/home/user/foo", true},
{"foo/**/bar/**/x", "/home/user/foo/bar", true},
{"foo/**/bar/**/x", "/home/user/foo/blaaa/blaz/bar/shared/work/x", true},
{"/foo/*/qux", "/foo/bar", true},
{"/foo/*/qux", "/foo/bar/boo", false},
{"/foo/*/qux", "/foo/bar/boo/xx", false},
{"/baz/bar", "/foo", false},
{"/foo", "/foo/bar", true},
{"/*", "/foo", true},
@ -179,20 +199,28 @@ func testchildpattern(t *testing.T, pattern, path string, shouldMatch bool) {
func TestChildMatch(t *testing.T) {
for _, test := range childMatchTests {
testchildpattern(t, test.pattern, test.path, test.match)
t.Run("", func(t *testing.T) {
testchildpattern(t, test.pattern, test.path, test.match)
})
// Test with native path separator
if filepath.Separator != '/' {
// Test with pattern as native
pattern := strings.Replace(test.pattern, "/", string(filepath.Separator), -1)
testchildpattern(t, pattern, test.path, test.match)
// Test with pattern as native
t.Run("pattern-native", func(t *testing.T) {
testchildpattern(t, pattern, test.path, test.match)
})
// Test with path as native
path := strings.Replace(test.path, "/", string(filepath.Separator), -1)
testchildpattern(t, test.pattern, path, test.match)
t.Run("path-native", func(t *testing.T) {
// Test with path as native
testchildpattern(t, test.pattern, path, test.match)
})
// Test with both pattern and path as native
testchildpattern(t, pattern, path, test.match)
t.Run("both-native", func(t *testing.T) {
// Test with both pattern and path as native
testchildpattern(t, pattern, path, test.match)
})
}
}
}

1
internal/walker/testing.go Normal file
View File

@ -0,0 +1 @@
package walker

138
internal/walker/walker.go Normal file
View File

@ -0,0 +1,138 @@
package walker
import (
"context"
"path"
"sort"
"github.com/pkg/errors"
"github.com/restic/restic/internal/restic"
)
// TreeLoader loads a tree from a repository.
type TreeLoader interface {
LoadTree(context.Context, restic.ID) (*restic.Tree, error)
}
// SkipNode is returned by WalkFunc when a dir node should not be walked.
var SkipNode = errors.New("skip this node")
// WalkFunc is the type of the function called for each node visited by Walk.
// Path is the slash-separated path from the root node. If there was a problem
// loading a node, err is set to a non-nil error. WalkFunc can chose to ignore
// it by returning nil.
//
// When the special value SkipNode is returned and node is a dir node, it is
// not walked. When the node is not a dir node, the remaining items in this
// tree are skipped.
//
// Setting ignore to true tells Walk that it should not visit the node again.
// For tree nodes, this means that the function is not called for the
// referenced tree. If the node is not a tree, and all nodes in the current
// tree have ignore set to true, the current tree will not be visited again.
// When err is not nil and different from SkipNode, the value returned for
// ignore is ignored.
type WalkFunc func(path string, node *restic.Node, nodeErr error) (ignore bool, err error)
// Walk calls walkFn recursively for each node in root. If walkFn returns an
// error, it is passed up the call stack. The trees in ignoreTrees are not
// walked. If walkFn ignores trees, these are added to the set.
func Walk(ctx context.Context, repo TreeLoader, root restic.ID, ignoreTrees restic.IDSet, walkFn WalkFunc) error {
tree, err := repo.LoadTree(ctx, root)
_, err = walkFn("/", nil, err)
if err != nil {
if err == SkipNode {
err = nil
}
return err
}
if ignoreTrees == nil {
ignoreTrees = restic.NewIDSet()
}
_, err = walk(ctx, repo, "/", tree, ignoreTrees, walkFn)
return err
}
// walk recursively traverses the tree, ignoring subtrees when the ID of the
// subtree is in ignoreTrees. If err is nil and ignore is true, the subtree ID
// will be added to ignoreTrees by walk.
func walk(ctx context.Context, repo TreeLoader, prefix string, tree *restic.Tree, ignoreTrees restic.IDSet, walkFn WalkFunc) (ignore bool, err error) {
var allNodesIgnored = true
sort.Slice(tree.Nodes, func(i, j int) bool {
return tree.Nodes[i].Name < tree.Nodes[j].Name
})
for _, node := range tree.Nodes {
p := path.Join(prefix, node.Name)
if node.Type == "" {
return false, errors.Errorf("node type is empty for node %q", node.Name)
}
if node.Type != "dir" {
ignore, err := walkFn(p, node, nil)
if err != nil {
if err == SkipNode {
// skip the remaining entries in this tree
return allNodesIgnored, nil
}
return false, err
}
if ignore == false {
allNodesIgnored = false
}
continue
}
if node.Subtree == nil {
return false, errors.Errorf("subtree for node %v in tree %v is nil", node.Name, p)
}
if ignoreTrees.Has(*node.Subtree) {
continue
}
subtree, err := repo.LoadTree(ctx, *node.Subtree)
ignore, err := walkFn(p, node, err)
if err != nil {
if err == SkipNode {
if ignore {
ignoreTrees.Insert(*node.Subtree)
}
continue
}
return false, err
}
if ignore {
ignoreTrees.Insert(*node.Subtree)
}
if !ignore {
allNodesIgnored = false
}
ignore, err = walk(ctx, repo, p, subtree, ignoreTrees, walkFn)
if err != nil {
return false, err
}
if ignore {
ignoreTrees.Insert(*node.Subtree)
}
if !ignore {
allNodesIgnored = false
}
}
return allNodesIgnored, nil
}

423
internal/walker/walker_test.go Normal file
View File

@ -0,0 +1,423 @@
package walker
import (
"context"
"encoding/json"
"fmt"
"testing"
"github.com/pkg/errors"
"github.com/restic/restic/internal/restic"
)
// TestTree is used to construct a list of trees for testing the walker.
type TestTree map[string]interface{}
// TestNode is used to test the walker.
type TestFile struct{}
func BuildTreeMap(tree TestTree) (m TreeMap, root restic.ID) {
m = TreeMap{}
id := buildTreeMap(tree, m)
return m, id
}
func buildTreeMap(tree TestTree, m TreeMap) restic.ID {
res := restic.NewTree()
for name, item := range tree {
switch elem := item.(type) {
case TestFile:
res.Insert(&restic.Node{
Name: name,
Type: "file",
})
case TestTree:
id := buildTreeMap(elem, m)
res.Insert(&restic.Node{
Name: name,
Subtree: &id,
Type: "dir",
})
default:
panic(fmt.Sprintf("invalid type %T", elem))
}
}
buf, err := json.Marshal(res)
if err != nil {
panic(err)
}
id := restic.Hash(buf)
if _, ok := m[id]; !ok {
m[id] = res
}
return id
}
// TreeMap returns the trees from the map on LoadTree.
type TreeMap map[restic.ID]*restic.Tree
func (t TreeMap) LoadTree(ctx context.Context, id restic.ID) (*restic.Tree, error) {
tree, ok := t[id]
if !ok {
return nil, errors.New("tree not found")
}
return tree, nil
}
// checkFunc returns a function suitable for walking the tree to check
// something, and a function which will check the final result.
type checkFunc func(t testing.TB) (walker WalkFunc, final func(testing.TB))
// checkItemOrder ensures that the order of the 'path' arguments is the one passed in as 'want'.
func checkItemOrder(want []string) checkFunc {
pos := 0
return func(t testing.TB) (walker WalkFunc, final func(testing.TB)) {
walker = func(path string, node *restic.Node, err error) (bool, error) {
if err != nil {
t.Errorf("error walking %v: %v", path, err)
return false, err
}
if pos >= len(want) {
t.Errorf("additional unexpected path found: %v", path)
return false, nil
}
if path != want[pos] {
t.Errorf("wrong path found, want %q, got %q", want[pos], path)
}
pos++
return false, nil
}
final = func(t testing.TB) {
if pos != len(want) {
t.Errorf("not enough items returned, want %d, got %d", len(want), pos)
}
}
return walker, final
}
}
// checkSkipFor returns SkipNode if path is in skipFor, it checks that the
// paths the walk func is called for are exactly the ones in wantPaths.
func checkSkipFor(skipFor map[string]struct{}, wantPaths []string) checkFunc {
var pos int
return func(t testing.TB) (walker WalkFunc, final func(testing.TB)) {
walker = func(path string, node *restic.Node, err error) (bool, error) {
if err != nil {
t.Errorf("error walking %v: %v", path, err)
return false, err
}
if pos >= len(wantPaths) {
t.Errorf("additional unexpected path found: %v", path)
return false, nil
}
if path != wantPaths[pos] {
t.Errorf("wrong path found, want %q, got %q", wantPaths[pos], path)
}
pos++
if _, ok := skipFor[path]; ok {
return false, SkipNode
}
return false, nil
}
final = func(t testing.TB) {
if pos != len(wantPaths) {
t.Errorf("wrong number of paths returned, want %d, got %d", len(wantPaths), pos)
}
}
return walker, final
}
}
// checkIgnore returns SkipNode if path is in skipFor and sets ignore according
// to ignoreFor. It checks that the paths the walk func is called for are exactly
// the ones in wantPaths.
func checkIgnore(skipFor map[string]struct{}, ignoreFor map[string]bool, wantPaths []string) checkFunc {
var pos int
return func(t testing.TB) (walker WalkFunc, final func(testing.TB)) {
walker = func(path string, node *restic.Node, err error) (bool, error) {
if err != nil {
t.Errorf("error walking %v: %v", path, err)
return false, err
}
if pos >= len(wantPaths) {
t.Errorf("additional unexpected path found: %v", path)
return ignoreFor[path], nil
}
if path != wantPaths[pos] {
t.Errorf("wrong path found, want %q, got %q", wantPaths[pos], path)
}
pos++
if _, ok := skipFor[path]; ok {
return ignoreFor[path], SkipNode
}
return ignoreFor[path], nil
}
final = func(t testing.TB) {
if pos != len(wantPaths) {
t.Errorf("wrong number of paths returned, want %d, got %d", len(wantPaths), pos)
}
}
return walker, final
}
}
func TestWalker(t *testing.T) {
var tests = []struct {
tree TestTree
checks []checkFunc
}{
{
tree: TestTree{
"foo": TestFile{},
"subdir": TestTree{
"subfile": TestFile{},
},
},
checks: []checkFunc{
checkItemOrder([]string{
"/",
"/foo",
"/subdir",
"/subdir/subfile",
}),
checkSkipFor(
map[string]struct{}{
"/subdir": struct{}{},
}, []string{
"/",
"/foo",
"/subdir",
},
),
checkIgnore(
map[string]struct{}{}, map[string]bool{
"/subdir": true,
}, []string{
"/",
"/foo",
"/subdir",
"/subdir/subfile",
},
),
},
},
{
tree: TestTree{
"foo": TestFile{},
"subdir1": TestTree{
"subfile1": TestFile{},
},
"subdir2": TestTree{
"subfile2": TestFile{},
"subsubdir2": TestTree{
"subsubfile3": TestFile{},
},
},
},
checks: []checkFunc{
checkItemOrder([]string{
"/",
"/foo",
"/subdir1",
"/subdir1/subfile1",
"/subdir2",
"/subdir2/subfile2",
"/subdir2/subsubdir2",
"/subdir2/subsubdir2/subsubfile3",
}),
checkSkipFor(
map[string]struct{}{
"/subdir1": struct{}{},
}, []string{
"/",
"/foo",
"/subdir1",
"/subdir2",
"/subdir2/subfile2",
"/subdir2/subsubdir2",
"/subdir2/subsubdir2/subsubfile3",
},
),
checkSkipFor(
map[string]struct{}{
"/subdir1": struct{}{},
"/subdir2/subsubdir2": struct{}{},
}, []string{
"/",
"/foo",
"/subdir1",
"/subdir2",
"/subdir2/subfile2",
"/subdir2/subsubdir2",
},
),
checkSkipFor(
map[string]struct{}{
"/foo": struct{}{},
}, []string{
"/",
"/foo",
},
),
},
},
{
tree: TestTree{
"foo": TestFile{},
"subdir1": TestTree{
"subfile1": TestFile{},
"subfile2": TestFile{},
"subfile3": TestFile{},
},
"subdir2": TestTree{
"subfile1": TestFile{},
"subfile2": TestFile{},
"subfile3": TestFile{},
},
"subdir3": TestTree{
"subfile1": TestFile{},
"subfile2": TestFile{},
"subfile3": TestFile{},
},
"zzz other": TestFile{},
},
checks: []checkFunc{
checkItemOrder([]string{
"/",
"/foo",
"/subdir1",
"/subdir1/subfile1",
"/subdir1/subfile2",
"/subdir1/subfile3",
"/subdir2",
"/subdir2/subfile1",
"/subdir2/subfile2",
"/subdir2/subfile3",
"/subdir3",
"/subdir3/subfile1",
"/subdir3/subfile2",
"/subdir3/subfile3",
"/zzz other",
}),
checkIgnore(
map[string]struct{}{
"/subdir1": struct{}{},
}, map[string]bool{
"/subdir1": true,
}, []string{
"/",
"/foo",
"/subdir1",
"/zzz other",
},
),
checkIgnore(
map[string]struct{}{}, map[string]bool{
"/subdir1": true,
}, []string{
"/",
"/foo",
"/subdir1",
"/subdir1/subfile1",
"/subdir1/subfile2",
"/subdir1/subfile3",
"/zzz other",
},
),
checkIgnore(
map[string]struct{}{
"/subdir2": struct{}{},
}, map[string]bool{
"/subdir2": true,
}, []string{
"/",
"/foo",
"/subdir1",
"/subdir1/subfile1",
"/subdir1/subfile2",
"/subdir1/subfile3",
"/subdir2",
"/zzz other",
},
),
checkIgnore(
map[string]struct{}{}, map[string]bool{
"/subdir1/subfile1": true,
"/subdir1/subfile2": true,
"/subdir1/subfile3": true,
}, []string{
"/",
"/foo",
"/subdir1",
"/subdir1/subfile1",
"/subdir1/subfile2",
"/subdir1/subfile3",
"/zzz other",
},
),
checkIgnore(
map[string]struct{}{}, map[string]bool{
"/subdir2/subfile1": true,
"/subdir2/subfile2": true,
"/subdir2/subfile3": true,
}, []string{
"/",
"/foo",
"/subdir1",
"/subdir1/subfile1",
"/subdir1/subfile2",
"/subdir1/subfile3",
"/subdir2",
"/subdir2/subfile1",
"/subdir2/subfile2",
"/subdir2/subfile3",
"/zzz other",
},
),
},
},
}
for _, test := range tests {
t.Run("", func(t *testing.T) {
repo, root := BuildTreeMap(test.tree)
for _, check := range test.checks {
t.Run("", func(t *testing.T) {
ctx, cancel := context.WithCancel(context.TODO())
defer cancel()
fn, last := check(t)
err := Walk(ctx, repo, root, restic.NewIDSet(), fn)
if err != nil {
t.Error(err)
}
last(t)
})
}
})
}
}