diff --git a/changelog/unreleased/issue-1825 b/changelog/unreleased/issue-1825 new file mode 100644 index 000000000..017d09162 --- /dev/null +++ b/changelog/unreleased/issue-1825 @@ -0,0 +1,12 @@ +Bugfix: Correct `find` to not skip snapshots + +Under certain circumstances, the `find` command was found to skip snapshots +containing directories with files to look for when the directories haven't been +modified at all, and were already printed as part of a different snapshot. This +is now corrected. + +In addition, we've switched to our own matching/pattern implementation, so now +things like `restic find "/home/user/foo/**/main.go"` are possible. + +https://github.com/restic/restic/issues/1825 +https://github.com/restic/restic/issues/1823 diff --git a/cmd/restic/cmd_backup.go b/cmd/restic/cmd_backup.go index 487d8d587..dc84ed280 100644 --- a/cmd/restic/cmd_backup.go +++ b/cmd/restic/cmd_backup.go @@ -336,6 +336,14 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina return err } + timeStamp := time.Now() + if opts.TimeStamp != "" { + timeStamp, err = time.Parse(TimeFormat, opts.TimeStamp) + if err != nil { + return errors.Fatalf("error in time option: %v\n", err) + } + } + var t tomb.Tomb p := ui.NewBackup(term, gopts.verbosity) @@ -402,14 +410,6 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina return true } - timeStamp := time.Now() - if opts.TimeStamp != "" { - timeStamp, err = time.Parse(TimeFormat, opts.TimeStamp) - if err != nil { - return errors.Fatalf("error in time option: %v\n", err) - } - } - var targetFS fs.FS = fs.Local{} if opts.Stdin { p.V("read data from stdin") diff --git a/cmd/restic/cmd_find.go b/cmd/restic/cmd_find.go index b2a67da5d..d21453859 100644 --- a/cmd/restic/cmd_find.go +++ b/cmd/restic/cmd_find.go @@ -3,7 +3,6 @@ package main import ( "context" "encoding/json" - "path/filepath" "strings" "time" @@ -11,7 +10,9 @@ import ( "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/filter" "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/walker" ) var cmdFind = &cobra.Command{ @@ -94,7 +95,7 @@ type statefulOutput struct { hits int } -func (s *statefulOutput) PrintJSON(prefix string, node *restic.Node) { +func (s *statefulOutput) PrintJSON(path string, node *restic.Node) { type findNode restic.Node b, err := json.Marshal(struct { // Add these attributes @@ -111,7 +112,7 @@ func (s *statefulOutput) PrintJSON(prefix string, node *restic.Node) { Content byte `json:"content,omitempty"` Subtree byte `json:"subtree,omitempty"` }{ - Path: filepath.Join(prefix, node.Name), + Path: path, Permissions: node.Mode.String(), findNode: (*findNode)(node), }) @@ -138,22 +139,22 @@ func (s *statefulOutput) PrintJSON(prefix string, node *restic.Node) { s.hits++ } -func (s *statefulOutput) PrintNormal(prefix string, node *restic.Node) { +func (s *statefulOutput) PrintNormal(path string, node *restic.Node) { if s.newsn != s.oldsn { if s.oldsn != nil { Verbosef("\n") } s.oldsn = s.newsn - Verbosef("Found matching entries in snapshot %s\n", s.oldsn.ID()) + Verbosef("Found matching entries in snapshot %s\n", s.oldsn.ID().Str()) } - Printf(formatNode(prefix, node, s.ListLong) + "\n") + Printf(formatNode(path, node, s.ListLong) + "\n") } -func (s *statefulOutput) Print(prefix string, node *restic.Node) { +func (s *statefulOutput) Print(path string, node *restic.Node) { if s.JSON { - s.PrintJSON(prefix, node) + s.PrintJSON(path, node) } else { - s.PrintNormal(prefix, node) + s.PrintNormal(path, node) } } @@ -174,74 +175,75 @@ func (s *statefulOutput) Finish() { // Finder bundles information needed to find a file or directory. type Finder struct { - repo restic.Repository - pat findPattern - out statefulOutput - notfound restic.IDSet + repo restic.Repository + pat findPattern + out statefulOutput + ignoreTrees restic.IDSet } -func (f *Finder) findInTree(ctx context.Context, treeID restic.ID, prefix string) error { - if f.notfound.Has(treeID) { - debug.Log("%v skipping tree %v, has already been checked", prefix, treeID) - return nil +func (f *Finder) findInSnapshot(ctx context.Context, sn *restic.Snapshot) error { + debug.Log("searching in snapshot %s\n for entries within [%s %s]", sn.ID(), f.pat.oldest, f.pat.newest) + + if sn.Tree == nil { + return errors.Errorf("snapshot %v has no tree", sn.ID().Str()) } - debug.Log("%v checking tree %v\n", prefix, treeID) + f.out.newsn = sn + return walker.Walk(ctx, f.repo, *sn.Tree, f.ignoreTrees, func(nodepath string, node *restic.Node, err error) (bool, error) { + if err != nil { + return false, err + } - tree, err := f.repo.LoadTree(ctx, treeID) - if err != nil { - return err - } - - var found bool - for _, node := range tree.Nodes { - debug.Log(" testing entry %q\n", node.Name) + if node == nil { + return false, nil + } name := node.Name if f.pat.ignoreCase { name = strings.ToLower(name) } - m, err := filepath.Match(f.pat.pattern, name) + foundMatch, err := filter.Match(f.pat.pattern, nodepath) if err != nil { - return err - } - - if m { - if !f.pat.oldest.IsZero() && node.ModTime.Before(f.pat.oldest) { - debug.Log(" ModTime is older than %s\n", f.pat.oldest) - continue - } - - if !f.pat.newest.IsZero() && node.ModTime.After(f.pat.newest) { - debug.Log(" ModTime is newer than %s\n", f.pat.newest) - continue - } - - debug.Log(" found match\n") - found = true - f.out.Print(prefix, node) + return false, err } + var ( + ignoreIfNoMatch = true + errIfNoMatch error + ) if node.Type == "dir" { - if err := f.findInTree(ctx, *node.Subtree, filepath.Join(prefix, node.Name)); err != nil { - return err + childMayMatch, err := filter.ChildMatch(f.pat.pattern, nodepath) + if err != nil { + return false, err + } + + if !childMayMatch { + ignoreIfNoMatch = true + errIfNoMatch = walker.SkipNode + } else { + ignoreIfNoMatch = false } } - } - if !found { - f.notfound.Insert(treeID) - } + if !foundMatch { + return ignoreIfNoMatch, errIfNoMatch + } - return nil -} + if !f.pat.oldest.IsZero() && node.ModTime.Before(f.pat.oldest) { + debug.Log(" ModTime is older than %s\n", f.pat.oldest) + return ignoreIfNoMatch, errIfNoMatch + } -func (f *Finder) findInSnapshot(ctx context.Context, sn *restic.Snapshot) error { - debug.Log("searching in snapshot %s\n for entries within [%s %s]", sn.ID(), f.pat.oldest, f.pat.newest) + if !f.pat.newest.IsZero() && node.ModTime.After(f.pat.newest) { + debug.Log(" ModTime is newer than %s\n", f.pat.newest) + return ignoreIfNoMatch, errIfNoMatch + } - f.out.newsn = sn - return f.findInTree(ctx, *sn.Tree, string(filepath.Separator)) + debug.Log(" found match\n") + f.out.Print(nodepath, node) + return false, nil + }) } func runFind(opts FindOptions, gopts GlobalOptions, args []string) error { @@ -289,10 +291,10 @@ func runFind(opts FindOptions, gopts GlobalOptions, args []string) error { defer cancel() f := &Finder{ - repo: repo, - pat: pat, - out: statefulOutput{ListLong: opts.ListLong, JSON: globalOptions.JSON}, - notfound: restic.NewIDSet(), + repo: repo, + pat: pat, + out: statefulOutput{ListLong: opts.ListLong, JSON: globalOptions.JSON}, + ignoreTrees: restic.NewIDSet(), } for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, opts.Snapshots) { if err = f.findInSnapshot(ctx, sn); err != nil { diff --git a/cmd/restic/cmd_ls.go b/cmd/restic/cmd_ls.go index d4a768d70..f7996b438 100644 --- a/cmd/restic/cmd_ls.go +++ b/cmd/restic/cmd_ls.go @@ -2,13 +2,12 @@ package main import ( "context" - "path/filepath" "github.com/spf13/cobra" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/walker" ) var cmdLs = &cobra.Command{ @@ -46,26 +45,6 @@ func init() { flags.StringArrayVar(&lsOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`, when no snapshot ID is given") } -func printTree(ctx context.Context, repo *repository.Repository, id *restic.ID, prefix string) error { - tree, err := repo.LoadTree(ctx, *id) - if err != nil { - return err - } - - for _, entry := range tree.Nodes { - Printf("%s\n", formatNode(prefix, entry, lsOptions.ListLong)) - - if entry.Type == "dir" && entry.Subtree != nil { - entryPath := prefix + string(filepath.Separator) + entry.Name - if err = printTree(ctx, repo, entry.Subtree, entryPath); err != nil { - return err - } - } - } - - return nil -} - func runLs(opts LsOptions, gopts GlobalOptions, args []string) error { if len(args) == 0 && opts.Host == "" && len(opts.Tags) == 0 && len(opts.Paths) == 0 { return errors.Fatal("Invalid arguments, either give one or more snapshot IDs or set filters.") @@ -85,7 +64,18 @@ func runLs(opts LsOptions, gopts GlobalOptions, args []string) error { for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, args) { Verbosef("snapshot %s of %v at %s):\n", sn.ID().Str(), sn.Paths, sn.Time) - if err = printTree(gopts.ctx, repo, sn.Tree, ""); err != nil { + err := walker.Walk(ctx, repo, *sn.Tree, nil, func(nodepath string, node *restic.Node, err error) (bool, error) { + if err != nil { + return false, err + } + + if node == nil { + return false, nil + } + Printf("%s\n", formatNode(nodepath, node, lsOptions.ListLong)) + return false, nil + }) + if err != nil { return err } } diff --git a/cmd/restic/format.go b/cmd/restic/format.go index 1f8ab366e..1de0335c9 100644 --- a/cmd/restic/format.go +++ b/cmd/restic/format.go @@ -3,7 +3,6 @@ package main import ( "fmt" "os" - "path/filepath" "time" "github.com/restic/restic/internal/restic" @@ -63,10 +62,9 @@ func formatDuration(d time.Duration) string { return formatSeconds(sec) } -func formatNode(prefix string, n *restic.Node, long bool) string { - nodepath := prefix + string(filepath.Separator) + n.Name +func formatNode(path string, n *restic.Node, long bool) string { if !long { - return nodepath + return path } var mode os.FileMode @@ -92,6 +90,6 @@ func formatNode(prefix string, n *restic.Node, long bool) string { return fmt.Sprintf("%s %5d %5d %6d %s %s%s", mode|n.Mode, n.UID, n.GID, n.Size, - n.ModTime.Format(TimeFormat), nodepath, + n.ModTime.Format(TimeFormat), path, target) } diff --git a/cmd/restic/integration_test.go b/cmd/restic/integration_test.go index 8ccf28b1e..c16097b97 100644 --- a/cmd/restic/integration_test.go +++ b/cmd/restic/integration_test.go @@ -387,23 +387,23 @@ func TestBackupExclude(t *testing.T) { testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, env.gopts) snapshots, snapshotID := lastSnapshot(snapshots, loadSnapshotMap(t, env.gopts)) files := testRunLs(t, env.gopts, snapshotID) - rtest.Assert(t, includes(files, filepath.Join(string(filepath.Separator), "testdata", "foo.tar.gz")), + rtest.Assert(t, includes(files, "/testdata/foo.tar.gz"), "expected file %q in first snapshot, but it's not included", "foo.tar.gz") opts.Excludes = []string{"*.tar.gz"} testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, env.gopts) snapshots, snapshotID = lastSnapshot(snapshots, loadSnapshotMap(t, env.gopts)) files = testRunLs(t, env.gopts, snapshotID) - rtest.Assert(t, !includes(files, filepath.Join(string(filepath.Separator), "testdata", "foo.tar.gz")), + rtest.Assert(t, !includes(files, "/testdata/foo.tar.gz"), "expected file %q not in first snapshot, but it's included", "foo.tar.gz") opts.Excludes = []string{"*.tar.gz", "private/secret"} testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, env.gopts) _, snapshotID = lastSnapshot(snapshots, loadSnapshotMap(t, env.gopts)) files = testRunLs(t, env.gopts, snapshotID) - rtest.Assert(t, !includes(files, filepath.Join(string(filepath.Separator), "testdata", "foo.tar.gz")), + rtest.Assert(t, !includes(files, "/testdata/foo.tar.gz"), "expected file %q not in first snapshot, but it's included", "foo.tar.gz") - rtest.Assert(t, !includes(files, filepath.Join(string(filepath.Separator), "testdata", "private", "secret", "passwords.txt")), + rtest.Assert(t, !includes(files, "/testdata/private/secret/passwords.txt"), "expected file %q not in first snapshot, but it's included", "passwords.txt") } diff --git a/internal/filter/filter.go b/internal/filter/filter.go index 0cc9512d4..74deddb03 100644 --- a/internal/filter/filter.go +++ b/internal/filter/filter.go @@ -83,6 +83,12 @@ func childMatch(patterns, strs []string) (matched bool, err error) { return true, nil } + ok, pos := hasDoubleWildcard(patterns) + if ok && len(strs) >= pos { + // cut off at the double wildcard + strs = strs[:pos] + } + // match path against absolute pattern prefix l := 0 if len(strs) > len(patterns) { diff --git a/internal/filter/filter_test.go b/internal/filter/filter_test.go index bd2070e56..97df452fb 100644 --- a/internal/filter/filter_test.go +++ b/internal/filter/filter_test.go @@ -83,6 +83,8 @@ var matchTests = []struct { {"foo/**/bar/*.go", "bar/main.go", false}, {"foo/**/bar", "/home/user/foo/x/y/bar", true}, {"foo/**/bar", "/home/user/foo/x/y/bar/main.go", true}, + {"foo/**/bar/**/x", "/home/user/foo/bar/x", true}, + {"foo/**/bar/**/x", "/home/user/foo/blaaa/blaz/bar/shared/work/x", true}, {"user/**/important*", "/home/user/work/x/y/hidden/x", false}, {"user/**/hidden*/**/c", "/home/user/work/x/y/hidden/z/a/b/c", true}, {"c:/foo/*test.*", "c:/foo/bar/test.go", false}, @@ -107,20 +109,28 @@ func testpattern(t *testing.T, pattern, path string, shouldMatch bool) { func TestMatch(t *testing.T) { for _, test := range matchTests { - testpattern(t, test.pattern, test.path, test.match) + t.Run("", func(t *testing.T) { + testpattern(t, test.pattern, test.path, test.match) + }) // Test with native path separator if filepath.Separator != '/' { - // Test with pattern as native pattern := strings.Replace(test.pattern, "/", string(filepath.Separator), -1) - testpattern(t, pattern, test.path, test.match) + // Test with pattern as native + t.Run("pattern-native", func(t *testing.T) { + testpattern(t, pattern, test.path, test.match) + }) - // Test with path as native path := strings.Replace(test.path, "/", string(filepath.Separator), -1) - testpattern(t, test.pattern, path, test.match) + t.Run("path-native", func(t *testing.T) { + // Test with path as native + testpattern(t, test.pattern, path, test.match) + }) - // Test with both pattern and path as native - testpattern(t, pattern, path, test.match) + t.Run("both-native", func(t *testing.T) { + // Test with both pattern and path as native + testpattern(t, pattern, path, test.match) + }) } } } @@ -147,6 +157,16 @@ var childMatchTests = []struct { {"/foo/**/baz", "/foo/bar/baz", true}, {"/foo/**/baz", "/foo/bar/baz/blah", true}, {"/foo/**/qux", "/foo/bar/baz/qux", true}, + {"/foo/**/qux", "/foo/bar/baz", true}, + {"/foo/**/qux", "/foo/bar/baz/boo", true}, + {"/foo/**", "/foo/bar/baz", true}, + {"/foo/**", "/foo/bar", true}, + {"foo/**/bar/**/x", "/home/user/foo", true}, + {"foo/**/bar/**/x", "/home/user/foo/bar", true}, + {"foo/**/bar/**/x", "/home/user/foo/blaaa/blaz/bar/shared/work/x", true}, + {"/foo/*/qux", "/foo/bar", true}, + {"/foo/*/qux", "/foo/bar/boo", false}, + {"/foo/*/qux", "/foo/bar/boo/xx", false}, {"/baz/bar", "/foo", false}, {"/foo", "/foo/bar", true}, {"/*", "/foo", true}, @@ -179,20 +199,28 @@ func testchildpattern(t *testing.T, pattern, path string, shouldMatch bool) { func TestChildMatch(t *testing.T) { for _, test := range childMatchTests { - testchildpattern(t, test.pattern, test.path, test.match) + t.Run("", func(t *testing.T) { + testchildpattern(t, test.pattern, test.path, test.match) + }) // Test with native path separator if filepath.Separator != '/' { - // Test with pattern as native pattern := strings.Replace(test.pattern, "/", string(filepath.Separator), -1) - testchildpattern(t, pattern, test.path, test.match) + // Test with pattern as native + t.Run("pattern-native", func(t *testing.T) { + testchildpattern(t, pattern, test.path, test.match) + }) - // Test with path as native path := strings.Replace(test.path, "/", string(filepath.Separator), -1) - testchildpattern(t, test.pattern, path, test.match) + t.Run("path-native", func(t *testing.T) { + // Test with path as native + testchildpattern(t, test.pattern, path, test.match) + }) - // Test with both pattern and path as native - testchildpattern(t, pattern, path, test.match) + t.Run("both-native", func(t *testing.T) { + // Test with both pattern and path as native + testchildpattern(t, pattern, path, test.match) + }) } } } diff --git a/internal/walker/testing.go b/internal/walker/testing.go new file mode 100644 index 000000000..c06778242 --- /dev/null +++ b/internal/walker/testing.go @@ -0,0 +1 @@ +package walker diff --git a/internal/walker/walker.go b/internal/walker/walker.go new file mode 100644 index 000000000..b679cbce0 --- /dev/null +++ b/internal/walker/walker.go @@ -0,0 +1,138 @@ +package walker + +import ( + "context" + "path" + "sort" + + "github.com/pkg/errors" + + "github.com/restic/restic/internal/restic" +) + +// TreeLoader loads a tree from a repository. +type TreeLoader interface { + LoadTree(context.Context, restic.ID) (*restic.Tree, error) +} + +// SkipNode is returned by WalkFunc when a dir node should not be walked. +var SkipNode = errors.New("skip this node") + +// WalkFunc is the type of the function called for each node visited by Walk. +// Path is the slash-separated path from the root node. If there was a problem +// loading a node, err is set to a non-nil error. WalkFunc can chose to ignore +// it by returning nil. +// +// When the special value SkipNode is returned and node is a dir node, it is +// not walked. When the node is not a dir node, the remaining items in this +// tree are skipped. +// +// Setting ignore to true tells Walk that it should not visit the node again. +// For tree nodes, this means that the function is not called for the +// referenced tree. If the node is not a tree, and all nodes in the current +// tree have ignore set to true, the current tree will not be visited again. +// When err is not nil and different from SkipNode, the value returned for +// ignore is ignored. +type WalkFunc func(path string, node *restic.Node, nodeErr error) (ignore bool, err error) + +// Walk calls walkFn recursively for each node in root. If walkFn returns an +// error, it is passed up the call stack. The trees in ignoreTrees are not +// walked. If walkFn ignores trees, these are added to the set. +func Walk(ctx context.Context, repo TreeLoader, root restic.ID, ignoreTrees restic.IDSet, walkFn WalkFunc) error { + tree, err := repo.LoadTree(ctx, root) + _, err = walkFn("/", nil, err) + + if err != nil { + if err == SkipNode { + err = nil + } + return err + } + + if ignoreTrees == nil { + ignoreTrees = restic.NewIDSet() + } + + _, err = walk(ctx, repo, "/", tree, ignoreTrees, walkFn) + return err +} + +// walk recursively traverses the tree, ignoring subtrees when the ID of the +// subtree is in ignoreTrees. If err is nil and ignore is true, the subtree ID +// will be added to ignoreTrees by walk. +func walk(ctx context.Context, repo TreeLoader, prefix string, tree *restic.Tree, ignoreTrees restic.IDSet, walkFn WalkFunc) (ignore bool, err error) { + var allNodesIgnored = true + + sort.Slice(tree.Nodes, func(i, j int) bool { + return tree.Nodes[i].Name < tree.Nodes[j].Name + }) + + for _, node := range tree.Nodes { + p := path.Join(prefix, node.Name) + + if node.Type == "" { + return false, errors.Errorf("node type is empty for node %q", node.Name) + } + + if node.Type != "dir" { + ignore, err := walkFn(p, node, nil) + if err != nil { + if err == SkipNode { + // skip the remaining entries in this tree + return allNodesIgnored, nil + } + + return false, err + } + + if ignore == false { + allNodesIgnored = false + } + + continue + } + + if node.Subtree == nil { + return false, errors.Errorf("subtree for node %v in tree %v is nil", node.Name, p) + } + + if ignoreTrees.Has(*node.Subtree) { + continue + } + + subtree, err := repo.LoadTree(ctx, *node.Subtree) + ignore, err := walkFn(p, node, err) + if err != nil { + if err == SkipNode { + if ignore { + ignoreTrees.Insert(*node.Subtree) + } + continue + } + return false, err + } + + if ignore { + ignoreTrees.Insert(*node.Subtree) + } + + if !ignore { + allNodesIgnored = false + } + + ignore, err = walk(ctx, repo, p, subtree, ignoreTrees, walkFn) + if err != nil { + return false, err + } + + if ignore { + ignoreTrees.Insert(*node.Subtree) + } + + if !ignore { + allNodesIgnored = false + } + } + + return allNodesIgnored, nil +} diff --git a/internal/walker/walker_test.go b/internal/walker/walker_test.go new file mode 100644 index 000000000..08b4fe405 --- /dev/null +++ b/internal/walker/walker_test.go @@ -0,0 +1,423 @@ +package walker + +import ( + "context" + "encoding/json" + "fmt" + "testing" + + "github.com/pkg/errors" + "github.com/restic/restic/internal/restic" +) + +// TestTree is used to construct a list of trees for testing the walker. +type TestTree map[string]interface{} + +// TestNode is used to test the walker. +type TestFile struct{} + +func BuildTreeMap(tree TestTree) (m TreeMap, root restic.ID) { + m = TreeMap{} + id := buildTreeMap(tree, m) + return m, id +} + +func buildTreeMap(tree TestTree, m TreeMap) restic.ID { + res := restic.NewTree() + + for name, item := range tree { + switch elem := item.(type) { + case TestFile: + res.Insert(&restic.Node{ + Name: name, + Type: "file", + }) + case TestTree: + id := buildTreeMap(elem, m) + res.Insert(&restic.Node{ + Name: name, + Subtree: &id, + Type: "dir", + }) + default: + panic(fmt.Sprintf("invalid type %T", elem)) + } + } + + buf, err := json.Marshal(res) + if err != nil { + panic(err) + } + + id := restic.Hash(buf) + + if _, ok := m[id]; !ok { + m[id] = res + } + + return id +} + +// TreeMap returns the trees from the map on LoadTree. +type TreeMap map[restic.ID]*restic.Tree + +func (t TreeMap) LoadTree(ctx context.Context, id restic.ID) (*restic.Tree, error) { + tree, ok := t[id] + if !ok { + return nil, errors.New("tree not found") + } + + return tree, nil +} + +// checkFunc returns a function suitable for walking the tree to check +// something, and a function which will check the final result. +type checkFunc func(t testing.TB) (walker WalkFunc, final func(testing.TB)) + +// checkItemOrder ensures that the order of the 'path' arguments is the one passed in as 'want'. +func checkItemOrder(want []string) checkFunc { + pos := 0 + return func(t testing.TB) (walker WalkFunc, final func(testing.TB)) { + walker = func(path string, node *restic.Node, err error) (bool, error) { + if err != nil { + t.Errorf("error walking %v: %v", path, err) + return false, err + } + + if pos >= len(want) { + t.Errorf("additional unexpected path found: %v", path) + return false, nil + } + + if path != want[pos] { + t.Errorf("wrong path found, want %q, got %q", want[pos], path) + } + pos++ + return false, nil + } + + final = func(t testing.TB) { + if pos != len(want) { + t.Errorf("not enough items returned, want %d, got %d", len(want), pos) + } + } + + return walker, final + } +} + +// checkSkipFor returns SkipNode if path is in skipFor, it checks that the +// paths the walk func is called for are exactly the ones in wantPaths. +func checkSkipFor(skipFor map[string]struct{}, wantPaths []string) checkFunc { + var pos int + + return func(t testing.TB) (walker WalkFunc, final func(testing.TB)) { + walker = func(path string, node *restic.Node, err error) (bool, error) { + if err != nil { + t.Errorf("error walking %v: %v", path, err) + return false, err + } + + if pos >= len(wantPaths) { + t.Errorf("additional unexpected path found: %v", path) + return false, nil + } + + if path != wantPaths[pos] { + t.Errorf("wrong path found, want %q, got %q", wantPaths[pos], path) + } + pos++ + + if _, ok := skipFor[path]; ok { + return false, SkipNode + } + + return false, nil + } + + final = func(t testing.TB) { + if pos != len(wantPaths) { + t.Errorf("wrong number of paths returned, want %d, got %d", len(wantPaths), pos) + } + } + + return walker, final + } +} + +// checkIgnore returns SkipNode if path is in skipFor and sets ignore according +// to ignoreFor. It checks that the paths the walk func is called for are exactly +// the ones in wantPaths. +func checkIgnore(skipFor map[string]struct{}, ignoreFor map[string]bool, wantPaths []string) checkFunc { + var pos int + + return func(t testing.TB) (walker WalkFunc, final func(testing.TB)) { + walker = func(path string, node *restic.Node, err error) (bool, error) { + if err != nil { + t.Errorf("error walking %v: %v", path, err) + return false, err + } + + if pos >= len(wantPaths) { + t.Errorf("additional unexpected path found: %v", path) + return ignoreFor[path], nil + } + + if path != wantPaths[pos] { + t.Errorf("wrong path found, want %q, got %q", wantPaths[pos], path) + } + pos++ + + if _, ok := skipFor[path]; ok { + return ignoreFor[path], SkipNode + } + + return ignoreFor[path], nil + } + + final = func(t testing.TB) { + if pos != len(wantPaths) { + t.Errorf("wrong number of paths returned, want %d, got %d", len(wantPaths), pos) + } + } + + return walker, final + } +} + +func TestWalker(t *testing.T) { + var tests = []struct { + tree TestTree + checks []checkFunc + }{ + { + tree: TestTree{ + "foo": TestFile{}, + "subdir": TestTree{ + "subfile": TestFile{}, + }, + }, + checks: []checkFunc{ + checkItemOrder([]string{ + "/", + "/foo", + "/subdir", + "/subdir/subfile", + }), + checkSkipFor( + map[string]struct{}{ + "/subdir": struct{}{}, + }, []string{ + "/", + "/foo", + "/subdir", + }, + ), + checkIgnore( + map[string]struct{}{}, map[string]bool{ + "/subdir": true, + }, []string{ + "/", + "/foo", + "/subdir", + "/subdir/subfile", + }, + ), + }, + }, + { + tree: TestTree{ + "foo": TestFile{}, + "subdir1": TestTree{ + "subfile1": TestFile{}, + }, + "subdir2": TestTree{ + "subfile2": TestFile{}, + "subsubdir2": TestTree{ + "subsubfile3": TestFile{}, + }, + }, + }, + checks: []checkFunc{ + checkItemOrder([]string{ + "/", + "/foo", + "/subdir1", + "/subdir1/subfile1", + "/subdir2", + "/subdir2/subfile2", + "/subdir2/subsubdir2", + "/subdir2/subsubdir2/subsubfile3", + }), + checkSkipFor( + map[string]struct{}{ + "/subdir1": struct{}{}, + }, []string{ + "/", + "/foo", + "/subdir1", + "/subdir2", + "/subdir2/subfile2", + "/subdir2/subsubdir2", + "/subdir2/subsubdir2/subsubfile3", + }, + ), + checkSkipFor( + map[string]struct{}{ + "/subdir1": struct{}{}, + "/subdir2/subsubdir2": struct{}{}, + }, []string{ + "/", + "/foo", + "/subdir1", + "/subdir2", + "/subdir2/subfile2", + "/subdir2/subsubdir2", + }, + ), + checkSkipFor( + map[string]struct{}{ + "/foo": struct{}{}, + }, []string{ + "/", + "/foo", + }, + ), + }, + }, + { + tree: TestTree{ + "foo": TestFile{}, + "subdir1": TestTree{ + "subfile1": TestFile{}, + "subfile2": TestFile{}, + "subfile3": TestFile{}, + }, + "subdir2": TestTree{ + "subfile1": TestFile{}, + "subfile2": TestFile{}, + "subfile3": TestFile{}, + }, + "subdir3": TestTree{ + "subfile1": TestFile{}, + "subfile2": TestFile{}, + "subfile3": TestFile{}, + }, + "zzz other": TestFile{}, + }, + checks: []checkFunc{ + checkItemOrder([]string{ + "/", + "/foo", + "/subdir1", + "/subdir1/subfile1", + "/subdir1/subfile2", + "/subdir1/subfile3", + "/subdir2", + "/subdir2/subfile1", + "/subdir2/subfile2", + "/subdir2/subfile3", + "/subdir3", + "/subdir3/subfile1", + "/subdir3/subfile2", + "/subdir3/subfile3", + "/zzz other", + }), + checkIgnore( + map[string]struct{}{ + "/subdir1": struct{}{}, + }, map[string]bool{ + "/subdir1": true, + }, []string{ + "/", + "/foo", + "/subdir1", + "/zzz other", + }, + ), + checkIgnore( + map[string]struct{}{}, map[string]bool{ + "/subdir1": true, + }, []string{ + "/", + "/foo", + "/subdir1", + "/subdir1/subfile1", + "/subdir1/subfile2", + "/subdir1/subfile3", + "/zzz other", + }, + ), + checkIgnore( + map[string]struct{}{ + "/subdir2": struct{}{}, + }, map[string]bool{ + "/subdir2": true, + }, []string{ + "/", + "/foo", + "/subdir1", + "/subdir1/subfile1", + "/subdir1/subfile2", + "/subdir1/subfile3", + "/subdir2", + "/zzz other", + }, + ), + checkIgnore( + map[string]struct{}{}, map[string]bool{ + "/subdir1/subfile1": true, + "/subdir1/subfile2": true, + "/subdir1/subfile3": true, + }, []string{ + "/", + "/foo", + "/subdir1", + "/subdir1/subfile1", + "/subdir1/subfile2", + "/subdir1/subfile3", + "/zzz other", + }, + ), + checkIgnore( + map[string]struct{}{}, map[string]bool{ + "/subdir2/subfile1": true, + "/subdir2/subfile2": true, + "/subdir2/subfile3": true, + }, []string{ + "/", + "/foo", + "/subdir1", + "/subdir1/subfile1", + "/subdir1/subfile2", + "/subdir1/subfile3", + "/subdir2", + "/subdir2/subfile1", + "/subdir2/subfile2", + "/subdir2/subfile3", + "/zzz other", + }, + ), + }, + }, + } + + for _, test := range tests { + t.Run("", func(t *testing.T) { + repo, root := BuildTreeMap(test.tree) + for _, check := range test.checks { + t.Run("", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.TODO()) + defer cancel() + + fn, last := check(t) + err := Walk(ctx, repo, root, restic.NewIDSet(), fn) + if err != nil { + t.Error(err) + } + last(t) + }) + } + }) + } +}