octoleo
/
restic
mirror of https://github.com/octoleo/restic.git
2
2
Fork 0
Code Issues Releases Wiki Activity

walk: Pass parent tree ID to WalkFunc

This commit is contained in:
Alexander Neumann 2018-08-19 12:28:06 +02:00 committed by Mikael Berthe
parent 1f27d17c0d
commit 15ad0e5bc7
5 changed files with 80 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
cmd/restic/cmd_find.go
View File

@ -258,7 +258,7 @@ func (f *Finder) findInSnapshot(ctx context.Context, sn *restic.Snapshot) error
}
f.out.newsn = sn
return walker.Walk(ctx, f.repo, *sn.Tree, f.ignoreTrees, func(nodepath string, node *restic.Node, treeID string, err error) (bool, error) {
return walker.Walk(ctx, f.repo, *sn.Tree, f.ignoreTrees, func(_ restic.ID, nodepath string, node *restic.Node, err error) (bool, error) {
if err != nil {
return false, err
}
@ -338,7 +338,7 @@ func (f *Finder) findIDs(ctx context.Context, sn *restic.Snapshot) error {
}
f.out.newsn = sn
return walker.Walk(ctx, f.repo, *sn.Tree, f.ignoreTrees, func(nodepath string, node *restic.Node, treeID string, err error) (bool, error) {
return walker.Walk(ctx, f.repo, *sn.Tree, f.ignoreTrees, func(parentTreeID restic.ID, nodepath string, node *restic.Node, err error) (bool, error) {
if err != nil {
return false, err
}
@ -379,7 +379,7 @@ func (f *Finder) findIDs(ctx context.Context, sn *restic.Snapshot) error {
f.blobIDs[idStr] = struct{}{}
delete(f.blobIDs, idStr[:shortStr])
}
f.out.PrintObject("blob", idStr, nodepath, treeID, sn)
f.out.PrintObject("blob", idStr, nodepath, parentTreeID.String(), sn)
break
}
}

2
cmd/restic/cmd_ls.go
View File

@ -203,7 +203,7 @@ func runLs(opts LsOptions, gopts GlobalOptions, args []string) error {
for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, args[:1]) {
printSnapshot(sn)
err := walker.Walk(ctx, repo, *sn.Tree, nil, func(nodepath string, node *restic.Node, treeID string, err error) (bool, error) {
err := walker.Walk(ctx, repo, *sn.Tree, nil, func(_ restic.ID, nodepath string, node *restic.Node, err error) (bool, error) {
if err != nil {
return false, err
}

2
cmd/restic/cmd_stats.go
View File

@ -188,7 +188,7 @@ func statsWalkSnapshot(ctx context.Context, snapshot *restic.Snapshot, repo rest
}
func statsWalkTree(repo restic.Repository, stats *statsContainer) walker.WalkFunc {
return func(npath string, node *restic.Node, parentTreeID string, nodeErr error) (bool, error) {
return func(_ restic.ID, npath string, node *restic.Node, nodeErr error) (bool, error) {
if nodeErr != nil {
return true, nodeErr
}

14
internal/walker/walker.go
View File

@ -33,14 +33,14 @@ var SkipNode = errors.New("skip this node")
// tree have ignore set to true, the current tree will not be visited again.
// When err is not nil and different from SkipNode, the value returned for
// ignore is ignored.
type WalkFunc func(path string, node *restic.Node, parentTreeID string, nodeErr error) (ignore bool, err error)
type WalkFunc func(parentTreeID restic.ID, path string, node *restic.Node, nodeErr error) (ignore bool, err error)
// Walk calls walkFn recursively for each node in root. If walkFn returns an
// error, it is passed up the call stack. The trees in ignoreTrees are not
// walked. If walkFn ignores trees, these are added to the set.
func Walk(ctx context.Context, repo TreeLoader, root restic.ID, ignoreTrees restic.IDSet, walkFn WalkFunc) error {
tree, err := repo.LoadTree(ctx, root)
_, err = walkFn("/", nil, "", err)
_, err = walkFn(root, "/", nil, err)
if err != nil {
if err == SkipNode {
@ -53,14 +53,14 @@ func Walk(ctx context.Context, repo TreeLoader, root restic.ID, ignoreTrees rest
ignoreTrees = restic.NewIDSet()
}
_, err = walk(ctx, repo, "/", root.String(), tree, ignoreTrees, walkFn)
_, err = walk(ctx, repo, "/", root, tree, ignoreTrees, walkFn)
return err
}
// walk recursively traverses the tree, ignoring subtrees when the ID of the
// subtree is in ignoreTrees. If err is nil and ignore is true, the subtree ID
// will be added to ignoreTrees by walk.
func walk(ctx context.Context, repo TreeLoader, prefix string, treeID string, tree *restic.Tree, ignoreTrees restic.IDSet, walkFn WalkFunc) (ignore bool, err error) {
func walk(ctx context.Context, repo TreeLoader, prefix string, parentTreeID restic.ID, tree *restic.Tree, ignoreTrees restic.IDSet, walkFn WalkFunc) (ignore bool, err error) {
var allNodesIgnored = true
if len(tree.Nodes) == 0 {
@ -79,7 +79,7 @@ func walk(ctx context.Context, repo TreeLoader, prefix string, treeID string, tr
}
if node.Type != "dir" {
ignore, err := walkFn(p, node, treeID, nil)
ignore, err := walkFn(parentTreeID, p, node, nil)
if err != nil {
if err == SkipNode {
// skip the remaining entries in this tree
@ -105,7 +105,7 @@ func walk(ctx context.Context, repo TreeLoader, prefix string, treeID string, tr
}
subtree, err := repo.LoadTree(ctx, *node.Subtree)
ignore, err := walkFn(p, node, treeID, err)
ignore, err := walkFn(parentTreeID, p, node, err)
if err != nil {
if err == SkipNode {
if ignore {
@ -124,7 +124,7 @@ func walk(ctx context.Context, repo TreeLoader, prefix string, treeID string, tr
allNodesIgnored = false
}
ignore, err = walk(ctx, repo, p, node.Subtree.String(), subtree, ignoreTrees, walkFn)
ignore, err = walk(ctx, repo, p, *node.Subtree, subtree, ignoreTrees, walkFn)
if err != nil {
return false, err
}

71
internal/walker/walker_test.go
View File

@ -78,7 +78,7 @@ type checkFunc func(t testing.TB) (walker WalkFunc, final func(testing.TB))
func checkItemOrder(want []string) checkFunc {
pos := 0
return func(t testing.TB) (walker WalkFunc, final func(testing.TB)) {
walker = func(path string, node *restic.Node, treeID string, err error) (bool, error) {
walker = func(treeID restic.ID, path string, node *restic.Node, err error) (bool, error) {
if err != nil {
t.Errorf("error walking %v: %v", path, err)
return false, err
@ -106,13 +106,45 @@ func checkItemOrder(want []string) checkFunc {
}
}
// checkParentTreeOrder ensures that the order of the 'parentID' arguments is the one passed in as 'want'.
func checkParentTreeOrder(want []string) checkFunc {
pos := 0
return func(t testing.TB) (walker WalkFunc, final func(testing.TB)) {
walker = func(treeID restic.ID, path string, node *restic.Node, err error) (bool, error) {
if err != nil {
t.Errorf("error walking %v: %v", path, err)
return false, err
}
if pos >= len(want) {
t.Errorf("additional unexpected parent tree ID found: %v", treeID)
return false, nil
}
if treeID.String() != want[pos] {
t.Errorf("wrong parent tree ID found, want %q, got %q", want[pos], treeID.String())
}
pos++
return false, nil
}
final = func(t testing.TB) {
if pos != len(want) {
t.Errorf("not enough items returned, want %d, got %d", len(want), pos)
}
}
return walker, final
}
}
// checkSkipFor returns SkipNode if path is in skipFor, it checks that the
// paths the walk func is called for are exactly the ones in wantPaths.
func checkSkipFor(skipFor map[string]struct{}, wantPaths []string) checkFunc {
var pos int
return func(t testing.TB) (walker WalkFunc, final func(testing.TB)) {
walker = func(path string, node *restic.Node, treeID string, err error) (bool, error) {
walker = func(treeID restic.ID, path string, node *restic.Node, err error) (bool, error) {
if err != nil {
t.Errorf("error walking %v: %v", path, err)
return false, err
@ -152,7 +184,7 @@ func checkIgnore(skipFor map[string]struct{}, ignoreFor map[string]bool, wantPat
var pos int
return func(t testing.TB) (walker WalkFunc, final func(testing.TB)) {
walker = func(path string, node *restic.Node, treeID string, err error) (bool, error) {
walker = func(treeID restic.ID, path string, node *restic.Node, err error) (bool, error) {
if err != nil {
t.Errorf("error walking %v: %v", path, err)
return false, err
@ -204,6 +236,12 @@ func TestWalker(t *testing.T) {
"/subdir",
"/subdir/subfile",
}),
checkParentTreeOrder([]string{
"2593e9dba52232c043d68c40d0f9c236b4448e37224941298ea6e223ca1e3a1b", // tree /
"2593e9dba52232c043d68c40d0f9c236b4448e37224941298ea6e223ca1e3a1b", // tree /
"2593e9dba52232c043d68c40d0f9c236b4448e37224941298ea6e223ca1e3a1b", // tree /
"a7f5be55bdd94db9df706a428e0726a4044720c9c94b9ebeb81000debe032087", // tree /subdir
}),
checkSkipFor(
map[string]struct{}{
"/subdir": struct{}{},
@ -249,6 +287,16 @@ func TestWalker(t *testing.T) {
"/subdir2/subsubdir2",
"/subdir2/subsubdir2/subsubfile3",
}),
checkParentTreeOrder([]string{
"31c86f0bc298086b787b5d24e9e33ea566c224be2939ed66a817f7fb6fdba700", // tree /
"31c86f0bc298086b787b5d24e9e33ea566c224be2939ed66a817f7fb6fdba700", // tree /
"31c86f0bc298086b787b5d24e9e33ea566c224be2939ed66a817f7fb6fdba700", // tree /
"af838dc7a83d353f0273c33d93fcdba3220d4517576f09694a971dd23b8e94dc", // tree /subdir1
"31c86f0bc298086b787b5d24e9e33ea566c224be2939ed66a817f7fb6fdba700", // tree /
"fb749ba6ae01a3814bed9b59d74af8d7593d3074a681d4112c4983d461089e5b", // tree /subdir2
"fb749ba6ae01a3814bed9b59d74af8d7593d3074a681d4112c4983d461089e5b", // tree /subdir2
"eb8dd587a9c5e6be87b69d2c5264a19622f75bf6704927aaebaee78d0992531d", // tree /subdir2/subsubdir2
}),
checkSkipFor(
map[string]struct{}{
"/subdir1": struct{}{},
@ -323,6 +371,23 @@ func TestWalker(t *testing.T) {
"/subdir3/subfile3",
"/zzz other",
}),
checkParentTreeOrder([]string{
"b37368f62fdd6f8f3d19f9ef23c6534988e26db4e5dddc21d206b16b6a17a58f", // tree /
"b37368f62fdd6f8f3d19f9ef23c6534988e26db4e5dddc21d206b16b6a17a58f", // tree /
"b37368f62fdd6f8f3d19f9ef23c6534988e26db4e5dddc21d206b16b6a17a58f", // tree /
"787b9260d4f0f8298f5cf58945681961982eb6aa1c526845206c5b353aeb4351", // tree /subdir1
"787b9260d4f0f8298f5cf58945681961982eb6aa1c526845206c5b353aeb4351", // tree /subdir1
"787b9260d4f0f8298f5cf58945681961982eb6aa1c526845206c5b353aeb4351", // tree /subdir1
"b37368f62fdd6f8f3d19f9ef23c6534988e26db4e5dddc21d206b16b6a17a58f", // tree /
"787b9260d4f0f8298f5cf58945681961982eb6aa1c526845206c5b353aeb4351", // tree /subdir2
"787b9260d4f0f8298f5cf58945681961982eb6aa1c526845206c5b353aeb4351", // tree /subdir2
"787b9260d4f0f8298f5cf58945681961982eb6aa1c526845206c5b353aeb4351", // tree /subdir2
"b37368f62fdd6f8f3d19f9ef23c6534988e26db4e5dddc21d206b16b6a17a58f", // tree /
"787b9260d4f0f8298f5cf58945681961982eb6aa1c526845206c5b353aeb4351", // tree /subdir3
"787b9260d4f0f8298f5cf58945681961982eb6aa1c526845206c5b353aeb4351", // tree /subdir3
"787b9260d4f0f8298f5cf58945681961982eb6aa1c526845206c5b353aeb4351", // tree /subdir3
"b37368f62fdd6f8f3d19f9ef23c6534988e26db4e5dddc21d206b16b6a17a58f", // tree /
}),
checkIgnore(
map[string]struct{}{
"/subdir1": struct{}{},