Merge pull request #4550 from ndecker/ls-ncdu

Ls ncdu

changelog/unreleased/issue-4549

@@ -0,0 +1,11 @@
+Enhancement: Add `--ncdu` option to `ls` command
+
+NCDU (NCurses Disk Usage) is a tool to analyse disk usage of directories.
+It has an option to save a directory tree and analyse it later.
+The `ls` command now supports the `--ncdu` option which outputs information
+about a snapshot in the NCDU format.
+
+You can use it as follows: `restic ls latest --ncdu | ncdu -f -`
+
+https://github.com/restic/restic/issues/4549
+https://github.com/restic/restic/pull/4550

cmd/restic/cmd_find.go

@@ -260,7 +260,7 @@ func (f *Finder) findInSnapshot(ctx context.Context, sn *restic.Snapshot) error
 	}
 
 	f.out.newsn = sn
-	return walker.Walk(ctx, f.repo, *sn.Tree, func(parentTreeID restic.ID, nodepath string, node *restic.Node, err error) error {
+	return walker.Walk(ctx, f.repo, *sn.Tree, walker.WalkVisitor{ProcessNode: func(parentTreeID restic.ID, nodepath string, node *restic.Node, err error) error {
 		if err != nil {
 			debug.Log("Error loading tree %v: %v", parentTreeID, err)
 
@@ -327,7 +327,7 @@ func (f *Finder) findInSnapshot(ctx context.Context, sn *restic.Snapshot) error
 		debug.Log("    found match\n")
 		f.out.PrintPattern(nodepath, node)
 		return nil
-	})
+	}})
 }
 
 func (f *Finder) findIDs(ctx context.Context, sn *restic.Snapshot) error {
@@ -338,7 +338,7 @@ func (f *Finder) findIDs(ctx context.Context, sn *restic.Snapshot) error {
 	}
 
 	f.out.newsn = sn
-	return walker.Walk(ctx, f.repo, *sn.Tree, func(parentTreeID restic.ID, nodepath string, node *restic.Node, err error) error {
+	return walker.Walk(ctx, f.repo, *sn.Tree, walker.WalkVisitor{ProcessNode: func(parentTreeID restic.ID, nodepath string, node *restic.Node, err error) error {
 		if err != nil {
 			debug.Log("Error loading tree %v: %v", parentTreeID, err)
 
@@ -388,7 +388,7 @@ func (f *Finder) findIDs(ctx context.Context, sn *restic.Snapshot) error {
 		}
 
 		return nil
-	})
+	}})
 }
 
 var errAllPacksFound = errors.New("all packs found")

cmd/restic/cmd_ls.go

@@ -3,6 +3,8 @@ package main
 import (
 	"context"
 	"encoding/json"
+	"fmt"
+	"io"
 	"os"
 	"strings"
 	"time"
@@ -51,6 +53,7 @@ type LsOptions struct {
 	restic.SnapshotFilter
 	Recursive     bool
 	HumanReadable bool
+	Ncdu          bool
 }
 
 var lsOptions LsOptions
@@ -63,16 +66,47 @@ func init() {
 	flags.BoolVarP(&lsOptions.ListLong, "long", "l", false, "use a long listing format showing size and mode")
 	flags.BoolVar(&lsOptions.Recursive, "recursive", false, "include files in subfolders of the listed directories")
 	flags.BoolVar(&lsOptions.HumanReadable, "human-readable", false, "print sizes in human readable format")
+	flags.BoolVar(&lsOptions.Ncdu, "ncdu", false, "output NCDU export format (pipe into 'ncdu -f -')")
 }
 
-type lsSnapshot struct {
+type lsPrinter interface {
+	Snapshot(sn *restic.Snapshot)
+	Node(path string, node *restic.Node)
+	LeaveDir(path string)
+	Close()
+}
+
+type jsonLsPrinter struct {
+	enc *json.Encoder
+}
+
+func (p *jsonLsPrinter) Snapshot(sn *restic.Snapshot) {
+	type lsSnapshot struct {
 		*restic.Snapshot
 		ID         *restic.ID `json:"id"`
 		ShortID    string     `json:"short_id"`
 		StructType string     `json:"struct_type"` // "snapshot"
+	}
+
+	err := p.enc.Encode(lsSnapshot{
+		Snapshot:   sn,
+		ID:         sn.ID(),
+		ShortID:    sn.ID().Str(),
+		StructType: "snapshot",
+	})
+	if err != nil {
+		Warnf("JSON encode failed: %v\n", err)
+	}
 }
 
 // Print node in our custom JSON format, followed by a newline.
+func (p *jsonLsPrinter) Node(path string, node *restic.Node) {
+	err := lsNodeJSON(p.enc, path, node)
+	if err != nil {
+		Warnf("JSON encode failed: %v\n", err)
+	}
+}
+
 func lsNodeJSON(enc *json.Encoder, path string, node *restic.Node) error {
 	n := &struct {
 		Name        string      `json:"name"`
@@ -114,10 +148,117 @@ func lsNodeJSON(enc *json.Encoder, path string, node *restic.Node) error {
 	return enc.Encode(n)
 }
 
+func (p *jsonLsPrinter) LeaveDir(_ string) {}
+func (p *jsonLsPrinter) Close()            {}
+
+type ncduLsPrinter struct {
+	out   io.Writer
+	depth int
+}
+
+// lsSnapshotNcdu prints a restic snapshot in Ncdu save format.
+// It opens the JSON list. Nodes are added with lsNodeNcdu and the list is closed by lsCloseNcdu.
+// Format documentation: https://dev.yorhel.nl/ncdu/jsonfmt
+func (p *ncduLsPrinter) Snapshot(sn *restic.Snapshot) {
+	const NcduMajorVer = 1
+	const NcduMinorVer = 2
+
+	snapshotBytes, err := json.Marshal(sn)
+	if err != nil {
+		Warnf("JSON encode failed: %v\n", err)
+	}
+	p.depth++
+	fmt.Fprintf(p.out, "[%d, %d, %s", NcduMajorVer, NcduMinorVer, string(snapshotBytes))
+}
+
+func lsNcduNode(_ string, node *restic.Node) ([]byte, error) {
+	type NcduNode struct {
+		Name   string `json:"name"`
+		Asize  uint64 `json:"asize"`
+		Dsize  uint64 `json:"dsize"`
+		Dev    uint64 `json:"dev"`
+		Ino    uint64 `json:"ino"`
+		NLink  uint64 `json:"nlink"`
+		NotReg bool   `json:"notreg"`
+		UID    uint32 `json:"uid"`
+		GID    uint32 `json:"gid"`
+		Mode   uint16 `json:"mode"`
+		Mtime  int64  `json:"mtime"`
+	}
+
+	outNode := NcduNode{
+		Name:   node.Name,
+		Asize:  node.Size,
+		Dsize:  node.Size,
+		Dev:    node.DeviceID,
+		Ino:    node.Inode,
+		NLink:  node.Links,
+		NotReg: node.Type != "dir" && node.Type != "file",
+		UID:    node.UID,
+		GID:    node.GID,
+		Mode:   uint16(node.Mode & os.ModePerm),
+		Mtime:  node.ModTime.Unix(),
+	}
+	//  bits according to inode(7) manpage
+	if node.Mode&os.ModeSetuid != 0 {
+		outNode.Mode |= 0o4000
+	}
+	if node.Mode&os.ModeSetgid != 0 {
+		outNode.Mode |= 0o2000
+	}
+	if node.Mode&os.ModeSticky != 0 {
+		outNode.Mode |= 0o1000
+	}
+
+	return json.Marshal(outNode)
+}
+
+func (p *ncduLsPrinter) Node(path string, node *restic.Node) {
+	out, err := lsNcduNode(path, node)
+	if err != nil {
+		Warnf("JSON encode failed: %v\n", err)
+	}
+
+	if node.Type == "dir" {
+		fmt.Fprintf(p.out, ",\n%s[\n%s%s", strings.Repeat("  ", p.depth), strings.Repeat("  ", p.depth+1), string(out))
+		p.depth++
+	} else {
+		fmt.Fprintf(p.out, ",\n%s%s", strings.Repeat("  ", p.depth), string(out))
+	}
+}
+
+func (p *ncduLsPrinter) LeaveDir(_ string) {
+	p.depth--
+	fmt.Fprintf(p.out, "\n%s]", strings.Repeat("  ", p.depth))
+}
+
+func (p *ncduLsPrinter) Close() {
+	fmt.Fprint(p.out, "\n]\n")
+}
+
+type textLsPrinter struct {
+	dirs          []string
+	ListLong      bool
+	HumanReadable bool
+}
+
+func (p *textLsPrinter) Snapshot(sn *restic.Snapshot) {
+	Verbosef("%v filtered by %v:\n", sn, p.dirs)
+}
+func (p *textLsPrinter) Node(path string, node *restic.Node) {
+	Printf("%s\n", formatNode(path, node, p.ListLong, p.HumanReadable))
+}
+
+func (p *textLsPrinter) LeaveDir(_ string) {}
+func (p *textLsPrinter) Close()            {}
+
 func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []string) error {
 	if len(args) == 0 {
 		return errors.Fatal("no snapshot ID specified, specify snapshot ID or use special ID 'latest'")
 	}
+	if opts.Ncdu && gopts.JSON {
+		return errors.Fatal("only either '--json' or '--ncdu' can be specified")
+	}
 
 	// extract any specific directories to walk
 	var dirs []string
@@ -179,38 +320,21 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri
 		return err
 	}
 
-	var (
+	var printer lsPrinter
-		printSnapshot func(sn *restic.Snapshot)
-		printNode     func(path string, node *restic.Node)
-	)
 
 	if gopts.JSON {
-		enc := json.NewEncoder(globalOptions.stdout)
+		printer = &jsonLsPrinter{
-
+			enc: json.NewEncoder(globalOptions.stdout),
-		printSnapshot = func(sn *restic.Snapshot) {
-			err := enc.Encode(lsSnapshot{
-				Snapshot:   sn,
-				ID:         sn.ID(),
-				ShortID:    sn.ID().Str(),
-				StructType: "snapshot",
-			})
-			if err != nil {
-				Warnf("JSON encode failed: %v\n", err)
-			}
-		}
-
-		printNode = func(path string, node *restic.Node) {
-			err := lsNodeJSON(enc, path, node)
-			if err != nil {
-				Warnf("JSON encode failed: %v\n", err)
 		}
+	} else if opts.Ncdu {
+		printer = &ncduLsPrinter{
+			out: globalOptions.stdout,
 		}
 	} else {
-		printSnapshot = func(sn *restic.Snapshot) {
+		printer = &textLsPrinter{
-			Verbosef("%v filtered by %v:\n", sn, dirs)
+			dirs:          dirs,
-		}
+			ListLong:      opts.ListLong,
-		printNode = func(path string, node *restic.Node) {
+			HumanReadable: opts.HumanReadable,
-			Printf("%s\n", formatNode(path, node, opts.ListLong, opts.HumanReadable))
 		}
 	}
 
@@ -228,9 +352,9 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri
 		return err
 	}
 
-	printSnapshot(sn)
+	printer.Snapshot(sn)
 
-	err = walker.Walk(ctx, repo, *sn.Tree, func(_ restic.ID, nodepath string, node *restic.Node, err error) error {
+	processNode := func(_ restic.ID, nodepath string, node *restic.Node, err error) error {
 		if err != nil {
 			return err
 		}
@@ -240,7 +364,7 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri
 
 		if withinDir(nodepath) {
 			// if we're within a dir, print the node
-			printNode(nodepath, node)
+			printer.Node(nodepath, node)
 
 			// if recursive listing is requested, signal the walker that it
 			// should continue walking recursively
@@ -261,11 +385,22 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri
 			return walker.ErrSkipNode
 		}
 		return nil
+	}
+
+	err = walker.Walk(ctx, repo, *sn.Tree, walker.WalkVisitor{
+		ProcessNode: processNode,
+		LeaveDir: func(path string) {
+			// the root path `/` has no corresponding node and is thus also skipped by processNode
+			if withinDir(path) && path != "/" {
+				printer.LeaveDir(path)
+			}
+		},
 	})
 
 	if err != nil {
 		return err
 	}
 
+	printer.Close()
 	return nil
 }

cmd/restic/cmd_ls_integration_test.go

@@ -2,18 +2,46 @@ package main
 
 import (
 	"context"
+	"encoding/json"
+	"path/filepath"
 	"strings"
 	"testing"
 
 	rtest "github.com/restic/restic/internal/test"
 )
 
-func testRunLs(t testing.TB, gopts GlobalOptions, snapshotID string) []string {
+func testRunLsWithOpts(t testing.TB, gopts GlobalOptions, opts LsOptions, args []string) []byte {
 	buf, err := withCaptureStdout(func() error {
 		gopts.Quiet = true
-		opts := LsOptions{}
+		return runLs(context.TODO(), opts, gopts, args)
-		return runLs(context.TODO(), opts, gopts, []string{snapshotID})
 	})
 	rtest.OK(t, err)
-	return strings.Split(buf.String(), "\n")
+	return buf.Bytes()
+}
+
+func testRunLs(t testing.TB, gopts GlobalOptions, snapshotID string) []string {
+	out := testRunLsWithOpts(t, gopts, LsOptions{}, []string{snapshotID})
+	return strings.Split(string(out), "\n")
+}
+
+func assertIsValidJSON(t *testing.T, data []byte) {
+	// Sanity check: output must be valid JSON.
+	var v interface{}
+	err := json.Unmarshal(data, &v)
+	rtest.OK(t, err)
+}
+
+func TestRunLsNcdu(t *testing.T) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+
+	testRunInit(t, env.gopts)
+	opts := BackupOptions{}
+	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, env.gopts)
+
+	ncdu := testRunLsWithOpts(t, env.gopts, LsOptions{Ncdu: true}, []string{"latest"})
+	assertIsValidJSON(t, ncdu)
+
+	ncdu = testRunLsWithOpts(t, env.gopts, LsOptions{Ncdu: true}, []string{"latest", "/testdata"})
+	assertIsValidJSON(t, ncdu)
 }

cmd/restic/cmd_ls_test.go

@@ -11,12 +11,12 @@ import (
 	rtest "github.com/restic/restic/internal/test"
 )
 
-func TestLsNodeJSON(t *testing.T) {
+type lsTestNode struct {
-	for _, c := range []struct {
 	path string
 	restic.Node
-		expect string
+}
-	}{
+
+var lsTestNodes = []lsTestNode{
 	// Mode is omitted when zero.
 	// Permissions, by convention is "-" per mode bit
 	{
@@ -32,7 +32,6 @@ func TestLsNodeJSON(t *testing.T) {
 			Group: "nobodies",
 			Links: 1,
 		},
-			expect: `{"name":"baz","type":"file","path":"/bar/baz","uid":10000000,"gid":20000000,"size":12345,"permissions":"----------","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","struct_type":"node"}`,
 	},
 
 	// Even empty files get an explicit size.
@@ -49,7 +48,6 @@ func TestLsNodeJSON(t *testing.T) {
 			Group: "not printed",
 			Links: 0xF00,
 		},
-			expect: `{"name":"empty","type":"file","path":"/foo/empty","uid":1001,"gid":1001,"size":0,"permissions":"----------","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","struct_type":"node"}`,
 	},
 
 	// Non-regular files do not get a size.
@@ -62,7 +60,6 @@ func TestLsNodeJSON(t *testing.T) {
 			Mode:       os.ModeSymlink | 0777,
 			LinkTarget: "not printed",
 		},
-			expect: `{"name":"link","type":"symlink","path":"/foo/link","uid":0,"gid":0,"mode":134218239,"permissions":"Lrwxrwxrwx","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","struct_type":"node"}`,
 	},
 
 	{
@@ -75,14 +72,33 @@ func TestLsNodeJSON(t *testing.T) {
 			AccessTime: time.Date(2021, 2, 3, 4, 5, 6, 7, time.UTC),
 			ChangeTime: time.Date(2022, 3, 4, 5, 6, 7, 8, time.UTC),
 		},
-			expect: `{"name":"directory","type":"dir","path":"/some/directory","uid":0,"gid":0,"mode":2147484141,"permissions":"drwxr-xr-x","mtime":"2020-01-02T03:04:05Z","atime":"2021-02-03T04:05:06.000000007Z","ctime":"2022-03-04T05:06:07.000000008Z","struct_type":"node"}`,
 	},
+
+	// Test encoding of setuid/setgid/sticky bit
+	{
+		path: "/some/sticky",
+		Node: restic.Node{
+			Name: "sticky",
+			Type: "dir",
+			Mode: os.ModeDir | 0755 | os.ModeSetuid | os.ModeSetgid | os.ModeSticky,
+		},
+	},
+}
+
+func TestLsNodeJSON(t *testing.T) {
+	for i, expect := range []string{
+		`{"name":"baz","type":"file","path":"/bar/baz","uid":10000000,"gid":20000000,"size":12345,"permissions":"----------","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","struct_type":"node"}`,
+		`{"name":"empty","type":"file","path":"/foo/empty","uid":1001,"gid":1001,"size":0,"permissions":"----------","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","struct_type":"node"}`,
+		`{"name":"link","type":"symlink","path":"/foo/link","uid":0,"gid":0,"mode":134218239,"permissions":"Lrwxrwxrwx","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","struct_type":"node"}`,
+		`{"name":"directory","type":"dir","path":"/some/directory","uid":0,"gid":0,"mode":2147484141,"permissions":"drwxr-xr-x","mtime":"2020-01-02T03:04:05Z","atime":"2021-02-03T04:05:06.000000007Z","ctime":"2022-03-04T05:06:07.000000008Z","struct_type":"node"}`,
+		`{"name":"sticky","type":"dir","path":"/some/sticky","uid":0,"gid":0,"mode":2161115629,"permissions":"dugtrwxr-xr-x","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","struct_type":"node"}`,
 	} {
+		c := lsTestNodes[i]
 		buf := new(bytes.Buffer)
 		enc := json.NewEncoder(buf)
 		err := lsNodeJSON(enc, c.path, &c.Node)
 		rtest.OK(t, err)
-		rtest.Equals(t, c.expect+"\n", buf.String())
+		rtest.Equals(t, expect+"\n", buf.String())
 
 		// Sanity check: output must be valid JSON.
 		var v interface{}
@@ -90,3 +106,54 @@ func TestLsNodeJSON(t *testing.T) {
 		rtest.OK(t, err)
 	}
 }
+
+func TestLsNcduNode(t *testing.T) {
+	for i, expect := range []string{
+		`{"name":"baz","asize":12345,"dsize":12345,"dev":0,"ino":0,"nlink":1,"notreg":false,"uid":10000000,"gid":20000000,"mode":0,"mtime":-62135596800}`,
+		`{"name":"empty","asize":0,"dsize":0,"dev":0,"ino":0,"nlink":3840,"notreg":false,"uid":1001,"gid":1001,"mode":0,"mtime":-62135596800}`,
+		`{"name":"link","asize":0,"dsize":0,"dev":0,"ino":0,"nlink":0,"notreg":true,"uid":0,"gid":0,"mode":511,"mtime":-62135596800}`,
+		`{"name":"directory","asize":0,"dsize":0,"dev":0,"ino":0,"nlink":0,"notreg":false,"uid":0,"gid":0,"mode":493,"mtime":1577934245}`,
+		`{"name":"sticky","asize":0,"dsize":0,"dev":0,"ino":0,"nlink":0,"notreg":false,"uid":0,"gid":0,"mode":4077,"mtime":-62135596800}`,
+	} {
+		c := lsTestNodes[i]
+		out, err := lsNcduNode(c.path, &c.Node)
+		rtest.OK(t, err)
+		rtest.Equals(t, expect, string(out))
+
+		// Sanity check: output must be valid JSON.
+		var v interface{}
+		err = json.Unmarshal(out, &v)
+		rtest.OK(t, err)
+	}
+}
+
+func TestLsNcdu(t *testing.T) {
+	var buf bytes.Buffer
+	printer := &ncduLsPrinter{
+		out: &buf,
+	}
+
+	printer.Snapshot(&restic.Snapshot{
+		Hostname: "host",
+		Paths:    []string{"/example"},
+	})
+	printer.Node("/directory", &restic.Node{
+		Type: "dir",
+		Name: "directory",
+	})
+	printer.Node("/directory/data", &restic.Node{
+		Type: "file",
+		Name: "data",
+		Size: 42,
+	})
+	printer.LeaveDir("/directory")
+	printer.Close()
+
+	rtest.Equals(t, `[1, 2, {"time":"0001-01-01T00:00:00Z","tree":null,"paths":["/example"],"hostname":"host"},
+  [
+    {"name":"directory","asize":0,"dsize":0,"dev":0,"ino":0,"nlink":0,"notreg":false,"uid":0,"gid":0,"mode":0,"mtime":-62135596800},
+    {"name":"data","asize":42,"dsize":42,"dev":0,"ino":0,"nlink":0,"notreg":false,"uid":0,"gid":0,"mode":0,"mtime":-62135596800}
+  ]
+]
+`, buf.String())
+}

cmd/restic/cmd_stats.go

@@ -203,7 +203,9 @@ func statsWalkSnapshot(ctx context.Context, snapshot *restic.Snapshot, repo rest
 	}
 
 	hardLinkIndex := restorer.NewHardlinkIndex[struct{}]()
-	err := walker.Walk(ctx, repo, *snapshot.Tree, statsWalkTree(repo, opts, stats, hardLinkIndex))
+	err := walker.Walk(ctx, repo, *snapshot.Tree, walker.WalkVisitor{
+		ProcessNode: statsWalkTree(repo, opts, stats, hardLinkIndex),
+	})
 	if err != nil {
 		return fmt.Errorf("walking tree %s: %v", *snapshot.Tree, err)
 	}

doc/045_working_with_repos.rst

@@ -82,6 +82,76 @@ Furthermore you can group the output by the same filters (host, paths, tags):
     1 snapshots
 
 
+Listing files in a snapshot
+===========================
+
+To get a list of the files in a specific snapshot you can use the ``ls`` command:
+
+.. code-block:: console
+
+    $ restic ls 073a90db
+
+    snapshot 073a90db of [/home/user/work.txt] filtered by [] at 2024-01-21 16:51:18.474558607 +0100 CET):
+    /home
+    /home/user
+    /home/user/work.txt
+
+The special snapshot ID ``latest`` can be used to list files and directories of the latest snapshot in the repository.
+The ``--host`` flag can be used in conjunction to select the latest snapshot originating from a certain host only.
+
+.. code-block:: console
+
+    $ restic ls --host kasimir latest
+
+    snapshot 073a90db of [/home/user/work.txt] filtered by [] at 2024-01-21 16:51:18.474558607 +0100 CET):
+    /home
+    /home/user
+    /home/user/work.txt
+
+By default, ``ls`` prints all files in a snapshot.
+
+File listings can optionally be filtered by directories. Any positional arguments after the snapshot ID are interpreted
+as absolute directory paths, and only files inside those directories will be listed. Files in subdirectories are not
+listed when filtering by directories. If the ``--recursive`` flag is used, then subdirectories are also included.
+Any directory paths specified must be absolute (starting with a path separator); paths use the forward slash '/'
+as separator.
+
+.. code-block:: console
+
+    $ restic ls latest /home
+    
+    snapshot 073a90db of [/home/user/work.txt] filtered by [/home] at 2024-01-21 16:51:18.474558607 +0100 CET):
+    /home
+    /home/user
+
+.. code-block:: console
+
+    $ restic ls --recursive latest /home
+
+    snapshot 073a90db of [/home/user/work.txt] filtered by [/home] at 2024-01-21 16:51:18.474558607 +0100 CET):
+    /home
+    /home/user
+    /home/user/work.txt
+
+To show more details about the files in a snapshot, you can use the ``--long`` option.  The colums include
+file permissions, UID, GID, file size, modification time and file path. For scripting usage, the
+``ls`` command supports the ``--json`` flag; the JSON output format is described at :ref:`ls json`.
+
+.. code-block:: console
+
+    $ restic ls --long latest
+
+    snapshot 073a90db of [/home/user/work.txt] filtered by [] at 2024-01-21 16:51:18.474558607 +0100 CET):
+    drwxr-xr-x     0     0      0 2024-01-21 16:50:52 /home
+    drwxr-xr-x     0     0      0 2024-01-21 16:51:03 /home/user
+    -rw-r--r--     0     0     18 2024-01-21 16:51:03 /home/user/work.txt
+
+NCDU (NCurses Disk Usage) is a tool to analyse disk usage of directories. The ``ls`` command supports
+outputting information about a snapshot in the NCDU format using the ``--ncdu`` option.
+
+You can use it as follows: ``restic ls latest --ncdu | ncdu -f -``
+
+
 Copying snapshots between repositories
 ======================================
 
@@ -242,6 +312,7 @@ Currently, rewriting the hostname and the time of the backup is supported.
 This is possible using the ``rewrite`` command with the option ``--new-host`` followed by the desired new hostname or the option ``--new-time`` followed by the desired new timestamp.
 
 .. code-block:: console
+
     $ restic rewrite --new-host newhost --new-time "1999-01-01 11:11:11"
 
     repository b7dbade3 opened (version 2, compression level auto)

doc/075_scripting.rst

@@ -409,6 +409,8 @@ The ``key list`` command returns an array of objects with the following structur
 +--------------+------------------------------------+
 
 
+.. _ls json:
+
 ls
 --
 

internal/dump/common.go

@@ -70,7 +70,7 @@ func sendNodes(ctx context.Context, repo restic.BlobLoader, root *restic.Node, c
 		return nil
 	}
 
-	err := walker.Walk(ctx, repo, *root.Subtree, func(_ restic.ID, nodepath string, node *restic.Node, err error) error {
+	err := walker.Walk(ctx, repo, *root.Subtree, walker.WalkVisitor{ProcessNode: func(_ restic.ID, nodepath string, node *restic.Node, err error) error {
 		if err != nil {
 			return err
 		}
@@ -91,7 +91,7 @@ func sendNodes(ctx context.Context, repo restic.BlobLoader, root *restic.Node, c
 		}
 
 		return nil
-	})
+	}})
 
 	return err
 }

internal/walker/walker.go

@@ -23,12 +23,20 @@ var ErrSkipNode = errors.New("skip this node")
 // tree are skipped.
 type WalkFunc func(parentTreeID restic.ID, path string, node *restic.Node, nodeErr error) (err error)
 
+type WalkVisitor struct {
+	// If the node is a `dir`, it will be entered afterwards unless `ErrSkipNode`
+	// was returned. This function is mandatory
+	ProcessNode WalkFunc
+	// Optional callback
+	LeaveDir func(path string)
+}
+
 // Walk calls walkFn recursively for each node in root. If walkFn returns an
 // error, it is passed up the call stack. The trees in ignoreTrees are not
 // walked. If walkFn ignores trees, these are added to the set.
-func Walk(ctx context.Context, repo restic.BlobLoader, root restic.ID, walkFn WalkFunc) error {
+func Walk(ctx context.Context, repo restic.BlobLoader, root restic.ID, visitor WalkVisitor) error {
 	tree, err := restic.LoadTree(ctx, repo, root)
-	err = walkFn(root, "/", nil, err)
+	err = visitor.ProcessNode(root, "/", nil, err)
 
 	if err != nil {
 		if err == ErrSkipNode {
@@ -37,13 +45,13 @@ func Walk(ctx context.Context, repo restic.BlobLoader, root restic.ID, walkFn Wa
 		return err
 	}
 
-	return walk(ctx, repo, "/", root, tree, walkFn)
+	return walk(ctx, repo, "/", root, tree, visitor)
 }
 
 // walk recursively traverses the tree, ignoring subtrees when the ID of the
 // subtree is in ignoreTrees. If err is nil and ignore is true, the subtree ID
 // will be added to ignoreTrees by walk.
-func walk(ctx context.Context, repo restic.BlobLoader, prefix string, parentTreeID restic.ID, tree *restic.Tree, walkFn WalkFunc) (err error) {
+func walk(ctx context.Context, repo restic.BlobLoader, prefix string, parentTreeID restic.ID, tree *restic.Tree, visitor WalkVisitor) (err error) {
 	sort.Slice(tree.Nodes, func(i, j int) bool {
 		return tree.Nodes[i].Name < tree.Nodes[j].Name
 	})
@@ -56,11 +64,11 @@ func walk(ctx context.Context, repo restic.BlobLoader, prefix string, parentTree
 		}
 
 		if node.Type != "dir" {
-			err := walkFn(parentTreeID, p, node, nil)
+			err := visitor.ProcessNode(parentTreeID, p, node, nil)
 			if err != nil {
 				if err == ErrSkipNode {
 					// skip the remaining entries in this tree
-					return nil
+					break
 				}
 
 				return err
@@ -74,18 +82,22 @@ func walk(ctx context.Context, repo restic.BlobLoader, prefix string, parentTree
 		}
 
 		subtree, err := restic.LoadTree(ctx, repo, *node.Subtree)
-		err = walkFn(parentTreeID, p, node, err)
+		err = visitor.ProcessNode(parentTreeID, p, node, err)
 		if err != nil {
 			if err == ErrSkipNode {
 				continue
 			}
 		}
 
-		err = walk(ctx, repo, p, *node.Subtree, subtree, walkFn)
+		err = walk(ctx, repo, p, *node.Subtree, subtree, visitor)
 		if err != nil {
 			return err
 		}
 	}
 
+	if visitor.LeaveDir != nil {
+		visitor.LeaveDir(prefix)
+	}
+
 	return nil
 }

internal/walker/walker_test.go

@@ -93,12 +93,12 @@ func (t TreeMap) Connections() uint {
 
 // checkFunc returns a function suitable for walking the tree to check
 // something, and a function which will check the final result.
-type checkFunc func(t testing.TB) (walker WalkFunc, final func(testing.TB))
+type checkFunc func(t testing.TB) (walker WalkFunc, leaveDir func(path string), final func(testing.TB))
 
 // checkItemOrder ensures that the order of the 'path' arguments is the one passed in as 'want'.
 func checkItemOrder(want []string) checkFunc {
 	pos := 0
-	return func(t testing.TB) (walker WalkFunc, final func(testing.TB)) {
+	return func(t testing.TB) (walker WalkFunc, leaveDir func(path string), final func(testing.TB)) {
 		walker = func(treeID restic.ID, path string, node *restic.Node, err error) error {
 			if err != nil {
 				t.Errorf("error walking %v: %v", path, err)
@@ -117,20 +117,24 @@ func checkItemOrder(want []string) checkFunc {
 			return nil
 		}
 
+		leaveDir = func(path string) {
+			_ = walker(restic.ID{}, "leave: "+path, nil, nil)
+		}
+
 		final = func(t testing.TB) {
 			if pos != len(want) {
 				t.Errorf("not enough items returned, want %d, got %d", len(want), pos)
 			}
 		}
 
-		return walker, final
+		return walker, leaveDir, final
 	}
 }
 
 // checkParentTreeOrder ensures that the order of the 'parentID' arguments is the one passed in as 'want'.
 func checkParentTreeOrder(want []string) checkFunc {
 	pos := 0
-	return func(t testing.TB) (walker WalkFunc, final func(testing.TB)) {
+	return func(t testing.TB) (walker WalkFunc, leaveDir func(path string), final func(testing.TB)) {
 		walker = func(treeID restic.ID, path string, node *restic.Node, err error) error {
 			if err != nil {
 				t.Errorf("error walking %v: %v", path, err)
@@ -155,7 +159,7 @@ func checkParentTreeOrder(want []string) checkFunc {
 			}
 		}
 
-		return walker, final
+		return walker, nil, final
 	}
 }
 
@@ -164,7 +168,7 @@ func checkParentTreeOrder(want []string) checkFunc {
 func checkSkipFor(skipFor map[string]struct{}, wantPaths []string) checkFunc {
 	var pos int
 
-	return func(t testing.TB) (walker WalkFunc, final func(testing.TB)) {
+	return func(t testing.TB) (walker WalkFunc, leaveDir func(path string), final func(testing.TB)) {
 		walker = func(treeID restic.ID, path string, node *restic.Node, err error) error {
 			if err != nil {
 				t.Errorf("error walking %v: %v", path, err)
@@ -188,13 +192,17 @@ func checkSkipFor(skipFor map[string]struct{}, wantPaths []string) checkFunc {
 			return nil
 		}
 
+		leaveDir = func(path string) {
+			_ = walker(restic.ID{}, "leave: "+path, nil, nil)
+		}
+
 		final = func(t testing.TB) {
 			if pos != len(wantPaths) {
 				t.Errorf("wrong number of paths returned, want %d, got %d", len(wantPaths), pos)
 			}
 		}
 
-		return walker, final
+		return walker, leaveDir, final
 	}
 }
 
@@ -216,6 +224,8 @@ func TestWalker(t *testing.T) {
 					"/foo",
 					"/subdir",
 					"/subdir/subfile",
+					"leave: /subdir",
+					"leave: /",
 				}),
 				checkParentTreeOrder([]string{
 					"a760536a8fd64dd63f8dd95d85d788d71fd1bee6828619350daf6959dcb499a0", // tree /
@@ -230,6 +240,7 @@ func TestWalker(t *testing.T) {
 						"/",
 						"/foo",
 						"/subdir",
+						"leave: /",
 					},
 				),
 				checkSkipFor(
@@ -260,10 +271,14 @@ func TestWalker(t *testing.T) {
 					"/foo",
 					"/subdir1",
 					"/subdir1/subfile1",
+					"leave: /subdir1",
 					"/subdir2",
 					"/subdir2/subfile2",
 					"/subdir2/subsubdir2",
 					"/subdir2/subsubdir2/subsubfile3",
+					"leave: /subdir2/subsubdir2",
+					"leave: /subdir2",
+					"leave: /",
 				}),
 				checkParentTreeOrder([]string{
 					"7a0e59b986cc83167d9fbeeefc54e4629770124c5825d391f7ee0598667fcdf1", // tree /
@@ -286,6 +301,9 @@ func TestWalker(t *testing.T) {
 						"/subdir2/subfile2",
 						"/subdir2/subsubdir2",
 						"/subdir2/subsubdir2/subsubfile3",
+						"leave: /subdir2/subsubdir2",
+						"leave: /subdir2",
+						"leave: /",
 					},
 				),
 				checkSkipFor(
@@ -299,6 +317,8 @@ func TestWalker(t *testing.T) {
 						"/subdir2",
 						"/subdir2/subfile2",
 						"/subdir2/subsubdir2",
+						"leave: /subdir2",
+						"leave: /",
 					},
 				),
 				checkSkipFor(
@@ -307,6 +327,7 @@ func TestWalker(t *testing.T) {
 					}, []string{
 						"/",
 						"/foo",
+						"leave: /",
 					},
 				),
 			},
@@ -339,15 +360,19 @@ func TestWalker(t *testing.T) {
 					"/subdir1/subfile1",
 					"/subdir1/subfile2",
 					"/subdir1/subfile3",
+					"leave: /subdir1",
 					"/subdir2",
 					"/subdir2/subfile1",
 					"/subdir2/subfile2",
 					"/subdir2/subfile3",
+					"leave: /subdir2",
 					"/subdir3",
 					"/subdir3/subfile1",
 					"/subdir3/subfile2",
 					"/subdir3/subfile3",
+					"leave: /subdir3",
 					"/zzz other",
+					"leave: /",
 				}),
 				checkParentTreeOrder([]string{
 					"c2efeff7f217a4dfa12a16e8bb3cefedd37c00873605c29e5271c6061030672f", // tree /
@@ -385,13 +410,20 @@ func TestWalker(t *testing.T) {
 				checkItemOrder([]string{
 					"/",
 					"/subdir1",
+					"leave: /subdir1",
 					"/subdir2",
+					"leave: /subdir2",
 					"/subdir3",
 					"/subdir3/file",
+					"leave: /subdir3",
 					"/subdir4",
 					"/subdir4/file",
+					"leave: /subdir4",
 					"/subdir5",
+					"leave: /subdir5",
 					"/subdir6",
+					"leave: /subdir6",
+					"leave: /",
 				}),
 			},
 		},
@@ -405,8 +437,11 @@ func TestWalker(t *testing.T) {
 					ctx, cancel := context.WithCancel(context.TODO())
 					defer cancel()
 
-					fn, last := check(t)
+					fn, leaveDir, last := check(t)
-					err := Walk(ctx, repo, root, fn)
+					err := Walk(ctx, repo, root, WalkVisitor{
+						ProcessNode: fn,
+						LeaveDir:    leaveDir,
+					})
 					if err != nil {
 						t.Error(err)
 					}