diff --git a/changelog/unreleased/issue-4549 b/changelog/unreleased/issue-4549 new file mode 100644 index 000000000..4829a9881 --- /dev/null +++ b/changelog/unreleased/issue-4549 @@ -0,0 +1,11 @@ +Enhancement: Add `--ncdu` option to `ls` command + +NCDU (NCurses Disk Usage) is a tool to analyse disk usage of directories. +It has an option to save a directory tree and analyse it later. +The `ls` command now supports the `--ncdu` option which outputs information +about a snapshot in the NCDU format. + +You can use it as follows: `restic ls latest --ncdu | ncdu -f -` + +https://github.com/restic/restic/issues/4549 +https://github.com/restic/restic/pull/4550 diff --git a/cmd/restic/cmd_find.go b/cmd/restic/cmd_find.go index 33fff864f..04e6ae3dd 100644 --- a/cmd/restic/cmd_find.go +++ b/cmd/restic/cmd_find.go @@ -260,7 +260,7 @@ func (f *Finder) findInSnapshot(ctx context.Context, sn *restic.Snapshot) error } f.out.newsn = sn - return walker.Walk(ctx, f.repo, *sn.Tree, func(parentTreeID restic.ID, nodepath string, node *restic.Node, err error) error { + return walker.Walk(ctx, f.repo, *sn.Tree, walker.WalkVisitor{ProcessNode: func(parentTreeID restic.ID, nodepath string, node *restic.Node, err error) error { if err != nil { debug.Log("Error loading tree %v: %v", parentTreeID, err) @@ -327,7 +327,7 @@ func (f *Finder) findInSnapshot(ctx context.Context, sn *restic.Snapshot) error debug.Log(" found match\n") f.out.PrintPattern(nodepath, node) return nil - }) + }}) } func (f *Finder) findIDs(ctx context.Context, sn *restic.Snapshot) error { @@ -338,7 +338,7 @@ func (f *Finder) findIDs(ctx context.Context, sn *restic.Snapshot) error { } f.out.newsn = sn - return walker.Walk(ctx, f.repo, *sn.Tree, func(parentTreeID restic.ID, nodepath string, node *restic.Node, err error) error { + return walker.Walk(ctx, f.repo, *sn.Tree, walker.WalkVisitor{ProcessNode: func(parentTreeID restic.ID, nodepath string, node *restic.Node, err error) error { if err != nil { debug.Log("Error loading tree %v: %v", parentTreeID, err) @@ -388,7 +388,7 @@ func (f *Finder) findIDs(ctx context.Context, sn *restic.Snapshot) error { } return nil - }) + }}) } var errAllPacksFound = errors.New("all packs found") diff --git a/cmd/restic/cmd_ls.go b/cmd/restic/cmd_ls.go index d30e2819c..f412546ae 100644 --- a/cmd/restic/cmd_ls.go +++ b/cmd/restic/cmd_ls.go @@ -3,6 +3,8 @@ package main import ( "context" "encoding/json" + "fmt" + "io" "os" "strings" "time" @@ -51,6 +53,7 @@ type LsOptions struct { restic.SnapshotFilter Recursive bool HumanReadable bool + Ncdu bool } var lsOptions LsOptions @@ -63,16 +66,47 @@ func init() { flags.BoolVarP(&lsOptions.ListLong, "long", "l", false, "use a long listing format showing size and mode") flags.BoolVar(&lsOptions.Recursive, "recursive", false, "include files in subfolders of the listed directories") flags.BoolVar(&lsOptions.HumanReadable, "human-readable", false, "print sizes in human readable format") + flags.BoolVar(&lsOptions.Ncdu, "ncdu", false, "output NCDU export format (pipe into 'ncdu -f -')") } -type lsSnapshot struct { - *restic.Snapshot - ID *restic.ID `json:"id"` - ShortID string `json:"short_id"` - StructType string `json:"struct_type"` // "snapshot" +type lsPrinter interface { + Snapshot(sn *restic.Snapshot) + Node(path string, node *restic.Node) + LeaveDir(path string) + Close() +} + +type jsonLsPrinter struct { + enc *json.Encoder +} + +func (p *jsonLsPrinter) Snapshot(sn *restic.Snapshot) { + type lsSnapshot struct { + *restic.Snapshot + ID *restic.ID `json:"id"` + ShortID string `json:"short_id"` + StructType string `json:"struct_type"` // "snapshot" + } + + err := p.enc.Encode(lsSnapshot{ + Snapshot: sn, + ID: sn.ID(), + ShortID: sn.ID().Str(), + StructType: "snapshot", + }) + if err != nil { + Warnf("JSON encode failed: %v\n", err) + } } // Print node in our custom JSON format, followed by a newline. +func (p *jsonLsPrinter) Node(path string, node *restic.Node) { + err := lsNodeJSON(p.enc, path, node) + if err != nil { + Warnf("JSON encode failed: %v\n", err) + } +} + func lsNodeJSON(enc *json.Encoder, path string, node *restic.Node) error { n := &struct { Name string `json:"name"` @@ -114,10 +148,117 @@ func lsNodeJSON(enc *json.Encoder, path string, node *restic.Node) error { return enc.Encode(n) } +func (p *jsonLsPrinter) LeaveDir(_ string) {} +func (p *jsonLsPrinter) Close() {} + +type ncduLsPrinter struct { + out io.Writer + depth int +} + +// lsSnapshotNcdu prints a restic snapshot in Ncdu save format. +// It opens the JSON list. Nodes are added with lsNodeNcdu and the list is closed by lsCloseNcdu. +// Format documentation: https://dev.yorhel.nl/ncdu/jsonfmt +func (p *ncduLsPrinter) Snapshot(sn *restic.Snapshot) { + const NcduMajorVer = 1 + const NcduMinorVer = 2 + + snapshotBytes, err := json.Marshal(sn) + if err != nil { + Warnf("JSON encode failed: %v\n", err) + } + p.depth++ + fmt.Fprintf(p.out, "[%d, %d, %s", NcduMajorVer, NcduMinorVer, string(snapshotBytes)) +} + +func lsNcduNode(_ string, node *restic.Node) ([]byte, error) { + type NcduNode struct { + Name string `json:"name"` + Asize uint64 `json:"asize"` + Dsize uint64 `json:"dsize"` + Dev uint64 `json:"dev"` + Ino uint64 `json:"ino"` + NLink uint64 `json:"nlink"` + NotReg bool `json:"notreg"` + UID uint32 `json:"uid"` + GID uint32 `json:"gid"` + Mode uint16 `json:"mode"` + Mtime int64 `json:"mtime"` + } + + outNode := NcduNode{ + Name: node.Name, + Asize: node.Size, + Dsize: node.Size, + Dev: node.DeviceID, + Ino: node.Inode, + NLink: node.Links, + NotReg: node.Type != "dir" && node.Type != "file", + UID: node.UID, + GID: node.GID, + Mode: uint16(node.Mode & os.ModePerm), + Mtime: node.ModTime.Unix(), + } + // bits according to inode(7) manpage + if node.Mode&os.ModeSetuid != 0 { + outNode.Mode |= 0o4000 + } + if node.Mode&os.ModeSetgid != 0 { + outNode.Mode |= 0o2000 + } + if node.Mode&os.ModeSticky != 0 { + outNode.Mode |= 0o1000 + } + + return json.Marshal(outNode) +} + +func (p *ncduLsPrinter) Node(path string, node *restic.Node) { + out, err := lsNcduNode(path, node) + if err != nil { + Warnf("JSON encode failed: %v\n", err) + } + + if node.Type == "dir" { + fmt.Fprintf(p.out, ",\n%s[\n%s%s", strings.Repeat(" ", p.depth), strings.Repeat(" ", p.depth+1), string(out)) + p.depth++ + } else { + fmt.Fprintf(p.out, ",\n%s%s", strings.Repeat(" ", p.depth), string(out)) + } +} + +func (p *ncduLsPrinter) LeaveDir(_ string) { + p.depth-- + fmt.Fprintf(p.out, "\n%s]", strings.Repeat(" ", p.depth)) +} + +func (p *ncduLsPrinter) Close() { + fmt.Fprint(p.out, "\n]\n") +} + +type textLsPrinter struct { + dirs []string + ListLong bool + HumanReadable bool +} + +func (p *textLsPrinter) Snapshot(sn *restic.Snapshot) { + Verbosef("%v filtered by %v:\n", sn, p.dirs) +} +func (p *textLsPrinter) Node(path string, node *restic.Node) { + Printf("%s\n", formatNode(path, node, p.ListLong, p.HumanReadable)) +} + +func (p *textLsPrinter) LeaveDir(_ string) {} +func (p *textLsPrinter) Close() {} + func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []string) error { if len(args) == 0 { return errors.Fatal("no snapshot ID specified, specify snapshot ID or use special ID 'latest'") } + if opts.Ncdu && gopts.JSON { + return errors.Fatal("only either '--json' or '--ncdu' can be specified") + } // extract any specific directories to walk var dirs []string @@ -179,38 +320,21 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri return err } - var ( - printSnapshot func(sn *restic.Snapshot) - printNode func(path string, node *restic.Node) - ) + var printer lsPrinter if gopts.JSON { - enc := json.NewEncoder(globalOptions.stdout) - - printSnapshot = func(sn *restic.Snapshot) { - err := enc.Encode(lsSnapshot{ - Snapshot: sn, - ID: sn.ID(), - ShortID: sn.ID().Str(), - StructType: "snapshot", - }) - if err != nil { - Warnf("JSON encode failed: %v\n", err) - } + printer = &jsonLsPrinter{ + enc: json.NewEncoder(globalOptions.stdout), } - - printNode = func(path string, node *restic.Node) { - err := lsNodeJSON(enc, path, node) - if err != nil { - Warnf("JSON encode failed: %v\n", err) - } + } else if opts.Ncdu { + printer = &ncduLsPrinter{ + out: globalOptions.stdout, } } else { - printSnapshot = func(sn *restic.Snapshot) { - Verbosef("%v filtered by %v:\n", sn, dirs) - } - printNode = func(path string, node *restic.Node) { - Printf("%s\n", formatNode(path, node, opts.ListLong, opts.HumanReadable)) + printer = &textLsPrinter{ + dirs: dirs, + ListLong: opts.ListLong, + HumanReadable: opts.HumanReadable, } } @@ -228,9 +352,9 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri return err } - printSnapshot(sn) + printer.Snapshot(sn) - err = walker.Walk(ctx, repo, *sn.Tree, func(_ restic.ID, nodepath string, node *restic.Node, err error) error { + processNode := func(_ restic.ID, nodepath string, node *restic.Node, err error) error { if err != nil { return err } @@ -240,7 +364,7 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri if withinDir(nodepath) { // if we're within a dir, print the node - printNode(nodepath, node) + printer.Node(nodepath, node) // if recursive listing is requested, signal the walker that it // should continue walking recursively @@ -261,11 +385,22 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri return walker.ErrSkipNode } return nil + } + + err = walker.Walk(ctx, repo, *sn.Tree, walker.WalkVisitor{ + ProcessNode: processNode, + LeaveDir: func(path string) { + // the root path `/` has no corresponding node and is thus also skipped by processNode + if withinDir(path) && path != "/" { + printer.LeaveDir(path) + } + }, }) if err != nil { return err } + printer.Close() return nil } diff --git a/cmd/restic/cmd_ls_integration_test.go b/cmd/restic/cmd_ls_integration_test.go index 39bf9c3b0..1b3c964e4 100644 --- a/cmd/restic/cmd_ls_integration_test.go +++ b/cmd/restic/cmd_ls_integration_test.go @@ -2,18 +2,46 @@ package main import ( "context" + "encoding/json" + "path/filepath" "strings" "testing" rtest "github.com/restic/restic/internal/test" ) -func testRunLs(t testing.TB, gopts GlobalOptions, snapshotID string) []string { +func testRunLsWithOpts(t testing.TB, gopts GlobalOptions, opts LsOptions, args []string) []byte { buf, err := withCaptureStdout(func() error { gopts.Quiet = true - opts := LsOptions{} - return runLs(context.TODO(), opts, gopts, []string{snapshotID}) + return runLs(context.TODO(), opts, gopts, args) }) rtest.OK(t, err) - return strings.Split(buf.String(), "\n") + return buf.Bytes() +} + +func testRunLs(t testing.TB, gopts GlobalOptions, snapshotID string) []string { + out := testRunLsWithOpts(t, gopts, LsOptions{}, []string{snapshotID}) + return strings.Split(string(out), "\n") +} + +func assertIsValidJSON(t *testing.T, data []byte) { + // Sanity check: output must be valid JSON. + var v interface{} + err := json.Unmarshal(data, &v) + rtest.OK(t, err) +} + +func TestRunLsNcdu(t *testing.T) { + env, cleanup := withTestEnvironment(t) + defer cleanup() + + testRunInit(t, env.gopts) + opts := BackupOptions{} + testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, env.gopts) + + ncdu := testRunLsWithOpts(t, env.gopts, LsOptions{Ncdu: true}, []string{"latest"}) + assertIsValidJSON(t, ncdu) + + ncdu = testRunLsWithOpts(t, env.gopts, LsOptions{Ncdu: true}, []string{"latest", "/testdata"}) + assertIsValidJSON(t, ncdu) } diff --git a/cmd/restic/cmd_ls_test.go b/cmd/restic/cmd_ls_test.go index 8a4fa51ee..41c235eab 100644 --- a/cmd/restic/cmd_ls_test.go +++ b/cmd/restic/cmd_ls_test.go @@ -11,78 +11,94 @@ import ( rtest "github.com/restic/restic/internal/test" ) +type lsTestNode struct { + path string + restic.Node +} + +var lsTestNodes = []lsTestNode{ + // Mode is omitted when zero. + // Permissions, by convention is "-" per mode bit + { + path: "/bar/baz", + Node: restic.Node{ + Name: "baz", + Type: "file", + Size: 12345, + UID: 10000000, + GID: 20000000, + + User: "nobody", + Group: "nobodies", + Links: 1, + }, + }, + + // Even empty files get an explicit size. + { + path: "/foo/empty", + Node: restic.Node{ + Name: "empty", + Type: "file", + Size: 0, + UID: 1001, + GID: 1001, + + User: "not printed", + Group: "not printed", + Links: 0xF00, + }, + }, + + // Non-regular files do not get a size. + // Mode is printed in decimal, including the type bits. + { + path: "/foo/link", + Node: restic.Node{ + Name: "link", + Type: "symlink", + Mode: os.ModeSymlink | 0777, + LinkTarget: "not printed", + }, + }, + + { + path: "/some/directory", + Node: restic.Node{ + Name: "directory", + Type: "dir", + Mode: os.ModeDir | 0755, + ModTime: time.Date(2020, 1, 2, 3, 4, 5, 0, time.UTC), + AccessTime: time.Date(2021, 2, 3, 4, 5, 6, 7, time.UTC), + ChangeTime: time.Date(2022, 3, 4, 5, 6, 7, 8, time.UTC), + }, + }, + + // Test encoding of setuid/setgid/sticky bit + { + path: "/some/sticky", + Node: restic.Node{ + Name: "sticky", + Type: "dir", + Mode: os.ModeDir | 0755 | os.ModeSetuid | os.ModeSetgid | os.ModeSticky, + }, + }, +} + func TestLsNodeJSON(t *testing.T) { - for _, c := range []struct { - path string - restic.Node - expect string - }{ - // Mode is omitted when zero. - // Permissions, by convention is "-" per mode bit - { - path: "/bar/baz", - Node: restic.Node{ - Name: "baz", - Type: "file", - Size: 12345, - UID: 10000000, - GID: 20000000, - - User: "nobody", - Group: "nobodies", - Links: 1, - }, - expect: `{"name":"baz","type":"file","path":"/bar/baz","uid":10000000,"gid":20000000,"size":12345,"permissions":"----------","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","struct_type":"node"}`, - }, - - // Even empty files get an explicit size. - { - path: "/foo/empty", - Node: restic.Node{ - Name: "empty", - Type: "file", - Size: 0, - UID: 1001, - GID: 1001, - - User: "not printed", - Group: "not printed", - Links: 0xF00, - }, - expect: `{"name":"empty","type":"file","path":"/foo/empty","uid":1001,"gid":1001,"size":0,"permissions":"----------","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","struct_type":"node"}`, - }, - - // Non-regular files do not get a size. - // Mode is printed in decimal, including the type bits. - { - path: "/foo/link", - Node: restic.Node{ - Name: "link", - Type: "symlink", - Mode: os.ModeSymlink | 0777, - LinkTarget: "not printed", - }, - expect: `{"name":"link","type":"symlink","path":"/foo/link","uid":0,"gid":0,"mode":134218239,"permissions":"Lrwxrwxrwx","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","struct_type":"node"}`, - }, - - { - path: "/some/directory", - Node: restic.Node{ - Name: "directory", - Type: "dir", - Mode: os.ModeDir | 0755, - ModTime: time.Date(2020, 1, 2, 3, 4, 5, 0, time.UTC), - AccessTime: time.Date(2021, 2, 3, 4, 5, 6, 7, time.UTC), - ChangeTime: time.Date(2022, 3, 4, 5, 6, 7, 8, time.UTC), - }, - expect: `{"name":"directory","type":"dir","path":"/some/directory","uid":0,"gid":0,"mode":2147484141,"permissions":"drwxr-xr-x","mtime":"2020-01-02T03:04:05Z","atime":"2021-02-03T04:05:06.000000007Z","ctime":"2022-03-04T05:06:07.000000008Z","struct_type":"node"}`, - }, + for i, expect := range []string{ + `{"name":"baz","type":"file","path":"/bar/baz","uid":10000000,"gid":20000000,"size":12345,"permissions":"----------","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","struct_type":"node"}`, + `{"name":"empty","type":"file","path":"/foo/empty","uid":1001,"gid":1001,"size":0,"permissions":"----------","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","struct_type":"node"}`, + `{"name":"link","type":"symlink","path":"/foo/link","uid":0,"gid":0,"mode":134218239,"permissions":"Lrwxrwxrwx","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","struct_type":"node"}`, + `{"name":"directory","type":"dir","path":"/some/directory","uid":0,"gid":0,"mode":2147484141,"permissions":"drwxr-xr-x","mtime":"2020-01-02T03:04:05Z","atime":"2021-02-03T04:05:06.000000007Z","ctime":"2022-03-04T05:06:07.000000008Z","struct_type":"node"}`, + `{"name":"sticky","type":"dir","path":"/some/sticky","uid":0,"gid":0,"mode":2161115629,"permissions":"dugtrwxr-xr-x","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","struct_type":"node"}`, } { + c := lsTestNodes[i] buf := new(bytes.Buffer) enc := json.NewEncoder(buf) err := lsNodeJSON(enc, c.path, &c.Node) rtest.OK(t, err) - rtest.Equals(t, c.expect+"\n", buf.String()) + rtest.Equals(t, expect+"\n", buf.String()) // Sanity check: output must be valid JSON. var v interface{} @@ -90,3 +106,54 @@ func TestLsNodeJSON(t *testing.T) { rtest.OK(t, err) } } + +func TestLsNcduNode(t *testing.T) { + for i, expect := range []string{ + `{"name":"baz","asize":12345,"dsize":12345,"dev":0,"ino":0,"nlink":1,"notreg":false,"uid":10000000,"gid":20000000,"mode":0,"mtime":-62135596800}`, + `{"name":"empty","asize":0,"dsize":0,"dev":0,"ino":0,"nlink":3840,"notreg":false,"uid":1001,"gid":1001,"mode":0,"mtime":-62135596800}`, + `{"name":"link","asize":0,"dsize":0,"dev":0,"ino":0,"nlink":0,"notreg":true,"uid":0,"gid":0,"mode":511,"mtime":-62135596800}`, + `{"name":"directory","asize":0,"dsize":0,"dev":0,"ino":0,"nlink":0,"notreg":false,"uid":0,"gid":0,"mode":493,"mtime":1577934245}`, + `{"name":"sticky","asize":0,"dsize":0,"dev":0,"ino":0,"nlink":0,"notreg":false,"uid":0,"gid":0,"mode":4077,"mtime":-62135596800}`, + } { + c := lsTestNodes[i] + out, err := lsNcduNode(c.path, &c.Node) + rtest.OK(t, err) + rtest.Equals(t, expect, string(out)) + + // Sanity check: output must be valid JSON. + var v interface{} + err = json.Unmarshal(out, &v) + rtest.OK(t, err) + } +} + +func TestLsNcdu(t *testing.T) { + var buf bytes.Buffer + printer := &ncduLsPrinter{ + out: &buf, + } + + printer.Snapshot(&restic.Snapshot{ + Hostname: "host", + Paths: []string{"/example"}, + }) + printer.Node("/directory", &restic.Node{ + Type: "dir", + Name: "directory", + }) + printer.Node("/directory/data", &restic.Node{ + Type: "file", + Name: "data", + Size: 42, + }) + printer.LeaveDir("/directory") + printer.Close() + + rtest.Equals(t, `[1, 2, {"time":"0001-01-01T00:00:00Z","tree":null,"paths":["/example"],"hostname":"host"}, + [ + {"name":"directory","asize":0,"dsize":0,"dev":0,"ino":0,"nlink":0,"notreg":false,"uid":0,"gid":0,"mode":0,"mtime":-62135596800}, + {"name":"data","asize":42,"dsize":42,"dev":0,"ino":0,"nlink":0,"notreg":false,"uid":0,"gid":0,"mode":0,"mtime":-62135596800} + ] +] +`, buf.String()) +} diff --git a/cmd/restic/cmd_stats.go b/cmd/restic/cmd_stats.go index 87c3f8d22..1bece21d0 100644 --- a/cmd/restic/cmd_stats.go +++ b/cmd/restic/cmd_stats.go @@ -203,7 +203,9 @@ func statsWalkSnapshot(ctx context.Context, snapshot *restic.Snapshot, repo rest } hardLinkIndex := restorer.NewHardlinkIndex[struct{}]() - err := walker.Walk(ctx, repo, *snapshot.Tree, statsWalkTree(repo, opts, stats, hardLinkIndex)) + err := walker.Walk(ctx, repo, *snapshot.Tree, walker.WalkVisitor{ + ProcessNode: statsWalkTree(repo, opts, stats, hardLinkIndex), + }) if err != nil { return fmt.Errorf("walking tree %s: %v", *snapshot.Tree, err) } diff --git a/doc/045_working_with_repos.rst b/doc/045_working_with_repos.rst index d74c9c240..48e5985dc 100644 --- a/doc/045_working_with_repos.rst +++ b/doc/045_working_with_repos.rst @@ -82,6 +82,76 @@ Furthermore you can group the output by the same filters (host, paths, tags): 1 snapshots +Listing files in a snapshot +=========================== + +To get a list of the files in a specific snapshot you can use the ``ls`` command: + +.. code-block:: console + + $ restic ls 073a90db + + snapshot 073a90db of [/home/user/work.txt] filtered by [] at 2024-01-21 16:51:18.474558607 +0100 CET): + /home + /home/user + /home/user/work.txt + +The special snapshot ID ``latest`` can be used to list files and directories of the latest snapshot in the repository. +The ``--host`` flag can be used in conjunction to select the latest snapshot originating from a certain host only. + +.. code-block:: console + + $ restic ls --host kasimir latest + + snapshot 073a90db of [/home/user/work.txt] filtered by [] at 2024-01-21 16:51:18.474558607 +0100 CET): + /home + /home/user + /home/user/work.txt + +By default, ``ls`` prints all files in a snapshot. + +File listings can optionally be filtered by directories. Any positional arguments after the snapshot ID are interpreted +as absolute directory paths, and only files inside those directories will be listed. Files in subdirectories are not +listed when filtering by directories. If the ``--recursive`` flag is used, then subdirectories are also included. +Any directory paths specified must be absolute (starting with a path separator); paths use the forward slash '/' +as separator. + +.. code-block:: console + + $ restic ls latest /home + + snapshot 073a90db of [/home/user/work.txt] filtered by [/home] at 2024-01-21 16:51:18.474558607 +0100 CET): + /home + /home/user + +.. code-block:: console + + $ restic ls --recursive latest /home + + snapshot 073a90db of [/home/user/work.txt] filtered by [/home] at 2024-01-21 16:51:18.474558607 +0100 CET): + /home + /home/user + /home/user/work.txt + +To show more details about the files in a snapshot, you can use the ``--long`` option. The colums include +file permissions, UID, GID, file size, modification time and file path. For scripting usage, the +``ls`` command supports the ``--json`` flag; the JSON output format is described at :ref:`ls json`. + +.. code-block:: console + + $ restic ls --long latest + + snapshot 073a90db of [/home/user/work.txt] filtered by [] at 2024-01-21 16:51:18.474558607 +0100 CET): + drwxr-xr-x 0 0 0 2024-01-21 16:50:52 /home + drwxr-xr-x 0 0 0 2024-01-21 16:51:03 /home/user + -rw-r--r-- 0 0 18 2024-01-21 16:51:03 /home/user/work.txt + +NCDU (NCurses Disk Usage) is a tool to analyse disk usage of directories. The ``ls`` command supports +outputting information about a snapshot in the NCDU format using the ``--ncdu`` option. + +You can use it as follows: ``restic ls latest --ncdu | ncdu -f -`` + + Copying snapshots between repositories ====================================== @@ -242,6 +312,7 @@ Currently, rewriting the hostname and the time of the backup is supported. This is possible using the ``rewrite`` command with the option ``--new-host`` followed by the desired new hostname or the option ``--new-time`` followed by the desired new timestamp. .. code-block:: console + $ restic rewrite --new-host newhost --new-time "1999-01-01 11:11:11" repository b7dbade3 opened (version 2, compression level auto) diff --git a/doc/075_scripting.rst b/doc/075_scripting.rst index f46572209..7279ee614 100644 --- a/doc/075_scripting.rst +++ b/doc/075_scripting.rst @@ -409,6 +409,8 @@ The ``key list`` command returns an array of objects with the following structur +--------------+------------------------------------+ +.. _ls json: + ls -- diff --git a/internal/dump/common.go b/internal/dump/common.go index 7a8d95ae9..016328835 100644 --- a/internal/dump/common.go +++ b/internal/dump/common.go @@ -70,7 +70,7 @@ func sendNodes(ctx context.Context, repo restic.BlobLoader, root *restic.Node, c return nil } - err := walker.Walk(ctx, repo, *root.Subtree, func(_ restic.ID, nodepath string, node *restic.Node, err error) error { + err := walker.Walk(ctx, repo, *root.Subtree, walker.WalkVisitor{ProcessNode: func(_ restic.ID, nodepath string, node *restic.Node, err error) error { if err != nil { return err } @@ -91,7 +91,7 @@ func sendNodes(ctx context.Context, repo restic.BlobLoader, root *restic.Node, c } return nil - }) + }}) return err } diff --git a/internal/walker/walker.go b/internal/walker/walker.go index aba2e39e5..091b05489 100644 --- a/internal/walker/walker.go +++ b/internal/walker/walker.go @@ -23,12 +23,20 @@ var ErrSkipNode = errors.New("skip this node") // tree are skipped. type WalkFunc func(parentTreeID restic.ID, path string, node *restic.Node, nodeErr error) (err error) +type WalkVisitor struct { + // If the node is a `dir`, it will be entered afterwards unless `ErrSkipNode` + // was returned. This function is mandatory + ProcessNode WalkFunc + // Optional callback + LeaveDir func(path string) +} + // Walk calls walkFn recursively for each node in root. If walkFn returns an // error, it is passed up the call stack. The trees in ignoreTrees are not // walked. If walkFn ignores trees, these are added to the set. -func Walk(ctx context.Context, repo restic.BlobLoader, root restic.ID, walkFn WalkFunc) error { +func Walk(ctx context.Context, repo restic.BlobLoader, root restic.ID, visitor WalkVisitor) error { tree, err := restic.LoadTree(ctx, repo, root) - err = walkFn(root, "/", nil, err) + err = visitor.ProcessNode(root, "/", nil, err) if err != nil { if err == ErrSkipNode { @@ -37,13 +45,13 @@ func Walk(ctx context.Context, repo restic.BlobLoader, root restic.ID, walkFn Wa return err } - return walk(ctx, repo, "/", root, tree, walkFn) + return walk(ctx, repo, "/", root, tree, visitor) } // walk recursively traverses the tree, ignoring subtrees when the ID of the // subtree is in ignoreTrees. If err is nil and ignore is true, the subtree ID // will be added to ignoreTrees by walk. -func walk(ctx context.Context, repo restic.BlobLoader, prefix string, parentTreeID restic.ID, tree *restic.Tree, walkFn WalkFunc) (err error) { +func walk(ctx context.Context, repo restic.BlobLoader, prefix string, parentTreeID restic.ID, tree *restic.Tree, visitor WalkVisitor) (err error) { sort.Slice(tree.Nodes, func(i, j int) bool { return tree.Nodes[i].Name < tree.Nodes[j].Name }) @@ -56,11 +64,11 @@ func walk(ctx context.Context, repo restic.BlobLoader, prefix string, parentTree } if node.Type != "dir" { - err := walkFn(parentTreeID, p, node, nil) + err := visitor.ProcessNode(parentTreeID, p, node, nil) if err != nil { if err == ErrSkipNode { // skip the remaining entries in this tree - return nil + break } return err @@ -74,18 +82,22 @@ func walk(ctx context.Context, repo restic.BlobLoader, prefix string, parentTree } subtree, err := restic.LoadTree(ctx, repo, *node.Subtree) - err = walkFn(parentTreeID, p, node, err) + err = visitor.ProcessNode(parentTreeID, p, node, err) if err != nil { if err == ErrSkipNode { continue } } - err = walk(ctx, repo, p, *node.Subtree, subtree, walkFn) + err = walk(ctx, repo, p, *node.Subtree, subtree, visitor) if err != nil { return err } } + if visitor.LeaveDir != nil { + visitor.LeaveDir(prefix) + } + return nil } diff --git a/internal/walker/walker_test.go b/internal/walker/walker_test.go index 786570e02..0f0009107 100644 --- a/internal/walker/walker_test.go +++ b/internal/walker/walker_test.go @@ -93,12 +93,12 @@ func (t TreeMap) Connections() uint { // checkFunc returns a function suitable for walking the tree to check // something, and a function which will check the final result. -type checkFunc func(t testing.TB) (walker WalkFunc, final func(testing.TB)) +type checkFunc func(t testing.TB) (walker WalkFunc, leaveDir func(path string), final func(testing.TB)) // checkItemOrder ensures that the order of the 'path' arguments is the one passed in as 'want'. func checkItemOrder(want []string) checkFunc { pos := 0 - return func(t testing.TB) (walker WalkFunc, final func(testing.TB)) { + return func(t testing.TB) (walker WalkFunc, leaveDir func(path string), final func(testing.TB)) { walker = func(treeID restic.ID, path string, node *restic.Node, err error) error { if err != nil { t.Errorf("error walking %v: %v", path, err) @@ -117,20 +117,24 @@ func checkItemOrder(want []string) checkFunc { return nil } + leaveDir = func(path string) { + _ = walker(restic.ID{}, "leave: "+path, nil, nil) + } + final = func(t testing.TB) { if pos != len(want) { t.Errorf("not enough items returned, want %d, got %d", len(want), pos) } } - return walker, final + return walker, leaveDir, final } } // checkParentTreeOrder ensures that the order of the 'parentID' arguments is the one passed in as 'want'. func checkParentTreeOrder(want []string) checkFunc { pos := 0 - return func(t testing.TB) (walker WalkFunc, final func(testing.TB)) { + return func(t testing.TB) (walker WalkFunc, leaveDir func(path string), final func(testing.TB)) { walker = func(treeID restic.ID, path string, node *restic.Node, err error) error { if err != nil { t.Errorf("error walking %v: %v", path, err) @@ -155,7 +159,7 @@ func checkParentTreeOrder(want []string) checkFunc { } } - return walker, final + return walker, nil, final } } @@ -164,7 +168,7 @@ func checkParentTreeOrder(want []string) checkFunc { func checkSkipFor(skipFor map[string]struct{}, wantPaths []string) checkFunc { var pos int - return func(t testing.TB) (walker WalkFunc, final func(testing.TB)) { + return func(t testing.TB) (walker WalkFunc, leaveDir func(path string), final func(testing.TB)) { walker = func(treeID restic.ID, path string, node *restic.Node, err error) error { if err != nil { t.Errorf("error walking %v: %v", path, err) @@ -188,13 +192,17 @@ func checkSkipFor(skipFor map[string]struct{}, wantPaths []string) checkFunc { return nil } + leaveDir = func(path string) { + _ = walker(restic.ID{}, "leave: "+path, nil, nil) + } + final = func(t testing.TB) { if pos != len(wantPaths) { t.Errorf("wrong number of paths returned, want %d, got %d", len(wantPaths), pos) } } - return walker, final + return walker, leaveDir, final } } @@ -216,6 +224,8 @@ func TestWalker(t *testing.T) { "/foo", "/subdir", "/subdir/subfile", + "leave: /subdir", + "leave: /", }), checkParentTreeOrder([]string{ "a760536a8fd64dd63f8dd95d85d788d71fd1bee6828619350daf6959dcb499a0", // tree / @@ -230,6 +240,7 @@ func TestWalker(t *testing.T) { "/", "/foo", "/subdir", + "leave: /", }, ), checkSkipFor( @@ -260,10 +271,14 @@ func TestWalker(t *testing.T) { "/foo", "/subdir1", "/subdir1/subfile1", + "leave: /subdir1", "/subdir2", "/subdir2/subfile2", "/subdir2/subsubdir2", "/subdir2/subsubdir2/subsubfile3", + "leave: /subdir2/subsubdir2", + "leave: /subdir2", + "leave: /", }), checkParentTreeOrder([]string{ "7a0e59b986cc83167d9fbeeefc54e4629770124c5825d391f7ee0598667fcdf1", // tree / @@ -286,6 +301,9 @@ func TestWalker(t *testing.T) { "/subdir2/subfile2", "/subdir2/subsubdir2", "/subdir2/subsubdir2/subsubfile3", + "leave: /subdir2/subsubdir2", + "leave: /subdir2", + "leave: /", }, ), checkSkipFor( @@ -299,6 +317,8 @@ func TestWalker(t *testing.T) { "/subdir2", "/subdir2/subfile2", "/subdir2/subsubdir2", + "leave: /subdir2", + "leave: /", }, ), checkSkipFor( @@ -307,6 +327,7 @@ func TestWalker(t *testing.T) { }, []string{ "/", "/foo", + "leave: /", }, ), }, @@ -339,15 +360,19 @@ func TestWalker(t *testing.T) { "/subdir1/subfile1", "/subdir1/subfile2", "/subdir1/subfile3", + "leave: /subdir1", "/subdir2", "/subdir2/subfile1", "/subdir2/subfile2", "/subdir2/subfile3", + "leave: /subdir2", "/subdir3", "/subdir3/subfile1", "/subdir3/subfile2", "/subdir3/subfile3", + "leave: /subdir3", "/zzz other", + "leave: /", }), checkParentTreeOrder([]string{ "c2efeff7f217a4dfa12a16e8bb3cefedd37c00873605c29e5271c6061030672f", // tree / @@ -385,13 +410,20 @@ func TestWalker(t *testing.T) { checkItemOrder([]string{ "/", "/subdir1", + "leave: /subdir1", "/subdir2", + "leave: /subdir2", "/subdir3", "/subdir3/file", + "leave: /subdir3", "/subdir4", "/subdir4/file", + "leave: /subdir4", "/subdir5", + "leave: /subdir5", "/subdir6", + "leave: /subdir6", + "leave: /", }), }, }, @@ -405,8 +437,11 @@ func TestWalker(t *testing.T) { ctx, cancel := context.WithCancel(context.TODO()) defer cancel() - fn, last := check(t) - err := Walk(ctx, repo, root, fn) + fn, leaveDir, last := check(t) + err := Walk(ctx, repo, root, WalkVisitor{ + ProcessNode: fn, + LeaveDir: leaveDir, + }) if err != nil { t.Error(err) }