octoleo/restic
2
2
Fork 0
mirror of https://github.com/octoleo/restic.git synced 2024-11-21 12:25:09 +00:00
Code Issues Releases Wiki Activity

Merge pull request #4550 from ndecker/ls-ncdu

Browse Source 
Ls ncdu
This commit is contained in:
Michael Eischer 2024-01-27 12:27:35 +00:00 committed by GitHub
commit e44e4b00a6
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
11 changed files with 492 additions and 129 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
changelog/unreleased/issue-4549 Normal file
View File

@ -0,0 +1,11 @@
Enhancement: Add `--ncdu` option to `ls` command
NCDU (NCurses Disk Usage) is a tool to analyse disk usage of directories.
It has an option to save a directory tree and analyse it later.
The `ls` command now supports the `--ncdu` option which outputs information
about a snapshot in the NCDU format.
You can use it as follows: `restic ls latest --ncdu | ncdu -f -`
https://github.com/restic/restic/issues/4549
https://github.com/restic/restic/pull/4550

8
cmd/restic/cmd_find.go
View File

@ -260,7 +260,7 @@ func (f *Finder) findInSnapshot(ctx context.Context, sn *restic.Snapshot) error
}
f.out.newsn = sn
return walker.Walk(ctx, f.repo, *sn.Tree, func(parentTreeID restic.ID, nodepath string, node *restic.Node, err error) error {
return walker.Walk(ctx, f.repo, *sn.Tree, walker.WalkVisitor{ProcessNode: func(parentTreeID restic.ID, nodepath string, node *restic.Node, err error) error {
if err != nil {
debug.Log("Error loading tree %v: %v", parentTreeID, err)
@ -327,7 +327,7 @@ func (f *Finder) findInSnapshot(ctx context.Context, sn *restic.Snapshot) error
debug.Log(" found match\n")
f.out.PrintPattern(nodepath, node)
return nil
})
}})
}
func (f *Finder) findIDs(ctx context.Context, sn *restic.Snapshot) error {
@ -338,7 +338,7 @@ func (f *Finder) findIDs(ctx context.Context, sn *restic.Snapshot) error {
}
f.out.newsn = sn
return walker.Walk(ctx, f.repo, *sn.Tree, func(parentTreeID restic.ID, nodepath string, node *restic.Node, err error) error {
return walker.Walk(ctx, f.repo, *sn.Tree, walker.WalkVisitor{ProcessNode: func(parentTreeID restic.ID, nodepath string, node *restic.Node, err error) error {
if err != nil {
debug.Log("Error loading tree %v: %v", parentTreeID, err)
@ -388,7 +388,7 @@ func (f *Finder) findIDs(ctx context.Context, sn *restic.Snapshot) error {
}
return nil
})
}})
}
var errAllPacksFound = errors.New("all packs found")

205
cmd/restic/cmd_ls.go
View File

@ -3,6 +3,8 @@ package main
import (
"context"
"encoding/json"
"fmt"
"io"
"os"
"strings"
"time"
@ -51,6 +53,7 @@ type LsOptions struct {
restic.SnapshotFilter
Recursive bool
HumanReadable bool
Ncdu bool
}
var lsOptions LsOptions
@ -63,16 +66,47 @@ func init() {
flags.BoolVarP(&lsOptions.ListLong, "long", "l", false, "use a long listing format showing size and mode")
flags.BoolVar(&lsOptions.Recursive, "recursive", false, "include files in subfolders of the listed directories")
flags.BoolVar(&lsOptions.HumanReadable, "human-readable", false, "print sizes in human readable format")
flags.BoolVar(&lsOptions.Ncdu, "ncdu", false, "output NCDU export format (pipe into 'ncdu -f -')")
}
type lsSnapshot struct {
*restic.Snapshot
ID *restic.ID `json:"id"`
ShortID string `json:"short_id"`
StructType string `json:"struct_type"` // "snapshot"
type lsPrinter interface {
Snapshot(sn *restic.Snapshot)
Node(path string, node *restic.Node)
LeaveDir(path string)
Close()
}
type jsonLsPrinter struct {
enc *json.Encoder
}
func (p *jsonLsPrinter) Snapshot(sn *restic.Snapshot) {
type lsSnapshot struct {
*restic.Snapshot
ID *restic.ID `json:"id"`
ShortID string `json:"short_id"`
StructType string `json:"struct_type"` // "snapshot"
}
err := p.enc.Encode(lsSnapshot{
Snapshot: sn,
ID: sn.ID(),
ShortID: sn.ID().Str(),
StructType: "snapshot",
})
if err != nil {
Warnf("JSON encode failed: %v\n", err)
}
}
// Print node in our custom JSON format, followed by a newline.
func (p *jsonLsPrinter) Node(path string, node *restic.Node) {
err := lsNodeJSON(p.enc, path, node)
if err != nil {
Warnf("JSON encode failed: %v\n", err)
}
}
func lsNodeJSON(enc *json.Encoder, path string, node *restic.Node) error {
n := &struct {
Name string `json:"name"`
@ -114,10 +148,117 @@ func lsNodeJSON(enc *json.Encoder, path string, node *restic.Node) error {
return enc.Encode(n)
}
func (p *jsonLsPrinter) LeaveDir(_ string) {}
func (p *jsonLsPrinter) Close() {}
type ncduLsPrinter struct {
out io.Writer
depth int
}
// lsSnapshotNcdu prints a restic snapshot in Ncdu save format.
// It opens the JSON list. Nodes are added with lsNodeNcdu and the list is closed by lsCloseNcdu.
// Format documentation: https://dev.yorhel.nl/ncdu/jsonfmt
func (p *ncduLsPrinter) Snapshot(sn *restic.Snapshot) {
const NcduMajorVer = 1
const NcduMinorVer = 2
snapshotBytes, err := json.Marshal(sn)
if err != nil {
Warnf("JSON encode failed: %v\n", err)
}
p.depth++
fmt.Fprintf(p.out, "[%d, %d, %s", NcduMajorVer, NcduMinorVer, string(snapshotBytes))
}
func lsNcduNode(_ string, node *restic.Node) ([]byte, error) {
type NcduNode struct {
Name string `json:"name"`
Asize uint64 `json:"asize"`
Dsize uint64 `json:"dsize"`
Dev uint64 `json:"dev"`
Ino uint64 `json:"ino"`
NLink uint64 `json:"nlink"`
NotReg bool `json:"notreg"`
UID uint32 `json:"uid"`
GID uint32 `json:"gid"`
Mode uint16 `json:"mode"`
Mtime int64 `json:"mtime"`
}
outNode := NcduNode{
Name: node.Name,
Asize: node.Size,
Dsize: node.Size,
Dev: node.DeviceID,
Ino: node.Inode,
NLink: node.Links,
NotReg: node.Type != "dir" && node.Type != "file",
UID: node.UID,
GID: node.GID,
Mode: uint16(node.Mode & os.ModePerm),
Mtime: node.ModTime.Unix(),
}
// bits according to inode(7) manpage
if node.Mode&os.ModeSetuid != 0 {
outNode.Mode |= 0o4000
}
if node.Mode&os.ModeSetgid != 0 {
outNode.Mode |= 0o2000
}
if node.Mode&os.ModeSticky != 0 {
outNode.Mode |= 0o1000
}
return json.Marshal(outNode)
}
func (p *ncduLsPrinter) Node(path string, node *restic.Node) {
out, err := lsNcduNode(path, node)
if err != nil {
Warnf("JSON encode failed: %v\n", err)
}
if node.Type == "dir" {
fmt.Fprintf(p.out, ",\n%s[\n%s%s", strings.Repeat(" ", p.depth), strings.Repeat(" ", p.depth+1), string(out))
p.depth++
} else {
fmt.Fprintf(p.out, ",\n%s%s", strings.Repeat(" ", p.depth), string(out))
}
}
func (p *ncduLsPrinter) LeaveDir(_ string) {
p.depth--
fmt.Fprintf(p.out, "\n%s]", strings.Repeat(" ", p.depth))
}
func (p *ncduLsPrinter) Close() {
fmt.Fprint(p.out, "\n]\n")
}
type textLsPrinter struct {
dirs []string
ListLong bool
HumanReadable bool
}
func (p *textLsPrinter) Snapshot(sn *restic.Snapshot) {
Verbosef("%v filtered by %v:\n", sn, p.dirs)
}
func (p *textLsPrinter) Node(path string, node *restic.Node) {
Printf("%s\n", formatNode(path, node, p.ListLong, p.HumanReadable))
}
func (p *textLsPrinter) LeaveDir(_ string) {}
func (p *textLsPrinter) Close() {}
func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []string) error {
if len(args) == 0 {
return errors.Fatal("no snapshot ID specified, specify snapshot ID or use special ID 'latest'")
}
if opts.Ncdu && gopts.JSON {
return errors.Fatal("only either '--json' or '--ncdu' can be specified")
}
// extract any specific directories to walk
var dirs []string
@ -179,38 +320,21 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri
return err
}
var (
printSnapshot func(sn *restic.Snapshot)
printNode func(path string, node *restic.Node)
)
var printer lsPrinter
if gopts.JSON {
enc := json.NewEncoder(globalOptions.stdout)
printSnapshot = func(sn *restic.Snapshot) {
err := enc.Encode(lsSnapshot{
Snapshot: sn,
ID: sn.ID(),
ShortID: sn.ID().Str(),
StructType: "snapshot",
})
if err != nil {
Warnf("JSON encode failed: %v\n", err)
}
printer = &jsonLsPrinter{
enc: json.NewEncoder(globalOptions.stdout),
}
printNode = func(path string, node *restic.Node) {
err := lsNodeJSON(enc, path, node)
if err != nil {
Warnf("JSON encode failed: %v\n", err)
}
} else if opts.Ncdu {
printer = &ncduLsPrinter{
out: globalOptions.stdout,
}
} else {
printSnapshot = func(sn *restic.Snapshot) {
Verbosef("%v filtered by %v:\n", sn, dirs)
}
printNode = func(path string, node *restic.Node) {
Printf("%s\n", formatNode(path, node, opts.ListLong, opts.HumanReadable))
printer = &textLsPrinter{
dirs: dirs,
ListLong: opts.ListLong,
HumanReadable: opts.HumanReadable,
}
}
@ -228,9 +352,9 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri
return err
}
printSnapshot(sn)
printer.Snapshot(sn)
err = walker.Walk(ctx, repo, *sn.Tree, func(_ restic.ID, nodepath string, node *restic.Node, err error) error {
processNode := func(_ restic.ID, nodepath string, node *restic.Node, err error) error {
if err != nil {
return err
}
@ -240,7 +364,7 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri
if withinDir(nodepath) {
// if we're within a dir, print the node
printNode(nodepath, node)
printer.Node(nodepath, node)
// if recursive listing is requested, signal the walker that it
// should continue walking recursively
@ -261,11 +385,22 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri
return walker.ErrSkipNode
}
return nil
}
err = walker.Walk(ctx, repo, *sn.Tree, walker.WalkVisitor{
ProcessNode: processNode,
LeaveDir: func(path string) {
// the root path `/` has no corresponding node and is thus also skipped by processNode
if withinDir(path) && path != "/" {
printer.LeaveDir(path)
}
},
})
if err != nil {
return err
}
printer.Close()
return nil
}

36
cmd/restic/cmd_ls_integration_test.go
View File

@ -2,18 +2,46 @@ package main
import (
"context"
"encoding/json"
"path/filepath"
"strings"
"testing"
rtest "github.com/restic/restic/internal/test"
)
func testRunLs(t testing.TB, gopts GlobalOptions, snapshotID string) []string {
func testRunLsWithOpts(t testing.TB, gopts GlobalOptions, opts LsOptions, args []string) []byte {
buf, err := withCaptureStdout(func() error {
gopts.Quiet = true
opts := LsOptions{}
return runLs(context.TODO(), opts, gopts, []string{snapshotID})
return runLs(context.TODO(), opts, gopts, args)
})
rtest.OK(t, err)
return strings.Split(buf.String(), "\n")
return buf.Bytes()
}
func testRunLs(t testing.TB, gopts GlobalOptions, snapshotID string) []string {
out := testRunLsWithOpts(t, gopts, LsOptions{}, []string{snapshotID})
return strings.Split(string(out), "\n")
}
func assertIsValidJSON(t *testing.T, data []byte) {
// Sanity check: output must be valid JSON.
var v interface{}
err := json.Unmarshal(data, &v)
rtest.OK(t, err)
}
func TestRunLsNcdu(t *testing.T) {
env, cleanup := withTestEnvironment(t)
defer cleanup()
testRunInit(t, env.gopts)
opts := BackupOptions{}
testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, env.gopts)
ncdu := testRunLsWithOpts(t, env.gopts, LsOptions{Ncdu: true}, []string{"latest"})
assertIsValidJSON(t, ncdu)
ncdu = testRunLsWithOpts(t, env.gopts, LsOptions{Ncdu: true}, []string{"latest", "/testdata"})
assertIsValidJSON(t, ncdu)
}

199
cmd/restic/cmd_ls_test.go
View File

@ -11,78 +11,94 @@ import (
rtest "github.com/restic/restic/internal/test"
)
type lsTestNode struct {
path string
restic.Node
}
var lsTestNodes = []lsTestNode{
// Mode is omitted when zero.
// Permissions, by convention is "-" per mode bit
{
path: "/bar/baz",
Node: restic.Node{
Name: "baz",
Type: "file",
Size: 12345,
UID: 10000000,
GID: 20000000,
User: "nobody",
Group: "nobodies",
Links: 1,
},
},
// Even empty files get an explicit size.
{
path: "/foo/empty",
Node: restic.Node{
Name: "empty",
Type: "file",
Size: 0,
UID: 1001,
GID: 1001,
User: "not printed",
Group: "not printed",
Links: 0xF00,
},
},
// Non-regular files do not get a size.
// Mode is printed in decimal, including the type bits.
{
path: "/foo/link",
Node: restic.Node{
Name: "link",
Type: "symlink",
Mode: os.ModeSymlink | 0777,
LinkTarget: "not printed",
},
},
{
path: "/some/directory",
Node: restic.Node{
Name: "directory",
Type: "dir",
Mode: os.ModeDir | 0755,
ModTime: time.Date(2020, 1, 2, 3, 4, 5, 0, time.UTC),
AccessTime: time.Date(2021, 2, 3, 4, 5, 6, 7, time.UTC),
ChangeTime: time.Date(2022, 3, 4, 5, 6, 7, 8, time.UTC),
},
},
// Test encoding of setuid/setgid/sticky bit
{
path: "/some/sticky",
Node: restic.Node{
Name: "sticky",
Type: "dir",
Mode: os.ModeDir | 0755 | os.ModeSetuid | os.ModeSetgid | os.ModeSticky,
},
},
}
func TestLsNodeJSON(t *testing.T) {
for _, c := range []struct {
path string
restic.Node
expect string
}{
// Mode is omitted when zero.
// Permissions, by convention is "-" per mode bit
{
path: "/bar/baz",
Node: restic.Node{
Name: "baz",
Type: "file",
Size: 12345,
UID: 10000000,
GID: 20000000,
User: "nobody",
Group: "nobodies",
Links: 1,
},
expect: `{"name":"baz","type":"file","path":"/bar/baz","uid":10000000,"gid":20000000,"size":12345,"permissions":"----------","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","struct_type":"node"}`,
},
// Even empty files get an explicit size.
{
path: "/foo/empty",
Node: restic.Node{
Name: "empty",
Type: "file",
Size: 0,
UID: 1001,
GID: 1001,
User: "not printed",
Group: "not printed",
Links: 0xF00,
},
expect: `{"name":"empty","type":"file","path":"/foo/empty","uid":1001,"gid":1001,"size":0,"permissions":"----------","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","struct_type":"node"}`,
},
// Non-regular files do not get a size.
// Mode is printed in decimal, including the type bits.
{
path: "/foo/link",
Node: restic.Node{
Name: "link",
Type: "symlink",
Mode: os.ModeSymlink | 0777,
LinkTarget: "not printed",
},
expect: `{"name":"link","type":"symlink","path":"/foo/link","uid":0,"gid":0,"mode":134218239,"permissions":"Lrwxrwxrwx","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","struct_type":"node"}`,
},
{
path: "/some/directory",
Node: restic.Node{
Name: "directory",
Type: "dir",
Mode: os.ModeDir | 0755,
ModTime: time.Date(2020, 1, 2, 3, 4, 5, 0, time.UTC),
AccessTime: time.Date(2021, 2, 3, 4, 5, 6, 7, time.UTC),
ChangeTime: time.Date(2022, 3, 4, 5, 6, 7, 8, time.UTC),
},
expect: `{"name":"directory","type":"dir","path":"/some/directory","uid":0,"gid":0,"mode":2147484141,"permissions":"drwxr-xr-x","mtime":"2020-01-02T03:04:05Z","atime":"2021-02-03T04:05:06.000000007Z","ctime":"2022-03-04T05:06:07.000000008Z","struct_type":"node"}`,
},
for i, expect := range []string{
`{"name":"baz","type":"file","path":"/bar/baz","uid":10000000,"gid":20000000,"size":12345,"permissions":"----------","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","struct_type":"node"}`,
`{"name":"empty","type":"file","path":"/foo/empty","uid":1001,"gid":1001,"size":0,"permissions":"----------","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","struct_type":"node"}`,
`{"name":"link","type":"symlink","path":"/foo/link","uid":0,"gid":0,"mode":134218239,"permissions":"Lrwxrwxrwx","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","struct_type":"node"}`,
`{"name":"directory","type":"dir","path":"/some/directory","uid":0,"gid":0,"mode":2147484141,"permissions":"drwxr-xr-x","mtime":"2020-01-02T03:04:05Z","atime":"2021-02-03T04:05:06.000000007Z","ctime":"2022-03-04T05:06:07.000000008Z","struct_type":"node"}`,
`{"name":"sticky","type":"dir","path":"/some/sticky","uid":0,"gid":0,"mode":2161115629,"permissions":"dugtrwxr-xr-x","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","struct_type":"node"}`,
} {
c := lsTestNodes[i]
buf := new(bytes.Buffer)
enc := json.NewEncoder(buf)
err := lsNodeJSON(enc, c.path, &c.Node)
rtest.OK(t, err)
rtest.Equals(t, c.expect+"\n", buf.String())
rtest.Equals(t, expect+"\n", buf.String())
// Sanity check: output must be valid JSON.
var v interface{}
@ -90,3 +106,54 @@ func TestLsNodeJSON(t *testing.T) {
rtest.OK(t, err)
}
}
func TestLsNcduNode(t *testing.T) {
for i, expect := range []string{
`{"name":"baz","asize":12345,"dsize":12345,"dev":0,"ino":0,"nlink":1,"notreg":false,"uid":10000000,"gid":20000000,"mode":0,"mtime":-62135596800}`,
`{"name":"empty","asize":0,"dsize":0,"dev":0,"ino":0,"nlink":3840,"notreg":false,"uid":1001,"gid":1001,"mode":0,"mtime":-62135596800}`,
`{"name":"link","asize":0,"dsize":0,"dev":0,"ino":0,"nlink":0,"notreg":true,"uid":0,"gid":0,"mode":511,"mtime":-62135596800}`,
`{"name":"directory","asize":0,"dsize":0,"dev":0,"ino":0,"nlink":0,"notreg":false,"uid":0,"gid":0,"mode":493,"mtime":1577934245}`,
`{"name":"sticky","asize":0,"dsize":0,"dev":0,"ino":0,"nlink":0,"notreg":false,"uid":0,"gid":0,"mode":4077,"mtime":-62135596800}`,
} {
c := lsTestNodes[i]
out, err := lsNcduNode(c.path, &c.Node)
rtest.OK(t, err)
rtest.Equals(t, expect, string(out))
// Sanity check: output must be valid JSON.
var v interface{}
err = json.Unmarshal(out, &v)
rtest.OK(t, err)
}
}
func TestLsNcdu(t *testing.T) {
var buf bytes.Buffer
printer := &ncduLsPrinter{
out: &buf,
}
printer.Snapshot(&restic.Snapshot{
Hostname: "host",
Paths: []string{"/example"},
})
printer.Node("/directory", &restic.Node{
Type: "dir",
Name: "directory",
})
printer.Node("/directory/data", &restic.Node{
Type: "file",
Name: "data",
Size: 42,
})
printer.LeaveDir("/directory")
printer.Close()
rtest.Equals(t, `[1, 2, {"time":"0001-01-01T00:00:00Z","tree":null,"paths":["/example"],"hostname":"host"},
[
{"name":"directory","asize":0,"dsize":0,"dev":0,"ino":0,"nlink":0,"notreg":false,"uid":0,"gid":0,"mode":0,"mtime":-62135596800},
{"name":"data","asize":42,"dsize":42,"dev":0,"ino":0,"nlink":0,"notreg":false,"uid":0,"gid":0,"mode":0,"mtime":-62135596800}
]
]
`, buf.String())
}

4
cmd/restic/cmd_stats.go
View File

@ -203,7 +203,9 @@ func statsWalkSnapshot(ctx context.Context, snapshot *restic.Snapshot, repo rest
}
hardLinkIndex := restorer.NewHardlinkIndex[struct{}]()
err := walker.Walk(ctx, repo, *snapshot.Tree, statsWalkTree(repo, opts, stats, hardLinkIndex))
err := walker.Walk(ctx, repo, *snapshot.Tree, walker.WalkVisitor{
ProcessNode: statsWalkTree(repo, opts, stats, hardLinkIndex),
})
if err != nil {
return fmt.Errorf("walking tree %s: %v", *snapshot.Tree, err)
}

71
doc/045_working_with_repos.rst
View File

@ -82,6 +82,76 @@ Furthermore you can group the output by the same filters (host, paths, tags):
1 snapshots
Listing files in a snapshot
===========================
To get a list of the files in a specific snapshot you can use the ``ls`` command:
.. code-block:: console
$ restic ls 073a90db
snapshot 073a90db of [/home/user/work.txt] filtered by [] at 2024-01-21 16:51:18.474558607 +0100 CET):
/home
/home/user
/home/user/work.txt
The special snapshot ID ``latest`` can be used to list files and directories of the latest snapshot in the repository.
The ``--host`` flag can be used in conjunction to select the latest snapshot originating from a certain host only.
.. code-block:: console
$ restic ls --host kasimir latest
snapshot 073a90db of [/home/user/work.txt] filtered by [] at 2024-01-21 16:51:18.474558607 +0100 CET):
/home
/home/user
/home/user/work.txt
By default, ``ls`` prints all files in a snapshot.
File listings can optionally be filtered by directories. Any positional arguments after the snapshot ID are interpreted
as absolute directory paths, and only files inside those directories will be listed. Files in subdirectories are not
listed when filtering by directories. If the ``--recursive`` flag is used, then subdirectories are also included.
Any directory paths specified must be absolute (starting with a path separator); paths use the forward slash '/'
as separator.
.. code-block:: console
$ restic ls latest /home
snapshot 073a90db of [/home/user/work.txt] filtered by [/home] at 2024-01-21 16:51:18.474558607 +0100 CET):
/home
/home/user
.. code-block:: console
$ restic ls --recursive latest /home
snapshot 073a90db of [/home/user/work.txt] filtered by [/home] at 2024-01-21 16:51:18.474558607 +0100 CET):
/home
/home/user
/home/user/work.txt
To show more details about the files in a snapshot, you can use the ``--long`` option. The colums include
file permissions, UID, GID, file size, modification time and file path. For scripting usage, the
``ls`` command supports the ``--json`` flag; the JSON output format is described at :ref:`ls json`.
.. code-block:: console
$ restic ls --long latest
snapshot 073a90db of [/home/user/work.txt] filtered by [] at 2024-01-21 16:51:18.474558607 +0100 CET):
drwxr-xr-x 0 0 0 2024-01-21 16:50:52 /home
drwxr-xr-x 0 0 0 2024-01-21 16:51:03 /home/user
-rw-r--r-- 0 0 18 2024-01-21 16:51:03 /home/user/work.txt
NCDU (NCurses Disk Usage) is a tool to analyse disk usage of directories. The ``ls`` command supports
outputting information about a snapshot in the NCDU format using the ``--ncdu`` option.
You can use it as follows: ``restic ls latest --ncdu | ncdu -f -``
Copying snapshots between repositories
======================================
@ -242,6 +312,7 @@ Currently, rewriting the hostname and the time of the backup is supported.
This is possible using the ``rewrite`` command with the option ``--new-host`` followed by the desired new hostname or the option ``--new-time`` followed by the desired new timestamp.
.. code-block:: console
$ restic rewrite --new-host newhost --new-time "1999-01-01 11:11:11"
repository b7dbade3 opened (version 2, compression level auto)

2
doc/075_scripting.rst
View File

@ -409,6 +409,8 @@ The ``key list`` command returns an array of objects with the following structur
+--------------+------------------------------------+
.. _ls json:
ls
--

4
internal/dump/common.go
View File

@ -70,7 +70,7 @@ func sendNodes(ctx context.Context, repo restic.BlobLoader, root *restic.Node, c
return nil
}
err := walker.Walk(ctx, repo, *root.Subtree, func(_ restic.ID, nodepath string, node *restic.Node, err error) error {
err := walker.Walk(ctx, repo, *root.Subtree, walker.WalkVisitor{ProcessNode: func(_ restic.ID, nodepath string, node *restic.Node, err error) error {
if err != nil {
return err
}
@ -91,7 +91,7 @@ func sendNodes(ctx context.Context, repo restic.BlobLoader, root *restic.Node, c
}
return nil
})
}})
return err
}

28
internal/walker/walker.go
View File

@ -23,12 +23,20 @@ var ErrSkipNode = errors.New("skip this node")
// tree are skipped.
type WalkFunc func(parentTreeID restic.ID, path string, node *restic.Node, nodeErr error) (err error)
type WalkVisitor struct {
// If the node is a `dir`, it will be entered afterwards unless `ErrSkipNode`
// was returned. This function is mandatory
ProcessNode WalkFunc
// Optional callback
LeaveDir func(path string)
}
// Walk calls walkFn recursively for each node in root. If walkFn returns an
// error, it is passed up the call stack. The trees in ignoreTrees are not
// walked. If walkFn ignores trees, these are added to the set.
func Walk(ctx context.Context, repo restic.BlobLoader, root restic.ID, walkFn WalkFunc) error {
func Walk(ctx context.Context, repo restic.BlobLoader, root restic.ID, visitor WalkVisitor) error {
tree, err := restic.LoadTree(ctx, repo, root)
err = walkFn(root, "/", nil, err)
err = visitor.ProcessNode(root, "/", nil, err)
if err != nil {
if err == ErrSkipNode {
@ -37,13 +45,13 @@ func Walk(ctx context.Context, repo restic.BlobLoader, root restic.ID, walkFn Wa
return err
}
return walk(ctx, repo, "/", root, tree, walkFn)
return walk(ctx, repo, "/", root, tree, visitor)
}
// walk recursively traverses the tree, ignoring subtrees when the ID of the
// subtree is in ignoreTrees. If err is nil and ignore is true, the subtree ID
// will be added to ignoreTrees by walk.
func walk(ctx context.Context, repo restic.BlobLoader, prefix string, parentTreeID restic.ID, tree *restic.Tree, walkFn WalkFunc) (err error) {
func walk(ctx context.Context, repo restic.BlobLoader, prefix string, parentTreeID restic.ID, tree *restic.Tree, visitor WalkVisitor) (err error) {
sort.Slice(tree.Nodes, func(i, j int) bool {
return tree.Nodes[i].Name < tree.Nodes[j].Name
})
@ -56,11 +64,11 @@ func walk(ctx context.Context, repo restic.BlobLoader, prefix string, parentTree
}
if node.Type != "dir" {
err := walkFn(parentTreeID, p, node, nil)
err := visitor.ProcessNode(parentTreeID, p, node, nil)
if err != nil {
if err == ErrSkipNode {
// skip the remaining entries in this tree
return nil
break
}
return err
@ -74,18 +82,22 @@ func walk(ctx context.Context, repo restic.BlobLoader, prefix string, parentTree
}
subtree, err := restic.LoadTree(ctx, repo, *node.Subtree)
err = walkFn(parentTreeID, p, node, err)
err = visitor.ProcessNode(parentTreeID, p, node, err)
if err != nil {
if err == ErrSkipNode {
continue
}
}
err = walk(ctx, repo, p, *node.Subtree, subtree, walkFn)
err = walk(ctx, repo, p, *node.Subtree, subtree, visitor)
if err != nil {
return err
}
}
if visitor.LeaveDir != nil {
visitor.LeaveDir(prefix)
}
return nil
}

53
internal/walker/walker_test.go
View File

@ -93,12 +93,12 @@ func (t TreeMap) Connections() uint {
// checkFunc returns a function suitable for walking the tree to check
// something, and a function which will check the final result.
type checkFunc func(t testing.TB) (walker WalkFunc, final func(testing.TB))
type checkFunc func(t testing.TB) (walker WalkFunc, leaveDir func(path string), final func(testing.TB))
// checkItemOrder ensures that the order of the 'path' arguments is the one passed in as 'want'.
func checkItemOrder(want []string) checkFunc {
pos := 0
return func(t testing.TB) (walker WalkFunc, final func(testing.TB)) {
return func(t testing.TB) (walker WalkFunc, leaveDir func(path string), final func(testing.TB)) {
walker = func(treeID restic.ID, path string, node *restic.Node, err error) error {
if err != nil {
t.Errorf("error walking %v: %v", path, err)
@ -117,20 +117,24 @@ func checkItemOrder(want []string) checkFunc {
return nil
}
leaveDir = func(path string) {
_ = walker(restic.ID{}, "leave: "+path, nil, nil)
}
final = func(t testing.TB) {
if pos != len(want) {
t.Errorf("not enough items returned, want %d, got %d", len(want), pos)
}
}
return walker, final
return walker, leaveDir, final
}
}
// checkParentTreeOrder ensures that the order of the 'parentID' arguments is the one passed in as 'want'.
func checkParentTreeOrder(want []string) checkFunc {
pos := 0
return func(t testing.TB) (walker WalkFunc, final func(testing.TB)) {
return func(t testing.TB) (walker WalkFunc, leaveDir func(path string), final func(testing.TB)) {
walker = func(treeID restic.ID, path string, node *restic.Node, err error) error {
if err != nil {
t.Errorf("error walking %v: %v", path, err)
@ -155,7 +159,7 @@ func checkParentTreeOrder(want []string) checkFunc {
}
}
return walker, final
return walker, nil, final
}
}
@ -164,7 +168,7 @@ func checkParentTreeOrder(want []string) checkFunc {
func checkSkipFor(skipFor map[string]struct{}, wantPaths []string) checkFunc {
var pos int
return func(t testing.TB) (walker WalkFunc, final func(testing.TB)) {
return func(t testing.TB) (walker WalkFunc, leaveDir func(path string), final func(testing.TB)) {
walker = func(treeID restic.ID, path string, node *restic.Node, err error) error {
if err != nil {
t.Errorf("error walking %v: %v", path, err)
@ -188,13 +192,17 @@ func checkSkipFor(skipFor map[string]struct{}, wantPaths []string) checkFunc {
return nil
}
leaveDir = func(path string) {
_ = walker(restic.ID{}, "leave: "+path, nil, nil)
}
final = func(t testing.TB) {
if pos != len(wantPaths) {
t.Errorf("wrong number of paths returned, want %d, got %d", len(wantPaths), pos)
}
}
return walker, final
return walker, leaveDir, final
}
}
@ -216,6 +224,8 @@ func TestWalker(t *testing.T) {
"/foo",
"/subdir",
"/subdir/subfile",
"leave: /subdir",
"leave: /",
}),
checkParentTreeOrder([]string{
"a760536a8fd64dd63f8dd95d85d788d71fd1bee6828619350daf6959dcb499a0", // tree /
@ -230,6 +240,7 @@ func TestWalker(t *testing.T) {
"/",
"/foo",
"/subdir",
"leave: /",
},
),
checkSkipFor(
@ -260,10 +271,14 @@ func TestWalker(t *testing.T) {
"/foo",
"/subdir1",
"/subdir1/subfile1",
"leave: /subdir1",
"/subdir2",
"/subdir2/subfile2",
"/subdir2/subsubdir2",
"/subdir2/subsubdir2/subsubfile3",
"leave: /subdir2/subsubdir2",
"leave: /subdir2",
"leave: /",
}),
checkParentTreeOrder([]string{
"7a0e59b986cc83167d9fbeeefc54e4629770124c5825d391f7ee0598667fcdf1", // tree /
@ -286,6 +301,9 @@ func TestWalker(t *testing.T) {
"/subdir2/subfile2",
"/subdir2/subsubdir2",
"/subdir2/subsubdir2/subsubfile3",
"leave: /subdir2/subsubdir2",
"leave: /subdir2",
"leave: /",
},
),
checkSkipFor(
@ -299,6 +317,8 @@ func TestWalker(t *testing.T) {
"/subdir2",
"/subdir2/subfile2",
"/subdir2/subsubdir2",
"leave: /subdir2",
"leave: /",
},
),
checkSkipFor(
@ -307,6 +327,7 @@ func TestWalker(t *testing.T) {
}, []string{
"/",
"/foo",
"leave: /",
},
),
},
@ -339,15 +360,19 @@ func TestWalker(t *testing.T) {
"/subdir1/subfile1",
"/subdir1/subfile2",
"/subdir1/subfile3",
"leave: /subdir1",
"/subdir2",
"/subdir2/subfile1",
"/subdir2/subfile2",
"/subdir2/subfile3",
"leave: /subdir2",
"/subdir3",
"/subdir3/subfile1",
"/subdir3/subfile2",
"/subdir3/subfile3",
"leave: /subdir3",
"/zzz other",
"leave: /",
}),
checkParentTreeOrder([]string{
"c2efeff7f217a4dfa12a16e8bb3cefedd37c00873605c29e5271c6061030672f", // tree /
@ -385,13 +410,20 @@ func TestWalker(t *testing.T) {
checkItemOrder([]string{
"/",
"/subdir1",
"leave: /subdir1",
"/subdir2",
"leave: /subdir2",
"/subdir3",
"/subdir3/file",
"leave: /subdir3",
"/subdir4",
"/subdir4/file",
"leave: /subdir4",
"/subdir5",
"leave: /subdir5",
"/subdir6",
"leave: /subdir6",
"leave: /",
}),
},
},
@ -405,8 +437,11 @@ func TestWalker(t *testing.T) {
ctx, cancel := context.WithCancel(context.TODO())
defer cancel()
fn, last := check(t)
err := Walk(ctx, repo, root, fn)
fn, leaveDir, last := check(t)
err := Walk(ctx, repo, root, WalkVisitor{
ProcessNode: fn,
LeaveDir: leaveDir,
})
if err != nil {
t.Error(err)
}