From 04c67d700d14f9583fad475521f3e927105e06e5 Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Sun, 19 Aug 2018 00:18:43 -0600 Subject: [PATCH 1/3] ls: Stream output when using --json option --- cmd/restic/cmd_ls.go | 53 ++++++++--------------------------------- cmd/restic/global.go | 4 +++- internal/restic/node.go | 2 +- 3 files changed, 14 insertions(+), 45 deletions(-) diff --git a/cmd/restic/cmd_ls.go b/cmd/restic/cmd_ls.go index d6297be04..3bfd53f9e 100644 --- a/cmd/restic/cmd_ls.go +++ b/cmd/restic/cmd_ls.go @@ -3,9 +3,7 @@ package main import ( "context" "encoding/json" - "os" "strings" - "time" "github.com/spf13/cobra" @@ -64,25 +62,14 @@ func init() { type lsSnapshot struct { *restic.Snapshot - ID *restic.ID `json:"id"` ShortID string `json:"short_id"` - Nodes []lsNode `json:"nodes"` StructType string `json:"struct_type"` // "snapshot" } type lsNode struct { - Name string `json:"name"` - Type string `json:"type"` - Path string `json:"path"` - UID uint32 `json:"uid"` - GID uint32 `json:"gid"` - Size uint64 `json:"size,omitempty"` - Mode os.FileMode `json:"mode,omitempty"` - ModTime time.Time `json:"mtime,omitempty"` - AccessTime time.Time `json:"atime,omitempty"` - ChangeTime time.Time `json:"ctime,omitempty"` - StructType string `json:"struct_type"` // "node" + *restic.Node + StructType string `json:"struct_type"` // "node" } func runLs(opts LsOptions, gopts GlobalOptions, args []string) error { @@ -150,54 +137,33 @@ func runLs(opts LsOptions, gopts GlobalOptions, args []string) error { var ( printSnapshot func(sn *restic.Snapshot) printNode func(path string, node *restic.Node) - printFinish func() error ) if gopts.JSON { - var lssnapshots []lsSnapshot + enc := json.NewEncoder(gopts.stdout) printSnapshot = func(sn *restic.Snapshot) { - lss := lsSnapshot{ + enc.Encode(lsSnapshot{ Snapshot: sn, ID: sn.ID(), ShortID: sn.ID().Str(), StructType: "snapshot", - } - lssnapshots = append(lssnapshots, lss) + }) } printNode = func(path string, node *restic.Node) { - lsn := lsNode{ - Name: node.Name, - Type: node.Type, - Path: path, - UID: node.UID, - GID: node.GID, - Size: node.Size, - Mode: node.Mode, - ModTime: node.ModTime, - AccessTime: node.AccessTime, - ChangeTime: node.ChangeTime, + enc.Encode(lsNode{ + Node: node, StructType: "node", - } - s := &lssnapshots[len(lssnapshots)-1] - s.Nodes = append(s.Nodes, lsn) - } - - printFinish = func() error { - return json.NewEncoder(gopts.stdout).Encode(lssnapshots) + }) } } else { - // default output methods printSnapshot = func(sn *restic.Snapshot) { Verbosef("snapshot %s of %v filtered by %v at %s):\n", sn.ID().Str(), sn.Paths, dirs, sn.Time) } printNode = func(path string, node *restic.Node) { Printf("%s\n", formatNode(path, node, lsOptions.ListLong)) } - printFinish = func() error { - return nil - } } for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, args[:1]) { @@ -240,5 +206,6 @@ func runLs(opts LsOptions, gopts GlobalOptions, args []string) error { return err } } - return printFinish() + + return nil } diff --git a/cmd/restic/global.go b/cmd/restic/global.go index d35b6ba4f..902734990 100644 --- a/cmd/restic/global.go +++ b/cmd/restic/global.go @@ -360,7 +360,9 @@ func OpenRepository(opts GlobalOptions) (*repository.Repository, error) { if len(id) > 8 { id = id[:8] } - Verbosef("repository %v opened successfully, password is correct\n", id) + if !opts.JSON { + Verbosef("repository %v opened successfully, password is correct\n", id) + } } if opts.NoCache { diff --git a/internal/restic/node.go b/internal/restic/node.go index cec7938e5..24fe4598e 100644 --- a/internal/restic/node.go +++ b/internal/restic/node.go @@ -50,7 +50,7 @@ type Node struct { Error string `json:"error,omitempty"` - Path string `json:"-"` + Path string `json:"path"` } // Nodes is a slice of nodes that can be sorted. From 22475729ce46c949d5c2825b40d774e52e3055fa Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Sun, 19 Aug 2018 09:03:47 -0600 Subject: [PATCH 2/3] Select specific Node fields for listing --- cmd/restic/cmd_ls.go | 26 +++++++++++++++++++++++--- internal/restic/node.go | 2 +- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/cmd/restic/cmd_ls.go b/cmd/restic/cmd_ls.go index 3bfd53f9e..c91cb5af7 100644 --- a/cmd/restic/cmd_ls.go +++ b/cmd/restic/cmd_ls.go @@ -3,7 +3,9 @@ package main import ( "context" "encoding/json" + "os" "strings" + "time" "github.com/spf13/cobra" @@ -68,8 +70,17 @@ type lsSnapshot struct { } type lsNode struct { - *restic.Node - StructType string `json:"struct_type"` // "node" + Name string `json:"name"` + Type string `json:"type"` + Path string `json:"path"` + UID uint32 `json:"uid"` + GID uint32 `json:"gid"` + Size uint64 `json:"size,omitempty"` + Mode os.FileMode `json:"mode,omitempty"` + ModTime time.Time `json:"mtime,omitempty"` + AccessTime time.Time `json:"atime,omitempty"` + ChangeTime time.Time `json:"ctime,omitempty"` + StructType string `json:"struct_type"` // "node" } func runLs(opts LsOptions, gopts GlobalOptions, args []string) error { @@ -153,7 +164,16 @@ func runLs(opts LsOptions, gopts GlobalOptions, args []string) error { printNode = func(path string, node *restic.Node) { enc.Encode(lsNode{ - Node: node, + Name: node.Name, + Type: node.Type, + Path: path, + UID: node.UID, + GID: node.GID, + Size: node.Size, + Mode: node.Mode, + ModTime: node.ModTime, + AccessTime: node.AccessTime, + ChangeTime: node.ChangeTime, StructType: "node", }) } diff --git a/internal/restic/node.go b/internal/restic/node.go index 24fe4598e..cec7938e5 100644 --- a/internal/restic/node.go +++ b/internal/restic/node.go @@ -50,7 +50,7 @@ type Node struct { Error string `json:"error,omitempty"` - Path string `json:"path"` + Path string `json:"-"` } // Nodes is a slice of nodes that can be sorted. From 9151eec24eb083a9f3f6389f7a497bb8b2d437e5 Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Sun, 19 Aug 2018 09:14:12 -0600 Subject: [PATCH 3/3] Add changelog entry --- changelog/unreleased/{1941 => pull-1941} | 0 changelog/unreleased/pull-1962 | 13 +++++++++++++ 2 files changed, 13 insertions(+) rename changelog/unreleased/{1941 => pull-1941} (100%) create mode 100644 changelog/unreleased/pull-1962 diff --git a/changelog/unreleased/1941 b/changelog/unreleased/pull-1941 similarity index 100% rename from changelog/unreleased/1941 rename to changelog/unreleased/pull-1941 diff --git a/changelog/unreleased/pull-1962 b/changelog/unreleased/pull-1962 new file mode 100644 index 000000000..e3fde4c48 --- /dev/null +++ b/changelog/unreleased/pull-1962 @@ -0,0 +1,13 @@ +Enhancement: Stream JSON output for ls command + +The `ls` command now supports JSON output with the global `--json` +flag, and this change streams out JSON messages one object at a time +rather than en entire array buffered in memory before encoding. The +advantage is it allows large listings to be handled efficiently. + +Two message types are printed: snapshots and nodes. A snapshot +object will precede node objects which belong to that snapshot. +The `struct_type` field can be used to determine which kind of +message an object is. + +https://github.com/restic/restic/pull/1962