Refactor duplicate code for grouping snapshots

This commit is a followup to the addition of the --group-by flag for the
snapshots command. Adding the grouping code there introduced duplicated
code (the forget command also does grouping). This commit refactors
boths sides to only use shared code.

cmd/restic/cmd_forget.go

@@ -4,10 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"io"
-	"sort"
-	"strings"
 
-	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/restic"
 	"github.com/spf13/cobra"
 )
@@ -91,178 +88,129 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error {
 		return err
 	}
 
-	// group by hostname and dirs
-	type key struct {
-		Hostname string
-		Paths    []string
-		Tags     []string
-	}
-	snapshotGroups := make(map[string]restic.Snapshots)
-
-	var GroupByTag bool
-	var GroupByHost bool
-	var GroupByPath bool
-	var GroupOptionList []string
-
-	GroupOptionList = strings.Split(opts.GroupBy, ",")
-
-	for _, option := range GroupOptionList {
-		switch option {
-		case "host":
-			GroupByHost = true
-		case "paths":
-			GroupByPath = true
-		case "tags":
-			GroupByTag = true
-		case "":
-		default:
-			return errors.Fatal("unknown grouping option: '" + option + "'")
-		}
-	}
-
 	removeSnapshots := 0
 
 	ctx, cancel := context.WithCancel(gopts.ctx)
 	defer cancel()
+
+	var snapshots restic.Snapshots
+
 	for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, args) {
-		if len(args) > 0 {
+		snapshots = append(snapshots, sn)
-			// When explicit snapshots args are given, remove them immediately.
+	}
+
+	if len(args) > 0 {
+		// When explicit snapshots args are given, remove them immediately.
+		for _, sn := range snapshots {
 			if !opts.DryRun {
 				h := restic.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()}
 				if err = repo.Backend().Remove(gopts.ctx, h); err != nil {
 					return err
 				}
-				Verbosef("removed snapshot %v\n", sn.ID().Str())
+				if !gopts.JSON {
+					Verbosef("removed snapshot %v\n", sn.ID().Str())
+				}
 				removeSnapshots++
 			} else {
-				Verbosef("would have removed snapshot %v\n", sn.ID().Str())
+				if !gopts.JSON {
+					Verbosef("would have removed snapshot %v\n", sn.ID().Str())
+				}
 			}
-		} else {
-			// Determining grouping-keys
-			var tags []string
-			var hostname string
-			var paths []string
-
-			if GroupByTag {
-				tags = sn.Tags
-				sort.StringSlice(tags).Sort()
-			}
-			if GroupByHost {
-				hostname = sn.Hostname
-			}
-			if GroupByPath {
-				paths = sn.Paths
-			}
-
-			sort.StringSlice(sn.Paths).Sort()
-			var k []byte
-			var err error
-
-			k, err = json.Marshal(key{Tags: tags, Hostname: hostname, Paths: paths})
-
-			if err != nil {
-				return err
-			}
-			snapshotGroups[string(k)] = append(snapshotGroups[string(k)], sn)
 		}
-	}
+	} else {
-
+		snapshotGroups, _, err := restic.GroupSnapshots(snapshots, opts.GroupBy)
-	policy := restic.ExpirePolicy{
+		if err != nil {
-		Last:    opts.Last,
+			return err
-		Hourly:  opts.Hourly,
-		Daily:   opts.Daily,
-		Weekly:  opts.Weekly,
-		Monthly: opts.Monthly,
-		Yearly:  opts.Yearly,
-		Within:  opts.Within,
-		Tags:    opts.KeepTags,
-	}
-
-	if policy.Empty() && len(args) == 0 {
-		Verbosef("no policy was specified, no snapshots will be removed\n")
-	}
-
-	if !policy.Empty() {
-		if !gopts.JSON {
-			Verbosef("Applying Policy: %v\n", policy)
 		}
 
-		var jsonGroups []*ForgetGroup
+		policy := restic.ExpirePolicy{
+			Last:    opts.Last,
+			Hourly:  opts.Hourly,
+			Daily:   opts.Daily,
+			Weekly:  opts.Weekly,
+			Monthly: opts.Monthly,
+			Yearly:  opts.Yearly,
+			Within:  opts.Within,
+			Tags:    opts.KeepTags,
+		}
 
-		for k, snapshotGroup := range snapshotGroups {
+		if policy.Empty() && len(args) == 0 {
-			var key key
-			if json.Unmarshal([]byte(k), &key) != nil {
-				return err
-			}
-
-			var fg ForgetGroup
-			// Info
 			if !gopts.JSON {
-				Verbosef("snapshots")
+				Verbosef("no policy was specified, no snapshots will be removed\n")
-			}
-			var infoStrings []string
-			if GroupByTag {
-				infoStrings = append(infoStrings, "tags ["+strings.Join(key.Tags, ", ")+"]")
-				fg.Tags = key.Tags
-			}
-			if GroupByHost {
-				infoStrings = append(infoStrings, "host ["+key.Hostname+"]")
-				fg.Host = key.Hostname
-			}
-			if GroupByPath {
-				infoStrings = append(infoStrings, "paths ["+strings.Join(key.Paths, ", ")+"]")
-				fg.Paths = key.Paths
-			}
-			if infoStrings != nil && !gopts.JSON {
-				Verbosef(" for (" + strings.Join(infoStrings, ", ") + ")")
 			}
+		}
+
+		if !policy.Empty() {
 			if !gopts.JSON {
-				Verbosef(":\n\n")
+				Verbosef("Applying Policy: %v\n", policy)
 			}
 
-			keep, remove, reasons := restic.ApplyPolicy(snapshotGroup, policy)
+			var jsonGroups []*ForgetGroup
 
-			if len(keep) != 0 && !gopts.Quiet && !gopts.JSON {
+			for k, snapshotGroup := range snapshotGroups {
-				Printf("keep %d snapshots:\n", len(keep))
+				if gopts.Verbose >= 1 && !gopts.JSON {
-				PrintSnapshots(globalOptions.stdout, keep, reasons, opts.Compact)
+					err = PrintSnapshotGroupHeader(gopts.stdout, k)
-				Printf("\n")
-			}
-			addJSONSnapshots(&fg.Keep, keep)
-
-			if len(remove) != 0 && !gopts.Quiet && !gopts.JSON {
-				Printf("remove %d snapshots:\n", len(remove))
-				PrintSnapshots(globalOptions.stdout, remove, nil, opts.Compact)
-				Printf("\n")
-			}
-			addJSONSnapshots(&fg.Remove, remove)
-
-			fg.Reasons = reasons
-
-			jsonGroups = append(jsonGroups, &fg)
-
-			removeSnapshots += len(remove)
-
-			if !opts.DryRun {
-				for _, sn := range remove {
-					h := restic.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()}
-					err = repo.Backend().Remove(gopts.ctx, h)
 					if err != nil {
 						return err
 					}
 				}
-			}
-		}
 
-		if gopts.JSON {
+				var key restic.SnapshotGroupKey
-			err = printJSONForget(gopts.stdout, jsonGroups)
+				if json.Unmarshal([]byte(k), &key) != nil {
-			if err != nil {
+					return err
-				return err
+				}
+
+				var fg ForgetGroup
+				fg.Tags = key.Tags
+				fg.Host = key.Hostname
+				fg.Paths = key.Paths
+
+				keep, remove, reasons := restic.ApplyPolicy(snapshotGroup, policy)
+
+				if len(keep) != 0 && !gopts.Quiet && !gopts.JSON {
+					Printf("keep %d snapshots:\n", len(keep))
+					PrintSnapshots(globalOptions.stdout, keep, reasons, opts.Compact)
+					Printf("\n")
+				}
+				addJSONSnapshots(&fg.Keep, keep)
+
+				if len(remove) != 0 && !gopts.Quiet && !gopts.JSON {
+					Printf("remove %d snapshots:\n", len(remove))
+					PrintSnapshots(globalOptions.stdout, remove, nil, opts.Compact)
+					Printf("\n")
+				}
+				addJSONSnapshots(&fg.Remove, remove)
+
+				fg.Reasons = reasons
+
+				jsonGroups = append(jsonGroups, &fg)
+
+				removeSnapshots += len(remove)
+
+				if !opts.DryRun {
+					for _, sn := range remove {
+						h := restic.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()}
+						err = repo.Backend().Remove(gopts.ctx, h)
+						if err != nil {
+							return err
+						}
+					}
+				}
+			}
+
+			if gopts.JSON {
+				err = printJSONForget(gopts.stdout, jsonGroups)
+				if err != nil {
+					return err
+				}
 			}
 		}
 	}
 
 	if removeSnapshots > 0 && opts.Prune {
-		Verbosef("%d snapshots have been removed, running prune\n", removeSnapshots)
+		if !gopts.JSON {
+			Verbosef("%d snapshots have been removed, running prune\n", removeSnapshots)
+		}
 		if !opts.DryRun {
 			return pruneRepository(gopts, repo)
 		}

cmd/restic/cmd_snapshots.go

@@ -8,7 +8,6 @@ import (
 	"sort"
 	"strings"
 
-	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/restic"
 	"github.com/restic/restic/internal/ui/table"
 	"github.com/spf13/cobra"
@@ -50,12 +49,6 @@ func init() {
 	f.StringVarP(&snapshotOptions.GroupBy, "group-by", "g", "", "string for grouping snapshots by host,paths,tags")
 }
 
-type groupKey struct {
-	Hostname string   `json:"hostname"`
-	Paths    []string `json:"paths"`
-	Tags     []string `json:"tags"`
-}
-
 func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) error {
 	repo, err := OpenRepository(gopts)
 	if err != nil {
@@ -77,7 +70,7 @@ func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) erro
 	for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, args) {
 		snapshots = append(snapshots, sn)
 	}
-	snapshotGroups, grouped, err := GroupSnapshots(snapshots, opts.GroupBy)
+	snapshotGroups, grouped, err := restic.GroupSnapshots(snapshots, opts.GroupBy)
 	if err != nil {
 		return err
 	}
@@ -148,65 +141,6 @@ func FilterLastSnapshots(list restic.Snapshots) restic.Snapshots {
 	return results
 }
 
-// GroupSnapshots takes a list of snapshots and a grouping criteria and creates
-// a group list of snapshots.
-func GroupSnapshots(snapshots restic.Snapshots, options string) (map[string]restic.Snapshots, bool, error) {
-	// group by hostname and dirs
-	snapshotGroups := make(map[string]restic.Snapshots)
-
-	var GroupByTag bool
-	var GroupByHost bool
-	var GroupByPath bool
-	var GroupOptionList []string
-
-	GroupOptionList = strings.Split(options, ",")
-
-	for _, option := range GroupOptionList {
-		switch option {
-		case "host":
-			GroupByHost = true
-		case "paths":
-			GroupByPath = true
-		case "tags":
-			GroupByTag = true
-		case "":
-		default:
-			return nil, false, errors.Fatal("unknown grouping option: '" + option + "'")
-		}
-	}
-
-	for _, sn := range snapshots {
-		// Determining grouping-keys
-		var tags []string
-		var hostname string
-		var paths []string
-
-		if GroupByTag {
-			tags = sn.Tags
-			sort.StringSlice(tags).Sort()
-		}
-		if GroupByHost {
-			hostname = sn.Hostname
-		}
-		if GroupByPath {
-			paths = sn.Paths
-		}
-
-		sort.StringSlice(sn.Paths).Sort()
-		var k []byte
-		var err error
-
-		k, err = json.Marshal(groupKey{Tags: tags, Hostname: hostname, Paths: paths})
-
-		if err != nil {
-			return nil, false, err
-		}
-		snapshotGroups[string(k)] = append(snapshotGroups[string(k)], sn)
-	}
-
-	return snapshotGroups, GroupByTag || GroupByHost || GroupByPath, nil
-}
-
 // PrintSnapshots prints a text table of the snapshots in list to stdout.
 func PrintSnapshots(stdout io.Writer, list restic.Snapshots, reasons []restic.KeepReason, compact bool) {
 	// keep the reasons a snasphot is being kept in a map, so that it doesn't
@@ -311,7 +245,7 @@ func PrintSnapshots(stdout io.Writer, list restic.Snapshots, reasons []restic.Ke
 // following snapshots belong to.
 // Prints nothing, if we did not group at all.
 func PrintSnapshotGroupHeader(stdout io.Writer, groupKeyJSON string) error {
-	var key groupKey
+	var key restic.SnapshotGroupKey
 	var err error
 
 	err = json.Unmarshal([]byte(groupKeyJSON), &key)
@@ -353,8 +287,8 @@ type Snapshot struct {
 
 // SnapshotGroup helps to print SnaphotGroups as JSON with their GroupReasons included.
 type SnapshotGroup struct {
-	GroupKey  groupKey   `json:"group_key"`
+	GroupKey  restic.SnapshotGroupKey `json:"group_key"`
-	Snapshots []Snapshot `json:"snapshots"`
+	Snapshots []Snapshot              `json:"snapshots"`
 }
 
 // printSnapshotsJSON writes the JSON representation of list to stdout.
@@ -363,7 +297,7 @@ func printSnapshotGroupJSON(stdout io.Writer, snGroups map[string]restic.Snapsho
 		var snapshotGroups []SnapshotGroup
 
 		for k, list := range snGroups {
-			var key groupKey
+			var key restic.SnapshotGroupKey
 			var err error
 			var snapshots []Snapshot
 

internal/restic/snapshot_group.go

@@ -0,0 +1,76 @@
+package restic
+
+import (
+	"encoding/json"
+	"sort"
+	"strings"
+
+	"github.com/restic/restic/internal/errors"
+)
+
+// SnapshotGroupKey is the structure for identifying groups in a grouped
+// snapshot list. This is used by GroupSnapshots()
+type SnapshotGroupKey struct {
+	Hostname string   `json:"hostname"`
+	Paths    []string `json:"paths"`
+	Tags     []string `json:"tags"`
+}
+
+// GroupSnapshots takes a list of snapshots and a grouping criteria and creates
+// a group list of snapshots.
+func GroupSnapshots(snapshots Snapshots, options string) (map[string]Snapshots, bool, error) {
+	// group by hostname and dirs
+	snapshotGroups := make(map[string]Snapshots)
+
+	var GroupByTag bool
+	var GroupByHost bool
+	var GroupByPath bool
+	var GroupOptionList []string
+
+	GroupOptionList = strings.Split(options, ",")
+
+	for _, option := range GroupOptionList {
+		switch option {
+		case "host":
+			GroupByHost = true
+		case "paths":
+			GroupByPath = true
+		case "tags":
+			GroupByTag = true
+		case "":
+		default:
+			return nil, false, errors.Fatal("unknown grouping option: '" + option + "'")
+		}
+	}
+
+	for _, sn := range snapshots {
+		// Determining grouping-keys
+		var tags []string
+		var hostname string
+		var paths []string
+
+		if GroupByTag {
+			tags = sn.Tags
+			sort.StringSlice(tags).Sort()
+		}
+		if GroupByHost {
+			hostname = sn.Hostname
+		}
+		if GroupByPath {
+			paths = sn.Paths
+		}
+
+		sort.StringSlice(sn.Paths).Sort()
+		var k []byte
+		var err error
+
+		k, err = json.Marshal(SnapshotGroupKey{Tags: tags, Hostname: hostname, Paths: paths})
+
+		if err != nil {
+			return nil, false, err
+		}
+		snapshotGroups[string(k)] = append(snapshotGroups[string(k)], sn)
+	}
+
+	return snapshotGroups, GroupByTag || GroupByHost || GroupByPath, nil
+}