Merge pull request #855 from middelink/fix-851

Add `tag` command to restic cli to manipulate tags on existing snapshots.

doc/Design.md

@@ -285,7 +285,7 @@ This way, the password can be changed without having to re-encrypt all data.
 Snapshots
 ---------
 
-A snapshots represents a directory with all files and sub-directories at a
+A snapshot represents a directory with all files and sub-directories at a
 given point in time. For each backup that is made, a new snapshot is created. A
 snapshot is a JSON document that is stored in an encrypted file below the
 directory `snapshots` in the repository. The filename is the storage ID. This
@@ -294,6 +294,31 @@ string is unique and used within restic to uniquely identify a snapshot.
 The command `restic cat snapshot` can be used as follows to decrypt and
 pretty-print the contents of a snapshot file:
 
+```console
+$ restic -r /tmp/restic-repo cat snapshot 251c2e58
+enter password for repository:
+{
+  "time": "2015-01-02T18:10:50.895208559+01:00",
+  "tree": "2da81727b6585232894cfbb8f8bdab8d1eccd3d8f7c92bc934d62e62e618ffdf",
+  "dir": "/tmp/testdata",
+  "hostname": "kasimir",
+  "username": "fd0",
+  "uid": 1000,
+  "gid": 100,
+  "tags": [
+    "NL"
+  ]
+}
+```
+
+Here it can be seen that this snapshot represents the contents of the directory
+`/tmp/testdata`. The most important field is `tree`. When the meta data (e.g.
+the tags) of a snapshot change, the snapshot needs to be re-encrypted and saved.
+This will change the storage ID, so in order to relate these seemingly
+different snapshots, a field `original` is introduced which contains the ID of
+the original snapshot, e.g. after adding the tag `DE` to the snapshot above it
+becomes:
+
 ```console
 $ restic -r /tmp/restic-repo cat snapshot 22a5af1b
 enter password for repository:
@@ -304,12 +329,17 @@ enter password for repository:
   "hostname": "kasimir",
   "username": "fd0",
   "uid": 1000,
-  "gid": 100
+  "gid": 100,
+  "tags": [
+    "NL",
+    "DE"
+  ],
+  "original": "251c2e5841355f743f9d4ffd3260bee765acee40a6229857e32b60446991b837"
 }
 ```
 
-Here it can be seen that this snapshot represents the contents of the directory
+Once introduced, the `original` field is not modified when the snapshot's meta
-`/tmp/testdata`. The most important field is `tree`.
+data is changed again.
 
 All content within a restic repository is referenced according to its SHA-256
 hash. Before saving, each file is split into variable sized Blobs of data. The

doc/Manual.md

@@ -73,6 +73,7 @@ Available Commands:
   rebuild-index build a new index file
   restore       extract the data from a snapshot
   snapshots     list all snapshots
+  tag           modifies tags on snapshots
   unlock        remove locks other processes created
   version       Print version information
 
@@ -394,6 +395,45 @@ enter password for repository:
 *eb78040b    username    kasimir   2015-08-12 13:29:57
 ```
 
+# Manage tags
+
+Managing tags on snapshots is done with the `tag` command. The existing set of
+tags can be replaced completely, tags can be added to removed. The result is
+directly visible in the `snapshots` command.
+
+Let's say we want to tag snapshot `590c8fc8` with the tags `NL` and `CH` and
+remove all other tags that may be present, the following command does that:
+
+```console
+$ restic -r /tmp/backup tag --set NL,CH 590c8fc8
+Create exclusive lock for repository
+Modified tags on 1 snapshots
+```
+
+Note the snapshot ID has changed, so between each change we need to look up
+the new ID of the snapshot. But there is an even better way, the `tag` command
+accepts `--tag` for a filter, so we can filter snapshots based on the tag we
+just added.
+
+So we can add and remove tags incrementally like this:
+
+```console
+$ restic -r /tmp/backup tag --tag NL --remove CH
+Create exclusive lock for repository
+Modified tags on 1 snapshots
+
+$ restic -r /tmp/backup tag --tag NL --add UK
+Create exclusive lock for repository
+Modified tags on 1 snapshots
+
+$ restic -r /tmp/backup tag --tag NL --remove NL
+Create exclusive lock for repository
+Modified tags on 1 snapshots
+
+$ restic -r /tmp/backup tag --tag NL --add SOMETHING
+No snapshots were modified
+```
+
 # Check integrity and consistency
 
 Imagine your repository is saved on a server that has a faulty hard drive, or

src/cmds/restic/cmd_snapshots.go

@@ -166,7 +166,7 @@ func printSnapshotsReadable(stdout io.Writer, list []*restic.Snapshot) {
 type Snapshot struct {
 	*restic.Snapshot
 
-	ID string `json:"id"`
+	ID *restic.ID `json:"id"`
 }
 
 // printSnapshotsJSON writes the JSON representation of list to stdout.
@@ -178,7 +178,7 @@ func printSnapshotsJSON(stdout io.Writer, list []*restic.Snapshot) error {
 
 		k := Snapshot{
 			Snapshot: sn,
-			ID:       sn.ID().String(),
+			ID:       sn.ID(),
 		}
 		snapshots = append(snapshots, k)
 	}

src/cmds/restic/cmd_tag.go

@@ -0,0 +1,172 @@
+package main
+
+import (
+	"github.com/spf13/cobra"
+
+	"restic"
+	"restic/debug"
+	"restic/errors"
+	"restic/repository"
+)
+
+var cmdTag = &cobra.Command{
+	Use:   "tag [flags] [snapshot-ID ...]",
+	Short: "modifies tags on snapshots",
+	Long: `
+The "tag" command allows you to modify tags on exiting snapshots.
+
+You can either set/replace the entire set of tags on a snapshot, or
+add tags to/remove tags from the existing set.
+
+When no snapshot-ID is given, all snapshots matching the host, tag and path filter criteria are modified.
+`,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		return runTag(tagOptions, globalOptions, args)
+	},
+}
+
+// TagOptions bundles all options for the 'tag' command.
+type TagOptions struct {
+	Host       string
+	Paths      []string
+	Tags       []string
+	SetTags    []string
+	AddTags    []string
+	RemoveTags []string
+}
+
+var tagOptions TagOptions
+
+func init() {
+	cmdRoot.AddCommand(cmdTag)
+
+	tagFlags := cmdTag.Flags()
+	tagFlags.StringSliceVar(&tagOptions.SetTags, "set", nil, "`tag` which will replace the existing tags (can be given multiple times)")
+	tagFlags.StringSliceVar(&tagOptions.AddTags, "add", nil, "`tag` which will be added to the existing tags (can be given multiple times)")
+	tagFlags.StringSliceVar(&tagOptions.RemoveTags, "remove", nil, "`tag` which will be removed from the existing tags (can be given multiple times)")
+
+	tagFlags.StringVarP(&tagOptions.Host, "host", "H", "", `only consider snapshots for this host, when no snapshot ID is given`)
+	tagFlags.StringSliceVar(&tagOptions.Tags, "tag", nil, "only consider snapshots which include this `tag`, when no snapshot-ID is given")
+	tagFlags.StringSliceVar(&tagOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`, when no snapshot-ID is given")
+}
+
+func changeTags(repo *repository.Repository, snapshotID restic.ID, setTags, addTags, removeTags, tags, paths []string, host string) (bool, error) {
+	var changed bool
+
+	sn, err := restic.LoadSnapshot(repo, snapshotID)
+	if err != nil {
+		return false, err
+	}
+	if (host != "" && host != sn.Hostname) || !sn.HasTags(tags) || !restic.SamePaths(sn.Paths, paths) {
+		return false, nil
+	}
+
+	if len(setTags) != 0 {
+		// Setting the tag to an empty string really means no tags.
+		if len(setTags) == 1 && setTags[0] == "" {
+			setTags = nil
+		}
+		sn.Tags = setTags
+		changed = true
+	} else {
+		changed = sn.AddTags(addTags)
+		if sn.RemoveTags(removeTags) {
+			changed = true
+		}
+	}
+
+	if changed {
+		// Retain the original snapshot id over all tag changes.
+		if sn.Original == nil {
+			sn.Original = sn.ID()
+		}
+
+		// Save the new snapshot.
+		id, err := repo.SaveJSONUnpacked(restic.SnapshotFile, sn)
+		if err != nil {
+			return false, err
+		}
+
+		debug.Log("new snapshot saved as %v", id.Str())
+
+		if err = repo.Flush(); err != nil {
+			return false, err
+		}
+
+		// Remove the old snapshot.
+		h := restic.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()}
+		if err = repo.Backend().Remove(h); err != nil {
+			return false, err
+		}
+
+		debug.Log("old snapshot %v removed", sn.ID())
+	}
+	return changed, nil
+}
+
+func runTag(opts TagOptions, gopts GlobalOptions, args []string) error {
+	if len(opts.SetTags) == 0 && len(opts.AddTags) == 0 && len(opts.RemoveTags) == 0 {
+		return errors.Fatal("nothing to do!")
+	}
+	if len(opts.SetTags) != 0 && (len(opts.AddTags) != 0 || len(opts.RemoveTags) != 0) {
+		return errors.Fatal("--set and --add/--remove cannot be given at the same time")
+	}
+
+	repo, err := OpenRepository(gopts)
+	if err != nil {
+		return err
+	}
+
+	if !gopts.NoLock {
+		Verbosef("Create exclusive lock for repository\n")
+		lock, err := lockRepoExclusive(repo)
+		defer unlockRepo(lock)
+		if err != nil {
+			return err
+		}
+	}
+
+	var ids restic.IDs
+	if len(args) != 0 {
+		// When explit snapshot-IDs are given, the filtering does not matter anymore.
+		opts.Host = ""
+		opts.Tags = nil
+		opts.Paths = nil
+
+		// Process all snapshot IDs given as arguments.
+		for _, s := range args {
+			snapshotID, err := restic.FindSnapshot(repo, s)
+			if err != nil {
+				Warnf("could not find a snapshot for ID %q, ignoring: %v\n", s, err)
+				continue
+			}
+			ids = append(ids, snapshotID)
+		}
+		ids = ids.Uniq()
+	} else {
+		// If there were no arguments, just get all snapshots.
+		done := make(chan struct{})
+		defer close(done)
+		for snapshotID := range repo.List(restic.SnapshotFile, done) {
+			ids = append(ids, snapshotID)
+		}
+	}
+
+	changeCnt := 0
+	for _, id := range ids {
+		changed, err := changeTags(repo, id, opts.SetTags, opts.AddTags, opts.RemoveTags, opts.Tags, opts.Paths, opts.Host)
+		if err != nil {
+			Warnf("unable to modify the tags for snapshot ID %q, ignoring: %v\n", id, err)
+			continue
+		}
+		if changed {
+			changeCnt++
+		}
+	}
+	if changeCnt == 0 {
+		Verbosef("No snapshots were modified\n")
+	} else {
+		Verbosef("Modified tags on %v snapshots\n", changeCnt)
+	}
+	return nil
+}

src/cmds/restic/integration_test.go

@@ -161,7 +161,7 @@ func testRunFind(t testing.TB, gopts GlobalOptions, pattern string) []string {
 	return strings.Split(string(buf.Bytes()), "\n")
 }
 
-func testRunSnapshots(t testing.TB, gopts GlobalOptions) (*Snapshot, map[string]Snapshot) {
+func testRunSnapshots(t testing.TB, gopts GlobalOptions) (newest *Snapshot, snapmap map[restic.ID]Snapshot) {
 	buf := bytes.NewBuffer(nil)
 	globalOptions.stdout = buf
 	globalOptions.JSON = true
@@ -177,15 +177,14 @@ func testRunSnapshots(t testing.TB, gopts GlobalOptions) (*Snapshot, map[string]
 	snapshots := []Snapshot{}
 	OK(t, json.Unmarshal(buf.Bytes(), &snapshots))
 
-	var newest *Snapshot
+	snapmap = make(map[restic.ID]Snapshot, len(snapshots))
-	snapmap := make(map[string]Snapshot, len(snapshots))
 	for _, sn := range snapshots {
-		snapmap[sn.ID] = sn
+		snapmap[*sn.ID] = sn
 		if newest == nil || sn.Time.After(newest.Time) {
 			newest = &sn
 		}
 	}
-	return newest, snapmap
+	return
 }
 
 func testRunForget(t testing.TB, gopts GlobalOptions, args ...string) {
@@ -655,6 +654,80 @@ func TestBackupTags(t *testing.T) {
 	})
 }
 
+func testRunTag(t testing.TB, opts TagOptions, gopts GlobalOptions) {
+	OK(t, runTag(opts, gopts, []string{}))
+}
+
+func TestTag(t *testing.T) {
+	withTestEnvironment(t, func(env *testEnvironment, gopts GlobalOptions) {
+		datafile := filepath.Join("testdata", "backup-data.tar.gz")
+		testRunInit(t, gopts)
+		SetupTarTestFixture(t, env.testdata, datafile)
+
+		testRunBackup(t, []string{env.testdata}, BackupOptions{}, gopts)
+		testRunCheck(t, gopts)
+		newest, _ := testRunSnapshots(t, gopts)
+		Assert(t, newest != nil, "expected a new backup, got nil")
+		Assert(t, len(newest.Tags) == 0,
+			"expected no tags, got %v", newest.Tags)
+		Assert(t, newest.Original == nil,
+			"expected original ID to be nil, got %v", newest.Original)
+		originalID := *newest.ID
+
+		testRunTag(t, TagOptions{SetTags: []string{"NL"}}, gopts)
+		testRunCheck(t, gopts)
+		newest, _ = testRunSnapshots(t, gopts)
+		Assert(t, newest != nil, "expected a new backup, got nil")
+		Assert(t, len(newest.Tags) == 1 && newest.Tags[0] == "NL",
+			"set failed, expected one NL tag, got %v", newest.Tags)
+		Assert(t, newest.Original != nil, "expected original snapshot id, got nil")
+		Assert(t, *newest.Original == originalID,
+			"expected original ID to be set to the first snapshot id")
+
+		testRunTag(t, TagOptions{AddTags: []string{"CH"}}, gopts)
+		testRunCheck(t, gopts)
+		newest, _ = testRunSnapshots(t, gopts)
+		Assert(t, newest != nil, "expected a new backup, got nil")
+		Assert(t, len(newest.Tags) == 2 && newest.Tags[0] == "NL" && newest.Tags[1] == "CH",
+			"add failed, expected CH,NL tags, got %v", newest.Tags)
+		Assert(t, newest.Original != nil, "expected original snapshot id, got nil")
+		Assert(t, *newest.Original == originalID,
+			"expected original ID to be set to the first snapshot id")
+
+		testRunTag(t, TagOptions{RemoveTags: []string{"NL"}}, gopts)
+		testRunCheck(t, gopts)
+		newest, _ = testRunSnapshots(t, gopts)
+		Assert(t, newest != nil, "expected a new backup, got nil")
+		Assert(t, len(newest.Tags) == 1 && newest.Tags[0] == "CH",
+			"remove failed, expected one CH tag, got %v", newest.Tags)
+		Assert(t, newest.Original != nil, "expected original snapshot id, got nil")
+		Assert(t, *newest.Original == originalID,
+			"expected original ID to be set to the first snapshot id")
+
+		testRunTag(t, TagOptions{AddTags: []string{"US", "RU"}}, gopts)
+		testRunTag(t, TagOptions{RemoveTags: []string{"CH", "US", "RU"}}, gopts)
+		testRunCheck(t, gopts)
+		newest, _ = testRunSnapshots(t, gopts)
+		Assert(t, newest != nil, "expected a new backup, got nil")
+		Assert(t, len(newest.Tags) == 0,
+			"expected no tags, got %v", newest.Tags)
+		Assert(t, newest.Original != nil, "expected original snapshot id, got nil")
+		Assert(t, *newest.Original == originalID,
+			"expected original ID to be set to the first snapshot id")
+
+		// Check special case of removing all tags.
+		testRunTag(t, TagOptions{SetTags: []string{""}}, gopts)
+		testRunCheck(t, gopts)
+		newest, _ = testRunSnapshots(t, gopts)
+		Assert(t, newest != nil, "expected a new backup, got nil")
+		Assert(t, len(newest.Tags) == 0,
+			"expected no tags, got %v", newest.Tags)
+		Assert(t, newest.Original != nil, "expected original snapshot id, got nil")
+		Assert(t, *newest.Original == originalID,
+			"expected original ID to be set to the first snapshot id")
+	})
+}
+
 func testRunKeyListOtherIDs(t testing.TB, gopts GlobalOptions) []string {
 	buf := bytes.NewBuffer(nil)
 

src/restic/snapshot.go

@@ -21,6 +21,7 @@ type Snapshot struct {
 	GID      uint32    `json:"gid,omitempty"`
 	Excludes []string  `json:"excludes,omitempty"`
 	Tags     []string  `json:"tags,omitempty"`
+	Original *ID       `json:"original,omitempty"`
 
 	id *ID // plaintext ID, used during restore
 }
@@ -73,8 +74,7 @@ func LoadAllSnapshots(repo Repository) (snapshots []*Snapshot, err error) {
 
 		snapshots = append(snapshots, sn)
 	}
-
+	return
-	return snapshots, nil
 }
 
 func (sn Snapshot) String() string {
@@ -99,6 +99,41 @@ func (sn *Snapshot) fillUserInfo() error {
 	return err
 }
 
+// AddTags adds the given tags to the snapshots tags, preventing duplicates.
+// It returns true if any changes were made.
+func (sn *Snapshot) AddTags(addTags []string) (changed bool) {
+nextTag:
+	for _, add := range addTags {
+		for _, tag := range sn.Tags {
+			if tag == add {
+				continue nextTag
+			}
+		}
+		sn.Tags = append(sn.Tags, add)
+		changed = true
+	}
+	return
+}
+
+// RemoveTags removes the given tags from the snapshots tags and
+// returns true if any changes were made.
+func (sn *Snapshot) RemoveTags(removeTags []string) (changed bool) {
+	for _, remove := range removeTags {
+		for i, tag := range sn.Tags {
+			if tag == remove {
+				// https://github.com/golang/go/wiki/SliceTricks
+				sn.Tags[i] = sn.Tags[len(sn.Tags)-1]
+				sn.Tags[len(sn.Tags)-1] = ""
+				sn.Tags = sn.Tags[:len(sn.Tags)-1]
+
+				changed = true
+				break
+			}
+		}
+	}
+	return
+}
+
 // HasTags returns true if the snapshot has all the tags.
 func (sn *Snapshot) HasTags(tags []string) bool {
 nextTag: