octoleo/restic
2
2
Fork 0
mirror of https://github.com/octoleo/restic.git synced 2024-11-02 19:49:44 +00:00
Code Issues Releases Wiki Activity

Merge pull request #855 from middelink/fix-851

Browse Source 
Add `tag` command to restic cli to manipulate tags on existing snapshots.
This commit is contained in:
Alexander Neumann 2017-03-05 20:20:20 +01:00
commit d50dc9f649
6 changed files with 363 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
doc/Design.md
View File

@ -285,7 +285,7 @@ This way, the password can be changed without having to re-encrypt all data.
Snapshots
---------
A snapshots represents a directory with all files and sub-directories at a
A snapshot represents a directory with all files and sub-directories at a
given point in time. For each backup that is made, a new snapshot is created. A
snapshot is a JSON document that is stored in an encrypted file below the
directory `snapshots` in the repository. The filename is the storage ID. This
@ -294,6 +294,31 @@ string is unique and used within restic to uniquely identify a snapshot.
The command `restic cat snapshot` can be used as follows to decrypt and
pretty-print the contents of a snapshot file:
```console
$ restic -r /tmp/restic-repo cat snapshot 251c2e58
enter password for repository:
{
"time": "2015-01-02T18:10:50.895208559+01:00",
"tree": "2da81727b6585232894cfbb8f8bdab8d1eccd3d8f7c92bc934d62e62e618ffdf",
"dir": "/tmp/testdata",
"hostname": "kasimir",
"username": "fd0",
"uid": 1000,
"gid": 100,
"tags": [
"NL"
]
}
```
Here it can be seen that this snapshot represents the contents of the directory
`/tmp/testdata`. The most important field is `tree`. When the meta data (e.g.
the tags) of a snapshot change, the snapshot needs to be re-encrypted and saved.
This will change the storage ID, so in order to relate these seemingly
different snapshots, a field `original` is introduced which contains the ID of
the original snapshot, e.g. after adding the tag `DE` to the snapshot above it
becomes:
```console
$ restic -r /tmp/restic-repo cat snapshot 22a5af1b
enter password for repository:
@ -304,12 +329,17 @@ enter password for repository:
"hostname": "kasimir",
"username": "fd0",
"uid": 1000,
"gid": 100
"gid": 100,
"tags": [
"NL",
"DE"
],
"original": "251c2e5841355f743f9d4ffd3260bee765acee40a6229857e32b60446991b837"
}
```
Here it can be seen that this snapshot represents the contents of the directory
`/tmp/testdata`. The most important field is `tree`.
Once introduced, the `original` field is not modified when the snapshot's meta
data is changed again.
All content within a restic repository is referenced according to its SHA-256
hash. Before saving, each file is split into variable sized Blobs of data. The

40
doc/Manual.md
View File

@ -73,6 +73,7 @@ Available Commands:
rebuild-index build a new index file
restore extract the data from a snapshot
snapshots list all snapshots
tag modifies tags on snapshots
unlock remove locks other processes created
version Print version information
@ -394,6 +395,45 @@ enter password for repository:
*eb78040b username kasimir 2015-08-12 13:29:57
```
# Manage tags
Managing tags on snapshots is done with the `tag` command. The existing set of
tags can be replaced completely, tags can be added to removed. The result is
directly visible in the `snapshots` command.
Let's say we want to tag snapshot `590c8fc8` with the tags `NL` and `CH` and
remove all other tags that may be present, the following command does that:
```console
$ restic -r /tmp/backup tag --set NL,CH 590c8fc8
Create exclusive lock for repository
Modified tags on 1 snapshots
```
Note the snapshot ID has changed, so between each change we need to look up
the new ID of the snapshot. But there is an even better way, the `tag` command
accepts `--tag` for a filter, so we can filter snapshots based on the tag we
just added.
So we can add and remove tags incrementally like this:
```console
$ restic -r /tmp/backup tag --tag NL --remove CH
Create exclusive lock for repository
Modified tags on 1 snapshots
$ restic -r /tmp/backup tag --tag NL --add UK
Create exclusive lock for repository
Modified tags on 1 snapshots
$ restic -r /tmp/backup tag --tag NL --remove NL
Create exclusive lock for repository
Modified tags on 1 snapshots
$ restic -r /tmp/backup tag --tag NL --add SOMETHING
No snapshots were modified
```
# Check integrity and consistency
Imagine your repository is saved on a server that has a faulty hard drive, or

4
src/cmds/restic/cmd_snapshots.go
View File

@ -166,7 +166,7 @@ func printSnapshotsReadable(stdout io.Writer, list []*restic.Snapshot) {
type Snapshot struct {
*restic.Snapshot
ID string `json:"id"`
ID *restic.ID `json:"id"`
}
// printSnapshotsJSON writes the JSON representation of list to stdout.
@ -178,7 +178,7 @@ func printSnapshotsJSON(stdout io.Writer, list []*restic.Snapshot) error {
k := Snapshot{
Snapshot: sn,
ID: sn.ID().String(),
ID: sn.ID(),
}
snapshots = append(snapshots, k)
}

172
src/cmds/restic/cmd_tag.go Normal file
View File

@ -0,0 +1,172 @@
package main
import (
"github.com/spf13/cobra"
"restic"
"restic/debug"
"restic/errors"
"restic/repository"
)
var cmdTag = &cobra.Command{
Use: "tag [flags] [snapshot-ID ...]",
Short: "modifies tags on snapshots",
Long: `
The "tag" command allows you to modify tags on exiting snapshots.
You can either set/replace the entire set of tags on a snapshot, or
add tags to/remove tags from the existing set.
When no snapshot-ID is given, all snapshots matching the host, tag and path filter criteria are modified.
`,
RunE: func(cmd *cobra.Command, args []string) error {
return runTag(tagOptions, globalOptions, args)
},
}
// TagOptions bundles all options for the 'tag' command.
type TagOptions struct {
Host string
Paths []string
Tags []string
SetTags []string
AddTags []string
RemoveTags []string
}
var tagOptions TagOptions
func init() {
cmdRoot.AddCommand(cmdTag)
tagFlags := cmdTag.Flags()
tagFlags.StringSliceVar(&tagOptions.SetTags, "set", nil, "`tag` which will replace the existing tags (can be given multiple times)")
tagFlags.StringSliceVar(&tagOptions.AddTags, "add", nil, "`tag` which will be added to the existing tags (can be given multiple times)")
tagFlags.StringSliceVar(&tagOptions.RemoveTags, "remove", nil, "`tag` which will be removed from the existing tags (can be given multiple times)")
tagFlags.StringVarP(&tagOptions.Host, "host", "H", "", `only consider snapshots for this host, when no snapshot ID is given`)
tagFlags.StringSliceVar(&tagOptions.Tags, "tag", nil, "only consider snapshots which include this `tag`, when no snapshot-ID is given")
tagFlags.StringSliceVar(&tagOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`, when no snapshot-ID is given")
}
func changeTags(repo *repository.Repository, snapshotID restic.ID, setTags, addTags, removeTags, tags, paths []string, host string) (bool, error) {
var changed bool
sn, err := restic.LoadSnapshot(repo, snapshotID)
if err != nil {
return false, err
}
if (host != "" && host != sn.Hostname) || !sn.HasTags(tags) || !restic.SamePaths(sn.Paths, paths) {
return false, nil
}
if len(setTags) != 0 {
// Setting the tag to an empty string really means no tags.
if len(setTags) == 1 && setTags[0] == "" {
setTags = nil
}
sn.Tags = setTags
changed = true
} else {
changed = sn.AddTags(addTags)
if sn.RemoveTags(removeTags) {
changed = true
}
}
if changed {
// Retain the original snapshot id over all tag changes.
if sn.Original == nil {
sn.Original = sn.ID()
}
// Save the new snapshot.
id, err := repo.SaveJSONUnpacked(restic.SnapshotFile, sn)
if err != nil {
return false, err
}
debug.Log("new snapshot saved as %v", id.Str())
if err = repo.Flush(); err != nil {
return false, err
}
// Remove the old snapshot.
h := restic.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()}
if err = repo.Backend().Remove(h); err != nil {
return false, err
}
debug.Log("old snapshot %v removed", sn.ID())
}
return changed, nil
}
func runTag(opts TagOptions, gopts GlobalOptions, args []string) error {
if len(opts.SetTags) == 0 && len(opts.AddTags) == 0 && len(opts.RemoveTags) == 0 {
return errors.Fatal("nothing to do!")
}
if len(opts.SetTags) != 0 && (len(opts.AddTags) != 0 || len(opts.RemoveTags) != 0) {
return errors.Fatal("--set and --add/--remove cannot be given at the same time")
}
repo, err := OpenRepository(gopts)
if err != nil {
return err
}
if !gopts.NoLock {
Verbosef("Create exclusive lock for repository\n")
lock, err := lockRepoExclusive(repo)
defer unlockRepo(lock)
if err != nil {
return err
}
}
var ids restic.IDs
if len(args) != 0 {
// When explit snapshot-IDs are given, the filtering does not matter anymore.
opts.Host = ""
opts.Tags = nil
opts.Paths = nil
// Process all snapshot IDs given as arguments.
for _, s := range args {
snapshotID, err := restic.FindSnapshot(repo, s)
if err != nil {
Warnf("could not find a snapshot for ID %q, ignoring: %v\n", s, err)
continue
}
ids = append(ids, snapshotID)
}
ids = ids.Uniq()
} else {
// If there were no arguments, just get all snapshots.
done := make(chan struct{})
defer close(done)
for snapshotID := range repo.List(restic.SnapshotFile, done) {
ids = append(ids, snapshotID)
}
}
changeCnt := 0
for _, id := range ids {
changed, err := changeTags(repo, id, opts.SetTags, opts.AddTags, opts.RemoveTags, opts.Tags, opts.Paths, opts.Host)
if err != nil {
Warnf("unable to modify the tags for snapshot ID %q, ignoring: %v\n", id, err)
continue
}
if changed {
changeCnt++
}
}
if changeCnt == 0 {
Verbosef("No snapshots were modified\n")
} else {
Verbosef("Modified tags on %v snapshots\n", changeCnt)
}
return nil
}

83
src/cmds/restic/integration_test.go
View File

@ -161,7 +161,7 @@ func testRunFind(t testing.TB, gopts GlobalOptions, pattern string) []string {
return strings.Split(string(buf.Bytes()), "\n")
}
func testRunSnapshots(t testing.TB, gopts GlobalOptions) (*Snapshot, map[string]Snapshot) {
func testRunSnapshots(t testing.TB, gopts GlobalOptions) (newest *Snapshot, snapmap map[restic.ID]Snapshot) {
buf := bytes.NewBuffer(nil)
globalOptions.stdout = buf
globalOptions.JSON = true
@ -177,15 +177,14 @@ func testRunSnapshots(t testing.TB, gopts GlobalOptions) (*Snapshot, map[string]
snapshots := []Snapshot{}
OK(t, json.Unmarshal(buf.Bytes(), &snapshots))
var newest *Snapshot
snapmap := make(map[string]Snapshot, len(snapshots))
snapmap = make(map[restic.ID]Snapshot, len(snapshots))
for _, sn := range snapshots {
snapmap[sn.ID] = sn
snapmap[*sn.ID] = sn
if newest == nil || sn.Time.After(newest.Time) {
newest = &sn
}
}
return newest, snapmap
return
}
func testRunForget(t testing.TB, gopts GlobalOptions, args ...string) {
@ -655,6 +654,80 @@ func TestBackupTags(t *testing.T) {
})
}
func testRunTag(t testing.TB, opts TagOptions, gopts GlobalOptions) {
OK(t, runTag(opts, gopts, []string{}))
}
func TestTag(t *testing.T) {
withTestEnvironment(t, func(env *testEnvironment, gopts GlobalOptions) {
datafile := filepath.Join("testdata", "backup-data.tar.gz")
testRunInit(t, gopts)
SetupTarTestFixture(t, env.testdata, datafile)
testRunBackup(t, []string{env.testdata}, BackupOptions{}, gopts)
testRunCheck(t, gopts)
newest, _ := testRunSnapshots(t, gopts)
Assert(t, newest != nil, "expected a new backup, got nil")
Assert(t, len(newest.Tags) == 0,
"expected no tags, got %v", newest.Tags)
Assert(t, newest.Original == nil,
"expected original ID to be nil, got %v", newest.Original)
originalID := *newest.ID
testRunTag(t, TagOptions{SetTags: []string{"NL"}}, gopts)
testRunCheck(t, gopts)
newest, _ = testRunSnapshots(t, gopts)
Assert(t, newest != nil, "expected a new backup, got nil")
Assert(t, len(newest.Tags) == 1 && newest.Tags[0] == "NL",
"set failed, expected one NL tag, got %v", newest.Tags)
Assert(t, newest.Original != nil, "expected original snapshot id, got nil")
Assert(t, *newest.Original == originalID,
"expected original ID to be set to the first snapshot id")
testRunTag(t, TagOptions{AddTags: []string{"CH"}}, gopts)
testRunCheck(t, gopts)
newest, _ = testRunSnapshots(t, gopts)
Assert(t, newest != nil, "expected a new backup, got nil")
Assert(t, len(newest.Tags) == 2 && newest.Tags[0] == "NL" && newest.Tags[1] == "CH",
"add failed, expected CH,NL tags, got %v", newest.Tags)
Assert(t, newest.Original != nil, "expected original snapshot id, got nil")
Assert(t, *newest.Original == originalID,
"expected original ID to be set to the first snapshot id")
testRunTag(t, TagOptions{RemoveTags: []string{"NL"}}, gopts)
testRunCheck(t, gopts)
newest, _ = testRunSnapshots(t, gopts)
Assert(t, newest != nil, "expected a new backup, got nil")
Assert(t, len(newest.Tags) == 1 && newest.Tags[0] == "CH",
"remove failed, expected one CH tag, got %v", newest.Tags)
Assert(t, newest.Original != nil, "expected original snapshot id, got nil")
Assert(t, *newest.Original == originalID,
"expected original ID to be set to the first snapshot id")
testRunTag(t, TagOptions{AddTags: []string{"US", "RU"}}, gopts)
testRunTag(t, TagOptions{RemoveTags: []string{"CH", "US", "RU"}}, gopts)
testRunCheck(t, gopts)
newest, _ = testRunSnapshots(t, gopts)
Assert(t, newest != nil, "expected a new backup, got nil")
Assert(t, len(newest.Tags) == 0,
"expected no tags, got %v", newest.Tags)
Assert(t, newest.Original != nil, "expected original snapshot id, got nil")
Assert(t, *newest.Original == originalID,
"expected original ID to be set to the first snapshot id")
// Check special case of removing all tags.
testRunTag(t, TagOptions{SetTags: []string{""}}, gopts)
testRunCheck(t, gopts)
newest, _ = testRunSnapshots(t, gopts)
Assert(t, newest != nil, "expected a new backup, got nil")
Assert(t, len(newest.Tags) == 0,
"expected no tags, got %v", newest.Tags)
Assert(t, newest.Original != nil, "expected original snapshot id, got nil")
Assert(t, *newest.Original == originalID,
"expected original ID to be set to the first snapshot id")
})
}
func testRunKeyListOtherIDs(t testing.TB, gopts GlobalOptions) []string {
buf := bytes.NewBuffer(nil)

39
src/restic/snapshot.go
View File

@ -21,6 +21,7 @@ type Snapshot struct {
GID uint32 `json:"gid,omitempty"`
Excludes []string `json:"excludes,omitempty"`
Tags []string `json:"tags,omitempty"`
Original *ID `json:"original,omitempty"`
id *ID // plaintext ID, used during restore
}
@ -73,8 +74,7 @@ func LoadAllSnapshots(repo Repository) (snapshots []*Snapshot, err error) {
snapshots = append(snapshots, sn)
}
return snapshots, nil
return
}
func (sn Snapshot) String() string {
@ -99,6 +99,41 @@ func (sn *Snapshot) fillUserInfo() error {
return err
}
// AddTags adds the given tags to the snapshots tags, preventing duplicates.
// It returns true if any changes were made.
func (sn *Snapshot) AddTags(addTags []string) (changed bool) {
nextTag:
for _, add := range addTags {
for _, tag := range sn.Tags {
if tag == add {
continue nextTag
}
}
sn.Tags = append(sn.Tags, add)
changed = true
}
return
}
// RemoveTags removes the given tags from the snapshots tags and
// returns true if any changes were made.
func (sn *Snapshot) RemoveTags(removeTags []string) (changed bool) {
for _, remove := range removeTags {
for i, tag := range sn.Tags {
if tag == remove {
// https://github.com/golang/go/wiki/SliceTricks
sn.Tags[i] = sn.Tags[len(sn.Tags)-1]
sn.Tags[len(sn.Tags)-1] = ""
sn.Tags = sn.Tags[:len(sn.Tags)-1]
changed = true
break
}
}
}
return
}
// HasTags returns true if the snapshot has all the tags.
func (sn *Snapshot) HasTags(tags []string) bool {
nextTag: