2
2
mirror of https://github.com/octoleo/restic.git synced 2024-12-22 19:08:55 +00:00

Merge pull request #4212 from greatroar/snapshotfilter

cmd, restic: Refactor and fix snapshot filtering
This commit is contained in:
Michael Eischer 2023-02-19 15:20:18 +01:00 committed by GitHub
commit cf6dfd6d36
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
19 changed files with 124 additions and 106 deletions

View File

@ -0,0 +1,8 @@
Bugfix: Restic dump now interprets --host and --path correctly
Restic dump previously confused its --host=<host> and --path=<path>
options: it looked for snapshots with paths called <host> from hosts
called <path>. It now treats the options as intended.
https://github.com/restic/restic/issues/4211
https://github.com/restic/restic/pull/4212

View File

@ -442,21 +442,18 @@ func findParentSnapshot(ctx context.Context, repo restic.Repository, opts Backup
if snName == "" { if snName == "" {
snName = "latest" snName = "latest"
} }
f := restic.SnapshotFilter{TimestampLimit: timeStampLimit}
var hosts []string
var paths []string
var tags []restic.TagList
if opts.GroupBy.Host { if opts.GroupBy.Host {
hosts = []string{opts.Host} f.Hosts = []string{opts.Host}
} }
if opts.GroupBy.Path { if opts.GroupBy.Path {
paths = targets f.Paths = targets
} }
if opts.GroupBy.Tag { if opts.GroupBy.Tag {
tags = []restic.TagList{opts.Tags.Flatten()} f.Tags = []restic.TagList{opts.Tags.Flatten()}
} }
sn, err := restic.FindFilteredSnapshot(ctx, repo.Backend(), repo, hosts, tags, paths, &timeStampLimit, snName) sn, err := f.FindLatest(ctx, repo.Backend(), repo, snName)
// Snapshot not found is ok if no explicit parent was set // Snapshot not found is ok if no explicit parent was set
if opts.Parent == "" && errors.Is(err, restic.ErrNoSnapshotFound) { if opts.Parent == "" && errors.Is(err, restic.ErrNoSnapshotFound) {
err = nil err = nil

View File

@ -39,7 +39,7 @@ new destination repository using the "init" command.
// CopyOptions bundles all options for the copy command. // CopyOptions bundles all options for the copy command.
type CopyOptions struct { type CopyOptions struct {
secondaryRepoOptions secondaryRepoOptions
snapshotFilterOptions restic.SnapshotFilter
} }
var copyOptions CopyOptions var copyOptions CopyOptions
@ -49,7 +49,7 @@ func init() {
f := cmdCopy.Flags() f := cmdCopy.Flags()
initSecondaryRepoOptions(f, &copyOptions.secondaryRepoOptions, "destination", "to copy snapshots from") initSecondaryRepoOptions(f, &copyOptions.secondaryRepoOptions, "destination", "to copy snapshots from")
initMultiSnapshotFilterOptions(f, &copyOptions.snapshotFilterOptions, true) initMultiSnapshotFilter(f, &copyOptions.SnapshotFilter, true)
} }
func runCopy(ctx context.Context, opts CopyOptions, gopts GlobalOptions, args []string) error { func runCopy(ctx context.Context, opts CopyOptions, gopts GlobalOptions, args []string) error {
@ -108,7 +108,7 @@ func runCopy(ctx context.Context, opts CopyOptions, gopts GlobalOptions, args []
} }
dstSnapshotByOriginal := make(map[restic.ID][]*restic.Snapshot) dstSnapshotByOriginal := make(map[restic.ID][]*restic.Snapshot)
for sn := range FindFilteredSnapshots(ctx, dstSnapshotLister, dstRepo, opts.Hosts, opts.Tags, opts.Paths, nil) { for sn := range FindFilteredSnapshots(ctx, dstSnapshotLister, dstRepo, &opts.SnapshotFilter, nil) {
if sn.Original != nil && !sn.Original.IsNull() { if sn.Original != nil && !sn.Original.IsNull() {
dstSnapshotByOriginal[*sn.Original] = append(dstSnapshotByOriginal[*sn.Original], sn) dstSnapshotByOriginal[*sn.Original] = append(dstSnapshotByOriginal[*sn.Original], sn)
} }
@ -119,8 +119,7 @@ func runCopy(ctx context.Context, opts CopyOptions, gopts GlobalOptions, args []
// remember already processed trees across all snapshots // remember already processed trees across all snapshots
visitedTrees := restic.NewIDSet() visitedTrees := restic.NewIDSet()
for sn := range FindFilteredSnapshots(ctx, srcSnapshotLister, srcRepo, opts.Hosts, opts.Tags, opts.Paths, args) { for sn := range FindFilteredSnapshots(ctx, srcSnapshotLister, srcRepo, &opts.SnapshotFilter, args) {
// check whether the destination has a snapshot with the same persistent ID which has similar snapshot fields // check whether the destination has a snapshot with the same persistent ID which has similar snapshot fields
srcOriginal := *sn.ID() srcOriginal := *sn.ID()
if sn.Original != nil { if sn.Original != nil {

View File

@ -40,7 +40,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
// DumpOptions collects all options for the dump command. // DumpOptions collects all options for the dump command.
type DumpOptions struct { type DumpOptions struct {
snapshotFilterOptions restic.SnapshotFilter
Archive string Archive string
} }
@ -50,7 +50,7 @@ func init() {
cmdRoot.AddCommand(cmdDump) cmdRoot.AddCommand(cmdDump)
flags := cmdDump.Flags() flags := cmdDump.Flags()
initSingleSnapshotFilterOptions(flags, &dumpOptions.snapshotFilterOptions) initSingleSnapshotFilter(flags, &dumpOptions.SnapshotFilter)
flags.StringVarP(&dumpOptions.Archive, "archive", "a", "tar", "set archive `format` as \"tar\" or \"zip\"") flags.StringVarP(&dumpOptions.Archive, "archive", "a", "tar", "set archive `format` as \"tar\" or \"zip\"")
} }
@ -139,7 +139,11 @@ func runDump(ctx context.Context, opts DumpOptions, gopts GlobalOptions, args []
} }
} }
sn, err := restic.FindFilteredSnapshot(ctx, repo.Backend(), repo, opts.Paths, opts.Tags, opts.Hosts, nil, snapshotIDString) sn, err := (&restic.SnapshotFilter{
Hosts: opts.Hosts,
Paths: opts.Paths,
Tags: opts.Tags,
}).FindLatest(ctx, repo.Backend(), repo, snapshotIDString)
if err != nil { if err != nil {
return errors.Fatalf("failed to find snapshot: %v", err) return errors.Fatalf("failed to find snapshot: %v", err)
} }

View File

@ -51,7 +51,7 @@ type FindOptions struct {
PackID, ShowPackID bool PackID, ShowPackID bool
CaseInsensitive bool CaseInsensitive bool
ListLong bool ListLong bool
snapshotFilterOptions restic.SnapshotFilter
} }
var findOptions FindOptions var findOptions FindOptions
@ -70,7 +70,7 @@ func init() {
f.BoolVarP(&findOptions.CaseInsensitive, "ignore-case", "i", false, "ignore case for pattern") f.BoolVarP(&findOptions.CaseInsensitive, "ignore-case", "i", false, "ignore case for pattern")
f.BoolVarP(&findOptions.ListLong, "long", "l", false, "use a long listing format showing size and mode") f.BoolVarP(&findOptions.ListLong, "long", "l", false, "use a long listing format showing size and mode")
initMultiSnapshotFilterOptions(f, &findOptions.snapshotFilterOptions, true) initMultiSnapshotFilter(f, &findOptions.SnapshotFilter, true)
} }
type findPattern struct { type findPattern struct {
@ -618,7 +618,7 @@ func runFind(ctx context.Context, opts FindOptions, gopts GlobalOptions, args []
} }
} }
for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, opts.Hosts, opts.Tags, opts.Paths, opts.Snapshots) { for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, &opts.SnapshotFilter, opts.Snapshots) {
if f.blobIDs != nil || f.treeIDs != nil { if f.blobIDs != nil || f.treeIDs != nil {
if err = f.findIDs(ctx, sn); err != nil && err.Error() != "OK" { if err = f.findIDs(ctx, sn); err != nil && err.Error() != "OK" {
return err return err

View File

@ -52,7 +52,7 @@ type ForgetOptions struct {
WithinYearly restic.Duration WithinYearly restic.Duration
KeepTags restic.TagLists KeepTags restic.TagLists
snapshotFilterOptions restic.SnapshotFilter
Compact bool Compact bool
// Grouping // Grouping
@ -81,7 +81,7 @@ func init() {
f.VarP(&forgetOptions.WithinYearly, "keep-within-yearly", "", "keep yearly snapshots that are newer than `duration` (eg. 1y5m7d2h) relative to the latest snapshot") f.VarP(&forgetOptions.WithinYearly, "keep-within-yearly", "", "keep yearly snapshots that are newer than `duration` (eg. 1y5m7d2h) relative to the latest snapshot")
f.Var(&forgetOptions.KeepTags, "keep-tag", "keep snapshots with this `taglist` (can be specified multiple times)") f.Var(&forgetOptions.KeepTags, "keep-tag", "keep snapshots with this `taglist` (can be specified multiple times)")
initMultiSnapshotFilterOptions(f, &forgetOptions.snapshotFilterOptions, false) initMultiSnapshotFilter(f, &forgetOptions.SnapshotFilter, false)
f.StringArrayVar(&forgetOptions.Hosts, "hostname", nil, "only consider snapshots with the given `hostname` (can be specified multiple times)") f.StringArrayVar(&forgetOptions.Hosts, "hostname", nil, "only consider snapshots with the given `hostname` (can be specified multiple times)")
err := f.MarkDeprecated("hostname", "use --host") err := f.MarkDeprecated("hostname", "use --host")
if err != nil { if err != nil {
@ -126,7 +126,7 @@ func runForget(ctx context.Context, opts ForgetOptions, gopts GlobalOptions, arg
var snapshots restic.Snapshots var snapshots restic.Snapshots
removeSnIDs := restic.NewIDSet() removeSnIDs := restic.NewIDSet()
for sn := range FindFilteredSnapshots(ctx, repo.Backend(), repo, opts.Hosts, opts.Tags, opts.Paths, args) { for sn := range FindFilteredSnapshots(ctx, repo.Backend(), repo, &opts.SnapshotFilter, args) {
snapshots = append(snapshots, sn) snapshots = append(snapshots, sn)
} }

View File

@ -49,7 +49,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
// LsOptions collects all options for the ls command. // LsOptions collects all options for the ls command.
type LsOptions struct { type LsOptions struct {
ListLong bool ListLong bool
snapshotFilterOptions restic.SnapshotFilter
Recursive bool Recursive bool
} }
@ -59,7 +59,7 @@ func init() {
cmdRoot.AddCommand(cmdLs) cmdRoot.AddCommand(cmdLs)
flags := cmdLs.Flags() flags := cmdLs.Flags()
initSingleSnapshotFilterOptions(flags, &lsOptions.snapshotFilterOptions) initSingleSnapshotFilter(flags, &lsOptions.SnapshotFilter)
flags.BoolVarP(&lsOptions.ListLong, "long", "l", false, "use a long listing format showing size and mode") flags.BoolVarP(&lsOptions.ListLong, "long", "l", false, "use a long listing format showing size and mode")
flags.BoolVar(&lsOptions.Recursive, "recursive", false, "include files in subfolders of the listed directories") flags.BoolVar(&lsOptions.Recursive, "recursive", false, "include files in subfolders of the listed directories")
} }
@ -210,7 +210,11 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri
} }
} }
sn, err := restic.FindFilteredSnapshot(ctx, snapshotLister, repo, opts.Hosts, opts.Tags, opts.Paths, nil, args[0]) sn, err := (&restic.SnapshotFilter{
Hosts: opts.Hosts,
Paths: opts.Paths,
Tags: opts.Tags,
}).FindLatest(ctx, snapshotLister, repo, args[0])
if err != nil { if err != nil {
return err return err
} }

View File

@ -77,7 +77,7 @@ type MountOptions struct {
OwnerRoot bool OwnerRoot bool
AllowOther bool AllowOther bool
NoDefaultPermissions bool NoDefaultPermissions bool
snapshotFilterOptions restic.SnapshotFilter
TimeTemplate string TimeTemplate string
PathTemplates []string PathTemplates []string
} }
@ -92,7 +92,7 @@ func init() {
mountFlags.BoolVar(&mountOptions.AllowOther, "allow-other", false, "allow other users to access the data in the mounted directory") mountFlags.BoolVar(&mountOptions.AllowOther, "allow-other", false, "allow other users to access the data in the mounted directory")
mountFlags.BoolVar(&mountOptions.NoDefaultPermissions, "no-default-permissions", false, "for 'allow-other', ignore Unix permissions and allow users to read all snapshot files") mountFlags.BoolVar(&mountOptions.NoDefaultPermissions, "no-default-permissions", false, "for 'allow-other', ignore Unix permissions and allow users to read all snapshot files")
initMultiSnapshotFilterOptions(mountFlags, &mountOptions.snapshotFilterOptions, true) initMultiSnapshotFilter(mountFlags, &mountOptions.SnapshotFilter, true)
mountFlags.StringArrayVar(&mountOptions.PathTemplates, "path-template", nil, "set `template` for path names (can be specified multiple times)") mountFlags.StringArrayVar(&mountOptions.PathTemplates, "path-template", nil, "set `template` for path names (can be specified multiple times)")
mountFlags.StringVar(&mountOptions.TimeTemplate, "snapshot-template", time.RFC3339, "set `template` to use for snapshot dirs") mountFlags.StringVar(&mountOptions.TimeTemplate, "snapshot-template", time.RFC3339, "set `template` to use for snapshot dirs")
@ -180,9 +180,7 @@ func runMount(ctx context.Context, opts MountOptions, gopts GlobalOptions, args
cfg := fuse.Config{ cfg := fuse.Config{
OwnerIsRoot: opts.OwnerRoot, OwnerIsRoot: opts.OwnerRoot,
Hosts: opts.Hosts, Filter: opts.SnapshotFilter,
Tags: opts.Tags,
Paths: opts.Paths,
TimeTemplate: opts.TimeTemplate, TimeTemplate: opts.TimeTemplate,
PathTemplates: opts.PathTemplates, PathTemplates: opts.PathTemplates,
} }

View File

@ -42,7 +42,7 @@ type RestoreOptions struct {
Include []string Include []string
InsensitiveInclude []string InsensitiveInclude []string
Target string Target string
snapshotFilterOptions restic.SnapshotFilter
Sparse bool Sparse bool
Verify bool Verify bool
} }
@ -59,7 +59,7 @@ func init() {
flags.StringArrayVar(&restoreOptions.InsensitiveInclude, "iinclude", nil, "same as `--include` but ignores the casing of filenames") flags.StringArrayVar(&restoreOptions.InsensitiveInclude, "iinclude", nil, "same as `--include` but ignores the casing of filenames")
flags.StringVarP(&restoreOptions.Target, "target", "t", "", "directory to extract data to") flags.StringVarP(&restoreOptions.Target, "target", "t", "", "directory to extract data to")
initSingleSnapshotFilterOptions(flags, &restoreOptions.snapshotFilterOptions) initSingleSnapshotFilter(flags, &restoreOptions.SnapshotFilter)
flags.BoolVar(&restoreOptions.Sparse, "sparse", false, "restore files as sparse") flags.BoolVar(&restoreOptions.Sparse, "sparse", false, "restore files as sparse")
flags.BoolVar(&restoreOptions.Verify, "verify", false, "verify restored files content") flags.BoolVar(&restoreOptions.Verify, "verify", false, "verify restored files content")
} }
@ -131,7 +131,11 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions, a
} }
} }
sn, err := restic.FindFilteredSnapshot(ctx, repo.Backend(), repo, opts.Hosts, opts.Tags, opts.Paths, nil, snapshotIDString) sn, err := (&restic.SnapshotFilter{
Hosts: opts.Hosts,
Paths: opts.Paths,
Tags: opts.Tags,
}).FindLatest(ctx, repo.Backend(), repo, snapshotIDString)
if err != nil { if err != nil {
return errors.Fatalf("failed to find snapshot: %v", err) return errors.Fatalf("failed to find snapshot: %v", err)
} }

View File

@ -51,7 +51,7 @@ type RewriteOptions struct {
Forget bool Forget bool
DryRun bool DryRun bool
snapshotFilterOptions restic.SnapshotFilter
excludePatternOptions excludePatternOptions
} }
@ -64,7 +64,7 @@ func init() {
f.BoolVarP(&rewriteOptions.Forget, "forget", "", false, "remove original snapshots after creating new ones") f.BoolVarP(&rewriteOptions.Forget, "forget", "", false, "remove original snapshots after creating new ones")
f.BoolVarP(&rewriteOptions.DryRun, "dry-run", "n", false, "do not do anything, just print what would be done") f.BoolVarP(&rewriteOptions.DryRun, "dry-run", "n", false, "do not do anything, just print what would be done")
initMultiSnapshotFilterOptions(f, &rewriteOptions.snapshotFilterOptions, true) initMultiSnapshotFilter(f, &rewriteOptions.SnapshotFilter, true)
initExcludePatternOptions(f, &rewriteOptions.excludePatternOptions) initExcludePatternOptions(f, &rewriteOptions.excludePatternOptions)
} }
@ -186,7 +186,7 @@ func runRewrite(ctx context.Context, opts RewriteOptions, gopts GlobalOptions, a
} }
changedCount := 0 changedCount := 0
for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, opts.Hosts, opts.Tags, opts.Paths, args) { for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, &opts.SnapshotFilter, args) {
Verbosef("\nsnapshot %s of %v at %s)\n", sn.ID().Str(), sn.Paths, sn.Time) Verbosef("\nsnapshot %s of %v at %s)\n", sn.ID().Str(), sn.Paths, sn.Time)
changed, err := rewriteSnapshot(ctx, repo, sn, opts) changed, err := rewriteSnapshot(ctx, repo, sn, opts)
if err != nil { if err != nil {

View File

@ -32,7 +32,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
// SnapshotOptions bundles all options for the snapshots command. // SnapshotOptions bundles all options for the snapshots command.
type SnapshotOptions struct { type SnapshotOptions struct {
snapshotFilterOptions restic.SnapshotFilter
Compact bool Compact bool
Last bool // This option should be removed in favour of Latest. Last bool // This option should be removed in favour of Latest.
Latest int Latest int
@ -45,7 +45,7 @@ func init() {
cmdRoot.AddCommand(cmdSnapshots) cmdRoot.AddCommand(cmdSnapshots)
f := cmdSnapshots.Flags() f := cmdSnapshots.Flags()
initMultiSnapshotFilterOptions(f, &snapshotOptions.snapshotFilterOptions, true) initMultiSnapshotFilter(f, &snapshotOptions.SnapshotFilter, true)
f.BoolVarP(&snapshotOptions.Compact, "compact", "c", false, "use compact output format") f.BoolVarP(&snapshotOptions.Compact, "compact", "c", false, "use compact output format")
f.BoolVar(&snapshotOptions.Last, "last", false, "only show the last snapshot for each host and path") f.BoolVar(&snapshotOptions.Last, "last", false, "only show the last snapshot for each host and path")
err := f.MarkDeprecated("last", "use --latest 1") err := f.MarkDeprecated("last", "use --latest 1")
@ -73,7 +73,7 @@ func runSnapshots(ctx context.Context, opts SnapshotOptions, gopts GlobalOptions
} }
var snapshots restic.Snapshots var snapshots restic.Snapshots
for sn := range FindFilteredSnapshots(ctx, repo.Backend(), repo, opts.Hosts, opts.Tags, opts.Paths, args) { for sn := range FindFilteredSnapshots(ctx, repo.Backend(), repo, &opts.SnapshotFilter, args) {
snapshots = append(snapshots, sn) snapshots = append(snapshots, sn)
} }
snapshotGroups, grouped, err := restic.GroupSnapshots(snapshots, opts.GroupBy) snapshotGroups, grouped, err := restic.GroupSnapshots(snapshots, opts.GroupBy)

View File

@ -58,7 +58,7 @@ type StatsOptions struct {
// the mode of counting to perform (see consts for available modes) // the mode of counting to perform (see consts for available modes)
countMode string countMode string
snapshotFilterOptions restic.SnapshotFilter
} }
var statsOptions StatsOptions var statsOptions StatsOptions
@ -67,7 +67,7 @@ func init() {
cmdRoot.AddCommand(cmdStats) cmdRoot.AddCommand(cmdStats)
f := cmdStats.Flags() f := cmdStats.Flags()
f.StringVar(&statsOptions.countMode, "mode", countModeRestoreSize, "counting mode: restore-size (default), files-by-contents, blobs-per-file or raw-data") f.StringVar(&statsOptions.countMode, "mode", countModeRestoreSize, "counting mode: restore-size (default), files-by-contents, blobs-per-file or raw-data")
initMultiSnapshotFilterOptions(f, &statsOptions.snapshotFilterOptions, true) initMultiSnapshotFilter(f, &statsOptions.SnapshotFilter, true)
} }
func runStats(ctx context.Context, gopts GlobalOptions, args []string) error { func runStats(ctx context.Context, gopts GlobalOptions, args []string) error {
@ -111,7 +111,7 @@ func runStats(ctx context.Context, gopts GlobalOptions, args []string) error {
SnapshotsCount: 0, SnapshotsCount: 0,
} }
for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, statsOptions.Hosts, statsOptions.Tags, statsOptions.Paths, args) { for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, &statsOptions.SnapshotFilter, args) {
err = statsWalkSnapshot(ctx, sn, repo, stats) err = statsWalkSnapshot(ctx, sn, repo, stats)
if err != nil { if err != nil {
return fmt.Errorf("error walking snapshot: %v", err) return fmt.Errorf("error walking snapshot: %v", err)

View File

@ -35,7 +35,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
// TagOptions bundles all options for the 'tag' command. // TagOptions bundles all options for the 'tag' command.
type TagOptions struct { type TagOptions struct {
snapshotFilterOptions restic.SnapshotFilter
SetTags restic.TagLists SetTags restic.TagLists
AddTags restic.TagLists AddTags restic.TagLists
RemoveTags restic.TagLists RemoveTags restic.TagLists
@ -50,7 +50,7 @@ func init() {
tagFlags.Var(&tagOptions.SetTags, "set", "`tags` which will replace the existing tags in the format `tag[,tag,...]` (can be given multiple times)") tagFlags.Var(&tagOptions.SetTags, "set", "`tags` which will replace the existing tags in the format `tag[,tag,...]` (can be given multiple times)")
tagFlags.Var(&tagOptions.AddTags, "add", "`tags` which will be added to the existing tags in the format `tag[,tag,...]` (can be given multiple times)") tagFlags.Var(&tagOptions.AddTags, "add", "`tags` which will be added to the existing tags in the format `tag[,tag,...]` (can be given multiple times)")
tagFlags.Var(&tagOptions.RemoveTags, "remove", "`tags` which will be removed from the existing tags in the format `tag[,tag,...]` (can be given multiple times)") tagFlags.Var(&tagOptions.RemoveTags, "remove", "`tags` which will be removed from the existing tags in the format `tag[,tag,...]` (can be given multiple times)")
initMultiSnapshotFilterOptions(tagFlags, &tagOptions.snapshotFilterOptions, true) initMultiSnapshotFilter(tagFlags, &tagOptions.SnapshotFilter, true)
} }
func changeTags(ctx context.Context, repo *repository.Repository, sn *restic.Snapshot, setTags, addTags, removeTags []string) (bool, error) { func changeTags(ctx context.Context, repo *repository.Repository, sn *restic.Snapshot, setTags, addTags, removeTags []string) (bool, error) {
@ -119,7 +119,7 @@ func runTag(ctx context.Context, opts TagOptions, gopts GlobalOptions, args []st
} }
changeCnt := 0 changeCnt := 0
for sn := range FindFilteredSnapshots(ctx, repo.Backend(), repo, opts.Hosts, opts.Tags, opts.Paths, args) { for sn := range FindFilteredSnapshots(ctx, repo.Backend(), repo, &opts.SnapshotFilter, args) {
changed, err := changeTags(ctx, repo, sn, opts.SetTags.Flatten(), opts.AddTags.Flatten(), opts.RemoveTags.Flatten()) changed, err := changeTags(ctx, repo, sn, opts.SetTags.Flatten(), opts.AddTags.Flatten(), opts.RemoveTags.Flatten())
if err != nil { if err != nil {
Warnf("unable to modify the tags for snapshot ID %q, ignoring: %v\n", sn.ID(), err) Warnf("unable to modify the tags for snapshot ID %q, ignoring: %v\n", sn.ID(), err)

View File

@ -8,34 +8,28 @@ import (
"github.com/spf13/pflag" "github.com/spf13/pflag"
) )
type snapshotFilterOptions struct { // initMultiSnapshotFilter is used for commands that work on multiple snapshots
Hosts []string
Tags restic.TagLists
Paths []string
}
// initMultiSnapshotFilterOptions is used for commands that work on multiple snapshots
// MUST be combined with restic.FindFilteredSnapshots or FindFilteredSnapshots // MUST be combined with restic.FindFilteredSnapshots or FindFilteredSnapshots
func initMultiSnapshotFilterOptions(flags *pflag.FlagSet, options *snapshotFilterOptions, addHostShorthand bool) { func initMultiSnapshotFilter(flags *pflag.FlagSet, filt *restic.SnapshotFilter, addHostShorthand bool) {
hostShorthand := "H" hostShorthand := "H"
if !addHostShorthand { if !addHostShorthand {
hostShorthand = "" hostShorthand = ""
} }
flags.StringArrayVarP(&options.Hosts, "host", hostShorthand, nil, "only consider snapshots for this `host` (can be specified multiple times)") flags.StringArrayVarP(&filt.Hosts, "host", hostShorthand, nil, "only consider snapshots for this `host` (can be specified multiple times)")
flags.Var(&options.Tags, "tag", "only consider snapshots including `tag[,tag,...]` (can be specified multiple times)") flags.Var(&filt.Tags, "tag", "only consider snapshots including `tag[,tag,...]` (can be specified multiple times)")
flags.StringArrayVar(&options.Paths, "path", nil, "only consider snapshots including this (absolute) `path` (can be specified multiple times)") flags.StringArrayVar(&filt.Paths, "path", nil, "only consider snapshots including this (absolute) `path` (can be specified multiple times)")
} }
// initSingleSnapshotFilterOptions is used for commands that work on a single snapshot // initSingleSnapshotFilter is used for commands that work on a single snapshot
// MUST be combined with restic.FindFilteredSnapshot // MUST be combined with restic.FindFilteredSnapshot
func initSingleSnapshotFilterOptions(flags *pflag.FlagSet, options *snapshotFilterOptions) { func initSingleSnapshotFilter(flags *pflag.FlagSet, filt *restic.SnapshotFilter) {
flags.StringArrayVarP(&options.Hosts, "host", "H", nil, "only consider snapshots for this `host`, when snapshot ID \"latest\" is given (can be specified multiple times)") flags.StringArrayVarP(&filt.Hosts, "host", "H", nil, "only consider snapshots for this `host`, when snapshot ID \"latest\" is given (can be specified multiple times)")
flags.Var(&options.Tags, "tag", "only consider snapshots including `tag[,tag,...]`, when snapshot ID \"latest\" is given (can be specified multiple times)") flags.Var(&filt.Tags, "tag", "only consider snapshots including `tag[,tag,...]`, when snapshot ID \"latest\" is given (can be specified multiple times)")
flags.StringArrayVar(&options.Paths, "path", nil, "only consider snapshots including this (absolute) `path`, when snapshot ID \"latest\" is given (can be specified multiple times)") flags.StringArrayVar(&filt.Paths, "path", nil, "only consider snapshots including this (absolute) `path`, when snapshot ID \"latest\" is given (can be specified multiple times)")
} }
// FindFilteredSnapshots yields Snapshots, either given explicitly by `snapshotIDs` or filtered from the list of all snapshots. // FindFilteredSnapshots yields Snapshots, either given explicitly by `snapshotIDs` or filtered from the list of all snapshots.
func FindFilteredSnapshots(ctx context.Context, be restic.Lister, loader restic.LoaderUnpacked, hosts []string, tags []restic.TagList, paths []string, snapshotIDs []string) <-chan *restic.Snapshot { func FindFilteredSnapshots(ctx context.Context, be restic.Lister, loader restic.LoaderUnpacked, f *restic.SnapshotFilter, snapshotIDs []string) <-chan *restic.Snapshot {
out := make(chan *restic.Snapshot) out := make(chan *restic.Snapshot)
go func() { go func() {
defer close(out) defer close(out)
@ -45,7 +39,7 @@ func FindFilteredSnapshots(ctx context.Context, be restic.Lister, loader restic.
return return
} }
err = restic.FindFilteredSnapshots(ctx, be, loader, hosts, tags, paths, snapshotIDs, func(id string, sn *restic.Snapshot, err error) error { err = f.FindAll(ctx, be, loader, snapshotIDs, func(id string, sn *restic.Snapshot, err error) error {
if err != nil { if err != nil {
Warnf("Ignoring %q: %v\n", id, err) Warnf("Ignoring %q: %v\n", id, err)
} else { } else {

View File

@ -106,7 +106,7 @@ func testRunRestore(t testing.TB, opts GlobalOptions, dir string, snapshotID res
func testRunRestoreLatest(t testing.TB, gopts GlobalOptions, dir string, paths []string, hosts []string) { func testRunRestoreLatest(t testing.TB, gopts GlobalOptions, dir string, paths []string, hosts []string) {
opts := RestoreOptions{ opts := RestoreOptions{
Target: dir, Target: dir,
snapshotFilterOptions: snapshotFilterOptions{ SnapshotFilter: restic.SnapshotFilter{
Hosts: hosts, Hosts: hosts,
Paths: paths, Paths: paths,
}, },
@ -2196,7 +2196,7 @@ func TestFindListOnce(t *testing.T) {
snapshotIDs := restic.NewIDSet() snapshotIDs := restic.NewIDSet()
// specify the two oldest snapshots explicitly and use "latest" to reference the newest one // specify the two oldest snapshots explicitly and use "latest" to reference the newest one
for sn := range FindFilteredSnapshots(context.TODO(), repo.Backend(), repo, nil, nil, nil, []string{ for sn := range FindFilteredSnapshots(context.TODO(), repo.Backend(), repo, &restic.SnapshotFilter{}, []string{
secondSnapshot[0].String(), secondSnapshot[0].String(),
secondSnapshot[1].String()[:8], secondSnapshot[1].String()[:8],
"latest", "latest",

View File

@ -16,9 +16,7 @@ import (
// Config holds settings for the fuse mount. // Config holds settings for the fuse mount.
type Config struct { type Config struct {
OwnerIsRoot bool OwnerIsRoot bool
Hosts []string Filter restic.SnapshotFilter
Tags []restic.TagList
Paths []string
TimeTemplate string TimeTemplate string
PathTemplates []string PathTemplates []string
} }

View File

@ -295,7 +295,7 @@ func (d *SnapshotsDirStructure) updateSnapshots(ctx context.Context) error {
} }
var snapshots restic.Snapshots var snapshots restic.Snapshots
err := restic.FindFilteredSnapshots(ctx, d.root.repo.Backend(), d.root.repo, d.root.cfg.Hosts, d.root.cfg.Tags, d.root.cfg.Paths, nil, func(id string, sn *restic.Snapshot, err error) error { err := d.root.cfg.Filter.FindAll(ctx, d.root.repo.Backend(), d.root.repo, nil, func(id string, sn *restic.Snapshot, err error) error {
if sn != nil { if sn != nil {
snapshots = append(snapshots, sn) snapshots = append(snapshots, sn)
} }

View File

@ -12,13 +12,32 @@ import (
// ErrNoSnapshotFound is returned when no snapshot for the given criteria could be found. // ErrNoSnapshotFound is returned when no snapshot for the given criteria could be found.
var ErrNoSnapshotFound = errors.New("no snapshot found") var ErrNoSnapshotFound = errors.New("no snapshot found")
// findLatestSnapshot finds latest snapshot with optional target/directory, tags, hostname, and timestamp filters. // A SnapshotFilter denotes a set of snapshots based on hosts, tags and paths.
func findLatestSnapshot(ctx context.Context, be Lister, loader LoaderUnpacked, hosts []string, type SnapshotFilter struct {
tags []TagList, paths []string, timeStampLimit *time.Time) (*Snapshot, error) { _ struct{} // Force naming fields in literals.
Hosts []string
Tags TagLists
Paths []string
// Match snapshots from before this timestamp. Zero for no limit.
TimestampLimit time.Time
}
func (f *SnapshotFilter) empty() bool {
return len(f.Hosts)+len(f.Tags)+len(f.Paths) == 0
}
func (f *SnapshotFilter) matches(sn *Snapshot) bool {
return sn.HasHostname(f.Hosts) && sn.HasTagList(f.Tags) && sn.HasPaths(f.Paths)
}
// findLatest finds the latest snapshot with optional target/directory,
// tags, hostname, and timestamp filters.
func (f *SnapshotFilter) findLatest(ctx context.Context, be Lister, loader LoaderUnpacked) (*Snapshot, error) {
var err error var err error
absTargets := make([]string, 0, len(paths)) absTargets := make([]string, 0, len(f.Paths))
for _, target := range paths { for _, target := range f.Paths {
if !filepath.IsAbs(target) { if !filepath.IsAbs(target) {
target, err = filepath.Abs(target) target, err = filepath.Abs(target)
if err != nil { if err != nil {
@ -35,7 +54,7 @@ func findLatestSnapshot(ctx context.Context, be Lister, loader LoaderUnpacked, h
return errors.Errorf("Error loading snapshot %v: %v", id.Str(), err) return errors.Errorf("Error loading snapshot %v: %v", id.Str(), err)
} }
if timeStampLimit != nil && snapshot.Time.After(*timeStampLimit) { if !f.TimestampLimit.IsZero() && snapshot.Time.After(f.TimestampLimit) {
return nil return nil
} }
@ -43,15 +62,7 @@ func findLatestSnapshot(ctx context.Context, be Lister, loader LoaderUnpacked, h
return nil return nil
} }
if !snapshot.HasHostname(hosts) { if !f.matches(snapshot) {
return nil
}
if !snapshot.HasTagList(tags) {
return nil
}
if !snapshot.HasPaths(absTargets) {
return nil return nil
} }
@ -85,12 +96,14 @@ func FindSnapshot(ctx context.Context, be Lister, loader LoaderUnpacked, s strin
return LoadSnapshot(ctx, loader, id) return LoadSnapshot(ctx, loader, id)
} }
// FindFilteredSnapshot returns either the latests from a filtered list of all snapshots or a snapshot specified by `snapshotID`. // FindLatest returns either the latest of a filtered list of all snapshots
func FindFilteredSnapshot(ctx context.Context, be Lister, loader LoaderUnpacked, hosts []string, tags []TagList, paths []string, timeStampLimit *time.Time, snapshotID string) (*Snapshot, error) { // or a snapshot specified by `snapshotID`.
func (f *SnapshotFilter) FindLatest(ctx context.Context, be Lister, loader LoaderUnpacked, snapshotID string) (*Snapshot, error) {
if snapshotID == "latest" { if snapshotID == "latest" {
sn, err := findLatestSnapshot(ctx, be, loader, hosts, tags, paths, timeStampLimit) sn, err := f.findLatest(ctx, be, loader)
if err == ErrNoSnapshotFound { if err == ErrNoSnapshotFound {
err = fmt.Errorf("snapshot filter (Paths:%v Tags:%v Hosts:%v): %w", paths, tags, hosts, err) err = fmt.Errorf("snapshot filter (Paths:%v Tags:%v Hosts:%v): %w",
f.Paths, f.Tags, f.Hosts, err)
} }
return sn, err return sn, err
} }
@ -99,8 +112,8 @@ func FindFilteredSnapshot(ctx context.Context, be Lister, loader LoaderUnpacked,
type SnapshotFindCb func(string, *Snapshot, error) error type SnapshotFindCb func(string, *Snapshot, error) error
// FindFilteredSnapshots yields Snapshots, either given explicitly by `snapshotIDs` or filtered from the list of all snapshots. // FindAll yields Snapshots, either given explicitly by `snapshotIDs` or filtered from the list of all snapshots.
func FindFilteredSnapshots(ctx context.Context, be Lister, loader LoaderUnpacked, hosts []string, tags []TagList, paths []string, snapshotIDs []string, fn SnapshotFindCb) error { func (f *SnapshotFilter) FindAll(ctx context.Context, be Lister, loader LoaderUnpacked, snapshotIDs []string, fn SnapshotFindCb) error {
if len(snapshotIDs) != 0 { if len(snapshotIDs) != 0 {
var err error var err error
usedFilter := false usedFilter := false
@ -116,9 +129,10 @@ func FindFilteredSnapshots(ctx context.Context, be Lister, loader LoaderUnpacked
usedFilter = true usedFilter = true
sn, err = findLatestSnapshot(ctx, be, loader, hosts, tags, paths, nil) sn, err = f.findLatest(ctx, be, loader)
if err == ErrNoSnapshotFound { if err == ErrNoSnapshotFound {
err = errors.Errorf("no snapshot matched given filter (Paths:%v Tags:%v Hosts:%v)", paths, tags, hosts) err = errors.Errorf("no snapshot matched given filter (Paths:%v Tags:%v Hosts:%v)",
f.Paths, f.Tags, f.Hosts)
} }
if sn != nil { if sn != nil {
ids.Insert(*sn.ID()) ids.Insert(*sn.ID())
@ -141,18 +155,14 @@ func FindFilteredSnapshots(ctx context.Context, be Lister, loader LoaderUnpacked
} }
// Give the user some indication their filters are not used. // Give the user some indication their filters are not used.
if !usedFilter && (len(hosts) != 0 || len(tags) != 0 || len(paths) != 0) { if !usedFilter && !f.empty() {
return fn("filters", nil, errors.Errorf("explicit snapshot ids are given")) return fn("filters", nil, errors.Errorf("explicit snapshot ids are given"))
} }
return nil return nil
} }
return ForAllSnapshots(ctx, be, loader, nil, func(id ID, sn *Snapshot, err error) error { return ForAllSnapshots(ctx, be, loader, nil, func(id ID, sn *Snapshot, err error) error {
if err != nil { if err == nil && !f.matches(sn) {
return fn(id.String(), sn, err)
}
if !sn.HasHostname(hosts) || !sn.HasTagList(tags) || !sn.HasPaths(paths) {
return nil return nil
} }

View File

@ -14,13 +14,14 @@ func TestFindLatestSnapshot(t *testing.T) {
restic.TestCreateSnapshot(t, repo, parseTimeUTC("2017-07-07 07:07:07"), 1, 0) restic.TestCreateSnapshot(t, repo, parseTimeUTC("2017-07-07 07:07:07"), 1, 0)
latestSnapshot := restic.TestCreateSnapshot(t, repo, parseTimeUTC("2019-09-09 09:09:09"), 1, 0) latestSnapshot := restic.TestCreateSnapshot(t, repo, parseTimeUTC("2019-09-09 09:09:09"), 1, 0)
sn, err := restic.FindFilteredSnapshot(context.TODO(), repo.Backend(), repo, []string{"foo"}, []restic.TagList{}, []string{}, nil, "latest") f := restic.SnapshotFilter{Hosts: []string{"foo"}}
sn, err := f.FindLatest(context.TODO(), repo.Backend(), repo, "latest")
if err != nil { if err != nil {
t.Fatalf("FindLatestSnapshot returned error: %v", err) t.Fatalf("FindLatest returned error: %v", err)
} }
if *sn.ID() != *latestSnapshot.ID() { if *sn.ID() != *latestSnapshot.ID() {
t.Errorf("FindLatestSnapshot returned wrong snapshot ID: %v", *sn.ID()) t.Errorf("FindLatest returned wrong snapshot ID: %v", *sn.ID())
} }
} }
@ -30,14 +31,15 @@ func TestFindLatestSnapshotWithMaxTimestamp(t *testing.T) {
desiredSnapshot := restic.TestCreateSnapshot(t, repo, parseTimeUTC("2017-07-07 07:07:07"), 1, 0) desiredSnapshot := restic.TestCreateSnapshot(t, repo, parseTimeUTC("2017-07-07 07:07:07"), 1, 0)
restic.TestCreateSnapshot(t, repo, parseTimeUTC("2019-09-09 09:09:09"), 1, 0) restic.TestCreateSnapshot(t, repo, parseTimeUTC("2019-09-09 09:09:09"), 1, 0)
maxTimestamp := parseTimeUTC("2018-08-08 08:08:08") sn, err := (&restic.SnapshotFilter{
Hosts: []string{"foo"},
sn, err := restic.FindFilteredSnapshot(context.TODO(), repo.Backend(), repo, []string{"foo"}, []restic.TagList{}, []string{}, &maxTimestamp, "latest") TimestampLimit: parseTimeUTC("2018-08-08 08:08:08"),
}).FindLatest(context.TODO(), repo.Backend(), repo, "latest")
if err != nil { if err != nil {
t.Fatalf("FindLatestSnapshot returned error: %v", err) t.Fatalf("FindLatest returned error: %v", err)
} }
if *sn.ID() != *desiredSnapshot.ID() { if *sn.ID() != *desiredSnapshot.ID() {
t.Errorf("FindLatestSnapshot returned wrong snapshot ID: %v", *sn.ID()) t.Errorf("FindLatest returned wrong snapshot ID: %v", *sn.ID())
} }
} }