From f1585af0f263e517f1cfbb468fabb8a63dc9bc76 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Tue, 27 Aug 2024 14:03:36 +0200 Subject: [PATCH] move include/exclude options to filter package --- cmd/restic/cmd_backup.go | 13 +- cmd/restic/cmd_restore.go | 13 +- cmd/restic/cmd_rewrite.go | 9 +- cmd/restic/cmd_rewrite_integration_test.go | 3 +- cmd/restic/exclude.go | 160 +---------------- cmd/restic/integration_filter_pattern_test.go | 25 +-- internal/archiver/exclude.go | 15 +- internal/filter/exclude.go | 162 ++++++++++++++++++ .../filter}/exclude_test.go | 6 +- {cmd/restic => internal/filter}/include.go | 37 ++-- .../filter}/include_test.go | 6 +- 11 files changed, 233 insertions(+), 216 deletions(-) create mode 100644 internal/filter/exclude.go rename {cmd/restic => internal/filter}/exclude_test.go (92%) rename {cmd/restic => internal/filter}/include.go (65%) rename {cmd/restic => internal/filter}/include_test.go (92%) diff --git a/cmd/restic/cmd_backup.go b/cmd/restic/cmd_backup.go index 8d72a27b0..43ef29ba2 100644 --- a/cmd/restic/cmd_backup.go +++ b/cmd/restic/cmd_backup.go @@ -20,6 +20,7 @@ import ( "github.com/restic/restic/internal/archiver" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/filter" "github.com/restic/restic/internal/fs" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" @@ -66,7 +67,7 @@ Exit status is 12 if the password is incorrect. // BackupOptions bundles all options for the backup command. type BackupOptions struct { - excludePatternOptions + filter.ExcludePatternOptions Parent string GroupBy restic.SnapshotGroupByOptions @@ -108,7 +109,7 @@ func init() { f.VarP(&backupOptions.GroupBy, "group-by", "g", "`group` snapshots by host, paths and/or tags, separated by comma (disable grouping with '')") f.BoolVarP(&backupOptions.Force, "force", "f", false, `force re-reading the source files/directories (overrides the "parent" flag)`) - backupOptions.excludePatternOptions.Add(f) + backupOptions.ExcludePatternOptions.Add(f) f.BoolVarP(&backupOptions.ExcludeOtherFS, "one-file-system", "x", false, "exclude other file systems, don't cross filesystem boundaries and subvolumes") f.StringArrayVar(&backupOptions.ExcludeIfPresent, "exclude-if-present", nil, "takes `filename[:header]`, exclude contents of directories containing filename (except filename itself) if header of that file is as provided (can be specified multiple times)") @@ -297,7 +298,7 @@ func (opts BackupOptions) Check(gopts GlobalOptions, args []string) error { // collectRejectByNameFuncs returns a list of all functions which may reject data // from being saved in a snapshot based on path only -func collectRejectByNameFuncs(opts BackupOptions, repo *repository.Repository) (fs []RejectByNameFunc, err error) { +func collectRejectByNameFuncs(opts BackupOptions, repo *repository.Repository) (fs []archiver.RejectByNameFunc, err error) { // exclude restic cache if repo.Cache != nil { f, err := rejectResticCache(repo) @@ -308,11 +309,13 @@ func collectRejectByNameFuncs(opts BackupOptions, repo *repository.Repository) ( fs = append(fs, f) } - fsPatterns, err := opts.excludePatternOptions.CollectPatterns() + fsPatterns, err := opts.ExcludePatternOptions.CollectPatterns(Warnf) if err != nil { return nil, err } - fs = append(fs, fsPatterns...) + for _, pat := range fsPatterns { + fs = append(fs, archiver.RejectByNameFunc(pat)) + } return fs, nil } diff --git a/cmd/restic/cmd_restore.go b/cmd/restic/cmd_restore.go index f20359dc0..82dd408a8 100644 --- a/cmd/restic/cmd_restore.go +++ b/cmd/restic/cmd_restore.go @@ -7,6 +7,7 @@ import ( "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/filter" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/restorer" "github.com/restic/restic/internal/ui" @@ -49,8 +50,8 @@ Exit status is 12 if the password is incorrect. // RestoreOptions collects all options for the restore command. type RestoreOptions struct { - excludePatternOptions - includePatternOptions + filter.ExcludePatternOptions + filter.IncludePatternOptions Target string restic.SnapshotFilter DryRun bool @@ -68,8 +69,8 @@ func init() { flags := cmdRestore.Flags() flags.StringVarP(&restoreOptions.Target, "target", "t", "", "directory to extract data to") - restoreOptions.excludePatternOptions.Add(flags) - restoreOptions.includePatternOptions.Add(flags) + restoreOptions.ExcludePatternOptions.Add(flags) + restoreOptions.IncludePatternOptions.Add(flags) initSingleSnapshotFilter(flags, &restoreOptions.SnapshotFilter) flags.BoolVar(&restoreOptions.DryRun, "dry-run", false, "do not write any data, just show what would be done") @@ -82,12 +83,12 @@ func init() { func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions, term *termstatus.Terminal, args []string) error { - excludePatternFns, err := opts.excludePatternOptions.CollectPatterns() + excludePatternFns, err := opts.ExcludePatternOptions.CollectPatterns(Warnf) if err != nil { return err } - includePatternFns, err := opts.includePatternOptions.CollectPatterns() + includePatternFns, err := opts.IncludePatternOptions.CollectPatterns(Warnf) if err != nil { return err } diff --git a/cmd/restic/cmd_rewrite.go b/cmd/restic/cmd_rewrite.go index fc9da5b60..a9f664110 100644 --- a/cmd/restic/cmd_rewrite.go +++ b/cmd/restic/cmd_rewrite.go @@ -10,6 +10,7 @@ import ( "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/filter" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/walker" @@ -88,7 +89,7 @@ type RewriteOptions struct { Metadata snapshotMetadataArgs restic.SnapshotFilter - excludePatternOptions + filter.ExcludePatternOptions } var rewriteOptions RewriteOptions @@ -103,7 +104,7 @@ func init() { f.StringVar(&rewriteOptions.Metadata.Time, "new-time", "", "replace time of the backup") initMultiSnapshotFilter(f, &rewriteOptions.SnapshotFilter, true) - rewriteOptions.excludePatternOptions.Add(f) + rewriteOptions.ExcludePatternOptions.Add(f) } type rewriteFilterFunc func(ctx context.Context, sn *restic.Snapshot) (restic.ID, error) @@ -113,7 +114,7 @@ func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *resti return false, errors.Errorf("snapshot %v has nil tree", sn.ID().Str()) } - rejectByNameFuncs, err := opts.excludePatternOptions.CollectPatterns() + rejectByNameFuncs, err := opts.ExcludePatternOptions.CollectPatterns(Warnf) if err != nil { return false, err } @@ -263,7 +264,7 @@ func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *r } func runRewrite(ctx context.Context, opts RewriteOptions, gopts GlobalOptions, args []string) error { - if opts.excludePatternOptions.Empty() && opts.Metadata.empty() { + if opts.ExcludePatternOptions.Empty() && opts.Metadata.empty() { return errors.Fatal("Nothing to do: no excludes provided and no new metadata provided") } diff --git a/cmd/restic/cmd_rewrite_integration_test.go b/cmd/restic/cmd_rewrite_integration_test.go index 781266184..6471d49ba 100644 --- a/cmd/restic/cmd_rewrite_integration_test.go +++ b/cmd/restic/cmd_rewrite_integration_test.go @@ -5,6 +5,7 @@ import ( "path/filepath" "testing" + "github.com/restic/restic/internal/filter" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" "github.com/restic/restic/internal/ui" @@ -12,7 +13,7 @@ import ( func testRunRewriteExclude(t testing.TB, gopts GlobalOptions, excludes []string, forget bool, metadata snapshotMetadataArgs) { opts := RewriteOptions{ - excludePatternOptions: excludePatternOptions{ + ExcludePatternOptions: filter.ExcludePatternOptions{ Excludes: excludes, }, Forget: forget, diff --git a/cmd/restic/exclude.go b/cmd/restic/exclude.go index a37f9c68e..99d1128a9 100644 --- a/cmd/restic/exclude.go +++ b/cmd/restic/exclude.go @@ -1,60 +1,16 @@ package main import ( - "bufio" - "bytes" - "fmt" - "os" - "strings" - + "github.com/restic/restic/internal/archiver" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/filter" "github.com/restic/restic/internal/fs" "github.com/restic/restic/internal/repository" - "github.com/restic/restic/internal/textfile" - "github.com/spf13/pflag" ) -// RejectByNameFunc is a function that takes a filename of a -// file that would be included in the backup. The function returns true if it -// should be excluded (rejected) from the backup. -type RejectByNameFunc func(path string) bool - -// rejectByPattern returns a RejectByNameFunc which rejects files that match -// one of the patterns. -func rejectByPattern(patterns []string) RejectByNameFunc { - parsedPatterns := filter.ParsePatterns(patterns) - return func(item string) bool { - matched, err := filter.List(parsedPatterns, item) - if err != nil { - Warnf("error for exclude pattern: %v", err) - } - - if matched { - debug.Log("path %q excluded by an exclude pattern", item) - return true - } - - return false - } -} - -// Same as `rejectByPattern` but case insensitive. -func rejectByInsensitivePattern(patterns []string) RejectByNameFunc { - for index, path := range patterns { - patterns[index] = strings.ToLower(path) - } - - rejFunc := rejectByPattern(patterns) - return func(item string) bool { - return rejFunc(strings.ToLower(item)) - } -} - // rejectResticCache returns a RejectByNameFunc that rejects the restic cache // directory (if set). -func rejectResticCache(repo *repository.Repository) (RejectByNameFunc, error) { +func rejectResticCache(repo *repository.Repository) (archiver.RejectByNameFunc, error) { if repo.Cache == nil { return func(string) bool { return false @@ -75,115 +31,3 @@ func rejectResticCache(repo *repository.Repository) (RejectByNameFunc, error) { return false }, nil } - -// readPatternsFromFiles reads all files and returns the list of -// patterns. For each line, leading and trailing white space is removed -// and comment lines are ignored. For each remaining pattern, environment -// variables are resolved. For adding a literal dollar sign ($), write $$ to -// the file. -func readPatternsFromFiles(files []string) ([]string, error) { - getenvOrDollar := func(s string) string { - if s == "$" { - return "$" - } - return os.Getenv(s) - } - - var patterns []string - for _, filename := range files { - err := func() (err error) { - data, err := textfile.Read(filename) - if err != nil { - return err - } - - scanner := bufio.NewScanner(bytes.NewReader(data)) - for scanner.Scan() { - line := strings.TrimSpace(scanner.Text()) - - // ignore empty lines - if line == "" { - continue - } - - // strip comments - if strings.HasPrefix(line, "#") { - continue - } - - line = os.Expand(line, getenvOrDollar) - patterns = append(patterns, line) - } - return scanner.Err() - }() - if err != nil { - return nil, fmt.Errorf("failed to read patterns from file %q: %w", filename, err) - } - } - return patterns, nil -} - -type excludePatternOptions struct { - Excludes []string - InsensitiveExcludes []string - ExcludeFiles []string - InsensitiveExcludeFiles []string -} - -func (opts *excludePatternOptions) Add(f *pflag.FlagSet) { - f.StringArrayVarP(&opts.Excludes, "exclude", "e", nil, "exclude a `pattern` (can be specified multiple times)") - f.StringArrayVar(&opts.InsensitiveExcludes, "iexclude", nil, "same as --exclude `pattern` but ignores the casing of filenames") - f.StringArrayVar(&opts.ExcludeFiles, "exclude-file", nil, "read exclude patterns from a `file` (can be specified multiple times)") - f.StringArrayVar(&opts.InsensitiveExcludeFiles, "iexclude-file", nil, "same as --exclude-file but ignores casing of `file`names in patterns") -} - -func (opts *excludePatternOptions) Empty() bool { - return len(opts.Excludes) == 0 && len(opts.InsensitiveExcludes) == 0 && len(opts.ExcludeFiles) == 0 && len(opts.InsensitiveExcludeFiles) == 0 -} - -func (opts excludePatternOptions) CollectPatterns() ([]RejectByNameFunc, error) { - var fs []RejectByNameFunc - // add patterns from file - if len(opts.ExcludeFiles) > 0 { - excludePatterns, err := readPatternsFromFiles(opts.ExcludeFiles) - if err != nil { - return nil, err - } - - if err := filter.ValidatePatterns(excludePatterns); err != nil { - return nil, errors.Fatalf("--exclude-file: %s", err) - } - - opts.Excludes = append(opts.Excludes, excludePatterns...) - } - - if len(opts.InsensitiveExcludeFiles) > 0 { - excludes, err := readPatternsFromFiles(opts.InsensitiveExcludeFiles) - if err != nil { - return nil, err - } - - if err := filter.ValidatePatterns(excludes); err != nil { - return nil, errors.Fatalf("--iexclude-file: %s", err) - } - - opts.InsensitiveExcludes = append(opts.InsensitiveExcludes, excludes...) - } - - if len(opts.InsensitiveExcludes) > 0 { - if err := filter.ValidatePatterns(opts.InsensitiveExcludes); err != nil { - return nil, errors.Fatalf("--iexclude: %s", err) - } - - fs = append(fs, rejectByInsensitivePattern(opts.InsensitiveExcludes)) - } - - if len(opts.Excludes) > 0 { - if err := filter.ValidatePatterns(opts.Excludes); err != nil { - return nil, errors.Fatalf("--exclude: %s", err) - } - - fs = append(fs, rejectByPattern(opts.Excludes)) - } - return fs, nil -} diff --git a/cmd/restic/integration_filter_pattern_test.go b/cmd/restic/integration_filter_pattern_test.go index dccbcc0a0..46badbe4f 100644 --- a/cmd/restic/integration_filter_pattern_test.go +++ b/cmd/restic/integration_filter_pattern_test.go @@ -5,6 +5,7 @@ import ( "path/filepath" "testing" + "github.com/restic/restic/internal/filter" rtest "github.com/restic/restic/internal/test" ) @@ -17,14 +18,14 @@ func TestBackupFailsWhenUsingInvalidPatterns(t *testing.T) { var err error // Test --exclude - err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{excludePatternOptions: excludePatternOptions{Excludes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts) + err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{ExcludePatternOptions: filter.ExcludePatternOptions{Excludes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts) rtest.Equals(t, `Fatal: --exclude: invalid pattern(s) provided: *[._]log[.-][0-9] !*[._]log[.-][0-9]`, err.Error()) // Test --iexclude - err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{excludePatternOptions: excludePatternOptions{InsensitiveExcludes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts) + err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{ExcludePatternOptions: filter.ExcludePatternOptions{InsensitiveExcludes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts) rtest.Equals(t, `Fatal: --iexclude: invalid pattern(s) provided: *[._]log[.-][0-9] @@ -47,14 +48,14 @@ func TestBackupFailsWhenUsingInvalidPatternsFromFile(t *testing.T) { var err error // Test --exclude-file: - err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{excludePatternOptions: excludePatternOptions{ExcludeFiles: []string{excludeFile}}}, env.gopts) + err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{ExcludePatternOptions: filter.ExcludePatternOptions{ExcludeFiles: []string{excludeFile}}}, env.gopts) rtest.Equals(t, `Fatal: --exclude-file: invalid pattern(s) provided: *[._]log[.-][0-9] !*[._]log[.-][0-9]`, err.Error()) // Test --iexclude-file - err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{excludePatternOptions: excludePatternOptions{InsensitiveExcludeFiles: []string{excludeFile}}}, env.gopts) + err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{ExcludePatternOptions: filter.ExcludePatternOptions{InsensitiveExcludeFiles: []string{excludeFile}}}, env.gopts) rtest.Equals(t, `Fatal: --iexclude-file: invalid pattern(s) provided: *[._]log[.-][0-9] @@ -70,28 +71,28 @@ func TestRestoreFailsWhenUsingInvalidPatterns(t *testing.T) { var err error // Test --exclude - err = testRunRestoreAssumeFailure("latest", RestoreOptions{excludePatternOptions: excludePatternOptions{Excludes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts) + err = testRunRestoreAssumeFailure("latest", RestoreOptions{ExcludePatternOptions: filter.ExcludePatternOptions{Excludes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts) rtest.Equals(t, `Fatal: --exclude: invalid pattern(s) provided: *[._]log[.-][0-9] !*[._]log[.-][0-9]`, err.Error()) // Test --iexclude - err = testRunRestoreAssumeFailure("latest", RestoreOptions{excludePatternOptions: excludePatternOptions{InsensitiveExcludes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts) + err = testRunRestoreAssumeFailure("latest", RestoreOptions{ExcludePatternOptions: filter.ExcludePatternOptions{InsensitiveExcludes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts) rtest.Equals(t, `Fatal: --iexclude: invalid pattern(s) provided: *[._]log[.-][0-9] !*[._]log[.-][0-9]`, err.Error()) // Test --include - err = testRunRestoreAssumeFailure("latest", RestoreOptions{includePatternOptions: includePatternOptions{Includes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts) + err = testRunRestoreAssumeFailure("latest", RestoreOptions{IncludePatternOptions: filter.IncludePatternOptions{Includes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts) rtest.Equals(t, `Fatal: --include: invalid pattern(s) provided: *[._]log[.-][0-9] !*[._]log[.-][0-9]`, err.Error()) // Test --iinclude - err = testRunRestoreAssumeFailure("latest", RestoreOptions{includePatternOptions: includePatternOptions{InsensitiveIncludes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts) + err = testRunRestoreAssumeFailure("latest", RestoreOptions{IncludePatternOptions: filter.IncludePatternOptions{InsensitiveIncludes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts) rtest.Equals(t, `Fatal: --iinclude: invalid pattern(s) provided: *[._]log[.-][0-9] @@ -111,22 +112,22 @@ func TestRestoreFailsWhenUsingInvalidPatternsFromFile(t *testing.T) { t.Fatalf("Could not write include file: %v", fileErr) } - err := testRunRestoreAssumeFailure("latest", RestoreOptions{includePatternOptions: includePatternOptions{IncludeFiles: []string{patternsFile}}}, env.gopts) + err := testRunRestoreAssumeFailure("latest", RestoreOptions{IncludePatternOptions: filter.IncludePatternOptions{IncludeFiles: []string{patternsFile}}}, env.gopts) rtest.Equals(t, `Fatal: --include-file: invalid pattern(s) provided: *[._]log[.-][0-9] !*[._]log[.-][0-9]`, err.Error()) - err = testRunRestoreAssumeFailure("latest", RestoreOptions{excludePatternOptions: excludePatternOptions{ExcludeFiles: []string{patternsFile}}}, env.gopts) + err = testRunRestoreAssumeFailure("latest", RestoreOptions{ExcludePatternOptions: filter.ExcludePatternOptions{ExcludeFiles: []string{patternsFile}}}, env.gopts) rtest.Equals(t, `Fatal: --exclude-file: invalid pattern(s) provided: *[._]log[.-][0-9] !*[._]log[.-][0-9]`, err.Error()) - err = testRunRestoreAssumeFailure("latest", RestoreOptions{includePatternOptions: includePatternOptions{InsensitiveIncludeFiles: []string{patternsFile}}}, env.gopts) + err = testRunRestoreAssumeFailure("latest", RestoreOptions{IncludePatternOptions: filter.IncludePatternOptions{InsensitiveIncludeFiles: []string{patternsFile}}}, env.gopts) rtest.Equals(t, `Fatal: --iinclude-file: invalid pattern(s) provided: *[._]log[.-][0-9] !*[._]log[.-][0-9]`, err.Error()) - err = testRunRestoreAssumeFailure("latest", RestoreOptions{excludePatternOptions: excludePatternOptions{InsensitiveExcludeFiles: []string{patternsFile}}}, env.gopts) + err = testRunRestoreAssumeFailure("latest", RestoreOptions{ExcludePatternOptions: filter.ExcludePatternOptions{InsensitiveExcludeFiles: []string{patternsFile}}}, env.gopts) rtest.Equals(t, `Fatal: --iexclude-file: invalid pattern(s) provided: *[._]log[.-][0-9] !*[._]log[.-][0-9]`, err.Error()) diff --git a/internal/archiver/exclude.go b/internal/archiver/exclude.go index f4444812c..62e4ea17e 100644 --- a/internal/archiver/exclude.go +++ b/internal/archiver/exclude.go @@ -14,6 +14,16 @@ import ( "github.com/restic/restic/internal/ui" ) +// RejectByNameFunc is a function that takes a filename of a +// file that would be included in the backup. The function returns true if it +// should be excluded (rejected) from the backup. +type RejectByNameFunc func(path string) bool + +// RejectFunc is a function that takes a filename and os.FileInfo of a +// file that would be included in the backup. The function returns true if it +// should be excluded (rejected) from the backup. +type RejectFunc func(path string, fi os.FileInfo, fs fs.FS) bool + type rejectionCache struct { m map[string]bool mtx sync.Mutex @@ -49,11 +59,6 @@ func (rc *rejectionCache) Store(dir string, rejected bool) { rc.m[dir] = rejected } -// RejectFunc is a function that takes a filename and os.FileInfo of a -// file that would be included in the backup. The function returns true if it -// should be excluded (rejected) from the backup. -type RejectFunc func(path string, fi os.FileInfo, fs fs.FS) bool - // RejectIfPresent returns a RejectByNameFunc which itself returns whether a path // should be excluded. The RejectByNameFunc considers a file to be excluded when // it resides in a directory with an exclusion file, that is specified by diff --git a/internal/filter/exclude.go b/internal/filter/exclude.go new file mode 100644 index 000000000..48ecdfddf --- /dev/null +++ b/internal/filter/exclude.go @@ -0,0 +1,162 @@ +package filter + +import ( + "bufio" + "bytes" + "fmt" + "os" + "strings" + + "github.com/restic/restic/internal/debug" + "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/textfile" + "github.com/spf13/pflag" +) + +// RejectByNameFunc is a function that takes a filename of a +// file that would be included in the backup. The function returns true if it +// should be excluded (rejected) from the backup. +type RejectByNameFunc func(path string) bool + +// RejectByPattern returns a RejectByNameFunc which rejects files that match +// one of the patterns. +func RejectByPattern(patterns []string, warnf func(msg string, args ...interface{})) RejectByNameFunc { + parsedPatterns := ParsePatterns(patterns) + return func(item string) bool { + matched, err := List(parsedPatterns, item) + if err != nil { + warnf("error for exclude pattern: %v", err) + } + + if matched { + debug.Log("path %q excluded by an exclude pattern", item) + return true + } + + return false + } +} + +// RejectByInsensitivePattern is like RejectByPattern but case insensitive. +func RejectByInsensitivePattern(patterns []string, warnf func(msg string, args ...interface{})) RejectByNameFunc { + for index, path := range patterns { + patterns[index] = strings.ToLower(path) + } + + rejFunc := RejectByPattern(patterns, warnf) + return func(item string) bool { + return rejFunc(strings.ToLower(item)) + } +} + +// readPatternsFromFiles reads all files and returns the list of +// patterns. For each line, leading and trailing white space is removed +// and comment lines are ignored. For each remaining pattern, environment +// variables are resolved. For adding a literal dollar sign ($), write $$ to +// the file. +func readPatternsFromFiles(files []string) ([]string, error) { + getenvOrDollar := func(s string) string { + if s == "$" { + return "$" + } + return os.Getenv(s) + } + + var patterns []string + for _, filename := range files { + err := func() (err error) { + data, err := textfile.Read(filename) + if err != nil { + return err + } + + scanner := bufio.NewScanner(bytes.NewReader(data)) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + + // ignore empty lines + if line == "" { + continue + } + + // strip comments + if strings.HasPrefix(line, "#") { + continue + } + + line = os.Expand(line, getenvOrDollar) + patterns = append(patterns, line) + } + return scanner.Err() + }() + if err != nil { + return nil, fmt.Errorf("failed to read patterns from file %q: %w", filename, err) + } + } + return patterns, nil +} + +type ExcludePatternOptions struct { + Excludes []string + InsensitiveExcludes []string + ExcludeFiles []string + InsensitiveExcludeFiles []string +} + +func (opts *ExcludePatternOptions) Add(f *pflag.FlagSet) { + f.StringArrayVarP(&opts.Excludes, "exclude", "e", nil, "exclude a `pattern` (can be specified multiple times)") + f.StringArrayVar(&opts.InsensitiveExcludes, "iexclude", nil, "same as --exclude `pattern` but ignores the casing of filenames") + f.StringArrayVar(&opts.ExcludeFiles, "exclude-file", nil, "read exclude patterns from a `file` (can be specified multiple times)") + f.StringArrayVar(&opts.InsensitiveExcludeFiles, "iexclude-file", nil, "same as --exclude-file but ignores casing of `file`names in patterns") +} + +func (opts *ExcludePatternOptions) Empty() bool { + return len(opts.Excludes) == 0 && len(opts.InsensitiveExcludes) == 0 && len(opts.ExcludeFiles) == 0 && len(opts.InsensitiveExcludeFiles) == 0 +} + +func (opts ExcludePatternOptions) CollectPatterns(warnf func(msg string, args ...interface{})) ([]RejectByNameFunc, error) { + var fs []RejectByNameFunc + // add patterns from file + if len(opts.ExcludeFiles) > 0 { + excludePatterns, err := readPatternsFromFiles(opts.ExcludeFiles) + if err != nil { + return nil, err + } + + if err := ValidatePatterns(excludePatterns); err != nil { + return nil, errors.Fatalf("--exclude-file: %s", err) + } + + opts.Excludes = append(opts.Excludes, excludePatterns...) + } + + if len(opts.InsensitiveExcludeFiles) > 0 { + excludes, err := readPatternsFromFiles(opts.InsensitiveExcludeFiles) + if err != nil { + return nil, err + } + + if err := ValidatePatterns(excludes); err != nil { + return nil, errors.Fatalf("--iexclude-file: %s", err) + } + + opts.InsensitiveExcludes = append(opts.InsensitiveExcludes, excludes...) + } + + if len(opts.InsensitiveExcludes) > 0 { + if err := ValidatePatterns(opts.InsensitiveExcludes); err != nil { + return nil, errors.Fatalf("--iexclude: %s", err) + } + + fs = append(fs, RejectByInsensitivePattern(opts.InsensitiveExcludes, warnf)) + } + + if len(opts.Excludes) > 0 { + if err := ValidatePatterns(opts.Excludes); err != nil { + return nil, errors.Fatalf("--exclude: %s", err) + } + + fs = append(fs, RejectByPattern(opts.Excludes, warnf)) + } + return fs, nil +} diff --git a/cmd/restic/exclude_test.go b/internal/filter/exclude_test.go similarity index 92% rename from cmd/restic/exclude_test.go rename to internal/filter/exclude_test.go index 177a81df2..738fb216d 100644 --- a/cmd/restic/exclude_test.go +++ b/internal/filter/exclude_test.go @@ -1,4 +1,4 @@ -package main +package filter import ( "testing" @@ -21,7 +21,7 @@ func TestRejectByPattern(t *testing.T) { for _, tc := range tests { t.Run("", func(t *testing.T) { - reject := rejectByPattern(patterns) + reject := RejectByPattern(patterns, nil) res := reject(tc.filename) if res != tc.reject { t.Fatalf("wrong result for filename %v: want %v, got %v", @@ -48,7 +48,7 @@ func TestRejectByInsensitivePattern(t *testing.T) { for _, tc := range tests { t.Run("", func(t *testing.T) { - reject := rejectByInsensitivePattern(patterns) + reject := RejectByInsensitivePattern(patterns, nil) res := reject(tc.filename) if res != tc.reject { t.Fatalf("wrong result for filename %v: want %v, got %v", diff --git a/cmd/restic/include.go b/internal/filter/include.go similarity index 65% rename from cmd/restic/include.go rename to internal/filter/include.go index 514a24016..87d5f1207 100644 --- a/cmd/restic/include.go +++ b/internal/filter/include.go @@ -1,10 +1,9 @@ -package main +package filter import ( "strings" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/filter" "github.com/spf13/pflag" ) @@ -12,21 +11,21 @@ import ( // in the restore process and returns whether it should be included. type IncludeByNameFunc func(item string) (matched bool, childMayMatch bool) -type includePatternOptions struct { +type IncludePatternOptions struct { Includes []string InsensitiveIncludes []string IncludeFiles []string InsensitiveIncludeFiles []string } -func (opts *includePatternOptions) Add(f *pflag.FlagSet) { +func (opts *IncludePatternOptions) Add(f *pflag.FlagSet) { f.StringArrayVarP(&opts.Includes, "include", "i", nil, "include a `pattern` (can be specified multiple times)") f.StringArrayVar(&opts.InsensitiveIncludes, "iinclude", nil, "same as --include `pattern` but ignores the casing of filenames") f.StringArrayVar(&opts.IncludeFiles, "include-file", nil, "read include patterns from a `file` (can be specified multiple times)") f.StringArrayVar(&opts.InsensitiveIncludeFiles, "iinclude-file", nil, "same as --include-file but ignores casing of `file`names in patterns") } -func (opts includePatternOptions) CollectPatterns() ([]IncludeByNameFunc, error) { +func (opts IncludePatternOptions) CollectPatterns(warnf func(msg string, args ...interface{})) ([]IncludeByNameFunc, error) { var fs []IncludeByNameFunc if len(opts.IncludeFiles) > 0 { includePatterns, err := readPatternsFromFiles(opts.IncludeFiles) @@ -34,7 +33,7 @@ func (opts includePatternOptions) CollectPatterns() ([]IncludeByNameFunc, error) return nil, err } - if err := filter.ValidatePatterns(includePatterns); err != nil { + if err := ValidatePatterns(includePatterns); err != nil { return nil, errors.Fatalf("--include-file: %s", err) } @@ -47,7 +46,7 @@ func (opts includePatternOptions) CollectPatterns() ([]IncludeByNameFunc, error) return nil, err } - if err := filter.ValidatePatterns(includePatterns); err != nil { + if err := ValidatePatterns(includePatterns); err != nil { return nil, errors.Fatalf("--iinclude-file: %s", err) } @@ -55,45 +54,45 @@ func (opts includePatternOptions) CollectPatterns() ([]IncludeByNameFunc, error) } if len(opts.InsensitiveIncludes) > 0 { - if err := filter.ValidatePatterns(opts.InsensitiveIncludes); err != nil { + if err := ValidatePatterns(opts.InsensitiveIncludes); err != nil { return nil, errors.Fatalf("--iinclude: %s", err) } - fs = append(fs, includeByInsensitivePattern(opts.InsensitiveIncludes)) + fs = append(fs, IncludeByInsensitivePattern(opts.InsensitiveIncludes, warnf)) } if len(opts.Includes) > 0 { - if err := filter.ValidatePatterns(opts.Includes); err != nil { + if err := ValidatePatterns(opts.Includes); err != nil { return nil, errors.Fatalf("--include: %s", err) } - fs = append(fs, includeByPattern(opts.Includes)) + fs = append(fs, IncludeByPattern(opts.Includes, warnf)) } return fs, nil } -// includeByPattern returns a IncludeByNameFunc which includes files that match +// IncludeByPattern returns a IncludeByNameFunc which includes files that match // one of the patterns. -func includeByPattern(patterns []string) IncludeByNameFunc { - parsedPatterns := filter.ParsePatterns(patterns) +func IncludeByPattern(patterns []string, warnf func(msg string, args ...interface{})) IncludeByNameFunc { + parsedPatterns := ParsePatterns(patterns) return func(item string) (matched bool, childMayMatch bool) { - matched, childMayMatch, err := filter.ListWithChild(parsedPatterns, item) + matched, childMayMatch, err := ListWithChild(parsedPatterns, item) if err != nil { - Warnf("error for include pattern: %v", err) + warnf("error for include pattern: %v", err) } return matched, childMayMatch } } -// includeByInsensitivePattern returns a IncludeByNameFunc which includes files that match +// IncludeByInsensitivePattern returns a IncludeByNameFunc which includes files that match // one of the patterns, ignoring the casing of the filenames. -func includeByInsensitivePattern(patterns []string) IncludeByNameFunc { +func IncludeByInsensitivePattern(patterns []string, warnf func(msg string, args ...interface{})) IncludeByNameFunc { for index, path := range patterns { patterns[index] = strings.ToLower(path) } - includeFunc := includeByPattern(patterns) + includeFunc := IncludeByPattern(patterns, warnf) return func(item string) (matched bool, childMayMatch bool) { return includeFunc(strings.ToLower(item)) } diff --git a/cmd/restic/include_test.go b/internal/filter/include_test.go similarity index 92% rename from cmd/restic/include_test.go rename to internal/filter/include_test.go index 751bfbb76..2f474622c 100644 --- a/cmd/restic/include_test.go +++ b/internal/filter/include_test.go @@ -1,4 +1,4 @@ -package main +package filter import ( "testing" @@ -21,7 +21,7 @@ func TestIncludeByPattern(t *testing.T) { for _, tc := range tests { t.Run(tc.filename, func(t *testing.T) { - includeFunc := includeByPattern(patterns) + includeFunc := IncludeByPattern(patterns, nil) matched, _ := includeFunc(tc.filename) if matched != tc.include { t.Fatalf("wrong result for filename %v: want %v, got %v", @@ -48,7 +48,7 @@ func TestIncludeByInsensitivePattern(t *testing.T) { for _, tc := range tests { t.Run(tc.filename, func(t *testing.T) { - includeFunc := includeByInsensitivePattern(patterns) + includeFunc := IncludeByInsensitivePattern(patterns, nil) matched, _ := includeFunc(tc.filename) if matched != tc.include { t.Fatalf("wrong result for filename %v: want %v, got %v",