octoleo/restic
2
2
Fork 0
mirror of https://github.com/octoleo/restic.git synced 2024-11-25 22:27:35 +00:00
Code Issues Releases Wiki Activity

move include/exclude options to filter package

Browse Source
This commit is contained in:
Michael Eischer 2024-08-27 14:03:36 +02:00
parent 5d58945718
commit f1585af0f2
11 changed files with 233 additions and 216 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
cmd/restic/cmd_backup.go
View File

@ -20,6 +20,7 @@ import (
"github.com/restic/restic/internal/archiver"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/filter"
"github.com/restic/restic/internal/fs"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
@ -66,7 +67,7 @@ Exit status is 12 if the password is incorrect.
// BackupOptions bundles all options for the backup command.
type BackupOptions struct {
excludePatternOptions
filter.ExcludePatternOptions
Parent string
GroupBy restic.SnapshotGroupByOptions
@ -108,7 +109,7 @@ func init() {
f.VarP(&backupOptions.GroupBy, "group-by", "g", "`group` snapshots by host, paths and/or tags, separated by comma (disable grouping with '')")
f.BoolVarP(&backupOptions.Force, "force", "f", false, `force re-reading the source files/directories (overrides the "parent" flag)`)
backupOptions.excludePatternOptions.Add(f)
backupOptions.ExcludePatternOptions.Add(f)
f.BoolVarP(&backupOptions.ExcludeOtherFS, "one-file-system", "x", false, "exclude other file systems, don't cross filesystem boundaries and subvolumes")
f.StringArrayVar(&backupOptions.ExcludeIfPresent, "exclude-if-present", nil, "takes `filename[:header]`, exclude contents of directories containing filename (except filename itself) if header of that file is as provided (can be specified multiple times)")
@ -297,7 +298,7 @@ func (opts BackupOptions) Check(gopts GlobalOptions, args []string) error {
// collectRejectByNameFuncs returns a list of all functions which may reject data
// from being saved in a snapshot based on path only
func collectRejectByNameFuncs(opts BackupOptions, repo *repository.Repository) (fs []RejectByNameFunc, err error) {
func collectRejectByNameFuncs(opts BackupOptions, repo *repository.Repository) (fs []archiver.RejectByNameFunc, err error) {
// exclude restic cache
if repo.Cache != nil {
f, err := rejectResticCache(repo)
@ -308,11 +309,13 @@ func collectRejectByNameFuncs(opts BackupOptions, repo *repository.Repository) (
fs = append(fs, f)
}
fsPatterns, err := opts.excludePatternOptions.CollectPatterns()
fsPatterns, err := opts.ExcludePatternOptions.CollectPatterns(Warnf)
if err != nil {
return nil, err
}
fs = append(fs, fsPatterns...)
for _, pat := range fsPatterns {
fs = append(fs, archiver.RejectByNameFunc(pat))
}
return fs, nil
}

13
cmd/restic/cmd_restore.go
View File

@ -7,6 +7,7 @@ import (
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/filter"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/restorer"
"github.com/restic/restic/internal/ui"
@ -49,8 +50,8 @@ Exit status is 12 if the password is incorrect.
// RestoreOptions collects all options for the restore command.
type RestoreOptions struct {
excludePatternOptions
includePatternOptions
filter.ExcludePatternOptions
filter.IncludePatternOptions
Target string
restic.SnapshotFilter
DryRun bool
@ -68,8 +69,8 @@ func init() {
flags := cmdRestore.Flags()
flags.StringVarP(&restoreOptions.Target, "target", "t", "", "directory to extract data to")
restoreOptions.excludePatternOptions.Add(flags)
restoreOptions.includePatternOptions.Add(flags)
restoreOptions.ExcludePatternOptions.Add(flags)
restoreOptions.IncludePatternOptions.Add(flags)
initSingleSnapshotFilter(flags, &restoreOptions.SnapshotFilter)
flags.BoolVar(&restoreOptions.DryRun, "dry-run", false, "do not write any data, just show what would be done")
@ -82,12 +83,12 @@ func init() {
func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions,
term *termstatus.Terminal, args []string) error {
excludePatternFns, err := opts.excludePatternOptions.CollectPatterns()
excludePatternFns, err := opts.ExcludePatternOptions.CollectPatterns(Warnf)
if err != nil {
return err
}
includePatternFns, err := opts.includePatternOptions.CollectPatterns()
includePatternFns, err := opts.IncludePatternOptions.CollectPatterns(Warnf)
if err != nil {
return err
}

9
cmd/restic/cmd_rewrite.go
View File

@ -10,6 +10,7 @@ import (
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/filter"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/walker"
@ -88,7 +89,7 @@ type RewriteOptions struct {
Metadata snapshotMetadataArgs
restic.SnapshotFilter
excludePatternOptions
filter.ExcludePatternOptions
}
var rewriteOptions RewriteOptions
@ -103,7 +104,7 @@ func init() {
f.StringVar(&rewriteOptions.Metadata.Time, "new-time", "", "replace time of the backup")
initMultiSnapshotFilter(f, &rewriteOptions.SnapshotFilter, true)
rewriteOptions.excludePatternOptions.Add(f)
rewriteOptions.ExcludePatternOptions.Add(f)
}
type rewriteFilterFunc func(ctx context.Context, sn *restic.Snapshot) (restic.ID, error)
@ -113,7 +114,7 @@ func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *resti
return false, errors.Errorf("snapshot %v has nil tree", sn.ID().Str())
}
rejectByNameFuncs, err := opts.excludePatternOptions.CollectPatterns()
rejectByNameFuncs, err := opts.ExcludePatternOptions.CollectPatterns(Warnf)
if err != nil {
return false, err
}
@ -263,7 +264,7 @@ func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *r
}
func runRewrite(ctx context.Context, opts RewriteOptions, gopts GlobalOptions, args []string) error {
if opts.excludePatternOptions.Empty() && opts.Metadata.empty() {
if opts.ExcludePatternOptions.Empty() && opts.Metadata.empty() {
return errors.Fatal("Nothing to do: no excludes provided and no new metadata provided")
}

3
cmd/restic/cmd_rewrite_integration_test.go
View File

@ -5,6 +5,7 @@ import (
"path/filepath"
"testing"
"github.com/restic/restic/internal/filter"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
"github.com/restic/restic/internal/ui"
@ -12,7 +13,7 @@ import (
func testRunRewriteExclude(t testing.TB, gopts GlobalOptions, excludes []string, forget bool, metadata snapshotMetadataArgs) {
opts := RewriteOptions{
excludePatternOptions: excludePatternOptions{
ExcludePatternOptions: filter.ExcludePatternOptions{
Excludes: excludes,
},
Forget: forget,

160
cmd/restic/exclude.go
View File

@ -1,60 +1,16 @@
package main
import (
"bufio"
"bytes"
"fmt"
"os"
"strings"
"github.com/restic/restic/internal/archiver"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/filter"
"github.com/restic/restic/internal/fs"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/textfile"
"github.com/spf13/pflag"
)
// RejectByNameFunc is a function that takes a filename of a
// file that would be included in the backup. The function returns true if it
// should be excluded (rejected) from the backup.
type RejectByNameFunc func(path string) bool
// rejectByPattern returns a RejectByNameFunc which rejects files that match
// one of the patterns.
func rejectByPattern(patterns []string) RejectByNameFunc {
parsedPatterns := filter.ParsePatterns(patterns)
return func(item string) bool {
matched, err := filter.List(parsedPatterns, item)
if err != nil {
Warnf("error for exclude pattern: %v", err)
}
if matched {
debug.Log("path %q excluded by an exclude pattern", item)
return true
}
return false
}
}
// Same as `rejectByPattern` but case insensitive.
func rejectByInsensitivePattern(patterns []string) RejectByNameFunc {
for index, path := range patterns {
patterns[index] = strings.ToLower(path)
}
rejFunc := rejectByPattern(patterns)
return func(item string) bool {
return rejFunc(strings.ToLower(item))
}
}
// rejectResticCache returns a RejectByNameFunc that rejects the restic cache
// directory (if set).
func rejectResticCache(repo *repository.Repository) (RejectByNameFunc, error) {
func rejectResticCache(repo *repository.Repository) (archiver.RejectByNameFunc, error) {
if repo.Cache == nil {
return func(string) bool {
return false
@ -75,115 +31,3 @@ func rejectResticCache(repo *repository.Repository) (RejectByNameFunc, error) {
return false
}, nil
}
// readPatternsFromFiles reads all files and returns the list of
// patterns. For each line, leading and trailing white space is removed
// and comment lines are ignored. For each remaining pattern, environment
// variables are resolved. For adding a literal dollar sign ($), write $$ to
// the file.
func readPatternsFromFiles(files []string) ([]string, error) {
getenvOrDollar := func(s string) string {
if s == "$" {
return "$"
}
return os.Getenv(s)
}
var patterns []string
for _, filename := range files {
err := func() (err error) {
data, err := textfile.Read(filename)
if err != nil {
return err
}
scanner := bufio.NewScanner(bytes.NewReader(data))
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
// ignore empty lines
if line == "" {
continue
}
// strip comments
if strings.HasPrefix(line, "#") {
continue
}
line = os.Expand(line, getenvOrDollar)
patterns = append(patterns, line)
}
return scanner.Err()
}()
if err != nil {
return nil, fmt.Errorf("failed to read patterns from file %q: %w", filename, err)
}
}
return patterns, nil
}
type excludePatternOptions struct {
Excludes []string
InsensitiveExcludes []string
ExcludeFiles []string
InsensitiveExcludeFiles []string
}
func (opts *excludePatternOptions) Add(f *pflag.FlagSet) {
f.StringArrayVarP(&opts.Excludes, "exclude", "e", nil, "exclude a `pattern` (can be specified multiple times)")
f.StringArrayVar(&opts.InsensitiveExcludes, "iexclude", nil, "same as --exclude `pattern` but ignores the casing of filenames")
f.StringArrayVar(&opts.ExcludeFiles, "exclude-file", nil, "read exclude patterns from a `file` (can be specified multiple times)")
f.StringArrayVar(&opts.InsensitiveExcludeFiles, "iexclude-file", nil, "same as --exclude-file but ignores casing of `file`names in patterns")
}
func (opts *excludePatternOptions) Empty() bool {
return len(opts.Excludes) == 0 && len(opts.InsensitiveExcludes) == 0 && len(opts.ExcludeFiles) == 0 && len(opts.InsensitiveExcludeFiles) == 0
}
func (opts excludePatternOptions) CollectPatterns() ([]RejectByNameFunc, error) {
var fs []RejectByNameFunc
// add patterns from file
if len(opts.ExcludeFiles) > 0 {
excludePatterns, err := readPatternsFromFiles(opts.ExcludeFiles)
if err != nil {
return nil, err
}
if err := filter.ValidatePatterns(excludePatterns); err != nil {
return nil, errors.Fatalf("--exclude-file: %s", err)
}
opts.Excludes = append(opts.Excludes, excludePatterns...)
}
if len(opts.InsensitiveExcludeFiles) > 0 {
excludes, err := readPatternsFromFiles(opts.InsensitiveExcludeFiles)
if err != nil {
return nil, err
}
if err := filter.ValidatePatterns(excludes); err != nil {
return nil, errors.Fatalf("--iexclude-file: %s", err)
}
opts.InsensitiveExcludes = append(opts.InsensitiveExcludes, excludes...)
}
if len(opts.InsensitiveExcludes) > 0 {
if err := filter.ValidatePatterns(opts.InsensitiveExcludes); err != nil {
return nil, errors.Fatalf("--iexclude: %s", err)
}
fs = append(fs, rejectByInsensitivePattern(opts.InsensitiveExcludes))
}
if len(opts.Excludes) > 0 {
if err := filter.ValidatePatterns(opts.Excludes); err != nil {
return nil, errors.Fatalf("--exclude: %s", err)
}
fs = append(fs, rejectByPattern(opts.Excludes))
}
return fs, nil
}

25
cmd/restic/integration_filter_pattern_test.go
View File

@ -5,6 +5,7 @@ import (
"path/filepath"
"testing"
"github.com/restic/restic/internal/filter"
rtest "github.com/restic/restic/internal/test"
)
@ -17,14 +18,14 @@ func TestBackupFailsWhenUsingInvalidPatterns(t *testing.T) {
var err error
// Test --exclude
err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{excludePatternOptions: excludePatternOptions{Excludes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts)
err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{ExcludePatternOptions: filter.ExcludePatternOptions{Excludes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts)
rtest.Equals(t, `Fatal: --exclude: invalid pattern(s) provided:
*[._]log[.-][0-9]
!*[._]log[.-][0-9]`, err.Error())
// Test --iexclude
err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{excludePatternOptions: excludePatternOptions{InsensitiveExcludes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts)
err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{ExcludePatternOptions: filter.ExcludePatternOptions{InsensitiveExcludes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts)
rtest.Equals(t, `Fatal: --iexclude: invalid pattern(s) provided:
*[._]log[.-][0-9]
@ -47,14 +48,14 @@ func TestBackupFailsWhenUsingInvalidPatternsFromFile(t *testing.T) {
var err error
// Test --exclude-file:
err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{excludePatternOptions: excludePatternOptions{ExcludeFiles: []string{excludeFile}}}, env.gopts)
err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{ExcludePatternOptions: filter.ExcludePatternOptions{ExcludeFiles: []string{excludeFile}}}, env.gopts)
rtest.Equals(t, `Fatal: --exclude-file: invalid pattern(s) provided:
*[._]log[.-][0-9]
!*[._]log[.-][0-9]`, err.Error())
// Test --iexclude-file
err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{excludePatternOptions: excludePatternOptions{InsensitiveExcludeFiles: []string{excludeFile}}}, env.gopts)
err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{ExcludePatternOptions: filter.ExcludePatternOptions{InsensitiveExcludeFiles: []string{excludeFile}}}, env.gopts)
rtest.Equals(t, `Fatal: --iexclude-file: invalid pattern(s) provided:
*[._]log[.-][0-9]
@ -70,28 +71,28 @@ func TestRestoreFailsWhenUsingInvalidPatterns(t *testing.T) {
var err error
// Test --exclude
err = testRunRestoreAssumeFailure("latest", RestoreOptions{excludePatternOptions: excludePatternOptions{Excludes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts)
err = testRunRestoreAssumeFailure("latest", RestoreOptions{ExcludePatternOptions: filter.ExcludePatternOptions{Excludes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts)
rtest.Equals(t, `Fatal: --exclude: invalid pattern(s) provided:
*[._]log[.-][0-9]
!*[._]log[.-][0-9]`, err.Error())
// Test --iexclude
err = testRunRestoreAssumeFailure("latest", RestoreOptions{excludePatternOptions: excludePatternOptions{InsensitiveExcludes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts)
err = testRunRestoreAssumeFailure("latest", RestoreOptions{ExcludePatternOptions: filter.ExcludePatternOptions{InsensitiveExcludes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts)
rtest.Equals(t, `Fatal: --iexclude: invalid pattern(s) provided:
*[._]log[.-][0-9]
!*[._]log[.-][0-9]`, err.Error())
// Test --include
err = testRunRestoreAssumeFailure("latest", RestoreOptions{includePatternOptions: includePatternOptions{Includes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts)
err = testRunRestoreAssumeFailure("latest", RestoreOptions{IncludePatternOptions: filter.IncludePatternOptions{Includes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts)
rtest.Equals(t, `Fatal: --include: invalid pattern(s) provided:
*[._]log[.-][0-9]
!*[._]log[.-][0-9]`, err.Error())
// Test --iinclude
err = testRunRestoreAssumeFailure("latest", RestoreOptions{includePatternOptions: includePatternOptions{InsensitiveIncludes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts)
err = testRunRestoreAssumeFailure("latest", RestoreOptions{IncludePatternOptions: filter.IncludePatternOptions{InsensitiveIncludes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts)
rtest.Equals(t, `Fatal: --iinclude: invalid pattern(s) provided:
*[._]log[.-][0-9]
@ -111,22 +112,22 @@ func TestRestoreFailsWhenUsingInvalidPatternsFromFile(t *testing.T) {
t.Fatalf("Could not write include file: %v", fileErr)
}
err := testRunRestoreAssumeFailure("latest", RestoreOptions{includePatternOptions: includePatternOptions{IncludeFiles: []string{patternsFile}}}, env.gopts)
err := testRunRestoreAssumeFailure("latest", RestoreOptions{IncludePatternOptions: filter.IncludePatternOptions{IncludeFiles: []string{patternsFile}}}, env.gopts)
rtest.Equals(t, `Fatal: --include-file: invalid pattern(s) provided:
*[._]log[.-][0-9]
!*[._]log[.-][0-9]`, err.Error())
err = testRunRestoreAssumeFailure("latest", RestoreOptions{excludePatternOptions: excludePatternOptions{ExcludeFiles: []string{patternsFile}}}, env.gopts)
err = testRunRestoreAssumeFailure("latest", RestoreOptions{ExcludePatternOptions: filter.ExcludePatternOptions{ExcludeFiles: []string{patternsFile}}}, env.gopts)
rtest.Equals(t, `Fatal: --exclude-file: invalid pattern(s) provided:
*[._]log[.-][0-9]
!*[._]log[.-][0-9]`, err.Error())
err = testRunRestoreAssumeFailure("latest", RestoreOptions{includePatternOptions: includePatternOptions{InsensitiveIncludeFiles: []string{patternsFile}}}, env.gopts)
err = testRunRestoreAssumeFailure("latest", RestoreOptions{IncludePatternOptions: filter.IncludePatternOptions{InsensitiveIncludeFiles: []string{patternsFile}}}, env.gopts)
rtest.Equals(t, `Fatal: --iinclude-file: invalid pattern(s) provided:
*[._]log[.-][0-9]
!*[._]log[.-][0-9]`, err.Error())
err = testRunRestoreAssumeFailure("latest", RestoreOptions{excludePatternOptions: excludePatternOptions{InsensitiveExcludeFiles: []string{patternsFile}}}, env.gopts)
err = testRunRestoreAssumeFailure("latest", RestoreOptions{ExcludePatternOptions: filter.ExcludePatternOptions{InsensitiveExcludeFiles: []string{patternsFile}}}, env.gopts)
rtest.Equals(t, `Fatal: --iexclude-file: invalid pattern(s) provided:
*[._]log[.-][0-9]
!*[._]log[.-][0-9]`, err.Error())

15
internal/archiver/exclude.go
View File

@ -14,6 +14,16 @@ import (
"github.com/restic/restic/internal/ui"
)
// RejectByNameFunc is a function that takes a filename of a
// file that would be included in the backup. The function returns true if it
// should be excluded (rejected) from the backup.
type RejectByNameFunc func(path string) bool
// RejectFunc is a function that takes a filename and os.FileInfo of a
// file that would be included in the backup. The function returns true if it
// should be excluded (rejected) from the backup.
type RejectFunc func(path string, fi os.FileInfo, fs fs.FS) bool
type rejectionCache struct {
m map[string]bool
mtx sync.Mutex
@ -49,11 +59,6 @@ func (rc *rejectionCache) Store(dir string, rejected bool) {
rc.m[dir] = rejected
}
// RejectFunc is a function that takes a filename and os.FileInfo of a
// file that would be included in the backup. The function returns true if it
// should be excluded (rejected) from the backup.
type RejectFunc func(path string, fi os.FileInfo, fs fs.FS) bool
// RejectIfPresent returns a RejectByNameFunc which itself returns whether a path
// should be excluded. The RejectByNameFunc considers a file to be excluded when
// it resides in a directory with an exclusion file, that is specified by

162
internal/filter/exclude.go Normal file
View File

@ -0,0 +1,162 @@
package filter
import (
"bufio"
"bytes"
"fmt"
"os"
"strings"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/textfile"
"github.com/spf13/pflag"
)
// RejectByNameFunc is a function that takes a filename of a
// file that would be included in the backup. The function returns true if it
// should be excluded (rejected) from the backup.
type RejectByNameFunc func(path string) bool
// RejectByPattern returns a RejectByNameFunc which rejects files that match
// one of the patterns.
func RejectByPattern(patterns []string, warnf func(msg string, args ...interface{})) RejectByNameFunc {
parsedPatterns := ParsePatterns(patterns)
return func(item string) bool {
matched, err := List(parsedPatterns, item)
if err != nil {
warnf("error for exclude pattern: %v", err)
}
if matched {
debug.Log("path %q excluded by an exclude pattern", item)
return true
}
return false
}
}
// RejectByInsensitivePattern is like RejectByPattern but case insensitive.
func RejectByInsensitivePattern(patterns []string, warnf func(msg string, args ...interface{})) RejectByNameFunc {
for index, path := range patterns {
patterns[index] = strings.ToLower(path)
}
rejFunc := RejectByPattern(patterns, warnf)
return func(item string) bool {
return rejFunc(strings.ToLower(item))
}
}
// readPatternsFromFiles reads all files and returns the list of
// patterns. For each line, leading and trailing white space is removed
// and comment lines are ignored. For each remaining pattern, environment
// variables are resolved. For adding a literal dollar sign ($), write $$ to
// the file.
func readPatternsFromFiles(files []string) ([]string, error) {
getenvOrDollar := func(s string) string {
if s == "$" {
return "$"
}
return os.Getenv(s)
}
var patterns []string
for _, filename := range files {
err := func() (err error) {
data, err := textfile.Read(filename)
if err != nil {
return err
}
scanner := bufio.NewScanner(bytes.NewReader(data))
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
// ignore empty lines
if line == "" {
continue
}
// strip comments
if strings.HasPrefix(line, "#") {
continue
}
line = os.Expand(line, getenvOrDollar)
patterns = append(patterns, line)
}
return scanner.Err()
}()
if err != nil {
return nil, fmt.Errorf("failed to read patterns from file %q: %w", filename, err)
}
}
return patterns, nil
}
type ExcludePatternOptions struct {
Excludes []string
InsensitiveExcludes []string
ExcludeFiles []string
InsensitiveExcludeFiles []string
}
func (opts *ExcludePatternOptions) Add(f *pflag.FlagSet) {
f.StringArrayVarP(&opts.Excludes, "exclude", "e", nil, "exclude a `pattern` (can be specified multiple times)")
f.StringArrayVar(&opts.InsensitiveExcludes, "iexclude", nil, "same as --exclude `pattern` but ignores the casing of filenames")
f.StringArrayVar(&opts.ExcludeFiles, "exclude-file", nil, "read exclude patterns from a `file` (can be specified multiple times)")
f.StringArrayVar(&opts.InsensitiveExcludeFiles, "iexclude-file", nil, "same as --exclude-file but ignores casing of `file`names in patterns")
}
func (opts *ExcludePatternOptions) Empty() bool {
return len(opts.Excludes) == 0 && len(opts.InsensitiveExcludes) == 0 && len(opts.ExcludeFiles) == 0 && len(opts.InsensitiveExcludeFiles) == 0
}
func (opts ExcludePatternOptions) CollectPatterns(warnf func(msg string, args ...interface{})) ([]RejectByNameFunc, error) {
var fs []RejectByNameFunc
// add patterns from file
if len(opts.ExcludeFiles) > 0 {
excludePatterns, err := readPatternsFromFiles(opts.ExcludeFiles)
if err != nil {
return nil, err
}
if err := ValidatePatterns(excludePatterns); err != nil {
return nil, errors.Fatalf("--exclude-file: %s", err)
}
opts.Excludes = append(opts.Excludes, excludePatterns...)
}
if len(opts.InsensitiveExcludeFiles) > 0 {
excludes, err := readPatternsFromFiles(opts.InsensitiveExcludeFiles)
if err != nil {
return nil, err
}
if err := ValidatePatterns(excludes); err != nil {
return nil, errors.Fatalf("--iexclude-file: %s", err)
}
opts.InsensitiveExcludes = append(opts.InsensitiveExcludes, excludes...)
}
if len(opts.InsensitiveExcludes) > 0 {
if err := ValidatePatterns(opts.InsensitiveExcludes); err != nil {
return nil, errors.Fatalf("--iexclude: %s", err)
}
fs = append(fs, RejectByInsensitivePattern(opts.InsensitiveExcludes, warnf))
}
if len(opts.Excludes) > 0 {
if err := ValidatePatterns(opts.Excludes); err != nil {
return nil, errors.Fatalf("--exclude: %s", err)
}
fs = append(fs, RejectByPattern(opts.Excludes, warnf))
}
return fs, nil
}

6
cmd/restic/exclude_test.go → internal/filter/exclude_test.go
View File

@ -1,4 +1,4 @@
package main
package filter
import (
"testing"
@ -21,7 +21,7 @@ func TestRejectByPattern(t *testing.T) {
for _, tc := range tests {
t.Run("", func(t *testing.T) {
reject := rejectByPattern(patterns)
reject := RejectByPattern(patterns, nil)
res := reject(tc.filename)
if res != tc.reject {
t.Fatalf("wrong result for filename %v: want %v, got %v",
@ -48,7 +48,7 @@ func TestRejectByInsensitivePattern(t *testing.T) {
for _, tc := range tests {
t.Run("", func(t *testing.T) {
reject := rejectByInsensitivePattern(patterns)
reject := RejectByInsensitivePattern(patterns, nil)
res := reject(tc.filename)
if res != tc.reject {
t.Fatalf("wrong result for filename %v: want %v, got %v",

37
cmd/restic/include.go → internal/filter/include.go
View File

@ -1,10 +1,9 @@
package main
package filter
import (
"strings"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/filter"
"github.com/spf13/pflag"
)
@ -12,21 +11,21 @@ import (
// in the restore process and returns whether it should be included.
type IncludeByNameFunc func(item string) (matched bool, childMayMatch bool)
type includePatternOptions struct {
type IncludePatternOptions struct {
Includes []string
InsensitiveIncludes []string
IncludeFiles []string
InsensitiveIncludeFiles []string
}
func (opts *includePatternOptions) Add(f *pflag.FlagSet) {
func (opts *IncludePatternOptions) Add(f *pflag.FlagSet) {
f.StringArrayVarP(&opts.Includes, "include", "i", nil, "include a `pattern` (can be specified multiple times)")
f.StringArrayVar(&opts.InsensitiveIncludes, "iinclude", nil, "same as --include `pattern` but ignores the casing of filenames")
f.StringArrayVar(&opts.IncludeFiles, "include-file", nil, "read include patterns from a `file` (can be specified multiple times)")
f.StringArrayVar(&opts.InsensitiveIncludeFiles, "iinclude-file", nil, "same as --include-file but ignores casing of `file`names in patterns")
}
func (opts includePatternOptions) CollectPatterns() ([]IncludeByNameFunc, error) {
func (opts IncludePatternOptions) CollectPatterns(warnf func(msg string, args ...interface{})) ([]IncludeByNameFunc, error) {
var fs []IncludeByNameFunc
if len(opts.IncludeFiles) > 0 {
includePatterns, err := readPatternsFromFiles(opts.IncludeFiles)
@ -34,7 +33,7 @@ func (opts includePatternOptions) CollectPatterns() ([]IncludeByNameFunc, error)
return nil, err
}
if err := filter.ValidatePatterns(includePatterns); err != nil {
if err := ValidatePatterns(includePatterns); err != nil {
return nil, errors.Fatalf("--include-file: %s", err)
}
@ -47,7 +46,7 @@ func (opts includePatternOptions) CollectPatterns() ([]IncludeByNameFunc, error)
return nil, err
}
if err := filter.ValidatePatterns(includePatterns); err != nil {
if err := ValidatePatterns(includePatterns); err != nil {
return nil, errors.Fatalf("--iinclude-file: %s", err)
}
@ -55,45 +54,45 @@ func (opts includePatternOptions) CollectPatterns() ([]IncludeByNameFunc, error)
}
if len(opts.InsensitiveIncludes) > 0 {
if err := filter.ValidatePatterns(opts.InsensitiveIncludes); err != nil {
if err := ValidatePatterns(opts.InsensitiveIncludes); err != nil {
return nil, errors.Fatalf("--iinclude: %s", err)
}
fs = append(fs, includeByInsensitivePattern(opts.InsensitiveIncludes))
fs = append(fs, IncludeByInsensitivePattern(opts.InsensitiveIncludes, warnf))
}
if len(opts.Includes) > 0 {
if err := filter.ValidatePatterns(opts.Includes); err != nil {
if err := ValidatePatterns(opts.Includes); err != nil {
return nil, errors.Fatalf("--include: %s", err)
}
fs = append(fs, includeByPattern(opts.Includes))
fs = append(fs, IncludeByPattern(opts.Includes, warnf))
}
return fs, nil
}
// includeByPattern returns a IncludeByNameFunc which includes files that match
// IncludeByPattern returns a IncludeByNameFunc which includes files that match
// one of the patterns.
func includeByPattern(patterns []string) IncludeByNameFunc {
parsedPatterns := filter.ParsePatterns(patterns)
func IncludeByPattern(patterns []string, warnf func(msg string, args ...interface{})) IncludeByNameFunc {
parsedPatterns := ParsePatterns(patterns)
return func(item string) (matched bool, childMayMatch bool) {
matched, childMayMatch, err := filter.ListWithChild(parsedPatterns, item)
matched, childMayMatch, err := ListWithChild(parsedPatterns, item)
if err != nil {
Warnf("error for include pattern: %v", err)
warnf("error for include pattern: %v", err)
}
return matched, childMayMatch
}
}
// includeByInsensitivePattern returns a IncludeByNameFunc which includes files that match
// IncludeByInsensitivePattern returns a IncludeByNameFunc which includes files that match
// one of the patterns, ignoring the casing of the filenames.
func includeByInsensitivePattern(patterns []string) IncludeByNameFunc {
func IncludeByInsensitivePattern(patterns []string, warnf func(msg string, args ...interface{})) IncludeByNameFunc {
for index, path := range patterns {
patterns[index] = strings.ToLower(path)
}
includeFunc := includeByPattern(patterns)
includeFunc := IncludeByPattern(patterns, warnf)
return func(item string) (matched bool, childMayMatch bool) {
return includeFunc(strings.ToLower(item))
}

6
cmd/restic/include_test.go → internal/filter/include_test.go
View File

@ -1,4 +1,4 @@
package main
package filter
import (
"testing"
@ -21,7 +21,7 @@ func TestIncludeByPattern(t *testing.T) {
for _, tc := range tests {
t.Run(tc.filename, func(t *testing.T) {
includeFunc := includeByPattern(patterns)
includeFunc := IncludeByPattern(patterns, nil)
matched, _ := includeFunc(tc.filename)
if matched != tc.include {
t.Fatalf("wrong result for filename %v: want %v, got %v",
@ -48,7 +48,7 @@ func TestIncludeByInsensitivePattern(t *testing.T) {
for _, tc := range tests {
t.Run(tc.filename, func(t *testing.T) {
includeFunc := includeByInsensitivePattern(patterns)
includeFunc := IncludeByInsensitivePattern(patterns, nil)
matched, _ := includeFunc(tc.filename)
if matched != tc.include {
t.Fatalf("wrong result for filename %v: want %v, got %v",