mirror of
https://github.com/octoleo/restic.git
synced 2024-12-24 11:55:28 +00:00
Merge pull request #4811 from konidev20/fix-gh-4781-restore-include-and-exclude-file-switches
Restore flags to include from file and exclude from file
This commit is contained in:
commit
be05a17e15
8
changelog/unreleased/issue-4781
Normal file
8
changelog/unreleased/issue-4781
Normal file
@ -0,0 +1,8 @@
|
|||||||
|
Enhancement: Add restore flags to read include and exclude patterns from files
|
||||||
|
|
||||||
|
Restic now supports reading include and exclude patterns from files using the
|
||||||
|
`--include-file`, `--exclude-file`, `--iinclude-file` and `--iexclude-file`
|
||||||
|
flags.
|
||||||
|
|
||||||
|
https://github.com/restic/restic/issues/4781
|
||||||
|
https://github.com/restic/restic/pull/4811
|
@ -2,12 +2,10 @@ package main
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"strings"
|
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/restic/restic/internal/debug"
|
"github.com/restic/restic/internal/debug"
|
||||||
"github.com/restic/restic/internal/errors"
|
"github.com/restic/restic/internal/errors"
|
||||||
"github.com/restic/restic/internal/filter"
|
|
||||||
"github.com/restic/restic/internal/restic"
|
"github.com/restic/restic/internal/restic"
|
||||||
"github.com/restic/restic/internal/restorer"
|
"github.com/restic/restic/internal/restorer"
|
||||||
"github.com/restic/restic/internal/ui"
|
"github.com/restic/restic/internal/ui"
|
||||||
@ -45,11 +43,9 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
|
|||||||
|
|
||||||
// RestoreOptions collects all options for the restore command.
|
// RestoreOptions collects all options for the restore command.
|
||||||
type RestoreOptions struct {
|
type RestoreOptions struct {
|
||||||
Exclude []string
|
excludePatternOptions
|
||||||
InsensitiveExclude []string
|
includePatternOptions
|
||||||
Include []string
|
Target string
|
||||||
InsensitiveInclude []string
|
|
||||||
Target string
|
|
||||||
restic.SnapshotFilter
|
restic.SnapshotFilter
|
||||||
Sparse bool
|
Sparse bool
|
||||||
Verify bool
|
Verify bool
|
||||||
@ -61,12 +57,11 @@ func init() {
|
|||||||
cmdRoot.AddCommand(cmdRestore)
|
cmdRoot.AddCommand(cmdRestore)
|
||||||
|
|
||||||
flags := cmdRestore.Flags()
|
flags := cmdRestore.Flags()
|
||||||
flags.StringArrayVarP(&restoreOptions.Exclude, "exclude", "e", nil, "exclude a `pattern` (can be specified multiple times)")
|
|
||||||
flags.StringArrayVar(&restoreOptions.InsensitiveExclude, "iexclude", nil, "same as --exclude but ignores the casing of `pattern`")
|
|
||||||
flags.StringArrayVarP(&restoreOptions.Include, "include", "i", nil, "include a `pattern`, exclude everything else (can be specified multiple times)")
|
|
||||||
flags.StringArrayVar(&restoreOptions.InsensitiveInclude, "iinclude", nil, "same as --include but ignores the casing of `pattern`")
|
|
||||||
flags.StringVarP(&restoreOptions.Target, "target", "t", "", "directory to extract data to")
|
flags.StringVarP(&restoreOptions.Target, "target", "t", "", "directory to extract data to")
|
||||||
|
|
||||||
|
initExcludePatternOptions(flags, &restoreOptions.excludePatternOptions)
|
||||||
|
initIncludePatternOptions(flags, &restoreOptions.includePatternOptions)
|
||||||
|
|
||||||
initSingleSnapshotFilter(flags, &restoreOptions.SnapshotFilter)
|
initSingleSnapshotFilter(flags, &restoreOptions.SnapshotFilter)
|
||||||
flags.BoolVar(&restoreOptions.Sparse, "sparse", false, "restore files as sparse")
|
flags.BoolVar(&restoreOptions.Sparse, "sparse", false, "restore files as sparse")
|
||||||
flags.BoolVar(&restoreOptions.Verify, "verify", false, "verify restored files content")
|
flags.BoolVar(&restoreOptions.Verify, "verify", false, "verify restored files content")
|
||||||
@ -75,38 +70,18 @@ func init() {
|
|||||||
func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions,
|
func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions,
|
||||||
term *termstatus.Terminal, args []string) error {
|
term *termstatus.Terminal, args []string) error {
|
||||||
|
|
||||||
hasExcludes := len(opts.Exclude) > 0 || len(opts.InsensitiveExclude) > 0
|
excludePatternFns, err := opts.excludePatternOptions.CollectPatterns()
|
||||||
hasIncludes := len(opts.Include) > 0 || len(opts.InsensitiveInclude) > 0
|
if err != nil {
|
||||||
|
return err
|
||||||
// Validate provided patterns
|
|
||||||
if len(opts.Exclude) > 0 {
|
|
||||||
if err := filter.ValidatePatterns(opts.Exclude); err != nil {
|
|
||||||
return errors.Fatalf("--exclude: %s", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(opts.InsensitiveExclude) > 0 {
|
|
||||||
if err := filter.ValidatePatterns(opts.InsensitiveExclude); err != nil {
|
|
||||||
return errors.Fatalf("--iexclude: %s", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(opts.Include) > 0 {
|
|
||||||
if err := filter.ValidatePatterns(opts.Include); err != nil {
|
|
||||||
return errors.Fatalf("--include: %s", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(opts.InsensitiveInclude) > 0 {
|
|
||||||
if err := filter.ValidatePatterns(opts.InsensitiveInclude); err != nil {
|
|
||||||
return errors.Fatalf("--iinclude: %s", err)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
for i, str := range opts.InsensitiveExclude {
|
includePatternFns, err := opts.includePatternOptions.CollectPatterns()
|
||||||
opts.InsensitiveExclude[i] = strings.ToLower(str)
|
if err != nil {
|
||||||
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
for i, str := range opts.InsensitiveInclude {
|
hasExcludes := len(excludePatternFns) > 0
|
||||||
opts.InsensitiveInclude[i] = strings.ToLower(str)
|
hasIncludes := len(includePatternFns) > 0
|
||||||
}
|
|
||||||
|
|
||||||
switch {
|
switch {
|
||||||
case len(args) == 0:
|
case len(args) == 0:
|
||||||
@ -174,44 +149,41 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions,
|
|||||||
msg.E("Warning: %s\n", message)
|
msg.E("Warning: %s\n", message)
|
||||||
}
|
}
|
||||||
|
|
||||||
excludePatterns := filter.ParsePatterns(opts.Exclude)
|
|
||||||
insensitiveExcludePatterns := filter.ParsePatterns(opts.InsensitiveExclude)
|
|
||||||
selectExcludeFilter := func(item string, _ string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool) {
|
selectExcludeFilter := func(item string, _ string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool) {
|
||||||
matched, err := filter.List(excludePatterns, item)
|
matched := false
|
||||||
if err != nil {
|
for _, rejectFn := range excludePatternFns {
|
||||||
msg.E("error for exclude pattern: %v", err)
|
matched = matched || rejectFn(item)
|
||||||
}
|
|
||||||
|
|
||||||
matchedInsensitive, err := filter.List(insensitiveExcludePatterns, strings.ToLower(item))
|
// implementing a short-circuit here to improve the performance
|
||||||
if err != nil {
|
// to prevent additional pattern matching once the first pattern
|
||||||
msg.E("error for iexclude pattern: %v", err)
|
// matches.
|
||||||
|
if matched {
|
||||||
|
break
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// An exclude filter is basically a 'wildcard but foo',
|
// An exclude filter is basically a 'wildcard but foo',
|
||||||
// so even if a childMayMatch, other children of a dir may not,
|
// so even if a childMayMatch, other children of a dir may not,
|
||||||
// therefore childMayMatch does not matter, but we should not go down
|
// therefore childMayMatch does not matter, but we should not go down
|
||||||
// unless the dir is selected for restore
|
// unless the dir is selected for restore
|
||||||
selectedForRestore = !matched && !matchedInsensitive
|
selectedForRestore = !matched
|
||||||
childMayBeSelected = selectedForRestore && node.Type == "dir"
|
childMayBeSelected = selectedForRestore && node.Type == "dir"
|
||||||
|
|
||||||
return selectedForRestore, childMayBeSelected
|
return selectedForRestore, childMayBeSelected
|
||||||
}
|
}
|
||||||
|
|
||||||
includePatterns := filter.ParsePatterns(opts.Include)
|
|
||||||
insensitiveIncludePatterns := filter.ParsePatterns(opts.InsensitiveInclude)
|
|
||||||
selectIncludeFilter := func(item string, _ string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool) {
|
selectIncludeFilter := func(item string, _ string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool) {
|
||||||
matched, childMayMatch, err := filter.ListWithChild(includePatterns, item)
|
selectedForRestore = false
|
||||||
if err != nil {
|
childMayBeSelected = false
|
||||||
msg.E("error for include pattern: %v", err)
|
for _, includeFn := range includePatternFns {
|
||||||
}
|
matched, childMayMatch := includeFn(item)
|
||||||
|
selectedForRestore = selectedForRestore || matched
|
||||||
|
childMayBeSelected = childMayBeSelected || childMayMatch
|
||||||
|
|
||||||
matchedInsensitive, childMayMatchInsensitive, err := filter.ListWithChild(insensitiveIncludePatterns, strings.ToLower(item))
|
if selectedForRestore && childMayBeSelected {
|
||||||
if err != nil {
|
break
|
||||||
msg.E("error for iexclude pattern: %v", err)
|
}
|
||||||
}
|
}
|
||||||
|
childMayBeSelected = childMayBeSelected && node.Type == "dir"
|
||||||
selectedForRestore = matched || matchedInsensitive
|
|
||||||
childMayBeSelected = (childMayMatch || childMayMatchInsensitive) && node.Type == "dir"
|
|
||||||
|
|
||||||
return selectedForRestore, childMayBeSelected
|
return selectedForRestore, childMayBeSelected
|
||||||
}
|
}
|
||||||
|
@ -7,12 +7,12 @@ import (
|
|||||||
"math/rand"
|
"math/rand"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
"syscall"
|
"syscall"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/restic/restic/internal/feature"
|
"github.com/restic/restic/internal/feature"
|
||||||
"github.com/restic/restic/internal/filter"
|
|
||||||
"github.com/restic/restic/internal/restic"
|
"github.com/restic/restic/internal/restic"
|
||||||
rtest "github.com/restic/restic/internal/test"
|
rtest "github.com/restic/restic/internal/test"
|
||||||
"github.com/restic/restic/internal/ui/termstatus"
|
"github.com/restic/restic/internal/ui/termstatus"
|
||||||
@ -24,9 +24,9 @@ func testRunRestore(t testing.TB, opts GlobalOptions, dir string, snapshotID res
|
|||||||
|
|
||||||
func testRunRestoreExcludes(t testing.TB, gopts GlobalOptions, dir string, snapshotID restic.ID, excludes []string) {
|
func testRunRestoreExcludes(t testing.TB, gopts GlobalOptions, dir string, snapshotID restic.ID, excludes []string) {
|
||||||
opts := RestoreOptions{
|
opts := RestoreOptions{
|
||||||
Target: dir,
|
Target: dir,
|
||||||
Exclude: excludes,
|
|
||||||
}
|
}
|
||||||
|
opts.Excludes = excludes
|
||||||
|
|
||||||
rtest.OK(t, testRunRestoreAssumeFailure(snapshotID.String(), opts, gopts))
|
rtest.OK(t, testRunRestoreAssumeFailure(snapshotID.String(), opts, gopts))
|
||||||
}
|
}
|
||||||
@ -51,22 +51,132 @@ func testRunRestoreLatest(t testing.TB, gopts GlobalOptions, dir string, paths [
|
|||||||
|
|
||||||
func testRunRestoreIncludes(t testing.TB, gopts GlobalOptions, dir string, snapshotID restic.ID, includes []string) {
|
func testRunRestoreIncludes(t testing.TB, gopts GlobalOptions, dir string, snapshotID restic.ID, includes []string) {
|
||||||
opts := RestoreOptions{
|
opts := RestoreOptions{
|
||||||
Target: dir,
|
Target: dir,
|
||||||
Include: includes,
|
|
||||||
}
|
}
|
||||||
|
opts.Includes = includes
|
||||||
|
|
||||||
rtest.OK(t, testRunRestoreAssumeFailure(snapshotID.String(), opts, gopts))
|
rtest.OK(t, testRunRestoreAssumeFailure(snapshotID.String(), opts, gopts))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func testRunRestoreIncludesFromFile(t testing.TB, gopts GlobalOptions, dir string, snapshotID restic.ID, includesFile string) {
|
||||||
|
opts := RestoreOptions{
|
||||||
|
Target: dir,
|
||||||
|
}
|
||||||
|
opts.IncludeFiles = []string{includesFile}
|
||||||
|
|
||||||
|
rtest.OK(t, testRunRestoreAssumeFailure(snapshotID.String(), opts, gopts))
|
||||||
|
}
|
||||||
|
|
||||||
|
func testRunRestoreExcludesFromFile(t testing.TB, gopts GlobalOptions, dir string, snapshotID restic.ID, excludesFile string) {
|
||||||
|
opts := RestoreOptions{
|
||||||
|
Target: dir,
|
||||||
|
}
|
||||||
|
opts.ExcludeFiles = []string{excludesFile}
|
||||||
|
|
||||||
|
rtest.OK(t, testRunRestoreAssumeFailure(snapshotID.String(), opts, gopts))
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRestoreMustFailWhenUsingBothIncludesAndExcludes(t *testing.T) {
|
||||||
|
env, cleanup := withTestEnvironment(t)
|
||||||
|
defer cleanup()
|
||||||
|
|
||||||
|
testRunInit(t, env.gopts)
|
||||||
|
|
||||||
|
// Add both include and exclude patterns
|
||||||
|
includePatterns := []string{"dir1/*include_me.txt", "dir2/**", "dir4/**/*_me.txt"}
|
||||||
|
excludePatterns := []string{"dir1/*include_me.txt", "dir2/**", "dir4/**/*_me.txt"}
|
||||||
|
|
||||||
|
restoredir := filepath.Join(env.base, "restore")
|
||||||
|
|
||||||
|
restoreOpts := RestoreOptions{
|
||||||
|
Target: restoredir,
|
||||||
|
}
|
||||||
|
restoreOpts.Includes = includePatterns
|
||||||
|
restoreOpts.Excludes = excludePatterns
|
||||||
|
|
||||||
|
err := testRunRestoreAssumeFailure("latest", restoreOpts, env.gopts)
|
||||||
|
rtest.Assert(t, err != nil && strings.Contains(err.Error(), "exclude and include patterns are mutually exclusive"),
|
||||||
|
"expected: %s error, got %v", "exclude and include patterns are mutually exclusive", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRestoreIncludes(t *testing.T) {
|
||||||
|
testfiles := []struct {
|
||||||
|
path string
|
||||||
|
size uint
|
||||||
|
include bool // Whether this file should be included in the restore
|
||||||
|
}{
|
||||||
|
{"dir1/include_me.txt", 100, true},
|
||||||
|
{"dir1/something_else.txt", 200, false},
|
||||||
|
{"dir2/also_include_me.txt", 150, true},
|
||||||
|
{"dir2/important_file.txt", 150, true},
|
||||||
|
{"dir3/not_included.txt", 180, false},
|
||||||
|
{"dir4/subdir/should_include_me.txt", 120, true},
|
||||||
|
}
|
||||||
|
|
||||||
|
env, cleanup := withTestEnvironment(t)
|
||||||
|
defer cleanup()
|
||||||
|
|
||||||
|
testRunInit(t, env.gopts)
|
||||||
|
|
||||||
|
// Create test files and directories
|
||||||
|
for _, testFile := range testfiles {
|
||||||
|
fullPath := filepath.Join(env.testdata, testFile.path)
|
||||||
|
rtest.OK(t, os.MkdirAll(filepath.Dir(fullPath), 0755))
|
||||||
|
rtest.OK(t, appendRandomData(fullPath, testFile.size))
|
||||||
|
}
|
||||||
|
|
||||||
|
opts := BackupOptions{}
|
||||||
|
|
||||||
|
// Perform backup
|
||||||
|
testRunBackup(t, filepath.Dir(env.testdata), []string{filepath.Base(env.testdata)}, opts, env.gopts)
|
||||||
|
testRunCheck(t, env.gopts)
|
||||||
|
|
||||||
|
snapshotID := testListSnapshots(t, env.gopts, 1)[0]
|
||||||
|
|
||||||
|
// Restore using includes
|
||||||
|
includePatterns := []string{"dir1/*include_me.txt", "dir2/**", "dir4/**/*_me.txt"}
|
||||||
|
restoredir := filepath.Join(env.base, "restore")
|
||||||
|
testRunRestoreIncludes(t, env.gopts, restoredir, snapshotID, includePatterns)
|
||||||
|
|
||||||
|
testRestoreFileInclusions := func(t *testing.T) {
|
||||||
|
// Check that only the included files are restored
|
||||||
|
for _, testFile := range testfiles {
|
||||||
|
restoredFilePath := filepath.Join(restoredir, "testdata", testFile.path)
|
||||||
|
_, err := os.Stat(restoredFilePath)
|
||||||
|
if testFile.include {
|
||||||
|
rtest.OK(t, err)
|
||||||
|
} else {
|
||||||
|
rtest.Assert(t, os.IsNotExist(err), "File %s should not have been restored", testFile.path)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
testRestoreFileInclusions(t)
|
||||||
|
|
||||||
|
// Create an include file with some patterns
|
||||||
|
patternsFile := env.base + "/patternsFile"
|
||||||
|
fileErr := os.WriteFile(patternsFile, []byte(strings.Join(includePatterns, "\n")), 0644)
|
||||||
|
if fileErr != nil {
|
||||||
|
t.Fatalf("Could not write include file: %v", fileErr)
|
||||||
|
}
|
||||||
|
|
||||||
|
restoredir = filepath.Join(env.base, "restore-include-from-file")
|
||||||
|
|
||||||
|
testRunRestoreIncludesFromFile(t, env.gopts, restoredir, snapshotID, patternsFile)
|
||||||
|
|
||||||
|
testRestoreFileInclusions(t)
|
||||||
|
}
|
||||||
|
|
||||||
func TestRestoreFilter(t *testing.T) {
|
func TestRestoreFilter(t *testing.T) {
|
||||||
testfiles := []struct {
|
testfiles := []struct {
|
||||||
name string
|
name string
|
||||||
size uint
|
size uint
|
||||||
|
exclude bool
|
||||||
}{
|
}{
|
||||||
{"testfile1.c", 100},
|
{"testfile1.c", 100, true},
|
||||||
{"testfile2.exe", 101},
|
{"testfile2.exe", 101, true},
|
||||||
{"subdir1/subdir2/testfile3.docx", 102},
|
{"subdir1/subdir2/testfile3.docx", 102, true},
|
||||||
{"subdir1/subdir2/testfile4.c", 102},
|
{"subdir1/subdir2/testfile4.c", 102, false},
|
||||||
}
|
}
|
||||||
|
|
||||||
env, cleanup := withTestEnvironment(t)
|
env, cleanup := withTestEnvironment(t)
|
||||||
@ -93,19 +203,38 @@ func TestRestoreFilter(t *testing.T) {
|
|||||||
rtest.OK(t, testFileSize(filepath.Join(env.base, "restore0", "testdata", testFile.name), int64(testFile.size)))
|
rtest.OK(t, testFileSize(filepath.Join(env.base, "restore0", "testdata", testFile.name), int64(testFile.size)))
|
||||||
}
|
}
|
||||||
|
|
||||||
for i, pat := range []string{"*.c", "*.exe", "*", "*file3*"} {
|
excludePatterns := []string{"testfile1.c", "*.exe", "*file3*"}
|
||||||
base := filepath.Join(env.base, fmt.Sprintf("restore%d", i+1))
|
|
||||||
testRunRestoreExcludes(t, env.gopts, base, snapshotID, []string{pat})
|
// checks if the files are excluded correctly
|
||||||
|
testRestoredFileExclusions := func(t *testing.T, restoredir string) {
|
||||||
for _, testFile := range testfiles {
|
for _, testFile := range testfiles {
|
||||||
err := testFileSize(filepath.Join(base, "testdata", testFile.name), int64(testFile.size))
|
restoredFilePath := filepath.Join(restoredir, "testdata", testFile.name)
|
||||||
if ok, _ := filter.Match(pat, filepath.Base(testFile.name)); !ok {
|
_, err := os.Stat(restoredFilePath)
|
||||||
rtest.OK(t, err)
|
if testFile.exclude {
|
||||||
|
rtest.Assert(t, os.IsNotExist(err), "File %s should not have been restored", testFile.name)
|
||||||
} else {
|
} else {
|
||||||
rtest.Assert(t, os.IsNotExist(err),
|
rtest.OK(t, testFileSize(restoredFilePath, int64(testFile.size)))
|
||||||
"expected %v to not exist in restore step %v, but it exists, err %v", testFile.name, i+1, err)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// restore with excludes
|
||||||
|
restoredir := filepath.Join(env.base, "restore-with-excludes")
|
||||||
|
testRunRestoreExcludes(t, env.gopts, restoredir, snapshotID, excludePatterns)
|
||||||
|
testRestoredFileExclusions(t, restoredir)
|
||||||
|
|
||||||
|
// Create an exclude file with some patterns
|
||||||
|
patternsFile := env.base + "/patternsFile"
|
||||||
|
fileErr := os.WriteFile(patternsFile, []byte(strings.Join(excludePatterns, "\n")), 0644)
|
||||||
|
if fileErr != nil {
|
||||||
|
t.Fatalf("Could not write include file: %v", fileErr)
|
||||||
|
}
|
||||||
|
|
||||||
|
// restore with excludes from file
|
||||||
|
restoredir = filepath.Join(env.base, "restore-with-exclude-from-file")
|
||||||
|
testRunRestoreExcludesFromFile(t, env.gopts, restoredir, snapshotID, patternsFile)
|
||||||
|
|
||||||
|
testRestoredFileExclusions(t, restoredir)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRestore(t *testing.T) {
|
func TestRestore(t *testing.T) {
|
||||||
|
@ -385,12 +385,12 @@ func rejectBySize(maxSizeStr string) (RejectFunc, error) {
|
|||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// readExcludePatternsFromFiles reads all exclude files and returns the list of
|
// readPatternsFromFiles reads all files and returns the list of
|
||||||
// exclude patterns. For each line, leading and trailing white space is removed
|
// patterns. For each line, leading and trailing white space is removed
|
||||||
// and comment lines are ignored. For each remaining pattern, environment
|
// and comment lines are ignored. For each remaining pattern, environment
|
||||||
// variables are resolved. For adding a literal dollar sign ($), write $$ to
|
// variables are resolved. For adding a literal dollar sign ($), write $$ to
|
||||||
// the file.
|
// the file.
|
||||||
func readExcludePatternsFromFiles(excludeFiles []string) ([]string, error) {
|
func readPatternsFromFiles(files []string) ([]string, error) {
|
||||||
getenvOrDollar := func(s string) string {
|
getenvOrDollar := func(s string) string {
|
||||||
if s == "$" {
|
if s == "$" {
|
||||||
return "$"
|
return "$"
|
||||||
@ -398,8 +398,8 @@ func readExcludePatternsFromFiles(excludeFiles []string) ([]string, error) {
|
|||||||
return os.Getenv(s)
|
return os.Getenv(s)
|
||||||
}
|
}
|
||||||
|
|
||||||
var excludes []string
|
var patterns []string
|
||||||
for _, filename := range excludeFiles {
|
for _, filename := range files {
|
||||||
err := func() (err error) {
|
err := func() (err error) {
|
||||||
data, err := textfile.Read(filename)
|
data, err := textfile.Read(filename)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -421,15 +421,15 @@ func readExcludePatternsFromFiles(excludeFiles []string) ([]string, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
line = os.Expand(line, getenvOrDollar)
|
line = os.Expand(line, getenvOrDollar)
|
||||||
excludes = append(excludes, line)
|
patterns = append(patterns, line)
|
||||||
}
|
}
|
||||||
return scanner.Err()
|
return scanner.Err()
|
||||||
}()
|
}()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("failed to read excludes from file %q: %w", filename, err)
|
return nil, fmt.Errorf("failed to read patterns from file %q: %w", filename, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return excludes, nil
|
return patterns, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
type excludePatternOptions struct {
|
type excludePatternOptions struct {
|
||||||
@ -454,7 +454,7 @@ func (opts excludePatternOptions) CollectPatterns() ([]RejectByNameFunc, error)
|
|||||||
var fs []RejectByNameFunc
|
var fs []RejectByNameFunc
|
||||||
// add patterns from file
|
// add patterns from file
|
||||||
if len(opts.ExcludeFiles) > 0 {
|
if len(opts.ExcludeFiles) > 0 {
|
||||||
excludePatterns, err := readExcludePatternsFromFiles(opts.ExcludeFiles)
|
excludePatterns, err := readPatternsFromFiles(opts.ExcludeFiles)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@ -467,7 +467,7 @@ func (opts excludePatternOptions) CollectPatterns() ([]RejectByNameFunc, error)
|
|||||||
}
|
}
|
||||||
|
|
||||||
if len(opts.InsensitiveExcludeFiles) > 0 {
|
if len(opts.InsensitiveExcludeFiles) > 0 {
|
||||||
excludes, err := readExcludePatternsFromFiles(opts.InsensitiveExcludeFiles)
|
excludes, err := readPatternsFromFiles(opts.InsensitiveExcludeFiles)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
100
cmd/restic/include.go
Normal file
100
cmd/restic/include.go
Normal file
@ -0,0 +1,100 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/restic/restic/internal/errors"
|
||||||
|
"github.com/restic/restic/internal/filter"
|
||||||
|
"github.com/spf13/pflag"
|
||||||
|
)
|
||||||
|
|
||||||
|
// IncludeByNameFunc is a function that takes a filename that should be included
|
||||||
|
// in the restore process and returns whether it should be included.
|
||||||
|
type IncludeByNameFunc func(item string) (matched bool, childMayMatch bool)
|
||||||
|
|
||||||
|
type includePatternOptions struct {
|
||||||
|
Includes []string
|
||||||
|
InsensitiveIncludes []string
|
||||||
|
IncludeFiles []string
|
||||||
|
InsensitiveIncludeFiles []string
|
||||||
|
}
|
||||||
|
|
||||||
|
func initIncludePatternOptions(f *pflag.FlagSet, opts *includePatternOptions) {
|
||||||
|
f.StringArrayVarP(&opts.Includes, "include", "i", nil, "include a `pattern` (can be specified multiple times)")
|
||||||
|
f.StringArrayVar(&opts.InsensitiveIncludes, "iinclude", nil, "same as --include `pattern` but ignores the casing of filenames")
|
||||||
|
f.StringArrayVar(&opts.IncludeFiles, "include-file", nil, "read include patterns from a `file` (can be specified multiple times)")
|
||||||
|
f.StringArrayVar(&opts.InsensitiveIncludeFiles, "iinclude-file", nil, "same as --include-file but ignores casing of `file`names in patterns")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (opts includePatternOptions) CollectPatterns() ([]IncludeByNameFunc, error) {
|
||||||
|
var fs []IncludeByNameFunc
|
||||||
|
if len(opts.IncludeFiles) > 0 {
|
||||||
|
includePatterns, err := readPatternsFromFiles(opts.IncludeFiles)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := filter.ValidatePatterns(includePatterns); err != nil {
|
||||||
|
return nil, errors.Fatalf("--include-file: %s", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
opts.Includes = append(opts.Includes, includePatterns...)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(opts.InsensitiveIncludeFiles) > 0 {
|
||||||
|
includePatterns, err := readPatternsFromFiles(opts.InsensitiveIncludeFiles)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := filter.ValidatePatterns(includePatterns); err != nil {
|
||||||
|
return nil, errors.Fatalf("--iinclude-file: %s", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
opts.InsensitiveIncludes = append(opts.InsensitiveIncludes, includePatterns...)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(opts.InsensitiveIncludes) > 0 {
|
||||||
|
if err := filter.ValidatePatterns(opts.InsensitiveIncludes); err != nil {
|
||||||
|
return nil, errors.Fatalf("--iinclude: %s", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
fs = append(fs, includeByInsensitivePattern(opts.InsensitiveIncludes))
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(opts.Includes) > 0 {
|
||||||
|
if err := filter.ValidatePatterns(opts.Includes); err != nil {
|
||||||
|
return nil, errors.Fatalf("--include: %s", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
fs = append(fs, includeByPattern(opts.Includes))
|
||||||
|
}
|
||||||
|
return fs, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// includeByPattern returns a IncludeByNameFunc which includes files that match
|
||||||
|
// one of the patterns.
|
||||||
|
func includeByPattern(patterns []string) IncludeByNameFunc {
|
||||||
|
parsedPatterns := filter.ParsePatterns(patterns)
|
||||||
|
return func(item string) (matched bool, childMayMatch bool) {
|
||||||
|
matched, childMayMatch, err := filter.ListWithChild(parsedPatterns, item)
|
||||||
|
if err != nil {
|
||||||
|
Warnf("error for include pattern: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return matched, childMayMatch
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// includeByInsensitivePattern returns a IncludeByNameFunc which includes files that match
|
||||||
|
// one of the patterns, ignoring the casing of the filenames.
|
||||||
|
func includeByInsensitivePattern(patterns []string) IncludeByNameFunc {
|
||||||
|
for index, path := range patterns {
|
||||||
|
patterns[index] = strings.ToLower(path)
|
||||||
|
}
|
||||||
|
|
||||||
|
includeFunc := includeByPattern(patterns)
|
||||||
|
return func(item string) (matched bool, childMayMatch bool) {
|
||||||
|
return includeFunc(strings.ToLower(item))
|
||||||
|
}
|
||||||
|
}
|
59
cmd/restic/include_test.go
Normal file
59
cmd/restic/include_test.go
Normal file
@ -0,0 +1,59 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestIncludeByPattern(t *testing.T) {
|
||||||
|
var tests = []struct {
|
||||||
|
filename string
|
||||||
|
include bool
|
||||||
|
}{
|
||||||
|
{filename: "/home/user/foo.go", include: true},
|
||||||
|
{filename: "/home/user/foo.c", include: false},
|
||||||
|
{filename: "/home/user/foobar", include: false},
|
||||||
|
{filename: "/home/user/foobar/x", include: false},
|
||||||
|
{filename: "/home/user/README", include: false},
|
||||||
|
{filename: "/home/user/README.md", include: true},
|
||||||
|
}
|
||||||
|
|
||||||
|
patterns := []string{"*.go", "README.md"}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.filename, func(t *testing.T) {
|
||||||
|
includeFunc := includeByPattern(patterns)
|
||||||
|
matched, _ := includeFunc(tc.filename)
|
||||||
|
if matched != tc.include {
|
||||||
|
t.Fatalf("wrong result for filename %v: want %v, got %v",
|
||||||
|
tc.filename, tc.include, matched)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestIncludeByInsensitivePattern(t *testing.T) {
|
||||||
|
var tests = []struct {
|
||||||
|
filename string
|
||||||
|
include bool
|
||||||
|
}{
|
||||||
|
{filename: "/home/user/foo.GO", include: true},
|
||||||
|
{filename: "/home/user/foo.c", include: false},
|
||||||
|
{filename: "/home/user/foobar", include: false},
|
||||||
|
{filename: "/home/user/FOObar/x", include: false},
|
||||||
|
{filename: "/home/user/README", include: false},
|
||||||
|
{filename: "/home/user/readme.MD", include: true},
|
||||||
|
}
|
||||||
|
|
||||||
|
patterns := []string{"*.go", "README.md"}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.filename, func(t *testing.T) {
|
||||||
|
includeFunc := includeByInsensitivePattern(patterns)
|
||||||
|
matched, _ := includeFunc(tc.filename)
|
||||||
|
if matched != tc.include {
|
||||||
|
t.Fatalf("wrong result for filename %v: want %v, got %v",
|
||||||
|
tc.filename, tc.include, matched)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
@ -70,30 +70,64 @@ func TestRestoreFailsWhenUsingInvalidPatterns(t *testing.T) {
|
|||||||
var err error
|
var err error
|
||||||
|
|
||||||
// Test --exclude
|
// Test --exclude
|
||||||
err = testRunRestoreAssumeFailure("latest", RestoreOptions{Exclude: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}, env.gopts)
|
err = testRunRestoreAssumeFailure("latest", RestoreOptions{excludePatternOptions: excludePatternOptions{Excludes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts)
|
||||||
|
|
||||||
rtest.Equals(t, `Fatal: --exclude: invalid pattern(s) provided:
|
rtest.Equals(t, `Fatal: --exclude: invalid pattern(s) provided:
|
||||||
*[._]log[.-][0-9]
|
*[._]log[.-][0-9]
|
||||||
!*[._]log[.-][0-9]`, err.Error())
|
!*[._]log[.-][0-9]`, err.Error())
|
||||||
|
|
||||||
// Test --iexclude
|
// Test --iexclude
|
||||||
err = testRunRestoreAssumeFailure("latest", RestoreOptions{InsensitiveExclude: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}, env.gopts)
|
err = testRunRestoreAssumeFailure("latest", RestoreOptions{excludePatternOptions: excludePatternOptions{InsensitiveExcludes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts)
|
||||||
|
|
||||||
rtest.Equals(t, `Fatal: --iexclude: invalid pattern(s) provided:
|
rtest.Equals(t, `Fatal: --iexclude: invalid pattern(s) provided:
|
||||||
*[._]log[.-][0-9]
|
*[._]log[.-][0-9]
|
||||||
!*[._]log[.-][0-9]`, err.Error())
|
!*[._]log[.-][0-9]`, err.Error())
|
||||||
|
|
||||||
// Test --include
|
// Test --include
|
||||||
err = testRunRestoreAssumeFailure("latest", RestoreOptions{Include: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}, env.gopts)
|
err = testRunRestoreAssumeFailure("latest", RestoreOptions{includePatternOptions: includePatternOptions{Includes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts)
|
||||||
|
|
||||||
rtest.Equals(t, `Fatal: --include: invalid pattern(s) provided:
|
rtest.Equals(t, `Fatal: --include: invalid pattern(s) provided:
|
||||||
*[._]log[.-][0-9]
|
*[._]log[.-][0-9]
|
||||||
!*[._]log[.-][0-9]`, err.Error())
|
!*[._]log[.-][0-9]`, err.Error())
|
||||||
|
|
||||||
// Test --iinclude
|
// Test --iinclude
|
||||||
err = testRunRestoreAssumeFailure("latest", RestoreOptions{InsensitiveInclude: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}, env.gopts)
|
err = testRunRestoreAssumeFailure("latest", RestoreOptions{includePatternOptions: includePatternOptions{InsensitiveIncludes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts)
|
||||||
|
|
||||||
rtest.Equals(t, `Fatal: --iinclude: invalid pattern(s) provided:
|
rtest.Equals(t, `Fatal: --iinclude: invalid pattern(s) provided:
|
||||||
*[._]log[.-][0-9]
|
*[._]log[.-][0-9]
|
||||||
!*[._]log[.-][0-9]`, err.Error())
|
!*[._]log[.-][0-9]`, err.Error())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRestoreFailsWhenUsingInvalidPatternsFromFile(t *testing.T) {
|
||||||
|
env, cleanup := withTestEnvironment(t)
|
||||||
|
defer cleanup()
|
||||||
|
|
||||||
|
testRunInit(t, env.gopts)
|
||||||
|
|
||||||
|
// Create an include file with some invalid patterns
|
||||||
|
patternsFile := env.base + "/patternsFile"
|
||||||
|
fileErr := os.WriteFile(patternsFile, []byte("*.go\n*[._]log[.-][0-9]\n!*[._]log[.-][0-9]"), 0644)
|
||||||
|
if fileErr != nil {
|
||||||
|
t.Fatalf("Could not write include file: %v", fileErr)
|
||||||
|
}
|
||||||
|
|
||||||
|
err := testRunRestoreAssumeFailure("latest", RestoreOptions{includePatternOptions: includePatternOptions{IncludeFiles: []string{patternsFile}}}, env.gopts)
|
||||||
|
rtest.Equals(t, `Fatal: --include-file: invalid pattern(s) provided:
|
||||||
|
*[._]log[.-][0-9]
|
||||||
|
!*[._]log[.-][0-9]`, err.Error())
|
||||||
|
|
||||||
|
err = testRunRestoreAssumeFailure("latest", RestoreOptions{excludePatternOptions: excludePatternOptions{ExcludeFiles: []string{patternsFile}}}, env.gopts)
|
||||||
|
rtest.Equals(t, `Fatal: --exclude-file: invalid pattern(s) provided:
|
||||||
|
*[._]log[.-][0-9]
|
||||||
|
!*[._]log[.-][0-9]`, err.Error())
|
||||||
|
|
||||||
|
err = testRunRestoreAssumeFailure("latest", RestoreOptions{includePatternOptions: includePatternOptions{InsensitiveIncludeFiles: []string{patternsFile}}}, env.gopts)
|
||||||
|
rtest.Equals(t, `Fatal: --iinclude-file: invalid pattern(s) provided:
|
||||||
|
*[._]log[.-][0-9]
|
||||||
|
!*[._]log[.-][0-9]`, err.Error())
|
||||||
|
|
||||||
|
err = testRunRestoreAssumeFailure("latest", RestoreOptions{excludePatternOptions: excludePatternOptions{InsensitiveExcludeFiles: []string{patternsFile}}}, env.gopts)
|
||||||
|
rtest.Equals(t, `Fatal: --iexclude-file: invalid pattern(s) provided:
|
||||||
|
*[._]log[.-][0-9]
|
||||||
|
!*[._]log[.-][0-9]`, err.Error())
|
||||||
|
}
|
||||||
|
@ -68,6 +68,9 @@ There are case insensitive variants of ``--exclude`` and ``--include`` called
|
|||||||
``--iexclude`` and ``--iinclude``. These options will behave the same way but
|
``--iexclude`` and ``--iinclude``. These options will behave the same way but
|
||||||
ignore the casing of paths.
|
ignore the casing of paths.
|
||||||
|
|
||||||
|
There are also ``--include-file``, ``--exclude-file``, ``--iinclude-file`` and
|
||||||
|
``--iexclude-file`` flags that read the include and exclude patterns from a file.
|
||||||
|
|
||||||
Restoring symbolic links on windows is only possible when the user has
|
Restoring symbolic links on windows is only possible when the user has
|
||||||
``SeCreateSymbolicLinkPrivilege`` privilege or is running as admin. This is a
|
``SeCreateSymbolicLinkPrivilege`` privilege or is running as admin. This is a
|
||||||
restriction of windows not restic.
|
restriction of windows not restic.
|
||||||
|
Loading…
Reference in New Issue
Block a user