From 2ee07ded2bfc5e6645daea360451204f5cdb9763 Mon Sep 17 00:00:00 2001 From: Vincent Bernat Date: Tue, 2 Jul 2019 21:36:23 +0200 Subject: [PATCH] filter: ability to use negative patterns This is quite similar to gitignore. If a pattern is suffixed by an exclamation mark and match a file that was previously matched by a regular pattern, the match is cancelled. Notably, this can be used with `--exclude-file` to cancel the exclusion of some files. Like for gitignore, once a directory is excluded, it is not possible to include files inside the directory. For example, a user wanting to only keep `*.c` in some directory should not use: ~/work !~/work/*.c But: ~/work/* !~/work/*.c I didn't write documentation or changelog entry. I would like to get feedback if this is the right approach for excluding/including files at will for backups. I use something like this as an exclude file to backup my home: $HOME/**/* !$HOME/Documents !$HOME/code !$HOME/.emacs.d !$HOME/games # [...] node_modules *~ *.o *.lo *.pyc # [...] $HOME/code/linux/* !$HOME/code/linux/.git # [...] There are some limitations for this change: - Patterns are not mixed accross methods: patterns from file are handled first and if a file is excluded with this method, it's not possible to reinclude it with `--exclude !something`. - Patterns starting with `!` are now interpreted as a negative pattern. I don't think anyone was relying on that. - The whole list of patterns is walked for each match. We may optimize later by exiting early if we know no pattern is starting with `!`. Fix #233 --- changelog/unreleased/issue-233 | 31 +++++++++++++++++++++++++++++++ doc/040_backup.rst | 22 ++++++++++++++++++++++ internal/filter/filter.go | 30 ++++++++++++++++++++---------- internal/filter/filter_test.go | 13 +++++++++++++ 4 files changed, 86 insertions(+), 10 deletions(-) create mode 100644 changelog/unreleased/issue-233 diff --git a/changelog/unreleased/issue-233 b/changelog/unreleased/issue-233 new file mode 100644 index 000000000..a1f9aa022 --- /dev/null +++ b/changelog/unreleased/issue-233 @@ -0,0 +1,31 @@ +Enhancement: Add negative patterns for include/exclude + +If a pattern is suffixed by an exclamation mark and match a file that +was previously matched by a regular pattern, the match is cancelled. +Notably, this can be used with `--exclude-file` to cancel the +exclusion of some files. + +It works similarly to `gitignore`, with the same limitation: once a +directory is excluded, it is not possible to include files inside the +directory. + +Example of use (as an exclude pattern for backup): + + $HOME/**/* + !$HOME/Documents + !$HOME/code + !$HOME/.emacs.d + !$HOME/games + # [...] + node_modules + *~ + *.o + *.lo + *.pyc + # [...] + $HOME/code/linux/* + !$HOME/code/linux/.git + # [...] + +https://github.com/restic/restic/issues/233 +https://github.com/restic/restic/pull/2311 diff --git a/doc/040_backup.rst b/doc/040_backup.rst index addcadef5..fa19a176b 100644 --- a/doc/040_backup.rst +++ b/doc/040_backup.rst @@ -289,6 +289,28 @@ On most Unixy shells, you can either quote or use backslashes. For example: * ``--exclude="foo bar star/foo.txt"`` * ``--exclude=foo\ bar\ star/foo.txt`` +If a pattern is suffixed by an exclamation mark and match a file that +was previously matched by a regular pattern, the match is cancelled. +It works similarly to ``gitignore``, with the same limitation: once a +directory is excluded, it is not possible to include files inside the +directory. Here is a complete example to backup a selection of +directories inside the home directory. It works by excluding any +directory, then selectively add back some of them. + +:: + + $HOME/**/* + !$HOME/Documents + !$HOME/code + !$HOME/.emacs.d + !$HOME/games + # [...] + node_modules + *~ + *.o + *.lo + *.pyc + By specifying the option ``--one-file-system`` you can instruct restic to only backup files from the file systems the initially specified files or directories reside on. In other words, it will prevent restic from crossing diff --git a/internal/filter/filter.go b/internal/filter/filter.go index d48b77ab5..ca8be7386 100644 --- a/internal/filter/filter.go +++ b/internal/filter/filter.go @@ -18,7 +18,8 @@ type patternPart struct { // Pattern represents a preparsed filter pattern type Pattern struct { - parts []patternPart + parts []patternPart + isNegated bool } func prepareStr(str string) ([]string, error) { @@ -29,6 +30,12 @@ func prepareStr(str string) ([]string, error) { } func preparePattern(patternStr string) Pattern { + var negate bool + if patternStr[0] == '!' { + negate = true + patternStr = patternStr[1:] + } + pathParts := splitPath(filepath.Clean(patternStr)) parts := make([]patternPart, len(pathParts)) for i, part := range pathParts { @@ -41,7 +48,7 @@ func preparePattern(patternStr string) Pattern { parts[i] = patternPart{part, isSimple} } - return Pattern{parts} + return Pattern{parts, negate} } // Split p into path components. Assuming p has been Cleaned, no component @@ -123,7 +130,7 @@ func childMatch(pattern Pattern, strs []string) (matched bool, err error) { } else { l = len(strs) } - return match(Pattern{pattern.parts[0:l]}, strs) + return match(Pattern{pattern.parts[0:l], pattern.isNegated}, strs) } func hasDoubleWildcard(list Pattern) (ok bool, pos int) { @@ -151,7 +158,7 @@ func match(pattern Pattern, strs []string) (matched bool, err error) { } newPat = append(newPat, pattern.parts[pos+1:]...) - matched, err := match(Pattern{newPat}, strs) + matched, err := match(Pattern{newPat, pattern.isNegated}, strs) if err != nil { return false, err } @@ -234,7 +241,9 @@ func ListWithChild(patterns []Pattern, str string) (matched bool, childMayMatch return list(patterns, true, str) } -// List returns true if str matches one of the patterns. Empty patterns are ignored. +// list returns true if str matches one of the patterns. Empty patterns are ignored. +// Patterns prefixed by "!" are negated: any matching file excluded by a previous pattern +// will become included again. func list(patterns []Pattern, checkChildMatches bool, str string) (matched bool, childMayMatch bool, err error) { if len(patterns) == 0 { return false, false, nil @@ -260,11 +269,12 @@ func list(patterns []Pattern, checkChildMatches bool, str string) (matched bool, c = true } - matched = matched || m - childMayMatch = childMayMatch || c - - if matched && childMayMatch { - return true, true, nil + if pat.isNegated { + matched = matched && !m + childMayMatch = childMayMatch && !m + } else { + matched = matched || m + childMayMatch = childMayMatch || c } } diff --git a/internal/filter/filter_test.go b/internal/filter/filter_test.go index 149e8cf16..72ed323f8 100644 --- a/internal/filter/filter_test.go +++ b/internal/filter/filter_test.go @@ -259,7 +259,20 @@ var filterListTests = []struct { {[]string{"/*/*/bar/test.*"}, "/foo/bar/bar", false, true}, {[]string{"/*/*/bar/test.*", "*.go"}, "/foo/bar/test.go", true, true}, {[]string{"", "*.c"}, "/foo/bar/test.go", false, true}, + {[]string{"!**", "*.go"}, "/foo/bar/test.go", true, true}, + {[]string{"!**", "*.c"}, "/foo/bar/test.go", false, true}, + {[]string{"/foo/*/test.*", "!*.c"}, "/foo/bar/test.c", false, false}, + {[]string{"/foo/*/test.*", "!*.c"}, "/foo/bar/test.go", true, true}, + {[]string{"/foo/*/*", "!test.*", "*.c"}, "/foo/bar/test.go", false, true}, + {[]string{"/foo/*/*", "!test.*", "*.c"}, "/foo/bar/test.c", true, true}, + {[]string{"/foo/*/*", "!test.*", "*.c"}, "/foo/bar/file.go", true, true}, + {[]string{"/**/*", "!/foo", "/foo/*", "!/foo/bar"}, "/foo/other/test.go", true, true}, + {[]string{"/**/*", "!/foo", "/foo/*", "!/foo/bar"}, "/foo/bar", false, false}, + {[]string{"/**/*", "!/foo", "/foo/*", "!/foo/bar"}, "/foo/bar/test.go", false, false}, + {[]string{"/**/*", "!/foo", "/foo/*", "!/foo/bar"}, "/foo/bar/test.go/child", false, false}, + {[]string{"/**/*", "!/foo", "/foo/*", "!/foo/bar", "/foo/bar/test*"}, "/foo/bar/test.go/child", true, true}, {[]string{"/foo/bar/*"}, "/foo", false, true}, + {[]string{"/foo/bar/*", "!/foo/bar/[a-m]*"}, "/foo", false, true}, {[]string{"/foo/**/test.c"}, "/foo/bar/foo/bar/test.c", true, true}, {[]string{"/foo/*/test.c"}, "/foo/bar/foo/bar/test.c", false, false}, }