From 375c2a56deee2a53bf035c943ff31b4bd050890c Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Wed, 7 Oct 2020 14:39:51 +0200 Subject: [PATCH] filter: Parse filter patterns only once MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit name old time/op new time/op delta FilterPatterns/Relative-4 30.3ms ±10% 23.6ms ±20% -22.12% (p=0.000 n=10+10) FilterPatterns/Absolute-4 49.0ms ± 3% 32.3ms ± 8% -33.94% (p=0.000 n=8+10) FilterPatterns/Wildcard-4 345ms ± 9% 334ms ±17% ~ (p=0.315 n=10+10) FilterPatterns/ManyNoMatch-4 3.93s ± 2% 0.71s ± 7% -81.98% (p=0.000 n=9+10) name old alloc/op new alloc/op delta FilterPatterns/Relative-4 4.63MB ± 0% 3.57MB ± 0% -22.98% (p=0.000 n=9+9) FilterPatterns/Absolute-4 8.54MB ± 0% 3.57MB ± 0% -58.20% (p=0.000 n=10+10) FilterPatterns/Wildcard-4 146MB ± 0% 141MB ± 0% -2.93% (p=0.000 n=9+9) FilterPatterns/ManyNoMatch-4 907MB ± 0% 4MB ± 0% -99.61% (p=0.000 n=9+9) name old allocs/op new allocs/op delta FilterPatterns/Relative-4 66.6k ± 0% 22.2k ± 0% -66.67% (p=0.000 n=10+10) FilterPatterns/Absolute-4 88.7k ± 0% 22.2k ± 0% -75.00% (p=0.000 n=10+10) FilterPatterns/Wildcard-4 1.70M ± 0% 1.63M ± 0% -3.92% (p=0.000 n=10+10) FilterPatterns/ManyNoMatch-4 4.46M ± 0% 0.02M ± 0% -99.50% (p=0.000 n=10+10) --- cmd/restic/cmd_restore.go | 12 ++++++---- cmd/restic/exclude.go | 3 ++- internal/filter/filter.go | 41 +++++++++++++++++++++------------- internal/filter/filter_test.go | 28 ++++++++++++----------- 4 files changed, 50 insertions(+), 34 deletions(-) diff --git a/cmd/restic/cmd_restore.go b/cmd/restic/cmd_restore.go index e87bafbba..331f2d86f 100644 --- a/cmd/restic/cmd_restore.go +++ b/cmd/restic/cmd_restore.go @@ -140,13 +140,15 @@ func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error { return nil } + excludePatterns := filter.ParsePatterns(opts.Exclude) + insensitiveExcludePatterns := filter.ParsePatterns(opts.InsensitiveExclude) selectExcludeFilter := func(item string, dstpath string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool) { - matched, _, err := filter.List(opts.Exclude, item) + matched, _, err := filter.List(excludePatterns, item) if err != nil { Warnf("error for exclude pattern: %v", err) } - matchedInsensitive, _, err := filter.List(opts.InsensitiveExclude, strings.ToLower(item)) + matchedInsensitive, _, err := filter.List(insensitiveExcludePatterns, strings.ToLower(item)) if err != nil { Warnf("error for iexclude pattern: %v", err) } @@ -161,13 +163,15 @@ func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error { return selectedForRestore, childMayBeSelected } + includePatterns := filter.ParsePatterns(opts.Include) + insensitiveIncludePatterns := filter.ParsePatterns(opts.InsensitiveInclude) selectIncludeFilter := func(item string, dstpath string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool) { - matched, childMayMatch, err := filter.List(opts.Include, item) + matched, childMayMatch, err := filter.List(includePatterns, item) if err != nil { Warnf("error for include pattern: %v", err) } - matchedInsensitive, childMayMatchInsensitive, err := filter.List(opts.InsensitiveInclude, strings.ToLower(item)) + matchedInsensitive, childMayMatchInsensitive, err := filter.List(insensitiveIncludePatterns, strings.ToLower(item)) if err != nil { Warnf("error for iexclude pattern: %v", err) } diff --git a/cmd/restic/exclude.go b/cmd/restic/exclude.go index 8d5585cfc..0295b00f1 100644 --- a/cmd/restic/exclude.go +++ b/cmd/restic/exclude.go @@ -74,8 +74,9 @@ type RejectFunc func(path string, fi os.FileInfo) bool // rejectByPattern returns a RejectByNameFunc which rejects files that match // one of the patterns. func rejectByPattern(patterns []string) RejectByNameFunc { + parsedPatterns := filter.ParsePatterns(patterns) return func(item string) bool { - matched, _, err := filter.List(patterns, item) + matched, _, err := filter.List(parsedPatterns, item) if err != nil { Warnf("error for exclude pattern: %v", err) } diff --git a/internal/filter/filter.go b/internal/filter/filter.go index 795afd179..67aa70f3e 100644 --- a/internal/filter/filter.go +++ b/internal/filter/filter.go @@ -11,7 +11,8 @@ import ( // second argument. var ErrBadString = errors.New("filter.Match: string is empty") -type filterPattern []string +// Pattern represents a preparsed filter pattern +type Pattern []string func prepareStr(str string) ([]string, error) { if str == "" { @@ -26,7 +27,7 @@ func prepareStr(str string) ([]string, error) { return strings.Split(str, "/"), nil } -func preparePattern(pattern string) filterPattern { +func preparePattern(pattern string) Pattern { pattern = filepath.Clean(pattern) // convert file path separator to '/' @@ -87,7 +88,7 @@ func ChildMatch(pattern, str string) (matched bool, err error) { return childMatch(patterns, strs) } -func childMatch(patterns, strs []string) (matched bool, err error) { +func childMatch(patterns Pattern, strs []string) (matched bool, err error) { if patterns[0] != "" { // relative pattern can always be nested down return true, nil @@ -109,7 +110,7 @@ func childMatch(patterns, strs []string) (matched bool, err error) { return match(patterns[0:l], strs) } -func hasDoubleWildcard(list []string) (ok bool, pos int) { +func hasDoubleWildcard(list Pattern) (ok bool, pos int) { for i, item := range list { if item == "**" { return true, i @@ -119,11 +120,11 @@ func hasDoubleWildcard(list []string) (ok bool, pos int) { return false, 0 } -func match(patterns, strs []string) (matched bool, err error) { +func match(patterns Pattern, strs []string) (matched bool, err error) { if ok, pos := hasDoubleWildcard(patterns); ok { // gradually expand '**' into separate wildcards for i := 0; i <= len(strs)-len(patterns)+1; i++ { - newPat := make([]string, pos) + newPat := make(Pattern, pos) copy(newPat, patterns[:pos]) for k := 0; k < i; k++ { newPat = append(newPat, "*") @@ -169,9 +170,22 @@ func match(patterns, strs []string) (matched bool, err error) { return false, nil } -// List returns true if str matches one of the patterns. Empty patterns are -// ignored. -func List(patterns []string, str string) (matched bool, childMayMatch bool, err error) { +// ParsePatterns prepares a list of patterns for use with List. +func ParsePatterns(patterns []string) []Pattern { + patpat := make([]Pattern, 0) + for _, pat := range patterns { + if pat == "" { + continue + } + + pats := preparePattern(pat) + patpat = append(patpat, pats) + } + return patpat +} + +// List returns true if str matches one of the patterns. Empty patterns are ignored. +func List(patterns []Pattern, str string) (matched bool, childMayMatch bool, err error) { if len(patterns) == 0 { return false, false, nil } @@ -181,17 +195,12 @@ func List(patterns []string, str string) (matched bool, childMayMatch bool, err return false, false, err } for _, pat := range patterns { - if pat == "" { - continue - } - - pats := preparePattern(pat) - m, err := match(pats, strs) + m, err := match(pat, strs) if err != nil { return false, false, err } - c, err := childMatch(pats, strs) + c, err := childMatch(pat, strs) if err != nil { return false, false, err } diff --git a/internal/filter/filter_test.go b/internal/filter/filter_test.go index 6ff83a43b..f7cf1dd82 100644 --- a/internal/filter/filter_test.go +++ b/internal/filter/filter_test.go @@ -259,7 +259,8 @@ var filterListTests = []struct { func TestList(t *testing.T) { for i, test := range filterListTests { - match, _, err := filter.List(test.patterns, test.path) + patterns := filter.ParsePatterns(test.patterns) + match, _, err := filter.List(patterns, test.path) if err != nil { t.Errorf("test %d failed: expected no error for patterns %q, but error returned: %v", i, test.patterns, err) @@ -274,7 +275,8 @@ func TestList(t *testing.T) { } func ExampleList() { - match, _, _ := filter.List([]string{"*.c", "*.go"}, "/home/user/file.go") + patterns := filter.ParsePatterns([]string{"*.c", "*.go"}) + match, _, _ := filter.List(patterns, "/home/user/file.go") fmt.Printf("match: %v\n", match) // Output: // match: true @@ -292,7 +294,7 @@ func TestInvalidStrs(t *testing.T) { } patterns := []string{"test"} - _, _, err = filter.List(patterns, "") + _, _, err = filter.List(filter.ParsePatterns(patterns), "") if err == nil { t.Error("List accepted invalid path") } @@ -300,13 +302,13 @@ func TestInvalidStrs(t *testing.T) { func TestInvalidPattern(t *testing.T) { patterns := []string{"test/["} - _, _, err := filter.List(patterns, "test/example") + _, _, err := filter.List(filter.ParsePatterns(patterns), "test/example") if err == nil { t.Error("List accepted invalid pattern") } patterns = []string{"test/**/["} - _, _, err = filter.List(patterns, "test/example") + _, _, err = filter.List(filter.ParsePatterns(patterns), "test/example") if err == nil { t.Error("List accepted invalid pattern") } @@ -403,25 +405,25 @@ func BenchmarkFilterPatterns(b *testing.B) { } tests := []struct { name string - patterns []string + patterns []filter.Pattern matches uint }{ - {"Relative", []string{ + {"Relative", filter.ParsePatterns([]string{ "does-not-match", "sdk/*", "*.html", - }, 22185}, - {"Absolute", []string{ + }), 22185}, + {"Absolute", filter.ParsePatterns([]string{ "/etc", "/home/*/test", "/usr/share/doc/libreoffice/sdk/docs/java", - }, 150}, - {"Wildcard", []string{ + }), 150}, + {"Wildcard", filter.ParsePatterns([]string{ "/etc/**/example", "/home/**/test", "/usr/**/java", - }, 150}, - {"ManyNoMatch", modlines, 0}, + }), 150}, + {"ManyNoMatch", filter.ParsePatterns(modlines), 0}, } for _, test := range tests {