From bcc3bddcf43ae008de314485dd5bc27ad65b369e Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Wed, 7 Oct 2020 19:46:41 +0200 Subject: [PATCH] filter: only check whether a child path could match when necessary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When checking excludes there is no need to test whether a child path could also match the pattern, as it is by definition excluded. Previously childMayMatch was calculated but then discarded. For simple absolute paths this can account for half the time spent for checking pattern matches. name old time/op new time/op delta FilterPatterns/Relative-4 23.3ms ± 9% 21.7ms ± 6% -6.68% (p=0.004 n=10+10) FilterPatterns/Absolute-4 13.9ms ± 7% 10.0ms ± 5% -27.61% (p=0.000 n=10+10) FilterPatterns/Wildcard-4 51.4ms ± 7% 47.0ms ± 7% -8.51% (p=0.001 n=9+9) FilterPatterns/ManyNoMatch-4 551ms ± 9% 190ms ± 1% -65.41% (p=0.000 n=10+8) name old alloc/op new alloc/op delta FilterPatterns/Relative-4 3.57MB ± 0% 3.57MB ± 0% ~ (p=0.665 n=10+9) FilterPatterns/Absolute-4 3.57MB ± 0% 3.57MB ± 0% ~ (p=0.480 n=9+10) FilterPatterns/Wildcard-4 14.3MB ± 0% 14.3MB ± 0% ~ (p=0.431 n=9+10) FilterPatterns/ManyNoMatch-4 3.57MB ± 0% 3.57MB ± 0% ~ (all equal) name old allocs/op new allocs/op delta FilterPatterns/Relative-4 22.2k ± 0% 22.2k ± 0% ~ (all equal) FilterPatterns/Absolute-4 22.2k ± 0% 22.2k ± 0% ~ (all equal) FilterPatterns/Wildcard-4 88.7k ± 0% 88.7k ± 0% ~ (all equal) FilterPatterns/ManyNoMatch-4 22.2k ± 0% 22.2k ± 0% ~ (all equal) --- cmd/restic/cmd_restore.go | 8 ++++---- cmd/restic/exclude.go | 2 +- internal/filter/filter.go | 24 ++++++++++++++++++++---- internal/filter/filter_test.go | 12 ++++++------ 4 files changed, 31 insertions(+), 15 deletions(-) diff --git a/cmd/restic/cmd_restore.go b/cmd/restic/cmd_restore.go index 331f2d86f..da3818b63 100644 --- a/cmd/restic/cmd_restore.go +++ b/cmd/restic/cmd_restore.go @@ -143,12 +143,12 @@ func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error { excludePatterns := filter.ParsePatterns(opts.Exclude) insensitiveExcludePatterns := filter.ParsePatterns(opts.InsensitiveExclude) selectExcludeFilter := func(item string, dstpath string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool) { - matched, _, err := filter.List(excludePatterns, item) + matched, err := filter.List(excludePatterns, item) if err != nil { Warnf("error for exclude pattern: %v", err) } - matchedInsensitive, _, err := filter.List(insensitiveExcludePatterns, strings.ToLower(item)) + matchedInsensitive, err := filter.List(insensitiveExcludePatterns, strings.ToLower(item)) if err != nil { Warnf("error for iexclude pattern: %v", err) } @@ -166,12 +166,12 @@ func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error { includePatterns := filter.ParsePatterns(opts.Include) insensitiveIncludePatterns := filter.ParsePatterns(opts.InsensitiveInclude) selectIncludeFilter := func(item string, dstpath string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool) { - matched, childMayMatch, err := filter.List(includePatterns, item) + matched, childMayMatch, err := filter.ListWithChild(includePatterns, item) if err != nil { Warnf("error for include pattern: %v", err) } - matchedInsensitive, childMayMatchInsensitive, err := filter.List(insensitiveIncludePatterns, strings.ToLower(item)) + matchedInsensitive, childMayMatchInsensitive, err := filter.ListWithChild(insensitiveIncludePatterns, strings.ToLower(item)) if err != nil { Warnf("error for iexclude pattern: %v", err) } diff --git a/cmd/restic/exclude.go b/cmd/restic/exclude.go index 0295b00f1..db603c04e 100644 --- a/cmd/restic/exclude.go +++ b/cmd/restic/exclude.go @@ -76,7 +76,7 @@ type RejectFunc func(path string, fi os.FileInfo) bool func rejectByPattern(patterns []string) RejectByNameFunc { parsedPatterns := filter.ParsePatterns(patterns) return func(item string) bool { - matched, _, err := filter.List(parsedPatterns, item) + matched, err := filter.List(parsedPatterns, item) if err != nil { Warnf("error for exclude pattern: %v", err) } diff --git a/internal/filter/filter.go b/internal/filter/filter.go index 5eb119f49..90526ddcc 100644 --- a/internal/filter/filter.go +++ b/internal/filter/filter.go @@ -194,7 +194,18 @@ func ParsePatterns(patterns []string) []Pattern { } // List returns true if str matches one of the patterns. Empty patterns are ignored. -func List(patterns []Pattern, str string) (matched bool, childMayMatch bool, err error) { +func List(patterns []Pattern, str string) (matched bool, err error) { + matched, _, err = list(patterns, false, str) + return matched, err +} + +// ListWithChild returns true if str matches one of the patterns. Empty patterns are ignored. +func ListWithChild(patterns []Pattern, str string) (matched bool, childMayMatch bool, err error) { + return list(patterns, true, str) +} + +// List returns true if str matches one of the patterns. Empty patterns are ignored. +func list(patterns []Pattern, checkChildMatches bool, str string) (matched bool, childMayMatch bool, err error) { if len(patterns) == 0 { return false, false, nil } @@ -209,9 +220,14 @@ func List(patterns []Pattern, str string) (matched bool, childMayMatch bool, err return false, false, err } - c, err := childMatch(pat, strs) - if err != nil { - return false, false, err + var c bool + if checkChildMatches { + c, err = childMatch(pat, strs) + if err != nil { + return false, false, err + } + } else { + c = true } matched = matched || m diff --git a/internal/filter/filter_test.go b/internal/filter/filter_test.go index f7cf1dd82..9d487a582 100644 --- a/internal/filter/filter_test.go +++ b/internal/filter/filter_test.go @@ -260,7 +260,7 @@ var filterListTests = []struct { func TestList(t *testing.T) { for i, test := range filterListTests { patterns := filter.ParsePatterns(test.patterns) - match, _, err := filter.List(patterns, test.path) + match, err := filter.List(patterns, test.path) if err != nil { t.Errorf("test %d failed: expected no error for patterns %q, but error returned: %v", i, test.patterns, err) @@ -276,7 +276,7 @@ func TestList(t *testing.T) { func ExampleList() { patterns := filter.ParsePatterns([]string{"*.c", "*.go"}) - match, _, _ := filter.List(patterns, "/home/user/file.go") + match, _ := filter.List(patterns, "/home/user/file.go") fmt.Printf("match: %v\n", match) // Output: // match: true @@ -294,7 +294,7 @@ func TestInvalidStrs(t *testing.T) { } patterns := []string{"test"} - _, _, err = filter.List(filter.ParsePatterns(patterns), "") + _, err = filter.List(filter.ParsePatterns(patterns), "") if err == nil { t.Error("List accepted invalid path") } @@ -302,13 +302,13 @@ func TestInvalidStrs(t *testing.T) { func TestInvalidPattern(t *testing.T) { patterns := []string{"test/["} - _, _, err := filter.List(filter.ParsePatterns(patterns), "test/example") + _, err := filter.List(filter.ParsePatterns(patterns), "test/example") if err == nil { t.Error("List accepted invalid pattern") } patterns = []string{"test/**/["} - _, _, err = filter.List(filter.ParsePatterns(patterns), "test/example") + _, err = filter.List(filter.ParsePatterns(patterns), "test/example") if err == nil { t.Error("List accepted invalid pattern") } @@ -435,7 +435,7 @@ func BenchmarkFilterPatterns(b *testing.B) { for i := 0; i < b.N; i++ { c = 0 for _, line := range lines { - match, _, err := filter.List(test.patterns, line) + match, err := filter.List(test.patterns, line) if err != nil { b.Fatal(err) }