octoleo
/
restic
mirror of https://github.com/octoleo/restic.git
2
2
Fork 0
Code Issues Releases Wiki Activity

filter: Parse filter patterns only once 

name                          old time/op    new time/op    delta
FilterPatterns/Relative-4       30.3ms ±10%    23.6ms ±20%  -22.12%  (p=0.000 n=10+10)
FilterPatterns/Absolute-4       49.0ms ± 3%    32.3ms ± 8%  -33.94%  (p=0.000 n=8+10)
FilterPatterns/Wildcard-4        345ms ± 9%     334ms ±17%     ~     (p=0.315 n=10+10)
FilterPatterns/ManyNoMatch-4     3.93s ± 2%     0.71s ± 7%  -81.98%  (p=0.000 n=9+10)

name                          old alloc/op   new alloc/op   delta
FilterPatterns/Relative-4       4.63MB ± 0%    3.57MB ± 0%  -22.98%  (p=0.000 n=9+9)
FilterPatterns/Absolute-4       8.54MB ± 0%    3.57MB ± 0%  -58.20%  (p=0.000 n=10+10)
FilterPatterns/Wildcard-4        146MB ± 0%     141MB ± 0%   -2.93%  (p=0.000 n=9+9)
FilterPatterns/ManyNoMatch-4     907MB ± 0%       4MB ± 0%  -99.61%  (p=0.000 n=9+9)

name                          old allocs/op  new allocs/op  delta
FilterPatterns/Relative-4        66.6k ± 0%     22.2k ± 0%  -66.67%  (p=0.000 n=10+10)
FilterPatterns/Absolute-4        88.7k ± 0%     22.2k ± 0%  -75.00%  (p=0.000 n=10+10)
FilterPatterns/Wildcard-4        1.70M ± 0%     1.63M ± 0%   -3.92%  (p=0.000 n=10+10)
FilterPatterns/ManyNoMatch-4     4.46M ± 0%     0.02M ± 0%  -99.50%  (p=0.000 n=10+10)
This commit is contained in:
Michael Eischer 2020-10-07 14:39:51 +02:00
parent b8eacd1364
commit 375c2a56de
4 changed files with 50 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
cmd/restic/cmd_restore.go
View File

@ -140,13 +140,15 @@ func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error {
return nil
}
excludePatterns := filter.ParsePatterns(opts.Exclude)
insensitiveExcludePatterns := filter.ParsePatterns(opts.InsensitiveExclude)
selectExcludeFilter := func(item string, dstpath string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool) {
matched, _, err := filter.List(opts.Exclude, item)
matched, _, err := filter.List(excludePatterns, item)
if err != nil {
Warnf("error for exclude pattern: %v", err)
}
matchedInsensitive, _, err := filter.List(opts.InsensitiveExclude, strings.ToLower(item))
matchedInsensitive, _, err := filter.List(insensitiveExcludePatterns, strings.ToLower(item))
if err != nil {
Warnf("error for iexclude pattern: %v", err)
}
@ -161,13 +163,15 @@ func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error {
return selectedForRestore, childMayBeSelected
}
includePatterns := filter.ParsePatterns(opts.Include)
insensitiveIncludePatterns := filter.ParsePatterns(opts.InsensitiveInclude)
selectIncludeFilter := func(item string, dstpath string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool) {
matched, childMayMatch, err := filter.List(opts.Include, item)
matched, childMayMatch, err := filter.List(includePatterns, item)
if err != nil {
Warnf("error for include pattern: %v", err)
}
matchedInsensitive, childMayMatchInsensitive, err := filter.List(opts.InsensitiveInclude, strings.ToLower(item))
matchedInsensitive, childMayMatchInsensitive, err := filter.List(insensitiveIncludePatterns, strings.ToLower(item))
if err != nil {
Warnf("error for iexclude pattern: %v", err)
}

3
cmd/restic/exclude.go
View File

@ -74,8 +74,9 @@ type RejectFunc func(path string, fi os.FileInfo) bool
// rejectByPattern returns a RejectByNameFunc which rejects files that match
// one of the patterns.
func rejectByPattern(patterns []string) RejectByNameFunc {
parsedPatterns := filter.ParsePatterns(patterns)
return func(item string) bool {
matched, _, err := filter.List(patterns, item)
matched, _, err := filter.List(parsedPatterns, item)
if err != nil {
Warnf("error for exclude pattern: %v", err)
}

41
internal/filter/filter.go
View File

@ -11,7 +11,8 @@ import (
// second argument.
var ErrBadString = errors.New("filter.Match: string is empty")
type filterPattern []string
// Pattern represents a preparsed filter pattern
type Pattern []string
func prepareStr(str string) ([]string, error) {
if str == "" {
@ -26,7 +27,7 @@ func prepareStr(str string) ([]string, error) {
return strings.Split(str, "/"), nil
}
func preparePattern(pattern string) filterPattern {
func preparePattern(pattern string) Pattern {
pattern = filepath.Clean(pattern)
// convert file path separator to '/'
@ -87,7 +88,7 @@ func ChildMatch(pattern, str string) (matched bool, err error) {
return childMatch(patterns, strs)
}
func childMatch(patterns, strs []string) (matched bool, err error) {
func childMatch(patterns Pattern, strs []string) (matched bool, err error) {
if patterns[0] != "" {
// relative pattern can always be nested down
return true, nil
@ -109,7 +110,7 @@ func childMatch(patterns, strs []string) (matched bool, err error) {
return match(patterns[0:l], strs)
}
func hasDoubleWildcard(list []string) (ok bool, pos int) {
func hasDoubleWildcard(list Pattern) (ok bool, pos int) {
for i, item := range list {
if item == "**" {
return true, i
@ -119,11 +120,11 @@ func hasDoubleWildcard(list []string) (ok bool, pos int) {
return false, 0
}
func match(patterns, strs []string) (matched bool, err error) {
func match(patterns Pattern, strs []string) (matched bool, err error) {
if ok, pos := hasDoubleWildcard(patterns); ok {
// gradually expand '**' into separate wildcards
for i := 0; i <= len(strs)-len(patterns)+1; i++ {
newPat := make([]string, pos)
newPat := make(Pattern, pos)
copy(newPat, patterns[:pos])
for k := 0; k < i; k++ {
newPat = append(newPat, "*")
@ -169,9 +170,22 @@ func match(patterns, strs []string) (matched bool, err error) {
return false, nil
}
// List returns true if str matches one of the patterns. Empty patterns are
// ignored.
func List(patterns []string, str string) (matched bool, childMayMatch bool, err error) {
// ParsePatterns prepares a list of patterns for use with List.
func ParsePatterns(patterns []string) []Pattern {
patpat := make([]Pattern, 0)
for _, pat := range patterns {
if pat == "" {
continue
}
pats := preparePattern(pat)
patpat = append(patpat, pats)
}
return patpat
}
// List returns true if str matches one of the patterns. Empty patterns are ignored.
func List(patterns []Pattern, str string) (matched bool, childMayMatch bool, err error) {
if len(patterns) == 0 {
return false, false, nil
}
@ -181,17 +195,12 @@ func List(patterns []string, str string) (matched bool, childMayMatch bool, err
return false, false, err
}
for _, pat := range patterns {
if pat == "" {
continue
}
pats := preparePattern(pat)
m, err := match(pats, strs)
m, err := match(pat, strs)
if err != nil {
return false, false, err
}
c, err := childMatch(pats, strs)
c, err := childMatch(pat, strs)
if err != nil {
return false, false, err
}

28
internal/filter/filter_test.go
View File

@ -259,7 +259,8 @@ var filterListTests = []struct {
func TestList(t *testing.T) {
for i, test := range filterListTests {
match, _, err := filter.List(test.patterns, test.path)
patterns := filter.ParsePatterns(test.patterns)
match, _, err := filter.List(patterns, test.path)
if err != nil {
t.Errorf("test %d failed: expected no error for patterns %q, but error returned: %v",
i, test.patterns, err)
@ -274,7 +275,8 @@ func TestList(t *testing.T) {
}
func ExampleList() {
match, _, _ := filter.List([]string{"*.c", "*.go"}, "/home/user/file.go")
patterns := filter.ParsePatterns([]string{"*.c", "*.go"})
match, _, _ := filter.List(patterns, "/home/user/file.go")
fmt.Printf("match: %v\n", match)
// Output:
// match: true
@ -292,7 +294,7 @@ func TestInvalidStrs(t *testing.T) {
}
patterns := []string{"test"}
_, _, err = filter.List(patterns, "")
_, _, err = filter.List(filter.ParsePatterns(patterns), "")
if err == nil {
t.Error("List accepted invalid path")
}
@ -300,13 +302,13 @@ func TestInvalidStrs(t *testing.T) {
func TestInvalidPattern(t *testing.T) {
patterns := []string{"test/["}
_, _, err := filter.List(patterns, "test/example")
_, _, err := filter.List(filter.ParsePatterns(patterns), "test/example")
if err == nil {
t.Error("List accepted invalid pattern")
}
patterns = []string{"test/**/["}
_, _, err = filter.List(patterns, "test/example")
_, _, err = filter.List(filter.ParsePatterns(patterns), "test/example")
if err == nil {
t.Error("List accepted invalid pattern")
}
@ -403,25 +405,25 @@ func BenchmarkFilterPatterns(b *testing.B) {
}
tests := []struct {
name string
patterns []string
patterns []filter.Pattern
matches uint
}{
{"Relative", []string{
{"Relative", filter.ParsePatterns([]string{
"does-not-match",
"sdk/*",
"*.html",
}, 22185},
{"Absolute", []string{
}), 22185},
{"Absolute", filter.ParsePatterns([]string{
"/etc",
"/home/*/test",
"/usr/share/doc/libreoffice/sdk/docs/java",
}, 150},
{"Wildcard", []string{
}), 150},
{"Wildcard", filter.ParsePatterns([]string{
"/etc/**/example",
"/home/**/test",
"/usr/**/java",
}, 150},
{"ManyNoMatch", modlines, 0},
}), 150},
{"ManyNoMatch", filter.ParsePatterns(modlines), 0},
}
for _, test := range tests {