Use more fnmatch-like matcher in .stignore (fixes #426)

fnmatch/fnmatch.go

@@ -0,0 +1,60 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+package fnmatch
+
+import (
+	"path/filepath"
+	"regexp"
+	"runtime"
+	"strings"
+)
+
+const (
+	FNM_NOESCAPE = (1 << iota)
+	FNM_PATHNAME
+	FNM_CASEFOLD
+)
+
+func Convert(pattern string, flags int) (*regexp.Regexp, error) {
+	if runtime.GOOS == "windows" {
+		flags |= FNM_NOESCAPE
+		pattern = filepath.FromSlash(pattern)
+	}
+
+	any := "."
+	if flags&FNM_PATHNAME != 0 {
+		any = "[^/]"
+	}
+	if flags&FNM_NOESCAPE != 0 {
+		pattern = strings.Replace(pattern, "\\", "\\\\", -1)
+	} else {
+		pattern = strings.Replace(pattern, "\\*", "[:escapedstar:]", -1)
+		pattern = strings.Replace(pattern, "\\?", "[:escapedques:]", -1)
+		pattern = strings.Replace(pattern, "\\.", "[:escapeddot:]", -1)
+	}
+	pattern = strings.Replace(pattern, ".", "\\.", -1)
+	pattern = strings.Replace(pattern, "**", "[:doublestar:]", -1)
+	pattern = strings.Replace(pattern, "*", any+"*", -1)
+	pattern = strings.Replace(pattern, "[:doublestar:]", ".*", -1)
+	pattern = strings.Replace(pattern, "?", any, -1)
+	pattern = strings.Replace(pattern, "[:escapedstar:]", "\\*", -1)
+	pattern = strings.Replace(pattern, "[:escapedques:]", "\\?", -1)
+	pattern = strings.Replace(pattern, "[:escapeddot:]", "\\.", -1)
+	pattern = "^" + pattern + "$"
+	if flags&FNM_CASEFOLD != 0 {
+		pattern = "(?i)" + pattern
+	}
+	return regexp.Compile(pattern)
+}
+
+// Matches the pattern against the string, with the given flags,
+// and returns true if the match is successful.
+func Match(pattern, s string, flags int) (bool, error) {
+	exp, err := Convert(pattern, flags)
+	if err != nil {
+		return false, err
+	}
+	return exp.MatchString(s), nil
+}

fnmatch/fnmatch_test.go

@@ -0,0 +1,74 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+package fnmatch
+
+import (
+	"testing"
+)
+
+var testCases = []struct {
+	pat   string
+	name  string
+	flags int
+	match bool
+}{
+	{"", "", 0, true},
+	{"*", "", 0, true},
+	{"*", "foo", 0, true},
+	{"*", "bar", 0, true},
+	{"*", "*", 0, true},
+	{"**", "f", 0, true},
+	{"**", "foo.txt", 0, true},
+	{"*.*", "foo.txt", 0, true},
+	{"foo*.txt", "foobar.txt", 0, true},
+	{"foo.txt", "foo.txt", 0, true},
+	{"foo\\.txt", "foo.txt", 0, true},
+	{"foo\\*.txt", "foo*.txt", 0, true},
+	{"foo\\.txt", "foo.txt", FNM_NOESCAPE, false},
+
+	{"foo.txt", "bar/foo.txt", 0, false},
+	{"*/foo.txt", "bar/foo.txt", 0, true},
+	{"f?o.txt", "foo.txt", 0, true},
+	{"f?o.txt", "fooo.txt", 0, false},
+	{"f[ab]o.txt", "foo.txt", 0, false},
+	{"f[ab]o.txt", "fao.txt", 0, true},
+	{"f[ab]o.txt", "fbo.txt", 0, true},
+	{"f[ab]o.txt", "fco.txt", 0, false},
+	{"f[ab]o.txt", "fabo.txt", 0, false},
+	{"f[ab]o.txt", "f[ab]o.txt", 0, false},
+	{"f\\[ab\\]o.txt", "f[ab]o.txt", 0, true},
+	{"f\\[ab\\]o.txt", "f[ab]o.txt", FNM_NOESCAPE, false},
+	{"f\\\\\\[ab\\\\\\]o.txt", "f\\[ab\\]o.txt", 0, true},
+
+	{"*foo.txt", "bar/foo.txt", 0, true},
+	{"*foo.txt", "bar/foo.txt", FNM_PATHNAME, false},
+	{"*/foo.txt", "bar/foo.txt", 0, true},
+	{"*/foo.txt", "bar/foo.txt", FNM_PATHNAME, true},
+	{"*/foo.txt", "bar/baz/foo.txt", 0, true},
+	{"*/foo.txt", "bar/baz/foo.txt", FNM_PATHNAME, false},
+	{"**/foo.txt", "bar/baz/foo.txt", 0, true},
+	{"**/foo.txt", "bar/baz/foo.txt", FNM_PATHNAME, true},
+
+	{"foo.txt", "foo.TXT", 0, false},
+	{"foo.txt", "foo.TXT", FNM_CASEFOLD, true},
+}
+
+func TestMatch(t *testing.T) {
+	for _, tc := range testCases {
+		if m, err := Match(tc.pat, tc.name, tc.flags); m != tc.match {
+			if err != nil {
+				t.Error(err)
+			} else {
+				t.Errorf("Match(%q, %q, %d) != %v", tc.pat, tc.name, tc.flags, tc.match)
+			}
+		}
+	}
+}
+
+func TestInvalid(t *testing.T) {
+	if _, err := Match("foo[bar", "...", 0); err == nil {
+		t.Error("Unexpected nil error")
+	}
+}

integration/test-folders.sh

@@ -75,6 +75,7 @@ testConvergence() {
 	fi
 }
 
+chmod -R +w s? s??-?
 rm -rf s? s??-?
 rm -rf f?/*.idx.gz f?/index
 

integration/test-http.sh

@@ -19,6 +19,7 @@ echo Building
 go build http.go
 
 echo Starting
+chmod -R +w s1 s2
 rm -rf s1 s2 h1/index h2/index
 syncthing -home h1 > 1.out 2>&1 &
 syncthing -home h2 > 2.out 2>&1 &

integration/test-merge.sh

@@ -114,6 +114,7 @@ alterFiles() {
 }
 
 rm -rf h?/*.idx.gz h?/index
+chmod -R +w s? s??-? s4d
 rm -rf s? s??-? s4d
 
 echo "Setting up files..."

model/model.go

@@ -800,7 +800,7 @@ func (m *Model) ScanRepoSub(repo, sub string) error {
 	}
 
 	m.setState(repo, RepoScanning)
-	fchan, _, err := w.Walk()
+	fchan, err := w.Walk()
 
 	if err != nil {
 		return err

scanner/walk.go

@@ -5,17 +5,19 @@
 package scanner
 
 import (
-	"bytes"
+	"bufio"
 	"errors"
-	"fmt"
+	"io"
-	"io/ioutil"
 	"os"
+	"path"
 	"path/filepath"
+	"regexp"
 	"runtime"
 	"strings"
 
 	"code.google.com/p/go.text/unicode/norm"
 
+	"github.com/syncthing/syncthing/fnmatch"
 	"github.com/syncthing/syncthing/lamport"
 	"github.com/syncthing/syncthing/protocol"
 )
@@ -53,29 +55,30 @@ type CurrentFiler interface {
 
 // Walk returns the list of files found in the local repository by scanning the
 // file system. Files are blockwise hashed.
-func (w *Walker) Walk() (chan protocol.FileInfo, map[string][]string, error) {
+func (w *Walker) Walk() (chan protocol.FileInfo, error) {
 	if debug {
 		l.Debugln("Walk", w.Dir, w.Sub, w.BlockSize, w.IgnoreFile)
 	}
 
 	err := checkDir(w.Dir)
 	if err != nil {
-		return nil, nil, err
+		return nil, err
 	}
 
-	ignore := make(map[string][]string)
 	files := make(chan protocol.FileInfo)
 	hashedFiles := make(chan protocol.FileInfo)
 	newParallelHasher(w.Dir, w.BlockSize, runtime.NumCPU(), hashedFiles, files)
-	hashFiles := w.walkAndHashFiles(files, ignore)
 
+	var ignores []*regexp.Regexp
 	go func() {
-		filepath.Walk(w.Dir, w.loadIgnoreFiles(w.Dir, ignore))
+		filepath.Walk(w.Dir, w.loadIgnoreFiles(w.Dir, &ignores))
+
+		hashFiles := w.walkAndHashFiles(files, ignores)
 		filepath.Walk(filepath.Join(w.Dir, w.Sub), hashFiles)
 		close(files)
 	}()
 
-	return hashedFiles, ignore, nil
+	return hashedFiles, nil
 }
 
 // CleanTempFiles removes all files that match the temporary filename pattern.
@@ -83,7 +86,7 @@ func (w *Walker) CleanTempFiles() {
 	filepath.Walk(w.Dir, w.cleanTempFile)
 }
 
-func (w *Walker) loadIgnoreFiles(dir string, ign map[string][]string) filepath.WalkFunc {
+func (w *Walker) loadIgnoreFiles(dir string, ignores *[]*regexp.Regexp) filepath.WalkFunc {
 	return func(p string, info os.FileInfo, err error) error {
 		if err != nil {
 			return nil
@@ -96,23 +99,78 @@ func (w *Walker) loadIgnoreFiles(dir string, ign map[string][]string) filepath.W
 
 		if pn, sn := filepath.Split(rn); sn == w.IgnoreFile {
 			pn := filepath.Clean(pn)
-			bs, _ := ioutil.ReadFile(p)
+			dirIgnores := loadIgnoreFile(p, pn)
-			lines := bytes.Split(bs, []byte("\n"))
+			*ignores = append(*ignores, dirIgnores...)
-			var patterns []string
-			for _, line := range lines {
-				lineStr := strings.TrimSpace(string(line))
-				if len(lineStr) > 0 {
-					patterns = append(patterns, lineStr)
-				}
-			}
-			ign[pn] = patterns
 		}
 
 		return nil
 	}
 }
 
-func (w *Walker) walkAndHashFiles(fchan chan protocol.FileInfo, ign map[string][]string) filepath.WalkFunc {
+func loadIgnoreFile(ignFile, base string) []*regexp.Regexp {
+	fd, err := os.Open(ignFile)
+	if err != nil {
+		return nil
+	}
+	defer fd.Close()
+	return parseIgnoreFile(fd, base)
+}
+
+func parseIgnoreFile(fd io.Reader, base string) []*regexp.Regexp {
+	var exps []*regexp.Regexp
+	scanner := bufio.NewScanner(fd)
+	for scanner.Scan() {
+		line := strings.TrimSpace(scanner.Text())
+		if line == "" {
+			continue
+		}
+
+		if strings.HasPrefix(line, "/") {
+			// Pattern is rooted in the current dir only
+			exp, err := fnmatch.Convert(path.Join(base, line[1:]), fnmatch.FNM_PATHNAME)
+			if err != nil {
+				l.Warnf("Invalid pattern %q in ignore file", line)
+				continue
+			}
+			exps = append(exps, exp)
+		} else if strings.HasPrefix(line, "**/") {
+			// Add the pattern as is, and without **/ so it matches in current dir
+			exp, err := fnmatch.Convert(line, fnmatch.FNM_PATHNAME)
+			if err != nil {
+				l.Warnf("Invalid pattern %q in ignore file", line)
+				continue
+			}
+			exps = append(exps, exp)
+
+			exp, err = fnmatch.Convert(path.Join(base, line[3:]), fnmatch.FNM_PATHNAME)
+			if err != nil {
+				l.Warnf("Invalid pattern %q in ignore file", line)
+				continue
+			}
+			exps = append(exps, exp)
+		} else {
+			// Path name or pattern, add it so it matches files both in
+			// current directory and subdirs.
+			exp, err := fnmatch.Convert(path.Join(base, line), fnmatch.FNM_PATHNAME)
+			if err != nil {
+				l.Warnf("Invalid pattern %q in ignore file", line)
+				continue
+			}
+			exps = append(exps, exp)
+
+			exp, err = fnmatch.Convert(path.Join(base, "**", line), fnmatch.FNM_PATHNAME)
+			if err != nil {
+				l.Warnf("Invalid pattern %q in ignore file", line)
+				continue
+			}
+			exps = append(exps, exp)
+		}
+	}
+
+	return exps
+}
+
+func (w *Walker) walkAndHashFiles(fchan chan protocol.FileInfo, ignores []*regexp.Regexp) filepath.WalkFunc {
 	return func(p string, info os.FileInfo, err error) error {
 		if err != nil {
 			if debug {
@@ -141,7 +199,7 @@ func (w *Walker) walkAndHashFiles(fchan chan protocol.FileInfo, ign map[string][
 			return nil
 		}
 
-		if sn := filepath.Base(rn); sn == w.IgnoreFile || sn == ".stversions" || w.ignoreFile(ign, rn) {
+		if sn := filepath.Base(rn); sn == w.IgnoreFile || sn == ".stversions" || w.ignoreFile(ignores, rn) {
 			// An ignored file
 			if debug {
 				l.Debugln("ignored:", rn)
@@ -225,15 +283,13 @@ func (w *Walker) cleanTempFile(path string, info os.FileInfo, err error) error {
 	return nil
 }
 
-func (w *Walker) ignoreFile(patterns map[string][]string, file string) bool {
+func (w *Walker) ignoreFile(patterns []*regexp.Regexp, file string) bool {
-	first, last := filepath.Split(file)
+	for _, pattern := range patterns {
-	for prefix, pats := range patterns {
+		if pattern.MatchString(file) {
-		if prefix == "." || prefix == first || strings.HasPrefix(first, fmt.Sprintf("%s%c", prefix, os.PathSeparator)) {
+			if debug {
-			for _, pattern := range pats {
+				l.Debugf("%q matches %v", file, pattern)
-				if match, _ := filepath.Match(pattern, last); match || pattern == last {
-					return true
-				}
 			}
+			return true
 		}
 	}
 	return false

scanner/walk_test.go

@@ -5,10 +5,9 @@
 package scanner
 
 import (
+	"bytes"
 	"fmt"
 	"path/filepath"
-	"reflect"
-	"runtime"
 	"sort"
 	"testing"
 	"time"
@@ -38,7 +37,7 @@ func TestWalkSub(t *testing.T) {
 		BlockSize:  128 * 1024,
 		IgnoreFile: ".stignore",
 	}
-	fchan, _, err := w.Walk()
+	fchan, err := w.Walk()
 	var files []protocol.FileInfo
 	for f := range fchan {
 		files = append(files, f)
@@ -61,7 +60,7 @@ func TestWalk(t *testing.T) {
 		BlockSize:  128 * 1024,
 		IgnoreFile: ".stignore",
 	}
-	fchan, ignores, err := w.Walk()
+	fchan, err := w.Walk()
 	var files []protocol.FileInfo
 	for f := range fchan {
 		files = append(files, f)
@@ -95,10 +94,6 @@ func TestWalk(t *testing.T) {
 			t.Errorf("Unrealistic modtime %d for test %d", mt, i)
 		}
 	}
-
-	if !reflect.DeepEqual(ignores, correctIgnores) {
-		t.Errorf("Incorrect ignores\n  %v\n  %v", correctIgnores, ignores)
-	}
 }
 
 func TestWalkError(t *testing.T) {
@@ -107,7 +102,7 @@ func TestWalkError(t *testing.T) {
 		BlockSize:  128 * 1024,
 		IgnoreFile: ".stignore",
 	}
-	_, _, err := w.Walk()
+	_, err := w.Walk()
 
 	if err == nil {
 		t.Error("no error from missing directory")
@@ -118,7 +113,7 @@ func TestWalkError(t *testing.T) {
 		BlockSize:  128 * 1024,
 		IgnoreFile: ".stignore",
 	}
-	_, _, err = w.Walk()
+	_, err = w.Walk()
 
 	if err == nil {
 		t.Error("no error from non-directory")
@@ -126,29 +121,41 @@ func TestWalkError(t *testing.T) {
 }
 
 func TestIgnore(t *testing.T) {
-	pattern := "q\\[abc\\]y"
+	patStr := bytes.NewBufferString(`
-	// On Windows, escaping is disabled.
+		t2
-	// Instead, '\\' is treated as path separator.
+		/t3
-	if runtime.GOOS == "windows" {
+		sub/dir/*
-		pattern = "q[abc]y"
+		*/other/test
-	}
+		**/deep
+	`)
+	patterns := parseIgnoreFile(patStr, "")
+
+	patStr = bytes.NewBufferString(`
+		bar
+		z*
+		q[abc]x
+	`)
+	patterns = append(patterns, parseIgnoreFile(patStr, "foo")...)
+
+	patStr = bytes.NewBufferString(`
+		quux
+		.*
+	`)
+	patterns = append(patterns, parseIgnoreFile(patStr, "foo/baz")...)
 
-	var patterns = map[string][]string{
-		".":   {"t2"},
-		"foo": {"bar", "z*", "q[abc]x", pattern},
-		filepath.Join("foo", "baz"): {"quux", ".*"},
-	}
 	var tests = []struct {
 		f string
 		r bool
 	}{
 		{filepath.Join("foo", "bar"), true},
+		{filepath.Join("t3"), true},
 		{filepath.Join("foofoo"), false},
 		{filepath.Join("foo", "quux"), false},
 		{filepath.Join("foo", "zuux"), true},
 		{filepath.Join("foo", "qzuux"), false},
 		{filepath.Join("foo", "baz", "t1"), false},
 		{filepath.Join("foo", "baz", "t2"), true},
+		{filepath.Join("foo", "baz", "t3"), false},
 		{filepath.Join("foo", "baz", "bar"), true},
 		{filepath.Join("foo", "baz", "quuxa"), false},
 		{filepath.Join("foo", "baz", "aquux"), false},
@@ -156,9 +163,14 @@ func TestIgnore(t *testing.T) {
 		{filepath.Join("foo", "baz", "zquux"), true},
 		{filepath.Join("foo", "baz", "quux"), true},
 		{filepath.Join("foo", "bazz", "quux"), false},
-		{filepath.Join("foo", "bazz", "q[abc]x"), true},
+		{filepath.Join("sub", "dir", "hej"), true},
-		{filepath.Join("foo", "bazz", "qax"), true},
+		{filepath.Join("deeper", "sub", "dir", "hej"), true},
-		{filepath.Join("foo", "bazz", "q[abc]y"), true},
+		{filepath.Join("other", "test"), false},
+		{filepath.Join("sub", "other", "test"), true},
+		{filepath.Join("deeper", "sub", "other", "test"), true},
+		{filepath.Join("deep"), true},
+		{filepath.Join("deeper", "deep"), true},
+		{filepath.Join("deeper", "deeper", "deep"), true},
 	}
 
 	w := Walker{}