Ignore files matching patterns in .stignore (fixes #7)

README.md

@@ -186,6 +186,15 @@ $ syncthing --gui 127.0.0.1:8080
 
 You then point your browser to the given address.
 
+Excluding Files
+---------------
+
+syncthing looks for files named `.stignore` while walking the
+repository. The file is expected to contain glob patterns of file names
+to ignore. Patterns are matched on file name only and apply to files in
+the same directory as the `.stignore` file and in directories lower down
+in the hierarchy.
+
 License
 =======
 

main.go

@@ -333,7 +333,7 @@ func connect(myID string, addr string, nodeAddrs map[string][]string, m *model.M
 }
 
 func updateLocalModel(m *model.Model) {
-	files := m.Walk(!opts.NoSymlinks)
+	files := m.FilteredWalk(!opts.NoSymlinks)
 	m.ReplaceLocal(files)
 	saveIndex(m)
 }

model/model_test.go

@@ -58,7 +58,7 @@ func init() {
 
 func TestUpdateLocal(t *testing.T) {
 	m := NewModel("testdata")
-	fs := m.Walk(false)
+	fs, _ := m.Walk(false)
 	m.ReplaceLocal(fs)
 
 	if len(m.need) > 0 {
@@ -100,7 +100,7 @@ func TestUpdateLocal(t *testing.T) {
 
 func TestRemoteUpdateExisting(t *testing.T) {
 	m := NewModel("testdata")
-	fs := m.Walk(false)
+	fs, _ := m.Walk(false)
 	m.ReplaceLocal(fs)
 
 	newFile := protocol.FileInfo{
@@ -117,7 +117,7 @@ func TestRemoteUpdateExisting(t *testing.T) {
 
 func TestRemoteAddNew(t *testing.T) {
 	m := NewModel("testdata")
-	fs := m.Walk(false)
+	fs, _ := m.Walk(false)
 	m.ReplaceLocal(fs)
 
 	newFile := protocol.FileInfo{
@@ -134,7 +134,7 @@ func TestRemoteAddNew(t *testing.T) {
 
 func TestRemoteUpdateOld(t *testing.T) {
 	m := NewModel("testdata")
-	fs := m.Walk(false)
+	fs, _ := m.Walk(false)
 	m.ReplaceLocal(fs)
 
 	oldTimeStamp := int64(1234)
@@ -152,7 +152,7 @@ func TestRemoteUpdateOld(t *testing.T) {
 
 func TestRemoteIndexUpdate(t *testing.T) {
 	m := NewModel("testdata")
-	fs := m.Walk(false)
+	fs, _ := m.Walk(false)
 	m.ReplaceLocal(fs)
 
 	foo := protocol.FileInfo{
@@ -185,7 +185,7 @@ func TestRemoteIndexUpdate(t *testing.T) {
 
 func TestDelete(t *testing.T) {
 	m := NewModel("testdata")
-	fs := m.Walk(false)
+	fs, _ := m.Walk(false)
 	m.ReplaceLocal(fs)
 
 	if l1, l2 := len(m.local), len(fs); l1 != l2 {
@@ -275,7 +275,7 @@ func TestDelete(t *testing.T) {
 
 func TestForgetNode(t *testing.T) {
 	m := NewModel("testdata")
-	fs := m.Walk(false)
+	fs, _ := m.Walk(false)
 	m.ReplaceLocal(fs)
 
 	if l1, l2 := len(m.local), len(fs); l1 != l2 {

model/walk.go

@@ -1,7 +1,9 @@
 package model
 
 import (
+	"bytes"
 	"fmt"
+	"io/ioutil"
 	"log"
 	"os"
 	"path"
@@ -35,7 +37,7 @@ func tempName(name string, modified int64) string {
 	return path.Join(tdir, tname)
 }
 
-func (m *Model) genWalker(res *[]File) filepath.WalkFunc {
+func (m *Model) genWalker(res *[]File, ign map[string][]string) filepath.WalkFunc {
 	return func(p string, info os.FileInfo, err error) error {
 		if err != nil {
 			return nil
@@ -45,12 +47,26 @@ func (m *Model) genWalker(res *[]File) filepath.WalkFunc {
 			return nil
 		}
 
-		if info.Mode()&os.ModeType == 0 {
-			rn, err := filepath.Rel(m.dir, p)
-			if err != nil {
-				return nil
-			}
+		rn, err := filepath.Rel(m.dir, p)
+		if err != nil {
+			return nil
+		}
 
+		if pn, sn := path.Split(rn); sn == ".stignore" {
+			pn := strings.Trim(pn, "/")
+			bs, _ := ioutil.ReadFile(p)
+			lines := bytes.Split(bs, []byte("\n"))
+			var patterns []string
+			for _, line := range lines {
+				if len(line) > 0 {
+					patterns = append(patterns, string(line))
+				}
+			}
+			ign[pn] = patterns
+			return nil
+		}
+
+		if info.Mode()&os.ModeType == 0 {
 			fi, err := os.Stat(p)
 			if err != nil {
 				return nil
@@ -94,21 +110,21 @@ func (m *Model) genWalker(res *[]File) filepath.WalkFunc {
 
 // Walk returns the list of files found in the local repository by scanning the
 // file system. Files are blockwise hashed.
-func (m *Model) Walk(followSymlinks bool) []File {
-	var files []File
-	fn := m.genWalker(&files)
+func (m *Model) Walk(followSymlinks bool) (files []File, ignore map[string][]string) {
+	ignore = make(map[string][]string)
+	fn := m.genWalker(&files, ignore)
 	filepath.Walk(m.dir, fn)
 
 	if followSymlinks {
 		d, err := os.Open(m.dir)
 		if err != nil {
-			return files
+			return
 		}
 		defer d.Close()
 
 		fis, err := d.Readdir(-1)
 		if err != nil {
-			return files
+			return
 		}
 
 		for _, fi := range fis {
@@ -118,7 +134,15 @@ func (m *Model) Walk(followSymlinks bool) []File {
 		}
 	}
 
-	return files
+	return
+}
+
+// Walk returns the list of files found in the local repository by scanning the
+// file system. Files are blockwise hashed. Patterns marked in .stignore files
+// are removed from the results.
+func (m *Model) FilteredWalk(followSymlinks bool) []File {
+	var files, ignored = m.Walk(followSymlinks)
+	return ignoreFilter(ignored, files)
 }
 
 func (m *Model) cleanTempFile(path string, info os.FileInfo, err error) error {
@@ -137,3 +161,21 @@ func (m *Model) cleanTempFile(path string, info os.FileInfo, err error) error {
 func (m *Model) cleanTempFiles() {
 	filepath.Walk(m.dir, m.cleanTempFile)
 }
+
+func ignoreFilter(patterns map[string][]string, files []File) (filtered []File) {
+nextFile:
+	for _, f := range files {
+		first, last := path.Split(f.Name)
+		for prefix, pats := range patterns {
+			if len(prefix) == 0 || prefix == first || strings.HasPrefix(first, prefix+"/") {
+				for _, pattern := range pats {
+					if match, _ := path.Match(pattern, last); match {
+						continue nextFile
+					}
+				}
+			}
+		}
+		filtered = append(filtered, f)
+	}
+	return filtered
+}

model/walk_test.go

@@ -2,6 +2,7 @@ package model
 
 import (
 	"fmt"
+	"reflect"
 	"testing"
 	"time"
 )
@@ -16,9 +17,13 @@ var testdata = []struct {
 	{"foo", 7, "aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f"},
 }
 
+var correctIgnores = map[string][]string{
+	"": {".*", "quux"},
+}
+
 func TestWalk(t *testing.T) {
 	m := NewModel("testdata")
-	files := m.Walk(false)
+	files, ignores := m.Walk(false)
 
 	if l1, l2 := len(files), len(testdata); l1 != l2 {
 		t.Fatalf("Incorrect number of walked files %d != %d", l1, l2)
@@ -39,4 +44,59 @@ func TestWalk(t *testing.T) {
 			t.Errorf("Unrealistic modtime %d for test %d", mt, i)
 		}
 	}
+
+	if !reflect.DeepEqual(ignores, correctIgnores) {
+		t.Errorf("Incorrect ignores\n  %v\n  %v", correctIgnores, ignores)
+	}
+}
+
+func TestFilteredWalk(t *testing.T) {
+	m := NewModel("testdata")
+	files := m.FilteredWalk(false)
+
+	if len(files) != 2 {
+		t.Fatalf("Incorrect number of walked filtered files %d != 2", len(files))
+	}
+	if files[0].Name != "bar" {
+		t.Error("Incorrect first file", files[0])
+	}
+	if files[1].Name != "foo" {
+		t.Error("Incorrect second file", files[1])
+	}
+}
+
+func TestIgnore(t *testing.T) {
+	var patterns = map[string][]string{
+		"":        {"t2"},
+		"foo":     {"bar", "z*"},
+		"foo/baz": {"quux", ".*"},
+	}
+	var files = []File{
+		{Name: "foo/bar"},
+		{Name: "foo/quux"},
+		{Name: "foo/zuux"},
+		{Name: "foo/qzuux"},
+		{Name: "foo/baz/t1"},
+		{Name: "foo/baz/t2"},
+		{Name: "foo/baz/bar"},
+		{Name: "foo/baz/quuxa"},
+		{Name: "foo/baz/aquux"},
+		{Name: "foo/baz/.quux"},
+		{Name: "foo/baz/zquux"},
+		{Name: "foo/baz/quux"},
+		{Name: "foo/bazz/quux"},
+	}
+	var remaining = []File{
+		{Name: "foo/quux"},
+		{Name: "foo/qzuux"},
+		{Name: "foo/baz/t1"},
+		{Name: "foo/baz/quuxa"},
+		{Name: "foo/baz/aquux"},
+		{Name: "foo/bazz/quux"},
+	}
+
+	var filtered = ignoreFilter(patterns, files)
+	if !reflect.DeepEqual(filtered, remaining) {
+		t.Errorf("Filtering mismatch\n  %v\n  %v", remaining, filtered)
+	}
 }