Rework .stignore functionality (fixes #561) (...)

- Only one .stignore is supported, at the repo root
 - Negative patterns (!) are supported
 - Ignore patterns affect sent and received indexes, not only scanning

cmd/syncthing/gui.go

@@ -255,7 +255,7 @@ func restGetModel(m *model.Model, w http.ResponseWriter, r *http.Request) {
 func restPostOverride(m *model.Model, w http.ResponseWriter, r *http.Request) {
 	var qs = r.URL.Query()
 	var repo = qs.Get("repo")
-	m.Override(repo)
+	go m.Override(repo)
 }
 
 func restGetNeed(m *model.Model, w http.ResponseWriter, r *http.Request) {

files/leveldb.go

@@ -186,18 +186,28 @@ func ldbGenericReplace(db *leveldb.DB, repo, node []byte, fs []protocol.FileInfo
 			if lv := ldbInsert(batch, repo, node, newName, fs[fsi]); lv > maxLocalVer {
 				maxLocalVer = lv
 			}
-			ldbUpdateGlobal(snap, batch, repo, node, newName, fs[fsi].Version)
+			if fs[fsi].IsInvalid() {
+				ldbRemoveFromGlobal(snap, batch, repo, node, newName)
+			} else {
+				ldbUpdateGlobal(snap, batch, repo, node, newName, fs[fsi].Version)
+			}
 			fsi++
 
 		case moreFs && moreDb && cmp == 0:
-			// File exists on both sides - compare versions.
+			// File exists on both sides - compare versions. We might get an
+			// update with the same version and different flags if a node has
+			// marked a file as invalid, so handle that too.
 			var ef protocol.FileInfoTruncated
 			ef.UnmarshalXDR(dbi.Value())
-			if fs[fsi].Version > ef.Version {
+			if fs[fsi].Version > ef.Version || fs[fsi].Version != ef.Version {
 				if lv := ldbInsert(batch, repo, node, newName, fs[fsi]); lv > maxLocalVer {
 					maxLocalVer = lv
 				}
-				ldbUpdateGlobal(snap, batch, repo, node, newName, fs[fsi].Version)
+				if fs[fsi].IsInvalid() {
+					ldbRemoveFromGlobal(snap, batch, repo, node, newName)
+				} else {
+					ldbUpdateGlobal(snap, batch, repo, node, newName, fs[fsi].Version)
+				}
 			}
 			// Iterate both sides.
 			fsi++
@@ -280,7 +290,11 @@ func ldbUpdate(db *leveldb.DB, repo, node []byte, fs []protocol.FileInfo) uint64
 			if lv := ldbInsert(batch, repo, node, name, f); lv > maxLocalVer {
 				maxLocalVer = lv
 			}
-			ldbUpdateGlobal(snap, batch, repo, node, name, f.Version)
+			if f.IsInvalid() {
+				ldbRemoveFromGlobal(snap, batch, repo, node, name)
+			} else {
+				ldbUpdateGlobal(snap, batch, repo, node, name, f.Version)
+			}
 			continue
 		}
 
@@ -289,11 +303,17 @@ func ldbUpdate(db *leveldb.DB, repo, node []byte, fs []protocol.FileInfo) uint64
 		if err != nil {
 			panic(err)
 		}
-		if ef.Version != f.Version {
+		// Flags might change without the version being bumped when we set the
+		// invalid flag on an existing file.
+		if ef.Version != f.Version || ef.Flags != f.Flags {
 			if lv := ldbInsert(batch, repo, node, name, f); lv > maxLocalVer {
 				maxLocalVer = lv
 			}
-			ldbUpdateGlobal(snap, batch, repo, node, name, f.Version)
+			if f.IsInvalid() {
+				ldbRemoveFromGlobal(snap, batch, repo, node, name)
+			} else {
+				ldbUpdateGlobal(snap, batch, repo, node, name, f.Version)
+			}
 		}
 	}
 
@@ -385,7 +405,9 @@ func ldbRemoveFromGlobal(db dbReader, batch dbWriter, repo, node, file []byte) {
 	gk := globalKey(repo, file)
 	svl, err := db.Get(gk, nil)
 	if err != nil {
-		panic(err)
+		// We might be called to "remove" a global version that doesn't exist
+		// if the first update for the file is already marked invalid.
+		return
 	}
 
 	var fl versionList

files/set_test.go

@@ -86,7 +86,7 @@ func (l fileList) String() string {
 	var b bytes.Buffer
 	b.WriteString("[]protocol.FileList{\n")
 	for _, f := range l {
-		fmt.Fprintf(&b, "  %q: #%d, %d bytes, %d blocks\n", f.Name, f.Version, f.Size(), len(f.Blocks))
+		fmt.Fprintf(&b, "  %q: #%d, %d bytes, %d blocks, flags=%o\n", f.Name, f.Version, f.Size(), len(f.Blocks), f.Flags)
 	}
 	b.WriteString("}")
 	return b.String()
@@ -280,6 +280,86 @@ func TestNeedWithInvalid(t *testing.T) {
 	}
 }
 
+func TestUpdateToInvalid(t *testing.T) {
+	lamport.Default = lamport.Clock{}
+
+	db, err := leveldb.Open(storage.NewMemStorage(), nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	s := files.NewSet("test", db)
+
+	localHave := fileList{
+		protocol.FileInfo{Name: "a", Version: 1000, Blocks: genBlocks(1)},
+		protocol.FileInfo{Name: "b", Version: 1001, Blocks: genBlocks(2)},
+		protocol.FileInfo{Name: "c", Version: 1002, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
+		protocol.FileInfo{Name: "d", Version: 1003, Blocks: genBlocks(7)},
+	}
+
+	s.ReplaceWithDelete(protocol.LocalNodeID, localHave)
+
+	have := fileList(haveList(s, protocol.LocalNodeID))
+	sort.Sort(have)
+
+	if fmt.Sprint(have) != fmt.Sprint(localHave) {
+		t.Errorf("Have incorrect before invalidation;\n A: %v !=\n E: %v", have, localHave)
+	}
+
+	localHave[1] = protocol.FileInfo{Name: "b", Version: 1001, Flags: protocol.FlagInvalid}
+	s.Update(protocol.LocalNodeID, localHave[1:2])
+
+	have = fileList(haveList(s, protocol.LocalNodeID))
+	sort.Sort(have)
+
+	if fmt.Sprint(have) != fmt.Sprint(localHave) {
+		t.Errorf("Have incorrect after invalidation;\n A: %v !=\n E: %v", have, localHave)
+	}
+}
+
+func TestInvalidAvailability(t *testing.T) {
+	lamport.Default = lamport.Clock{}
+
+	db, err := leveldb.Open(storage.NewMemStorage(), nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	s := files.NewSet("test", db)
+
+	remote0Have := fileList{
+		protocol.FileInfo{Name: "both", Version: 1001, Blocks: genBlocks(2)},
+		protocol.FileInfo{Name: "r1only", Version: 1002, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
+		protocol.FileInfo{Name: "r0only", Version: 1003, Blocks: genBlocks(7)},
+		protocol.FileInfo{Name: "none", Version: 1004, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
+	}
+	remote1Have := fileList{
+		protocol.FileInfo{Name: "both", Version: 1001, Blocks: genBlocks(2)},
+		protocol.FileInfo{Name: "r1only", Version: 1002, Blocks: genBlocks(7)},
+		protocol.FileInfo{Name: "r0only", Version: 1003, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
+		protocol.FileInfo{Name: "none", Version: 1004, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
+	}
+
+	s.Replace(remoteNode0, remote0Have)
+	s.Replace(remoteNode1, remote1Have)
+
+	if av := s.Availability("both"); len(av) != 2 {
+		t.Error("Incorrect availability for 'both':", av)
+	}
+
+	if av := s.Availability("r0only"); len(av) != 1 || av[0] != remoteNode0 {
+		t.Error("Incorrect availability for 'r0only':", av)
+	}
+
+	if av := s.Availability("r1only"); len(av) != 1 || av[0] != remoteNode1 {
+		t.Error("Incorrect availability for 'r1only':", av)
+	}
+
+	if av := s.Availability("none"); len(av) != 0 {
+		t.Error("Incorrect availability for 'none':", av)
+	}
+}
+
 func TestLocalDeleted(t *testing.T) {
 	db, err := leveldb.Open(storage.NewMemStorage(), nil)
 	if err != nil {

gui/app.js

@@ -971,9 +971,9 @@ function debounce(func, wait) {
         } else {
             timeout = null;
             if (again) {
+                again = false;
                 result = func.apply(context, args);
                 context = args = null;
-                again = false;
             }
         }
     };

ignore/ignore.go

@@ -0,0 +1,146 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+package ignore
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+
+	"github.com/syncthing/syncthing/fnmatch"
+)
+
+type Pattern struct {
+	match   *regexp.Regexp
+	include bool
+}
+
+type Patterns []Pattern
+
+func Load(file string) (Patterns, error) {
+	seen := make(map[string]bool)
+	return loadIgnoreFile(file, seen)
+}
+
+func Parse(r io.Reader, file string) (Patterns, error) {
+	seen := map[string]bool{
+		file: true,
+	}
+	return parseIgnoreFile(r, file, seen)
+}
+
+func (l Patterns) Match(file string) bool {
+	for _, pattern := range l {
+		if pattern.match.MatchString(file) {
+			return pattern.include
+		}
+	}
+	return false
+}
+
+func loadIgnoreFile(file string, seen map[string]bool) (Patterns, error) {
+	if seen[file] {
+		return nil, fmt.Errorf("Multiple include of ignore file %q", file)
+	}
+	seen[file] = true
+
+	fd, err := os.Open(file)
+	if err != nil {
+		return nil, err
+	}
+	defer fd.Close()
+
+	return parseIgnoreFile(fd, file, seen)
+}
+
+func parseIgnoreFile(fd io.Reader, currentFile string, seen map[string]bool) (Patterns, error) {
+	var exps Patterns
+
+	addPattern := func(line string) error {
+		include := true
+		if strings.HasPrefix(line, "!") {
+			line = line[1:]
+			include = false
+		}
+
+		if strings.HasPrefix(line, "/") {
+			// Pattern is rooted in the current dir only
+			exp, err := fnmatch.Convert(line[1:], fnmatch.FNM_PATHNAME)
+			if err != nil {
+				return fmt.Errorf("Invalid pattern %q in ignore file", line)
+			}
+			exps = append(exps, Pattern{exp, include})
+		} else if strings.HasPrefix(line, "**/") {
+			// Add the pattern as is, and without **/ so it matches in current dir
+			exp, err := fnmatch.Convert(line, fnmatch.FNM_PATHNAME)
+			if err != nil {
+				return fmt.Errorf("Invalid pattern %q in ignore file", line)
+			}
+			exps = append(exps, Pattern{exp, include})
+
+			exp, err = fnmatch.Convert(line[3:], fnmatch.FNM_PATHNAME)
+			if err != nil {
+				return fmt.Errorf("Invalid pattern %q in ignore file", line)
+			}
+			exps = append(exps, Pattern{exp, include})
+		} else if strings.HasPrefix(line, "#include ") {
+			includeFile := filepath.Join(filepath.Dir(currentFile), line[len("#include "):])
+			includes, err := loadIgnoreFile(includeFile, seen)
+			if err != nil {
+				return err
+			} else {
+				exps = append(exps, includes...)
+			}
+		} else {
+			// Path name or pattern, add it so it matches files both in
+			// current directory and subdirs.
+			exp, err := fnmatch.Convert(line, fnmatch.FNM_PATHNAME)
+			if err != nil {
+				return fmt.Errorf("Invalid pattern %q in ignore file", line)
+			}
+			exps = append(exps, Pattern{exp, include})
+
+			exp, err = fnmatch.Convert("**/"+line, fnmatch.FNM_PATHNAME)
+			if err != nil {
+				return fmt.Errorf("Invalid pattern %q in ignore file", line)
+			}
+			exps = append(exps, Pattern{exp, include})
+		}
+		return nil
+	}
+
+	scanner := bufio.NewScanner(fd)
+	var err error
+	for scanner.Scan() {
+		line := strings.TrimSpace(scanner.Text())
+		switch {
+		case line == "":
+			continue
+		case strings.HasPrefix(line, "#"):
+			err = addPattern(line)
+		case strings.HasSuffix(line, "/**"):
+			err = addPattern(line)
+		case strings.HasSuffix(line, "/"):
+			err = addPattern(line)
+			if err == nil {
+				err = addPattern(line + "**")
+			}
+		default:
+			err = addPattern(line)
+			if err == nil {
+				err = addPattern(line + "/**")
+			}
+		}
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return exps, nil
+}

ignore/ignore_test.go

@@ -0,0 +1,104 @@
+package ignore_test
+
+import (
+	"bytes"
+	"path/filepath"
+	"testing"
+
+	"github.com/syncthing/syncthing/ignore"
+)
+
+func TestIgnore(t *testing.T) {
+	pats, err := ignore.Load("testdata/.stignore")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var tests = []struct {
+		f string
+		r bool
+	}{
+		{"afile", false},
+		{"bfile", true},
+		{"cfile", false},
+		{"dfile", false},
+		{"efile", true},
+		{"ffile", true},
+
+		{"dir1", false},
+		{filepath.Join("dir1", "cfile"), true},
+		{filepath.Join("dir1", "dfile"), false},
+		{filepath.Join("dir1", "efile"), true},
+		{filepath.Join("dir1", "ffile"), false},
+
+		{"dir2", false},
+		{filepath.Join("dir2", "cfile"), false},
+		{filepath.Join("dir2", "dfile"), true},
+		{filepath.Join("dir2", "efile"), true},
+		{filepath.Join("dir2", "ffile"), false},
+
+		{filepath.Join("dir3"), true},
+		{filepath.Join("dir3", "afile"), true},
+	}
+
+	for i, tc := range tests {
+		if r := pats.Match(tc.f); r != tc.r {
+			t.Errorf("Incorrect ignoreFile() #%d (%s); E: %v, A: %v", i, tc.f, tc.r, r)
+		}
+	}
+}
+
+func TestExcludes(t *testing.T) {
+	stignore := `
+	!iex2
+	!ign1/ex
+	ign1
+	i*2
+	!ign2
+	`
+	pats, err := ignore.Parse(bytes.NewBufferString(stignore), ".stignore")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var tests = []struct {
+		f string
+		r bool
+	}{
+		{"ign1", true},
+		{"ign2", true},
+		{"ibla2", true},
+		{"iex2", false},
+		{"ign1/ign", true},
+		{"ign1/ex", false},
+		{"ign1/iex2", false},
+		{"iex2/ign", false},
+		{"foo/bar/ign1", true},
+		{"foo/bar/ign2", true},
+		{"foo/bar/iex2", false},
+	}
+
+	for _, tc := range tests {
+		if r := pats.Match(tc.f); r != tc.r {
+			t.Errorf("Incorrect match for %s: %v != %v", tc.f, r, tc.r)
+		}
+	}
+}
+
+func TestBadPatterns(t *testing.T) {
+	var badPatterns = []string{
+		"[",
+		"/[",
+		"**/[",
+		"#include nonexistent",
+		"#include .stignore",
+		"!#include makesnosense",
+	}
+
+	for _, pat := range badPatterns {
+		parsed, err := ignore.Parse(bytes.NewBufferString(pat), ".stignore")
+		if err == nil {
+			t.Errorf("No error for pattern %q: %v", pat, parsed)
+		}
+	}
+}

ignore/testdata/.stignore

@@ -0,0 +1,6 @@
+#include excludes
+
+bfile
+dir1/cfile
+**/efile
+/ffile

ignore/testdata/dir3/cfile

@@ -0,0 +1 @@
+baz

ignore/testdata/dir3/dfile

@@ -0,0 +1 @@
+quux

ignore/testdata/excludes

@@ -0,0 +1,2 @@
+dir2/dfile
+#include further-excludes

ignore/testdata/further-excludes

@@ -0,0 +1 @@
+dir3

integration/all.sh

@@ -2,7 +2,7 @@
 set -euo pipefail
 IFS=$'\n\t'
 
-go test -tags integration -v
+#go test -tags integration -v
 ./test-http.sh
 ./test-merge.sh
 ./test-delupd.sh

model/model.go

@@ -19,6 +19,7 @@ import (
 	"github.com/syncthing/syncthing/config"
 	"github.com/syncthing/syncthing/events"
 	"github.com/syncthing/syncthing/files"
+	"github.com/syncthing/syncthing/ignore"
 	"github.com/syncthing/syncthing/lamport"
 	"github.com/syncthing/syncthing/protocol"
 	"github.com/syncthing/syncthing/scanner"
@@ -72,6 +73,7 @@ type Model struct {
 	repoNodes    map[string][]protocol.NodeID                       // repo -> nodeIDs
 	nodeRepos    map[protocol.NodeID][]string                       // nodeID -> repos
 	nodeStatRefs map[protocol.NodeID]*stats.NodeStatisticsReference // nodeID -> statsRef
+	repoIgnores  map[string]ignore.Patterns                         // repo -> list of ignore patterns
 	rmut         sync.RWMutex                                       // protects the above
 
 	repoState        map[string]repoState // repo -> state
@@ -108,6 +110,7 @@ func NewModel(indexDir string, cfg *config.Configuration, nodeName, clientName,
 		repoNodes:        make(map[string][]protocol.NodeID),
 		nodeRepos:        make(map[protocol.NodeID][]string),
 		nodeStatRefs:     make(map[protocol.NodeID]*stats.NodeStatisticsReference),
+		repoIgnores:      make(map[string]ignore.Patterns),
 		repoState:        make(map[string]repoState),
 		repoStateChanged: make(map[string]time.Time),
 		protoConn:        make(map[protocol.NodeID]protocol.Connection),
@@ -289,6 +292,9 @@ func (m *Model) LocalSize(repo string) (files, deleted int, bytes int64) {
 	defer m.rmut.RUnlock()
 	if rf, ok := m.repoFiles[repo]; ok {
 		rf.WithHaveTruncated(protocol.LocalNodeID, func(f protocol.FileIntf) bool {
+			if f.IsInvalid() {
+				return true
+			}
 			fs, de, by := sizeOfFile(f)
 			files += fs
 			deleted += de
@@ -348,24 +354,32 @@ func (m *Model) Index(nodeID protocol.NodeID, repo string, fs []protocol.FileInf
 		return
 	}
 
-	for i := range fs {
-		lamport.Default.Tick(fs[i].Version)
-	}
-
 	m.rmut.RLock()
-	r, ok := m.repoFiles[repo]
+	files, ok := m.repoFiles[repo]
+	ignores, _ := m.repoIgnores[repo]
 	m.rmut.RUnlock()
-	if ok {
+
-		r.Replace(nodeID, fs)
+	if !ok {
-	} else {
 		l.Fatalf("Index for nonexistant repo %q", repo)
 	}
 
+	for i := 0; i < len(fs); {
+		lamport.Default.Tick(fs[i].Version)
+		if ignores.Match(fs[i].Name) {
+			fs[i] = fs[len(fs)-1]
+			fs = fs[:len(fs)-1]
+		} else {
+			i++
+		}
+	}
+
+	files.Replace(nodeID, fs)
+
 	events.Default.Log(events.RemoteIndexUpdated, map[string]interface{}{
 		"node":    nodeID.String(),
 		"repo":    repo,
 		"items":   len(fs),
-		"version": r.LocalVersion(nodeID),
+		"version": files.LocalVersion(nodeID),
 	})
 }
 
@@ -381,24 +395,32 @@ func (m *Model) IndexUpdate(nodeID protocol.NodeID, repo string, fs []protocol.F
 		return
 	}
 
-	for i := range fs {
-		lamport.Default.Tick(fs[i].Version)
-	}
-
 	m.rmut.RLock()
-	r, ok := m.repoFiles[repo]
+	files, ok := m.repoFiles[repo]
+	ignores, _ := m.repoIgnores[repo]
 	m.rmut.RUnlock()
-	if ok {
+
-		r.Update(nodeID, fs)
+	if !ok {
-	} else {
 		l.Fatalf("IndexUpdate for nonexistant repo %q", repo)
 	}
 
+	for i := 0; i < len(fs); {
+		lamport.Default.Tick(fs[i].Version)
+		if ignores.Match(fs[i].Name) {
+			fs[i] = fs[len(fs)-1]
+			fs = fs[:len(fs)-1]
+		} else {
+			i++
+		}
+	}
+
+	files.Update(nodeID, fs)
+
 	events.Default.Log(events.RemoteIndexUpdated, map[string]interface{}{
 		"node":    nodeID.String(),
 		"repo":    repo,
 		"items":   len(fs),
-		"version": r.LocalVersion(nodeID),
+		"version": files.LocalVersion(nodeID),
 	})
 }
 
@@ -572,7 +594,7 @@ func (m *Model) AddConnection(rawConn io.Closer, protoConn protocol.Connection)
 	m.rmut.RLock()
 	for _, repo := range m.nodeRepos[nodeID] {
 		fs := m.repoFiles[repo]
-		go sendIndexes(protoConn, repo, fs)
+		go sendIndexes(protoConn, repo, fs, m.repoIgnores[repo])
 	}
 	if statRef, ok := m.nodeStatRefs[nodeID]; ok {
 		statRef.WasSeen()
@@ -583,7 +605,7 @@ func (m *Model) AddConnection(rawConn io.Closer, protoConn protocol.Connection)
 	m.pmut.Unlock()
 }
 
-func sendIndexes(conn protocol.Connection, repo string, fs *files.Set) {
+func sendIndexes(conn protocol.Connection, repo string, fs *files.Set, ignores ignore.Patterns) {
 	nodeID := conn.ID()
 	name := conn.Name()
 	var err error
@@ -598,7 +620,7 @@ func sendIndexes(conn protocol.Connection, repo string, fs *files.Set) {
 		}
 	}()
 
-	minLocalVer, err := sendIndexTo(true, 0, conn, repo, fs)
+	minLocalVer, err := sendIndexTo(true, 0, conn, repo, fs, ignores)
 
 	for err == nil {
 		time.Sleep(5 * time.Second)
@@ -606,11 +628,11 @@ func sendIndexes(conn protocol.Connection, repo string, fs *files.Set) {
 			continue
 		}
 
-		minLocalVer, err = sendIndexTo(false, minLocalVer, conn, repo, fs)
+		minLocalVer, err = sendIndexTo(false, minLocalVer, conn, repo, fs, ignores)
 	}
 }
 
-func sendIndexTo(initial bool, minLocalVer uint64, conn protocol.Connection, repo string, fs *files.Set) (uint64, error) {
+func sendIndexTo(initial bool, minLocalVer uint64, conn protocol.Connection, repo string, fs *files.Set, ignores ignore.Patterns) (uint64, error) {
 	nodeID := conn.ID()
 	name := conn.Name()
 	batch := make([]protocol.FileInfo, 0, indexBatchSize)
@@ -628,6 +650,10 @@ func sendIndexTo(initial bool, minLocalVer uint64, conn protocol.Connection, rep
 			maxLocalVer = f.LocalVersion
 		}
 
+		if ignores.Match(f.Name) {
+			return true
+		}
+
 		if len(batch) == indexBatchSize || currentBatchSize > indexTargetSize {
 			if initial {
 				if err = conn.Index(repo, batch); err != nil {
@@ -781,10 +807,13 @@ func (m *Model) ScanRepoSub(repo, sub string) error {
 	fs, ok := m.repoFiles[repo]
 	dir := m.repoCfgs[repo].Directory
 
+	ignores, _ := ignore.Load(filepath.Join(dir, ".stignore"))
+	m.repoIgnores[repo] = ignores
+
 	w := &scanner.Walker{
 		Dir:          dir,
 		Sub:          sub,
-		IgnoreFile:   ".stignore",
+		Ignores:      ignores,
 		BlockSize:    scanner.StandardBlockSize,
 		TempNamer:    defTempNamer,
 		CurrentFiler: cFiler{m, repo},
@@ -827,15 +856,40 @@ func (m *Model) ScanRepoSub(repo, sub string) error {
 	fs.WithHaveTruncated(protocol.LocalNodeID, func(fi protocol.FileIntf) bool {
 		f := fi.(protocol.FileInfoTruncated)
 		if !strings.HasPrefix(f.Name, sub) {
+			// Return true so that we keep iterating, until we get to the part
+			// of the tree we are interested in. Then return false so we stop
+			// iterating when we've passed the end of the subtree.
 			return !seenPrefix
 		}
+
 		seenPrefix = true
 		if !protocol.IsDeleted(f.Flags) {
+			if f.IsInvalid() {
+				return true
+			}
+
 			if len(batch) == batchSize {
 				fs.Update(protocol.LocalNodeID, batch)
 				batch = batch[:0]
 			}
-			if _, err := os.Stat(filepath.Join(dir, f.Name)); err != nil && os.IsNotExist(err) {
+
+			if ignores.Match(f.Name) {
+				// File has been ignored. Set invalid bit.
+				nf := protocol.FileInfo{
+					Name:     f.Name,
+					Flags:    f.Flags | protocol.FlagInvalid,
+					Modified: f.Modified,
+					Version:  f.Version, // The file is still the same, so don't bump version
+				}
+				events.Default.Log(events.LocalIndexUpdated, map[string]interface{}{
+					"repo":     repo,
+					"name":     f.Name,
+					"modified": time.Unix(f.Modified, 0),
+					"flags":    fmt.Sprintf("0%o", f.Flags),
+					"size":     f.Size(),
+				})
+				batch = append(batch, nf)
+			} else if _, err := os.Stat(filepath.Join(dir, f.Name)); err != nil && os.IsNotExist(err) {
 				// File has been deleted
 				nf := protocol.FileInfo{
 					Name:     f.Name,
@@ -928,6 +982,7 @@ func (m *Model) Override(repo string) {
 	fs := m.repoFiles[repo]
 	m.rmut.RUnlock()
 
+	m.setState(repo, RepoScanning)
 	batch := make([]protocol.FileInfo, 0, indexBatchSize)
 	fs.WithNeed(protocol.LocalNodeID, func(fi protocol.FileIntf) bool {
 		need := fi.(protocol.FileInfo)
@@ -953,6 +1008,7 @@ func (m *Model) Override(repo string) {
 	if len(batch) > 0 {
 		fs.Update(protocol.LocalNodeID, batch)
 	}
+	m.setState(repo, RepoIdle)
 }
 
 // Version returns the change version for the given repository. This is

scanner/testdata/.stignore

@@ -1,5 +1,4 @@
 #include excludes
-#include nonexistent-file
 
 bfile
 dir1/cfile

scanner/testdata/excludes

@@ -1,4 +1,2 @@
 dir2/dfile
-#include excludes
 #include further-excludes
-#include loop-excludes

scanner/testdata/loop-excludes

@@ -1 +0,0 @@
-#include excludes

scanner/walk.go

@@ -5,19 +5,14 @@
 package scanner
 
 import (
-	"bufio"
 	"errors"
-	"io"
 	"os"
-	"path"
 	"path/filepath"
-	"regexp"
 	"runtime"
-	"strings"
 
 	"code.google.com/p/go.text/unicode/norm"
 
-	"github.com/syncthing/syncthing/fnmatch"
+	"github.com/syncthing/syncthing/ignore"
 	"github.com/syncthing/syncthing/lamport"
 	"github.com/syncthing/syncthing/protocol"
 )
@@ -29,8 +24,8 @@ type Walker struct {
 	Sub string
 	// BlockSize controls the size of the block used when hashing.
 	BlockSize int
-	// If IgnoreFile is not empty, it is the name used for the file that holds ignore patterns.
+	// List of patterns to ignore
-	IgnoreFile string
+	Ignores ignore.Patterns
 	// If TempNamer is not nil, it is used to ignore tempory files when walking.
 	TempNamer TempNamer
 	// If CurrentFiler is not nil, it is queried for the current file before rescanning.
@@ -57,7 +52,7 @@ type CurrentFiler interface {
 // file system. Files are blockwise hashed.
 func (w *Walker) Walk() (chan protocol.FileInfo, error) {
 	if debug {
-		l.Debugln("Walk", w.Dir, w.Sub, w.BlockSize, w.IgnoreFile)
+		l.Debugln("Walk", w.Dir, w.Sub, w.BlockSize, w.Ignores)
 	}
 
 	err := checkDir(w.Dir)
@@ -69,11 +64,8 @@ func (w *Walker) Walk() (chan protocol.FileInfo, error) {
 	hashedFiles := make(chan protocol.FileInfo)
 	newParallelHasher(w.Dir, w.BlockSize, runtime.NumCPU(), hashedFiles, files)
 
-	var ignores []*regexp.Regexp
 	go func() {
-		filepath.Walk(w.Dir, w.loadIgnoreFiles(w.Dir, &ignores))
+		hashFiles := w.walkAndHashFiles(files)
-
-		hashFiles := w.walkAndHashFiles(files, ignores)
 		filepath.Walk(filepath.Join(w.Dir, w.Sub), hashFiles)
 		close(files)
 	}()
@@ -86,113 +78,7 @@ func (w *Walker) CleanTempFiles() {
 	filepath.Walk(w.Dir, w.cleanTempFile)
 }
 
-func (w *Walker) loadIgnoreFiles(dir string, ignores *[]*regexp.Regexp) filepath.WalkFunc {
+func (w *Walker) walkAndHashFiles(fchan chan protocol.FileInfo) filepath.WalkFunc {
-	return func(p string, info os.FileInfo, err error) error {
-		if err != nil {
-			return nil
-		}
-
-		rn, err := filepath.Rel(dir, p)
-		if err != nil {
-			return nil
-		}
-
-		if pn, sn := filepath.Split(rn); sn == w.IgnoreFile {
-			pn := filepath.Clean(pn)
-			filesSeen := make(map[string]map[string]bool)
-			dirIgnores := loadIgnoreFile(p, pn, filesSeen)
-			*ignores = append(*ignores, dirIgnores...)
-		}
-
-		return nil
-	}
-}
-
-func loadIgnoreFile(ignFile, base string, filesSeen map[string]map[string]bool) []*regexp.Regexp {
-	fd, err := os.Open(ignFile)
-	if err != nil {
-		return nil
-	}
-	defer fd.Close()
-	return parseIgnoreFile(fd, base, ignFile, filesSeen)
-}
-
-func parseIgnoreFile(fd io.Reader, base, currentFile string, filesSeen map[string]map[string]bool) []*regexp.Regexp {
-	var exps []*regexp.Regexp
-	scanner := bufio.NewScanner(fd)
-	for scanner.Scan() {
-		line := strings.TrimSpace(scanner.Text())
-		if line == "" {
-			continue
-		}
-
-		if strings.HasPrefix(line, "/") {
-			// Pattern is rooted in the current dir only
-			exp, err := fnmatch.Convert(path.Join(base, line[1:]), fnmatch.FNM_PATHNAME)
-			if err != nil {
-				l.Warnf("Invalid pattern %q in ignore file", line)
-				continue
-			}
-			exps = append(exps, exp)
-		} else if strings.HasPrefix(line, "**/") {
-			// Add the pattern as is, and without **/ so it matches in current dir
-			exp, err := fnmatch.Convert(line, fnmatch.FNM_PATHNAME)
-			if err != nil {
-				l.Warnf("Invalid pattern %q in ignore file", line)
-				continue
-			}
-			exps = append(exps, exp)
-
-			exp, err = fnmatch.Convert(path.Join(base, line[3:]), fnmatch.FNM_PATHNAME)
-			if err != nil {
-				l.Warnf("Invalid pattern %q in ignore file", line)
-				continue
-			}
-			exps = append(exps, exp)
-		} else if strings.HasPrefix(line, "#include ") {
-			includeFile := filepath.Join(filepath.Dir(currentFile), strings.Replace(line, "#include ", "", 1))
-			if _, err := os.Stat(includeFile); os.IsNotExist(err) {
-				l.Infoln("Could not open ignore include file", includeFile)
-			} else {
-				seen := false
-				if seenByCurrent, ok := filesSeen[currentFile]; ok {
-					_, seen = seenByCurrent[includeFile]
-				}
-
-				if seen {
-					l.Warnf("Recursion detected while including %s from %s", includeFile, currentFile)
-				} else {
-					if filesSeen[currentFile] == nil {
-						filesSeen[currentFile] = make(map[string]bool)
-					}
-					filesSeen[currentFile][includeFile] = true
-					includes := loadIgnoreFile(includeFile, base, filesSeen)
-					exps = append(exps, includes...)
-				}
-			}
-		} else {
-			// Path name or pattern, add it so it matches files both in
-			// current directory and subdirs.
-			exp, err := fnmatch.Convert(path.Join(base, line), fnmatch.FNM_PATHNAME)
-			if err != nil {
-				l.Warnf("Invalid pattern %q in ignore file", line)
-				continue
-			}
-			exps = append(exps, exp)
-
-			exp, err = fnmatch.Convert(path.Join(base, "**", line), fnmatch.FNM_PATHNAME)
-			if err != nil {
-				l.Warnf("Invalid pattern %q in ignore file", line)
-				continue
-			}
-			exps = append(exps, exp)
-		}
-	}
-
-	return exps
-}
-
-func (w *Walker) walkAndHashFiles(fchan chan protocol.FileInfo, ignores []*regexp.Regexp) filepath.WalkFunc {
 	return func(p string, info os.FileInfo, err error) error {
 		if err != nil {
 			if debug {
@@ -221,7 +107,7 @@ func (w *Walker) walkAndHashFiles(fchan chan protocol.FileInfo, ignores []*regex
 			return nil
 		}
 
-		if sn := filepath.Base(rn); sn == w.IgnoreFile || sn == ".stversions" || w.ignoreFile(ignores, rn) {
+		if sn := filepath.Base(rn); sn == ".stignore" || sn == ".stversions" || w.Ignores.Match(rn) {
 			// An ignored file
 			if debug {
 				l.Debugln("ignored:", rn)
@@ -305,18 +191,6 @@ func (w *Walker) cleanTempFile(path string, info os.FileInfo, err error) error {
 	return nil
 }
 
-func (w *Walker) ignoreFile(patterns []*regexp.Regexp, file string) bool {
-	for _, pattern := range patterns {
-		if pattern.MatchString(file) {
-			if debug {
-				l.Debugf("%q matches %v", file, pattern)
-			}
-			return true
-		}
-	}
-	return false
-}
-
 func checkDir(dir string) error {
 	if info, err := os.Lstat(dir); err != nil {
 		return err

scanner/walk_test.go

@@ -13,6 +13,7 @@ import (
 	"sort"
 	"testing"
 
+	"github.com/syncthing/syncthing/ignore"
 	"github.com/syncthing/syncthing/protocol"
 )
 
@@ -30,9 +31,8 @@ var testdata = testfileList{
 	{filepath.Join("dir1", "dfile"), 5, "49ae93732fcf8d63fe1cce759664982dbd5b23161f007dba8561862adc96d063"},
 	{"dir2", 128, ""},
 	{filepath.Join("dir2", "cfile"), 4, "bf07a7fbb825fc0aae7bf4a1177b2b31fcf8a3feeaf7092761e18c859ee52a9c"},
-	{"excludes", 78, "1f5ac95d9e6fb2516629a029d788d27953c7bb2f4dc09184b660fdda0c8f2f04"},
+	{"excludes", 37, "df90b52f0c55dba7a7a940affe482571563b1ac57bd5be4d8a0291e7de928e06"},
 	{"further-excludes", 5, "7eb0a548094fa6295f7fd9200d69973e5f5ec5c04f2a86d998080ac43ecf89f1"},
-	{"loop-excludes", 18, "2db057aa82a8b8fe4b1367ccc875259ed4b8020255820d4e3d4bfe78f0dd3f2a"},
 }
 
 var correctIgnores = map[string][]string{
@@ -47,11 +47,16 @@ func init() {
 }
 
 func TestWalkSub(t *testing.T) {
+	ignores, err := ignore.Load("testdata/.stignore")
+	if err != nil {
+		t.Fatal(err)
+	}
+
 	w := Walker{
-		Dir:        "testdata",
+		Dir:       "testdata",
-		Sub:        "dir2",
+		Sub:       "dir2",
-		BlockSize:  128 * 1024,
+		BlockSize: 128 * 1024,
-		IgnoreFile: ".stignore",
+		Ignores:   ignores,
 	}
 	fchan, err := w.Walk()
 	var files []protocol.FileInfo
@@ -77,10 +82,16 @@ func TestWalkSub(t *testing.T) {
 }
 
 func TestWalk(t *testing.T) {
+	ignores, err := ignore.Load("testdata/.stignore")
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Log(ignores)
+
 	w := Walker{
-		Dir:        "testdata",
+		Dir:       "testdata",
-		BlockSize:  128 * 1024,
+		BlockSize: 128 * 1024,
-		IgnoreFile: ".stignore",
+		Ignores:   ignores,
 	}
 
 	fchan, err := w.Walk()
@@ -102,9 +113,8 @@ func TestWalk(t *testing.T) {
 
 func TestWalkError(t *testing.T) {
 	w := Walker{
-		Dir:        "testdata-missing",
+		Dir:       "testdata-missing",
-		BlockSize:  128 * 1024,
+		BlockSize: 128 * 1024,
-		IgnoreFile: ".stignore",
 	}
 	_, err := w.Walk()
 
@@ -113,9 +123,8 @@ func TestWalkError(t *testing.T) {
 	}
 
 	w = Walker{
-		Dir:        "testdata/bar",
+		Dir:       "testdata/bar",
-		BlockSize:  128 * 1024,
+		BlockSize: 128 * 1024,
-		IgnoreFile: ".stignore",
 	}
 	_, err = w.Walk()
 
@@ -124,67 +133,6 @@ func TestWalkError(t *testing.T) {
 	}
 }
 
-func TestIgnore(t *testing.T) {
-	patStr := bytes.NewBufferString(`
-		t2
-		/t3
-		sub/dir/*
-		*/other/test
-		**/deep
-	`)
-	patterns := parseIgnoreFile(patStr, "", "", make(map[string]map[string]bool))
-
-	patStr = bytes.NewBufferString(`
-		bar
-		z*
-		q[abc]x
-	`)
-	patterns = append(patterns, parseIgnoreFile(patStr, "foo", "", make(map[string]map[string]bool))...)
-
-	patStr = bytes.NewBufferString(`
-		quux
-		.*
-	`)
-	patterns = append(patterns, parseIgnoreFile(patStr, "foo/baz", "", make(map[string]map[string]bool))...)
-
-	var tests = []struct {
-		f string
-		r bool
-	}{
-		{filepath.Join("foo", "bar"), true},
-		{filepath.Join("t3"), true},
-		{filepath.Join("foofoo"), false},
-		{filepath.Join("foo", "quux"), false},
-		{filepath.Join("foo", "zuux"), true},
-		{filepath.Join("foo", "qzuux"), false},
-		{filepath.Join("foo", "baz", "t1"), false},
-		{filepath.Join("foo", "baz", "t2"), true},
-		{filepath.Join("foo", "baz", "t3"), false},
-		{filepath.Join("foo", "baz", "bar"), true},
-		{filepath.Join("foo", "baz", "quuxa"), false},
-		{filepath.Join("foo", "baz", "aquux"), false},
-		{filepath.Join("foo", "baz", ".quux"), true},
-		{filepath.Join("foo", "baz", "zquux"), true},
-		{filepath.Join("foo", "baz", "quux"), true},
-		{filepath.Join("foo", "bazz", "quux"), false},
-		{filepath.Join("sub", "dir", "hej"), true},
-		{filepath.Join("deeper", "sub", "dir", "hej"), true},
-		{filepath.Join("other", "test"), false},
-		{filepath.Join("sub", "other", "test"), true},
-		{filepath.Join("deeper", "sub", "other", "test"), true},
-		{filepath.Join("deep"), true},
-		{filepath.Join("deeper", "deep"), true},
-		{filepath.Join("deeper", "deeper", "deep"), true},
-	}
-
-	w := Walker{}
-	for i, tc := range tests {
-		if r := w.ignoreFile(patterns, tc.f); r != tc.r {
-			t.Errorf("Incorrect ignoreFile() #%d (%s); E: %v, A: %v", i, tc.f, tc.r, r)
-		}
-	}
-}
-
 type fileList []protocol.FileInfo
 
 func (f fileList) Len() int {