lib/ignore: Fast reload of unchanged ignores (fixes #3394)

This changes the "seen" map that we're anyway keeping around to track
the modtimes of loaded files instead. When doing a Load() we check that
1) the file we are loading is in the modtime set, and 2) that none of
the files in the modtime set have changed modtimes. If that's the case
we do a quick return without parsing anything or clearing the cache.

This required adding two one seconds sleeps in the tests to make sure
the modtimes were updated when we expect cache reloads, because I'm on a
crappy filesystem with one second timestamp granularity. That also
proves it works...

GitHub-Pull-Request: https://github.com/syncthing/syncthing/pull/3754

lib/ignore/ignore.go

@@ -69,6 +69,7 @@ type Matcher struct {
 	matches   *cache
 	curHash   string
 	stop      chan struct{}
+	modtimes  map[string]time.Time
 	mut       sync.Mutex
 }
 
@@ -85,25 +86,41 @@ func New(withCache bool) *Matcher {
 }
 
 func (m *Matcher) Load(file string) error {
-	// No locking, Parse() does the locking
+	m.mut.Lock()
+	defer m.mut.Unlock()
+
+	if m.patternsUnchanged(file) {
+		return nil
+	}
 
 	fd, err := os.Open(file)
 	if err != nil {
-		// We do a parse with empty patterns to clear out the hash, cache etc.
+		m.parseLocked(&bytes.Buffer{}, file)
-		m.Parse(&bytes.Buffer{}, file)
 		return err
 	}
 	defer fd.Close()
 
-	return m.Parse(fd, file)
+	info, err := fd.Stat()
+	if err != nil {
+		m.parseLocked(&bytes.Buffer{}, file)
+		return err
+	}
+
+	m.modtimes = map[string]time.Time{
+		file: info.ModTime(),
+	}
+
+	return m.parseLocked(fd, file)
 }
 
 func (m *Matcher) Parse(r io.Reader, file string) error {
 	m.mut.Lock()
 	defer m.mut.Unlock()
+	return m.parseLocked(r, file)
+}
 
-	seen := map[string]bool{file: true}
+func (m *Matcher) parseLocked(r io.Reader, file string) error {
-	patterns, err := parseIgnoreFile(r, file, seen)
+	patterns, err := parseIgnoreFile(r, file, m.modtimes)
 	// Error is saved and returned at the end. We process the patterns
 	// (possibly blank) anyway.
 
@@ -122,6 +139,26 @@ func (m *Matcher) Parse(r io.Reader, file string) error {
 	return err
 }
 
+// patternsUnchanged returns true if none of the files making up the loaded
+// patterns have changed since last check.
+func (m *Matcher) patternsUnchanged(file string) bool {
+	if _, ok := m.modtimes[file]; !ok {
+		return false
+	}
+
+	for filename, modtime := range m.modtimes {
+		info, err := os.Stat(filename)
+		if err != nil {
+			return false
+		}
+		if !info.ModTime().Equal(modtime) {
+			return false
+		}
+	}
+
+	return true
+}
+
 func (m *Matcher) Match(file string) (result Result) {
 	if m == nil {
 		return resultNotMatched
@@ -221,11 +258,10 @@ func hashPatterns(patterns []Pattern) string {
 	return fmt.Sprintf("%x", h.Sum(nil))
 }
 
-func loadIgnoreFile(file string, seen map[string]bool) ([]Pattern, error) {
+func loadIgnoreFile(file string, modtimes map[string]time.Time) ([]Pattern, error) {
-	if seen[file] {
+	if _, ok := modtimes[file]; ok {
 		return nil, fmt.Errorf("Multiple include of ignore file %q", file)
 	}
-	seen[file] = true
 
 	fd, err := os.Open(file)
 	if err != nil {
@@ -233,10 +269,16 @@ func loadIgnoreFile(file string, seen map[string]bool) ([]Pattern, error) {
 	}
 	defer fd.Close()
 
-	return parseIgnoreFile(fd, file, seen)
+	info, err := fd.Stat()
+	if err != nil {
+		return nil, err
+	}
+	modtimes[file] = info.ModTime()
+
+	return parseIgnoreFile(fd, file, modtimes)
 }
 
-func parseIgnoreFile(fd io.Reader, currentFile string, seen map[string]bool) ([]Pattern, error) {
+func parseIgnoreFile(fd io.Reader, currentFile string, modtimes map[string]time.Time) ([]Pattern, error) {
 	var patterns []Pattern
 
 	defaultResult := resultInclude
@@ -302,7 +344,7 @@ func parseIgnoreFile(fd io.Reader, currentFile string, seen map[string]bool) ([]
 		} else if strings.HasPrefix(line, "#include ") {
 			includeRel := line[len("#include "):]
 			includeFile := filepath.Join(filepath.Dir(currentFile), includeRel)
-			includes, err := loadIgnoreFile(includeFile, seen)
+			includes, err := loadIgnoreFile(includeFile, modtimes)
 			if err != nil {
 				return fmt.Errorf("include of %q: %v", includeRel, err)
 			}

lib/ignore/ignore_test.go

@@ -14,6 +14,7 @@ import (
 	"path/filepath"
 	"runtime"
 	"testing"
+	"time"
 )
 
 func TestIgnore(t *testing.T) {
@@ -276,9 +277,12 @@ func TestCaching(t *testing.T) {
 		t.Fatal("Expected 4 cached results")
 	}
 
-	// Modify the include file, expect empty cache
+	// Modify the include file, expect empty cache. Ensure the timestamp on
+	// the file changes.
 
+	time.Sleep(time.Second)
 	fd2.WriteString("/z/\n")
+	fd2.Sync()
 
 	err = pats.Load(fd1.Name())
 	if err != nil {
@@ -307,7 +311,9 @@ func TestCaching(t *testing.T) {
 
 	// Modify the root file, expect cache to be invalidated
 
+	time.Sleep(time.Second)
 	fd1.WriteString("/a/\n")
+	fd1.Sync()
 
 	err = pats.Load(fd1.Name())
 	if err != nil {
@@ -472,6 +478,8 @@ func TestCacheReload(t *testing.T) {
 
 	// Rewrite file to match f1 and f3
 
+	time.Sleep(time.Second)
+
 	err = fd.Truncate(0)
 	if err != nil {
 		t.Fatal(err)