Suppress frequent changes to files (fixes #12)

model/model.go

@@ -50,6 +50,9 @@ type Model struct {
 	delete         bool
 
 	trace map[string]bool
+
+	fileLastChanged   map[string]time.Time
+	fileWasSuppressed map[string]int
 }
 
 const (
@@ -57,6 +60,9 @@ const (
 
 	idxBcastHoldtime = 15 * time.Second  // Wait at least this long after the last index modification
 	idxBcastMaxDelay = 120 * time.Second // Unless we've already waited this long
+
+	minFileHoldTimeS = 60  // Never allow file changes more often than this
+	maxFileHoldTimeS = 600 // Always allow file changes at least this often
 )
 
 var ErrNoSuchFile = errors.New("no such file")
@@ -66,15 +72,17 @@ var ErrNoSuchFile = errors.New("no such file")
 // for file data without altering the local repository in any way.
 func NewModel(dir string) *Model {
 	m := &Model{
-		dir:          dir,
+		dir:               dir,
-		global:       make(map[string]File),
+		global:            make(map[string]File),
-		local:        make(map[string]File),
+		local:             make(map[string]File),
-		remote:       make(map[string]map[string]File),
+		remote:            make(map[string]map[string]File),
-		need:         make(map[string]bool),
+		need:              make(map[string]bool),
-		nodes:        make(map[string]*protocol.Connection),
+		nodes:             make(map[string]*protocol.Connection),
-		rawConn:      make(map[string]io.ReadWriteCloser),
+		rawConn:           make(map[string]io.ReadWriteCloser),
-		lastIdxBcast: time.Now(),
+		lastIdxBcast:      time.Now(),
-		trace:        make(map[string]bool),
+		trace:             make(map[string]bool),
+		fileLastChanged:   make(map[string]time.Time),
+		fileWasSuppressed: make(map[string]int),
 	}
 
 	go m.broadcastIndexLoop()
@@ -304,6 +312,7 @@ func (m *Model) Request(nodeID, name string, offset uint64, size uint32, hash []
 }
 
 // ReplaceLocal replaces the local repository index with the given list of files.
+// Change suppression is applied to files changing too often.
 func (m *Model) ReplaceLocal(fs []File) {
 	m.Lock()
 	defer m.Unlock()
@@ -391,6 +400,28 @@ func (m *Model) AddConnection(conn io.ReadWriteCloser, nodeID string) {
 	}()
 }
 
+func (m *Model) shouldSuppressChange(name string) bool {
+	sup := shouldSuppressChange(m.fileLastChanged[name], m.fileWasSuppressed[name])
+	if sup {
+		m.fileWasSuppressed[name]++
+	} else {
+		m.fileWasSuppressed[name] = 0
+		m.fileLastChanged[name] = time.Now()
+	}
+	return sup
+}
+
+func shouldSuppressChange(lastChange time.Time, numChanges int) bool {
+	sinceLast := time.Since(lastChange)
+	if sinceLast > maxFileHoldTimeS*time.Second {
+		return false
+	}
+	if sinceLast < time.Duration((numChanges+2)*minFileHoldTimeS)*time.Second {
+		return true
+	}
+	return false
+}
+
 // protocolIndex returns the current local index in protocol data types.
 // Must be called with the read lock held.
 func (m *Model) protocolIndex() []protocol.FileInfo {

model/model_test.go

@@ -340,3 +340,28 @@ func TestRequest(t *testing.T) {
 		t.Errorf("Unexpected non nil data on insecure file read: %q", string(bs))
 	}
 }
+
+func TestSuppression(t *testing.T) {
+	var testdata = []struct {
+		lastChange time.Time
+		hold       int
+		result     bool
+	}{
+		{time.Unix(0, 0), 0, false},                    // First change
+		{time.Now().Add(-1 * time.Second), 0, true},    // Changed once one second ago, suppress
+		{time.Now().Add(-119 * time.Second), 0, true},  // Changed once 119 seconds ago, suppress
+		{time.Now().Add(-121 * time.Second), 0, false}, // Changed once 121 seconds ago, permit
+
+		{time.Now().Add(-179 * time.Second), 1, true},  // Suppressed once 179 seconds ago, suppress again
+		{time.Now().Add(-181 * time.Second), 1, false}, // Suppressed once 181 seconds ago, permit
+
+		{time.Now().Add(-599 * time.Second), 99, true},  // Suppressed lots of times, last allowed 599 seconds ago, suppress again
+		{time.Now().Add(-601 * time.Second), 99, false}, // Suppressed lots of times, last allowed 601 seconds ago, permit
+	}
+
+	for i, tc := range testdata {
+		if shouldSuppressChange(tc.lastChange, tc.hold) != tc.result {
+			t.Errorf("Incorrect result for test #%d: %v", i, tc)
+		}
+	}
+}

model/walk.go

@@ -9,6 +9,7 @@ import (
 	"path"
 	"path/filepath"
 	"strings"
+	"time"
 )
 
 const BlockSize = 128 * 1024
@@ -81,17 +82,38 @@ func (m *Model) genWalker(res *[]File, ign map[string][]string) filepath.WalkFun
 				// No change
 				*res = append(*res, hf)
 			} else {
+				m.Lock()
+				if m.shouldSuppressChange(rn) {
+					if m.trace["file"] {
+						log.Println("FILE: SUPPRESS:", rn, m.fileWasSuppressed[rn], time.Since(m.fileLastChanged[rn]))
+					}
+
+					if ok {
+						// Files that are ignored will be suppressed but don't actually exist in the local model
+						*res = append(*res, hf)
+					}
+					m.Unlock()
+					return nil
+				}
+				m.Unlock()
+
 				if m.trace["file"] {
 					log.Printf("FILE: Hash %q", p)
 				}
 				fd, err := os.Open(p)
 				if err != nil {
+					if m.trace["file"] {
+						log.Printf("FILE: %q: %v", p, err)
+					}
 					return nil
 				}
 				defer fd.Close()
 
 				blocks, err := Blocks(fd, BlockSize)
 				if err != nil {
+					if m.trace["file"] {
+						log.Printf("FILE: %q: %v", p, err)
+					}
 					return nil
 				}
 				f := File{