all: Display fs watcher status and retry starting it (ref #4552) (#4713)

cmd/syncthing/gui.go

@@ -112,6 +112,7 @@ type modelIntf interface {
 	State(folder string) (string, time.Time, error)
 	UsageReportingStats(version int, preview bool) map[string]interface{}
 	PullErrors(folder string) ([]model.FileError, error)
+	WatchError(folder string) error
 }
 
 type configIntf interface {
@@ -733,6 +734,11 @@ func folderSummary(cfg configIntf, m modelIntf, folder string) (map[string]inter
 		}
 	}
 
+	err = m.WatchError(folder)
+	if err != nil {
+		res["watchError"] = err.Error()
+	}
+
 	return res, nil
 }
 

cmd/syncthing/mocked_model_test.go

@@ -136,3 +136,7 @@ func (m *mockedModel) UsageReportingStats(version int, preview bool) map[string]
 func (m *mockedModel) PullErrors(folder string) ([]model.FileError, error) {
 	return nil, nil
 }
+
+func (m *mockedModel) WatchError(folder string) error {
+	return nil
+}

gui/default/assets/lang/lang-en.json

@@ -65,6 +65,7 @@
    "Device that last modified the item": "Device that last modified the item",
    "Devices": "Devices",
    "Disabled": "Disabled",
+   "Disabled periodic scanning": "Disabled periodic scanning",
    "Disconnected": "Disconnected",
    "Discovered": "Discovered",
    "Discovery": "Discovery",
@@ -89,6 +90,7 @@
    "Error": "Error",
    "External File Versioning": "External File Versioning",
    "Failed Items": "Failed Items",
+   "Failed to setup, retrying": "Failed to setup, retrying",
    "Failure to connect to IPv6 servers is expected if there is no IPv6 connectivity.": "Failure to connect to IPv6 servers is expected if there is no IPv6 connectivity.",
    "File Pull Order": "File Pull Order",
    "File Versioning": "File Versioning",
@@ -183,6 +185,7 @@
    "Pause": "Pause",
    "Pause All": "Pause All",
    "Paused": "Paused",
+   "Periodic scan every": "Periodic scan every",
    "Please consult the release notes before performing a major upgrade.": "Please consult the release notes before performing a major upgrade.",
    "Please set a GUI Authentication User and Password in the Settings dialog.": "Please set a GUI Authentication User and Password in the Settings dialog.",
    "Please wait": "Please wait",
@@ -205,6 +208,7 @@
    "Rescan": "Rescan",
    "Rescan All": "Rescan All",
    "Rescan Interval": "Rescan Interval",
+   "Rescans": "Rescans",
    "Restart": "Restart",
    "Restart Needed": "Restart Needed",
    "Restarting": "Restarting",
@@ -213,6 +217,7 @@
    "Resume": "Resume",
    "Resume All": "Resume All",
    "Reused": "Reused",
+   "Running": "Running",
    "Save": "Save",
    "Scan Time Remaining": "Scan Time Remaining",
    "Scanning": "Scanning",

gui/default/index.html

@@ -371,16 +371,35 @@
                         <span translate>Yes</span>
                       </td>
                     </tr>
-                    <tr ng-if="folder.fsNotifications">
+                    <tr>
-                      <th><span class="fa fa-fw fa-bolt"></span>&nbsp;<span translate>Filesystem Notifications</span></th>
+                      <th><span class="fa fa-fw fa-refresh"></span>&nbsp;<span translate>Rescans</span></th>
                       <td class="text-right">
-                        <span translate>Yes</span>
+                        <span ng-if="folder.rescanIntervalS > 0 && !folder.fsWatcherEnabled" tooltip data-original-title="{{'Periodic scan every' | translate}} {{folder.rescanIntervalS | duration}} {{'and disabled watching for changes' | translate}}">
+                          <span class="fa fa-clock-o"></span>&nbsp;{{folder.rescanIntervalS | duration}}&ensp;
+                          <span class="fa fa-eye-slash"></span><span translate>Disabled</span>
+                        </span>
+                        <span ng-if="folder.rescanIntervalS > 0 && folder.fsWatcherEnabled && !model[folder.id].watchError" tooltip data-original-title="{{'Periodic scan every' | translate}} {{folder.rescanIntervalS | duration}} {{'and watching for changes' | translate}}">
+                          <span class="fa fa-clock-o"></span>{{folder.rescanIntervalS | duration}}&ensp;
+                          <span class="fa fa-eye"></span>&nbsp;<span translate>Running</span>
+                        </span>
+                        <span ng-if="folder.rescanIntervalS > 0 && folder.fsWatcherEnabled && model[folder.id].watchError" tooltip data-original-title="{{'Periodic scan every' | translate}} {{folder.rescanIntervalS | duration}} {{'and failed setting up watching for changes, retrying every 1m:' | translate}}<br/>{{model[folder.id].watchError}}">
+                          <span class="fa fa-clock-o"></span>{{folder.rescanIntervalS | duration}}&ensp;
+                          <span class="fa fa-eye-slash"></span>&nbsp;<span translate>Failed to setup, retrying</span>
+                        </span>
+                        <span ng-if="folder.rescanIntervalS <= 0 && !folder.fsWatcherEnabled" tooltip data-original-title="{{'Disabled periodic scanning' | translate}} {{'and disabled watching for changes' | translate}}">
+                          <span class="fa fa-clock-o"></span>&nbsp;<span translate>Disabled</span>&ensp;
+                          <span class="fa fa-eye-slash"></span>&nbsp;<span translate>Disabled</span>
+                        </span>
+                        <span ng-if="folder.rescanIntervalS <= 0 && folder.fsWatcherEnabled && !model[folder.id].watchError" tooltip data-original-title="{{'Disabled periodic scanning' | translate}} {{'and watching for changes' | translate}}">
+                          <span class="fa fa-clock-o"></span>&nbsp;<span translate>Disabled</span>&ensp;
+                          <span class="fa fa-eye"></span>&nbsp;<span translate>Running</span>
+                        </span>
+                        <span ng-if="folder.rescanIntervalS <= 0 && folder.fsWatcherEnabled && model[folder.id].watchError" tooltip data-original-title="{{'Disabled periodic scanning' | translate}} {{'and failed setting up watching for changes, retrying every 1m:' | translate}}<br/>{{model[folder.id].watchError}}">
+                          <span class="fa fa-clock-o"></span>&nbsp;<span translate>Disabled</span>&ensp;
+                          <span class="fa fa-eye-slash"></span>&nbsp;<span translate>Failed to setup, retrying</span>
+                        </span>
                       </td>
                     </tr>
-                    <tr ng-if="(folder.rescanIntervalS != 60 && !folder.fsNotifications) || (folder.rescanIntervalS != 3600 && folder.fsNotifications)">
-                      <th><span class="fa fa-fw fa-refresh"></span>&nbsp;<span translate>Rescan Interval</span></th>
-                      <td class="text-right">{{folder.rescanIntervalS}} s</td>
-                    </tr>
                     <tr ng-if="folder.order != 'random'">
                       <th><span class="fa fa-fw fa-sort"></span>&nbsp;<span translate>File Pull Order</span></th>
                       <td class="text-right" ng-switch="folder.order">

lib/model/folder.go

@@ -8,12 +8,17 @@ package model
 
 import (
 	"context"
+	"errors"
 	"time"
 
 	"github.com/syncthing/syncthing/lib/config"
+	"github.com/syncthing/syncthing/lib/ignore"
+	"github.com/syncthing/syncthing/lib/sync"
 	"github.com/syncthing/syncthing/lib/watchaggregator"
 )
 
+var errWatchNotStarted error = errors.New("not started")
+
 type folder struct {
 	stateTracker
 	config.FolderConfiguration
@@ -26,6 +31,8 @@ type folder struct {
 	watchCancel         context.CancelFunc
 	watchChan           chan []string
 	restartWatchChan    chan struct{}
+	watchErr            error
+	watchErrMut         sync.Mutex
 }
 
 func newFolder(model *Model, cfg config.FolderConfiguration) folder {
@@ -41,6 +48,8 @@ func newFolder(model *Model, cfg config.FolderConfiguration) folder {
 		model:               model,
 		initialScanFinished: make(chan struct{}),
 		watchCancel:         func() {},
+		watchErr:            errWatchNotStarted,
+		watchErrMut:         sync.NewMutex(),
 	}
 }
 
@@ -127,28 +136,22 @@ func (f *folder) scanTimerFired() {
 	f.scan.Reschedule()
 }
 
-func (f *folder) startWatch() {
+func (f *folder) WatchError() error {
-	ctx, cancel := context.WithCancel(f.ctx)
+	f.watchErrMut.Lock()
-	f.model.fmut.RLock()
+	defer f.watchErrMut.Unlock()
-	ignores := f.model.folderIgnores[f.folderID]
+	return f.watchErr
-	f.model.fmut.RUnlock()
-	eventChan, err := f.Filesystem().Watch(".", ignores, ctx, f.IgnorePerms)
-	if err != nil {
-		l.Warnf("Failed to start filesystem watcher for folder %s: %v", f.Description(), err)
-	} else {
-		f.watchChan = make(chan []string)
-		f.watchCancel = cancel
-		watchaggregator.Aggregate(eventChan, f.watchChan, f.FolderConfiguration, f.model.cfg, ctx)
-		l.Infoln("Started filesystem watcher for folder", f.Description())
-	}
 }
 
-func (f *folder) restartWatch() {
+// stopWatch immediately aborts watching and may be called asynchronously
+func (f *folder) stopWatch() {
 	f.watchCancel()
-	f.startWatch()
+	f.watchErrMut.Lock()
-	f.Scan(nil)
+	f.watchErr = errWatchNotStarted
+	f.watchErrMut.Unlock()
 }
 
+// scheduleWatchRestart makes sure watching is restarted from the main for loop
+// in a folder's Serve and thus may be called asynchronously (e.g. when ignores change).
 func (f *folder) scheduleWatchRestart() {
 	select {
 	case f.restartWatchChan <- struct{}{}:
@@ -159,6 +162,56 @@ func (f *folder) scheduleWatchRestart() {
 	}
 }
 
+// restartWatch should only ever be called synchronously. If you want to use
+// this asynchronously, you should probably use scheduleWatchRestart instead.
+func (f *folder) restartWatch() {
+	f.stopWatch()
+	f.startWatch()
+	f.Scan(nil)
+}
+
+// startWatch should only ever be called synchronously. If you want to use
+// this asynchronously, you should probably use scheduleWatchRestart instead.
+func (f *folder) startWatch() {
+	ctx, cancel := context.WithCancel(f.ctx)
+	f.model.fmut.RLock()
+	ignores := f.model.folderIgnores[f.folderID]
+	f.model.fmut.RUnlock()
+	f.watchChan = make(chan []string)
+	f.watchCancel = cancel
+	go f.startWatchAsync(ctx, ignores)
+}
+
+// startWatchAsync tries to start the filesystem watching and retries every minute on failure.
+// It is a convenience function that should not be used except in startWatch.
+func (f *folder) startWatchAsync(ctx context.Context, ignores *ignore.Matcher) {
+	timer := time.NewTimer(0)
+	for {
+		select {
+		case <-timer.C:
+			eventChan, err := f.Filesystem().Watch(".", ignores, ctx, f.IgnorePerms)
+			f.watchErrMut.Lock()
+			prevErr := f.watchErr
+			f.watchErr = err
+			f.watchErrMut.Unlock()
+			if err != nil {
+				if prevErr == errWatchNotStarted {
+					l.Warnf("Failed to start filesystem watcher for folder %s: %v", f.Description(), err)
+				} else {
+					l.Debugf("Failed to start filesystem watcher for folder %s again: %v", f.Description(), err)
+				}
+				timer.Reset(time.Minute)
+				continue
+			}
+			watchaggregator.Aggregate(eventChan, f.watchChan, f.FolderConfiguration, f.model.cfg, ctx)
+			l.Debugln("Started filesystem watcher for folder", f.Description())
+			return
+		case <-ctx.Done():
+			return
+		}
+	}
+}
+
 func (f *folder) setError(err error) {
 	_, _, oldErr := f.getState()
 	if (err != nil && oldErr != nil && oldErr.Error() == err.Error()) || (err == nil && oldErr == nil) {
@@ -177,7 +230,7 @@ func (f *folder) setError(err error) {
 
 	if f.FSWatcherEnabled {
 		if err != nil {
-			f.watchCancel()
+			f.stopWatch()
 		} else {
 			f.scheduleWatchRestart()
 		}

lib/model/model.go

@@ -65,6 +65,7 @@ type service interface {
 	Stop()
 	CheckHealth() error
 	PullErrors() []FileError
+	WatchError() error
 
 	getState() (folderState, time.Time, error)
 	setState(state folderState)
@@ -2213,6 +2214,15 @@ func (m *Model) PullErrors(folder string) ([]FileError, error) {
 	return m.folderRunners[folder].PullErrors(), nil
 }
 
+func (m *Model) WatchError(folder string) error {
+	m.fmut.RLock()
+	defer m.fmut.RUnlock()
+	if err := m.checkFolderRunningLocked(folder); err != nil {
+		return err
+	}
+	return m.folderRunners[folder].WatchError()
+}
+
 func (m *Model) Override(folder string) {
 	m.fmut.RLock()
 	fs, ok := m.folderFiles[folder]