all: Fix FS watcher restarting and web UI indication (fixes #4923) (#4962)

cmd/syncthing/summaryservice.go

@@ -60,7 +60,7 @@ func (c *folderSummaryService) Stop() {
 // listenForUpdates subscribes to the event bus and makes note of folders that
 // need their data recalculated.
 func (c *folderSummaryService) listenForUpdates() {
-	sub := events.Default.Subscribe(events.LocalIndexUpdated | events.RemoteIndexUpdated | events.StateChanged | events.RemoteDownloadProgress | events.DeviceConnected)
+	sub := events.Default.Subscribe(events.LocalIndexUpdated | events.RemoteIndexUpdated | events.StateChanged | events.RemoteDownloadProgress | events.DeviceConnected | events.FolderWatchStateChanged)
 	defer events.Default.Unsubscribe(sub)
 
 	for {
@@ -105,14 +105,14 @@ func (c *folderSummaryService) listenForUpdates() {
 					// c.immediate must be nonblocking so that we can continue
 					// handling events.
 
+					c.foldersMut.Lock()
 					select {
 					case c.immediate <- folder:
-						c.foldersMut.Lock()
 						delete(c.folders, folder)
-						c.foldersMut.Unlock()
-
 					default:
+						c.folders[folder] = struct{}{}
 					}
+					c.foldersMut.Unlock()
 				}
 
 			default:

gui/default/assets/lang/lang-en.json

@@ -109,6 +109,7 @@
    "Files are moved to date stamped versions in a .stversions folder when replaced or deleted by Syncthing.": "Files are moved to date stamped versions in a .stversions folder when replaced or deleted by Syncthing.",
    "Files are protected from changes made on other devices, but changes made on this device will be sent to the rest of the cluster.": "Files are protected from changes made on other devices, but changes made on this device will be sent to the rest of the cluster.",
    "Filesystem Notifications": "Filesystem Notifications",
+   "Filesystem Watcher Errors": "Filesystem Watcher Errors",
    "Filter by date": "Filter by date",
    "Filter by name": "Filter by name",
    "Folder": "Folder",
@@ -117,6 +118,7 @@
    "Folder Path": "Folder Path",
    "Folder Type": "Folder Type",
    "Folders": "Folders",
+   "For the following folders an error occurred while starting to watch for changes. It will be retried every minute, so the errors might go away soon. If they persist, try to fix the underlying issue and ask for help if you can't.": "For the following folders an error occurred while starting to watch for changes. It will be retried every minute, so the errors might go away soon. If they persist, try to fix the underlying issue and ask for help if you can't.",
    "Full Rescan Interval (s)": "Full Rescan Interval (s)",
    "GUI": "GUI",
    "GUI Authentication Password": "GUI Authentication Password",

gui/default/index.html

@@ -257,6 +257,33 @@
       </div>
     </div>
 
+    <!-- Panel: FS watcher errors -->
+
+    <div ng-if="sizeOf(fsWatcherErrorMap()) > 0" class="row">
+      <div class="col-md-12">
+        <div class="panel panel-warning">
+          <div class="panel-heading">
+            <h3 class="panel-title">
+              <div class="panel-icon">
+                <span class="fas fa-exclamation-circle"></span>
+              </div>
+              <span translate>Filesystem Watcher Errors</span>
+            </h3>
+          </div>
+          <div class="panel-body">
+            <p>
+              <span translate>For the following folders an error occurred while starting to watch for changes. It will be retried every minute, so the errors might go away soon. If they persist, try to fix the underlying issue and ask for help if you can't.</span>&emsp;<a href="https://forum.syncthing.net" target="_blank"><span class="fas fa-question-circle"></span>&nbsp;<span translate>Support</span></a>
+            </p>
+            <table>
+              <tr ng-repeat="(id, err) in fsWatcherErrorMap()">
+                <td>{{folderLabel(id)}}</td><td>{{err}}</td>
+              </tr>
+            </table>
+          </div>
+        </div>
+      </div>
+    </div>
+
     <div ng-if="config && config.options && config.options.unackedNotificationIDs" ng-include="'syncthing/core/notifications.html'"></div>
 
     <!-- First regular row -->
@@ -378,11 +405,11 @@
                             <span class="far fa-clock"></span>&nbsp;{{folder.rescanIntervalS | duration}}&ensp;
                             <span class="fas fa-eye-slash"></span>&nbsp;<span translate>Disabled</span>
                           </span>
-                          <span ng-if="folder.fsWatcherEnabled && (!model[folder.id].watchError || folder.paused)" tooltip data-original-title="{{'Periodic scanning at given interval and enabled watching for changes' | translate}}">
+                          <span ng-if="folder.fsWatcherEnabled && (!model[folder.id].watchError || folder.paused || folderStatus(folder) === 'stopped')" tooltip data-original-title="{{'Periodic scanning at given interval and enabled watching for changes' | translate}}">
                             <span class="far fa-clock"></span>&nbsp;{{folder.rescanIntervalS | duration}}&ensp;
                             <span class="fas fa-eye"></span>&nbsp;<span translate>Enabled</span>
                           </span>
-                          <span ng-if="folder.fsWatcherEnabled && !folder.paused && model[folder.id].watchError" tooltip data-original-title="{{'Periodic scanning at given interval and failed setting up watching for changes, retrying every 1m:' | translate}}<br/>{{model[folder.id].watchError}}">
+                          <span ng-if="folder.fsWatcherEnabled && !folder.paused && folderStatus(folder) !== 'stopped' && model[folder.id].watchError" tooltip data-original-title="{{'Periodic scanning at given interval and failed setting up watching for changes, retrying every 1m:' | translate}}<br/>{{model[folder.id].watchError}}">
                             <span class="far fa-clock"></span>&nbsp;{{folder.rescanIntervalS | duration}}&ensp;
                             <span class="fas fa-eye-slash"></span>&nbsp;<span translate>Failed to setup, retrying</span>
                           </span>
@@ -392,11 +419,11 @@
                             <span class="far fa-clock"></span>&nbsp;<span translate>Disabled</span>&ensp;
                             <span class="fas fa-eye-slash"></span>&nbsp;<span translate>Disabled</span>
                           </span>
-                          <span ng-if="folder.fsWatcherEnabled && (!model[folder.id].watchError || folder.paused)" tooltip data-original-title="{{'Disabled periodic scanning and enabled watching for changes' | translate}}">
+                          <span ng-if="folder.fsWatcherEnabled && (!model[folder.id].watchError || folder.paused || folderStatus(folder) === 'stopped')" tooltip data-original-title="{{'Disabled periodic scanning and enabled watching for changes' | translate}}">
                             <span class="far fa-clock"></span>&nbsp;<span translate>Disabled</span>&ensp;
                             <span class="fas fa-eye"></span>&nbsp;<span translate>Enabled</span>
                           </span>
-                          <span ng-if="folder.fsWatcherEnabled && !folder.paused && model[folder.id].watchError" tooltip data-original-title="{{'Disabled periodic scanning and failed setting up watching for changes, retrying every 1m:' | translate}}<br/>{{model[folder.id].watchError}}">
+                          <span ng-if="folder.fsWatcherEnabled && !folder.paused && folderStatus(folder) !== 'stopped' && model[folder.id].watchError" tooltip data-original-title="{{'Disabled periodic scanning and failed setting up watching for changes, retrying every 1m:' | translate}}<br/>{{model[folder.id].watchError}}">
                             <span class="far fa-clock"></span>&nbsp;<span translate>Disabled</span>&ensp;
                             <span class="fas fa-eye-slash"></span>&nbsp;<span translate>Failed to setup, retrying</span>
                           </span>

gui/default/syncthing/core/syncthingController.js

@@ -1466,6 +1466,16 @@ angular.module('syncthing.core')
             $http.post(urlbase + '/system/error/clear');
         };
 
+        $scope.fsWatcherErrorMap = function () {
+            var errs = {}
+            $.each($scope.folders, function (id, cfg) {
+                if (cfg.fsWatcherEnabled && $scope.model[cfg.id] && $scope.model[id].watchError && !cfg.paused && $scope.folderStatus(cfg) !== 'stopped') {
+                    errs[id] = $scope.model[id].watchError;
+                }
+            });
+            return errs;
+        }
+
         $scope.friendlyDevices = function (str) {
             for (var i = 0; i < $scope.devices.length; i++) {
                 var cfg = $scope.devices[i];
@@ -2248,6 +2258,9 @@ angular.module('syncthing.core')
         };
 
         $scope.sizeOf = function (dict) {
+            if (dict === undefined) {
+                return 0;
+            }
             return Object.keys(dict).length;
         };
 

lib/events/events.go

@@ -44,6 +44,7 @@ const (
 	FolderScanProgress
 	FolderPaused
 	FolderResumed
+	FolderWatchStateChanged
 	ListenAddressesChanged
 	LoginAttempt
 
@@ -110,6 +111,8 @@ func (t EventType) String() string {
 		return "ListenAddressesChanged"
 	case LoginAttempt:
 		return "LoginAttempt"
+	case FolderWatchStateChanged:
+		return "FolderWatchStateChanged"
 	default:
 		return "Unknown"
 	}
@@ -187,6 +190,8 @@ func UnmarshalEventType(s string) EventType {
 		return ListenAddressesChanged
 	case "LoginAttempt":
 		return LoginAttempt
+	case "FolderWatchStateChanged":
+		return FolderWatchStateChanged
 	default:
 		return 0
 	}

lib/model/folder.go

@@ -15,6 +15,7 @@ import (
 
 	"github.com/syncthing/syncthing/lib/config"
 	"github.com/syncthing/syncthing/lib/db"
+	"github.com/syncthing/syncthing/lib/events"
 	"github.com/syncthing/syncthing/lib/ignore"
 	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syncthing/syncthing/lib/sync"
@@ -76,13 +77,14 @@ func newFolder(model *Model, cfg config.FolderConfiguration) folder {
 		scanNow:             make(chan rescanRequest),
 		scanDelay:           make(chan time.Duration),
 		initialScanFinished: make(chan struct{}),
+		stopped:             make(chan struct{}),
 
 		pullScheduled: make(chan struct{}, 1), // This needs to be 1-buffered so that we queue a pull if we're busy when it comes.
 
-		watchCancel: func() {},
+		watchCancel:      func() {},
-		watchErr:    errWatchNotStarted,
+		restartWatchChan: make(chan struct{}, 1),
-		watchErrMut: sync.NewMutex(),
+		watchErr:         errWatchNotStarted,
-		stopped:     make(chan struct{}),
+		watchErrMut:      sync.NewMutex(),
 	}
 }
 
@@ -295,8 +297,19 @@ func (f *folder) WatchError() error {
 func (f *folder) stopWatch() {
 	f.watchCancel()
 	f.watchErrMut.Lock()
+	prevErr := f.watchErr
 	f.watchErr = errWatchNotStarted
 	f.watchErrMut.Unlock()
+	if prevErr != errWatchNotStarted {
+		data := map[string]interface{}{
+			"folder": f.ID,
+			"to":     errWatchNotStarted.Error(),
+		}
+		if prevErr != nil {
+			data["from"] = prevErr.Error()
+		}
+		events.Default.Log(events.FolderWatchStateChanged, data)
+	}
 }
 
 // scheduleWatchRestart makes sure watching is restarted from the main for loop
@@ -316,7 +329,7 @@ func (f *folder) scheduleWatchRestart() {
 func (f *folder) restartWatch() {
 	f.stopWatch()
 	f.startWatch()
-	f.Scan(nil)
+	f.scanSubdirs(nil)
 }
 
 // startWatch should only ever be called synchronously. If you want to use
@@ -343,11 +356,23 @@ func (f *folder) startWatchAsync(ctx context.Context, ignores *ignore.Matcher) {
 			prevErr := f.watchErr
 			f.watchErr = err
 			f.watchErrMut.Unlock()
+			if err != prevErr {
+				data := map[string]interface{}{
+					"folder": f.ID,
+				}
+				if prevErr != nil {
+					data["from"] = prevErr.Error()
+				}
+				if err != nil {
+					data["to"] = err.Error()
+				}
+				events.Default.Log(events.FolderWatchStateChanged, data)
+			}
 			if err != nil {
 				if prevErr == errWatchNotStarted {
-					l.Warnf("Failed to start filesystem watcher for folder %s: %v", f.Description(), err)
+					l.Infof("Error while trying to start filesystem watcher for folder %s, trying again in 1min: %v", f.Description(), err)
 				} else {
-					l.Debugf("Failed to start filesystem watcher for folder %s again: %v", f.Description(), err)
+					l.Debugf("Repeat error while trying to start filesystem watcher for folder %s, trying again in 1min: %v", f.Description(), err)
 				}
 				timer.Reset(time.Minute)
 				continue