cmd/syncthing: Add more stats to usage reports (ref #3628)

GitHub-Pull-Request: https://github.com/syncthing/syncthing/pull/4347

cmd/syncthing/gui.go

@@ -100,6 +100,7 @@ type modelIntf interface {
 	CurrentSequence(folder string) (int64, bool)
 	RemoteSequence(folder string) (int64, bool)
 	State(folder string) (string, time.Time, error)
+	UsageReportingStats(version int) map[string]interface{}
 }
 
 type configIntf interface {
@@ -119,6 +120,7 @@ type configIntf interface {
 
 type connectionsIntf interface {
 	Status() map[string]interface{}
+	NATType() string
 }
 
 type rater interface {
@@ -800,18 +802,6 @@ func (s *apiService) postSystemConfig(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
-	// Fixup usage reporting settings
-
-	if curAcc := s.cfg.Options().URAccepted; to.Options.URAccepted > curAcc {
-		// UR was enabled
-		to.Options.URAccepted = usageReportVersion
-		to.Options.URUniqueID = rand.String(8)
-	} else if to.Options.URAccepted < curAcc {
-		// UR was disabled
-		to.Options.URAccepted = -1
-		to.Options.URUniqueID = ""
-	}
-
 	// Activate and save
 
 	if err := s.cfg.Replace(to); err != nil {
@@ -903,6 +893,7 @@ func (s *apiService) getSystemStatus(w http.ResponseWriter, r *http.Request) {
 	// gives us percent
 	res["cpuPercent"] = s.cpu.Rate() / 10 / float64(runtime.NumCPU())
 	res["pathSeparator"] = string(filepath.Separator)
+	res["urVersionMax"] = usageReportVersion
 	res["uptime"] = int(time.Since(startTime).Seconds())
 	res["startTime"] = startTime
 
@@ -981,7 +972,11 @@ func (s *apiService) getSystemDiscovery(w http.ResponseWriter, r *http.Request)
 }
 
 func (s *apiService) getReport(w http.ResponseWriter, r *http.Request) {
-	sendJSON(w, reportData(s.cfg, s.model))
+	version := usageReportVersion
+	if val, _ := strconv.Atoi(r.URL.Query().Get("version")); val > 0 {
+		version = val
+	}
+	sendJSON(w, reportData(s.cfg, s.model, s.connectionsService, version))
 }
 
 func (s *apiService) getRandomString(w http.ResponseWriter, r *http.Request) {

cmd/syncthing/main.go

@@ -882,24 +882,14 @@ func syncthingMain(runtimeOptions RuntimeOptions) {
 		// Unique ID will be set and config saved below if necessary.
 	}
 
-	if opts.URAccepted > 0 && opts.URAccepted < usageReportVersion {
+	if opts.URUniqueID == "" {
-		l.Infoln("Anonymous usage report has changed; revoking acceptance")
-		opts.URAccepted = 0
-		opts.URUniqueID = ""
-		cfg.SetOptions(opts)
-	}
-
-	if opts.URAccepted >= usageReportVersion && opts.URUniqueID == "" {
-		// Generate and save a new unique ID if it is missing.
 		opts.URUniqueID = rand.String(8)
 		cfg.SetOptions(opts)
 		cfg.Save()
 	}
 
-	// The usageReportingManager registers itself to listen to configuration
+	usageReportingSvc := newUsageReportingService(cfg, m, connectionsService)
-	// changes, and there's nothing more we need to tell it from the outside.
+	mainService.Add(usageReportingSvc)
-	// Hence we don't keep the returned pointer.
-	newUsageReportingManager(cfg, m)
 
 	if opts.RestartOnWakeup {
 		go standbyMonitor()

cmd/syncthing/mocked_connections_test.go

@@ -11,3 +11,7 @@ type mockedConnections struct{}
 func (m *mockedConnections) Status() map[string]interface{} {
 	return nil
 }
+
+func (m *mockedConnections) NATType() string {
+	return ""
+}

cmd/syncthing/mocked_model_test.go

@@ -114,3 +114,7 @@ func (m *mockedModel) RemoteSequence(folder string) (int64, bool) {
 func (m *mockedModel) State(folder string) (string, time.Time, error) {
 	return "", time.Time{}, nil
 }
+
+func (m *mockedModel) UsageReportingStats(version int) map[string]interface{} {
+	return nil
+}

cmd/syncthing/usage_report.go

@@ -12,7 +12,6 @@ import (
 	"crypto/rand"
 	"crypto/tls"
 	"encoding/json"
-	"fmt"
 	"net/http"
 	"runtime"
 	"sort"
@@ -20,71 +19,25 @@ import (
 	"time"
 
 	"github.com/syncthing/syncthing/lib/config"
+	"github.com/syncthing/syncthing/lib/connections"
 	"github.com/syncthing/syncthing/lib/dialer"
 	"github.com/syncthing/syncthing/lib/model"
 	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syncthing/syncthing/lib/scanner"
 	"github.com/syncthing/syncthing/lib/upgrade"
-	"github.com/thejerf/suture"
 )
 
 // Current version number of the usage report, for acceptance purposes. If
 // fields are added or changed this integer must be incremented so that users
 // are prompted for acceptance of the new report.
-const usageReportVersion = 2
+const usageReportVersion = 3
-
-type usageReportingManager struct {
-	cfg   *config.Wrapper
-	model *model.Model
-	sup   *suture.Supervisor
-}
-
-func newUsageReportingManager(cfg *config.Wrapper, m *model.Model) *usageReportingManager {
-	mgr := &usageReportingManager{
-		cfg:   cfg,
-		model: m,
-	}
-
-	// Start UR if it's enabled.
-	mgr.CommitConfiguration(config.Configuration{}, cfg.RawCopy())
-
-	// Listen to future config changes so that we can start and stop as
-	// appropriate.
-	cfg.Subscribe(mgr)
-
-	return mgr
-}
-
-func (m *usageReportingManager) VerifyConfiguration(from, to config.Configuration) error {
-	return nil
-}
-
-func (m *usageReportingManager) CommitConfiguration(from, to config.Configuration) bool {
-	if to.Options.URAccepted >= usageReportVersion && m.sup == nil {
-		// Usage reporting was turned on; lets start it.
-		service := newUsageReportingService(m.cfg, m.model)
-		m.sup = suture.NewSimple("usageReporting")
-		m.sup.Add(service)
-		m.sup.ServeBackground()
-	} else if to.Options.URAccepted < usageReportVersion && m.sup != nil {
-		// Usage reporting was turned off
-		m.sup.Stop()
-		m.sup = nil
-	}
-
-	return true
-}
-
-func (m *usageReportingManager) String() string {
-	return fmt.Sprintf("usageReportingManager@%p", m)
-}
 
 // reportData returns the data to be sent in a usage report. It's used in
 // various places, so not part of the usageReportingManager object.
-func reportData(cfg configIntf, m modelIntf) map[string]interface{} {
+func reportData(cfg configIntf, m modelIntf, connectionsService connectionsIntf, version int) map[string]interface{} {
 	opts := cfg.Options()
 	res := make(map[string]interface{})
-	res["urVersion"] = usageReportVersion
+	res["urVersion"] = version
 	res["uniqueID"] = opts.URUniqueID
 	res["version"] = Version
 	res["longVersion"] = LongVersion
@@ -227,25 +180,40 @@ func reportData(cfg configIntf, m modelIntf) map[string]interface{} {
 	res["upgradeAllowedAuto"] = !(upgrade.DisabledByCompilation || noUpgradeFromEnv) && opts.AutoUpgradeIntervalH > 0
 	res["upgradeAllowedPre"] = !(upgrade.DisabledByCompilation || noUpgradeFromEnv) && opts.AutoUpgradeIntervalH > 0 && opts.UpgradeToPreReleases
 
+	if version >= 3 {
+		res["uptime"] = time.Now().Sub(startTime).Seconds()
+		res["natType"] = connectionsService.NATType()
+	}
+
+	for key, value := range m.UsageReportingStats(version) {
+		res[key] = value
+	}
+
 	return res
 }
 
 type usageReportingService struct {
-	cfg   *config.Wrapper
+	cfg                *config.Wrapper
-	model *model.Model
+	model              *model.Model
-	stop  chan struct{}
+	connectionsService *connections.Service
+	forceRun           chan struct{}
+	stop               chan struct{}
 }
 
-func newUsageReportingService(cfg *config.Wrapper, model *model.Model) *usageReportingService {
+func newUsageReportingService(cfg *config.Wrapper, model *model.Model, connectionsService *connections.Service) *usageReportingService {
-	return &usageReportingService{
+	svc := &usageReportingService{
-		cfg:   cfg,
+		cfg:                cfg,
-		model: model,
+		model:              model,
-		stop:  make(chan struct{}),
+		connectionsService: connectionsService,
+		forceRun:           make(chan struct{}),
+		stop:               make(chan struct{}),
 	}
+	cfg.Subscribe(svc)
+	return svc
 }
 
 func (s *usageReportingService) sendUsageReport() error {
-	d := reportData(s.cfg, s.model)
+	d := reportData(s.cfg, s.model, s.connectionsService, s.cfg.Options().URAccepted)
 	var b bytes.Buffer
 	json.NewEncoder(&b).Encode(d)
 
@@ -264,27 +232,45 @@ func (s *usageReportingService) sendUsageReport() error {
 
 func (s *usageReportingService) Serve() {
 	s.stop = make(chan struct{})
-
+	t := time.NewTimer(time.Duration(s.cfg.Options().URInitialDelayS) * time.Second)
-	l.Infoln("Starting usage reporting")
-	defer l.Infoln("Stopping usage reporting")
-
-	t := time.NewTimer(time.Duration(s.cfg.Options().URInitialDelayS) * time.Second) // time to initial report at start
 	for {
 		select {
 		case <-s.stop:
 			return
+		case <-s.forceRun:
+			t.Reset(0)
 		case <-t.C:
-			err := s.sendUsageReport()
+			if s.cfg.Options().URAccepted >= 2 {
-			if err != nil {
+				err := s.sendUsageReport()
-				l.Infoln("Usage report:", err)
+				if err != nil {
+					l.Infoln("Usage report:", err)
+				} else {
+					l.Infof("Sent usage report (version %d)", s.cfg.Options().URAccepted)
+				}
 			}
 			t.Reset(24 * time.Hour) // next report tomorrow
 		}
 	}
 }
 
+func (s *usageReportingService) VerifyConfiguration(from, to config.Configuration) error {
+	return nil
+}
+
+func (s *usageReportingService) CommitConfiguration(from, to config.Configuration) bool {
+	if from.Options.URAccepted != to.Options.URAccepted || from.Options.URUniqueID != to.Options.URUniqueID || from.Options.URURL != to.Options.URURL {
+		s.forceRun <- struct{}{}
+	}
+	return true
+}
+
 func (s *usageReportingService) Stop() {
 	close(s.stop)
+	close(s.forceRun)
+}
+
+func (usageReportingService) String() string {
+	return "usageReportingService"
 }
 
 // cpuBench returns CPU performance as a measure of single threaded SHA-256 MiB/s

gui/default/syncthing/core/modalDirective.js

@@ -1,6 +1,8 @@
 angular.module('syncthing.core')
     .directive('modal', function () {
         return {
+            // If you ever change any of the petroglyphs below, please search for $parent.$parent,
+            // as some templates rely on the way scope is composed in this case.
             restrict: 'E',
             templateUrl: 'modal.html',
             replace: true,

gui/default/syncthing/core/syncthingController.js

@@ -33,6 +33,8 @@ angular.module('syncthing.core')
         $scope.folderRejections = {};
         $scope.protocolChanged = false;
         $scope.reportData = {};
+        $scope.reportDataPreview = {};
+        $scope.reportDataPreviewVersion = '';
         $scope.reportPreview = false;
         $scope.folders = {};
         $scope.seenError = '';
@@ -133,7 +135,11 @@ angular.module('syncthing.core')
             }).error($scope.emitHTTPError);
 
             $http.get(urlbase + '/svc/report').success(function (data) {
-                $scope.reportData = data;
+                $scope.reportDataPreview = $scope.reportData = data;
+                if ($scope.system && $scope.config.options.urSeen < $scope.system.urVersionMax) {
+                    // Usage reporting format has changed, prompt the user to re-accept.
+                    $('#ur').modal();
+                }
             }).error($scope.emitHTTPError);
 
             $http.get(urlbase + '/system/upgrade').success(function (data) {
@@ -376,6 +382,7 @@ angular.module('syncthing.core')
             $scope.config = config;
             $scope.config.options._listenAddressesStr = $scope.config.options.listenAddresses.join(', ');
             $scope.config.options._globalAnnounceServersStr = $scope.config.options.globalAnnounceServers.join(', ');
+            $scope.config.options._urAcceptedStr = "" + $scope.config.options.urAccepted;
 
             $scope.devices = $scope.config.devices;
             $scope.devices.forEach(function (deviceCfg) {
@@ -412,6 +419,10 @@ angular.module('syncthing.core')
                 $scope.myID = data.myID;
                 $scope.system = data;
 
+                if ($scope.reportDataPreviewVersion === '') {
+                    $scope.reportDataPreviewVersion = $scope.system.urVersionMax;
+                }
+
                 var listenersFailed = [];
                 for (var address in data.connectionServiceStatus) {
                     if (data.connectionServiceStatus[address].error) {
@@ -1058,7 +1069,6 @@ angular.module('syncthing.core')
         $scope.editSettings = function () {
             // Make a working copy
             $scope.tmpOptions = angular.copy($scope.config.options);
-            $scope.tmpOptions.urEnabled = ($scope.tmpOptions.urAccepted > 0);
             $scope.tmpOptions.deviceName = $scope.thisDevice().name;
             $scope.tmpOptions.upgrades = "none";
             if ($scope.tmpOptions.autoUpgradeIntervalH > 0) {
@@ -1088,18 +1098,31 @@ angular.module('syncthing.core')
             }).error($scope.emitHTTPError);
         };
 
+        $scope.urVersions = function() {
+            var result = [];
+            if ($scope.system) {
+                for (var i = $scope.system.urVersionMax; i >= 2; i--) {
+                    result.push("" + i);
+                }
+            }
+            return result;
+        };
+
         $scope.saveSettings = function () {
             // Make sure something changed
             var changed = !angular.equals($scope.config.options, $scope.tmpOptions) || !angular.equals($scope.config.gui, $scope.tmpGUI);
             var themeChanged = $scope.config.gui.theme !== $scope.tmpGUI.theme;
             if (changed) {
+                // Angular has issues with selects with numeric values, so we handle strings here.
+                $scope.tmpOptions.urAccepted = parseInt($scope.tmpOptions._urAcceptedStr);
                 // Check if auto-upgrade has been enabled or disabled. This
                 // also has an effect on usage reporting, so do the check
                 // for that later.
                 if ($scope.tmpOptions.upgrades == "candidate") {
                     $scope.tmpOptions.autoUpgradeIntervalH = $scope.tmpOptions.autoUpgradeIntervalH || 12;
                     $scope.tmpOptions.upgradeToPreReleases = true;
-                    $scope.tmpOptions.urEnabled = true;
+                    $scope.tmpOptions.urAccepted = $scope.system.urVersionMax;
+                    $scope.tmpOptions.urSeen = $scope.system.urVersionMax;
                 } else if ($scope.tmpOptions.upgrades == "stable") {
                     $scope.tmpOptions.autoUpgradeIntervalH = $scope.tmpOptions.autoUpgradeIntervalH || 12;
                     $scope.tmpOptions.upgradeToPreReleases = false;
@@ -1107,13 +1130,6 @@ angular.module('syncthing.core')
                     $scope.tmpOptions.autoUpgradeIntervalH = 0;
                 }
 
-                // Check if usage reporting has been enabled or disabled
-                if ($scope.tmpOptions.urEnabled && $scope.tmpOptions.urAccepted <= 0) {
-                    $scope.tmpOptions.urAccepted = 1000;
-                } else if (!$scope.tmpOptions.urEnabled && $scope.tmpOptions.urAccepted > 0) {
-                    $scope.tmpOptions.urAccepted = -1;
-                }
-
                 // Check if protocol will need to be changed on restart
                 if ($scope.config.gui.useTLS !== $scope.tmpGUI.useTLS) {
                     $scope.protocolChanged = true;
@@ -1691,13 +1707,17 @@ angular.module('syncthing.core')
         };
 
         $scope.acceptUR = function () {
-            $scope.config.options.urAccepted = 1000; // Larger than the largest existing report version
+            $scope.config.options.urAccepted = $scope.system.urVersionMax;
+            $scope.config.options.urSeen = $scope.system.urVersionMax;
             $scope.saveConfig();
             $('#ur').modal('hide');
         };
 
         $scope.declineUR = function () {
-            $scope.config.options.urAccepted = -1;
+            if ($scope.config.options.urAccepted === 0) {
+                $scope.config.options.urAccepted = -1;
+            }
+            $scope.config.options.urSeen = $scope.system.urVersionMax;
             $scope.saveConfig();
             $('#ur').modal('hide');
         };
@@ -1747,6 +1767,13 @@ angular.module('syncthing.core')
             $scope.reportPreview = true;
         };
 
+        $scope.refreshReportDataPreview = function () {
+            $scope.reportDataPreview = '';
+            $http.get(urlbase + '/svc/report?version=' + $scope.reportDataPreviewVersion).success(function (data) {
+                $scope.reportDataPreview = data;
+            }).error($scope.emitHTTPError);
+        };
+
         $scope.rescanAllFolders = function () {
             $http.post(urlbase + "/db/scan");
         };

gui/default/syncthing/settings/settingsModalView.html

@@ -139,10 +139,14 @@
           </div>
 
           <div class="form-group">
-            <div class="checkbox" ng-if="tmpOptions.upgrades != 'candidate'">
+            <div ng-if="tmpOptions.upgrades != 'candidate'">
-              <label>
+              <label translate for="urVersion">Anonymous Usage Reporting</label> (<a href="" translate data-toggle="modal" data-target="#urPreview">Preview</a>)
-                <input id="UREnabled" type="checkbox" ng-model="tmpOptions.urEnabled"/> <span translate>Anonymous Usage Reporting</span> (<a href="" translate data-toggle="modal" data-target="#urPreview">Preview</a>)
+              <select class="form-control" id="urVersion" ng-model="tmpOptions._urAcceptedStr">
-              </label>
+                <option ng-repeat="n in urVersions()" value="{{n}}">Version {{n}}</option>
+                <!-- 1 does not exist, as we did not support incremental formats back then. -->
+                <option value="0" translate>Undecided (will prompt)</option>
+                <option value="-1" translate>Disabled</option>
+              </select>
             </div>
             <p class="help-block" ng-if="tmpOptions.upgrades == 'candidate'">
               <span translate>Usage reporting is always enabled for candidate releases.</span> (<a href="" translate data-toggle="modal" data-target="#urPreview">Preview</a>)

gui/default/syncthing/usagereport/usageReportModalView.html

@@ -1,8 +1,13 @@
 <modal id="ur" status="info" icon="bar-chart" heading="{{'Allow Anonymous Usage Reporting?' | translate}}" large="yes" closeable="no">
   <div class="modal-body">
-    <p translate>The encrypted usage report is sent daily. It is used to track common platforms, folder sizes and app versions. If the reported data set is changed you will be prompted with this dialog again.</p>
+    <div ng-if="config.options.urAccepted > 0 && config.options.urAccepted < system.urVersionMax">
-    <p translate>The aggregated statistics are publicly available at the URL below.</p>
+      <p>Anonymous Usage report format has changed. Would you like to move to the new format?</p>
-    <p><a href="https://data.syncthing.net/" target="_blank">https://data.syncthing.net/</a></p>
+    </div>
+    <div ng-if="!(config.options.urAccepted > 0 && config.options.urAccepted < system.urVersionMax)">
+      <p translate>The encrypted usage report is sent daily. It is used to track common platforms, folder sizes and app versions. If the reported data set is changed you will be prompted with this dialog again.</p>
+      <p translate>The aggregated statistics are publicly available at the URL below.</p>
+      <p><a href="https://data.syncthing.net/" target="_blank">https://data.syncthing.net/</a></p>
+    </div>
     <button type="button" class="btn btn-default btn-sm" ng-click="showReportPreview()" ng-show="!reportPreview">
       <span class="fa fa-file-text-o"></span>&nbsp;<span translate>Preview Usage Report</span>
     </button>

gui/default/syncthing/usagereport/usageReportPreviewModalView.html

@@ -5,8 +5,13 @@
     </p>
     <p translate>The aggregated statistics are publicly available at the URL below.</p>
     <p><a href="https://data.syncthing.net/" target="_blank">https://data.syncthing.net/</a></p>
+    <label translate>Version</label>
+    <select id="urPreviewVersion" class="form-control" ng-model="$parent.$parent.reportDataPreviewVersion" ng-change="refreshReportDataPreview()" >
+      <option ng-repeat="n in urVersions()" value="{{n}}">Version {{n}}</option>
+    </select>
+    <hr>
     <form>
-      <textarea class="form-control" rows="20">{{reportData | json}}</textarea>
+      <textarea class="form-control" rows="20" ng-if="reportDataPreview">{{reportDataPreview | json}}</textarea>
     </form>
   </div>
   <div class="modal-footer">

lib/config/config.go

@@ -32,7 +32,7 @@ import (
 
 const (
 	OldestHandledVersion = 10
-	CurrentVersion       = 23
+	CurrentVersion       = 24
 	MaxRescanIntervalS   = 365 * 24 * 60 * 60
 )
 
@@ -326,6 +326,9 @@ func (cfg *Configuration) clean() error {
 	if cfg.Version == 22 {
 		convertV22V23(cfg)
 	}
+	if cfg.Version == 23 {
+		convertV23V24(cfg)
+	}
 
 	// Build a list of available devices
 	existingDevices := make(map[protocol.DeviceID]bool)
@@ -375,6 +378,12 @@ func (cfg *Configuration) clean() error {
 	return nil
 }
 
+func convertV23V24(cfg *Configuration) {
+	cfg.Options.URSeen = 2
+
+	cfg.Version = 24
+}
+
 func convertV22V23(cfg *Configuration) {
 	permBits := fs.FileMode(0777)
 	if runtime.GOOS == "windows" {

lib/config/config_test.go

@@ -200,6 +200,7 @@ func TestOverriddenValues(t *testing.T) {
 		ProgressUpdateIntervalS: 10,
 		LimitBandwidthInLan:     true,
 		MinHomeDiskFree:         Size{5.2, "%"},
+		URSeen:                  2,
 		URURL:                   "https://localhost/newdata",
 		URInitialDelayS:         800,
 		URPostInsecurely:        true,

lib/config/optionsconfiguration.go

@@ -112,6 +112,7 @@ type OptionsConfiguration struct {
 	NATRenewalM             int                     `xml:"natRenewalMinutes" json:"natRenewalMinutes" default:"30"`
 	NATTimeoutS             int                     `xml:"natTimeoutSeconds" json:"natTimeoutSeconds" default:"10"`
 	URAccepted              int                     `xml:"urAccepted" json:"urAccepted"` // Accepted usage reporting version; 0 for off (undecided), -1 for off (permanently)
+	URSeen                  int                     `xml:"urSeen" json:"urSeen"`         // Report which the user has been prompted for.
 	URUniqueID              string                  `xml:"urUniqueID" json:"urUniqueId"` // Unique ID for reporting purposes, regenerated when UR is turned on.
 	URURL                   string                  `xml:"urURL" json:"urURL" default:"https://data.syncthing.net/newdata"`
 	URPostInsecurely        bool                    `xml:"urPostInsecurely" json:"urPostInsecurely" default:"false"` // For testing

lib/config/testdata/v24.xml

@@ -0,0 +1,16 @@
+<configuration version="22">
+    <folder id="test" path="testdata" type="readonly" ignorePerms="false" rescanIntervalS="600" autoNormalize="true">
+        <filesystemType>basic</filesystemType>
+        <device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR"></device>
+        <device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2"></device>
+        <minDiskFree unit="%">1</minDiskFree>
+        <maxConflicts>-1</maxConflicts>
+        <fsync>true</fsync>
+    </folder>
+    <device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR" name="node one" compression="metadata">
+        <address>tcp://a</address>
+    </device>
+    <device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" name="node two" compression="metadata">
+        <address>tcp://b</address>
+    </device>
+</configuration>

lib/connections/kcp_listen.go

@@ -12,14 +12,16 @@ import (
 	"net/url"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/AudriusButkevicius/kcp-go"
 	"github.com/AudriusButkevicius/pfilter"
 	"github.com/ccding/go-stun/stun"
+	"github.com/xtaci/smux"
+
 	"github.com/syncthing/syncthing/lib/config"
 	"github.com/syncthing/syncthing/lib/nat"
-	"github.com/xtaci/smux"
 )
 
 func init() {
@@ -38,6 +40,7 @@ type kcpListener struct {
 	stop    chan struct{}
 	conns   chan internalConn
 	factory listenerFactory
+	nat     atomic.Value
 
 	address *url.URL
 	err     error
@@ -183,6 +186,14 @@ func (t *kcpListener) Factory() listenerFactory {
 	return t.factory
 }
 
+func (t *kcpListener) NATType() string {
+	v := t.nat.Load().(stun.NATType)
+	if v == stun.NATUnknown || v == stun.NATError {
+		return "unknown"
+	}
+	return v.String()
+}
+
 func (t *kcpListener) stunRenewal(listener net.PacketConn) {
 	client := stun.NewClientWithConnection(listener)
 	client.SetSoftwareName("syncthing")
@@ -199,6 +210,7 @@ func (t *kcpListener) stunRenewal(listener net.PacketConn) {
 		if t.cfg.Options().StunKeepaliveS < 1 {
 			time.Sleep(time.Second)
 			oldType = stun.NATUnknown
+			t.nat.Store(stun.NATUnknown)
 			t.mut.Lock()
 			t.address = nil
 			t.mut.Unlock()
@@ -222,6 +234,7 @@ func (t *kcpListener) stunRenewal(listener net.PacketConn) {
 
 			if oldType != natType {
 				l.Infof("%s detected NAT type: %s", t.uri, natType)
+				t.nat.Store(natType)
 			}
 
 			for {
@@ -273,7 +286,7 @@ func (t *kcpListener) stunRenewal(listener net.PacketConn) {
 type kcpListenerFactory struct{}
 
 func (f *kcpListenerFactory) New(uri *url.URL, cfg *config.Wrapper, tlsCfg *tls.Config, conns chan internalConn, natService *nat.Service) genericListener {
-	return &kcpListener{
+	l := &kcpListener{
 		uri:     fixupPort(uri, config.DefaultKCPPort),
 		cfg:     cfg,
 		tlsCfg:  tlsCfg,
@@ -281,6 +294,8 @@ func (f *kcpListenerFactory) New(uri *url.URL, cfg *config.Wrapper, tlsCfg *tls.
 		stop:    make(chan struct{}),
 		factory: f,
 	}
+	l.nat.Store(stun.NATUnknown)
+	return l
 }
 
 func (kcpListenerFactory) Enabled(cfg config.Configuration) bool {

lib/connections/relay_listen.go

@@ -171,6 +171,10 @@ func (t *relayListener) String() string {
 	return t.uri.String()
 }
 
+func (t *relayListener) NATType() string {
+	return "unknown"
+}
+
 type relayListenerFactory struct{}
 
 func (f *relayListenerFactory) New(uri *url.URL, cfg *config.Wrapper, tlsCfg *tls.Config, conns chan internalConn, natService *nat.Service) genericListener {

lib/connections/service.go

@@ -574,6 +574,18 @@ func (s *Service) Status() map[string]interface{} {
 	return result
 }
 
+func (s *Service) NATType() string {
+	s.listenersMut.RLock()
+	defer s.listenersMut.RUnlock()
+	for _, listener := range s.listeners {
+		natType := listener.NATType()
+		if natType != "unknown" {
+			return natType
+		}
+	}
+	return "unknown"
+}
+
 func (s *Service) getDialerFactory(cfg config.Configuration, uri *url.URL) (dialerFactory, error) {
 	dialerFactory, ok := dialers[uri.Scheme]
 	if !ok {

lib/connections/structs.go

@@ -25,6 +25,7 @@ type Connection interface {
 	protocol.Connection
 	io.Closer
 	Type() string
+	Transport() string
 	RemoteAddr() net.Addr
 }
 
@@ -74,10 +75,27 @@ func (t connType) String() string {
 	}
 }
 
+func (t connType) Transport() string {
+	switch t {
+	case connTypeRelayClient, connTypeRelayServer:
+		return "relay"
+	case connTypeTCPClient, connTypeTCPServer:
+		return "tcp"
+	case connTypeKCPClient, connTypeKCPServer:
+		return "kcp"
+	default:
+		return "unknown"
+	}
+}
+
 func (c internalConn) Type() string {
 	return c.connType.String()
 }
 
+func (c internalConn) Transport() string {
+	return c.connType.Transport()
+}
+
 func (c internalConn) String() string {
 	return fmt.Sprintf("%s-%s/%s", c.LocalAddr(), c.RemoteAddr(), c.connType.String())
 }
@@ -116,6 +134,7 @@ type genericListener interface {
 	OnAddressesChanged(func(genericListener))
 	String() string
 	Factory() listenerFactory
+	NATType() string
 }
 
 type Model interface {

lib/connections/tcp_listen.go

@@ -176,6 +176,10 @@ func (t *tcpListener) Factory() listenerFactory {
 	return t.factory
 }
 
+func (t *tcpListener) NATType() string {
+	return "unknown"
+}
+
 type tcpListenerFactory struct{}
 
 func (f *tcpListenerFactory) New(uri *url.URL, cfg *config.Wrapper, tlsCfg *tls.Config, conns chan internalConn, natService *nat.Service) genericListener {

lib/model/folder.go

@@ -60,6 +60,10 @@ func (f *folder) Jobs() ([]string, []string) {
 
 func (f *folder) BringToFront(string) {}
 
+func (f *folder) BlockStats() map[string]int {
+	return nil
+}
+
 func (f *folder) scanSubdirs(subDirs []string) error {
 	if err := f.model.internalScanFolderSubdirs(f.ctx, f.folderID, subDirs); err != nil {
 		// Potentially sets the error twice, once in the scanner just

lib/model/model.go

@@ -52,6 +52,7 @@ type service interface {
 	Scan(subs []string) error
 	Serve()
 	Stop()
+	BlockStats() map[string]int
 
 	getState() (folderState, time.Time, error)
 	setState(state folderState)
@@ -405,6 +406,105 @@ func (m *Model) RestartFolder(cfg config.FolderConfiguration) {
 	m.fmut.Unlock()
 }
 
+func (m *Model) UsageReportingStats(version int) map[string]interface{} {
+	stats := make(map[string]interface{})
+	if version >= 3 {
+		// Block stats
+		m.fmut.Lock()
+		blockStats := make(map[string]int)
+		for _, folder := range m.folderRunners {
+			for k, v := range folder.BlockStats() {
+				blockStats[k] += v
+			}
+		}
+		m.fmut.Unlock()
+		stats["blockStats"] = blockStats
+
+		// Transport stats
+		m.pmut.Lock()
+		transportStats := make(map[string]int)
+		for _, conn := range m.conn {
+			transportStats[conn.Transport()]++
+		}
+		m.pmut.Unlock()
+		stats["transportStats"] = transportStats
+
+		// Ignore stats
+		ignoreStats := map[string]int{
+			"lines":           0,
+			"inverts":         0,
+			"folded":          0,
+			"deletable":       0,
+			"rooted":          0,
+			"includes":        0,
+			"escapedIncludes": 0,
+			"doubleStars":     0,
+			"stars":           0,
+		}
+		var seenPrefix [3]bool
+		for folder := range m.cfg.Folders() {
+			lines, _, err := m.GetIgnores(folder)
+			if err != nil {
+				continue
+			}
+			ignoreStats["lines"] += len(lines)
+
+			for _, line := range lines {
+				// Allow prefixes to be specified in any order, but only once.
+				for {
+					if strings.HasPrefix(line, "!") && !seenPrefix[0] {
+						seenPrefix[0] = true
+						line = line[1:]
+						ignoreStats["inverts"] += 1
+					} else if strings.HasPrefix(line, "(?i)") && !seenPrefix[1] {
+						seenPrefix[1] = true
+						line = line[4:]
+						ignoreStats["folded"] += 1
+					} else if strings.HasPrefix(line, "(?d)") && !seenPrefix[2] {
+						seenPrefix[2] = true
+						line = line[4:]
+						ignoreStats["deletable"] += 1
+					} else {
+						seenPrefix[0] = false
+						seenPrefix[1] = false
+						seenPrefix[2] = false
+						break
+					}
+				}
+
+				// Noops, remove
+				if strings.HasSuffix(line, "**") {
+					line = line[:len(line)-2]
+				}
+				if strings.HasPrefix(line, "**/") {
+					line = line[3:]
+				}
+
+				if strings.HasPrefix(line, "/") {
+					ignoreStats["rooted"] += 1
+				} else if strings.HasPrefix(line, "#include ") {
+					ignoreStats["includes"] += 1
+					if strings.Contains(line, "..") {
+						ignoreStats["escapedIncludes"] += 1
+					}
+				}
+
+				if strings.Contains(line, "**") {
+					ignoreStats["doubleStars"] += 1
+					// Remove not to trip up star checks.
+					strings.Replace(line, "**", "", -1)
+				}
+
+				if strings.Contains(line, "*") {
+					ignoreStats["stars"] += 1
+				}
+			}
+		}
+		stats["ignoreStats"] = ignoreStats
+	}
+	return stats
+}
+
 type ConnectionInfo struct {
 	protocol.Statistics
 	Connected     bool
@@ -2449,6 +2549,7 @@ func (m *Model) CommitConfiguration(from, to config.Configuration) bool {
 	// Some options don't require restart as those components handle it fine
 	// by themselves.
 	from.Options.URAccepted = to.Options.URAccepted
+	from.Options.URSeen = to.Options.URSeen
 	from.Options.URUniqueID = to.Options.URUniqueID
 	from.Options.ListenAddresses = to.Options.ListenAddresses
 	from.Options.RelaysEnabled = to.Options.RelaysEnabled

lib/model/model_test.go

@@ -308,6 +308,9 @@ func (f *fakeConnection) RemoteAddr() net.Addr {
 func (f *fakeConnection) Type() string {
 	return "fake"
 }
+func (f *fakeConnection) Transport() string {
+	return "fake"
+}
 
 func (f *fakeConnection) DownloadProgress(folder string, updates []protocol.FileDownloadProgressUpdate) {
 	f.downloadProgressMessages = append(f.downloadProgressMessages, downloadProgressMessage{

lib/model/rwfolder.go

@@ -93,6 +93,9 @@ type sendReceiveFolder struct {
 
 	errors    map[string]string // path -> error string
 	errorsMut sync.Mutex
+
+	blockStats    map[string]int
+	blockStatsMut sync.Mutex
 }
 
 func newSendReceiveFolder(model *Model, cfg config.FolderConfiguration, ver versioner.Versioner, fs fs.Filesystem) service {
@@ -107,6 +110,9 @@ func newSendReceiveFolder(model *Model, cfg config.FolderConfiguration, ver vers
 		remoteIndex: make(chan struct{}, 1), // This needs to be 1-buffered so that we queue a notification if we're busy doing a pull when it comes.
 
 		errorsMut: sync.NewMutex(),
+
+		blockStats:    make(map[string]int),
+		blockStatsMut: sync.NewMutex(),
 	}
 
 	f.configureCopiersAndPullers()
@@ -875,6 +881,11 @@ func (f *sendReceiveFolder) renameFile(source, target protocol.FileInfo) {
 	}
 
 	if err == nil {
+		f.blockStatsMut.Lock()
+		f.blockStats["total"] += len(target.Blocks)
+		f.blockStats["renamed"] += len(target.Blocks)
+		f.blockStatsMut.Unlock()
+
 		// The file was renamed, so we have handled both the necessary delete
 		// of the source and the creation of the target. Fix-up the metadata,
 		// and update the local index of the target file.
@@ -1443,6 +1454,15 @@ func (f *sendReceiveFolder) finisherRoutine(in <-chan *sharedPullerState) {
 			if err != nil {
 				l.Debugln("Puller: final:", err)
 				f.newError(state.file.Name, err)
+			} else {
+				f.blockStatsMut.Lock()
+				f.blockStats["total"] += state.reused + state.copyTotal + state.pullTotal
+				f.blockStats["reused"] += state.reused
+				f.blockStats["pulled"] += state.pullTotal
+				f.blockStats["copyOrigin"] += state.copyOrigin
+				f.blockStats["copyOriginShifted"] += state.copyOriginShifted
+				f.blockStats["copyElsewhere"] += state.copyTotal - state.copyOrigin
+				f.blockStatsMut.Unlock()
 			}
 			events.Default.Log(events.ItemFinished, map[string]interface{}{
 				"folder": f.folderID,
@@ -1459,6 +1479,16 @@ func (f *sendReceiveFolder) finisherRoutine(in <-chan *sharedPullerState) {
 	}
 }
 
+func (f *sendReceiveFolder) BlockStats() map[string]int {
+	f.blockStatsMut.Lock()
+	stats := make(map[string]int)
+	for k, v := range f.blockStats {
+		stats[k] = v
+	}
+	f.blockStatsMut.Unlock()
+	return stats
+}
+
 // Moves the given filename to the front of the job queue
 func (f *sendReceiveFolder) BringToFront(filename string) {
 	f.queue.BringToFront(filename)