From 2760d032ca672435e06a1eeb7832f5fe1af14f92 Mon Sep 17 00:00:00 2001 From: Audrius Butkevicius Date: Thu, 12 Oct 2017 06:16:46 +0000 Subject: [PATCH] cmd/syncthing: Add more stats to usage reports (ref #3628) GitHub-Pull-Request: https://github.com/syncthing/syncthing/pull/4347 --- cmd/syncthing/gui.go | 21 ++- cmd/syncthing/main.go | 16 +-- cmd/syncthing/mocked_connections_test.go | 4 + cmd/syncthing/mocked_model_test.go | 4 + cmd/syncthing/usage_report.go | 122 ++++++++---------- gui/default/syncthing/core/modalDirective.js | 2 + .../syncthing/core/syncthingController.js | 51 ++++++-- .../syncthing/settings/settingsModalView.html | 12 +- .../usagereport/usageReportModalView.html | 11 +- .../usageReportPreviewModalView.html | 7 +- lib/config/config.go | 11 +- lib/config/config_test.go | 1 + lib/config/optionsconfiguration.go | 1 + lib/config/testdata/v24.xml | 16 +++ lib/connections/kcp_listen.go | 19 ++- lib/connections/relay_listen.go | 4 + lib/connections/service.go | 12 ++ lib/connections/structs.go | 19 +++ lib/connections/tcp_listen.go | 4 + lib/model/folder.go | 4 + lib/model/model.go | 101 +++++++++++++++ lib/model/model_test.go | 3 + lib/model/rwfolder.go | 30 +++++ 23 files changed, 358 insertions(+), 117 deletions(-) create mode 100644 lib/config/testdata/v24.xml diff --git a/cmd/syncthing/gui.go b/cmd/syncthing/gui.go index 57134d452..640f4778f 100644 --- a/cmd/syncthing/gui.go +++ b/cmd/syncthing/gui.go @@ -100,6 +100,7 @@ type modelIntf interface { CurrentSequence(folder string) (int64, bool) RemoteSequence(folder string) (int64, bool) State(folder string) (string, time.Time, error) + UsageReportingStats(version int) map[string]interface{} } type configIntf interface { @@ -119,6 +120,7 @@ type configIntf interface { type connectionsIntf interface { Status() map[string]interface{} + NATType() string } type rater interface { @@ -800,18 +802,6 @@ func (s *apiService) postSystemConfig(w http.ResponseWriter, r *http.Request) { } } - // Fixup usage reporting settings - - if curAcc := s.cfg.Options().URAccepted; to.Options.URAccepted > curAcc { - // UR was enabled - to.Options.URAccepted = usageReportVersion - to.Options.URUniqueID = rand.String(8) - } else if to.Options.URAccepted < curAcc { - // UR was disabled - to.Options.URAccepted = -1 - to.Options.URUniqueID = "" - } - // Activate and save if err := s.cfg.Replace(to); err != nil { @@ -903,6 +893,7 @@ func (s *apiService) getSystemStatus(w http.ResponseWriter, r *http.Request) { // gives us percent res["cpuPercent"] = s.cpu.Rate() / 10 / float64(runtime.NumCPU()) res["pathSeparator"] = string(filepath.Separator) + res["urVersionMax"] = usageReportVersion res["uptime"] = int(time.Since(startTime).Seconds()) res["startTime"] = startTime @@ -981,7 +972,11 @@ func (s *apiService) getSystemDiscovery(w http.ResponseWriter, r *http.Request) } func (s *apiService) getReport(w http.ResponseWriter, r *http.Request) { - sendJSON(w, reportData(s.cfg, s.model)) + version := usageReportVersion + if val, _ := strconv.Atoi(r.URL.Query().Get("version")); val > 0 { + version = val + } + sendJSON(w, reportData(s.cfg, s.model, s.connectionsService, version)) } func (s *apiService) getRandomString(w http.ResponseWriter, r *http.Request) { diff --git a/cmd/syncthing/main.go b/cmd/syncthing/main.go index 949ee4283..e86024686 100644 --- a/cmd/syncthing/main.go +++ b/cmd/syncthing/main.go @@ -882,24 +882,14 @@ func syncthingMain(runtimeOptions RuntimeOptions) { // Unique ID will be set and config saved below if necessary. } - if opts.URAccepted > 0 && opts.URAccepted < usageReportVersion { - l.Infoln("Anonymous usage report has changed; revoking acceptance") - opts.URAccepted = 0 - opts.URUniqueID = "" - cfg.SetOptions(opts) - } - - if opts.URAccepted >= usageReportVersion && opts.URUniqueID == "" { - // Generate and save a new unique ID if it is missing. + if opts.URUniqueID == "" { opts.URUniqueID = rand.String(8) cfg.SetOptions(opts) cfg.Save() } - // The usageReportingManager registers itself to listen to configuration - // changes, and there's nothing more we need to tell it from the outside. - // Hence we don't keep the returned pointer. - newUsageReportingManager(cfg, m) + usageReportingSvc := newUsageReportingService(cfg, m, connectionsService) + mainService.Add(usageReportingSvc) if opts.RestartOnWakeup { go standbyMonitor() diff --git a/cmd/syncthing/mocked_connections_test.go b/cmd/syncthing/mocked_connections_test.go index 82286af89..ebb90503b 100644 --- a/cmd/syncthing/mocked_connections_test.go +++ b/cmd/syncthing/mocked_connections_test.go @@ -11,3 +11,7 @@ type mockedConnections struct{} func (m *mockedConnections) Status() map[string]interface{} { return nil } + +func (m *mockedConnections) NATType() string { + return "" +} diff --git a/cmd/syncthing/mocked_model_test.go b/cmd/syncthing/mocked_model_test.go index 3ca25bd8a..8f9acd08f 100644 --- a/cmd/syncthing/mocked_model_test.go +++ b/cmd/syncthing/mocked_model_test.go @@ -114,3 +114,7 @@ func (m *mockedModel) RemoteSequence(folder string) (int64, bool) { func (m *mockedModel) State(folder string) (string, time.Time, error) { return "", time.Time{}, nil } + +func (m *mockedModel) UsageReportingStats(version int) map[string]interface{} { + return nil +} diff --git a/cmd/syncthing/usage_report.go b/cmd/syncthing/usage_report.go index c43958015..aa5ea6a0a 100644 --- a/cmd/syncthing/usage_report.go +++ b/cmd/syncthing/usage_report.go @@ -12,7 +12,6 @@ import ( "crypto/rand" "crypto/tls" "encoding/json" - "fmt" "net/http" "runtime" "sort" @@ -20,71 +19,25 @@ import ( "time" "github.com/syncthing/syncthing/lib/config" + "github.com/syncthing/syncthing/lib/connections" "github.com/syncthing/syncthing/lib/dialer" "github.com/syncthing/syncthing/lib/model" "github.com/syncthing/syncthing/lib/protocol" "github.com/syncthing/syncthing/lib/scanner" "github.com/syncthing/syncthing/lib/upgrade" - "github.com/thejerf/suture" ) // Current version number of the usage report, for acceptance purposes. If // fields are added or changed this integer must be incremented so that users // are prompted for acceptance of the new report. -const usageReportVersion = 2 - -type usageReportingManager struct { - cfg *config.Wrapper - model *model.Model - sup *suture.Supervisor -} - -func newUsageReportingManager(cfg *config.Wrapper, m *model.Model) *usageReportingManager { - mgr := &usageReportingManager{ - cfg: cfg, - model: m, - } - - // Start UR if it's enabled. - mgr.CommitConfiguration(config.Configuration{}, cfg.RawCopy()) - - // Listen to future config changes so that we can start and stop as - // appropriate. - cfg.Subscribe(mgr) - - return mgr -} - -func (m *usageReportingManager) VerifyConfiguration(from, to config.Configuration) error { - return nil -} - -func (m *usageReportingManager) CommitConfiguration(from, to config.Configuration) bool { - if to.Options.URAccepted >= usageReportVersion && m.sup == nil { - // Usage reporting was turned on; lets start it. - service := newUsageReportingService(m.cfg, m.model) - m.sup = suture.NewSimple("usageReporting") - m.sup.Add(service) - m.sup.ServeBackground() - } else if to.Options.URAccepted < usageReportVersion && m.sup != nil { - // Usage reporting was turned off - m.sup.Stop() - m.sup = nil - } - - return true -} - -func (m *usageReportingManager) String() string { - return fmt.Sprintf("usageReportingManager@%p", m) -} +const usageReportVersion = 3 // reportData returns the data to be sent in a usage report. It's used in // various places, so not part of the usageReportingManager object. -func reportData(cfg configIntf, m modelIntf) map[string]interface{} { +func reportData(cfg configIntf, m modelIntf, connectionsService connectionsIntf, version int) map[string]interface{} { opts := cfg.Options() res := make(map[string]interface{}) - res["urVersion"] = usageReportVersion + res["urVersion"] = version res["uniqueID"] = opts.URUniqueID res["version"] = Version res["longVersion"] = LongVersion @@ -227,25 +180,40 @@ func reportData(cfg configIntf, m modelIntf) map[string]interface{} { res["upgradeAllowedAuto"] = !(upgrade.DisabledByCompilation || noUpgradeFromEnv) && opts.AutoUpgradeIntervalH > 0 res["upgradeAllowedPre"] = !(upgrade.DisabledByCompilation || noUpgradeFromEnv) && opts.AutoUpgradeIntervalH > 0 && opts.UpgradeToPreReleases + if version >= 3 { + res["uptime"] = time.Now().Sub(startTime).Seconds() + res["natType"] = connectionsService.NATType() + } + + for key, value := range m.UsageReportingStats(version) { + res[key] = value + } + return res } type usageReportingService struct { - cfg *config.Wrapper - model *model.Model - stop chan struct{} + cfg *config.Wrapper + model *model.Model + connectionsService *connections.Service + forceRun chan struct{} + stop chan struct{} } -func newUsageReportingService(cfg *config.Wrapper, model *model.Model) *usageReportingService { - return &usageReportingService{ - cfg: cfg, - model: model, - stop: make(chan struct{}), +func newUsageReportingService(cfg *config.Wrapper, model *model.Model, connectionsService *connections.Service) *usageReportingService { + svc := &usageReportingService{ + cfg: cfg, + model: model, + connectionsService: connectionsService, + forceRun: make(chan struct{}), + stop: make(chan struct{}), } + cfg.Subscribe(svc) + return svc } func (s *usageReportingService) sendUsageReport() error { - d := reportData(s.cfg, s.model) + d := reportData(s.cfg, s.model, s.connectionsService, s.cfg.Options().URAccepted) var b bytes.Buffer json.NewEncoder(&b).Encode(d) @@ -264,27 +232,45 @@ func (s *usageReportingService) sendUsageReport() error { func (s *usageReportingService) Serve() { s.stop = make(chan struct{}) - - l.Infoln("Starting usage reporting") - defer l.Infoln("Stopping usage reporting") - - t := time.NewTimer(time.Duration(s.cfg.Options().URInitialDelayS) * time.Second) // time to initial report at start + t := time.NewTimer(time.Duration(s.cfg.Options().URInitialDelayS) * time.Second) for { select { case <-s.stop: return + case <-s.forceRun: + t.Reset(0) case <-t.C: - err := s.sendUsageReport() - if err != nil { - l.Infoln("Usage report:", err) + if s.cfg.Options().URAccepted >= 2 { + err := s.sendUsageReport() + if err != nil { + l.Infoln("Usage report:", err) + } else { + l.Infof("Sent usage report (version %d)", s.cfg.Options().URAccepted) + } } t.Reset(24 * time.Hour) // next report tomorrow } } } +func (s *usageReportingService) VerifyConfiguration(from, to config.Configuration) error { + return nil +} + +func (s *usageReportingService) CommitConfiguration(from, to config.Configuration) bool { + if from.Options.URAccepted != to.Options.URAccepted || from.Options.URUniqueID != to.Options.URUniqueID || from.Options.URURL != to.Options.URURL { + s.forceRun <- struct{}{} + } + return true +} + func (s *usageReportingService) Stop() { close(s.stop) + close(s.forceRun) +} + +func (usageReportingService) String() string { + return "usageReportingService" } // cpuBench returns CPU performance as a measure of single threaded SHA-256 MiB/s diff --git a/gui/default/syncthing/core/modalDirective.js b/gui/default/syncthing/core/modalDirective.js index def2ff3fa..43553ac0d 100644 --- a/gui/default/syncthing/core/modalDirective.js +++ b/gui/default/syncthing/core/modalDirective.js @@ -1,6 +1,8 @@ angular.module('syncthing.core') .directive('modal', function () { return { + // If you ever change any of the petroglyphs below, please search for $parent.$parent, + // as some templates rely on the way scope is composed in this case. restrict: 'E', templateUrl: 'modal.html', replace: true, diff --git a/gui/default/syncthing/core/syncthingController.js b/gui/default/syncthing/core/syncthingController.js index 8aad64176..710131ef8 100755 --- a/gui/default/syncthing/core/syncthingController.js +++ b/gui/default/syncthing/core/syncthingController.js @@ -33,6 +33,8 @@ angular.module('syncthing.core') $scope.folderRejections = {}; $scope.protocolChanged = false; $scope.reportData = {}; + $scope.reportDataPreview = {}; + $scope.reportDataPreviewVersion = ''; $scope.reportPreview = false; $scope.folders = {}; $scope.seenError = ''; @@ -133,7 +135,11 @@ angular.module('syncthing.core') }).error($scope.emitHTTPError); $http.get(urlbase + '/svc/report').success(function (data) { - $scope.reportData = data; + $scope.reportDataPreview = $scope.reportData = data; + if ($scope.system && $scope.config.options.urSeen < $scope.system.urVersionMax) { + // Usage reporting format has changed, prompt the user to re-accept. + $('#ur').modal(); + } }).error($scope.emitHTTPError); $http.get(urlbase + '/system/upgrade').success(function (data) { @@ -376,6 +382,7 @@ angular.module('syncthing.core') $scope.config = config; $scope.config.options._listenAddressesStr = $scope.config.options.listenAddresses.join(', '); $scope.config.options._globalAnnounceServersStr = $scope.config.options.globalAnnounceServers.join(', '); + $scope.config.options._urAcceptedStr = "" + $scope.config.options.urAccepted; $scope.devices = $scope.config.devices; $scope.devices.forEach(function (deviceCfg) { @@ -412,6 +419,10 @@ angular.module('syncthing.core') $scope.myID = data.myID; $scope.system = data; + if ($scope.reportDataPreviewVersion === '') { + $scope.reportDataPreviewVersion = $scope.system.urVersionMax; + } + var listenersFailed = []; for (var address in data.connectionServiceStatus) { if (data.connectionServiceStatus[address].error) { @@ -1058,7 +1069,6 @@ angular.module('syncthing.core') $scope.editSettings = function () { // Make a working copy $scope.tmpOptions = angular.copy($scope.config.options); - $scope.tmpOptions.urEnabled = ($scope.tmpOptions.urAccepted > 0); $scope.tmpOptions.deviceName = $scope.thisDevice().name; $scope.tmpOptions.upgrades = "none"; if ($scope.tmpOptions.autoUpgradeIntervalH > 0) { @@ -1088,18 +1098,31 @@ angular.module('syncthing.core') }).error($scope.emitHTTPError); }; + $scope.urVersions = function() { + var result = []; + if ($scope.system) { + for (var i = $scope.system.urVersionMax; i >= 2; i--) { + result.push("" + i); + } + } + return result; + }; + $scope.saveSettings = function () { // Make sure something changed var changed = !angular.equals($scope.config.options, $scope.tmpOptions) || !angular.equals($scope.config.gui, $scope.tmpGUI); var themeChanged = $scope.config.gui.theme !== $scope.tmpGUI.theme; if (changed) { + // Angular has issues with selects with numeric values, so we handle strings here. + $scope.tmpOptions.urAccepted = parseInt($scope.tmpOptions._urAcceptedStr); // Check if auto-upgrade has been enabled or disabled. This // also has an effect on usage reporting, so do the check // for that later. if ($scope.tmpOptions.upgrades == "candidate") { $scope.tmpOptions.autoUpgradeIntervalH = $scope.tmpOptions.autoUpgradeIntervalH || 12; $scope.tmpOptions.upgradeToPreReleases = true; - $scope.tmpOptions.urEnabled = true; + $scope.tmpOptions.urAccepted = $scope.system.urVersionMax; + $scope.tmpOptions.urSeen = $scope.system.urVersionMax; } else if ($scope.tmpOptions.upgrades == "stable") { $scope.tmpOptions.autoUpgradeIntervalH = $scope.tmpOptions.autoUpgradeIntervalH || 12; $scope.tmpOptions.upgradeToPreReleases = false; @@ -1107,13 +1130,6 @@ angular.module('syncthing.core') $scope.tmpOptions.autoUpgradeIntervalH = 0; } - // Check if usage reporting has been enabled or disabled - if ($scope.tmpOptions.urEnabled && $scope.tmpOptions.urAccepted <= 0) { - $scope.tmpOptions.urAccepted = 1000; - } else if (!$scope.tmpOptions.urEnabled && $scope.tmpOptions.urAccepted > 0) { - $scope.tmpOptions.urAccepted = -1; - } - // Check if protocol will need to be changed on restart if ($scope.config.gui.useTLS !== $scope.tmpGUI.useTLS) { $scope.protocolChanged = true; @@ -1691,13 +1707,17 @@ angular.module('syncthing.core') }; $scope.acceptUR = function () { - $scope.config.options.urAccepted = 1000; // Larger than the largest existing report version + $scope.config.options.urAccepted = $scope.system.urVersionMax; + $scope.config.options.urSeen = $scope.system.urVersionMax; $scope.saveConfig(); $('#ur').modal('hide'); }; $scope.declineUR = function () { - $scope.config.options.urAccepted = -1; + if ($scope.config.options.urAccepted === 0) { + $scope.config.options.urAccepted = -1; + } + $scope.config.options.urSeen = $scope.system.urVersionMax; $scope.saveConfig(); $('#ur').modal('hide'); }; @@ -1747,6 +1767,13 @@ angular.module('syncthing.core') $scope.reportPreview = true; }; + $scope.refreshReportDataPreview = function () { + $scope.reportDataPreview = ''; + $http.get(urlbase + '/svc/report?version=' + $scope.reportDataPreviewVersion).success(function (data) { + $scope.reportDataPreview = data; + }).error($scope.emitHTTPError); + }; + $scope.rescanAllFolders = function () { $http.post(urlbase + "/db/scan"); }; diff --git a/gui/default/syncthing/settings/settingsModalView.html b/gui/default/syncthing/settings/settingsModalView.html index 77540ec35..782c9f5f4 100644 --- a/gui/default/syncthing/settings/settingsModalView.html +++ b/gui/default/syncthing/settings/settingsModalView.html @@ -139,10 +139,14 @@
-
- +
+ (Preview) +

Usage reporting is always enabled for candidate releases. (Preview) diff --git a/gui/default/syncthing/usagereport/usageReportModalView.html b/gui/default/syncthing/usagereport/usageReportModalView.html index 76b1d294a..620a98ad2 100644 --- a/gui/default/syncthing/usagereport/usageReportModalView.html +++ b/gui/default/syncthing/usagereport/usageReportModalView.html @@ -1,8 +1,13 @@

-

The encrypted usage report is sent daily. It is used to track common platforms, folder sizes and app versions. If the reported data set is changed you will be prompted with this dialog again.

-

The aggregated statistics are publicly available at the URL below.

-

https://data.syncthing.net/

+
+

Anonymous Usage report format has changed. Would you like to move to the new format?

+
+
+

The encrypted usage report is sent daily. It is used to track common platforms, folder sizes and app versions. If the reported data set is changed you will be prompted with this dialog again.

+

The aggregated statistics are publicly available at the URL below.

+

https://data.syncthing.net/

+
diff --git a/gui/default/syncthing/usagereport/usageReportPreviewModalView.html b/gui/default/syncthing/usagereport/usageReportPreviewModalView.html index 8435be216..5a2217cc5 100644 --- a/gui/default/syncthing/usagereport/usageReportPreviewModalView.html +++ b/gui/default/syncthing/usagereport/usageReportPreviewModalView.html @@ -5,8 +5,13 @@

The aggregated statistics are publicly available at the URL below.

https://data.syncthing.net/

+ + +
- +
diff --git a/lib/config/config.go b/lib/config/config.go index 0aff1509e..36c140423 100644 --- a/lib/config/config.go +++ b/lib/config/config.go @@ -32,7 +32,7 @@ import ( const ( OldestHandledVersion = 10 - CurrentVersion = 23 + CurrentVersion = 24 MaxRescanIntervalS = 365 * 24 * 60 * 60 ) @@ -326,6 +326,9 @@ func (cfg *Configuration) clean() error { if cfg.Version == 22 { convertV22V23(cfg) } + if cfg.Version == 23 { + convertV23V24(cfg) + } // Build a list of available devices existingDevices := make(map[protocol.DeviceID]bool) @@ -375,6 +378,12 @@ func (cfg *Configuration) clean() error { return nil } +func convertV23V24(cfg *Configuration) { + cfg.Options.URSeen = 2 + + cfg.Version = 24 +} + func convertV22V23(cfg *Configuration) { permBits := fs.FileMode(0777) if runtime.GOOS == "windows" { diff --git a/lib/config/config_test.go b/lib/config/config_test.go index eb6f96fb3..93c6a1369 100644 --- a/lib/config/config_test.go +++ b/lib/config/config_test.go @@ -200,6 +200,7 @@ func TestOverriddenValues(t *testing.T) { ProgressUpdateIntervalS: 10, LimitBandwidthInLan: true, MinHomeDiskFree: Size{5.2, "%"}, + URSeen: 2, URURL: "https://localhost/newdata", URInitialDelayS: 800, URPostInsecurely: true, diff --git a/lib/config/optionsconfiguration.go b/lib/config/optionsconfiguration.go index 24e8b1f74..c1c55f259 100644 --- a/lib/config/optionsconfiguration.go +++ b/lib/config/optionsconfiguration.go @@ -112,6 +112,7 @@ type OptionsConfiguration struct { NATRenewalM int `xml:"natRenewalMinutes" json:"natRenewalMinutes" default:"30"` NATTimeoutS int `xml:"natTimeoutSeconds" json:"natTimeoutSeconds" default:"10"` URAccepted int `xml:"urAccepted" json:"urAccepted"` // Accepted usage reporting version; 0 for off (undecided), -1 for off (permanently) + URSeen int `xml:"urSeen" json:"urSeen"` // Report which the user has been prompted for. URUniqueID string `xml:"urUniqueID" json:"urUniqueId"` // Unique ID for reporting purposes, regenerated when UR is turned on. URURL string `xml:"urURL" json:"urURL" default:"https://data.syncthing.net/newdata"` URPostInsecurely bool `xml:"urPostInsecurely" json:"urPostInsecurely" default:"false"` // For testing diff --git a/lib/config/testdata/v24.xml b/lib/config/testdata/v24.xml new file mode 100644 index 000000000..2a577fc50 --- /dev/null +++ b/lib/config/testdata/v24.xml @@ -0,0 +1,16 @@ + + + basic + + + 1 + -1 + true + + +
tcp://a
+ + +
tcp://b
+ + diff --git a/lib/connections/kcp_listen.go b/lib/connections/kcp_listen.go index 1bf7f95c2..368c4ce1e 100644 --- a/lib/connections/kcp_listen.go +++ b/lib/connections/kcp_listen.go @@ -12,14 +12,16 @@ import ( "net/url" "strings" "sync" + "sync/atomic" "time" "github.com/AudriusButkevicius/kcp-go" "github.com/AudriusButkevicius/pfilter" "github.com/ccding/go-stun/stun" + "github.com/xtaci/smux" + "github.com/syncthing/syncthing/lib/config" "github.com/syncthing/syncthing/lib/nat" - "github.com/xtaci/smux" ) func init() { @@ -38,6 +40,7 @@ type kcpListener struct { stop chan struct{} conns chan internalConn factory listenerFactory + nat atomic.Value address *url.URL err error @@ -183,6 +186,14 @@ func (t *kcpListener) Factory() listenerFactory { return t.factory } +func (t *kcpListener) NATType() string { + v := t.nat.Load().(stun.NATType) + if v == stun.NATUnknown || v == stun.NATError { + return "unknown" + } + return v.String() +} + func (t *kcpListener) stunRenewal(listener net.PacketConn) { client := stun.NewClientWithConnection(listener) client.SetSoftwareName("syncthing") @@ -199,6 +210,7 @@ func (t *kcpListener) stunRenewal(listener net.PacketConn) { if t.cfg.Options().StunKeepaliveS < 1 { time.Sleep(time.Second) oldType = stun.NATUnknown + t.nat.Store(stun.NATUnknown) t.mut.Lock() t.address = nil t.mut.Unlock() @@ -222,6 +234,7 @@ func (t *kcpListener) stunRenewal(listener net.PacketConn) { if oldType != natType { l.Infof("%s detected NAT type: %s", t.uri, natType) + t.nat.Store(natType) } for { @@ -273,7 +286,7 @@ func (t *kcpListener) stunRenewal(listener net.PacketConn) { type kcpListenerFactory struct{} func (f *kcpListenerFactory) New(uri *url.URL, cfg *config.Wrapper, tlsCfg *tls.Config, conns chan internalConn, natService *nat.Service) genericListener { - return &kcpListener{ + l := &kcpListener{ uri: fixupPort(uri, config.DefaultKCPPort), cfg: cfg, tlsCfg: tlsCfg, @@ -281,6 +294,8 @@ func (f *kcpListenerFactory) New(uri *url.URL, cfg *config.Wrapper, tlsCfg *tls. stop: make(chan struct{}), factory: f, } + l.nat.Store(stun.NATUnknown) + return l } func (kcpListenerFactory) Enabled(cfg config.Configuration) bool { diff --git a/lib/connections/relay_listen.go b/lib/connections/relay_listen.go index e9aca1db1..fa754aa55 100644 --- a/lib/connections/relay_listen.go +++ b/lib/connections/relay_listen.go @@ -171,6 +171,10 @@ func (t *relayListener) String() string { return t.uri.String() } +func (t *relayListener) NATType() string { + return "unknown" +} + type relayListenerFactory struct{} func (f *relayListenerFactory) New(uri *url.URL, cfg *config.Wrapper, tlsCfg *tls.Config, conns chan internalConn, natService *nat.Service) genericListener { diff --git a/lib/connections/service.go b/lib/connections/service.go index 1d857aa82..df8a1d672 100644 --- a/lib/connections/service.go +++ b/lib/connections/service.go @@ -574,6 +574,18 @@ func (s *Service) Status() map[string]interface{} { return result } +func (s *Service) NATType() string { + s.listenersMut.RLock() + defer s.listenersMut.RUnlock() + for _, listener := range s.listeners { + natType := listener.NATType() + if natType != "unknown" { + return natType + } + } + return "unknown" +} + func (s *Service) getDialerFactory(cfg config.Configuration, uri *url.URL) (dialerFactory, error) { dialerFactory, ok := dialers[uri.Scheme] if !ok { diff --git a/lib/connections/structs.go b/lib/connections/structs.go index 405d140ac..0a044d0b8 100644 --- a/lib/connections/structs.go +++ b/lib/connections/structs.go @@ -25,6 +25,7 @@ type Connection interface { protocol.Connection io.Closer Type() string + Transport() string RemoteAddr() net.Addr } @@ -74,10 +75,27 @@ func (t connType) String() string { } } +func (t connType) Transport() string { + switch t { + case connTypeRelayClient, connTypeRelayServer: + return "relay" + case connTypeTCPClient, connTypeTCPServer: + return "tcp" + case connTypeKCPClient, connTypeKCPServer: + return "kcp" + default: + return "unknown" + } +} + func (c internalConn) Type() string { return c.connType.String() } +func (c internalConn) Transport() string { + return c.connType.Transport() +} + func (c internalConn) String() string { return fmt.Sprintf("%s-%s/%s", c.LocalAddr(), c.RemoteAddr(), c.connType.String()) } @@ -116,6 +134,7 @@ type genericListener interface { OnAddressesChanged(func(genericListener)) String() string Factory() listenerFactory + NATType() string } type Model interface { diff --git a/lib/connections/tcp_listen.go b/lib/connections/tcp_listen.go index d3e5a2e74..f84c6e20e 100644 --- a/lib/connections/tcp_listen.go +++ b/lib/connections/tcp_listen.go @@ -176,6 +176,10 @@ func (t *tcpListener) Factory() listenerFactory { return t.factory } +func (t *tcpListener) NATType() string { + return "unknown" +} + type tcpListenerFactory struct{} func (f *tcpListenerFactory) New(uri *url.URL, cfg *config.Wrapper, tlsCfg *tls.Config, conns chan internalConn, natService *nat.Service) genericListener { diff --git a/lib/model/folder.go b/lib/model/folder.go index 284d4a925..e0dc1764f 100644 --- a/lib/model/folder.go +++ b/lib/model/folder.go @@ -60,6 +60,10 @@ func (f *folder) Jobs() ([]string, []string) { func (f *folder) BringToFront(string) {} +func (f *folder) BlockStats() map[string]int { + return nil +} + func (f *folder) scanSubdirs(subDirs []string) error { if err := f.model.internalScanFolderSubdirs(f.ctx, f.folderID, subDirs); err != nil { // Potentially sets the error twice, once in the scanner just diff --git a/lib/model/model.go b/lib/model/model.go index f74d540a5..a3c01d5a5 100644 --- a/lib/model/model.go +++ b/lib/model/model.go @@ -52,6 +52,7 @@ type service interface { Scan(subs []string) error Serve() Stop() + BlockStats() map[string]int getState() (folderState, time.Time, error) setState(state folderState) @@ -405,6 +406,105 @@ func (m *Model) RestartFolder(cfg config.FolderConfiguration) { m.fmut.Unlock() } +func (m *Model) UsageReportingStats(version int) map[string]interface{} { + stats := make(map[string]interface{}) + if version >= 3 { + // Block stats + m.fmut.Lock() + blockStats := make(map[string]int) + for _, folder := range m.folderRunners { + for k, v := range folder.BlockStats() { + blockStats[k] += v + } + } + m.fmut.Unlock() + stats["blockStats"] = blockStats + + // Transport stats + m.pmut.Lock() + transportStats := make(map[string]int) + for _, conn := range m.conn { + transportStats[conn.Transport()]++ + } + m.pmut.Unlock() + stats["transportStats"] = transportStats + + // Ignore stats + ignoreStats := map[string]int{ + "lines": 0, + "inverts": 0, + "folded": 0, + "deletable": 0, + "rooted": 0, + "includes": 0, + "escapedIncludes": 0, + "doubleStars": 0, + "stars": 0, + } + var seenPrefix [3]bool + for folder := range m.cfg.Folders() { + lines, _, err := m.GetIgnores(folder) + if err != nil { + continue + } + ignoreStats["lines"] += len(lines) + + for _, line := range lines { + // Allow prefixes to be specified in any order, but only once. + for { + if strings.HasPrefix(line, "!") && !seenPrefix[0] { + seenPrefix[0] = true + line = line[1:] + ignoreStats["inverts"] += 1 + } else if strings.HasPrefix(line, "(?i)") && !seenPrefix[1] { + seenPrefix[1] = true + line = line[4:] + ignoreStats["folded"] += 1 + } else if strings.HasPrefix(line, "(?d)") && !seenPrefix[2] { + seenPrefix[2] = true + line = line[4:] + ignoreStats["deletable"] += 1 + } else { + seenPrefix[0] = false + seenPrefix[1] = false + seenPrefix[2] = false + break + } + } + + // Noops, remove + if strings.HasSuffix(line, "**") { + line = line[:len(line)-2] + } + if strings.HasPrefix(line, "**/") { + line = line[3:] + } + + if strings.HasPrefix(line, "/") { + ignoreStats["rooted"] += 1 + } else if strings.HasPrefix(line, "#include ") { + ignoreStats["includes"] += 1 + if strings.Contains(line, "..") { + ignoreStats["escapedIncludes"] += 1 + } + } + + if strings.Contains(line, "**") { + ignoreStats["doubleStars"] += 1 + // Remove not to trip up star checks. + strings.Replace(line, "**", "", -1) + } + + if strings.Contains(line, "*") { + ignoreStats["stars"] += 1 + } + } + } + stats["ignoreStats"] = ignoreStats + } + return stats +} + type ConnectionInfo struct { protocol.Statistics Connected bool @@ -2449,6 +2549,7 @@ func (m *Model) CommitConfiguration(from, to config.Configuration) bool { // Some options don't require restart as those components handle it fine // by themselves. from.Options.URAccepted = to.Options.URAccepted + from.Options.URSeen = to.Options.URSeen from.Options.URUniqueID = to.Options.URUniqueID from.Options.ListenAddresses = to.Options.ListenAddresses from.Options.RelaysEnabled = to.Options.RelaysEnabled diff --git a/lib/model/model_test.go b/lib/model/model_test.go index 7aa4f9ae2..c54e211e0 100644 --- a/lib/model/model_test.go +++ b/lib/model/model_test.go @@ -308,6 +308,9 @@ func (f *fakeConnection) RemoteAddr() net.Addr { func (f *fakeConnection) Type() string { return "fake" } +func (f *fakeConnection) Transport() string { + return "fake" +} func (f *fakeConnection) DownloadProgress(folder string, updates []protocol.FileDownloadProgressUpdate) { f.downloadProgressMessages = append(f.downloadProgressMessages, downloadProgressMessage{ diff --git a/lib/model/rwfolder.go b/lib/model/rwfolder.go index f84d2c2f7..acb3348de 100644 --- a/lib/model/rwfolder.go +++ b/lib/model/rwfolder.go @@ -93,6 +93,9 @@ type sendReceiveFolder struct { errors map[string]string // path -> error string errorsMut sync.Mutex + + blockStats map[string]int + blockStatsMut sync.Mutex } func newSendReceiveFolder(model *Model, cfg config.FolderConfiguration, ver versioner.Versioner, fs fs.Filesystem) service { @@ -107,6 +110,9 @@ func newSendReceiveFolder(model *Model, cfg config.FolderConfiguration, ver vers remoteIndex: make(chan struct{}, 1), // This needs to be 1-buffered so that we queue a notification if we're busy doing a pull when it comes. errorsMut: sync.NewMutex(), + + blockStats: make(map[string]int), + blockStatsMut: sync.NewMutex(), } f.configureCopiersAndPullers() @@ -875,6 +881,11 @@ func (f *sendReceiveFolder) renameFile(source, target protocol.FileInfo) { } if err == nil { + f.blockStatsMut.Lock() + f.blockStats["total"] += len(target.Blocks) + f.blockStats["renamed"] += len(target.Blocks) + f.blockStatsMut.Unlock() + // The file was renamed, so we have handled both the necessary delete // of the source and the creation of the target. Fix-up the metadata, // and update the local index of the target file. @@ -1443,6 +1454,15 @@ func (f *sendReceiveFolder) finisherRoutine(in <-chan *sharedPullerState) { if err != nil { l.Debugln("Puller: final:", err) f.newError(state.file.Name, err) + } else { + f.blockStatsMut.Lock() + f.blockStats["total"] += state.reused + state.copyTotal + state.pullTotal + f.blockStats["reused"] += state.reused + f.blockStats["pulled"] += state.pullTotal + f.blockStats["copyOrigin"] += state.copyOrigin + f.blockStats["copyOriginShifted"] += state.copyOriginShifted + f.blockStats["copyElsewhere"] += state.copyTotal - state.copyOrigin + f.blockStatsMut.Unlock() } events.Default.Log(events.ItemFinished, map[string]interface{}{ "folder": f.folderID, @@ -1459,6 +1479,16 @@ func (f *sendReceiveFolder) finisherRoutine(in <-chan *sharedPullerState) { } } +func (f *sendReceiveFolder) BlockStats() map[string]int { + f.blockStatsMut.Lock() + stats := make(map[string]int) + for k, v := range f.blockStats { + stats[k] = v + } + f.blockStatsMut.Unlock() + return stats +} + // Moves the given filename to the front of the job queue func (f *sendReceiveFolder) BringToFront(filename string) { f.queue.BringToFront(filename)