cmd/syncthing: Add more stats to usage reports (ref #3628)

GitHub-Pull-Request: https://github.com/syncthing/syncthing/pull/4347
This commit is contained in:
Audrius Butkevicius 2017-10-12 06:16:46 +00:00 committed by Jakob Borg
parent 813e6ddf83
commit 2760d032ca
23 changed files with 358 additions and 117 deletions

View File

@ -100,6 +100,7 @@ type modelIntf interface {
CurrentSequence(folder string) (int64, bool) CurrentSequence(folder string) (int64, bool)
RemoteSequence(folder string) (int64, bool) RemoteSequence(folder string) (int64, bool)
State(folder string) (string, time.Time, error) State(folder string) (string, time.Time, error)
UsageReportingStats(version int) map[string]interface{}
} }
type configIntf interface { type configIntf interface {
@ -119,6 +120,7 @@ type configIntf interface {
type connectionsIntf interface { type connectionsIntf interface {
Status() map[string]interface{} Status() map[string]interface{}
NATType() string
} }
type rater interface { type rater interface {
@ -800,18 +802,6 @@ func (s *apiService) postSystemConfig(w http.ResponseWriter, r *http.Request) {
} }
} }
// Fixup usage reporting settings
if curAcc := s.cfg.Options().URAccepted; to.Options.URAccepted > curAcc {
// UR was enabled
to.Options.URAccepted = usageReportVersion
to.Options.URUniqueID = rand.String(8)
} else if to.Options.URAccepted < curAcc {
// UR was disabled
to.Options.URAccepted = -1
to.Options.URUniqueID = ""
}
// Activate and save // Activate and save
if err := s.cfg.Replace(to); err != nil { if err := s.cfg.Replace(to); err != nil {
@ -903,6 +893,7 @@ func (s *apiService) getSystemStatus(w http.ResponseWriter, r *http.Request) {
// gives us percent // gives us percent
res["cpuPercent"] = s.cpu.Rate() / 10 / float64(runtime.NumCPU()) res["cpuPercent"] = s.cpu.Rate() / 10 / float64(runtime.NumCPU())
res["pathSeparator"] = string(filepath.Separator) res["pathSeparator"] = string(filepath.Separator)
res["urVersionMax"] = usageReportVersion
res["uptime"] = int(time.Since(startTime).Seconds()) res["uptime"] = int(time.Since(startTime).Seconds())
res["startTime"] = startTime res["startTime"] = startTime
@ -981,7 +972,11 @@ func (s *apiService) getSystemDiscovery(w http.ResponseWriter, r *http.Request)
} }
func (s *apiService) getReport(w http.ResponseWriter, r *http.Request) { func (s *apiService) getReport(w http.ResponseWriter, r *http.Request) {
sendJSON(w, reportData(s.cfg, s.model)) version := usageReportVersion
if val, _ := strconv.Atoi(r.URL.Query().Get("version")); val > 0 {
version = val
}
sendJSON(w, reportData(s.cfg, s.model, s.connectionsService, version))
} }
func (s *apiService) getRandomString(w http.ResponseWriter, r *http.Request) { func (s *apiService) getRandomString(w http.ResponseWriter, r *http.Request) {

View File

@ -882,24 +882,14 @@ func syncthingMain(runtimeOptions RuntimeOptions) {
// Unique ID will be set and config saved below if necessary. // Unique ID will be set and config saved below if necessary.
} }
if opts.URAccepted > 0 && opts.URAccepted < usageReportVersion { if opts.URUniqueID == "" {
l.Infoln("Anonymous usage report has changed; revoking acceptance")
opts.URAccepted = 0
opts.URUniqueID = ""
cfg.SetOptions(opts)
}
if opts.URAccepted >= usageReportVersion && opts.URUniqueID == "" {
// Generate and save a new unique ID if it is missing.
opts.URUniqueID = rand.String(8) opts.URUniqueID = rand.String(8)
cfg.SetOptions(opts) cfg.SetOptions(opts)
cfg.Save() cfg.Save()
} }
// The usageReportingManager registers itself to listen to configuration usageReportingSvc := newUsageReportingService(cfg, m, connectionsService)
// changes, and there's nothing more we need to tell it from the outside. mainService.Add(usageReportingSvc)
// Hence we don't keep the returned pointer.
newUsageReportingManager(cfg, m)
if opts.RestartOnWakeup { if opts.RestartOnWakeup {
go standbyMonitor() go standbyMonitor()

View File

@ -11,3 +11,7 @@ type mockedConnections struct{}
func (m *mockedConnections) Status() map[string]interface{} { func (m *mockedConnections) Status() map[string]interface{} {
return nil return nil
} }
func (m *mockedConnections) NATType() string {
return ""
}

View File

@ -114,3 +114,7 @@ func (m *mockedModel) RemoteSequence(folder string) (int64, bool) {
func (m *mockedModel) State(folder string) (string, time.Time, error) { func (m *mockedModel) State(folder string) (string, time.Time, error) {
return "", time.Time{}, nil return "", time.Time{}, nil
} }
func (m *mockedModel) UsageReportingStats(version int) map[string]interface{} {
return nil
}

View File

@ -12,7 +12,6 @@ import (
"crypto/rand" "crypto/rand"
"crypto/tls" "crypto/tls"
"encoding/json" "encoding/json"
"fmt"
"net/http" "net/http"
"runtime" "runtime"
"sort" "sort"
@ -20,71 +19,25 @@ import (
"time" "time"
"github.com/syncthing/syncthing/lib/config" "github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/connections"
"github.com/syncthing/syncthing/lib/dialer" "github.com/syncthing/syncthing/lib/dialer"
"github.com/syncthing/syncthing/lib/model" "github.com/syncthing/syncthing/lib/model"
"github.com/syncthing/syncthing/lib/protocol" "github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/scanner" "github.com/syncthing/syncthing/lib/scanner"
"github.com/syncthing/syncthing/lib/upgrade" "github.com/syncthing/syncthing/lib/upgrade"
"github.com/thejerf/suture"
) )
// Current version number of the usage report, for acceptance purposes. If // Current version number of the usage report, for acceptance purposes. If
// fields are added or changed this integer must be incremented so that users // fields are added or changed this integer must be incremented so that users
// are prompted for acceptance of the new report. // are prompted for acceptance of the new report.
const usageReportVersion = 2 const usageReportVersion = 3
type usageReportingManager struct {
cfg *config.Wrapper
model *model.Model
sup *suture.Supervisor
}
func newUsageReportingManager(cfg *config.Wrapper, m *model.Model) *usageReportingManager {
mgr := &usageReportingManager{
cfg: cfg,
model: m,
}
// Start UR if it's enabled.
mgr.CommitConfiguration(config.Configuration{}, cfg.RawCopy())
// Listen to future config changes so that we can start and stop as
// appropriate.
cfg.Subscribe(mgr)
return mgr
}
func (m *usageReportingManager) VerifyConfiguration(from, to config.Configuration) error {
return nil
}
func (m *usageReportingManager) CommitConfiguration(from, to config.Configuration) bool {
if to.Options.URAccepted >= usageReportVersion && m.sup == nil {
// Usage reporting was turned on; lets start it.
service := newUsageReportingService(m.cfg, m.model)
m.sup = suture.NewSimple("usageReporting")
m.sup.Add(service)
m.sup.ServeBackground()
} else if to.Options.URAccepted < usageReportVersion && m.sup != nil {
// Usage reporting was turned off
m.sup.Stop()
m.sup = nil
}
return true
}
func (m *usageReportingManager) String() string {
return fmt.Sprintf("usageReportingManager@%p", m)
}
// reportData returns the data to be sent in a usage report. It's used in // reportData returns the data to be sent in a usage report. It's used in
// various places, so not part of the usageReportingManager object. // various places, so not part of the usageReportingManager object.
func reportData(cfg configIntf, m modelIntf) map[string]interface{} { func reportData(cfg configIntf, m modelIntf, connectionsService connectionsIntf, version int) map[string]interface{} {
opts := cfg.Options() opts := cfg.Options()
res := make(map[string]interface{}) res := make(map[string]interface{})
res["urVersion"] = usageReportVersion res["urVersion"] = version
res["uniqueID"] = opts.URUniqueID res["uniqueID"] = opts.URUniqueID
res["version"] = Version res["version"] = Version
res["longVersion"] = LongVersion res["longVersion"] = LongVersion
@ -227,25 +180,40 @@ func reportData(cfg configIntf, m modelIntf) map[string]interface{} {
res["upgradeAllowedAuto"] = !(upgrade.DisabledByCompilation || noUpgradeFromEnv) && opts.AutoUpgradeIntervalH > 0 res["upgradeAllowedAuto"] = !(upgrade.DisabledByCompilation || noUpgradeFromEnv) && opts.AutoUpgradeIntervalH > 0
res["upgradeAllowedPre"] = !(upgrade.DisabledByCompilation || noUpgradeFromEnv) && opts.AutoUpgradeIntervalH > 0 && opts.UpgradeToPreReleases res["upgradeAllowedPre"] = !(upgrade.DisabledByCompilation || noUpgradeFromEnv) && opts.AutoUpgradeIntervalH > 0 && opts.UpgradeToPreReleases
if version >= 3 {
res["uptime"] = time.Now().Sub(startTime).Seconds()
res["natType"] = connectionsService.NATType()
}
for key, value := range m.UsageReportingStats(version) {
res[key] = value
}
return res return res
} }
type usageReportingService struct { type usageReportingService struct {
cfg *config.Wrapper cfg *config.Wrapper
model *model.Model model *model.Model
stop chan struct{} connectionsService *connections.Service
forceRun chan struct{}
stop chan struct{}
} }
func newUsageReportingService(cfg *config.Wrapper, model *model.Model) *usageReportingService { func newUsageReportingService(cfg *config.Wrapper, model *model.Model, connectionsService *connections.Service) *usageReportingService {
return &usageReportingService{ svc := &usageReportingService{
cfg: cfg, cfg: cfg,
model: model, model: model,
stop: make(chan struct{}), connectionsService: connectionsService,
forceRun: make(chan struct{}),
stop: make(chan struct{}),
} }
cfg.Subscribe(svc)
return svc
} }
func (s *usageReportingService) sendUsageReport() error { func (s *usageReportingService) sendUsageReport() error {
d := reportData(s.cfg, s.model) d := reportData(s.cfg, s.model, s.connectionsService, s.cfg.Options().URAccepted)
var b bytes.Buffer var b bytes.Buffer
json.NewEncoder(&b).Encode(d) json.NewEncoder(&b).Encode(d)
@ -264,27 +232,45 @@ func (s *usageReportingService) sendUsageReport() error {
func (s *usageReportingService) Serve() { func (s *usageReportingService) Serve() {
s.stop = make(chan struct{}) s.stop = make(chan struct{})
t := time.NewTimer(time.Duration(s.cfg.Options().URInitialDelayS) * time.Second)
l.Infoln("Starting usage reporting")
defer l.Infoln("Stopping usage reporting")
t := time.NewTimer(time.Duration(s.cfg.Options().URInitialDelayS) * time.Second) // time to initial report at start
for { for {
select { select {
case <-s.stop: case <-s.stop:
return return
case <-s.forceRun:
t.Reset(0)
case <-t.C: case <-t.C:
err := s.sendUsageReport() if s.cfg.Options().URAccepted >= 2 {
if err != nil { err := s.sendUsageReport()
l.Infoln("Usage report:", err) if err != nil {
l.Infoln("Usage report:", err)
} else {
l.Infof("Sent usage report (version %d)", s.cfg.Options().URAccepted)
}
} }
t.Reset(24 * time.Hour) // next report tomorrow t.Reset(24 * time.Hour) // next report tomorrow
} }
} }
} }
func (s *usageReportingService) VerifyConfiguration(from, to config.Configuration) error {
return nil
}
func (s *usageReportingService) CommitConfiguration(from, to config.Configuration) bool {
if from.Options.URAccepted != to.Options.URAccepted || from.Options.URUniqueID != to.Options.URUniqueID || from.Options.URURL != to.Options.URURL {
s.forceRun <- struct{}{}
}
return true
}
func (s *usageReportingService) Stop() { func (s *usageReportingService) Stop() {
close(s.stop) close(s.stop)
close(s.forceRun)
}
func (usageReportingService) String() string {
return "usageReportingService"
} }
// cpuBench returns CPU performance as a measure of single threaded SHA-256 MiB/s // cpuBench returns CPU performance as a measure of single threaded SHA-256 MiB/s

View File

@ -1,6 +1,8 @@
angular.module('syncthing.core') angular.module('syncthing.core')
.directive('modal', function () { .directive('modal', function () {
return { return {
// If you ever change any of the petroglyphs below, please search for $parent.$parent,
// as some templates rely on the way scope is composed in this case.
restrict: 'E', restrict: 'E',
templateUrl: 'modal.html', templateUrl: 'modal.html',
replace: true, replace: true,

View File

@ -33,6 +33,8 @@ angular.module('syncthing.core')
$scope.folderRejections = {}; $scope.folderRejections = {};
$scope.protocolChanged = false; $scope.protocolChanged = false;
$scope.reportData = {}; $scope.reportData = {};
$scope.reportDataPreview = {};
$scope.reportDataPreviewVersion = '';
$scope.reportPreview = false; $scope.reportPreview = false;
$scope.folders = {}; $scope.folders = {};
$scope.seenError = ''; $scope.seenError = '';
@ -133,7 +135,11 @@ angular.module('syncthing.core')
}).error($scope.emitHTTPError); }).error($scope.emitHTTPError);
$http.get(urlbase + '/svc/report').success(function (data) { $http.get(urlbase + '/svc/report').success(function (data) {
$scope.reportData = data; $scope.reportDataPreview = $scope.reportData = data;
if ($scope.system && $scope.config.options.urSeen < $scope.system.urVersionMax) {
// Usage reporting format has changed, prompt the user to re-accept.
$('#ur').modal();
}
}).error($scope.emitHTTPError); }).error($scope.emitHTTPError);
$http.get(urlbase + '/system/upgrade').success(function (data) { $http.get(urlbase + '/system/upgrade').success(function (data) {
@ -376,6 +382,7 @@ angular.module('syncthing.core')
$scope.config = config; $scope.config = config;
$scope.config.options._listenAddressesStr = $scope.config.options.listenAddresses.join(', '); $scope.config.options._listenAddressesStr = $scope.config.options.listenAddresses.join(', ');
$scope.config.options._globalAnnounceServersStr = $scope.config.options.globalAnnounceServers.join(', '); $scope.config.options._globalAnnounceServersStr = $scope.config.options.globalAnnounceServers.join(', ');
$scope.config.options._urAcceptedStr = "" + $scope.config.options.urAccepted;
$scope.devices = $scope.config.devices; $scope.devices = $scope.config.devices;
$scope.devices.forEach(function (deviceCfg) { $scope.devices.forEach(function (deviceCfg) {
@ -412,6 +419,10 @@ angular.module('syncthing.core')
$scope.myID = data.myID; $scope.myID = data.myID;
$scope.system = data; $scope.system = data;
if ($scope.reportDataPreviewVersion === '') {
$scope.reportDataPreviewVersion = $scope.system.urVersionMax;
}
var listenersFailed = []; var listenersFailed = [];
for (var address in data.connectionServiceStatus) { for (var address in data.connectionServiceStatus) {
if (data.connectionServiceStatus[address].error) { if (data.connectionServiceStatus[address].error) {
@ -1058,7 +1069,6 @@ angular.module('syncthing.core')
$scope.editSettings = function () { $scope.editSettings = function () {
// Make a working copy // Make a working copy
$scope.tmpOptions = angular.copy($scope.config.options); $scope.tmpOptions = angular.copy($scope.config.options);
$scope.tmpOptions.urEnabled = ($scope.tmpOptions.urAccepted > 0);
$scope.tmpOptions.deviceName = $scope.thisDevice().name; $scope.tmpOptions.deviceName = $scope.thisDevice().name;
$scope.tmpOptions.upgrades = "none"; $scope.tmpOptions.upgrades = "none";
if ($scope.tmpOptions.autoUpgradeIntervalH > 0) { if ($scope.tmpOptions.autoUpgradeIntervalH > 0) {
@ -1088,18 +1098,31 @@ angular.module('syncthing.core')
}).error($scope.emitHTTPError); }).error($scope.emitHTTPError);
}; };
$scope.urVersions = function() {
var result = [];
if ($scope.system) {
for (var i = $scope.system.urVersionMax; i >= 2; i--) {
result.push("" + i);
}
}
return result;
};
$scope.saveSettings = function () { $scope.saveSettings = function () {
// Make sure something changed // Make sure something changed
var changed = !angular.equals($scope.config.options, $scope.tmpOptions) || !angular.equals($scope.config.gui, $scope.tmpGUI); var changed = !angular.equals($scope.config.options, $scope.tmpOptions) || !angular.equals($scope.config.gui, $scope.tmpGUI);
var themeChanged = $scope.config.gui.theme !== $scope.tmpGUI.theme; var themeChanged = $scope.config.gui.theme !== $scope.tmpGUI.theme;
if (changed) { if (changed) {
// Angular has issues with selects with numeric values, so we handle strings here.
$scope.tmpOptions.urAccepted = parseInt($scope.tmpOptions._urAcceptedStr);
// Check if auto-upgrade has been enabled or disabled. This // Check if auto-upgrade has been enabled or disabled. This
// also has an effect on usage reporting, so do the check // also has an effect on usage reporting, so do the check
// for that later. // for that later.
if ($scope.tmpOptions.upgrades == "candidate") { if ($scope.tmpOptions.upgrades == "candidate") {
$scope.tmpOptions.autoUpgradeIntervalH = $scope.tmpOptions.autoUpgradeIntervalH || 12; $scope.tmpOptions.autoUpgradeIntervalH = $scope.tmpOptions.autoUpgradeIntervalH || 12;
$scope.tmpOptions.upgradeToPreReleases = true; $scope.tmpOptions.upgradeToPreReleases = true;
$scope.tmpOptions.urEnabled = true; $scope.tmpOptions.urAccepted = $scope.system.urVersionMax;
$scope.tmpOptions.urSeen = $scope.system.urVersionMax;
} else if ($scope.tmpOptions.upgrades == "stable") { } else if ($scope.tmpOptions.upgrades == "stable") {
$scope.tmpOptions.autoUpgradeIntervalH = $scope.tmpOptions.autoUpgradeIntervalH || 12; $scope.tmpOptions.autoUpgradeIntervalH = $scope.tmpOptions.autoUpgradeIntervalH || 12;
$scope.tmpOptions.upgradeToPreReleases = false; $scope.tmpOptions.upgradeToPreReleases = false;
@ -1107,13 +1130,6 @@ angular.module('syncthing.core')
$scope.tmpOptions.autoUpgradeIntervalH = 0; $scope.tmpOptions.autoUpgradeIntervalH = 0;
} }
// Check if usage reporting has been enabled or disabled
if ($scope.tmpOptions.urEnabled && $scope.tmpOptions.urAccepted <= 0) {
$scope.tmpOptions.urAccepted = 1000;
} else if (!$scope.tmpOptions.urEnabled && $scope.tmpOptions.urAccepted > 0) {
$scope.tmpOptions.urAccepted = -1;
}
// Check if protocol will need to be changed on restart // Check if protocol will need to be changed on restart
if ($scope.config.gui.useTLS !== $scope.tmpGUI.useTLS) { if ($scope.config.gui.useTLS !== $scope.tmpGUI.useTLS) {
$scope.protocolChanged = true; $scope.protocolChanged = true;
@ -1691,13 +1707,17 @@ angular.module('syncthing.core')
}; };
$scope.acceptUR = function () { $scope.acceptUR = function () {
$scope.config.options.urAccepted = 1000; // Larger than the largest existing report version $scope.config.options.urAccepted = $scope.system.urVersionMax;
$scope.config.options.urSeen = $scope.system.urVersionMax;
$scope.saveConfig(); $scope.saveConfig();
$('#ur').modal('hide'); $('#ur').modal('hide');
}; };
$scope.declineUR = function () { $scope.declineUR = function () {
$scope.config.options.urAccepted = -1; if ($scope.config.options.urAccepted === 0) {
$scope.config.options.urAccepted = -1;
}
$scope.config.options.urSeen = $scope.system.urVersionMax;
$scope.saveConfig(); $scope.saveConfig();
$('#ur').modal('hide'); $('#ur').modal('hide');
}; };
@ -1747,6 +1767,13 @@ angular.module('syncthing.core')
$scope.reportPreview = true; $scope.reportPreview = true;
}; };
$scope.refreshReportDataPreview = function () {
$scope.reportDataPreview = '';
$http.get(urlbase + '/svc/report?version=' + $scope.reportDataPreviewVersion).success(function (data) {
$scope.reportDataPreview = data;
}).error($scope.emitHTTPError);
};
$scope.rescanAllFolders = function () { $scope.rescanAllFolders = function () {
$http.post(urlbase + "/db/scan"); $http.post(urlbase + "/db/scan");
}; };

View File

@ -139,10 +139,14 @@
</div> </div>
<div class="form-group"> <div class="form-group">
<div class="checkbox" ng-if="tmpOptions.upgrades != 'candidate'"> <div ng-if="tmpOptions.upgrades != 'candidate'">
<label> <label translate for="urVersion">Anonymous Usage Reporting</label> (<a href="" translate data-toggle="modal" data-target="#urPreview">Preview</a>)
<input id="UREnabled" type="checkbox" ng-model="tmpOptions.urEnabled"/> <span translate>Anonymous Usage Reporting</span> (<a href="" translate data-toggle="modal" data-target="#urPreview">Preview</a>) <select class="form-control" id="urVersion" ng-model="tmpOptions._urAcceptedStr">
</label> <option ng-repeat="n in urVersions()" value="{{n}}">Version {{n}}</option>
<!-- 1 does not exist, as we did not support incremental formats back then. -->
<option value="0" translate>Undecided (will prompt)</option>
<option value="-1" translate>Disabled</option>
</select>
</div> </div>
<p class="help-block" ng-if="tmpOptions.upgrades == 'candidate'"> <p class="help-block" ng-if="tmpOptions.upgrades == 'candidate'">
<span translate>Usage reporting is always enabled for candidate releases.</span> (<a href="" translate data-toggle="modal" data-target="#urPreview">Preview</a>) <span translate>Usage reporting is always enabled for candidate releases.</span> (<a href="" translate data-toggle="modal" data-target="#urPreview">Preview</a>)

View File

@ -1,8 +1,13 @@
<modal id="ur" status="info" icon="bar-chart" heading="{{'Allow Anonymous Usage Reporting?' | translate}}" large="yes" closeable="no"> <modal id="ur" status="info" icon="bar-chart" heading="{{'Allow Anonymous Usage Reporting?' | translate}}" large="yes" closeable="no">
<div class="modal-body"> <div class="modal-body">
<p translate>The encrypted usage report is sent daily. It is used to track common platforms, folder sizes and app versions. If the reported data set is changed you will be prompted with this dialog again.</p> <div ng-if="config.options.urAccepted > 0 && config.options.urAccepted < system.urVersionMax">
<p translate>The aggregated statistics are publicly available at the URL below.</p> <p>Anonymous Usage report format has changed. Would you like to move to the new format?</p>
<p><a href="https://data.syncthing.net/" target="_blank">https://data.syncthing.net/</a></p> </div>
<div ng-if="!(config.options.urAccepted > 0 && config.options.urAccepted < system.urVersionMax)">
<p translate>The encrypted usage report is sent daily. It is used to track common platforms, folder sizes and app versions. If the reported data set is changed you will be prompted with this dialog again.</p>
<p translate>The aggregated statistics are publicly available at the URL below.</p>
<p><a href="https://data.syncthing.net/" target="_blank">https://data.syncthing.net/</a></p>
</div>
<button type="button" class="btn btn-default btn-sm" ng-click="showReportPreview()" ng-show="!reportPreview"> <button type="button" class="btn btn-default btn-sm" ng-click="showReportPreview()" ng-show="!reportPreview">
<span class="fa fa-file-text-o"></span>&nbsp;<span translate>Preview Usage Report</span> <span class="fa fa-file-text-o"></span>&nbsp;<span translate>Preview Usage Report</span>
</button> </button>

View File

@ -5,8 +5,13 @@
</p> </p>
<p translate>The aggregated statistics are publicly available at the URL below.</p> <p translate>The aggregated statistics are publicly available at the URL below.</p>
<p><a href="https://data.syncthing.net/" target="_blank">https://data.syncthing.net/</a></p> <p><a href="https://data.syncthing.net/" target="_blank">https://data.syncthing.net/</a></p>
<label translate>Version</label>
<select id="urPreviewVersion" class="form-control" ng-model="$parent.$parent.reportDataPreviewVersion" ng-change="refreshReportDataPreview()" >
<option ng-repeat="n in urVersions()" value="{{n}}">Version {{n}}</option>
</select>
<hr>
<form> <form>
<textarea class="form-control" rows="20">{{reportData | json}}</textarea> <textarea class="form-control" rows="20" ng-if="reportDataPreview">{{reportDataPreview | json}}</textarea>
</form> </form>
</div> </div>
<div class="modal-footer"> <div class="modal-footer">

View File

@ -32,7 +32,7 @@ import (
const ( const (
OldestHandledVersion = 10 OldestHandledVersion = 10
CurrentVersion = 23 CurrentVersion = 24
MaxRescanIntervalS = 365 * 24 * 60 * 60 MaxRescanIntervalS = 365 * 24 * 60 * 60
) )
@ -326,6 +326,9 @@ func (cfg *Configuration) clean() error {
if cfg.Version == 22 { if cfg.Version == 22 {
convertV22V23(cfg) convertV22V23(cfg)
} }
if cfg.Version == 23 {
convertV23V24(cfg)
}
// Build a list of available devices // Build a list of available devices
existingDevices := make(map[protocol.DeviceID]bool) existingDevices := make(map[protocol.DeviceID]bool)
@ -375,6 +378,12 @@ func (cfg *Configuration) clean() error {
return nil return nil
} }
func convertV23V24(cfg *Configuration) {
cfg.Options.URSeen = 2
cfg.Version = 24
}
func convertV22V23(cfg *Configuration) { func convertV22V23(cfg *Configuration) {
permBits := fs.FileMode(0777) permBits := fs.FileMode(0777)
if runtime.GOOS == "windows" { if runtime.GOOS == "windows" {

View File

@ -200,6 +200,7 @@ func TestOverriddenValues(t *testing.T) {
ProgressUpdateIntervalS: 10, ProgressUpdateIntervalS: 10,
LimitBandwidthInLan: true, LimitBandwidthInLan: true,
MinHomeDiskFree: Size{5.2, "%"}, MinHomeDiskFree: Size{5.2, "%"},
URSeen: 2,
URURL: "https://localhost/newdata", URURL: "https://localhost/newdata",
URInitialDelayS: 800, URInitialDelayS: 800,
URPostInsecurely: true, URPostInsecurely: true,

View File

@ -112,6 +112,7 @@ type OptionsConfiguration struct {
NATRenewalM int `xml:"natRenewalMinutes" json:"natRenewalMinutes" default:"30"` NATRenewalM int `xml:"natRenewalMinutes" json:"natRenewalMinutes" default:"30"`
NATTimeoutS int `xml:"natTimeoutSeconds" json:"natTimeoutSeconds" default:"10"` NATTimeoutS int `xml:"natTimeoutSeconds" json:"natTimeoutSeconds" default:"10"`
URAccepted int `xml:"urAccepted" json:"urAccepted"` // Accepted usage reporting version; 0 for off (undecided), -1 for off (permanently) URAccepted int `xml:"urAccepted" json:"urAccepted"` // Accepted usage reporting version; 0 for off (undecided), -1 for off (permanently)
URSeen int `xml:"urSeen" json:"urSeen"` // Report which the user has been prompted for.
URUniqueID string `xml:"urUniqueID" json:"urUniqueId"` // Unique ID for reporting purposes, regenerated when UR is turned on. URUniqueID string `xml:"urUniqueID" json:"urUniqueId"` // Unique ID for reporting purposes, regenerated when UR is turned on.
URURL string `xml:"urURL" json:"urURL" default:"https://data.syncthing.net/newdata"` URURL string `xml:"urURL" json:"urURL" default:"https://data.syncthing.net/newdata"`
URPostInsecurely bool `xml:"urPostInsecurely" json:"urPostInsecurely" default:"false"` // For testing URPostInsecurely bool `xml:"urPostInsecurely" json:"urPostInsecurely" default:"false"` // For testing

16
lib/config/testdata/v24.xml vendored Normal file
View File

@ -0,0 +1,16 @@
<configuration version="22">
<folder id="test" path="testdata" type="readonly" ignorePerms="false" rescanIntervalS="600" autoNormalize="true">
<filesystemType>basic</filesystemType>
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR"></device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2"></device>
<minDiskFree unit="%">1</minDiskFree>
<maxConflicts>-1</maxConflicts>
<fsync>true</fsync>
</folder>
<device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR" name="node one" compression="metadata">
<address>tcp://a</address>
</device>
<device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" name="node two" compression="metadata">
<address>tcp://b</address>
</device>
</configuration>

View File

@ -12,14 +12,16 @@ import (
"net/url" "net/url"
"strings" "strings"
"sync" "sync"
"sync/atomic"
"time" "time"
"github.com/AudriusButkevicius/kcp-go" "github.com/AudriusButkevicius/kcp-go"
"github.com/AudriusButkevicius/pfilter" "github.com/AudriusButkevicius/pfilter"
"github.com/ccding/go-stun/stun" "github.com/ccding/go-stun/stun"
"github.com/xtaci/smux"
"github.com/syncthing/syncthing/lib/config" "github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/nat" "github.com/syncthing/syncthing/lib/nat"
"github.com/xtaci/smux"
) )
func init() { func init() {
@ -38,6 +40,7 @@ type kcpListener struct {
stop chan struct{} stop chan struct{}
conns chan internalConn conns chan internalConn
factory listenerFactory factory listenerFactory
nat atomic.Value
address *url.URL address *url.URL
err error err error
@ -183,6 +186,14 @@ func (t *kcpListener) Factory() listenerFactory {
return t.factory return t.factory
} }
func (t *kcpListener) NATType() string {
v := t.nat.Load().(stun.NATType)
if v == stun.NATUnknown || v == stun.NATError {
return "unknown"
}
return v.String()
}
func (t *kcpListener) stunRenewal(listener net.PacketConn) { func (t *kcpListener) stunRenewal(listener net.PacketConn) {
client := stun.NewClientWithConnection(listener) client := stun.NewClientWithConnection(listener)
client.SetSoftwareName("syncthing") client.SetSoftwareName("syncthing")
@ -199,6 +210,7 @@ func (t *kcpListener) stunRenewal(listener net.PacketConn) {
if t.cfg.Options().StunKeepaliveS < 1 { if t.cfg.Options().StunKeepaliveS < 1 {
time.Sleep(time.Second) time.Sleep(time.Second)
oldType = stun.NATUnknown oldType = stun.NATUnknown
t.nat.Store(stun.NATUnknown)
t.mut.Lock() t.mut.Lock()
t.address = nil t.address = nil
t.mut.Unlock() t.mut.Unlock()
@ -222,6 +234,7 @@ func (t *kcpListener) stunRenewal(listener net.PacketConn) {
if oldType != natType { if oldType != natType {
l.Infof("%s detected NAT type: %s", t.uri, natType) l.Infof("%s detected NAT type: %s", t.uri, natType)
t.nat.Store(natType)
} }
for { for {
@ -273,7 +286,7 @@ func (t *kcpListener) stunRenewal(listener net.PacketConn) {
type kcpListenerFactory struct{} type kcpListenerFactory struct{}
func (f *kcpListenerFactory) New(uri *url.URL, cfg *config.Wrapper, tlsCfg *tls.Config, conns chan internalConn, natService *nat.Service) genericListener { func (f *kcpListenerFactory) New(uri *url.URL, cfg *config.Wrapper, tlsCfg *tls.Config, conns chan internalConn, natService *nat.Service) genericListener {
return &kcpListener{ l := &kcpListener{
uri: fixupPort(uri, config.DefaultKCPPort), uri: fixupPort(uri, config.DefaultKCPPort),
cfg: cfg, cfg: cfg,
tlsCfg: tlsCfg, tlsCfg: tlsCfg,
@ -281,6 +294,8 @@ func (f *kcpListenerFactory) New(uri *url.URL, cfg *config.Wrapper, tlsCfg *tls.
stop: make(chan struct{}), stop: make(chan struct{}),
factory: f, factory: f,
} }
l.nat.Store(stun.NATUnknown)
return l
} }
func (kcpListenerFactory) Enabled(cfg config.Configuration) bool { func (kcpListenerFactory) Enabled(cfg config.Configuration) bool {

View File

@ -171,6 +171,10 @@ func (t *relayListener) String() string {
return t.uri.String() return t.uri.String()
} }
func (t *relayListener) NATType() string {
return "unknown"
}
type relayListenerFactory struct{} type relayListenerFactory struct{}
func (f *relayListenerFactory) New(uri *url.URL, cfg *config.Wrapper, tlsCfg *tls.Config, conns chan internalConn, natService *nat.Service) genericListener { func (f *relayListenerFactory) New(uri *url.URL, cfg *config.Wrapper, tlsCfg *tls.Config, conns chan internalConn, natService *nat.Service) genericListener {

View File

@ -574,6 +574,18 @@ func (s *Service) Status() map[string]interface{} {
return result return result
} }
func (s *Service) NATType() string {
s.listenersMut.RLock()
defer s.listenersMut.RUnlock()
for _, listener := range s.listeners {
natType := listener.NATType()
if natType != "unknown" {
return natType
}
}
return "unknown"
}
func (s *Service) getDialerFactory(cfg config.Configuration, uri *url.URL) (dialerFactory, error) { func (s *Service) getDialerFactory(cfg config.Configuration, uri *url.URL) (dialerFactory, error) {
dialerFactory, ok := dialers[uri.Scheme] dialerFactory, ok := dialers[uri.Scheme]
if !ok { if !ok {

View File

@ -25,6 +25,7 @@ type Connection interface {
protocol.Connection protocol.Connection
io.Closer io.Closer
Type() string Type() string
Transport() string
RemoteAddr() net.Addr RemoteAddr() net.Addr
} }
@ -74,10 +75,27 @@ func (t connType) String() string {
} }
} }
func (t connType) Transport() string {
switch t {
case connTypeRelayClient, connTypeRelayServer:
return "relay"
case connTypeTCPClient, connTypeTCPServer:
return "tcp"
case connTypeKCPClient, connTypeKCPServer:
return "kcp"
default:
return "unknown"
}
}
func (c internalConn) Type() string { func (c internalConn) Type() string {
return c.connType.String() return c.connType.String()
} }
func (c internalConn) Transport() string {
return c.connType.Transport()
}
func (c internalConn) String() string { func (c internalConn) String() string {
return fmt.Sprintf("%s-%s/%s", c.LocalAddr(), c.RemoteAddr(), c.connType.String()) return fmt.Sprintf("%s-%s/%s", c.LocalAddr(), c.RemoteAddr(), c.connType.String())
} }
@ -116,6 +134,7 @@ type genericListener interface {
OnAddressesChanged(func(genericListener)) OnAddressesChanged(func(genericListener))
String() string String() string
Factory() listenerFactory Factory() listenerFactory
NATType() string
} }
type Model interface { type Model interface {

View File

@ -176,6 +176,10 @@ func (t *tcpListener) Factory() listenerFactory {
return t.factory return t.factory
} }
func (t *tcpListener) NATType() string {
return "unknown"
}
type tcpListenerFactory struct{} type tcpListenerFactory struct{}
func (f *tcpListenerFactory) New(uri *url.URL, cfg *config.Wrapper, tlsCfg *tls.Config, conns chan internalConn, natService *nat.Service) genericListener { func (f *tcpListenerFactory) New(uri *url.URL, cfg *config.Wrapper, tlsCfg *tls.Config, conns chan internalConn, natService *nat.Service) genericListener {

View File

@ -60,6 +60,10 @@ func (f *folder) Jobs() ([]string, []string) {
func (f *folder) BringToFront(string) {} func (f *folder) BringToFront(string) {}
func (f *folder) BlockStats() map[string]int {
return nil
}
func (f *folder) scanSubdirs(subDirs []string) error { func (f *folder) scanSubdirs(subDirs []string) error {
if err := f.model.internalScanFolderSubdirs(f.ctx, f.folderID, subDirs); err != nil { if err := f.model.internalScanFolderSubdirs(f.ctx, f.folderID, subDirs); err != nil {
// Potentially sets the error twice, once in the scanner just // Potentially sets the error twice, once in the scanner just

View File

@ -52,6 +52,7 @@ type service interface {
Scan(subs []string) error Scan(subs []string) error
Serve() Serve()
Stop() Stop()
BlockStats() map[string]int
getState() (folderState, time.Time, error) getState() (folderState, time.Time, error)
setState(state folderState) setState(state folderState)
@ -405,6 +406,105 @@ func (m *Model) RestartFolder(cfg config.FolderConfiguration) {
m.fmut.Unlock() m.fmut.Unlock()
} }
func (m *Model) UsageReportingStats(version int) map[string]interface{} {
stats := make(map[string]interface{})
if version >= 3 {
// Block stats
m.fmut.Lock()
blockStats := make(map[string]int)
for _, folder := range m.folderRunners {
for k, v := range folder.BlockStats() {
blockStats[k] += v
}
}
m.fmut.Unlock()
stats["blockStats"] = blockStats
// Transport stats
m.pmut.Lock()
transportStats := make(map[string]int)
for _, conn := range m.conn {
transportStats[conn.Transport()]++
}
m.pmut.Unlock()
stats["transportStats"] = transportStats
// Ignore stats
ignoreStats := map[string]int{
"lines": 0,
"inverts": 0,
"folded": 0,
"deletable": 0,
"rooted": 0,
"includes": 0,
"escapedIncludes": 0,
"doubleStars": 0,
"stars": 0,
}
var seenPrefix [3]bool
for folder := range m.cfg.Folders() {
lines, _, err := m.GetIgnores(folder)
if err != nil {
continue
}
ignoreStats["lines"] += len(lines)
for _, line := range lines {
// Allow prefixes to be specified in any order, but only once.
for {
if strings.HasPrefix(line, "!") && !seenPrefix[0] {
seenPrefix[0] = true
line = line[1:]
ignoreStats["inverts"] += 1
} else if strings.HasPrefix(line, "(?i)") && !seenPrefix[1] {
seenPrefix[1] = true
line = line[4:]
ignoreStats["folded"] += 1
} else if strings.HasPrefix(line, "(?d)") && !seenPrefix[2] {
seenPrefix[2] = true
line = line[4:]
ignoreStats["deletable"] += 1
} else {
seenPrefix[0] = false
seenPrefix[1] = false
seenPrefix[2] = false
break
}
}
// Noops, remove
if strings.HasSuffix(line, "**") {
line = line[:len(line)-2]
}
if strings.HasPrefix(line, "**/") {
line = line[3:]
}
if strings.HasPrefix(line, "/") {
ignoreStats["rooted"] += 1
} else if strings.HasPrefix(line, "#include ") {
ignoreStats["includes"] += 1
if strings.Contains(line, "..") {
ignoreStats["escapedIncludes"] += 1
}
}
if strings.Contains(line, "**") {
ignoreStats["doubleStars"] += 1
// Remove not to trip up star checks.
strings.Replace(line, "**", "", -1)
}
if strings.Contains(line, "*") {
ignoreStats["stars"] += 1
}
}
}
stats["ignoreStats"] = ignoreStats
}
return stats
}
type ConnectionInfo struct { type ConnectionInfo struct {
protocol.Statistics protocol.Statistics
Connected bool Connected bool
@ -2449,6 +2549,7 @@ func (m *Model) CommitConfiguration(from, to config.Configuration) bool {
// Some options don't require restart as those components handle it fine // Some options don't require restart as those components handle it fine
// by themselves. // by themselves.
from.Options.URAccepted = to.Options.URAccepted from.Options.URAccepted = to.Options.URAccepted
from.Options.URSeen = to.Options.URSeen
from.Options.URUniqueID = to.Options.URUniqueID from.Options.URUniqueID = to.Options.URUniqueID
from.Options.ListenAddresses = to.Options.ListenAddresses from.Options.ListenAddresses = to.Options.ListenAddresses
from.Options.RelaysEnabled = to.Options.RelaysEnabled from.Options.RelaysEnabled = to.Options.RelaysEnabled

View File

@ -308,6 +308,9 @@ func (f *fakeConnection) RemoteAddr() net.Addr {
func (f *fakeConnection) Type() string { func (f *fakeConnection) Type() string {
return "fake" return "fake"
} }
func (f *fakeConnection) Transport() string {
return "fake"
}
func (f *fakeConnection) DownloadProgress(folder string, updates []protocol.FileDownloadProgressUpdate) { func (f *fakeConnection) DownloadProgress(folder string, updates []protocol.FileDownloadProgressUpdate) {
f.downloadProgressMessages = append(f.downloadProgressMessages, downloadProgressMessage{ f.downloadProgressMessages = append(f.downloadProgressMessages, downloadProgressMessage{

View File

@ -93,6 +93,9 @@ type sendReceiveFolder struct {
errors map[string]string // path -> error string errors map[string]string // path -> error string
errorsMut sync.Mutex errorsMut sync.Mutex
blockStats map[string]int
blockStatsMut sync.Mutex
} }
func newSendReceiveFolder(model *Model, cfg config.FolderConfiguration, ver versioner.Versioner, fs fs.Filesystem) service { func newSendReceiveFolder(model *Model, cfg config.FolderConfiguration, ver versioner.Versioner, fs fs.Filesystem) service {
@ -107,6 +110,9 @@ func newSendReceiveFolder(model *Model, cfg config.FolderConfiguration, ver vers
remoteIndex: make(chan struct{}, 1), // This needs to be 1-buffered so that we queue a notification if we're busy doing a pull when it comes. remoteIndex: make(chan struct{}, 1), // This needs to be 1-buffered so that we queue a notification if we're busy doing a pull when it comes.
errorsMut: sync.NewMutex(), errorsMut: sync.NewMutex(),
blockStats: make(map[string]int),
blockStatsMut: sync.NewMutex(),
} }
f.configureCopiersAndPullers() f.configureCopiersAndPullers()
@ -875,6 +881,11 @@ func (f *sendReceiveFolder) renameFile(source, target protocol.FileInfo) {
} }
if err == nil { if err == nil {
f.blockStatsMut.Lock()
f.blockStats["total"] += len(target.Blocks)
f.blockStats["renamed"] += len(target.Blocks)
f.blockStatsMut.Unlock()
// The file was renamed, so we have handled both the necessary delete // The file was renamed, so we have handled both the necessary delete
// of the source and the creation of the target. Fix-up the metadata, // of the source and the creation of the target. Fix-up the metadata,
// and update the local index of the target file. // and update the local index of the target file.
@ -1443,6 +1454,15 @@ func (f *sendReceiveFolder) finisherRoutine(in <-chan *sharedPullerState) {
if err != nil { if err != nil {
l.Debugln("Puller: final:", err) l.Debugln("Puller: final:", err)
f.newError(state.file.Name, err) f.newError(state.file.Name, err)
} else {
f.blockStatsMut.Lock()
f.blockStats["total"] += state.reused + state.copyTotal + state.pullTotal
f.blockStats["reused"] += state.reused
f.blockStats["pulled"] += state.pullTotal
f.blockStats["copyOrigin"] += state.copyOrigin
f.blockStats["copyOriginShifted"] += state.copyOriginShifted
f.blockStats["copyElsewhere"] += state.copyTotal - state.copyOrigin
f.blockStatsMut.Unlock()
} }
events.Default.Log(events.ItemFinished, map[string]interface{}{ events.Default.Log(events.ItemFinished, map[string]interface{}{
"folder": f.folderID, "folder": f.folderID,
@ -1459,6 +1479,16 @@ func (f *sendReceiveFolder) finisherRoutine(in <-chan *sharedPullerState) {
} }
} }
func (f *sendReceiveFolder) BlockStats() map[string]int {
f.blockStatsMut.Lock()
stats := make(map[string]int)
for k, v := range f.blockStats {
stats[k] = v
}
f.blockStatsMut.Unlock()
return stats
}
// Moves the given filename to the front of the job queue // Moves the given filename to the front of the job queue
func (f *sendReceiveFolder) BringToFront(filename string) { func (f *sendReceiveFolder) BringToFront(filename string) {
f.queue.BringToFront(filename) f.queue.BringToFront(filename)