octoleo/syncthing
1
0
Fork 0
mirror of https://github.com/octoleo/syncthing.git synced 2024-11-09 14:50:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

lib/ur: Normalise contract between syncthing and ursrv (#6770)

Browse Source 
* Fix ui, hide report date

* Undo Goland madness

* UR now web scale

* Fix migration

* Fix marshaling, force tick on start

* Fix tests

* Darwin build

* Split "all" build target, add package name as a tag

* Remove pq and sql dep from syncthing, split build targets

* Empty line

* Revert "Empty line"

This reverts commit f74af2b067dadda8a343714123512bd545a643c3.

* Revert "Remove pq and sql dep from syncthing, split build targets"

This reverts commit 8fc295ad007c5bb7886c557f492dacf51be307ad.

* Revert "Split "all" build target, add package name as a tag"

This reverts commit f4dc88995106d2b06042f30bea781a0feb08e55f.

* Normalise contract types

* Fix build add more logging
This commit is contained in:
Audrius Butkevicius 2020-06-23 09:47:15 +01:00 committed by GitHub
parent 72f954dcab
commit 689cf2a5ee
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 976 additions and 855 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
cmd/uraggregate/main.go
View File

@ -175,13 +175,13 @@ func aggregateVersionSummary(db *sql.DB, since time.Time) (int64, error) {
res, err := db.Exec(`INSERT INTO VersionSummary (
SELECT
DATE_TRUNC('day', Received) AS Day,
SUBSTRING(Version FROM '^v\d.\d+') AS Ver,
SUBSTRING(Report->>'version' FROM '^v\d.\d+') AS Ver,
COUNT(*) AS Count
FROM Reports
FROM ReportsJson
WHERE
DATE_TRUNC('day', Received) > $1
AND DATE_TRUNC('day', Received) < DATE_TRUNC('day', NOW())
AND Version like 'v_.%'
AND Report->>'version' like 'v_.%'
GROUP BY Day, Ver
);
`, since)
@ -195,11 +195,11 @@ func aggregateVersionSummary(db *sql.DB, since time.Time) (int64, error) {
func aggregateUserMovement(db *sql.DB) (int64, error) {
rows, err := db.Query(`SELECT
DATE_TRUNC('day', Received) AS Day,
UniqueID
FROM Reports
Report->>'uniqueID'
FROM ReportsJson
WHERE
DATE_TRUNC('day', Received) < DATE_TRUNC('day', NOW())
AND Version like 'v_.%'
AND Report->>'version' like 'v_.%'
ORDER BY Day
`)
if err != nil {
@ -276,16 +276,16 @@ func aggregatePerformance(db *sql.DB, since time.Time) (int64, error) {
res, err := db.Exec(`INSERT INTO Performance (
SELECT
DATE_TRUNC('day', Received) AS Day,
AVG(TotFiles) As TotFiles,
AVG(TotMiB) As TotMiB,
AVG(SHA256Perf) As SHA256Perf,
AVG(MemorySize) As MemorySize,
AVG(MemoryUsageMiB) As MemoryUsageMiB
FROM Reports
AVG((Report->>'totFiles')::numeric) As TotFiles,
AVG((Report->>'totMiB')::numeric) As TotMiB,
AVG((Report->>'sha256Perf')::numeric) As SHA256Perf,
AVG((Report->>'memorySize')::numeric) As MemorySize,
AVG((Report->>'memoryUsageMiB')::numeric) As MemoryUsageMiB
FROM ReportsJson
WHERE
DATE_TRUNC('day', Received) > $1
AND DATE_TRUNC('day', Received) < DATE_TRUNC('day', NOW())
AND Version like 'v_.%'
AND Report->>'version' like 'v_.%'
GROUP BY Day
);
`, since)
@ -303,22 +303,22 @@ func aggregateBlockStats(db *sql.DB, since time.Time) (int64, error) {
SELECT
DATE_TRUNC('day', Received) AS Day,
COUNT(1) As Reports,
SUM(BlocksTotal) AS Total,
SUM(BlocksRenamed) AS Renamed,
SUM(BlocksReused) AS Reused,
SUM(BlocksPulled) AS Pulled,
SUM(BlocksCopyOrigin) AS CopyOrigin,
SUM(BlocksCopyOriginShifted) AS CopyOriginShifted,
SUM(BlocksCopyElsewhere) AS CopyElsewhere
FROM Reports
SUM((Report->'blockStats'->>'total')::numeric) AS Total,
SUM((Report->'blockStats'->>'renamed')::numeric) AS Renamed,
SUM((Report->'blockStats'->>'reused')::numeric) AS Reused,
SUM((Report->'blockStats'->>'pulled')::numeric) AS Pulled,
SUM((Report->'blockStats'->>'copyOrigin')::numeric) AS CopyOrigin,
SUM((Report->'blockStats'->>'copyOriginShifted')::numeric) AS CopyOriginShifted,
SUM((Report->'blockStats'->>'copyElsewhere')::numeric) AS CopyElsewhere
FROM ReportsJson
WHERE
DATE_TRUNC('day', Received) > $1
AND DATE_TRUNC('day', Received) < DATE_TRUNC('day', NOW())
AND ReportVersion = 3
AND Version like 'v_.%'
AND Version NOT LIKE 'v0.14.40%'
AND Version NOT LIKE 'v0.14.39%'
AND Version NOT LIKE 'v0.14.38%'
AND (Report->>'urVersion')::numeric >= 3
AND Report->>'version' like 'v_.%'
AND Report->>'version' NOT LIKE 'v0.14.40%'
AND Report->>'version' NOT LIKE 'v0.14.39%'
AND Report->>'version' NOT LIKE 'v0.14.38%'
GROUP BY Day
);
`, since)

17
cmd/ursrv/analytics.go
View File

@ -114,6 +114,23 @@ func statsForInts(data []int) [4]float64 {
return res
}
func statsForInt64s(data []int64) [4]float64 {
var res [4]float64
if len(data) == 0 {
return res
}
sort.Slice(data, func(a, b int) bool {
return data[a] < data[b]
})
res[0] = float64(data[int(float64(len(data))*0.05)])
res[1] = float64(data[len(data)/2])
res[2] = float64(data[int(float64(len(data))*0.95)])
res[3] = float64(data[len(data)-1])
return res
}
func statsForFloats(data []float64) [4]float64 {
var res [4]float64
if len(data) == 0 {

613
cmd/ursrv/main.go
View File

@ -10,10 +10,7 @@ import (
"bytes"
"crypto/tls"
"database/sql"
"database/sql/driver"
"encoding/json"
"errors"
"fmt"
"html/template"
"io"
"io/ioutil"
@ -29,8 +26,9 @@ import (
"time"
"unicode"
"github.com/lib/pq"
geoip2 "github.com/oschwald/geoip2-golang"
"github.com/oschwald/geoip2-golang"
"github.com/syncthing/syncthing/lib/ur/contract"
)
var (
@ -103,585 +101,44 @@ func getEnvDefault(key, def string) string {
return def
}
type IntMap map[string]int
func (p IntMap) Value() (driver.Value, error) {
return json.Marshal(p)
}
func (p *IntMap) Scan(src interface{}) error {
source, ok := src.([]byte)
if !ok {
return errors.New("Type assertion .([]byte) failed.")
}
var i map[string]int
err := json.Unmarshal(source, &i)
if err != nil {
return err
}
*p = i
return nil
}
type report struct {
Received time.Time // Only from DB
UniqueID string
Version string
LongVersion string
Platform string
NumFolders int
NumDevices int
TotFiles int
FolderMaxFiles int
TotMiB int
FolderMaxMiB int
MemoryUsageMiB int
SHA256Perf float64
MemorySize int
// v2 fields
URVersion int
NumCPU int
FolderUses struct {
SendOnly int
ReceiveOnly int
IgnorePerms int
IgnoreDelete int
AutoNormalize int
SimpleVersioning int
ExternalVersioning int
StaggeredVersioning int
TrashcanVersioning int
}
DeviceUses struct {
Introducer int
CustomCertName int
CompressAlways int
CompressMetadata int
CompressNever int
DynamicAddr int
StaticAddr int
}
Announce struct {
GlobalEnabled bool
LocalEnabled bool
DefaultServersDNS int
DefaultServersIP int
OtherServers int
}
Relays struct {
Enabled bool
DefaultServers int
OtherServers int
}
UsesRateLimit bool
UpgradeAllowedManual bool
UpgradeAllowedAuto bool
// V2.5 fields (fields that were in v2 but never added to the database
UpgradeAllowedPre bool
RescanIntvs pq.Int64Array
// v3 fields
Uptime int
NATType string
AlwaysLocalNets bool
CacheIgnoredFiles bool
OverwriteRemoteDeviceNames bool
ProgressEmitterEnabled bool
CustomDefaultFolderPath bool
WeakHashSelection string
CustomTrafficClass bool
CustomTempIndexMinBlocks bool
TemporariesDisabled bool
TemporariesCustom bool
LimitBandwidthInLan bool
CustomReleaseURL bool
RestartOnWakeup bool
CustomStunServers bool
FolderUsesV3 struct {
ScanProgressDisabled int
ConflictsDisabled int
ConflictsUnlimited int
ConflictsOther int
DisableSparseFiles int
DisableTempIndexes int
AlwaysWeakHash int
CustomWeakHashThreshold int
FsWatcherEnabled int
PullOrder IntMap
FilesystemType IntMap
FsWatcherDelays pq.Int64Array
}
GUIStats struct {
Enabled int
UseTLS int
UseAuth int
InsecureAdminAccess int
Debugging int
InsecureSkipHostCheck int
InsecureAllowFrameLoading int
ListenLocal int
ListenUnspecified int
Theme IntMap
}
BlockStats struct {
Total int
Renamed int
Reused int
Pulled int
CopyOrigin int
CopyOriginShifted int
CopyElsewhere int
}
TransportStats IntMap
IgnoreStats struct {
Lines int
Inverts int
Folded int
Deletable int
Rooted int
Includes int
EscapedIncludes int
DoubleStars int
Stars int
}
// V3 fields added late in the RC
WeakHashEnabled bool
// Generated
Date string
Address string
}
func (r *report) Validate() error {
if r.UniqueID == "" || r.Version == "" || r.Platform == "" {
return errors.New("missing required field")
}
if len(r.Date) != 8 {
return errors.New("date not initialized")
}
// Some fields may not be null.
if r.RescanIntvs == nil {
r.RescanIntvs = []int64{}
}
if r.FolderUsesV3.FsWatcherDelays == nil {
r.FolderUsesV3.FsWatcherDelays = []int64{}
}
return nil
}
func (r *report) FieldPointers() []interface{} {
// All the fields of the report, in the same order as the database fields.
return []interface{}{
&r.Received, &r.UniqueID, &r.Version, &r.LongVersion, &r.Platform,
&r.NumFolders, &r.NumDevices, &r.TotFiles, &r.FolderMaxFiles,
&r.TotMiB, &r.FolderMaxMiB, &r.MemoryUsageMiB, &r.SHA256Perf,
&r.MemorySize, &r.Date,
// V2
&r.URVersion, &r.NumCPU, &r.FolderUses.SendOnly, &r.FolderUses.IgnorePerms,
&r.FolderUses.IgnoreDelete, &r.FolderUses.AutoNormalize, &r.DeviceUses.Introducer,
&r.DeviceUses.CustomCertName, &r.DeviceUses.CompressAlways,
&r.DeviceUses.CompressMetadata, &r.DeviceUses.CompressNever,
&r.DeviceUses.DynamicAddr, &r.DeviceUses.StaticAddr,
&r.Announce.GlobalEnabled, &r.Announce.LocalEnabled,
&r.Announce.DefaultServersDNS, &r.Announce.DefaultServersIP,
&r.Announce.OtherServers, &r.Relays.Enabled, &r.Relays.DefaultServers,
&r.Relays.OtherServers, &r.UsesRateLimit, &r.UpgradeAllowedManual,
&r.UpgradeAllowedAuto, &r.FolderUses.SimpleVersioning,
&r.FolderUses.ExternalVersioning, &r.FolderUses.StaggeredVersioning,
&r.FolderUses.TrashcanVersioning,
// V2.5
&r.UpgradeAllowedPre, &r.RescanIntvs,
// V3
&r.Uptime, &r.NATType, &r.AlwaysLocalNets, &r.CacheIgnoredFiles,
&r.OverwriteRemoteDeviceNames, &r.ProgressEmitterEnabled, &r.CustomDefaultFolderPath,
&r.WeakHashSelection, &r.CustomTrafficClass, &r.CustomTempIndexMinBlocks,
&r.TemporariesDisabled, &r.TemporariesCustom, &r.LimitBandwidthInLan,
&r.CustomReleaseURL, &r.RestartOnWakeup, &r.CustomStunServers,
&r.FolderUsesV3.ScanProgressDisabled, &r.FolderUsesV3.ConflictsDisabled,
&r.FolderUsesV3.ConflictsUnlimited, &r.FolderUsesV3.ConflictsOther,
&r.FolderUsesV3.DisableSparseFiles, &r.FolderUsesV3.DisableTempIndexes,
&r.FolderUsesV3.AlwaysWeakHash, &r.FolderUsesV3.CustomWeakHashThreshold,
&r.FolderUsesV3.FsWatcherEnabled,
&r.FolderUsesV3.PullOrder, &r.FolderUsesV3.FilesystemType,
&r.FolderUsesV3.FsWatcherDelays,
&r.GUIStats.Enabled, &r.GUIStats.UseTLS, &r.GUIStats.UseAuth,
&r.GUIStats.InsecureAdminAccess,
&r.GUIStats.Debugging, &r.GUIStats.InsecureSkipHostCheck,
&r.GUIStats.InsecureAllowFrameLoading, &r.GUIStats.ListenLocal,
&r.GUIStats.ListenUnspecified, &r.GUIStats.Theme,
&r.BlockStats.Total, &r.BlockStats.Renamed,
&r.BlockStats.Reused, &r.BlockStats.Pulled, &r.BlockStats.CopyOrigin,
&r.BlockStats.CopyOriginShifted, &r.BlockStats.CopyElsewhere,
&r.TransportStats,
&r.IgnoreStats.Lines, &r.IgnoreStats.Inverts, &r.IgnoreStats.Folded,
&r.IgnoreStats.Deletable, &r.IgnoreStats.Rooted, &r.IgnoreStats.Includes,
&r.IgnoreStats.EscapedIncludes, &r.IgnoreStats.DoubleStars, &r.IgnoreStats.Stars,
// V3 added late in the RC
&r.WeakHashEnabled,
&r.Address,
// Receive only folders
&r.FolderUses.ReceiveOnly,
}
}
func (r *report) FieldNames() []string {
// The database fields that back this struct in PostgreSQL
return []string{
// V1
"Received",
"UniqueID",
"Version",
"LongVersion",
"Platform",
"NumFolders",
"NumDevices",
"TotFiles",
"FolderMaxFiles",
"TotMiB",
"FolderMaxMiB",
"MemoryUsageMiB",
"SHA256Perf",
"MemorySize",
"Date",
// V2
"ReportVersion",
"NumCPU",
"FolderRO",
"FolderIgnorePerms",
"FolderIgnoreDelete",
"FolderAutoNormalize",
"DeviceIntroducer",
"DeviceCustomCertName",
"DeviceCompressAlways",
"DeviceCompressMetadata",
"DeviceCompressNever",
"DeviceDynamicAddr",
"DeviceStaticAddr",
"AnnounceGlobalEnabled",
"AnnounceLocalEnabled",
"AnnounceDefaultServersDNS",
"AnnounceDefaultServersIP",
"AnnounceOtherServers",
"RelayEnabled",
"RelayDefaultServers",
"RelayOtherServers",
"RateLimitEnabled",
"UpgradeAllowedManual",
"UpgradeAllowedAuto",
// v0.12.19+
"FolderSimpleVersioning",
"FolderExternalVersioning",
"FolderStaggeredVersioning",
"FolderTrashcanVersioning",
// V2.5
"UpgradeAllowedPre",
"RescanIntvs",
// V3
"Uptime",
"NATType",
"AlwaysLocalNets",
"CacheIgnoredFiles",
"OverwriteRemoteDeviceNames",
"ProgressEmitterEnabled",
"CustomDefaultFolderPath",
"WeakHashSelection",
"CustomTrafficClass",
"CustomTempIndexMinBlocks",
"TemporariesDisabled",
"TemporariesCustom",
"LimitBandwidthInLan",
"CustomReleaseURL",
"RestartOnWakeup",
"CustomStunServers",
"FolderScanProgressDisabled",
"FolderConflictsDisabled",
"FolderConflictsUnlimited",
"FolderConflictsOther",
"FolderDisableSparseFiles",
"FolderDisableTempIndexes",
"FolderAlwaysWeakHash",
"FolderCustomWeakHashThreshold",
"FolderFsWatcherEnabled",
"FolderPullOrder",
"FolderFilesystemType",
"FolderFsWatcherDelays",
"GUIEnabled",
"GUIUseTLS",
"GUIUseAuth",
"GUIInsecureAdminAccess",
"GUIDebugging",
"GUIInsecureSkipHostCheck",
"GUIInsecureAllowFrameLoading",
"GUIListenLocal",
"GUIListenUnspecified",
"GUITheme",
"BlocksTotal",
"BlocksRenamed",
"BlocksReused",
"BlocksPulled",
"BlocksCopyOrigin",
"BlocksCopyOriginShifted",
"BlocksCopyElsewhere",
"Transport",
"IgnoreLines",
"IgnoreInverts",
"IgnoreFolded",
"IgnoreDeletable",
"IgnoreRooted",
"IgnoreIncludes",
"IgnoreEscapedIncludes",
"IgnoreDoubleStars",
"IgnoreStars",
// V3 added late in the RC
"WeakHashEnabled",
"Address",
// Receive only folders
"FolderRecvOnly",
}
}
func setupDB(db *sql.DB) error {
_, err := db.Exec(`CREATE TABLE IF NOT EXISTS Reports (
_, err := db.Exec(`CREATE TABLE IF NOT EXISTS ReportsJson (
Received TIMESTAMP NOT NULL,
UniqueID VARCHAR(32) NOT NULL,
Version VARCHAR(32) NOT NULL,
LongVersion VARCHAR(256) NOT NULL,
Platform VARCHAR(32) NOT NULL,
NumFolders INTEGER NOT NULL,
NumDevices INTEGER NOT NULL,
TotFiles INTEGER NOT NULL,
FolderMaxFiles INTEGER NOT NULL,
TotMiB INTEGER NOT NULL,
FolderMaxMiB INTEGER NOT NULL,
MemoryUsageMiB INTEGER NOT NULL,
SHA256Perf DOUBLE PRECISION NOT NULL,
MemorySize INTEGER NOT NULL,
Date VARCHAR(8) NOT NULL
Report JSONB NOT NULL
)`)
if err != nil {
return err
}
var t string
row := db.QueryRow(`SELECT 'UniqueIDIndex'::regclass`)
if err := row.Scan(&t); err != nil {
if _, err = db.Exec(`CREATE UNIQUE INDEX UniqueIDIndex ON Reports (Date, UniqueID)`); err != nil {
if err := db.QueryRow(`SELECT 'UniqueIDJsonIndex'::regclass`).Scan(&t); err != nil {
if _, err = db.Exec(`CREATE UNIQUE INDEX UniqueIDJsonIndex ON ReportsJson ((Report->>'date'), (Report->>'uniqueID'))`); err != nil {
return err
}
}
row = db.QueryRow(`SELECT 'ReceivedIndex'::regclass`)
if err := row.Scan(&t); err != nil {
if _, err = db.Exec(`CREATE INDEX ReceivedIndex ON Reports (Received)`); err != nil {
if err := db.QueryRow(`SELECT 'ReceivedJsonIndex'::regclass`).Scan(&t); err != nil {
if _, err = db.Exec(`CREATE INDEX ReceivedJsonIndex ON ReportsJson (Received)`); err != nil {
return err
}
}
// V2
row = db.QueryRow(`SELECT attname FROM pg_attribute WHERE attrelid = (SELECT oid FROM pg_class WHERE relname = 'reports') AND attname = 'reportversion'`)
if err := row.Scan(&t); err != nil {
// The ReportVersion column doesn't exist; add the new columns.
_, err = db.Exec(`ALTER TABLE Reports
ADD COLUMN ReportVersion INTEGER NOT NULL DEFAULT 0,
ADD COLUMN NumCPU INTEGER NOT NULL DEFAULT 0,
ADD COLUMN FolderRO INTEGER NOT NULL DEFAULT 0,
ADD COLUMN FolderIgnorePerms INTEGER NOT NULL DEFAULT 0,
ADD COLUMN FolderIgnoreDelete INTEGER NOT NULL DEFAULT 0,
ADD COLUMN FolderAutoNormalize INTEGER NOT NULL DEFAULT 0,
ADD COLUMN DeviceIntroducer INTEGER NOT NULL DEFAULT 0,
ADD COLUMN DeviceCustomCertName INTEGER NOT NULL DEFAULT 0,
ADD COLUMN DeviceCompressAlways INTEGER NOT NULL DEFAULT 0,
ADD COLUMN DeviceCompressMetadata INTEGER NOT NULL DEFAULT 0,
ADD COLUMN DeviceCompressNever INTEGER NOT NULL DEFAULT 0,
ADD COLUMN DeviceDynamicAddr INTEGER NOT NULL DEFAULT 0,
ADD COLUMN DeviceStaticAddr INTEGER NOT NULL DEFAULT 0,
ADD COLUMN AnnounceGlobalEnabled BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN AnnounceLocalEnabled BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN AnnounceDefaultServersDNS INTEGER NOT NULL DEFAULT 0,
ADD COLUMN AnnounceDefaultServersIP INTEGER NOT NULL DEFAULT 0,
ADD COLUMN AnnounceOtherServers INTEGER NOT NULL DEFAULT 0,
ADD COLUMN RelayEnabled BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN RelayDefaultServers INTEGER NOT NULL DEFAULT 0,
ADD COLUMN RelayOtherServers INTEGER NOT NULL DEFAULT 0,
ADD COLUMN RateLimitEnabled BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN UpgradeAllowedManual BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN UpgradeAllowedAuto BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN FolderSimpleVersioning INTEGER NOT NULL DEFAULT 0,
ADD COLUMN FolderExternalVersioning INTEGER NOT NULL DEFAULT 0,
ADD COLUMN FolderStaggeredVersioning INTEGER NOT NULL DEFAULT 0,
ADD COLUMN FolderTrashcanVersioning INTEGER NOT NULL DEFAULT 0
`)
if err != nil {
if err := db.QueryRow(`SELECT 'ReportVersionJsonIndex'::regclass`).Scan(&t); err != nil {
if _, err = db.Exec(`CREATE INDEX ReportVersionJsonIndex ON ReportsJson (cast((Report->>'urVersion') as numeric))`); err != nil {
return err
}
}
row = db.QueryRow(`SELECT 'ReportVersionIndex'::regclass`)
if err := row.Scan(&t); err != nil {
if _, err = db.Exec(`CREATE INDEX ReportVersionIndex ON Reports (ReportVersion)`); err != nil {
return err
}
}
// V2.5
row = db.QueryRow(`SELECT attname FROM pg_attribute WHERE attrelid = (SELECT oid FROM pg_class WHERE relname = 'reports') AND attname = 'upgradeallowedpre'`)
if err := row.Scan(&t); err != nil {
// The ReportVersion column doesn't exist; add the new columns.
_, err = db.Exec(`ALTER TABLE Reports
ADD COLUMN UpgradeAllowedPre BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN RescanIntvs INT[] NOT NULL DEFAULT '{}'
`)
if err != nil {
return err
}
}
// V3
row = db.QueryRow(`SELECT attname FROM pg_attribute WHERE attrelid = (SELECT oid FROM pg_class WHERE relname = 'reports') AND attname = 'uptime'`)
if err := row.Scan(&t); err != nil {
// The Uptime column doesn't exist; add the new columns.
_, err = db.Exec(`ALTER TABLE Reports
ADD COLUMN Uptime INTEGER NOT NULL DEFAULT 0,
ADD COLUMN NATType VARCHAR(32) NOT NULL DEFAULT '',
ADD COLUMN AlwaysLocalNets BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN CacheIgnoredFiles BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN OverwriteRemoteDeviceNames BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN ProgressEmitterEnabled BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN CustomDefaultFolderPath BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN WeakHashSelection VARCHAR(32) NOT NULL DEFAULT '',
ADD COLUMN CustomTrafficClass BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN CustomTempIndexMinBlocks BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN TemporariesDisabled BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN TemporariesCustom BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN LimitBandwidthInLan BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN CustomReleaseURL BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN RestartOnWakeup BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN CustomStunServers BOOLEAN NOT NULL DEFAULT FALSE,
ADD COLUMN FolderScanProgressDisabled INTEGER NOT NULL DEFAULT 0,
ADD COLUMN FolderConflictsDisabled INTEGER NOT NULL DEFAULT 0,
ADD COLUMN FolderConflictsUnlimited INTEGER NOT NULL DEFAULT 0,
ADD COLUMN FolderConflictsOther INTEGER NOT NULL DEFAULT 0,
ADD COLUMN FolderDisableSparseFiles INTEGER NOT NULL DEFAULT 0,
ADD COLUMN FolderDisableTempIndexes INTEGER NOT NULL DEFAULT 0,
ADD COLUMN FolderAlwaysWeakHash INTEGER NOT NULL DEFAULT 0,
ADD COLUMN FolderCustomWeakHashThreshold INTEGER NOT NULL DEFAULT 0,
ADD COLUMN FolderFsWatcherEnabled INTEGER NOT NULL DEFAULT 0,
ADD COLUMN FolderPullOrder JSONB NOT NULL DEFAULT '{}',
ADD COLUMN FolderFilesystemType JSONB NOT NULL DEFAULT '{}',
ADD COLUMN FolderFsWatcherDelays INT[] NOT NULL DEFAULT '{}',
ADD COLUMN GUIEnabled INTEGER NOT NULL DEFAULT 0,
ADD COLUMN GUIUseTLS INTEGER NOT NULL DEFAULT 0,
ADD COLUMN GUIUseAuth INTEGER NOT NULL DEFAULT 0,
ADD COLUMN GUIInsecureAdminAccess INTEGER NOT NULL DEFAULT 0,
ADD COLUMN GUIDebugging INTEGER NOT NULL DEFAULT 0,
ADD COLUMN GUIInsecureSkipHostCheck INTEGER NOT NULL DEFAULT 0,
ADD COLUMN GUIInsecureAllowFrameLoading INTEGER NOT NULL DEFAULT 0,
ADD COLUMN GUIListenLocal INTEGER NOT NULL DEFAULT 0,
ADD COLUMN GUIListenUnspecified INTEGER NOT NULL DEFAULT 0,
ADD COLUMN GUITheme JSONB NOT NULL DEFAULT '{}',
ADD COLUMN BlocksTotal INTEGER NOT NULL DEFAULT 0,
ADD COLUMN BlocksRenamed INTEGER NOT NULL DEFAULT 0,
ADD COLUMN BlocksReused INTEGER NOT NULL DEFAULT 0,
ADD COLUMN BlocksPulled INTEGER NOT NULL DEFAULT 0,
ADD COLUMN BlocksCopyOrigin INTEGER NOT NULL DEFAULT 0,
ADD COLUMN BlocksCopyOriginShifted INTEGER NOT NULL DEFAULT 0,
ADD COLUMN BlocksCopyElsewhere INTEGER NOT NULL DEFAULT 0,
ADD COLUMN Transport JSONB NOT NULL DEFAULT '{}',
ADD COLUMN IgnoreLines INTEGER NOT NULL DEFAULT 0,
ADD COLUMN IgnoreInverts INTEGER NOT NULL DEFAULT 0,
ADD COLUMN IgnoreFolded INTEGER NOT NULL DEFAULT 0,
ADD COLUMN IgnoreDeletable INTEGER NOT NULL DEFAULT 0,
ADD COLUMN IgnoreRooted INTEGER NOT NULL DEFAULT 0,
ADD COLUMN IgnoreIncludes INTEGER NOT NULL DEFAULT 0,
ADD COLUMN IgnoreEscapedIncludes INTEGER NOT NULL DEFAULT 0,
ADD COLUMN IgnoreDoubleStars INTEGER NOT NULL DEFAULT 0,
ADD COLUMN IgnoreStars INTEGER NOT NULL DEFAULT 0
`)
if err != nil {
return err
}
}
// V3 added late in the RC
row = db.QueryRow(`SELECT attname FROM pg_attribute WHERE attrelid = (SELECT oid FROM pg_class WHERE relname = 'reports') AND attname = 'weakhashenabled'`)
if err := row.Scan(&t); err != nil {
// The WeakHashEnabled column doesn't exist; add the new columns.
_, err = db.Exec(`ALTER TABLE Reports
ADD COLUMN WeakHashEnabled BOOLEAN NOT NULL DEFAULT FALSE
ADD COLUMN Address VARCHAR(45) NOT NULL DEFAULT ''
`)
if err != nil {
return err
}
}
// Receive only added ad-hoc
row = db.QueryRow(`SELECT attname FROM pg_attribute WHERE attrelid = (SELECT oid FROM pg_class WHERE relname = 'reports') AND attname = 'folderrecvonly'`)
if err := row.Scan(&t); err != nil {
// The RecvOnly column doesn't exist; add it.
_, err = db.Exec(`ALTER TABLE Reports
ADD COLUMN FolderRecvOnly INTEGER NOT NULL DEFAULT 0
`)
if err != nil {
return err
}
// Migrate from old schema to new schema if the table exists.
if err := migrate(db); err != nil {
return err
}
return nil
}
func insertReport(db *sql.DB, r report) error {
r.Received = time.Now().UTC()
fields := r.FieldPointers()
params := make([]string, len(fields))
for i := range params {
params[i] = fmt.Sprintf("$%d", i+1)
}
query := "INSERT INTO Reports (" + strings.Join(r.FieldNames(), ", ") + ") VALUES (" + strings.Join(params, ", ") + ")"
_, err := db.Exec(query, fields...)
func insertReport(db *sql.DB, r contract.Report) error {
_, err := db.Exec("INSERT INTO ReportsJson (Report, Received) VALUES ($1, $2)", r, time.Now().UTC())
return err
}
@ -689,9 +146,9 @@ func insertReport(db *sql.DB, r report) error {
type withDBFunc func(*sql.DB, http.ResponseWriter, *http.Request)
func withDB(db *sql.DB, f withDBFunc) http.HandlerFunc {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
return func(w http.ResponseWriter, r *http.Request) {
f(db, w, r)
})
}
}
func main() {
@ -778,7 +235,7 @@ const maxCacheTime = 15 * time.Minute
func cacheRefresher(db *sql.DB) {
ticker := time.NewTicker(maxCacheTime - time.Minute)
defer ticker.Stop()
for range ticker.C {
for ; true; <-ticker.C {
cacheMut.Lock()
if err := refreshCacheLocked(db); err != nil {
log.Println(err)
@ -861,7 +318,7 @@ func newDataHandler(db *sql.DB, w http.ResponseWriter, r *http.Request) {
addr = ""
}
var rep report
var rep contract.Report
rep.Date = time.Now().UTC().Format("20060102")
rep.Address = addr
@ -1069,11 +526,11 @@ func getReport(db *sql.DB) map[string]interface{} {
var numDevices []int
var totFiles []int
var maxFiles []int
var totMiB []int
var maxMiB []int
var memoryUsage []int
var totMiB []int64
var maxMiB []int64
var memoryUsage []int64
var sha256Perf []float64
var memorySize []int
var memorySize []int64
var uptime []int
var compilers []string
var builders []string
@ -1112,9 +569,9 @@ func getReport(db *sql.DB) map[string]interface{} {
var numCPU []int
var rep report
var rep contract.Report
rows, err := db.Query(`SELECT ` + strings.Join(rep.FieldNames(), ",") + ` FROM Reports WHERE Received > now() - '1 day'::INTERVAL`)
rows, err := db.Query(`SELECT Received, Report FROM ReportsJson WHERE Received > now() - '1 day'::INTERVAL`)
if err != nil {
log.Println("sql:", err)
return nil
@ -1122,7 +579,7 @@ func getReport(db *sql.DB) map[string]interface{} {
defer rows.Close()
for rows.Next() {
err := rows.Scan(rep.FieldPointers()...)
err := rows.Scan(&rep.Received, &rep)
if err != nil {
log.Println("sql:", err)
@ -1173,19 +630,19 @@ func getReport(db *sql.DB) map[string]interface{} {
maxFiles = append(maxFiles, rep.FolderMaxFiles)
}
if rep.TotMiB > 0 {
totMiB = append(totMiB, rep.TotMiB*(1<<20))
totMiB = append(totMiB, int64(rep.TotMiB)*(1<<20))
}
if rep.FolderMaxMiB > 0 {
maxMiB = append(maxMiB, rep.FolderMaxMiB*(1<<20))
maxMiB = append(maxMiB, int64(rep.FolderMaxMiB)*(1<<20))
}
if rep.MemoryUsageMiB > 0 {
memoryUsage = append(memoryUsage, rep.MemoryUsageMiB*(1<<20))
memoryUsage = append(memoryUsage, int64(rep.MemoryUsageMiB)*(1<<20))
}
if rep.SHA256Perf > 0 {
sha256Perf = append(sha256Perf, rep.SHA256Perf*(1<<20))
}
if rep.MemorySize > 0 {
memorySize = append(memorySize, rep.MemorySize*(1<<20))
memorySize = append(memorySize, int64(rep.MemorySize)*(1<<20))
}
if rep.Uptime > 0 {
uptime = append(uptime, rep.Uptime)
@ -1336,14 +793,14 @@ func getReport(db *sql.DB) map[string]interface{} {
})
categories = append(categories, category{
Values: statsForInts(totMiB),
Values: statsForInt64s(totMiB),
Descr: "Data Managed per Device",
Unit: "B",
Type: NumberBinary,
})
categories = append(categories, category{
Values: statsForInts(maxMiB),
Values: statsForInt64s(maxMiB),
Descr: "Data in Largest Folder",
Unit: "B",
Type: NumberBinary,
@ -1360,14 +817,14 @@ func getReport(db *sql.DB) map[string]interface{} {
})
categories = append(categories, category{
Values: statsForInts(memoryUsage),
Values: statsForInt64s(memoryUsage),
Descr: "Memory Usage",
Unit: "B",
Type: NumberBinary,
})
categories = append(categories, category{
Values: statsForInts(memorySize),
Values: statsForInt64s(memorySize),
Descr: "System Memory",
Unit: "B",
Type: NumberBinary,

143
cmd/ursrv/migration.go Normal file
View File

@ -0,0 +1,143 @@
// Copyright (C) 2020 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package main
import (
"database/sql"
"database/sql/driver"
"encoding/json"
"errors"
"log"
"strings"
"github.com/lib/pq"
"github.com/syncthing/syncthing/lib/ur/contract"
)
func migrate(db *sql.DB) error {
var count uint64
log.Println("Checking old table row count, this might take a while...")
if err := db.QueryRow(`SELECT COUNT(1) FROM Reports`).Scan(&count); err != nil || count == 0 {
// err != nil most likely means table does not exist.
return nil
}
log.Printf("Found %d records, will perform migration.", count)
tx, err := db.Begin()
if err != nil {
log.Println("sql:", err)
return err
}
defer tx.Rollback()
// These must be lower case, because we don't quote them when creating, so postgres creates them lower case.
// Yet pg.CopyIn quotes them, which makes them case sensitive.
stmt, err := tx.Prepare(pq.CopyIn("reportsjson", "received", "report"))
if err != nil {
log.Println("sql:", err)
return err
}
// Custom types used in the old struct.
var rep contract.Report
var rescanIntvs pq.Int64Array
var fsWatcherDelay pq.Int64Array
pullOrder := make(IntMap)
fileSystemType := make(IntMap)
themes := make(IntMap)
transportStats := make(IntMap)
rows, err := db.Query(`SELECT ` + strings.Join(rep.FieldNames(), ", ") + `, FolderFsWatcherDelays, RescanIntvs, FolderPullOrder, FolderFilesystemType, GUITheme, Transport FROM Reports`)
if err != nil {
log.Println("sql:", err)
return err
}
defer rows.Close()
var done uint64
pct := count / 100
for rows.Next() {
err := rows.Scan(append(rep.FieldPointers(), &fsWatcherDelay, &rescanIntvs, &pullOrder, &fileSystemType, &themes, &transportStats)...)
if err != nil {
log.Println("sql scan:", err)
return err
}
// Patch up parts that used to use custom types
rep.RescanIntvs = make([]int, len(rescanIntvs))
for i := range rescanIntvs {
rep.RescanIntvs[i] = int(rescanIntvs[i])
}
rep.FolderUsesV3.FsWatcherDelays = make([]int, len(fsWatcherDelay))
for i := range fsWatcherDelay {
rep.FolderUsesV3.FsWatcherDelays[i] = int(fsWatcherDelay[i])
}
rep.FolderUsesV3.PullOrder = pullOrder
rep.FolderUsesV3.FilesystemType = fileSystemType
rep.GUIStats.Theme = themes
rep.TransportStats = transportStats
_, err = stmt.Exec(rep.Received, rep)
if err != nil {
log.Println("sql insert:", err)
return err
}
done++
if done%pct == 0 {
log.Printf("Migration progress %d/%d (%d%%)", done, count, (100*done)/count)
}
}
// Tell the driver bulk copy is finished
_, err = stmt.Exec()
if err != nil {
log.Println("sql stmt exec:", err)
return err
}
err = stmt.Close()
if err != nil {
log.Println("sql stmt close:", err)
return err
}
_, err = tx.Exec("DROP TABLE Reports")
if err != nil {
log.Println("sql drop:", err)
return err
}
err = tx.Commit()
if err != nil {
log.Println("sql commit:", err)
return err
}
return nil
}
type IntMap map[string]int
func (p IntMap) Value() (driver.Value, error) {
return json.Marshal(p)
}
func (p *IntMap) Scan(src interface{}) error {
source, ok := src.([]byte)
if !ok {
return errors.New("Type assertion .([]byte) failed.")
}
var i map[string]int
err := json.Unmarshal(source, &i)
if err != nil {
return err
}
*p = i
return nil
}

10
gui/default/syncthing/core/syncthingController.js
View File

@ -32,8 +32,6 @@ angular.module('syncthing.core')
$scope.protocolChanged = false;
$scope.reportData = {};
$scope.reportDataPreview = '';
$scope.reportDataPreviewVersion = '';
$scope.reportDataPreviewDiff = false;
$scope.reportPreview = false;
$scope.folders = {};
$scope.seenError = '';
@ -2322,13 +2320,13 @@ angular.module('syncthing.core')
$scope.reportPreview = true;
};
$scope.refreshReportDataPreview = function () {
$scope.refreshReportDataPreview = function (ver, diff) {
$scope.reportDataPreview = '';
if (!$scope.reportDataPreviewVersion) {
if (!ver) {
return;
}
var version = parseInt($scope.reportDataPreviewVersion);
if ($scope.reportDataPreviewDiff && version > 2) {
var version = parseInt(ver);
if (diff && version > 2) {
$q.all([
$http.get(urlbase + '/svc/report?version=' + version),
$http.get(urlbase + '/svc/report?version=' + (version - 1)),

6
gui/default/syncthing/usagereport/usageReportPreviewModalView.html
View File

@ -6,13 +6,13 @@
<p translate>The aggregated statistics are publicly available at the URL below.</p>
<p><a href="https://data.syncthing.net/" target="_blank">https://data.syncthing.net/</a></p>
<label translate>Version</label>
<select id="urPreviewVersion" class="form-control" ng-model="$parent.$parent.reportDataPreviewVersion" ng-change="refreshReportDataPreview()">
<select id="urPreviewVersion" class="form-control" ng-model="reportDataPreviewVersion" ng-change="refreshReportDataPreview(reportDataPreviewVersion, reportDataPreviewDiff)">
<option selected value translate>Select a version</option>
<option ng-repeat="n in urVersions()" value="{{n}}">{{'Version' | translate}} {{n}}</option>
</select>
<div class="checkbox" ng-if="$parent.$parent.reportDataPreviewVersion > 2">
<div class="checkbox" ng-if="reportDataPreviewVersion > 2">
<label>
<input type="checkbox" ng-model="$parent.$parent.$parent.reportDataPreviewDiff" ng-change="refreshReportDataPreview()" />
<input type="checkbox" ng-model="reportDataPreviewDiff" ng-change="refreshReportDataPreview(reportDataPreviewVersion, reportDataPreviewDiff)" />
<span translate>Show diff with previous version</span>
</label>
</div>

19
lib/api/api.go
View File

@ -1063,10 +1063,15 @@ func (s *service) getSupportBundle(w http.ResponseWriter, r *http.Request) {
}
// Report Data as a JSON
if usageReportingData, err := json.MarshalIndent(s.urService.ReportData(context.TODO()), "", " "); err != nil {
l.Warnln("Support bundle: failed to create versionPlatform.json:", err)
if r, err := s.urService.ReportData(context.TODO()); err != nil {
l.Warnln("Support bundle: failed to create usage-reporting.json.txt:", err)
} else {
files = append(files, fileEntry{name: "usage-reporting.json.txt", data: usageReportingData})
if usageReportingData, err := json.MarshalIndent(r, "", " "); err != nil {
l.Warnln("Support bundle: failed to serialize usage-reporting.json.txt", err)
} else {
files = append(files, fileEntry{name: "usage-reporting.json.txt", data: usageReportingData})
}
}
// Heap and CPU Proofs as a pprof extension
@ -1148,7 +1153,13 @@ func (s *service) getReport(w http.ResponseWriter, r *http.Request) {
if val, _ := strconv.Atoi(r.URL.Query().Get("version")); val > 0 {
version = val
}
sendJSON(w, s.urService.ReportDataPreview(context.TODO(), version))
if r, err := s.urService.ReportDataPreview(context.TODO(), version); err != nil {
http.Error(w, err.Error(), 500)
return
} else {
sendJSON(w, r)
}
}
func (s *service) getRandomString(w http.ResponseWriter, r *http.Request) {

4
lib/api/mocked_model_test.go
View File

@ -15,6 +15,7 @@ import (
"github.com/syncthing/syncthing/lib/model"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/stats"
"github.com/syncthing/syncthing/lib/ur/contract"
"github.com/syncthing/syncthing/lib/versioner"
)
@ -112,8 +113,7 @@ func (m *mockedModel) State(folder string) (string, time.Time, error) {
return "", time.Time{}, nil
}
func (m *mockedModel) UsageReportingStats(version int, preview bool) map[string]interface{} {
return nil
func (m *mockedModel) UsageReportingStats(r *contract.Report, version int, preview bool) {
}
func (m *mockedModel) FolderErrors(folder string) ([]model.FileError, error) {

63
lib/model/model.go
View File

@ -34,6 +34,7 @@ import (
"github.com/syncthing/syncthing/lib/scanner"
"github.com/syncthing/syncthing/lib/stats"
"github.com/syncthing/syncthing/lib/sync"
"github.com/syncthing/syncthing/lib/ur/contract"
"github.com/syncthing/syncthing/lib/util"
"github.com/syncthing/syncthing/lib/versioner"
)
@ -101,7 +102,7 @@ type Model interface {
ConnectionStats() map[string]interface{}
DeviceStatistics() (map[string]stats.DeviceStatistics, error)
FolderStatistics() (map[string]stats.FolderStatistics, error)
UsageReportingStats(version int, preview bool) map[string]interface{}
UsageReportingStats(report *contract.Report, version int, preview bool)
StartDeadlockDetector(timeout time.Duration)
GlobalDirectoryTree(folder, prefix string, levels int, dirsonly bool) map[string]interface{}
@ -443,7 +444,7 @@ func (m *model) stopFolder(cfg config.FolderConfiguration, err error) {
// Need to hold lock on m.fmut when calling this.
func (m *model) cleanupFolderLocked(cfg config.FolderConfiguration) {
// Clean up our config maps
// clear up our config maps
delete(m.folderCfgs, cfg.ID)
delete(m.folderFiles, cfg.ID)
delete(m.folderIgnores, cfg.ID)
@ -520,49 +521,49 @@ func (m *model) newFolder(cfg config.FolderConfiguration) {
m.addAndStartFolderLocked(cfg, fset)
}
func (m *model) UsageReportingStats(version int, preview bool) map[string]interface{} {
stats := make(map[string]interface{})
func (m *model) UsageReportingStats(report *contract.Report, version int, preview bool) {
if version >= 3 {
// Block stats
blockStatsMut.Lock()
copyBlockStats := make(map[string]int)
for k, v := range blockStats {
copyBlockStats[k] = v
switch k {
case "total":
report.BlockStats.Total = v
case "renamed":
report.BlockStats.Renamed = v
case "reused":
report.BlockStats.Reused = v
case "pulled":
report.BlockStats.Pulled = v
case "copyOrigin":
report.BlockStats.CopyOrigin = v
case "copyOriginShifted":
report.BlockStats.CopyOriginShifted = v
case "copyElsewhere":
report.BlockStats.CopyElsewhere = v
}
// Reset counts, as these are incremental
if !preview {
blockStats[k] = 0
}
}
blockStatsMut.Unlock()
stats["blockStats"] = copyBlockStats
// Transport stats
m.pmut.RLock()
transportStats := make(map[string]int)
for _, conn := range m.conn {
transportStats[conn.Transport()]++
report.TransportStats[conn.Transport()]++
}
m.pmut.RUnlock()
stats["transportStats"] = transportStats
// Ignore stats
ignoreStats := map[string]int{
"lines": 0,
"inverts": 0,
"folded": 0,
"deletable": 0,
"rooted": 0,
"includes": 0,
"escapedIncludes": 0,
"doubleStars": 0,
"stars": 0,
}
var seenPrefix [3]bool
for folder := range m.cfg.Folders() {
lines, _, err := m.GetIgnores(folder)
if err != nil {
continue
}
ignoreStats["lines"] += len(lines)
report.IgnoreStats.Lines += len(lines)
for _, line := range lines {
// Allow prefixes to be specified in any order, but only once.
@ -570,15 +571,15 @@ func (m *model) UsageReportingStats(version int, preview bool) map[string]interf
if strings.HasPrefix(line, "!") && !seenPrefix[0] {
seenPrefix[0] = true
line = line[1:]
ignoreStats["inverts"] += 1
report.IgnoreStats.Inverts++
} else if strings.HasPrefix(line, "(?i)") && !seenPrefix[1] {
seenPrefix[1] = true
line = line[4:]
ignoreStats["folded"] += 1
report.IgnoreStats.Folded++
} else if strings.HasPrefix(line, "(?d)") && !seenPrefix[2] {
seenPrefix[2] = true
line = line[4:]
ignoreStats["deletable"] += 1
report.IgnoreStats.Deletable++
} else {
seenPrefix[0] = false
seenPrefix[1] = false
@ -592,28 +593,26 @@ func (m *model) UsageReportingStats(version int, preview bool) map[string]interf
line = strings.TrimPrefix(line, "**/")
if strings.HasPrefix(line, "/") {
ignoreStats["rooted"] += 1
report.IgnoreStats.Rooted++
} else if strings.HasPrefix(line, "#include ") {
ignoreStats["includes"] += 1
report.IgnoreStats.Includes++
if strings.Contains(line, "..") {
ignoreStats["escapedIncludes"] += 1
report.IgnoreStats.EscapedIncludes++
}
}
if strings.Contains(line, "**") {
ignoreStats["doubleStars"] += 1
report.IgnoreStats.DoubleStars++
// Remove not to trip up star checks.
line = strings.Replace(line, "**", "", -1)
}
if strings.Contains(line, "*") {
ignoreStats["stars"] += 1
report.IgnoreStats.Stars++
}
}
}
stats["ignoreStats"] = ignoreStats
}
return stats
}
type ConnectionInfo struct {

425
lib/ur/contract/contract.go Normal file
View File

@ -0,0 +1,425 @@
// Copyright (C) 2020 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package contract
import (
"database/sql/driver"
"encoding/json"
"errors"
"reflect"
"strconv"
"time"
)
type Report struct {
// Generated
Received time.Time `json:"-"` // Only from DB
Date string `json:"date,omitempty"`
Address string `json:"address,omitempty"`
// v1 fields
UniqueID string `json:"uniqueID,omitempty" since:"1"`
Version string `json:"version,omitempty" since:"1"`
LongVersion string `json:"longVersion,omitempty" since:"1"`
Platform string `json:"platform,omitempty" since:"1"`
NumFolders int `json:"numFolders,omitempty" since:"1"`
NumDevices int `json:"numDevices,omitempty" since:"1"`
TotFiles int `json:"totFiles,omitempty" since:"1"`
FolderMaxFiles int `json:"folderMaxFiles,omitempty" since:"1"`
TotMiB int `json:"totMiB,omitempty" since:"1"`
FolderMaxMiB int `json:"folderMaxMiB,omitempty" since:"1"`
MemoryUsageMiB int `json:"memoryUsageMiB,omitempty" since:"1"`
SHA256Perf float64 `json:"sha256Perf,omitempty" since:"1"`
HashPerf float64 `json:"hashPerf,omitempty" since:"1"` // Was previously not stored server-side
MemorySize int `json:"memorySize,omitempty" since:"1"`
// v2 fields
URVersion int `json:"urVersion,omitempty" since:"2"`
NumCPU int `json:"numCPU,omitempty" since:"2"`
FolderUses struct {
SendOnly int `json:"sendonly,omitempty" since:"2"`
SendReceive int `json:"sendreceive,omitempty" since:"2"` // Was previously not stored server-side
ReceiveOnly int `json:"receiveonly,omitempty" since:"2"`
IgnorePerms int `json:"ignorePerms,omitempty" since:"2"`
IgnoreDelete int `json:"ignoreDelete,omitempty" since:"2"`
AutoNormalize int `json:"autoNormalize,omitempty" since:"2"`
SimpleVersioning int `json:"simpleVersioning,omitempty" since:"2"`
ExternalVersioning int `json:"externalVersioning,omitempty" since:"2"`
StaggeredVersioning int `json:"staggeredVersioning,omitempty" since:"2"`
TrashcanVersioning int `json:"trashcanVersioning,omitempty" since:"2"`
} `json:"folderUses,omitempty" since:"2"`
DeviceUses struct {
Introducer int `json:"introducer,omitempty" since:"2"`
CustomCertName int `json:"customCertName,omitempty" since:"2"`
CompressAlways int `json:"compressAlways,omitempty" since:"2"`
CompressMetadata int `json:"compressMetadata,omitempty" since:"2"`
CompressNever int `json:"compressNever,omitempty" since:"2"`
DynamicAddr int `json:"dynamicAddr,omitempty" since:"2"`
StaticAddr int `json:"staticAddr,omitempty" since:"2"`
} `json:"deviceUses,omitempty" since:"2"`
Announce struct {
GlobalEnabled bool `json:"globalEnabled,omitempty" since:"2"`
LocalEnabled bool `json:"localEnabled,omitempty" since:"2"`
DefaultServersDNS int `json:"defaultServersDNS,omitempty" since:"2"`
DefaultServersIP int `json:"defaultServersIP,omitempty" since:"2"` // Deprecated and not provided client-side anymore
OtherServers int `json:"otherServers,omitempty" since:"2"`
} `json:"announce,omitempty" since:"2"`
Relays struct {
Enabled bool `json:"enabled,omitempty" since:"2"`
DefaultServers int `json:"defaultServers,omitempty" since:"2"`
OtherServers int `json:"otherServers,omitempty" since:"2"`
} `json:"relays,omitempty" since:"2"`
UsesRateLimit bool `json:"usesRateLimit,omitempty" since:"2"`
UpgradeAllowedManual bool `json:"upgradeAllowedManual,omitempty" since:"2"`
UpgradeAllowedAuto bool `json:"upgradeAllowedAuto,omitempty" since:"2"`
// V2.5 fields (fields that were in v2 but never added to the database
UpgradeAllowedPre bool `json:"upgradeAllowedPre,omitempty" since:"2"`
RescanIntvs []int `json:"rescanIntvs,omitempty" since:"2"`
// v3 fields
Uptime int `json:"uptime,omitempty" since:"3"`
NATType string `json:"natType,omitempty" since:"3"`
AlwaysLocalNets bool `json:"alwaysLocalNets,omitempty" since:"3"`
CacheIgnoredFiles bool `json:"cacheIgnoredFiles,omitempty" since:"3"`
OverwriteRemoteDeviceNames bool `json:"overwriteRemoteDeviceNames,omitempty" since:"3"`
ProgressEmitterEnabled bool `json:"progressEmitterEnabled,omitempty" since:"3"`
CustomDefaultFolderPath bool `json:"customDefaultFolderPath,omitempty" since:"3"`
WeakHashSelection string `json:"weakHashSelection,omitempty" since:"3"` // Deprecated and not provided client-side anymore
CustomTrafficClass bool `json:"customTrafficClass,omitempty" since:"3"`
CustomTempIndexMinBlocks bool `json:"customTempIndexMinBlocks,omitempty" since:"3"`
TemporariesDisabled bool `json:"temporariesDisabled,omitempty" since:"3"`
TemporariesCustom bool `json:"temporariesCustom,omitempty" since:"3"`
LimitBandwidthInLan bool `json:"limitBandwidthInLan,omitempty" since:"3"`
CustomReleaseURL bool `json:"customReleaseURL,omitempty" since:"3"`
RestartOnWakeup bool `json:"restartOnWakeup,omitempty" since:"3"`
CustomStunServers bool `json:"customStunServers,omitempty" since:"3"`
FolderUsesV3 struct {
ScanProgressDisabled int `json:"scanProgressDisabled,omitempty" since:"3"`
ConflictsDisabled int `json:"conflictsDisabled,omitempty" since:"3"`
ConflictsUnlimited int `json:"conflictsUnlimited,omitempty" since:"3"`
ConflictsOther int `json:"conflictsOther,omitempty" since:"3"`
DisableSparseFiles int `json:"disableSparseFiles,omitempty" since:"3"`
DisableTempIndexes int `json:"disableTempIndexes,omitempty" since:"3"`
AlwaysWeakHash int `json:"alwaysWeakHash,omitempty" since:"3"`
CustomWeakHashThreshold int `json:"customWeakHashThreshold,omitempty" since:"3"`
FsWatcherEnabled int `json:"fsWatcherEnabled,omitempty" since:"3"`
PullOrder map[string]int `json:"pullOrder,omitempty" since:"3"`
FilesystemType map[string]int `json:"filesystemType,omitempty" since:"3"`
FsWatcherDelays []int `json:"fsWatcherDelays,omitempty" since:"3"`
} `json:"folderUsesV3,omitempty" since:"3"`
GUIStats struct {
Enabled int `json:"enabled,omitempty" since:"3"`
UseTLS int `json:"useTLS,omitempty" since:"3"`
UseAuth int `json:"useAuth,omitempty" since:"3"`
InsecureAdminAccess int `json:"insecureAdminAccess,omitempty" since:"3"`
Debugging int `json:"debugging,omitempty" since:"3"`
InsecureSkipHostCheck int `json:"insecureSkipHostCheck,omitempty" since:"3"`
InsecureAllowFrameLoading int `json:"insecureAllowFrameLoading,omitempty" since:"3"`
ListenLocal int `json:"listenLocal,omitempty" since:"3"`
ListenUnspecified int `json:"listenUnspecified,omitempty" since:"3"`
Theme map[string]int `json:"theme,omitempty" since:"3"`
} `json:"guiStats,omitempty" since:"3"`
BlockStats struct {
Total int `json:"total,omitempty" since:"3"`
Renamed int `json:"renamed,omitempty" since:"3"`
Reused int `json:"reused,omitempty" since:"3"`
Pulled int `json:"pulled,omitempty" since:"3"`
CopyOrigin int `json:"copyOrigin,omitempty" since:"3"`
CopyOriginShifted int `json:"copyOriginShifted,omitempty" since:"3"`
CopyElsewhere int `json:"copyElsewhere,omitempty" since:"3"`
} `json:"blockStats,omitempty" since:"3"`
TransportStats map[string]int `json:"transportStats,omitempty" since:"3"`
IgnoreStats struct {
Lines int `json:"lines,omitempty" since:"3"`
Inverts int `json:"inverts,omitempty" since:"3"`
Folded int `json:"folded,omitempty" since:"3"`
Deletable int `json:"deletable,omitempty" since:"3"`
Rooted int `json:"rooted,omitempty" since:"3"`
Includes int `json:"includes,omitempty" since:"3"`
EscapedIncludes int `json:"escapedIncludes,omitempty" since:"3"`
DoubleStars int `json:"doubleStars,omitempty" since:"3"`
Stars int `json:"stars,omitempty" since:"3"`
} `json:"ignoreStats,omitempty" since:"3"`
// V3 fields added late in the RC
WeakHashEnabled bool `json:"weakHashEnabled,omitempty" since:"3"` // Deprecated and not provided client-side anymore
}
func New() *Report {
r := &Report{}
r.FolderUsesV3.PullOrder = make(map[string]int)
r.FolderUsesV3.FilesystemType = make(map[string]int)
r.GUIStats.Theme = make(map[string]int)
r.TransportStats = make(map[string]int)
r.RescanIntvs = make([]int, 0)
r.FolderUsesV3.FsWatcherDelays = make([]int, 0)
return r
}
func (r *Report) Validate() error {
if r.UniqueID == "" || r.Version == "" || r.Platform == "" {
return errors.New("missing required field")
}
if len(r.Date) != 8 {
return errors.New("date not initialized")
}
// Some fields may not be null.
if r.RescanIntvs == nil {
r.RescanIntvs = []int{}
}
if r.FolderUsesV3.FsWatcherDelays == nil {
r.FolderUsesV3.FsWatcherDelays = []int{}
}
return nil
}
func (r *Report) ClearForVersion(version int) error {
return clear(r, version)
}
func (r *Report) FieldPointers() []interface{} {
// All the fields of the Report, in the same order as the database fields.
return []interface{}{
&r.Received, &r.UniqueID, &r.Version, &r.LongVersion, &r.Platform,
&r.NumFolders, &r.NumDevices, &r.TotFiles, &r.FolderMaxFiles,
&r.TotMiB, &r.FolderMaxMiB, &r.MemoryUsageMiB, &r.SHA256Perf,
&r.MemorySize, &r.Date,
// V2
&r.URVersion, &r.NumCPU, &r.FolderUses.SendOnly, &r.FolderUses.IgnorePerms,
&r.FolderUses.IgnoreDelete, &r.FolderUses.AutoNormalize, &r.DeviceUses.Introducer,
&r.DeviceUses.CustomCertName, &r.DeviceUses.CompressAlways,
&r.DeviceUses.CompressMetadata, &r.DeviceUses.CompressNever,
&r.DeviceUses.DynamicAddr, &r.DeviceUses.StaticAddr,
&r.Announce.GlobalEnabled, &r.Announce.LocalEnabled,
&r.Announce.DefaultServersDNS, &r.Announce.DefaultServersIP,
&r.Announce.OtherServers, &r.Relays.Enabled, &r.Relays.DefaultServers,
&r.Relays.OtherServers, &r.UsesRateLimit, &r.UpgradeAllowedManual,
&r.UpgradeAllowedAuto, &r.FolderUses.SimpleVersioning,
&r.FolderUses.ExternalVersioning, &r.FolderUses.StaggeredVersioning,
&r.FolderUses.TrashcanVersioning,
// V2.5
&r.UpgradeAllowedPre,
// V3
&r.Uptime, &r.NATType, &r.AlwaysLocalNets, &r.CacheIgnoredFiles,
&r.OverwriteRemoteDeviceNames, &r.ProgressEmitterEnabled, &r.CustomDefaultFolderPath,
&r.WeakHashSelection, &r.CustomTrafficClass, &r.CustomTempIndexMinBlocks,
&r.TemporariesDisabled, &r.TemporariesCustom, &r.LimitBandwidthInLan,
&r.CustomReleaseURL, &r.RestartOnWakeup, &r.CustomStunServers,
&r.FolderUsesV3.ScanProgressDisabled, &r.FolderUsesV3.ConflictsDisabled,
&r.FolderUsesV3.ConflictsUnlimited, &r.FolderUsesV3.ConflictsOther,
&r.FolderUsesV3.DisableSparseFiles, &r.FolderUsesV3.DisableTempIndexes,
&r.FolderUsesV3.AlwaysWeakHash, &r.FolderUsesV3.CustomWeakHashThreshold,
&r.FolderUsesV3.FsWatcherEnabled,
&r.GUIStats.Enabled, &r.GUIStats.UseTLS, &r.GUIStats.UseAuth,
&r.GUIStats.InsecureAdminAccess,
&r.GUIStats.Debugging, &r.GUIStats.InsecureSkipHostCheck,
&r.GUIStats.InsecureAllowFrameLoading, &r.GUIStats.ListenLocal,
&r.GUIStats.ListenUnspecified,
&r.BlockStats.Total, &r.BlockStats.Renamed,
&r.BlockStats.Reused, &r.BlockStats.Pulled, &r.BlockStats.CopyOrigin,
&r.BlockStats.CopyOriginShifted, &r.BlockStats.CopyElsewhere,
&r.IgnoreStats.Lines, &r.IgnoreStats.Inverts, &r.IgnoreStats.Folded,
&r.IgnoreStats.Deletable, &r.IgnoreStats.Rooted, &r.IgnoreStats.Includes,
&r.IgnoreStats.EscapedIncludes, &r.IgnoreStats.DoubleStars, &r.IgnoreStats.Stars,
// V3 added late in the RC
&r.WeakHashEnabled,
&r.Address,
// Receive only folders
&r.FolderUses.ReceiveOnly,
}
}
func (r *Report) FieldNames() []string {
// The database fields that back this struct in PostgreSQL
return []string{
// V1
"Received",
"UniqueID",
"Version",
"LongVersion",
"Platform",
"NumFolders",
"NumDevices",
"TotFiles",
"FolderMaxFiles",
"TotMiB",
"FolderMaxMiB",
"MemoryUsageMiB",
"SHA256Perf",
"MemorySize",
"Date",
// V2
"ReportVersion",
"NumCPU",
"FolderRO",
"FolderIgnorePerms",
"FolderIgnoreDelete",
"FolderAutoNormalize",
"DeviceIntroducer",
"DeviceCustomCertName",
"DeviceCompressAlways",
"DeviceCompressMetadata",
"DeviceCompressNever",
"DeviceDynamicAddr",
"DeviceStaticAddr",
"AnnounceGlobalEnabled",
"AnnounceLocalEnabled",
"AnnounceDefaultServersDNS",
"AnnounceDefaultServersIP",
"AnnounceOtherServers",
"RelayEnabled",
"RelayDefaultServers",
"RelayOtherServers",
"RateLimitEnabled",
"UpgradeAllowedManual",
"UpgradeAllowedAuto",
// v0.12.19+
"FolderSimpleVersioning",
"FolderExternalVersioning",
"FolderStaggeredVersioning",
"FolderTrashcanVersioning",
// V2.5
"UpgradeAllowedPre",
// V3
"Uptime",
"NATType",
"AlwaysLocalNets",
"CacheIgnoredFiles",
"OverwriteRemoteDeviceNames",
"ProgressEmitterEnabled",
"CustomDefaultFolderPath",
"WeakHashSelection",
"CustomTrafficClass",
"CustomTempIndexMinBlocks",
"TemporariesDisabled",
"TemporariesCustom",
"LimitBandwidthInLan",
"CustomReleaseURL",
"RestartOnWakeup",
"CustomStunServers",
"FolderScanProgressDisabled",
"FolderConflictsDisabled",
"FolderConflictsUnlimited",
"FolderConflictsOther",
"FolderDisableSparseFiles",
"FolderDisableTempIndexes",
"FolderAlwaysWeakHash",
"FolderCustomWeakHashThreshold",
"FolderFsWatcherEnabled",
"GUIEnabled",
"GUIUseTLS",
"GUIUseAuth",
"GUIInsecureAdminAccess",
"GUIDebugging",
"GUIInsecureSkipHostCheck",
"GUIInsecureAllowFrameLoading",
"GUIListenLocal",
"GUIListenUnspecified",
"BlocksTotal",
"BlocksRenamed",
"BlocksReused",
"BlocksPulled",
"BlocksCopyOrigin",
"BlocksCopyOriginShifted",
"BlocksCopyElsewhere",
"IgnoreLines",
"IgnoreInverts",
"IgnoreFolded",
"IgnoreDeletable",
"IgnoreRooted",
"IgnoreIncludes",
"IgnoreEscapedIncludes",
"IgnoreDoubleStars",
"IgnoreStars",
// V3 added late in the RC
"WeakHashEnabled",
"Address",
// Receive only folders
"FolderRecvOnly",
}
}
func (r Report) Value() (driver.Value, error) {
// This needs to be string, yet we read back bytes..
bs, err := json.Marshal(r)
return string(bs), err
}
func (r *Report) Scan(value interface{}) error {
b, ok := value.([]byte)
if !ok {
return errors.New("type assertion to []byte failed")
}
return json.Unmarshal(b, &r)
}
func clear(v interface{}, since int) error {
s := reflect.ValueOf(v).Elem()
t := s.Type()
for i := 0; i < s.NumField(); i++ {
f := s.Field(i)
tag := t.Field(i).Tag
v := tag.Get("since")
if len(v) == 0 {
f.Set(reflect.Zero(f.Type()))
continue
}
vn, err := strconv.Atoi(v)
if err != nil {
return err
}
if vn > since {
f.Set(reflect.Zero(f.Type()))
continue
}
// Dive deeper
if f.Kind() == reflect.Ptr {
f = f.Elem()
}
if f.Kind() == reflect.Struct {
if err := clear(f.Addr().Interface(), since); err != nil {
return err
}
}
}
return nil
}

130
lib/ur/contract/contract_test.go Normal file
View File

@ -0,0 +1,130 @@
// Copyright (C) 2020 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package contract
import (
"reflect"
"testing"
)
type PtrStruct struct {
A string `since:"2"`
B map[string]int `since:"3"`
}
type Nested struct {
A float32 `since:"4"`
B [4]int `since:"5"`
C bool `since:"1"`
}
type TestStruct struct {
A int
B map[string]string `since:"1"`
C []string `since:"2"`
Nested Nested `since:"3"`
Ptr *PtrStruct `since:"2"`
}
func testValue() TestStruct {
return TestStruct{
A: 1,
B: map[string]string{
"foo": "bar",
},
C: []string{"a", "b"},
Nested: Nested{
A: 0.10,
B: [4]int{1, 2, 3, 4},
C: true,
},
Ptr: &PtrStruct{
A: "value",
B: map[string]int{
"x": 1,
"b": 2,
},
},
}
}
func TestClean(t *testing.T) {
expect(t, 0, TestStruct{})
expect(t, 1, TestStruct{
// A unset, since it does not have "since"
B: map[string]string{
"foo": "bar",
},
})
expect(t, 2, TestStruct{
// A unset, since it does not have "since"
B: map[string]string{
"foo": "bar",
},
C: []string{"a", "b"},
Ptr: &PtrStruct{
A: "value",
},
})
expect(t, 3, TestStruct{
// A unset, since it does not have "since"
B: map[string]string{
"foo": "bar",
},
C: []string{"a", "b"},
Nested: Nested{
C: true,
},
Ptr: &PtrStruct{
A: "value",
B: map[string]int{
"x": 1,
"b": 2,
},
},
})
expect(t, 4, TestStruct{
// A unset, since it does not have "since"
B: map[string]string{
"foo": "bar",
},
C: []string{"a", "b"},
Nested: Nested{
A: 0.10,
C: true,
},
Ptr: &PtrStruct{
A: "value",
B: map[string]int{
"x": 1,
"b": 2,
},
},
})
x := testValue()
x.A = 0
expect(t, 5, x)
expect(t, 6, x)
}
func expect(t *testing.T, since int, b interface{}) {
t.Helper()
x := testValue()
if err := clear(&x, since); err != nil {
t.Fatal(err.Error())
}
if !reflect.DeepEqual(x, b) {
t.Errorf("%#v != %#v", x, b)
}
}

7
lib/ur/memsize_darwin.go
View File

@ -8,7 +8,10 @@ package ur
import "golang.org/x/sys/unix"
func memorySize() (int64, error) {
func memorySize() int64 {
mem, err := unix.SysctlUint64("hw.memsize")
return int64(mem), err
if err != nil {
return 0
}
return int64(mem)
}

13
lib/ur/memsize_linux.go
View File

@ -8,32 +8,31 @@ package ur
import (
"bufio"
"errors"
"os"
"strconv"
"strings"
)
func memorySize() (int64, error) {
func memorySize() int64 {
f, err := os.Open("/proc/meminfo")
if err != nil {
return 0, err
return 0
}
s := bufio.NewScanner(f)
if !s.Scan() {
return 0, errors.New("/proc/meminfo parse error 1")
return 0
}
l := s.Text()
fs := strings.Fields(l)
if len(fs) != 3 || fs[2] != "kB" {
return 0, errors.New("/proc/meminfo parse error 2")
return 0
}
kb, err := strconv.ParseInt(fs[1], 10, 64)
if err != nil {
return 0, err
return 0
}
return kb * 1024, nil
return kb * 1024
}

11
lib/ur/memsize_netbsd.go
View File

@ -7,25 +7,24 @@
package ur
import (
"errors"
"os/exec"
"strconv"
"strings"
)
func memorySize() (int64, error) {
func memorySize() int64 {
cmd := exec.Command("/sbin/sysctl", "hw.physmem64")
out, err := cmd.Output()
if err != nil {
return 0, err
return 0
}
fs := strings.Fields(string(out))
if len(fs) != 3 {
return 0, errors.New("sysctl parse error")
return 0
}
bytes, err := strconv.ParseInt(fs[2], 10, 64)
if err != nil {
return 0, err
return 0
}
return bytes, nil
return bytes
}

8
lib/ur/memsize_solaris.go
View File

@ -13,16 +13,16 @@ import (
"strconv"
)
func memorySize() (int64, error) {
func memorySize() int64 {
cmd := exec.Command("prtconf", "-m")
out, err := cmd.CombinedOutput()
if err != nil {
return 0, err
return 0
}
mb, err := strconv.ParseInt(string(out), 10, 64)
if err != nil {
return 0, err
return 0
}
return mb * 1024 * 1024, nil
return mb * 1024 * 1024
}

6
lib/ur/memsize_unimpl.go
View File

@ -8,8 +8,6 @@
package ur
import "errors"
func memorySize() (int64, error) {
return 0, errors.New("not implemented")
func memorySize() int64 {
return 0
}

8
lib/ur/memsize_windows.go
View File

@ -17,14 +17,14 @@ var (
globalMemoryStatusEx, _ = syscall.GetProcAddress(kernel32, "GlobalMemoryStatusEx")
)
func memorySize() (int64, error) {
func memorySize() int64 {
var memoryStatusEx [64]byte
binary.LittleEndian.PutUint32(memoryStatusEx[:], 64)
ret, _, callErr := syscall.Syscall(uintptr(globalMemoryStatusEx), 1, uintptr(unsafe.Pointer(&memoryStatusEx[0])), 0, 0)
ret, _, _ := syscall.Syscall(uintptr(globalMemoryStatusEx), 1, uintptr(unsafe.Pointer(&memoryStatusEx[0])), 0, 0)
if ret == 0 {
return 0, callErr
return 0
}
return int64(binary.LittleEndian.Uint64(memoryStatusEx[8:])), nil
return int64(binary.LittleEndian.Uint64(memoryStatusEx[8:]))
}

296
lib/ur/usage_report.go
View File

@ -28,6 +28,7 @@ import (
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/scanner"
"github.com/syncthing/syncthing/lib/upgrade"
"github.com/syncthing/syncthing/lib/ur/contract"
"github.com/syncthing/syncthing/lib/util"
"github.com/thejerf/suture"
@ -63,27 +64,19 @@ func New(cfg config.Wrapper, m model.Model, connectionsService connections.Servi
// ReportData returns the data to be sent in a usage report with the currently
// configured usage reporting version.
func (s *Service) ReportData(ctx context.Context) map[string]interface{} {
func (s *Service) ReportData(ctx context.Context) (*contract.Report, error) {
urVersion := s.cfg.Options().URAccepted
return s.reportData(ctx, urVersion, false)
}
// ReportDataPreview returns a preview of the data to be sent in a usage report
// with the given version.
func (s *Service) ReportDataPreview(ctx context.Context, urVersion int) map[string]interface{} {
func (s *Service) ReportDataPreview(ctx context.Context, urVersion int) (*contract.Report, error) {
return s.reportData(ctx, urVersion, true)
}
func (s *Service) reportData(ctx context.Context, urVersion int, preview bool) map[string]interface{} {
func (s *Service) reportData(ctx context.Context, urVersion int, preview bool) (*contract.Report, error) {
opts := s.cfg.Options()
res := make(map[string]interface{})
res["urVersion"] = urVersion
res["uniqueID"] = opts.URUniqueID
res["version"] = build.Version
res["longVersion"] = build.LongVersion
res["platform"] = runtime.GOOS + "-" + runtime.GOARCH
res["numFolders"] = len(s.cfg.Folders())
res["numDevices"] = len(s.cfg.Devices())
var totFiles, maxFiles int
var totBytes, maxBytes int64
@ -104,264 +97,211 @@ func (s *Service) reportData(ctx context.Context, urVersion int, preview bool) m
}
}
res["totFiles"] = totFiles
res["folderMaxFiles"] = maxFiles
res["totMiB"] = totBytes / 1024 / 1024
res["folderMaxMiB"] = maxBytes / 1024 / 1024
var mem runtime.MemStats
runtime.ReadMemStats(&mem)
res["memoryUsageMiB"] = (mem.Sys - mem.HeapReleased) / 1024 / 1024
res["sha256Perf"] = CpuBench(ctx, 5, 125*time.Millisecond, false)
res["hashPerf"] = CpuBench(ctx, 5, 125*time.Millisecond, true)
bytes, err := memorySize()
if err == nil {
res["memorySize"] = bytes / 1024 / 1024
}
res["numCPU"] = runtime.NumCPU()
report := contract.New()
report.URVersion = urVersion
report.UniqueID = opts.URUniqueID
report.Version = build.Version
report.LongVersion = build.LongVersion
report.Platform = runtime.GOOS + "-" + runtime.GOARCH
report.NumFolders = len(s.cfg.Folders())
report.NumDevices = len(s.cfg.Devices())
report.TotFiles = totFiles
report.FolderMaxFiles = maxFiles
report.TotMiB = int(totBytes / 1024 / 1024)
report.FolderMaxMiB = int(maxBytes / 1024 / 1024)
report.MemoryUsageMiB = int((mem.Sys - mem.HeapReleased) / 1024 / 1024)
report.SHA256Perf = CpuBench(ctx, 5, 125*time.Millisecond, false)
report.HashPerf = CpuBench(ctx, 5, 125*time.Millisecond, true)
report.MemorySize = int(memorySize() / 1024 / 1024)
report.NumCPU = runtime.NumCPU()
var rescanIntvs []int
folderUses := map[string]int{
"sendonly": 0,
"sendreceive": 0,
"receiveonly": 0,
"ignorePerms": 0,
"ignoreDelete": 0,
"autoNormalize": 0,
"simpleVersioning": 0,
"externalVersioning": 0,
"staggeredVersioning": 0,
"trashcanVersioning": 0,
}
for _, cfg := range s.cfg.Folders() {
rescanIntvs = append(rescanIntvs, cfg.RescanIntervalS)
report.RescanIntvs = append(report.RescanIntvs, cfg.RescanIntervalS)
switch cfg.Type {
case config.FolderTypeSendOnly:
folderUses["sendonly"]++
report.FolderUses.SendOnly++
case config.FolderTypeSendReceive:
folderUses["sendreceive"]++
report.FolderUses.SendReceive++
case config.FolderTypeReceiveOnly:
folderUses["receiveonly"]++
report.FolderUses.ReceiveOnly++
}
if cfg.IgnorePerms {
folderUses["ignorePerms"]++
report.FolderUses.IgnorePerms++
}
if cfg.IgnoreDelete {
folderUses["ignoreDelete"]++
report.FolderUses.IgnoreDelete++
}
if cfg.AutoNormalize {
folderUses["autoNormalize"]++
report.FolderUses.AutoNormalize++
}
if cfg.Versioning.Type != "" {
folderUses[cfg.Versioning.Type+"Versioning"]++
switch cfg.Versioning.Type {
case "":
// None
case "simple":
report.FolderUses.SimpleVersioning++
case "staggered":
report.FolderUses.StaggeredVersioning++
case "external":
report.FolderUses.ExternalVersioning++
case "trashcan":
report.FolderUses.TrashcanVersioning++
default:
l.Warnf("Unhandled versioning type for usage reports: %s", cfg.Versioning.Type)
}
}
sort.Ints(rescanIntvs)
res["rescanIntvs"] = rescanIntvs
res["folderUses"] = folderUses
sort.Ints(report.RescanIntvs)
deviceUses := map[string]int{
"introducer": 0,
"customCertName": 0,
"compressAlways": 0,
"compressMetadata": 0,
"compressNever": 0,
"dynamicAddr": 0,
"staticAddr": 0,
}
for _, cfg := range s.cfg.Devices() {
if cfg.Introducer {
deviceUses["introducer"]++
report.DeviceUses.Introducer++
}
if cfg.CertName != "" && cfg.CertName != "syncthing" {
deviceUses["customCertName"]++
report.DeviceUses.CustomCertName++
}
if cfg.Compression == protocol.CompressAlways {
deviceUses["compressAlways"]++
} else if cfg.Compression == protocol.CompressMetadata {
deviceUses["compressMetadata"]++
} else if cfg.Compression == protocol.CompressNever {
deviceUses["compressNever"]++
switch cfg.Compression {
case protocol.CompressAlways:
report.DeviceUses.CompressAlways++
case protocol.CompressMetadata:
report.DeviceUses.CompressMetadata++
case protocol.CompressNever:
report.DeviceUses.CompressNever++
default:
l.Warnf("Unhandled versioning type for usage reports: %s", cfg.Compression)
}
for _, addr := range cfg.Addresses {
if addr == "dynamic" {
deviceUses["dynamicAddr"]++
report.DeviceUses.DynamicAddr++
} else {
deviceUses["staticAddr"]++
report.DeviceUses.StaticAddr++
}
}
}
res["deviceUses"] = deviceUses
defaultAnnounceServersDNS, defaultAnnounceServersIP, otherAnnounceServers := 0, 0, 0
report.Announce.GlobalEnabled = opts.GlobalAnnEnabled
report.Announce.LocalEnabled = opts.LocalAnnEnabled
for _, addr := range opts.RawGlobalAnnServers {
if addr == "default" || addr == "default-v4" || addr == "default-v6" {
defaultAnnounceServersDNS++
report.Announce.DefaultServersDNS++
} else {
otherAnnounceServers++
report.Announce.OtherServers++
}
}
res["announce"] = map[string]interface{}{
"globalEnabled": opts.GlobalAnnEnabled,
"localEnabled": opts.LocalAnnEnabled,
"defaultServersDNS": defaultAnnounceServersDNS,
"defaultServersIP": defaultAnnounceServersIP,
"otherServers": otherAnnounceServers,
}
defaultRelayServers, otherRelayServers := 0, 0
report.Relays.Enabled = opts.RelaysEnabled
for _, addr := range s.cfg.Options().ListenAddresses() {
switch {
case addr == "dynamic+https://relays.syncthing.net/endpoint":
defaultRelayServers++
report.Relays.DefaultServers++
case strings.HasPrefix(addr, "relay://") || strings.HasPrefix(addr, "dynamic+http"):
otherRelayServers++
report.Relays.OtherServers++
}
}
res["relays"] = map[string]interface{}{
"enabled": defaultRelayServers+otherAnnounceServers > 0,
"defaultServers": defaultRelayServers,
"otherServers": otherRelayServers,
}
res["usesRateLimit"] = opts.MaxRecvKbps > 0 || opts.MaxSendKbps > 0
report.UsesRateLimit = opts.MaxRecvKbps > 0 || opts.MaxSendKbps > 0
report.UpgradeAllowedManual = !(upgrade.DisabledByCompilation || s.noUpgrade)
report.UpgradeAllowedAuto = !(upgrade.DisabledByCompilation || s.noUpgrade) && opts.AutoUpgradeIntervalH > 0
report.UpgradeAllowedPre = !(upgrade.DisabledByCompilation || s.noUpgrade) && opts.AutoUpgradeIntervalH > 0 && opts.UpgradeToPreReleases
res["upgradeAllowedManual"] = !(upgrade.DisabledByCompilation || s.noUpgrade)
res["upgradeAllowedAuto"] = !(upgrade.DisabledByCompilation || s.noUpgrade) && opts.AutoUpgradeIntervalH > 0
res["upgradeAllowedPre"] = !(upgrade.DisabledByCompilation || s.noUpgrade) && opts.AutoUpgradeIntervalH > 0 && opts.UpgradeToPreReleases
// V3
if urVersion >= 3 {
res["uptime"] = s.UptimeS()
res["natType"] = s.connectionsService.NATType()
res["alwaysLocalNets"] = len(opts.AlwaysLocalNets) > 0
res["cacheIgnoredFiles"] = opts.CacheIgnoredFiles
res["overwriteRemoteDeviceNames"] = opts.OverwriteRemoteDevNames
res["progressEmitterEnabled"] = opts.ProgressUpdateIntervalS > -1
res["customDefaultFolderPath"] = opts.DefaultFolderPath != "~"
res["customTrafficClass"] = opts.TrafficClass != 0
res["customTempIndexMinBlocks"] = opts.TempIndexMinBlocks != 10
res["temporariesDisabled"] = opts.KeepTemporariesH == 0
res["temporariesCustom"] = opts.KeepTemporariesH != 24
res["limitBandwidthInLan"] = opts.LimitBandwidthInLan
res["customReleaseURL"] = opts.ReleasesURL != "https://upgrades.syncthing.net/meta.json"
res["restartOnWakeup"] = opts.RestartOnWakeup
report.Uptime = s.UptimeS()
report.NATType = s.connectionsService.NATType()
report.AlwaysLocalNets = len(opts.AlwaysLocalNets) > 0
report.CacheIgnoredFiles = opts.CacheIgnoredFiles
report.OverwriteRemoteDeviceNames = opts.OverwriteRemoteDevNames
report.ProgressEmitterEnabled = opts.ProgressUpdateIntervalS > -1
report.CustomDefaultFolderPath = opts.DefaultFolderPath != "~"
report.CustomTrafficClass = opts.TrafficClass != 0
report.CustomTempIndexMinBlocks = opts.TempIndexMinBlocks != 10
report.TemporariesDisabled = opts.KeepTemporariesH == 0
report.TemporariesCustom = opts.KeepTemporariesH != 24
report.LimitBandwidthInLan = opts.LimitBandwidthInLan
report.CustomReleaseURL = opts.ReleasesURL != "https=//upgrades.syncthing.net/meta.json"
report.RestartOnWakeup = opts.RestartOnWakeup
report.CustomStunServers = len(opts.RawStunServers) != 1 || opts.RawStunServers[0] != "default"
folderUsesV3 := map[string]int{
"scanProgressDisabled": 0,
"conflictsDisabled": 0,
"conflictsUnlimited": 0,
"conflictsOther": 0,
"disableSparseFiles": 0,
"disableTempIndexes": 0,
"alwaysWeakHash": 0,
"customWeakHashThreshold": 0,
"fsWatcherEnabled": 0,
}
pullOrder := make(map[string]int)
filesystemType := make(map[string]int)
var fsWatcherDelays []int
for _, cfg := range s.cfg.Folders() {
if cfg.ScanProgressIntervalS < 0 {
folderUsesV3["scanProgressDisabled"]++
report.FolderUsesV3.ScanProgressDisabled++
}
if cfg.MaxConflicts == 0 {
folderUsesV3["conflictsDisabled"]++
report.FolderUsesV3.ConflictsDisabled++
} else if cfg.MaxConflicts < 0 {
folderUsesV3["conflictsUnlimited"]++
report.FolderUsesV3.ConflictsUnlimited++
} else {
folderUsesV3["conflictsOther"]++
report.FolderUsesV3.ConflictsOther++
}
if cfg.DisableSparseFiles {
folderUsesV3["disableSparseFiles"]++
report.FolderUsesV3.DisableSparseFiles++
}
if cfg.DisableTempIndexes {
folderUsesV3["disableTempIndexes"]++
report.FolderUsesV3.DisableTempIndexes++
}
if cfg.WeakHashThresholdPct < 0 {
folderUsesV3["alwaysWeakHash"]++
report.FolderUsesV3.AlwaysWeakHash++
} else if cfg.WeakHashThresholdPct != 25 {
folderUsesV3["customWeakHashThreshold"]++
report.FolderUsesV3.CustomWeakHashThreshold++
}
if cfg.FSWatcherEnabled {
folderUsesV3["fsWatcherEnabled"]++
report.FolderUsesV3.FsWatcherEnabled++
}
pullOrder[cfg.Order.String()]++
filesystemType[cfg.FilesystemType.String()]++
fsWatcherDelays = append(fsWatcherDelays, cfg.FSWatcherDelayS)
report.FolderUsesV3.PullOrder[cfg.Order.String()]++
report.FolderUsesV3.FilesystemType[cfg.FilesystemType.String()]++
report.FolderUsesV3.FsWatcherDelays = append(report.FolderUsesV3.FsWatcherDelays, cfg.FSWatcherDelayS)
}
sort.Ints(fsWatcherDelays)
folderUsesV3Interface := map[string]interface{}{
"pullOrder": pullOrder,
"filesystemType": filesystemType,
"fsWatcherDelays": fsWatcherDelays,
}
for key, value := range folderUsesV3 {
folderUsesV3Interface[key] = value
}
res["folderUsesV3"] = folderUsesV3Interface
sort.Ints(report.FolderUsesV3.FsWatcherDelays)
guiCfg := s.cfg.GUI()
// Anticipate multiple GUI configs in the future, hence store counts.
guiStats := map[string]int{
"enabled": 0,
"useTLS": 0,
"useAuth": 0,
"insecureAdminAccess": 0,
"debugging": 0,
"insecureSkipHostCheck": 0,
"insecureAllowFrameLoading": 0,
"listenLocal": 0,
"listenUnspecified": 0,
}
theme := make(map[string]int)
if guiCfg.Enabled {
guiStats["enabled"]++
report.GUIStats.Enabled++
if guiCfg.UseTLS() {
guiStats["useTLS"]++
report.GUIStats.UseTLS++
}
if len(guiCfg.User) > 0 && len(guiCfg.Password) > 0 {
guiStats["useAuth"]++
report.GUIStats.UseAuth++
}
if guiCfg.InsecureAdminAccess {
guiStats["insecureAdminAccess"]++
report.GUIStats.InsecureAdminAccess++
}
if guiCfg.Debugging {
guiStats["debugging"]++
report.GUIStats.Debugging++
}
if guiCfg.InsecureSkipHostCheck {
guiStats["insecureSkipHostCheck"]++
report.GUIStats.InsecureSkipHostCheck++
}
if guiCfg.InsecureAllowFrameLoading {
guiStats["insecureAllowFrameLoading"]++
report.GUIStats.InsecureAllowFrameLoading++
}
addr, err := net.ResolveTCPAddr("tcp", guiCfg.Address())
if err == nil {
if addr.IP.IsLoopback() {
guiStats["listenLocal"]++
report.GUIStats.ListenLocal++
} else if addr.IP.IsUnspecified() {
guiStats["listenUnspecified"]++
report.GUIStats.ListenUnspecified++
}
}
theme[guiCfg.Theme]++
report.GUIStats.Theme[guiCfg.Theme]++
}
guiStatsInterface := map[string]interface{}{
"theme": theme,
}
for key, value := range guiStats {
guiStatsInterface[key] = value
}
res["guiStats"] = guiStatsInterface
}
for key, value := range s.model.UsageReportingStats(urVersion, preview) {
res[key] = value
s.model.UsageReportingStats(report, urVersion, preview)
if err := report.ClearForVersion(urVersion); err != nil {
return nil, err
}
return res
return report, nil
}
func (s *Service) UptimeS() int {
@ -369,7 +309,10 @@ func (s *Service) UptimeS() int {
}
func (s *Service) sendUsageReport(ctx context.Context) error {
d := s.ReportData(ctx)
d, err := s.ReportData(ctx)
if err != nil {
return err
}
var b bytes.Buffer
if err := json.NewEncoder(&b).Encode(d); err != nil {
return err
@ -384,12 +327,11 @@ func (s *Service) sendUsageReport(ctx context.Context) error {
},
},
}
req, err := http.NewRequest("POST", s.cfg.Options().URURL, &b)
req, err := http.NewRequestWithContext(ctx, "POST", s.cfg.Options().URURL, &b)
if err != nil {
return err
}
req.Header.Set("Content-Type", "application/json")
req.Cancel = ctx.Done()
resp, err := client.Do(req)
if err != nil {
return err