octoleo/syncthing
1
0
Fork 0
mirror of https://github.com/octoleo/syncthing.git synced 2024-12-22 19:08:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

lib/weakhash, lib/model, cmd/syncthing: Decide if to use weakhash on startup (fixes #3938)

Browse Source 
GitHub-Pull-Request: https://github.com/syncthing/syncthing/pull/3949
This commit is contained in:
Audrius Butkevicius 2017-02-06 10:27:11 +00:00
parent 237893ead3
commit 67acef1794
12 changed files with 232 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
cmd/syncthing/main.go
View File

@ -46,6 +46,7 @@ import (
"github.com/syncthing/syncthing/lib/symlinks"
"github.com/syncthing/syncthing/lib/tlsutil"
"github.com/syncthing/syncthing/lib/upgrade"
"github.com/syncthing/syncthing/lib/weakhash"
"github.com/thejerf/suture"
@ -623,8 +624,10 @@ func syncthingMain(runtimeOptions RuntimeOptions) {
sha256.SelectAlgo()
sha256.Report()
perf := cpuBench(3, 150*time.Millisecond)
l.Infof("Actual hashing performance is %.02f MB/s", perf)
perfWithWeakHash := cpuBench(3, 150*time.Millisecond, true)
l.Infof("Hashing performance with weak hash is %.02f MB/s", perfWithWeakHash)
perfWithoutWeakHash := cpuBench(3, 150*time.Millisecond, false)
l.Infof("Hashing performance without weak hash is %.02f MB/s", perfWithoutWeakHash)
// Emit the Starting event, now that we know who we are.
@ -680,6 +683,22 @@ func syncthingMain(runtimeOptions RuntimeOptions) {
symlinks.Supported = false
}
if opts.WeakHashSelectionMethod == config.WeakHashAuto {
if perfWithoutWeakHash*0.8 > perfWithWeakHash {
l.Infof("Weak hash disabled, as it has an unacceptable performance impact.")
weakhash.Enabled = false
} else {
l.Infof("Weak hash enabled, as it has an acceptable performance impact.")
weakhash.Enabled = true
}
} else if opts.WeakHashSelectionMethod == config.WeakHashNever {
l.Infof("Disabling weak hash")
weakhash.Enabled = false
} else if opts.WeakHashSelectionMethod == config.WeakHashAlways {
l.Infof("Enabling weak hash")
weakhash.Enabled = true
}
if (opts.MaxRecvKbps > 0 || opts.MaxSendKbps > 0) && !opts.LimitBandwidthInLan {
lans, _ = osutil.GetLans()
for _, lan := range opts.AlwaysLocalNets {

5
cmd/syncthing/mocked_config_test.go
View File

@ -9,6 +9,7 @@ package main
import (
"github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/util"
)
type mockedConfig struct {
@ -24,7 +25,9 @@ func (c *mockedConfig) ListenAddresses() []string {
}
func (c *mockedConfig) RawCopy() config.Configuration {
return config.Configuration{}
cfg := config.Configuration{}
util.SetDefaults(&cfg.Options)
return cfg
}
func (c *mockedConfig) Options() config.OptionsConfiguration {

11
cmd/syncthing/usage_report.go
View File

@ -113,7 +113,8 @@ func reportData(cfg configIntf, m modelIntf) map[string]interface{} {
var mem runtime.MemStats
runtime.ReadMemStats(&mem)
res["memoryUsageMiB"] = (mem.Sys - mem.HeapReleased) / 1024 / 1024
res["sha256Perf"] = cpuBench(5, 125*time.Millisecond)
res["sha256Perf"] = cpuBench(5, 125*time.Millisecond, false)
res["hashPerf"] = cpuBench(5, 125*time.Millisecond, true)
bytes, err := memorySize()
if err == nil {
@ -286,10 +287,10 @@ func (s *usageReportingService) Stop() {
}
// cpuBench returns CPU performance as a measure of single threaded SHA-256 MiB/s
func cpuBench(iterations int, duration time.Duration) float64 {
func cpuBench(iterations int, duration time.Duration, useWeakHash bool) float64 {
var perf float64
for i := 0; i < iterations; i++ {
if v := cpuBenchOnce(duration); v > perf {
if v := cpuBenchOnce(duration, useWeakHash); v > perf {
perf = v
}
}
@ -299,7 +300,7 @@ func cpuBench(iterations int, duration time.Duration) float64 {
var blocksResult []protocol.BlockInfo // so the result is not optimized away
func cpuBenchOnce(duration time.Duration) float64 {
func cpuBenchOnce(duration time.Duration, useWeakHash bool) float64 {
dataSize := 16 * protocol.BlockSize
bs := make([]byte, dataSize)
rand.Reader.Read(bs)
@ -308,7 +309,7 @@ func cpuBenchOnce(duration time.Duration) float64 {
b := 0
for time.Since(t0) < duration {
r := bytes.NewReader(bs)
blocksResult, _ = scanner.Blocks(r, protocol.BlockSize, int64(dataSize), nil, true)
blocksResult, _ = scanner.Blocks(r, protocol.BlockSize, int64(dataSize), nil, useWeakHash)
b += dataSize
}
d := time.Since(t0)

26
lib/config/config.go
View File

@ -332,6 +332,19 @@ func convertV17V18(cfg *Configuration) {
cfg.Version = 18
}
func convertV16V17(cfg *Configuration) {
for i := range cfg.Folders {
cfg.Folders[i].Fsync = true
}
cfg.Version = 17
}
func convertV15V16(cfg *Configuration) {
// Triggers a database tweak
cfg.Version = 16
}
func convertV14V15(cfg *Configuration) {
// Undo v0.13.0 broken migration
@ -347,19 +360,6 @@ func convertV14V15(cfg *Configuration) {
cfg.Version = 15
}
func convertV15V16(cfg *Configuration) {
// Triggers a database tweak
cfg.Version = 16
}
func convertV16V17(cfg *Configuration) {
for i := range cfg.Folders {
cfg.Folders[i].Fsync = true
}
cfg.Version = 17
}
func convertV13V14(cfg *Configuration) {
// Not using the ignore cache is the new default. Disable it on existing
// configurations.

2
lib/config/config_test.go
View File

@ -65,6 +65,7 @@ func TestDefaultValues(t *testing.T) {
OverwriteRemoteDevNames: false,
TempIndexMinBlocks: 10,
UnackedNotificationIDs: []string{},
WeakHashSelectionMethod: WeakHashAuto,
}
cfg := New(device1)
@ -203,6 +204,7 @@ func TestOverriddenValues(t *testing.T) {
UnackedNotificationIDs: []string{
"channelNotification", // added in 17->18 migration
},
WeakHashSelectionMethod: WeakHashNever,
}
os.Unsetenv("STNOUPGRADE")

161
lib/config/optionsconfiguration.go
View File

@ -6,43 +6,132 @@
package config
import (
"encoding/json"
"encoding/xml"
"fmt"
)
type WeakHashSelectionMethod int
const (
WeakHashAuto WeakHashSelectionMethod = iota
WeakHashAlways
WeakHashNever
)
func (m WeakHashSelectionMethod) MarshalString() (string, error) {
switch m {
case WeakHashAuto:
return "auto", nil
case WeakHashAlways:
return "always", nil
case WeakHashNever:
return "never", nil
default:
return "", fmt.Errorf("unrecognized hash selection method")
}
}
func (m WeakHashSelectionMethod) String() string {
s, err := m.MarshalString()
if err != nil {
panic(err)
}
return s
}
func (m *WeakHashSelectionMethod) UnmarshalString(value string) error {
switch value {
case "auto":
*m = WeakHashAuto
return nil
case "always":
*m = WeakHashAlways
return nil
case "never":
*m = WeakHashNever
return nil
}
return fmt.Errorf("unrecognized hash selection method")
}
func (m WeakHashSelectionMethod) MarshalJSON() ([]byte, error) {
val, err := m.MarshalString()
if err != nil {
return nil, err
}
return json.Marshal(val)
}
func (m *WeakHashSelectionMethod) UnmarshalJSON(data []byte) error {
var value string
if err := json.Unmarshal(data, &value); err != nil {
return err
}
return m.UnmarshalString(value)
}
func (m WeakHashSelectionMethod) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
val, err := m.MarshalString()
if err != nil {
return err
}
return e.EncodeElement(val, start)
}
func (m *WeakHashSelectionMethod) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
var value string
if err := d.DecodeElement(&value, &start); err != nil {
return err
}
return m.UnmarshalString(value)
}
func (WeakHashSelectionMethod) ParseDefault(value string) (interface{}, error) {
var m WeakHashSelectionMethod
err := m.UnmarshalString(value)
return m, err
}
type OptionsConfiguration struct {
ListenAddresses []string `xml:"listenAddress" json:"listenAddresses" default:"default"`
GlobalAnnServers []string `xml:"globalAnnounceServer" json:"globalAnnounceServers" json:"globalAnnounceServer" default:"default"`
GlobalAnnEnabled bool `xml:"globalAnnounceEnabled" json:"globalAnnounceEnabled" default:"true"`
LocalAnnEnabled bool `xml:"localAnnounceEnabled" json:"localAnnounceEnabled" default:"true"`
LocalAnnPort int `xml:"localAnnouncePort" json:"localAnnouncePort" default:"21027"`
LocalAnnMCAddr string `xml:"localAnnounceMCAddr" json:"localAnnounceMCAddr" default:"[ff12::8384]:21027"`
MaxSendKbps int `xml:"maxSendKbps" json:"maxSendKbps"`
MaxRecvKbps int `xml:"maxRecvKbps" json:"maxRecvKbps"`
ReconnectIntervalS int `xml:"reconnectionIntervalS" json:"reconnectionIntervalS" default:"60"`
RelaysEnabled bool `xml:"relaysEnabled" json:"relaysEnabled" default:"true"`
RelayReconnectIntervalM int `xml:"relayReconnectIntervalM" json:"relayReconnectIntervalM" default:"10"`
StartBrowser bool `xml:"startBrowser" json:"startBrowser" default:"true"`
NATEnabled bool `xml:"natEnabled" json:"natEnabled" default:"true"`
NATLeaseM int `xml:"natLeaseMinutes" json:"natLeaseMinutes" default:"60"`
NATRenewalM int `xml:"natRenewalMinutes" json:"natRenewalMinutes" default:"30"`
NATTimeoutS int `xml:"natTimeoutSeconds" json:"natTimeoutSeconds" default:"10"`
URAccepted int `xml:"urAccepted" json:"urAccepted"` // Accepted usage reporting version; 0 for off (undecided), -1 for off (permanently)
URUniqueID string `xml:"urUniqueID" json:"urUniqueId"` // Unique ID for reporting purposes, regenerated when UR is turned on.
URURL string `xml:"urURL" json:"urURL" default:"https://data.syncthing.net/newdata"`
URPostInsecurely bool `xml:"urPostInsecurely" json:"urPostInsecurely" default:"false"` // For testing
URInitialDelayS int `xml:"urInitialDelayS" json:"urInitialDelayS" default:"1800"`
RestartOnWakeup bool `xml:"restartOnWakeup" json:"restartOnWakeup" default:"true"`
AutoUpgradeIntervalH int `xml:"autoUpgradeIntervalH" json:"autoUpgradeIntervalH" default:"12"` // 0 for off
UpgradeToPreReleases bool `xml:"upgradeToPreReleases" json:"upgradeToPreReleases"` // when auto upgrades are enabled
KeepTemporariesH int `xml:"keepTemporariesH" json:"keepTemporariesH" default:"24"` // 0 for off
CacheIgnoredFiles bool `xml:"cacheIgnoredFiles" json:"cacheIgnoredFiles" default:"false"`
ProgressUpdateIntervalS int `xml:"progressUpdateIntervalS" json:"progressUpdateIntervalS" default:"5"`
SymlinksEnabled bool `xml:"symlinksEnabled" json:"symlinksEnabled" default:"true"`
LimitBandwidthInLan bool `xml:"limitBandwidthInLan" json:"limitBandwidthInLan" default:"false"`
MinHomeDiskFreePct float64 `xml:"minHomeDiskFreePct" json:"minHomeDiskFreePct" default:"1"`
ReleasesURL string `xml:"releasesURL" json:"releasesURL" default:"https://upgrades.syncthing.net/meta.json"`
AlwaysLocalNets []string `xml:"alwaysLocalNet" json:"alwaysLocalNets"`
OverwriteRemoteDevNames bool `xml:"overwriteRemoteDeviceNamesOnConnect" json:"overwriteRemoteDeviceNamesOnConnect" default:"false"`
TempIndexMinBlocks int `xml:"tempIndexMinBlocks" json:"tempIndexMinBlocks" default:"10"`
UnackedNotificationIDs []string `xml:"unackedNotificationID" json:"unackedNotificationIDs"`
TrafficClass int `xml:"trafficClass" json:"trafficClass"`
ListenAddresses []string `xml:"listenAddress" json:"listenAddresses" default:"default"`
GlobalAnnServers []string `xml:"globalAnnounceServer" json:"globalAnnounceServers" json:"globalAnnounceServer" default:"default"`
GlobalAnnEnabled bool `xml:"globalAnnounceEnabled" json:"globalAnnounceEnabled" default:"true"`
LocalAnnEnabled bool `xml:"localAnnounceEnabled" json:"localAnnounceEnabled" default:"true"`
LocalAnnPort int `xml:"localAnnouncePort" json:"localAnnouncePort" default:"21027"`
LocalAnnMCAddr string `xml:"localAnnounceMCAddr" json:"localAnnounceMCAddr" default:"[ff12::8384]:21027"`
MaxSendKbps int `xml:"maxSendKbps" json:"maxSendKbps"`
MaxRecvKbps int `xml:"maxRecvKbps" json:"maxRecvKbps"`
ReconnectIntervalS int `xml:"reconnectionIntervalS" json:"reconnectionIntervalS" default:"60"`
RelaysEnabled bool `xml:"relaysEnabled" json:"relaysEnabled" default:"true"`
RelayReconnectIntervalM int `xml:"relayReconnectIntervalM" json:"relayReconnectIntervalM" default:"10"`
StartBrowser bool `xml:"startBrowser" json:"startBrowser" default:"true"`
NATEnabled bool `xml:"natEnabled" json:"natEnabled" default:"true"`
NATLeaseM int `xml:"natLeaseMinutes" json:"natLeaseMinutes" default:"60"`
NATRenewalM int `xml:"natRenewalMinutes" json:"natRenewalMinutes" default:"30"`
NATTimeoutS int `xml:"natTimeoutSeconds" json:"natTimeoutSeconds" default:"10"`
URAccepted int `xml:"urAccepted" json:"urAccepted"` // Accepted usage reporting version; 0 for off (undecided), -1 for off (permanently)
URUniqueID string `xml:"urUniqueID" json:"urUniqueId"` // Unique ID for reporting purposes, regenerated when UR is turned on.
URURL string `xml:"urURL" json:"urURL" default:"https://data.syncthing.net/newdata"`
URPostInsecurely bool `xml:"urPostInsecurely" json:"urPostInsecurely" default:"false"` // For testing
URInitialDelayS int `xml:"urInitialDelayS" json:"urInitialDelayS" default:"1800"`
RestartOnWakeup bool `xml:"restartOnWakeup" json:"restartOnWakeup" default:"true"`
AutoUpgradeIntervalH int `xml:"autoUpgradeIntervalH" json:"autoUpgradeIntervalH" default:"12"` // 0 for off
UpgradeToPreReleases bool `xml:"upgradeToPreReleases" json:"upgradeToPreReleases"` // when auto upgrades are enabled
KeepTemporariesH int `xml:"keepTemporariesH" json:"keepTemporariesH" default:"24"` // 0 for off
CacheIgnoredFiles bool `xml:"cacheIgnoredFiles" json:"cacheIgnoredFiles" default:"false"`
ProgressUpdateIntervalS int `xml:"progressUpdateIntervalS" json:"progressUpdateIntervalS" default:"5"`
SymlinksEnabled bool `xml:"symlinksEnabled" json:"symlinksEnabled" default:"true"`
LimitBandwidthInLan bool `xml:"limitBandwidthInLan" json:"limitBandwidthInLan" default:"false"`
MinHomeDiskFreePct float64 `xml:"minHomeDiskFreePct" json:"minHomeDiskFreePct" default:"1"`
ReleasesURL string `xml:"releasesURL" json:"releasesURL" default:"https://upgrades.syncthing.net/meta.json"`
AlwaysLocalNets []string `xml:"alwaysLocalNet" json:"alwaysLocalNets"`
OverwriteRemoteDevNames bool `xml:"overwriteRemoteDeviceNamesOnConnect" json:"overwriteRemoteDeviceNamesOnConnect" default:"false"`
TempIndexMinBlocks int `xml:"tempIndexMinBlocks" json:"tempIndexMinBlocks" default:"10"`
UnackedNotificationIDs []string `xml:"unackedNotificationID" json:"unackedNotificationIDs"`
TrafficClass int `xml:"trafficClass" json:"trafficClass"`
WeakHashSelectionMethod WeakHashSelectionMethod `xml:"weakHashSelectionMethod" json:"weakHashSelectionMethod"`
DeprecatedUPnPEnabled bool `xml:"upnpEnabled,omitempty" json:"-"`
DeprecatedUPnPLeaseM int `xml:"upnpLeaseMinutes,omitempty" json:"-"`

1
lib/config/testdata/overridenvalues.xml vendored
View File

@ -34,5 +34,6 @@
<releasesURL>https://localhost/releases</releasesURL>
<overwriteRemoteDeviceNamesOnConnect>true</overwriteRemoteDeviceNamesOnConnect>
<tempIndexMinBlocks>100</tempIndexMinBlocks>
<weakHashSelectionMethod>never</weakHashSelectionMethod>
</options>
</configuration>

3
lib/model/model.go
View File

@ -37,6 +37,7 @@ import (
"github.com/syncthing/syncthing/lib/sync"
"github.com/syncthing/syncthing/lib/upgrade"
"github.com/syncthing/syncthing/lib/versioner"
"github.com/syncthing/syncthing/lib/weakhash"
"github.com/thejerf/suture"
)
@ -1813,7 +1814,7 @@ func (m *Model) internalScanFolderSubdirs(folder string, subDirs []string) error
ShortID: m.shortID,
ProgressTickIntervalS: folderCfg.ScanProgressIntervalS,
Cancel: cancel,
UseWeakHashes: folderCfg.WeakHashThresholdPct < 100,
UseWeakHashes: weakhash.Enabled,
})
if err != nil {

37
lib/model/rwfolder.go
View File

@ -1221,23 +1221,34 @@ func (f *sendReceiveFolder) copierRoutine(in <-chan copyBlocksState, pullChan ch
f.model.fmut.RUnlock()
var weakHashFinder *weakhash.Finder
blocksPercentChanged := 0
if tot := len(state.file.Blocks); tot > 0 {
blocksPercentChanged = (tot - state.have) * 100 / tot
}
if blocksPercentChanged >= f.WeakHashThresholdPct {
hashesToFind := make([]uint32, 0, len(state.blocks))
for _, block := range state.blocks {
if block.WeakHash != 0 {
hashesToFind = append(hashesToFind, block.WeakHash)
if weakhash.Enabled {
blocksPercentChanged := 0
if tot := len(state.file.Blocks); tot > 0 {
blocksPercentChanged = (tot - state.have) * 100 / tot
}
if blocksPercentChanged >= f.WeakHashThresholdPct {
hashesToFind := make([]uint32, 0, len(state.blocks))
for _, block := range state.blocks {
if block.WeakHash != 0 {
hashesToFind = append(hashesToFind, block.WeakHash)
}
}
}
weakHashFinder, err = weakhash.NewFinder(state.realName, protocol.BlockSize, hashesToFind)
if err != nil {
l.Debugln("weak hasher", err)
if len(hashesToFind) > 0 {
weakHashFinder, err = weakhash.NewFinder(state.realName, protocol.BlockSize, hashesToFind)
if err != nil {
l.Debugln("weak hasher", err)
}
} else {
l.Debugf("not weak hashing %s. file did not contain any weak hashes", state.file.Name)
}
} else {
l.Debugf("not weak hashing %s. not enough changed %.02f < %d", state.file.Name, blocksPercentChanged, f.WeakHashThresholdPct)
}
} else {
l.Debugf("not weak hashing %s. weak hashing disabled", state.file.Name)
}
for _, block := range state.blocks {

11
lib/util/utils.go
View File

@ -25,6 +25,17 @@ func SetDefaults(data interface{}) error {
v := tag.Get("default")
if len(v) > 0 {
if parser, ok := f.Interface().(interface {
ParseDefault(string) (interface{}, error)
}); ok {
val, err := parser.ParseDefault(v)
if err != nil {
panic(err)
}
f.Set(reflect.ValueOf(val))
continue
}
switch f.Interface().(type) {
case string:
f.SetString(v)

21
lib/util/utils_test.go
View File

@ -8,12 +8,21 @@ package util
import "testing"
type Defaulter struct {
Value string
}
func (Defaulter) ParseDefault(v string) (interface{}, error) {
return Defaulter{Value: v}, nil
}
func TestSetDefaults(t *testing.T) {
x := &struct {
A string `default:"string"`
B int `default:"2"`
C float64 `default:"2.2"`
D bool `default:"true"`
A string `default:"string"`
B int `default:"2"`
C float64 `default:"2.2"`
D bool `default:"true"`
E Defaulter `default:"defaulter"`
}{}
if x.A != "" {
@ -24,6 +33,8 @@ func TestSetDefaults(t *testing.T) {
t.Errorf("float failed")
} else if x.D {
t.Errorf("bool failed")
} else if x.E.Value != "" {
t.Errorf("defaulter failed")
}
if err := SetDefaults(x); err != nil {
@ -38,6 +49,8 @@ func TestSetDefaults(t *testing.T) {
t.Errorf("float failed")
} else if !x.D {
t.Errorf("bool failed")
} else if x.E.Value != "defaulter" {
t.Errorf("defaulter failed")
}
}

8
lib/weakhash/weakhash.go
View File

@ -21,11 +21,15 @@ const (
maxWeakhashFinderHits = 10
)
var (
Enabled = true
)
// Find finds all the blocks of the given size within io.Reader that matches
// the hashes provided, and returns a hash -> slice of offsets within reader
// map, that produces the same weak hash.
func Find(ir io.Reader, hashesToFind []uint32, size int) (map[uint32][]int64, error) {
if ir == nil {
if ir == nil || len(hashesToFind) == 0 {
return nil, nil
}
@ -45,7 +49,7 @@ func Find(ir io.Reader, hashesToFind []uint32, size int) (map[uint32][]int64, er
offsets := make(map[uint32][]int64)
for _, hashToFind := range hashesToFind {
offsets[hashToFind] = nil
offsets[hashToFind] = make([]int64, 0, maxWeakhashFinderHits)
}
var i int64