diff --git a/cmd/stdiscosrv/stdiscosrv b/cmd/stdiscosrv/stdiscosrv new file mode 100755 index 000000000..86caf06d7 Binary files /dev/null and b/cmd/stdiscosrv/stdiscosrv differ diff --git a/cmd/syncthing/main.go b/cmd/syncthing/main.go index d40d4078c..4e7bae210 100644 --- a/cmd/syncthing/main.go +++ b/cmd/syncthing/main.go @@ -41,6 +41,7 @@ import ( "github.com/syncthing/syncthing/lib/osutil" "github.com/syncthing/syncthing/lib/protocol" "github.com/syncthing/syncthing/lib/rand" + "github.com/syncthing/syncthing/lib/sha256" "github.com/syncthing/syncthing/lib/symlinks" "github.com/syncthing/syncthing/lib/tlsutil" "github.com/syncthing/syncthing/lib/upgrade" @@ -166,6 +167,11 @@ are mostly useful for developers. Use with care. STNOUPGRADE Disable automatic upgrades. + STHASHING Select the SHA256 hashing package to use. Possible values + are "standard" for the Go standard library implementation, + "minio" for the github.com/minio/sha256-simd implementation, + and blank (the default) for auto detection. + GOMAXPROCS Set the maximum number of CPU cores to use. Defaults to all available CPU cores. @@ -567,7 +573,9 @@ func syncthingMain(runtimeOptions RuntimeOptions) { l.Infoln(LongVersion) l.Infoln("My ID:", myID) - printHashRate() + + sha256.SelectAlgo() + sha256.Report() // Emit the Starting event, now that we know who we are. @@ -840,22 +848,6 @@ func setupSignalHandling() { }() } -// printHashRate prints the hashing performance in MB/s, formatting it with -// appropriate precision for the value, i.e. 182 MB/s, 18 MB/s, 1.8 MB/s, 0.18 -// MB/s. -func printHashRate() { - hashRate := cpuBench(3, 100*time.Millisecond) - - decimals := 0 - if hashRate < 1 { - decimals = 2 - } else if hashRate < 10 { - decimals = 1 - } - - l.Infof("Single thread hash performance is ~%.*f MB/s", decimals, hashRate) -} - func loadConfig() (*config.Wrapper, error) { cfgFile := locations[locConfigFile] cfg, err := config.Load(cfgFile, myID) diff --git a/cmd/syncthing/monitor.go b/cmd/syncthing/monitor.go index 2c10bf003..9f9d64b46 100644 --- a/cmd/syncthing/monitor.go +++ b/cmd/syncthing/monitor.go @@ -178,6 +178,22 @@ func copyStderr(stderr io.Reader, dst io.Writer) { if panicFd == nil { dst.Write([]byte(line)) + if strings.Contains(line, "SIGILL") { + l.Warnln(` +******************************************************************************* +* Crash due to illegal instruction detected. This is most likely due to a CPU * +* incompatibility with the high performance hashing package. Switching to the * +* standard hashing package instead. Please report this issue at: * +* * +* https://github.com/syncthing/syncthing/issues * +* * +* Include the details of your CPU. * +******************************************************************************* +`) + os.Setenv("STHASHING", "standard") + return + } + if strings.HasPrefix(line, "panic:") || strings.HasPrefix(line, "fatal error:") { panicFd, err = os.Create(timestampedLoc(locPanicLog)) if err != nil { diff --git a/cmd/syncthing/usage_report.go b/cmd/syncthing/usage_report.go index 26a2b6842..9cfb869a1 100644 --- a/cmd/syncthing/usage_report.go +++ b/cmd/syncthing/usage_report.go @@ -18,12 +18,11 @@ import ( "strings" "time" - "github.com/minio/sha256-simd" - "github.com/syncthing/syncthing/lib/config" "github.com/syncthing/syncthing/lib/dialer" "github.com/syncthing/syncthing/lib/model" "github.com/syncthing/syncthing/lib/protocol" + "github.com/syncthing/syncthing/lib/sha256" "github.com/syncthing/syncthing/lib/upgrade" "github.com/thejerf/suture" ) diff --git a/lib/protocol/bep_extensions.go b/lib/protocol/bep_extensions.go index 61f791b02..12e0c88f7 100644 --- a/lib/protocol/bep_extensions.go +++ b/lib/protocol/bep_extensions.go @@ -12,9 +12,8 @@ import ( "fmt" "time" - "github.com/minio/sha256-simd" - "github.com/syncthing/syncthing/lib/rand" + "github.com/syncthing/syncthing/lib/sha256" ) const ( diff --git a/lib/protocol/deviceid.go b/lib/protocol/deviceid.go index 961689b3b..919ce8003 100644 --- a/lib/protocol/deviceid.go +++ b/lib/protocol/deviceid.go @@ -11,7 +11,7 @@ import ( "regexp" "strings" - "github.com/minio/sha256-simd" + "github.com/syncthing/syncthing/lib/sha256" "github.com/calmh/luhn" ) diff --git a/lib/scanner/blocks.go b/lib/scanner/blocks.go index aac00900a..3d9919ff4 100644 --- a/lib/scanner/blocks.go +++ b/lib/scanner/blocks.go @@ -11,9 +11,8 @@ import ( "fmt" "io" - "github.com/minio/sha256-simd" - "github.com/syncthing/syncthing/lib/protocol" + "github.com/syncthing/syncthing/lib/sha256" ) var SHA256OfNothing = []uint8{0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a, 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, 0x27, 0xae, 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, 0xa4, 0x95, 0x99, 0x1b, 0x78, 0x52, 0xb8, 0x55} diff --git a/lib/sha256/sha256.go b/lib/sha256/sha256.go new file mode 100644 index 000000000..469167573 --- /dev/null +++ b/lib/sha256/sha256.go @@ -0,0 +1,136 @@ +// Copyright (C) 2016 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package sha256 + +import ( + "crypto/rand" + cryptoSha256 "crypto/sha256" + "fmt" + "hash" + "os" + "time" + + minioSha256 "github.com/minio/sha256-simd" + "github.com/syncthing/syncthing/lib/logger" +) + +var l = logger.DefaultLogger.NewFacility("sha256", "SHA256 hashing package") + +const ( + benchmarkingIterations = 3 + benchmarkingDuration = 150 * time.Millisecond + defaultImpl = "crypto/sha256" + minioImpl = "minio/sha256-simd" +) + +const ( + BlockSize = cryptoSha256.BlockSize + Size = cryptoSha256.Size +) + +// May be switched out for another implementation +var ( + New = cryptoSha256.New + Sum256 = cryptoSha256.Sum256 +) + +var ( + selectedImpl = defaultImpl + cryptoPerf float64 + minioPerf float64 +) + +func SelectAlgo() { + switch os.Getenv("STHASHING") { + case "": + // When unset, probe for the fastest implementation. + benchmark() + if minioPerf > cryptoPerf { + selectMinio() + } + + case "minio": + // When set to "minio", use that. Benchmark anyway to be able to + // present the difference. + benchmark() + selectMinio() + + default: + // When set to anything else, such as "standard", use the default Go + // implementation. Benchmark that anyway, so we can report something + // useful in Report(). Make sure not to touch the minio + // implementation as it may be disabled for incompatibility reasons. + cryptoPerf = cpuBenchOnce(benchmarkingIterations*benchmarkingDuration, cryptoSha256.New) + } +} + +// Report prints a line with the measured hash performance rates for the +// selected and alternate implementation. +func Report() { + var otherImpl string + var selectedRate, otherRate float64 + + switch selectedImpl { + case defaultImpl: + selectedRate = cryptoPerf + otherRate = minioPerf + otherImpl = minioImpl + + case minioImpl: + selectedRate = minioPerf + otherRate = cryptoPerf + otherImpl = defaultImpl + } + + l.Infof("Single thread hash performance is %s using %s (%s using %s).", formatRate(selectedRate), selectedImpl, formatRate(otherRate), otherImpl) +} + +func selectMinio() { + New = minioSha256.New + Sum256 = minioSha256.Sum256 + selectedImpl = minioImpl +} + +func benchmark() { + // Interleave the tests to achieve some sort of fairness if the CPU is + // just in the process of spinning up to full speed. + for i := 0; i < benchmarkingIterations; i++ { + if perf := cpuBenchOnce(benchmarkingDuration, cryptoSha256.New); perf > cryptoPerf { + cryptoPerf = perf + } + if perf := cpuBenchOnce(benchmarkingDuration, minioSha256.New); perf > minioPerf { + minioPerf = perf + } + } +} + +func cpuBenchOnce(duration time.Duration, newFn func() hash.Hash) float64 { + chunkSize := 100 * 1 << 10 + h := newFn() + bs := make([]byte, chunkSize) + rand.Reader.Read(bs) + + t0 := time.Now() + b := 0 + for time.Since(t0) < duration { + h.Write(bs) + b += chunkSize + } + h.Sum(nil) + d := time.Since(t0) + return float64(int(float64(b)/d.Seconds()/(1<<20)*100)) / 100 +} + +func formatRate(rate float64) string { + decimals := 0 + if rate < 1 { + decimals = 2 + } else if rate < 10 { + decimals = 1 + } + return fmt.Sprintf("%.*f MB/s", decimals, rate) +} diff --git a/lib/signature/signature.go b/lib/signature/signature.go index a0afb641a..c73b70382 100644 --- a/lib/signature/signature.go +++ b/lib/signature/signature.go @@ -20,7 +20,7 @@ import ( "io" "math/big" - "github.com/minio/sha256-simd" + "github.com/syncthing/syncthing/lib/sha256" ) // GenerateKeys returns a new key pair, with the private and public key