octoleo/syncthing
1
0
Fork 0
mirror of https://github.com/octoleo/syncthing.git synced 2024-11-17 10:35:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

lib/sha256: Remove it (#9643)

Browse Source 
### Purpose

Remove the `lib/sha256` package, because it's no longer necessary. Go's
standard library now has the same performance and is on par with
`sha256-simd` since [Since Go
1.21](1a64574f42).
Therefore using `sha256-simd` has no benefits anymore.

ARM already has optimized sha256 assembly code since
7b8a7f8272,
`sha256-simd` published their results before that optimized assembly was
implemented,
f941fedda8.
The assembly looks very similar and the benchmarks in the Go commit
match that of `sha256-simd`.

This patch removes all of the related code of `lib/sha256` and makes
`crypto/sha256` the 'default'.

Benchmark of `sha256-simd` and `crypto/sha256`:
<details>

```
cpu: AMD Ryzen 5 3600X 6-Core Processor
                │  simd.txt   │               go.txt                │
                │   sec/op    │    sec/op     vs base               │
Hash/8Bytes-12    63.25n ± 1%    73.38n ± 1%  +16.02% (p=0.002 n=6)
Hash/64Bytes-12   98.73n ± 1%   105.30n ± 1%   +6.65% (p=0.002 n=6)
Hash/1K-12        567.2n ± 1%    572.8n ± 1%   +0.99% (p=0.002 n=6)
Hash/8K-12        4.062µ ± 1%    4.062µ ± 1%        ~ (p=0.396 n=6)
Hash/1M-12        512.1µ ± 0%    510.6µ ± 1%        ~ (p=0.485 n=6)
Hash/5M-12        2.556m ± 1%    2.564m ± 0%        ~ (p=0.093 n=6)
Hash/10M-12       5.112m ± 0%    5.127m ± 0%        ~ (p=0.093 n=6)
geomean           13.82µ         14.27µ        +3.28%

                │   simd.txt   │               go.txt                │
                │     B/s      │     B/s       vs base               │
Hash/8Bytes-12    120.6Mi ± 1%   104.0Mi ± 1%  -13.81% (p=0.002 n=6)
Hash/64Bytes-12   618.2Mi ± 1%   579.8Mi ± 1%   -6.22% (p=0.002 n=6)
Hash/1K-12        1.682Gi ± 1%   1.665Gi ± 1%   -0.98% (p=0.002 n=6)
Hash/8K-12        1.878Gi ± 1%   1.878Gi ± 1%        ~ (p=0.310 n=6)
Hash/1M-12        1.907Gi ± 0%   1.913Gi ± 1%        ~ (p=0.485 n=6)
Hash/5M-12        1.911Gi ± 1%   1.904Gi ± 0%        ~ (p=0.093 n=6)
Hash/10M-12       1.910Gi ± 0%   1.905Gi ± 0%        ~ (p=0.093 n=6)
geomean           1.066Gi        1.032Gi        -3.18%
```

</details>


### Testing

Compiled and tested on Linux.

### Documentation

https://github.com/syncthing/docs/pull/874
This commit is contained in:
Gusted 2024-08-10 13:58:20 +02:00 committed by GitHub
parent 19693734a3
commit 356c5055ad
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
26 changed files with 17 additions and 217 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/stcompdirs/main.go
View File

@ -7,6 +7,7 @@
package main
import (
"crypto/sha256"
"errors"
"flag"
"fmt"
@ -16,7 +17,6 @@ import (
"path/filepath"
_ "github.com/syncthing/syncthing/lib/automaxprocs"
"github.com/syncthing/syncthing/lib/sha256"
)
func main() {

2
cmd/stcrashreceiver/main.go
View File

@ -14,6 +14,7 @@ package main
import (
"context"
"crypto/sha256"
"encoding/json"
"fmt"
"io"
@ -29,7 +30,6 @@ import (
"github.com/prometheus/client_golang/prometheus/promhttp"
_ "github.com/syncthing/syncthing/lib/automaxprocs"
"github.com/syncthing/syncthing/lib/build"
"github.com/syncthing/syncthing/lib/sha256"
"github.com/syncthing/syncthing/lib/ur"
)

3
cmd/stcrashreceiver/util.go
View File

@ -9,14 +9,13 @@ package main
import (
"bytes"
"compress/gzip"
"crypto/sha256"
"fmt"
"net"
"net/http"
"os"
"path/filepath"
"time"
"github.com/syncthing/syncthing/lib/sha256"
)
// userIDFor returns a string we can use as the user ID for the purpose of

2
cmd/stwatchfile/main.go
View File

@ -7,6 +7,7 @@
package main
import (
"crypto/sha256"
"flag"
"fmt"
"io"
@ -14,7 +15,6 @@ import (
"time"
_ "github.com/syncthing/syncthing/lib/automaxprocs"
"github.com/syncthing/syncthing/lib/sha256"
)
func main() {

3
cmd/syncthing/crash_reporting.go
View File

@ -9,6 +9,7 @@ package main
import (
"bytes"
"context"
"crypto/sha256"
"fmt"
"net/http"
"os"
@ -16,8 +17,6 @@ import (
"sort"
"strings"
"time"
"github.com/syncthing/syncthing/lib/sha256"
)
const (

5
cmd/syncthing/main.go
View File

@ -92,11 +92,6 @@ above.
STLOCKTHRESHOLD Used for debugging internal deadlocks; sets debug
sensitivity. Use only under direction of a developer.
STHASHING Select the SHA256 hashing package to use. Possible values
are "standard" for the Go standard library implementation,
"minio" for the github.com/minio/sha256-simd implementation,
and blank (the default) for auto detection.
STVERSIONEXTRA Add extra information to the version string in logs and the
version line in the GUI. Can be set to the name of a wrapper
or tool controlling syncthing to communicate this to the end

16
cmd/syncthing/monitor.go
View File

@ -241,22 +241,6 @@ func copyStderr(stderr io.Reader, dst io.Writer) {
if panicFd == nil {
dst.Write([]byte(line))
if strings.Contains(line, "SIGILL") {
l.Warnln(`
*******************************************************************************
* Crash due to illegal instruction detected. This is most likely due to a CPU *
* incompatibility with the high performance hashing package. Switching to the *
* standard hashing package instead. Please report this issue at: *
* *
* https://github.com/syncthing/syncthing/issues *
* *
* Include the details of your CPU. *
*******************************************************************************
`)
os.Setenv("STHASHING", "standard")
return
}
if strings.HasPrefix(line, "panic:") || strings.HasPrefix(line, "fatal error:") {
panicFd, err = os.Create(locations.GetTimestamped(locations.PanicLog))
if err != nil {

2
go.mod
View File

@ -25,7 +25,6 @@ require (
github.com/maruel/panicparse/v2 v2.3.1
github.com/maxbrunsfeld/counterfeiter/v6 v6.8.1
github.com/maxmind/geoipupdate/v6 v6.1.0
github.com/minio/sha256-simd v1.0.1
github.com/miscreant/miscreant.go v0.0.0-20200214223636-26d376326b75
github.com/oschwald/geoip2-golang v1.11.0
github.com/pierrec/lz4/v4 v4.1.21
@ -68,7 +67,6 @@ require (
github.com/google/uuid v1.6.0 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/klauspost/cpuid/v2 v2.2.8 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/nxadm/tail v1.4.11 // indirect
github.com/onsi/ginkgo/v2 v2.20.0 // indirect

4
go.sum
View File

@ -130,8 +130,6 @@ github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNU
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM=
github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
@ -147,8 +145,6 @@ github.com/maxbrunsfeld/counterfeiter/v6 v6.8.1/go.mod h1:eyp4DdUJAKkr9tvxR3jWhw
github.com/maxmind/geoipupdate/v6 v6.1.0 h1:sdtTHzzQNJlXF5+fd/EoPTucRHyMonYt/Cok8xzzfqA=
github.com/maxmind/geoipupdate/v6 v6.1.0/go.mod h1:cZYCDzfMzTY4v6dKRdV7KTB6SStxtn3yFkiJ1btTGGc=
github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM=
github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8=
github.com/miscreant/miscreant.go v0.0.0-20200214223636-26d376326b75 h1:cUVxyR+UfmdEAZGJ8IiKld1O0dbGotEnkMolG5hfMSY=
github.com/miscreant/miscreant.go v0.0.0-20200214223636-26d376326b75/go.mod h1:pBbZyGwC5i16IBkjVKoy/sznA8jPD/K9iedwe1ESE6w=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=

1
gui/default/syncthing/core/aboutModalView.html
View File

@ -65,7 +65,6 @@ Jakob Borg, Audrius Butkevicius, Jesse Lucas, Simon Frei, Tomasz Wilczyński, Al
<li><a href="https://github.com/lib/pq">lib/pq</a>, Copyright &copy; 2011-2013, 'pq' Contributors, portions Copyright &copy; 2011 Blake Mizerany.</li>
<li><a href="https://github.com/mattn/go-isatty">mattn/go-isatty</a>, Copyright &copy; Yasuhiro MATSUMOTO.</li>
<li><a href="https://github.com/matttproud/golang_protobuf_extensions">matttproud/golang_protobuf_extensions</a>, Copyright &copy; 2012 Matt T. Proud.</li>
<li><a href="https://github.com/minio/sha256-simd">minio/sha256-simd</a>, Copyright &copy; 2016-2017 Minio, Inc.</li>
<li><a href="https://github.com/oschwald/geoip2-golang">oschwald/geoip2-golang</a>, Copyright &copy; 2015, Gregory J. Oschwald.</li>
<li><a href="https://github.com/oschwald/maxminddb-golang">oschwald/maxminddb-golang</a>, Copyright &copy; 2015, Gregory J. Oschwald.</li>
<li><a href="https://github.com/petermattis/goid">petermattis/goid</a>, Copyright &copy; 2015-2016 Peter Mattis.</li>

1
lib/build/build.go
View File

@ -40,7 +40,6 @@ var (
envTags = []string{
"STGUIASSETS",
"STHASHING",
"STNORESTART",
"STNOUPGRADE",
}

2
lib/db/lowlevel.go
View File

@ -9,6 +9,7 @@ package db
import (
"bytes"
"context"
"crypto/sha256"
"encoding/binary"
"errors"
"fmt"
@ -22,7 +23,6 @@ import (
"github.com/syncthing/syncthing/lib/events"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/sha256"
"github.com/syncthing/syncthing/lib/stringutil"
"github.com/syncthing/syncthing/lib/svcutil"
"github.com/syncthing/syncthing/lib/sync"

2
lib/fs/tempname.go
View File

@ -7,12 +7,12 @@
package fs
import (
"crypto/sha256"
"fmt"
"path/filepath"
"strings"
"github.com/syncthing/syncthing/lib/build"
"github.com/syncthing/syncthing/lib/sha256"
)
const (

2
lib/ignore/ignore.go
View File

@ -9,6 +9,7 @@ package ignore
import (
"bufio"
"bytes"
"crypto/sha256"
"errors"
"fmt"
"io"
@ -21,7 +22,6 @@ import (
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/ignore/ignoreresult"
"github.com/syncthing/syncthing/lib/osutil"
"github.com/syncthing/syncthing/lib/sha256"
"github.com/syncthing/syncthing/lib/sync"
)

2
lib/model/folder_sendrecv.go
View File

@ -9,6 +9,7 @@ package model
import (
"bytes"
"context"
"crypto/sha256"
"errors"
"fmt"
"io"
@ -28,7 +29,6 @@ import (
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/scanner"
"github.com/syncthing/syncthing/lib/semaphore"
"github.com/syncthing/syncthing/lib/sha256"
"github.com/syncthing/syncthing/lib/sync"
"github.com/syncthing/syncthing/lib/versioner"
"github.com/syncthing/syncthing/lib/weakhash"

2
lib/protocol/bep_extensions.go
View File

@ -4,6 +4,7 @@ package protocol
import (
"bytes"
"crypto/sha256"
"encoding/binary"
"encoding/json"
"errors"
@ -12,7 +13,6 @@ import (
"github.com/syncthing/syncthing/lib/build"
"github.com/syncthing/syncthing/lib/rand"
"github.com/syncthing/syncthing/lib/sha256"
)
const (

3
lib/protocol/deviceid.go
View File

@ -4,13 +4,12 @@ package protocol
import (
"bytes"
"crypto/sha256"
"encoding/base32"
"encoding/binary"
"errors"
"fmt"
"strings"
"github.com/syncthing/syncthing/lib/sha256"
)
const (

2
lib/protocol/encryption.go
View File

@ -8,6 +8,7 @@ package protocol
import (
"context"
"crypto/sha256"
"encoding/base32"
"encoding/binary"
"errors"
@ -20,7 +21,6 @@ import (
lru "github.com/hashicorp/golang-lru/v2"
"github.com/miscreant/miscreant.go"
"github.com/syncthing/syncthing/lib/rand"
"github.com/syncthing/syncthing/lib/sha256"
"golang.org/x/crypto/chacha20poly1305"
"golang.org/x/crypto/hkdf"
"golang.org/x/crypto/scrypt"

2
lib/scanner/blocks.go
View File

@ -9,12 +9,12 @@ package scanner
import (
"bytes"
"context"
"crypto/sha256"
"hash"
"hash/adler32"
"io"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/sha256"
)
var SHA256OfNothing = []uint8{0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a, 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, 0x27, 0xae, 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, 0xa4, 0x95, 0x99, 0x1b, 0x78, 0x52, 0xb8, 0x55}

2
lib/scanner/blocks_test.go
View File

@ -10,6 +10,7 @@ import (
"bytes"
"context"
"crypto/rand"
"crypto/sha256"
"fmt"
origAdler32 "hash/adler32"
mrand "math/rand"
@ -18,7 +19,6 @@ import (
rollingAdler32 "github.com/chmduquesne/rollinghash/adler32"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/sha256"
)
var blocksTestData = []struct {

2
lib/scanner/walk_test.go
View File

@ -9,6 +9,7 @@ package scanner
import (
"bytes"
"context"
"crypto/sha256"
"errors"
"fmt"
"io"
@ -26,7 +27,6 @@ import (
"github.com/syncthing/syncthing/lib/ignore"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/rand"
"github.com/syncthing/syncthing/lib/sha256"
"golang.org/x/text/unicode/norm"
)

157
lib/sha256/sha256.go
View File

@ -1,157 +0,0 @@
// Copyright (C) 2016 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package sha256
import (
cryptoSha256 "crypto/sha256"
"encoding/hex"
"fmt"
"hash"
"math/rand"
"os"
"time"
minioSha256 "github.com/minio/sha256-simd"
"github.com/syncthing/syncthing/lib/logger"
)
var l = logger.DefaultLogger.NewFacility("sha256", "SHA256 hashing package")
const (
benchmarkingIterations = 3
benchmarkingDuration = 150 * time.Millisecond
defaultImpl = "crypto/sha256"
minioImpl = "minio/sha256-simd"
Size = cryptoSha256.Size
)
// May be switched out for another implementation
var (
New = cryptoSha256.New
Sum256 = cryptoSha256.Sum256
)
var (
selectedImpl = defaultImpl
cryptoPerf float64
minioPerf float64
)
func SelectAlgo() {
switch os.Getenv("STHASHING") {
case "":
// When unset, probe for the fastest implementation.
benchmark()
if minioPerf > cryptoPerf {
selectMinio()
}
case "minio":
// When set to "minio", use that.
selectMinio()
default:
// When set to anything else, such as "standard", use the default Go
// implementation. Make sure not to touch the minio
// implementation as it may be disabled for incompatibility reasons.
}
verifyCorrectness()
}
// Report prints a line with the measured hash performance rates for the
// selected and alternate implementation.
func Report() {
var otherImpl string
var selectedRate, otherRate float64
switch selectedImpl {
case defaultImpl:
selectedRate = cryptoPerf
otherRate = minioPerf
otherImpl = minioImpl
case minioImpl:
selectedRate = minioPerf
otherRate = cryptoPerf
otherImpl = defaultImpl
}
if selectedRate == 0 {
return
}
l.Infof("Single thread SHA256 performance is %s using %s (%s using %s).", formatRate(selectedRate), selectedImpl, formatRate(otherRate), otherImpl)
}
func selectMinio() {
New = minioSha256.New
Sum256 = minioSha256.Sum256
selectedImpl = minioImpl
}
func benchmark() {
// Interleave the tests to achieve some sort of fairness if the CPU is
// just in the process of spinning up to full speed.
for i := 0; i < benchmarkingIterations; i++ {
if perf := cpuBenchOnce(benchmarkingDuration, cryptoSha256.New); perf > cryptoPerf {
cryptoPerf = perf
}
if perf := cpuBenchOnce(benchmarkingDuration, minioSha256.New); perf > minioPerf {
minioPerf = perf
}
}
}
func cpuBenchOnce(duration time.Duration, newFn func() hash.Hash) float64 {
chunkSize := 100 * 1 << 10
h := newFn()
bs := make([]byte, chunkSize)
r := rand.New(rand.NewSource(time.Now().UnixNano()))
r.Read(bs)
t0 := time.Now()
b := 0
for time.Since(t0) < duration {
h.Write(bs)
b += chunkSize
}
h.Sum(nil)
d := time.Since(t0)
return float64(int(float64(b)/d.Seconds()/(1<<20)*100)) / 100
}
func formatRate(rate float64) string {
decimals := 0
if rate < 1 {
decimals = 2
} else if rate < 10 {
decimals = 1
}
return fmt.Sprintf("%.*f MB/s", decimals, rate)
}
func verifyCorrectness() {
// The currently selected algo should in fact perform a SHA256 calculation.
// $ echo "Syncthing Magic Testing Value" | openssl dgst -sha256 -hex
correct := "87f6cfd24131724c6ec43495594c5c22abc7d2b86bcc134bc6f10b7ec3dda4ee"
input := "Syncthing Magic Testing Value\n"
h := New()
h.Write([]byte(input))
sum := hex.EncodeToString(h.Sum(nil))
if sum != correct {
panic("sha256 is broken")
}
arr := Sum256([]byte(input))
sum = hex.EncodeToString(arr[:])
if sum != correct {
panic("sha256 is broken")
}
}

2
lib/signature/signature.go
View File

@ -11,6 +11,7 @@ package signature
import (
"crypto/ecdsa"
"crypto/elliptic"
"crypto/sha256"
"crypto/x509"
"encoding/asn1"
"encoding/pem"
@ -20,7 +21,6 @@ import (
"math/big"
"github.com/syncthing/syncthing/lib/rand"
"github.com/syncthing/syncthing/lib/sha256"
)
// GenerateKeys returns a new key pair, with the private and public key

6
lib/syncthing/syncthing.go
View File

@ -37,7 +37,6 @@ import (
"github.com/syncthing/syncthing/lib/osutil"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/rand"
"github.com/syncthing/syncthing/lib/sha256"
"github.com/syncthing/syncthing/lib/svcutil"
"github.com/syncthing/syncthing/lib/tlsutil"
"github.com/syncthing/syncthing/lib/upgrade"
@ -153,11 +152,6 @@ func (a *App) startup() error {
l.SetPrefix(fmt.Sprintf("[%s] ", a.myID.String()[:5]))
l.Infoln("My ID:", a.myID)
// Select SHA256 implementation and report. Affected by the
// STHASHING environment variable.
sha256.SelectAlgo()
sha256.Report()
// Emit the Starting event, now that we know who we are.
a.evLogger.Log(events.Starting, map[string]string{

5
man/syncthing.1
View File

@ -555,11 +555,6 @@ passing \fI\%\-\-gui\-apikey\fP\&.
Directory to load GUI assets from. Overrides compiled in assets. Useful for
developing webgui, commonly use \fBSTGUIASSETS=gui bin/syncthing\fP\&.
.TP
.B STHASHING
Specify which hashing package to use. Defaults to automatic based on
performance. Specify “minio” (compatibility) or “standard” for the default
Go implementation.
.TP
.B STHEAPPROFILE
Write heap profiles to \fBheap\-$pid\-$timestamp.pprof\fP each time heap usage
increases.

2
test/util.go
View File

@ -11,6 +11,7 @@ package integration
import (
cr "crypto/rand"
"crypto/sha256"
"errors"
"fmt"
"io"
@ -27,7 +28,6 @@ import (
"github.com/syncthing/syncthing/lib/build"
"github.com/syncthing/syncthing/lib/rc"
"github.com/syncthing/syncthing/lib/sha256"
)
func init() {