From 42a3db05b02bf3dd23b75c8d1ab5135ab74ae3cb Mon Sep 17 00:00:00 2001 From: greatroar <@> Date: Thu, 19 Mar 2020 11:27:19 +0100 Subject: [PATCH 1/2] Use Minio's optimized SHA-256 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit internal/repository benchmarks on an Intel i7-3770k: name old speed new speed delta PackerManager-8 209MB/s ± 1% 291MB/s ± 1% +38.94% (p=0.008 n=5+5) SaveAndEncrypt-8 112MB/s ± 1% 135MB/s ± 1% +20.25% (p=0.008 n=5+5) --- cmd/restic/cmd_stats.go | 3 ++- go.mod | 1 + internal/repository/packer_manager.go | 3 ++- internal/repository/repository.go | 3 ++- internal/restic/id.go | 3 ++- 5 files changed, 9 insertions(+), 4 deletions(-) diff --git a/cmd/restic/cmd_stats.go b/cmd/restic/cmd_stats.go index 6587f8382..6d3d7bd3d 100644 --- a/cmd/restic/cmd_stats.go +++ b/cmd/restic/cmd_stats.go @@ -2,7 +2,6 @@ package main import ( "context" - "crypto/sha256" "encoding/json" "fmt" "os" @@ -11,6 +10,8 @@ import ( "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/walker" + + "github.com/minio/sha256-simd" "github.com/spf13/cobra" ) diff --git a/go.mod b/go.mod index 7bc073c64..55e25b549 100644 --- a/go.mod +++ b/go.mod @@ -21,6 +21,7 @@ require ( github.com/kurin/blazer v0.5.3 github.com/marstr/guid v1.1.0 // indirect github.com/minio/minio-go/v6 v6.0.43 + github.com/minio/sha256-simd v0.1.1 github.com/ncw/swift v1.0.47 github.com/pkg/errors v0.8.1 github.com/pkg/profile v1.3.0 diff --git a/internal/repository/packer_manager.go b/internal/repository/packer_manager.go index 4884e0885..caa5a9661 100644 --- a/internal/repository/packer_manager.go +++ b/internal/repository/packer_manager.go @@ -2,7 +2,6 @@ package repository import ( "context" - "crypto/sha256" "os" "sync" @@ -14,6 +13,8 @@ import ( "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/fs" "github.com/restic/restic/internal/pack" + + "github.com/minio/sha256-simd" ) // Saver implements saving data in a backend. diff --git a/internal/repository/repository.go b/internal/repository/repository.go index 08241f683..bb928aedb 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -3,7 +3,6 @@ package repository import ( "bytes" "context" - "crypto/sha256" "encoding/json" "fmt" "io" @@ -17,6 +16,8 @@ import ( "github.com/restic/restic/internal/hashing" "github.com/restic/restic/internal/pack" "github.com/restic/restic/internal/restic" + + "github.com/minio/sha256-simd" "golang.org/x/sync/errgroup" ) diff --git a/internal/restic/id.go b/internal/restic/id.go index bc9749e77..c098dbfb3 100644 --- a/internal/restic/id.go +++ b/internal/restic/id.go @@ -2,13 +2,14 @@ package restic import ( "crypto/rand" - "crypto/sha256" "encoding/hex" "encoding/json" "fmt" "io" "github.com/restic/restic/internal/errors" + + "github.com/minio/sha256-simd" ) // Hash returns the ID for data. From f97a68088764939d53bf10fd8706c48c75ad0fa4 Mon Sep 17 00:00:00 2001 From: greatroar <@> Date: Tue, 28 Apr 2020 07:57:29 +0200 Subject: [PATCH 2/2] Fix repository benchmarks BenchmarkLoad{AndDecrypt,Blob} were spending between 38% and 50% of their time measuring SHA-256 performance in their checks. --- internal/repository/repository_test.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/internal/repository/repository_test.go b/internal/repository/repository_test.go index b08448ca8..ada6ce67d 100644 --- a/internal/repository/repository_test.go +++ b/internal/repository/repository_test.go @@ -197,6 +197,10 @@ func BenchmarkLoadBlob(b *testing.B) { for i := 0; i < b.N; i++ { var err error buf, err = repo.LoadBlob(context.TODO(), restic.DataBlob, id, buf) + + // Checking the SHA-256 with restic.Hash can make up 38% of the time + // spent in this loop, so pause the timer. + b.StopTimer() rtest.OK(b, err) if len(buf) != length { b.Errorf("wanted %d bytes, got %d", length, len(buf)) @@ -206,6 +210,7 @@ func BenchmarkLoadBlob(b *testing.B) { if !id.Equal(id2) { b.Errorf("wrong data returned, wanted %v, got %v", id.Str(), id2.Str()) } + b.StartTimer() } } @@ -230,6 +235,9 @@ func BenchmarkLoadAndDecrypt(b *testing.B) { for i := 0; i < b.N; i++ { data, err := repo.LoadAndDecrypt(context.TODO(), nil, restic.DataFile, storageID) rtest.OK(b, err) + + // See comment in BenchmarkLoadBlob. + b.StopTimer() if len(data) != length { b.Errorf("wanted %d bytes, got %d", length, len(data)) } @@ -238,6 +246,7 @@ func BenchmarkLoadAndDecrypt(b *testing.B) { if !dataID.Equal(id2) { b.Errorf("wrong data returned, wanted %v, got %v", storageID.Str(), id2.Str()) } + b.StartTimer() } }