syncthing/lib/weakhash/benchmark_test.go

152 lines
2.7 KiB
Go
Raw Normal View History

// Copyright (C) 2016 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package weakhash
import (
lib/scanner: Use standard adler32 when we don't need rolling (#5556) * lib/scanner: Use standard adler32 when we don't need rolling Seems the rolling adler32 implementation is super slow when executed on large blocks, even tho I can't explain why. BenchmarkFind1MFile-16 100 18991667 ns/op 55.21 MB/s 398844 B/op 20 allocs/op BenchmarkBlock/adler32-131072/#00-16 200 9726519 ns/op 1078.06 MB/s 2654936 B/op 163 allocs/op BenchmarkBlock/bozo32-131072/#00-16 20 73435540 ns/op 142.79 MB/s 2654928 B/op 163 allocs/op BenchmarkBlock/buzhash32-131072/#00-16 20 61482005 ns/op 170.55 MB/s 2654928 B/op 163 allocs/op BenchmarkBlock/buzhash64-131072/#00-16 20 61673660 ns/op 170.02 MB/s 2654928 B/op 163 allocs/op BenchmarkBlock/vanilla-adler32-131072/#00-16 300 4377307 ns/op 2395.48 MB/s 2654935 B/op 163 allocs/op BenchmarkBlock/adler32-16777216/#00-16 2 544010100 ns/op 19.27 MB/s 65624 B/op 5 allocs/op BenchmarkBlock/bozo32-16777216/#00-16 1 4678108500 ns/op 2.24 MB/s 51970144 B/op 24 allocs/op BenchmarkBlock/buzhash32-16777216/#00-16 1 3880370700 ns/op 2.70 MB/s 51970144 B/op 24 allocs/op BenchmarkBlock/buzhash64-16777216/#00-16 1 3875911700 ns/op 2.71 MB/s 51970144 B/op 24 allocs/op BenchmarkBlock/vanilla-adler32-16777216/#00-16 300 4010279 ns/op 2614.72 MB/s 65624 B/op 5 allocs/op BenchmarkRoll/adler32-131072/#00-16 2000 974279 ns/op 134.53 MB/s 270 B/op 0 allocs/op BenchmarkRoll/bozo32-131072/#00-16 2000 791770 ns/op 165.54 MB/s 270 B/op 0 allocs/op BenchmarkRoll/buzhash32-131072/#00-16 2000 917409 ns/op 142.87 MB/s 270 B/op 0 allocs/op BenchmarkRoll/buzhash64-131072/#00-16 2000 881125 ns/op 148.76 MB/s 270 B/op 0 allocs/op BenchmarkRoll/adler32-16777216/#00-16 10 124000400 ns/op 135.30 MB/s 7548937 B/op 0 allocs/op BenchmarkRoll/bozo32-16777216/#00-16 10 118008080 ns/op 142.17 MB/s 7548928 B/op 0 allocs/op BenchmarkRoll/buzhash32-16777216/#00-16 10 126794440 ns/op 132.32 MB/s 7548928 B/op 0 allocs/op BenchmarkRoll/buzhash64-16777216/#00-16 10 126631960 ns/op 132.49 MB/s 7548928 B/op 0 allocs/op * Update benchmark_test.go * gofmt * fixup benchmark
2019-02-25 09:29:31 +00:00
"bytes"
"context"
lib/scanner: Use standard adler32 when we don't need rolling (#5556) * lib/scanner: Use standard adler32 when we don't need rolling Seems the rolling adler32 implementation is super slow when executed on large blocks, even tho I can't explain why. BenchmarkFind1MFile-16 100 18991667 ns/op 55.21 MB/s 398844 B/op 20 allocs/op BenchmarkBlock/adler32-131072/#00-16 200 9726519 ns/op 1078.06 MB/s 2654936 B/op 163 allocs/op BenchmarkBlock/bozo32-131072/#00-16 20 73435540 ns/op 142.79 MB/s 2654928 B/op 163 allocs/op BenchmarkBlock/buzhash32-131072/#00-16 20 61482005 ns/op 170.55 MB/s 2654928 B/op 163 allocs/op BenchmarkBlock/buzhash64-131072/#00-16 20 61673660 ns/op 170.02 MB/s 2654928 B/op 163 allocs/op BenchmarkBlock/vanilla-adler32-131072/#00-16 300 4377307 ns/op 2395.48 MB/s 2654935 B/op 163 allocs/op BenchmarkBlock/adler32-16777216/#00-16 2 544010100 ns/op 19.27 MB/s 65624 B/op 5 allocs/op BenchmarkBlock/bozo32-16777216/#00-16 1 4678108500 ns/op 2.24 MB/s 51970144 B/op 24 allocs/op BenchmarkBlock/buzhash32-16777216/#00-16 1 3880370700 ns/op 2.70 MB/s 51970144 B/op 24 allocs/op BenchmarkBlock/buzhash64-16777216/#00-16 1 3875911700 ns/op 2.71 MB/s 51970144 B/op 24 allocs/op BenchmarkBlock/vanilla-adler32-16777216/#00-16 300 4010279 ns/op 2614.72 MB/s 65624 B/op 5 allocs/op BenchmarkRoll/adler32-131072/#00-16 2000 974279 ns/op 134.53 MB/s 270 B/op 0 allocs/op BenchmarkRoll/bozo32-131072/#00-16 2000 791770 ns/op 165.54 MB/s 270 B/op 0 allocs/op BenchmarkRoll/buzhash32-131072/#00-16 2000 917409 ns/op 142.87 MB/s 270 B/op 0 allocs/op BenchmarkRoll/buzhash64-131072/#00-16 2000 881125 ns/op 148.76 MB/s 270 B/op 0 allocs/op BenchmarkRoll/adler32-16777216/#00-16 10 124000400 ns/op 135.30 MB/s 7548937 B/op 0 allocs/op BenchmarkRoll/bozo32-16777216/#00-16 10 118008080 ns/op 142.17 MB/s 7548928 B/op 0 allocs/op BenchmarkRoll/buzhash32-16777216/#00-16 10 126794440 ns/op 132.32 MB/s 7548928 B/op 0 allocs/op BenchmarkRoll/buzhash64-16777216/#00-16 10 126631960 ns/op 132.49 MB/s 7548928 B/op 0 allocs/op * Update benchmark_test.go * gofmt * fixup benchmark
2019-02-25 09:29:31 +00:00
"fmt"
"hash"
vadler32 "hash/adler32"
"io"
"math/rand"
"os"
"testing"
"github.com/chmduquesne/rollinghash/adler32"
"github.com/chmduquesne/rollinghash/bozo32"
"github.com/chmduquesne/rollinghash/buzhash32"
"github.com/chmduquesne/rollinghash/buzhash64"
)
lib/scanner: Use standard adler32 when we don't need rolling (#5556) * lib/scanner: Use standard adler32 when we don't need rolling Seems the rolling adler32 implementation is super slow when executed on large blocks, even tho I can't explain why. BenchmarkFind1MFile-16 100 18991667 ns/op 55.21 MB/s 398844 B/op 20 allocs/op BenchmarkBlock/adler32-131072/#00-16 200 9726519 ns/op 1078.06 MB/s 2654936 B/op 163 allocs/op BenchmarkBlock/bozo32-131072/#00-16 20 73435540 ns/op 142.79 MB/s 2654928 B/op 163 allocs/op BenchmarkBlock/buzhash32-131072/#00-16 20 61482005 ns/op 170.55 MB/s 2654928 B/op 163 allocs/op BenchmarkBlock/buzhash64-131072/#00-16 20 61673660 ns/op 170.02 MB/s 2654928 B/op 163 allocs/op BenchmarkBlock/vanilla-adler32-131072/#00-16 300 4377307 ns/op 2395.48 MB/s 2654935 B/op 163 allocs/op BenchmarkBlock/adler32-16777216/#00-16 2 544010100 ns/op 19.27 MB/s 65624 B/op 5 allocs/op BenchmarkBlock/bozo32-16777216/#00-16 1 4678108500 ns/op 2.24 MB/s 51970144 B/op 24 allocs/op BenchmarkBlock/buzhash32-16777216/#00-16 1 3880370700 ns/op 2.70 MB/s 51970144 B/op 24 allocs/op BenchmarkBlock/buzhash64-16777216/#00-16 1 3875911700 ns/op 2.71 MB/s 51970144 B/op 24 allocs/op BenchmarkBlock/vanilla-adler32-16777216/#00-16 300 4010279 ns/op 2614.72 MB/s 65624 B/op 5 allocs/op BenchmarkRoll/adler32-131072/#00-16 2000 974279 ns/op 134.53 MB/s 270 B/op 0 allocs/op BenchmarkRoll/bozo32-131072/#00-16 2000 791770 ns/op 165.54 MB/s 270 B/op 0 allocs/op BenchmarkRoll/buzhash32-131072/#00-16 2000 917409 ns/op 142.87 MB/s 270 B/op 0 allocs/op BenchmarkRoll/buzhash64-131072/#00-16 2000 881125 ns/op 148.76 MB/s 270 B/op 0 allocs/op BenchmarkRoll/adler32-16777216/#00-16 10 124000400 ns/op 135.30 MB/s 7548937 B/op 0 allocs/op BenchmarkRoll/bozo32-16777216/#00-16 10 118008080 ns/op 142.17 MB/s 7548928 B/op 0 allocs/op BenchmarkRoll/buzhash32-16777216/#00-16 10 126794440 ns/op 132.32 MB/s 7548928 B/op 0 allocs/op BenchmarkRoll/buzhash64-16777216/#00-16 10 126631960 ns/op 132.49 MB/s 7548928 B/op 0 allocs/op * Update benchmark_test.go * gofmt * fixup benchmark
2019-02-25 09:29:31 +00:00
const testFile = "../model/testdata/tmpfile"
const size = 128 << 10
func BenchmarkFind1MFile(b *testing.B) {
b.ReportAllocs()
b.SetBytes(1 << 20)
for i := 0; i < b.N; i++ {
fd, err := os.Open(testFile)
if err != nil {
b.Fatal(err)
}
_, err = Find(context.Background(), fd, []uint32{0, 1, 2}, size)
if err != nil {
b.Fatal(err)
}
fd.Close()
}
}
lib/scanner: Use standard adler32 when we don't need rolling (#5556) * lib/scanner: Use standard adler32 when we don't need rolling Seems the rolling adler32 implementation is super slow when executed on large blocks, even tho I can't explain why. BenchmarkFind1MFile-16 100 18991667 ns/op 55.21 MB/s 398844 B/op 20 allocs/op BenchmarkBlock/adler32-131072/#00-16 200 9726519 ns/op 1078.06 MB/s 2654936 B/op 163 allocs/op BenchmarkBlock/bozo32-131072/#00-16 20 73435540 ns/op 142.79 MB/s 2654928 B/op 163 allocs/op BenchmarkBlock/buzhash32-131072/#00-16 20 61482005 ns/op 170.55 MB/s 2654928 B/op 163 allocs/op BenchmarkBlock/buzhash64-131072/#00-16 20 61673660 ns/op 170.02 MB/s 2654928 B/op 163 allocs/op BenchmarkBlock/vanilla-adler32-131072/#00-16 300 4377307 ns/op 2395.48 MB/s 2654935 B/op 163 allocs/op BenchmarkBlock/adler32-16777216/#00-16 2 544010100 ns/op 19.27 MB/s 65624 B/op 5 allocs/op BenchmarkBlock/bozo32-16777216/#00-16 1 4678108500 ns/op 2.24 MB/s 51970144 B/op 24 allocs/op BenchmarkBlock/buzhash32-16777216/#00-16 1 3880370700 ns/op 2.70 MB/s 51970144 B/op 24 allocs/op BenchmarkBlock/buzhash64-16777216/#00-16 1 3875911700 ns/op 2.71 MB/s 51970144 B/op 24 allocs/op BenchmarkBlock/vanilla-adler32-16777216/#00-16 300 4010279 ns/op 2614.72 MB/s 65624 B/op 5 allocs/op BenchmarkRoll/adler32-131072/#00-16 2000 974279 ns/op 134.53 MB/s 270 B/op 0 allocs/op BenchmarkRoll/bozo32-131072/#00-16 2000 791770 ns/op 165.54 MB/s 270 B/op 0 allocs/op BenchmarkRoll/buzhash32-131072/#00-16 2000 917409 ns/op 142.87 MB/s 270 B/op 0 allocs/op BenchmarkRoll/buzhash64-131072/#00-16 2000 881125 ns/op 148.76 MB/s 270 B/op 0 allocs/op BenchmarkRoll/adler32-16777216/#00-16 10 124000400 ns/op 135.30 MB/s 7548937 B/op 0 allocs/op BenchmarkRoll/bozo32-16777216/#00-16 10 118008080 ns/op 142.17 MB/s 7548928 B/op 0 allocs/op BenchmarkRoll/buzhash32-16777216/#00-16 10 126794440 ns/op 132.32 MB/s 7548928 B/op 0 allocs/op BenchmarkRoll/buzhash64-16777216/#00-16 10 126631960 ns/op 132.49 MB/s 7548928 B/op 0 allocs/op * Update benchmark_test.go * gofmt * fixup benchmark
2019-02-25 09:29:31 +00:00
type RollingHash interface {
hash.Hash
Roll(byte)
}
lib/scanner: Use standard adler32 when we don't need rolling (#5556) * lib/scanner: Use standard adler32 when we don't need rolling Seems the rolling adler32 implementation is super slow when executed on large blocks, even tho I can't explain why. BenchmarkFind1MFile-16 100 18991667 ns/op 55.21 MB/s 398844 B/op 20 allocs/op BenchmarkBlock/adler32-131072/#00-16 200 9726519 ns/op 1078.06 MB/s 2654936 B/op 163 allocs/op BenchmarkBlock/bozo32-131072/#00-16 20 73435540 ns/op 142.79 MB/s 2654928 B/op 163 allocs/op BenchmarkBlock/buzhash32-131072/#00-16 20 61482005 ns/op 170.55 MB/s 2654928 B/op 163 allocs/op BenchmarkBlock/buzhash64-131072/#00-16 20 61673660 ns/op 170.02 MB/s 2654928 B/op 163 allocs/op BenchmarkBlock/vanilla-adler32-131072/#00-16 300 4377307 ns/op 2395.48 MB/s 2654935 B/op 163 allocs/op BenchmarkBlock/adler32-16777216/#00-16 2 544010100 ns/op 19.27 MB/s 65624 B/op 5 allocs/op BenchmarkBlock/bozo32-16777216/#00-16 1 4678108500 ns/op 2.24 MB/s 51970144 B/op 24 allocs/op BenchmarkBlock/buzhash32-16777216/#00-16 1 3880370700 ns/op 2.70 MB/s 51970144 B/op 24 allocs/op BenchmarkBlock/buzhash64-16777216/#00-16 1 3875911700 ns/op 2.71 MB/s 51970144 B/op 24 allocs/op BenchmarkBlock/vanilla-adler32-16777216/#00-16 300 4010279 ns/op 2614.72 MB/s 65624 B/op 5 allocs/op BenchmarkRoll/adler32-131072/#00-16 2000 974279 ns/op 134.53 MB/s 270 B/op 0 allocs/op BenchmarkRoll/bozo32-131072/#00-16 2000 791770 ns/op 165.54 MB/s 270 B/op 0 allocs/op BenchmarkRoll/buzhash32-131072/#00-16 2000 917409 ns/op 142.87 MB/s 270 B/op 0 allocs/op BenchmarkRoll/buzhash64-131072/#00-16 2000 881125 ns/op 148.76 MB/s 270 B/op 0 allocs/op BenchmarkRoll/adler32-16777216/#00-16 10 124000400 ns/op 135.30 MB/s 7548937 B/op 0 allocs/op BenchmarkRoll/bozo32-16777216/#00-16 10 118008080 ns/op 142.17 MB/s 7548928 B/op 0 allocs/op BenchmarkRoll/buzhash32-16777216/#00-16 10 126794440 ns/op 132.32 MB/s 7548928 B/op 0 allocs/op BenchmarkRoll/buzhash64-16777216/#00-16 10 126631960 ns/op 132.49 MB/s 7548928 B/op 0 allocs/op * Update benchmark_test.go * gofmt * fixup benchmark
2019-02-25 09:29:31 +00:00
func BenchmarkBlock(b *testing.B) {
tests := []struct {
name string
hash hash.Hash
}{
{
"adler32", adler32.New(),
},
{
"bozo32", bozo32.New(),
},
{
"buzhash32", buzhash32.New(),
},
{
"buzhash64", buzhash64.New(),
},
{
"vanilla-adler32", vadler32.New(),
},
}
lib/scanner: Use standard adler32 when we don't need rolling (#5556) * lib/scanner: Use standard adler32 when we don't need rolling Seems the rolling adler32 implementation is super slow when executed on large blocks, even tho I can't explain why. BenchmarkFind1MFile-16 100 18991667 ns/op 55.21 MB/s 398844 B/op 20 allocs/op BenchmarkBlock/adler32-131072/#00-16 200 9726519 ns/op 1078.06 MB/s 2654936 B/op 163 allocs/op BenchmarkBlock/bozo32-131072/#00-16 20 73435540 ns/op 142.79 MB/s 2654928 B/op 163 allocs/op BenchmarkBlock/buzhash32-131072/#00-16 20 61482005 ns/op 170.55 MB/s 2654928 B/op 163 allocs/op BenchmarkBlock/buzhash64-131072/#00-16 20 61673660 ns/op 170.02 MB/s 2654928 B/op 163 allocs/op BenchmarkBlock/vanilla-adler32-131072/#00-16 300 4377307 ns/op 2395.48 MB/s 2654935 B/op 163 allocs/op BenchmarkBlock/adler32-16777216/#00-16 2 544010100 ns/op 19.27 MB/s 65624 B/op 5 allocs/op BenchmarkBlock/bozo32-16777216/#00-16 1 4678108500 ns/op 2.24 MB/s 51970144 B/op 24 allocs/op BenchmarkBlock/buzhash32-16777216/#00-16 1 3880370700 ns/op 2.70 MB/s 51970144 B/op 24 allocs/op BenchmarkBlock/buzhash64-16777216/#00-16 1 3875911700 ns/op 2.71 MB/s 51970144 B/op 24 allocs/op BenchmarkBlock/vanilla-adler32-16777216/#00-16 300 4010279 ns/op 2614.72 MB/s 65624 B/op 5 allocs/op BenchmarkRoll/adler32-131072/#00-16 2000 974279 ns/op 134.53 MB/s 270 B/op 0 allocs/op BenchmarkRoll/bozo32-131072/#00-16 2000 791770 ns/op 165.54 MB/s 270 B/op 0 allocs/op BenchmarkRoll/buzhash32-131072/#00-16 2000 917409 ns/op 142.87 MB/s 270 B/op 0 allocs/op BenchmarkRoll/buzhash64-131072/#00-16 2000 881125 ns/op 148.76 MB/s 270 B/op 0 allocs/op BenchmarkRoll/adler32-16777216/#00-16 10 124000400 ns/op 135.30 MB/s 7548937 B/op 0 allocs/op BenchmarkRoll/bozo32-16777216/#00-16 10 118008080 ns/op 142.17 MB/s 7548928 B/op 0 allocs/op BenchmarkRoll/buzhash32-16777216/#00-16 10 126794440 ns/op 132.32 MB/s 7548928 B/op 0 allocs/op BenchmarkRoll/buzhash64-16777216/#00-16 10 126631960 ns/op 132.49 MB/s 7548928 B/op 0 allocs/op * Update benchmark_test.go * gofmt * fixup benchmark
2019-02-25 09:29:31 +00:00
sizes := []int64{128 << 10, 16 << 20}
buf := make([]byte, 16<<20)
rand.Read(buf)
for _, testSize := range sizes {
for _, test := range tests {
b.Run(test.name+"-"+fmt.Sprint(testSize), func(bb *testing.B) {
bb.Run("", func(bbb *testing.B) {
bbb.ResetTimer()
for i := 0; i < bbb.N; i++ {
lr := io.LimitReader(bytes.NewReader(buf), testSize)
n, err := io.Copy(test.hash, lr)
if err != nil {
bbb.Error(err)
}
if n != testSize {
bbb.Errorf("%d != %d", n, testSize)
}
test.hash.Sum(nil)
test.hash.Reset()
}
bbb.SetBytes(testSize)
lib/scanner: Use standard adler32 when we don't need rolling (#5556) * lib/scanner: Use standard adler32 when we don't need rolling Seems the rolling adler32 implementation is super slow when executed on large blocks, even tho I can't explain why. BenchmarkFind1MFile-16 100 18991667 ns/op 55.21 MB/s 398844 B/op 20 allocs/op BenchmarkBlock/adler32-131072/#00-16 200 9726519 ns/op 1078.06 MB/s 2654936 B/op 163 allocs/op BenchmarkBlock/bozo32-131072/#00-16 20 73435540 ns/op 142.79 MB/s 2654928 B/op 163 allocs/op BenchmarkBlock/buzhash32-131072/#00-16 20 61482005 ns/op 170.55 MB/s 2654928 B/op 163 allocs/op BenchmarkBlock/buzhash64-131072/#00-16 20 61673660 ns/op 170.02 MB/s 2654928 B/op 163 allocs/op BenchmarkBlock/vanilla-adler32-131072/#00-16 300 4377307 ns/op 2395.48 MB/s 2654935 B/op 163 allocs/op BenchmarkBlock/adler32-16777216/#00-16 2 544010100 ns/op 19.27 MB/s 65624 B/op 5 allocs/op BenchmarkBlock/bozo32-16777216/#00-16 1 4678108500 ns/op 2.24 MB/s 51970144 B/op 24 allocs/op BenchmarkBlock/buzhash32-16777216/#00-16 1 3880370700 ns/op 2.70 MB/s 51970144 B/op 24 allocs/op BenchmarkBlock/buzhash64-16777216/#00-16 1 3875911700 ns/op 2.71 MB/s 51970144 B/op 24 allocs/op BenchmarkBlock/vanilla-adler32-16777216/#00-16 300 4010279 ns/op 2614.72 MB/s 65624 B/op 5 allocs/op BenchmarkRoll/adler32-131072/#00-16 2000 974279 ns/op 134.53 MB/s 270 B/op 0 allocs/op BenchmarkRoll/bozo32-131072/#00-16 2000 791770 ns/op 165.54 MB/s 270 B/op 0 allocs/op BenchmarkRoll/buzhash32-131072/#00-16 2000 917409 ns/op 142.87 MB/s 270 B/op 0 allocs/op BenchmarkRoll/buzhash64-131072/#00-16 2000 881125 ns/op 148.76 MB/s 270 B/op 0 allocs/op BenchmarkRoll/adler32-16777216/#00-16 10 124000400 ns/op 135.30 MB/s 7548937 B/op 0 allocs/op BenchmarkRoll/bozo32-16777216/#00-16 10 118008080 ns/op 142.17 MB/s 7548928 B/op 0 allocs/op BenchmarkRoll/buzhash32-16777216/#00-16 10 126794440 ns/op 132.32 MB/s 7548928 B/op 0 allocs/op BenchmarkRoll/buzhash64-16777216/#00-16 10 126631960 ns/op 132.49 MB/s 7548928 B/op 0 allocs/op * Update benchmark_test.go * gofmt * fixup benchmark
2019-02-25 09:29:31 +00:00
bbb.ReportAllocs()
})
})
}
}
}
lib/scanner: Use standard adler32 when we don't need rolling (#5556) * lib/scanner: Use standard adler32 when we don't need rolling Seems the rolling adler32 implementation is super slow when executed on large blocks, even tho I can't explain why. BenchmarkFind1MFile-16 100 18991667 ns/op 55.21 MB/s 398844 B/op 20 allocs/op BenchmarkBlock/adler32-131072/#00-16 200 9726519 ns/op 1078.06 MB/s 2654936 B/op 163 allocs/op BenchmarkBlock/bozo32-131072/#00-16 20 73435540 ns/op 142.79 MB/s 2654928 B/op 163 allocs/op BenchmarkBlock/buzhash32-131072/#00-16 20 61482005 ns/op 170.55 MB/s 2654928 B/op 163 allocs/op BenchmarkBlock/buzhash64-131072/#00-16 20 61673660 ns/op 170.02 MB/s 2654928 B/op 163 allocs/op BenchmarkBlock/vanilla-adler32-131072/#00-16 300 4377307 ns/op 2395.48 MB/s 2654935 B/op 163 allocs/op BenchmarkBlock/adler32-16777216/#00-16 2 544010100 ns/op 19.27 MB/s 65624 B/op 5 allocs/op BenchmarkBlock/bozo32-16777216/#00-16 1 4678108500 ns/op 2.24 MB/s 51970144 B/op 24 allocs/op BenchmarkBlock/buzhash32-16777216/#00-16 1 3880370700 ns/op 2.70 MB/s 51970144 B/op 24 allocs/op BenchmarkBlock/buzhash64-16777216/#00-16 1 3875911700 ns/op 2.71 MB/s 51970144 B/op 24 allocs/op BenchmarkBlock/vanilla-adler32-16777216/#00-16 300 4010279 ns/op 2614.72 MB/s 65624 B/op 5 allocs/op BenchmarkRoll/adler32-131072/#00-16 2000 974279 ns/op 134.53 MB/s 270 B/op 0 allocs/op BenchmarkRoll/bozo32-131072/#00-16 2000 791770 ns/op 165.54 MB/s 270 B/op 0 allocs/op BenchmarkRoll/buzhash32-131072/#00-16 2000 917409 ns/op 142.87 MB/s 270 B/op 0 allocs/op BenchmarkRoll/buzhash64-131072/#00-16 2000 881125 ns/op 148.76 MB/s 270 B/op 0 allocs/op BenchmarkRoll/adler32-16777216/#00-16 10 124000400 ns/op 135.30 MB/s 7548937 B/op 0 allocs/op BenchmarkRoll/bozo32-16777216/#00-16 10 118008080 ns/op 142.17 MB/s 7548928 B/op 0 allocs/op BenchmarkRoll/buzhash32-16777216/#00-16 10 126794440 ns/op 132.32 MB/s 7548928 B/op 0 allocs/op BenchmarkRoll/buzhash64-16777216/#00-16 10 126631960 ns/op 132.49 MB/s 7548928 B/op 0 allocs/op * Update benchmark_test.go * gofmt * fixup benchmark
2019-02-25 09:29:31 +00:00
func BenchmarkRoll(b *testing.B) {
tests := []struct {
name string
hash RollingHash
}{
{
"adler32", adler32.New(),
},
{
"bozo32", bozo32.New(),
},
{
"buzhash32", buzhash32.New(),
},
{
"buzhash64", buzhash64.New(),
},
}
lib/scanner: Use standard adler32 when we don't need rolling (#5556) * lib/scanner: Use standard adler32 when we don't need rolling Seems the rolling adler32 implementation is super slow when executed on large blocks, even tho I can't explain why. BenchmarkFind1MFile-16 100 18991667 ns/op 55.21 MB/s 398844 B/op 20 allocs/op BenchmarkBlock/adler32-131072/#00-16 200 9726519 ns/op 1078.06 MB/s 2654936 B/op 163 allocs/op BenchmarkBlock/bozo32-131072/#00-16 20 73435540 ns/op 142.79 MB/s 2654928 B/op 163 allocs/op BenchmarkBlock/buzhash32-131072/#00-16 20 61482005 ns/op 170.55 MB/s 2654928 B/op 163 allocs/op BenchmarkBlock/buzhash64-131072/#00-16 20 61673660 ns/op 170.02 MB/s 2654928 B/op 163 allocs/op BenchmarkBlock/vanilla-adler32-131072/#00-16 300 4377307 ns/op 2395.48 MB/s 2654935 B/op 163 allocs/op BenchmarkBlock/adler32-16777216/#00-16 2 544010100 ns/op 19.27 MB/s 65624 B/op 5 allocs/op BenchmarkBlock/bozo32-16777216/#00-16 1 4678108500 ns/op 2.24 MB/s 51970144 B/op 24 allocs/op BenchmarkBlock/buzhash32-16777216/#00-16 1 3880370700 ns/op 2.70 MB/s 51970144 B/op 24 allocs/op BenchmarkBlock/buzhash64-16777216/#00-16 1 3875911700 ns/op 2.71 MB/s 51970144 B/op 24 allocs/op BenchmarkBlock/vanilla-adler32-16777216/#00-16 300 4010279 ns/op 2614.72 MB/s 65624 B/op 5 allocs/op BenchmarkRoll/adler32-131072/#00-16 2000 974279 ns/op 134.53 MB/s 270 B/op 0 allocs/op BenchmarkRoll/bozo32-131072/#00-16 2000 791770 ns/op 165.54 MB/s 270 B/op 0 allocs/op BenchmarkRoll/buzhash32-131072/#00-16 2000 917409 ns/op 142.87 MB/s 270 B/op 0 allocs/op BenchmarkRoll/buzhash64-131072/#00-16 2000 881125 ns/op 148.76 MB/s 270 B/op 0 allocs/op BenchmarkRoll/adler32-16777216/#00-16 10 124000400 ns/op 135.30 MB/s 7548937 B/op 0 allocs/op BenchmarkRoll/bozo32-16777216/#00-16 10 118008080 ns/op 142.17 MB/s 7548928 B/op 0 allocs/op BenchmarkRoll/buzhash32-16777216/#00-16 10 126794440 ns/op 132.32 MB/s 7548928 B/op 0 allocs/op BenchmarkRoll/buzhash64-16777216/#00-16 10 126631960 ns/op 132.49 MB/s 7548928 B/op 0 allocs/op * Update benchmark_test.go * gofmt * fixup benchmark
2019-02-25 09:29:31 +00:00
sizes := []int64{128 << 10, 16 << 20}
lib/scanner: Use standard adler32 when we don't need rolling (#5556) * lib/scanner: Use standard adler32 when we don't need rolling Seems the rolling adler32 implementation is super slow when executed on large blocks, even tho I can't explain why. BenchmarkFind1MFile-16 100 18991667 ns/op 55.21 MB/s 398844 B/op 20 allocs/op BenchmarkBlock/adler32-131072/#00-16 200 9726519 ns/op 1078.06 MB/s 2654936 B/op 163 allocs/op BenchmarkBlock/bozo32-131072/#00-16 20 73435540 ns/op 142.79 MB/s 2654928 B/op 163 allocs/op BenchmarkBlock/buzhash32-131072/#00-16 20 61482005 ns/op 170.55 MB/s 2654928 B/op 163 allocs/op BenchmarkBlock/buzhash64-131072/#00-16 20 61673660 ns/op 170.02 MB/s 2654928 B/op 163 allocs/op BenchmarkBlock/vanilla-adler32-131072/#00-16 300 4377307 ns/op 2395.48 MB/s 2654935 B/op 163 allocs/op BenchmarkBlock/adler32-16777216/#00-16 2 544010100 ns/op 19.27 MB/s 65624 B/op 5 allocs/op BenchmarkBlock/bozo32-16777216/#00-16 1 4678108500 ns/op 2.24 MB/s 51970144 B/op 24 allocs/op BenchmarkBlock/buzhash32-16777216/#00-16 1 3880370700 ns/op 2.70 MB/s 51970144 B/op 24 allocs/op BenchmarkBlock/buzhash64-16777216/#00-16 1 3875911700 ns/op 2.71 MB/s 51970144 B/op 24 allocs/op BenchmarkBlock/vanilla-adler32-16777216/#00-16 300 4010279 ns/op 2614.72 MB/s 65624 B/op 5 allocs/op BenchmarkRoll/adler32-131072/#00-16 2000 974279 ns/op 134.53 MB/s 270 B/op 0 allocs/op BenchmarkRoll/bozo32-131072/#00-16 2000 791770 ns/op 165.54 MB/s 270 B/op 0 allocs/op BenchmarkRoll/buzhash32-131072/#00-16 2000 917409 ns/op 142.87 MB/s 270 B/op 0 allocs/op BenchmarkRoll/buzhash64-131072/#00-16 2000 881125 ns/op 148.76 MB/s 270 B/op 0 allocs/op BenchmarkRoll/adler32-16777216/#00-16 10 124000400 ns/op 135.30 MB/s 7548937 B/op 0 allocs/op BenchmarkRoll/bozo32-16777216/#00-16 10 118008080 ns/op 142.17 MB/s 7548928 B/op 0 allocs/op BenchmarkRoll/buzhash32-16777216/#00-16 10 126794440 ns/op 132.32 MB/s 7548928 B/op 0 allocs/op BenchmarkRoll/buzhash64-16777216/#00-16 10 126631960 ns/op 132.49 MB/s 7548928 B/op 0 allocs/op * Update benchmark_test.go * gofmt * fixup benchmark
2019-02-25 09:29:31 +00:00
for _, testSize := range sizes {
for _, test := range tests {
b.Run(test.name+"-"+fmt.Sprint(testSize), func(bb *testing.B) {
bb.Run("", func(bbb *testing.B) {
data := make([]byte, testSize)
lib/scanner: Use standard adler32 when we don't need rolling (#5556) * lib/scanner: Use standard adler32 when we don't need rolling Seems the rolling adler32 implementation is super slow when executed on large blocks, even tho I can't explain why. BenchmarkFind1MFile-16 100 18991667 ns/op 55.21 MB/s 398844 B/op 20 allocs/op BenchmarkBlock/adler32-131072/#00-16 200 9726519 ns/op 1078.06 MB/s 2654936 B/op 163 allocs/op BenchmarkBlock/bozo32-131072/#00-16 20 73435540 ns/op 142.79 MB/s 2654928 B/op 163 allocs/op BenchmarkBlock/buzhash32-131072/#00-16 20 61482005 ns/op 170.55 MB/s 2654928 B/op 163 allocs/op BenchmarkBlock/buzhash64-131072/#00-16 20 61673660 ns/op 170.02 MB/s 2654928 B/op 163 allocs/op BenchmarkBlock/vanilla-adler32-131072/#00-16 300 4377307 ns/op 2395.48 MB/s 2654935 B/op 163 allocs/op BenchmarkBlock/adler32-16777216/#00-16 2 544010100 ns/op 19.27 MB/s 65624 B/op 5 allocs/op BenchmarkBlock/bozo32-16777216/#00-16 1 4678108500 ns/op 2.24 MB/s 51970144 B/op 24 allocs/op BenchmarkBlock/buzhash32-16777216/#00-16 1 3880370700 ns/op 2.70 MB/s 51970144 B/op 24 allocs/op BenchmarkBlock/buzhash64-16777216/#00-16 1 3875911700 ns/op 2.71 MB/s 51970144 B/op 24 allocs/op BenchmarkBlock/vanilla-adler32-16777216/#00-16 300 4010279 ns/op 2614.72 MB/s 65624 B/op 5 allocs/op BenchmarkRoll/adler32-131072/#00-16 2000 974279 ns/op 134.53 MB/s 270 B/op 0 allocs/op BenchmarkRoll/bozo32-131072/#00-16 2000 791770 ns/op 165.54 MB/s 270 B/op 0 allocs/op BenchmarkRoll/buzhash32-131072/#00-16 2000 917409 ns/op 142.87 MB/s 270 B/op 0 allocs/op BenchmarkRoll/buzhash64-131072/#00-16 2000 881125 ns/op 148.76 MB/s 270 B/op 0 allocs/op BenchmarkRoll/adler32-16777216/#00-16 10 124000400 ns/op 135.30 MB/s 7548937 B/op 0 allocs/op BenchmarkRoll/bozo32-16777216/#00-16 10 118008080 ns/op 142.17 MB/s 7548928 B/op 0 allocs/op BenchmarkRoll/buzhash32-16777216/#00-16 10 126794440 ns/op 132.32 MB/s 7548928 B/op 0 allocs/op BenchmarkRoll/buzhash64-16777216/#00-16 10 126631960 ns/op 132.49 MB/s 7548928 B/op 0 allocs/op * Update benchmark_test.go * gofmt * fixup benchmark
2019-02-25 09:29:31 +00:00
if _, err := test.hash.Write(data); err != nil {
bbb.Error(err)
}
lib/scanner: Use standard adler32 when we don't need rolling (#5556) * lib/scanner: Use standard adler32 when we don't need rolling Seems the rolling adler32 implementation is super slow when executed on large blocks, even tho I can't explain why. BenchmarkFind1MFile-16 100 18991667 ns/op 55.21 MB/s 398844 B/op 20 allocs/op BenchmarkBlock/adler32-131072/#00-16 200 9726519 ns/op 1078.06 MB/s 2654936 B/op 163 allocs/op BenchmarkBlock/bozo32-131072/#00-16 20 73435540 ns/op 142.79 MB/s 2654928 B/op 163 allocs/op BenchmarkBlock/buzhash32-131072/#00-16 20 61482005 ns/op 170.55 MB/s 2654928 B/op 163 allocs/op BenchmarkBlock/buzhash64-131072/#00-16 20 61673660 ns/op 170.02 MB/s 2654928 B/op 163 allocs/op BenchmarkBlock/vanilla-adler32-131072/#00-16 300 4377307 ns/op 2395.48 MB/s 2654935 B/op 163 allocs/op BenchmarkBlock/adler32-16777216/#00-16 2 544010100 ns/op 19.27 MB/s 65624 B/op 5 allocs/op BenchmarkBlock/bozo32-16777216/#00-16 1 4678108500 ns/op 2.24 MB/s 51970144 B/op 24 allocs/op BenchmarkBlock/buzhash32-16777216/#00-16 1 3880370700 ns/op 2.70 MB/s 51970144 B/op 24 allocs/op BenchmarkBlock/buzhash64-16777216/#00-16 1 3875911700 ns/op 2.71 MB/s 51970144 B/op 24 allocs/op BenchmarkBlock/vanilla-adler32-16777216/#00-16 300 4010279 ns/op 2614.72 MB/s 65624 B/op 5 allocs/op BenchmarkRoll/adler32-131072/#00-16 2000 974279 ns/op 134.53 MB/s 270 B/op 0 allocs/op BenchmarkRoll/bozo32-131072/#00-16 2000 791770 ns/op 165.54 MB/s 270 B/op 0 allocs/op BenchmarkRoll/buzhash32-131072/#00-16 2000 917409 ns/op 142.87 MB/s 270 B/op 0 allocs/op BenchmarkRoll/buzhash64-131072/#00-16 2000 881125 ns/op 148.76 MB/s 270 B/op 0 allocs/op BenchmarkRoll/adler32-16777216/#00-16 10 124000400 ns/op 135.30 MB/s 7548937 B/op 0 allocs/op BenchmarkRoll/bozo32-16777216/#00-16 10 118008080 ns/op 142.17 MB/s 7548928 B/op 0 allocs/op BenchmarkRoll/buzhash32-16777216/#00-16 10 126794440 ns/op 132.32 MB/s 7548928 B/op 0 allocs/op BenchmarkRoll/buzhash64-16777216/#00-16 10 126631960 ns/op 132.49 MB/s 7548928 B/op 0 allocs/op * Update benchmark_test.go * gofmt * fixup benchmark
2019-02-25 09:29:31 +00:00
bbb.ResetTimer()
lib/scanner: Use standard adler32 when we don't need rolling (#5556) * lib/scanner: Use standard adler32 when we don't need rolling Seems the rolling adler32 implementation is super slow when executed on large blocks, even tho I can't explain why. BenchmarkFind1MFile-16 100 18991667 ns/op 55.21 MB/s 398844 B/op 20 allocs/op BenchmarkBlock/adler32-131072/#00-16 200 9726519 ns/op 1078.06 MB/s 2654936 B/op 163 allocs/op BenchmarkBlock/bozo32-131072/#00-16 20 73435540 ns/op 142.79 MB/s 2654928 B/op 163 allocs/op BenchmarkBlock/buzhash32-131072/#00-16 20 61482005 ns/op 170.55 MB/s 2654928 B/op 163 allocs/op BenchmarkBlock/buzhash64-131072/#00-16 20 61673660 ns/op 170.02 MB/s 2654928 B/op 163 allocs/op BenchmarkBlock/vanilla-adler32-131072/#00-16 300 4377307 ns/op 2395.48 MB/s 2654935 B/op 163 allocs/op BenchmarkBlock/adler32-16777216/#00-16 2 544010100 ns/op 19.27 MB/s 65624 B/op 5 allocs/op BenchmarkBlock/bozo32-16777216/#00-16 1 4678108500 ns/op 2.24 MB/s 51970144 B/op 24 allocs/op BenchmarkBlock/buzhash32-16777216/#00-16 1 3880370700 ns/op 2.70 MB/s 51970144 B/op 24 allocs/op BenchmarkBlock/buzhash64-16777216/#00-16 1 3875911700 ns/op 2.71 MB/s 51970144 B/op 24 allocs/op BenchmarkBlock/vanilla-adler32-16777216/#00-16 300 4010279 ns/op 2614.72 MB/s 65624 B/op 5 allocs/op BenchmarkRoll/adler32-131072/#00-16 2000 974279 ns/op 134.53 MB/s 270 B/op 0 allocs/op BenchmarkRoll/bozo32-131072/#00-16 2000 791770 ns/op 165.54 MB/s 270 B/op 0 allocs/op BenchmarkRoll/buzhash32-131072/#00-16 2000 917409 ns/op 142.87 MB/s 270 B/op 0 allocs/op BenchmarkRoll/buzhash64-131072/#00-16 2000 881125 ns/op 148.76 MB/s 270 B/op 0 allocs/op BenchmarkRoll/adler32-16777216/#00-16 10 124000400 ns/op 135.30 MB/s 7548937 B/op 0 allocs/op BenchmarkRoll/bozo32-16777216/#00-16 10 118008080 ns/op 142.17 MB/s 7548928 B/op 0 allocs/op BenchmarkRoll/buzhash32-16777216/#00-16 10 126794440 ns/op 132.32 MB/s 7548928 B/op 0 allocs/op BenchmarkRoll/buzhash64-16777216/#00-16 10 126631960 ns/op 132.49 MB/s 7548928 B/op 0 allocs/op * Update benchmark_test.go * gofmt * fixup benchmark
2019-02-25 09:29:31 +00:00
for i := 0; i < bbb.N; i++ {
for j := int64(0); j <= testSize; j++ {
test.hash.Roll('a')
}
}
lib/scanner: Use standard adler32 when we don't need rolling (#5556) * lib/scanner: Use standard adler32 when we don't need rolling Seems the rolling adler32 implementation is super slow when executed on large blocks, even tho I can't explain why. BenchmarkFind1MFile-16 100 18991667 ns/op 55.21 MB/s 398844 B/op 20 allocs/op BenchmarkBlock/adler32-131072/#00-16 200 9726519 ns/op 1078.06 MB/s 2654936 B/op 163 allocs/op BenchmarkBlock/bozo32-131072/#00-16 20 73435540 ns/op 142.79 MB/s 2654928 B/op 163 allocs/op BenchmarkBlock/buzhash32-131072/#00-16 20 61482005 ns/op 170.55 MB/s 2654928 B/op 163 allocs/op BenchmarkBlock/buzhash64-131072/#00-16 20 61673660 ns/op 170.02 MB/s 2654928 B/op 163 allocs/op BenchmarkBlock/vanilla-adler32-131072/#00-16 300 4377307 ns/op 2395.48 MB/s 2654935 B/op 163 allocs/op BenchmarkBlock/adler32-16777216/#00-16 2 544010100 ns/op 19.27 MB/s 65624 B/op 5 allocs/op BenchmarkBlock/bozo32-16777216/#00-16 1 4678108500 ns/op 2.24 MB/s 51970144 B/op 24 allocs/op BenchmarkBlock/buzhash32-16777216/#00-16 1 3880370700 ns/op 2.70 MB/s 51970144 B/op 24 allocs/op BenchmarkBlock/buzhash64-16777216/#00-16 1 3875911700 ns/op 2.71 MB/s 51970144 B/op 24 allocs/op BenchmarkBlock/vanilla-adler32-16777216/#00-16 300 4010279 ns/op 2614.72 MB/s 65624 B/op 5 allocs/op BenchmarkRoll/adler32-131072/#00-16 2000 974279 ns/op 134.53 MB/s 270 B/op 0 allocs/op BenchmarkRoll/bozo32-131072/#00-16 2000 791770 ns/op 165.54 MB/s 270 B/op 0 allocs/op BenchmarkRoll/buzhash32-131072/#00-16 2000 917409 ns/op 142.87 MB/s 270 B/op 0 allocs/op BenchmarkRoll/buzhash64-131072/#00-16 2000 881125 ns/op 148.76 MB/s 270 B/op 0 allocs/op BenchmarkRoll/adler32-16777216/#00-16 10 124000400 ns/op 135.30 MB/s 7548937 B/op 0 allocs/op BenchmarkRoll/bozo32-16777216/#00-16 10 118008080 ns/op 142.17 MB/s 7548928 B/op 0 allocs/op BenchmarkRoll/buzhash32-16777216/#00-16 10 126794440 ns/op 132.32 MB/s 7548928 B/op 0 allocs/op BenchmarkRoll/buzhash64-16777216/#00-16 10 126631960 ns/op 132.49 MB/s 7548928 B/op 0 allocs/op * Update benchmark_test.go * gofmt * fixup benchmark
2019-02-25 09:29:31 +00:00
bbb.SetBytes(testSize)
bbb.ReportAllocs()
})
lib/scanner: Use standard adler32 when we don't need rolling (#5556) * lib/scanner: Use standard adler32 when we don't need rolling Seems the rolling adler32 implementation is super slow when executed on large blocks, even tho I can't explain why. BenchmarkFind1MFile-16 100 18991667 ns/op 55.21 MB/s 398844 B/op 20 allocs/op BenchmarkBlock/adler32-131072/#00-16 200 9726519 ns/op 1078.06 MB/s 2654936 B/op 163 allocs/op BenchmarkBlock/bozo32-131072/#00-16 20 73435540 ns/op 142.79 MB/s 2654928 B/op 163 allocs/op BenchmarkBlock/buzhash32-131072/#00-16 20 61482005 ns/op 170.55 MB/s 2654928 B/op 163 allocs/op BenchmarkBlock/buzhash64-131072/#00-16 20 61673660 ns/op 170.02 MB/s 2654928 B/op 163 allocs/op BenchmarkBlock/vanilla-adler32-131072/#00-16 300 4377307 ns/op 2395.48 MB/s 2654935 B/op 163 allocs/op BenchmarkBlock/adler32-16777216/#00-16 2 544010100 ns/op 19.27 MB/s 65624 B/op 5 allocs/op BenchmarkBlock/bozo32-16777216/#00-16 1 4678108500 ns/op 2.24 MB/s 51970144 B/op 24 allocs/op BenchmarkBlock/buzhash32-16777216/#00-16 1 3880370700 ns/op 2.70 MB/s 51970144 B/op 24 allocs/op BenchmarkBlock/buzhash64-16777216/#00-16 1 3875911700 ns/op 2.71 MB/s 51970144 B/op 24 allocs/op BenchmarkBlock/vanilla-adler32-16777216/#00-16 300 4010279 ns/op 2614.72 MB/s 65624 B/op 5 allocs/op BenchmarkRoll/adler32-131072/#00-16 2000 974279 ns/op 134.53 MB/s 270 B/op 0 allocs/op BenchmarkRoll/bozo32-131072/#00-16 2000 791770 ns/op 165.54 MB/s 270 B/op 0 allocs/op BenchmarkRoll/buzhash32-131072/#00-16 2000 917409 ns/op 142.87 MB/s 270 B/op 0 allocs/op BenchmarkRoll/buzhash64-131072/#00-16 2000 881125 ns/op 148.76 MB/s 270 B/op 0 allocs/op BenchmarkRoll/adler32-16777216/#00-16 10 124000400 ns/op 135.30 MB/s 7548937 B/op 0 allocs/op BenchmarkRoll/bozo32-16777216/#00-16 10 118008080 ns/op 142.17 MB/s 7548928 B/op 0 allocs/op BenchmarkRoll/buzhash32-16777216/#00-16 10 126794440 ns/op 132.32 MB/s 7548928 B/op 0 allocs/op BenchmarkRoll/buzhash64-16777216/#00-16 10 126631960 ns/op 132.49 MB/s 7548928 B/op 0 allocs/op * Update benchmark_test.go * gofmt * fixup benchmark
2019-02-25 09:29:31 +00:00
})
}
}
}