mirror of
https://github.com/octoleo/restic.git
synced 2024-12-24 11:55:28 +00:00
8d2996eaaa
In Go 1.17.1, maphash has become quite a bit faster than siphash, so we can drop one third-party dependency. maphash is just an interface to the standard Go map's hash function, which we already trust for other use cases. Benchmark results on linux/amd64, -benchtime=3s: name old time/op new time/op delta IndexHasUnknown-8 50.6ns ±10% 41.0ns ±19% -18.92% (p=0.000 n=9+10) IndexHasKnown-8 52.6ns ±12% 41.5ns ±12% -21.13% (p=0.000 n=9+10) IndexMapHash-8 3.64µs ± 1% 2.00µs ± 0% -45.09% (p=0.000 n=10+9) IndexAlloc-8 700ms ± 1% 601ms ± 6% -14.18% (p=0.000 n=8+10) IndexAllocParallel-8 205ms ± 5% 192ms ± 8% -6.18% (p=0.043 n=10+10) MasterIndexAlloc-8 319ms ± 1% 279ms ± 5% -12.58% (p=0.000 n=10+10) MasterIndexLookupSingleIndex-8 156ns ± 8% 147ns ± 6% -5.46% (p=0.023 n=10+10) MasterIndexLookupMultipleIndex-8 150ns ± 7% 142ns ± 8% -5.69% (p=0.007 n=10+10) MasterIndexLookupSingleIndexUnknown-8 74.4ns ± 6% 72.0ns ± 9% ~ (p=0.175 n=10+9) MasterIndexLookupMultipleIndexUnknown-8 67.4ns ± 9% 65.5ns ± 7% ~ (p=0.340 n=9+9) MasterIndexLookupParallel/known,indices=25-8 461ns ± 2% 445ns ± 2% -3.49% (p=0.000 n=10+10) MasterIndexLookupParallel/unknown,indices=25-8 408ns ±11% 378ns ± 5% -7.22% (p=0.035 n=10+9) MasterIndexLookupParallel/known,indices=50-8 479ns ± 1% 437ns ± 4% -8.82% (p=0.000 n=10+10) MasterIndexLookupParallel/unknown,indices=50-8 406ns ± 8% 343ns ±15% -15.44% (p=0.001 n=10+10) MasterIndexLookupParallel/known,indices=100-8 480ns ± 1% 455ns ± 5% -5.15% (p=0.000 n=8+10) MasterIndexLookupParallel/unknown,indices=100-8 391ns ±18% 382ns ± 8% ~ (p=0.315 n=10+10) MasterIndexLookupBlobSize-8 71.0ns ± 8% 57.2ns ±11% -19.36% (p=0.000 n=9+10) PackerManager-8 254ms ± 1% 254ms ± 1% ~ (p=0.285 n=15+15) name old speed new speed delta IndexMapHash-8 1.12GB/s ± 1% 2.05GB/s ± 0% +82.13% (p=0.000 n=10+9) PackerManager-8 208MB/s ± 1% 207MB/s ± 1% ~ (p=0.281 n=15+15) name old alloc/op new alloc/op delta IndexMapHash-8 0.00B 0.00B ~ (all equal) IndexAlloc-8 400MB ± 0% 400MB ± 0% ~ (p=1.000 n=9+10) IndexAllocParallel-8 401MB ± 0% 401MB ± 0% +0.00% (p=0.000 n=10+10) MasterIndexAlloc-8 258MB ± 0% 262MB ± 0% +1.42% (p=0.000 n=9+10) PackerManager-8 73.1kB ± 0% 73.1kB ± 0% ~ (p=0.382 n=13+13) name old allocs/op new allocs/op delta IndexMapHash-8 0.00 0.00 ~ (all equal) IndexAlloc-8 907k ± 0% 907k ± 0% -0.00% (p=0.000 n=10+10) IndexAllocParallel-8 907k ± 0% 907k ± 0% +0.00% (p=0.009 n=10+10) MasterIndexAlloc-8 327k ± 0% 317k ± 0% -3.06% (p=0.000 n=10+10) PackerManager-8 744 ± 0% 744 ± 0% ~ (all equal)
130 lines
2.4 KiB
Go
130 lines
2.4 KiB
Go
package repository
|
|
|
|
import (
|
|
"math/rand"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/restic/restic/internal/restic"
|
|
rtest "github.com/restic/restic/internal/test"
|
|
)
|
|
|
|
func TestIndexMapBasic(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
var (
|
|
id restic.ID
|
|
m indexMap
|
|
r = rand.New(rand.NewSource(98765))
|
|
)
|
|
|
|
for i := 1; i <= 400; i++ {
|
|
r.Read(id[:])
|
|
rtest.Assert(t, m.get(id) == nil, "%v retrieved but not added", id)
|
|
|
|
m.add(id, 0, 0, 0)
|
|
rtest.Assert(t, m.get(id) != nil, "%v added but not retrieved", id)
|
|
rtest.Equals(t, uint(i), m.len())
|
|
}
|
|
}
|
|
|
|
func TestIndexMapForeach(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
const N = 10
|
|
|
|
var m indexMap
|
|
|
|
// Don't crash on empty map.
|
|
m.foreach(func(*indexEntry) bool { return true })
|
|
|
|
for i := 0; i < N; i++ {
|
|
var id restic.ID
|
|
id[0] = byte(i)
|
|
m.add(id, i, uint32(i), uint32(i))
|
|
}
|
|
|
|
seen := make(map[int]struct{})
|
|
m.foreach(func(e *indexEntry) bool {
|
|
i := int(e.id[0])
|
|
rtest.Assert(t, i < N, "unknown id %v in indexMap", e.id)
|
|
rtest.Equals(t, i, e.packIndex)
|
|
rtest.Equals(t, i, int(e.length))
|
|
rtest.Equals(t, i, int(e.offset))
|
|
|
|
seen[i] = struct{}{}
|
|
return true
|
|
})
|
|
|
|
rtest.Equals(t, N, len(seen))
|
|
|
|
ncalls := 0
|
|
m.foreach(func(*indexEntry) bool {
|
|
ncalls++
|
|
return false
|
|
})
|
|
rtest.Equals(t, 1, ncalls)
|
|
}
|
|
|
|
func TestIndexMapForeachWithID(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
const ndups = 3
|
|
|
|
var (
|
|
id restic.ID
|
|
m indexMap
|
|
r = rand.New(rand.NewSource(1234321))
|
|
)
|
|
r.Read(id[:])
|
|
|
|
// No result (and no crash) for empty map.
|
|
n := 0
|
|
m.foreachWithID(id, func(*indexEntry) { n++ })
|
|
rtest.Equals(t, 0, n)
|
|
|
|
// Test insertion and retrieval of duplicates.
|
|
for i := 0; i < ndups; i++ {
|
|
m.add(id, i, 0, 0)
|
|
}
|
|
|
|
for i := 0; i < 100; i++ {
|
|
var otherid restic.ID
|
|
r.Read(otherid[:])
|
|
m.add(otherid, -1, 0, 0)
|
|
}
|
|
|
|
n = 0
|
|
var packs [ndups]bool
|
|
m.foreachWithID(id, func(e *indexEntry) {
|
|
packs[e.packIndex] = true
|
|
n++
|
|
})
|
|
rtest.Equals(t, ndups, n)
|
|
|
|
for i := range packs {
|
|
rtest.Assert(t, packs[i], "duplicate from pack %d not retrieved", i)
|
|
}
|
|
}
|
|
|
|
func BenchmarkIndexMapHash(b *testing.B) {
|
|
var m indexMap
|
|
m.add(restic.ID{}, 0, 0, 0) // Trigger lazy initialization.
|
|
|
|
ids := make([]restic.ID, 128) // 4 KiB.
|
|
r := rand.New(rand.NewSource(time.Now().UnixNano()))
|
|
for i := range ids {
|
|
r.Read(ids[i][:])
|
|
}
|
|
|
|
b.ReportAllocs()
|
|
b.SetBytes(int64(len(restic.ID{}) * len(ids)))
|
|
b.ResetTimer()
|
|
|
|
for i := 0; i < b.N; i++ {
|
|
for _, id := range ids {
|
|
m.hash(id)
|
|
}
|
|
}
|
|
}
|