mirror of
https://github.com/octoleo/restic.git
synced 2024-12-23 19:38:57 +00:00
7bda28f31f
These are faster to construct but slower to access. The allocation rate is halved, the peak memory usage almost halved compared to standard map. Benchmark results on linux/amd64, -benchtime=3s -count=20: name old time/op new time/op delta PackerManager-8 178ms ± 0% 178ms ± 0% ~ (p=0.231 n=20+20) DecodeIndex-8 4.54s ± 0% 4.30s ± 0% -5.20% (p=0.000 n=18+17) DecodeIndexParallel-8 4.54s ± 0% 4.30s ± 0% -5.22% (p=0.000 n=19+18) IndexHasUnknown-8 44.4ns ± 5% 50.5ns ±11% +13.82% (p=0.000 n=19+17) IndexHasKnown-8 48.3ns ± 0% 51.5ns ±12% +6.68% (p=0.001 n=16+20) IndexAlloc-8 758ms ± 1% 616ms ± 1% -18.69% (p=0.000 n=19+19) IndexAllocParallel-8 234ms ± 3% 204ms ± 2% -12.60% (p=0.000 n=20+18) MasterIndexLookupSingleIndex-8 122ns ± 0% 145ns ± 9% +18.44% (p=0.000 n=14+20) MasterIndexLookupMultipleIndex-8 369ns ± 2% 429ns ± 8% +16.27% (p=0.000 n=20+20) MasterIndexLookupSingleIndexUnknown-8 68.4ns ± 5% 74.9ns ±13% +9.47% (p=0.000 n=20+20) MasterIndexLookupMultipleIndexUnknown-8 315ns ± 3% 369ns ±11% +17.14% (p=0.000 n=20+20) MasterIndexLookupParallel/known,indices=5-8 743ns ± 1% 816ns ± 2% +9.87% (p=0.000 n=17+17) MasterIndexLookupParallel/unknown,indices=5-8 238ns ± 1% 260ns ± 2% +9.14% (p=0.000 n=19+20) MasterIndexLookupParallel/known,indices=10-8 1.01µs ± 3% 1.11µs ± 2% +9.79% (p=0.000 n=19+20) MasterIndexLookupParallel/unknown,indices=10-8 222ns ± 0% 269ns ± 2% +20.83% (p=0.000 n=16+20) MasterIndexLookupParallel/known,indices=20-8 1.06µs ± 2% 1.19µs ± 2% +12.95% (p=0.000 n=19+18) MasterIndexLookupParallel/unknown,indices=20-8 413ns ± 1% 530ns ± 1% +28.19% (p=0.000 n=18+20) SaveAndEncrypt-8 30.2ms ± 1% 30.4ms ± 0% +0.71% (p=0.000 n=19+19) LoadTree-8 540µs ± 1% 576µs ± 1% +6.73% (p=0.000 n=20+20) LoadBlob-8 5.64ms ± 0% 5.64ms ± 0% ~ (p=0.883 n=18+17) LoadAndDecrypt-8 5.93ms ± 0% 5.95ms ± 1% ~ (p=0.247 n=20+19) LoadIndex-8 25.1ms ± 0% 24.5ms ± 1% -2.54% (p=0.000 n=18+17) name old speed new speed delta PackerManager-8 296MB/s ± 0% 296MB/s ± 0% ~ (p=0.229 n=20+20) SaveAndEncrypt-8 139MB/s ± 1% 138MB/s ± 0% -0.71% (p=0.000 n=19+19) LoadBlob-8 177MB/s ± 0% 177MB/s ± 0% ~ (p=0.890 n=18+17) LoadAndDecrypt-8 169MB/s ± 0% 168MB/s ± 1% ~ (p=0.227 n=20+19) name old alloc/op new alloc/op delta PackerManager-8 91.8kB ± 0% 91.8kB ± 0% ~ (p=0.772 n=12+19) IndexAlloc-8 786MB ± 0% 400MB ± 0% -49.04% (p=0.000 n=20+18) IndexAllocParallel-8 786MB ± 0% 401MB ± 0% -49.04% (p=0.000 n=19+15) SaveAndEncrypt-8 21.0MB ± 0% 21.0MB ± 0% +0.00% (p=0.000 n=19+19) name old allocs/op new allocs/op delta PackerManager-8 1.41k ± 0% 1.41k ± 0% ~ (all equal) IndexAlloc-8 977k ± 0% 907k ± 0% -7.18% (p=0.000 n=20+20) IndexAllocParallel-8 977k ± 0% 907k ± 0% -7.17% (p=0.000 n=19+15) SaveAndEncrypt-8 73.0 ± 0% 73.0 ± 0% ~ (all equal)
156 lines
2.9 KiB
Go
156 lines
2.9 KiB
Go
package repository
|
|
|
|
import (
|
|
"math/rand"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/restic/restic/internal/restic"
|
|
rtest "github.com/restic/restic/internal/test"
|
|
)
|
|
|
|
func TestIndexMapBasic(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
var (
|
|
id restic.ID
|
|
m indexMap
|
|
r = rand.New(rand.NewSource(98765))
|
|
)
|
|
|
|
for i := 1; i <= 400; i++ {
|
|
r.Read(id[:])
|
|
rtest.Assert(t, m.get(id) == nil, "%v retrieved but not added", id)
|
|
|
|
m.add(id, 0, 0, 0)
|
|
rtest.Assert(t, m.get(id) != nil, "%v added but not retrieved", id)
|
|
rtest.Equals(t, uint(i), m.len())
|
|
}
|
|
}
|
|
|
|
func TestIndexMapForeach(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
const N = 10
|
|
|
|
var m indexMap
|
|
|
|
// Don't crash on empty map.
|
|
m.foreach(func(*indexEntry) bool { return true })
|
|
|
|
for i := 0; i < N; i++ {
|
|
var id restic.ID
|
|
id[0] = byte(i)
|
|
m.add(id, i, uint32(i), uint32(i))
|
|
}
|
|
|
|
seen := make(map[int]struct{})
|
|
m.foreach(func(e *indexEntry) bool {
|
|
i := int(e.id[0])
|
|
rtest.Assert(t, i < N, "unknown id %v in indexMap", e.id)
|
|
rtest.Equals(t, i, e.packIndex)
|
|
rtest.Equals(t, i, int(e.length))
|
|
rtest.Equals(t, i, int(e.offset))
|
|
|
|
seen[i] = struct{}{}
|
|
return true
|
|
})
|
|
|
|
rtest.Equals(t, N, len(seen))
|
|
|
|
ncalls := 0
|
|
m.foreach(func(*indexEntry) bool {
|
|
ncalls++
|
|
return false
|
|
})
|
|
rtest.Equals(t, 1, ncalls)
|
|
}
|
|
|
|
func TestIndexMapForeachWithID(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
const ndups = 3
|
|
|
|
var (
|
|
id restic.ID
|
|
m indexMap
|
|
r = rand.New(rand.NewSource(1234321))
|
|
)
|
|
r.Read(id[:])
|
|
|
|
// No result (and no crash) for empty map.
|
|
n := 0
|
|
m.foreachWithID(id, func(*indexEntry) { n++ })
|
|
rtest.Equals(t, 0, n)
|
|
|
|
// Test insertion and retrieval of duplicates.
|
|
for i := 0; i < ndups; i++ {
|
|
m.add(id, i, 0, 0)
|
|
}
|
|
|
|
for i := 0; i < 100; i++ {
|
|
var otherid restic.ID
|
|
r.Read(otherid[:])
|
|
m.add(otherid, -1, 0, 0)
|
|
}
|
|
|
|
n = 0
|
|
var packs [ndups]bool
|
|
m.foreachWithID(id, func(e *indexEntry) {
|
|
packs[e.packIndex] = true
|
|
n++
|
|
})
|
|
rtest.Equals(t, ndups, n)
|
|
|
|
for i := range packs {
|
|
rtest.Assert(t, packs[i], "duplicate from pack %d not retrieved", i)
|
|
}
|
|
}
|
|
|
|
func TestIndexMapHash(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
var m1, m2 indexMap
|
|
|
|
id := restic.NewRandomID()
|
|
// Add to both maps to initialize them.
|
|
m1.add(id, 0, 0, 0)
|
|
m2.add(id, 0, 0, 0)
|
|
|
|
h1 := m1.hash(id)
|
|
h2 := m2.hash(id)
|
|
|
|
rtest.Equals(t, len(m1.buckets), len(m2.buckets)) // just to be sure
|
|
|
|
if h1 == h2 {
|
|
// The probability of the zero key should be 2^(-128).
|
|
if m1.key0 == 0 && m1.key1 == 0 {
|
|
t.Error("siphash key not set for m1")
|
|
}
|
|
if m2.key0 == 0 && m2.key1 == 0 {
|
|
t.Error("siphash key not set for m2")
|
|
}
|
|
}
|
|
}
|
|
|
|
func BenchmarkIndexMapHash(b *testing.B) {
|
|
var m indexMap
|
|
m.add(restic.ID{}, 0, 0, 0) // Trigger lazy initialization.
|
|
|
|
ids := make([]restic.ID, 128) // 4 KiB.
|
|
r := rand.New(rand.NewSource(time.Now().UnixNano()))
|
|
for i := range ids {
|
|
r.Read(ids[i][:])
|
|
}
|
|
|
|
b.ReportAllocs()
|
|
b.SetBytes(int64(len(restic.ID{}) * len(ids)))
|
|
b.ResetTimer()
|
|
|
|
for i := 0; i < b.N; i++ {
|
|
for _, id := range ids {
|
|
m.hash(id)
|
|
}
|
|
}
|
|
}
|