mirror of
https://github.com/octoleo/restic.git
synced 2024-11-22 12:55:18 +00:00
bloblru: Upgrade to hashicorp/golang-lru/v2
The new genericized LRU cache no longer needs to have the IDs separately allocated: name old time/op new time/op delta Add-8 494ns ± 2% 388ns ± 2% -21.46% (p=0.000 n=10+9) name old alloc/op new alloc/op delta Add-8 176B ± 0% 152B ± 0% -13.64% (p=0.000 n=10+10) name old allocs/op new allocs/op delta Add-8 5.00 ± 0% 3.00 ± 0% -40.00% (p=0.000 n=10+10)
This commit is contained in:
parent
05cebc1c4b
commit
e5d597fd22
2
go.mod
2
go.mod
@ -9,7 +9,7 @@ require (
|
|||||||
github.com/elithrar/simple-scrypt v1.3.0
|
github.com/elithrar/simple-scrypt v1.3.0
|
||||||
github.com/go-ole/go-ole v1.2.6
|
github.com/go-ole/go-ole v1.2.6
|
||||||
github.com/google/go-cmp v0.5.9
|
github.com/google/go-cmp v0.5.9
|
||||||
github.com/hashicorp/golang-lru v0.5.4
|
github.com/hashicorp/golang-lru/v2 v2.0.1
|
||||||
github.com/juju/ratelimit v1.0.2
|
github.com/juju/ratelimit v1.0.2
|
||||||
github.com/klauspost/compress v1.15.9
|
github.com/klauspost/compress v1.15.9
|
||||||
github.com/kurin/blazer v0.5.4-0.20211030221322-ba894c124ac6
|
github.com/kurin/blazer v0.5.4-0.20211030221322-ba894c124ac6
|
||||||
|
4
go.sum
4
go.sum
@ -109,8 +109,8 @@ github.com/googleapis/enterprise-certificate-proxy v0.2.0 h1:y8Yozv7SZtlU//QXbez
|
|||||||
github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg=
|
github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg=
|
||||||
github.com/googleapis/gax-go/v2 v2.7.0 h1:IcsPKeInNvYi7eqSaDjiZqDDKu5rsmunY0Y1YupQSSQ=
|
github.com/googleapis/gax-go/v2 v2.7.0 h1:IcsPKeInNvYi7eqSaDjiZqDDKu5rsmunY0Y1YupQSSQ=
|
||||||
github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8=
|
github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8=
|
||||||
github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
|
github.com/hashicorp/golang-lru/v2 v2.0.1 h1:5pv5N1lT1fjLg2VQ5KWc7kmucp2x/kvFOnxuVTqZ6x4=
|
||||||
github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
|
github.com/hashicorp/golang-lru/v2 v2.0.1/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
|
||||||
github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w=
|
github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w=
|
||||||
github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc=
|
github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc=
|
||||||
github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
||||||
|
@ -6,7 +6,7 @@ import (
|
|||||||
"github.com/restic/restic/internal/debug"
|
"github.com/restic/restic/internal/debug"
|
||||||
"github.com/restic/restic/internal/restic"
|
"github.com/restic/restic/internal/restic"
|
||||||
|
|
||||||
"github.com/hashicorp/golang-lru/simplelru"
|
"github.com/hashicorp/golang-lru/v2/simplelru"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Crude estimate of the overhead per blob: a SHA-256, a linked list node
|
// Crude estimate of the overhead per blob: a SHA-256, a linked list node
|
||||||
@ -17,7 +17,7 @@ const overhead = len(restic.ID{}) + 64
|
|||||||
// It is safe for concurrent access.
|
// It is safe for concurrent access.
|
||||||
type Cache struct {
|
type Cache struct {
|
||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
c *simplelru.LRU
|
c *simplelru.LRU[restic.ID, []byte]
|
||||||
|
|
||||||
free, size int // Current and max capacity, in bytes.
|
free, size int // Current and max capacity, in bytes.
|
||||||
}
|
}
|
||||||
@ -33,7 +33,7 @@ func New(size int) *Cache {
|
|||||||
// The actual maximum will be smaller than size/overhead, because we
|
// The actual maximum will be smaller than size/overhead, because we
|
||||||
// evict entries (RemoveOldest in add) to maintain our size bound.
|
// evict entries (RemoveOldest in add) to maintain our size bound.
|
||||||
maxEntries := size / overhead
|
maxEntries := size / overhead
|
||||||
lru, err := simplelru.NewLRU(maxEntries, c.evict)
|
lru, err := simplelru.NewLRU[restic.ID, []byte](maxEntries, c.evict)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
panic(err) // Can only be maxEntries <= 0.
|
panic(err) // Can only be maxEntries <= 0.
|
||||||
}
|
}
|
||||||
@ -55,24 +55,21 @@ func (c *Cache) Add(id restic.ID, blob []byte) (old []byte) {
|
|||||||
c.mu.Lock()
|
c.mu.Lock()
|
||||||
defer c.mu.Unlock()
|
defer c.mu.Unlock()
|
||||||
|
|
||||||
var key interface{} = id
|
if c.c.Contains(id) { // Doesn't update the recency list.
|
||||||
|
|
||||||
if c.c.Contains(key) { // Doesn't update the recency list.
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// This loop takes at most min(maxEntries, maxchunksize/overhead)
|
// This loop takes at most min(maxEntries, maxchunksize/overhead)
|
||||||
// iterations.
|
// iterations.
|
||||||
for size > c.free {
|
for size > c.free {
|
||||||
_, val, _ := c.c.RemoveOldest()
|
_, b, _ := c.c.RemoveOldest()
|
||||||
b := val.([]byte)
|
|
||||||
if cap(b) > cap(old) {
|
if cap(b) > cap(old) {
|
||||||
// We can only return one buffer, so pick the largest.
|
// We can only return one buffer, so pick the largest.
|
||||||
old = b
|
old = b
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
c.c.Add(key, blob)
|
c.c.Add(id, blob)
|
||||||
c.free -= size
|
c.free -= size
|
||||||
|
|
||||||
return old
|
return old
|
||||||
@ -80,17 +77,15 @@ func (c *Cache) Add(id restic.ID, blob []byte) (old []byte) {
|
|||||||
|
|
||||||
func (c *Cache) Get(id restic.ID) ([]byte, bool) {
|
func (c *Cache) Get(id restic.ID) ([]byte, bool) {
|
||||||
c.mu.Lock()
|
c.mu.Lock()
|
||||||
value, ok := c.c.Get(id)
|
blob, ok := c.c.Get(id)
|
||||||
c.mu.Unlock()
|
c.mu.Unlock()
|
||||||
|
|
||||||
debug.Log("bloblru.Cache: get %v, hit %v", id, ok)
|
debug.Log("bloblru.Cache: get %v, hit %v", id, ok)
|
||||||
|
|
||||||
blob, ok := value.([]byte)
|
|
||||||
return blob, ok
|
return blob, ok
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Cache) evict(key, value interface{}) {
|
func (c *Cache) evict(key restic.ID, blob []byte) {
|
||||||
blob := value.([]byte)
|
|
||||||
debug.Log("bloblru.Cache: evict %v, %d bytes", key, cap(blob))
|
debug.Log("bloblru.Cache: evict %v, %d bytes", key, cap(blob))
|
||||||
c.free += cap(blob) + overhead
|
c.free += cap(blob) + overhead
|
||||||
}
|
}
|
||||||
|
@ -1,6 +1,7 @@
|
|||||||
package bloblru
|
package bloblru
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"math/rand"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/restic/restic/internal/restic"
|
"github.com/restic/restic/internal/restic"
|
||||||
@ -50,3 +51,29 @@ func TestCache(t *testing.T) {
|
|||||||
rtest.Equals(t, cacheSize, c.size)
|
rtest.Equals(t, cacheSize, c.size)
|
||||||
rtest.Equals(t, cacheSize, c.free)
|
rtest.Equals(t, cacheSize, c.free)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func BenchmarkAdd(b *testing.B) {
|
||||||
|
const (
|
||||||
|
MiB = 1 << 20
|
||||||
|
nblobs = 64
|
||||||
|
)
|
||||||
|
|
||||||
|
c := New(64 * MiB)
|
||||||
|
|
||||||
|
buf := make([]byte, 8*MiB)
|
||||||
|
ids := make([]restic.ID, nblobs)
|
||||||
|
sizes := make([]int, nblobs)
|
||||||
|
|
||||||
|
r := rand.New(rand.NewSource(100))
|
||||||
|
for i := range ids {
|
||||||
|
r.Read(ids[i][:])
|
||||||
|
sizes[i] = r.Intn(8 * MiB)
|
||||||
|
}
|
||||||
|
|
||||||
|
b.ResetTimer()
|
||||||
|
b.ReportAllocs()
|
||||||
|
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
c.Add(ids[i%nblobs], buf[:sizes[i%nblobs]])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user