octoleo/restic
2
2
Fork 0
mirror of https://github.com/octoleo/restic.git synced 2024-06-04 01:50:48 +00:00
Code Issues Releases Wiki Activity

Add chunker pool

Browse Source
This commit is contained in:
Alexander Neumann 2015-02-09 23:38:50 +01:00
parent 3daeee8e2c
commit bbb3145f11
2 changed files with 54 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
archiver.go
View File

@ -1,7 +1,6 @@
package restic
import (
"crypto/sha256"
"encoding/json"
"errors"
"fmt"
@ -18,7 +17,9 @@ import (
const (
maxConcurrentFiles = 16
maxConcurrentBlobs = 16
chunkerBufSize = 512 * chunker.KiB
// chunkerBufSize is used in pool.go
chunkerBufSize = 512 * chunker.KiB
)
type Archiver struct {
@ -161,9 +162,10 @@ func (arch *Archiver) SaveFile(node *Node) (Blobs, error) {
var blobs Blobs
// store all chunks
chnker := chunker.New(file, chunkerBufSize, sha256.New)
chnker := GetChunker("archiver.SaveFile")
chnker.Reset(file)
chans := [](<-chan Blob){}
//defer chnker.Free()
defer FreeChunker("archiver.SaveFile", chnker)
chunks := 0

55
pools.go
View File

@ -1,8 +1,10 @@
package restic
import (
"crypto/sha256"
"sync"
"github.com/restic/restic/chunker"
"github.com/restic/restic/debug"
)
@ -12,6 +14,7 @@ type poolStats struct {
mput map[string]int
mmax map[string]int
new int
get int
put int
max int
@ -63,23 +66,42 @@ func newPoolStats() *poolStats {
}
var (
chunkPool = sync.Pool{New: newChunkBuf}
nodePool = sync.Pool{New: newNode}
chunkPool = sync.Pool{New: newChunkBuf}
nodePool = sync.Pool{New: newNode}
chunkerPool = sync.Pool{New: newChunker}
chunkStats = newPoolStats()
nodeStats = newPoolStats()
chunkStats = newPoolStats()
nodeStats = newPoolStats()
chunkerStats = newPoolStats()
)
func newChunkBuf() interface{} {
chunkStats.m.Lock()
defer chunkStats.m.Unlock()
chunkStats.new++
// create buffer for iv, data and hmac
return make([]byte, maxCiphertextSize)
}
func newNode() interface{} {
nodeStats.m.Lock()
defer nodeStats.m.Unlock()
nodeStats.new++
// create buffer for iv, data and hmac
return new(Node)
}
func newChunker() interface{} {
chunkStats.m.Lock()
defer chunkStats.m.Unlock()
chunkStats.new++
// create a new chunker with a nil reader
return chunker.New(nil, chunkerBufSize, sha256.New)
}
func GetChunkBuf(s string) []byte {
chunkStats.Get(s)
return chunkPool.Get().([]byte)
@ -100,14 +122,33 @@ func FreeNode(n *Node) {
nodePool.Put(n)
}
func GetChunker(s string) *chunker.Chunker {
chunkerStats.Get(s)
return chunkerPool.Get().(*chunker.Chunker)
}
func FreeChunker(s string, ch *chunker.Chunker) {
chunkerStats.Put(s)
chunkerPool.Put(ch)
}
func PoolAlloc() {
debug.Log("pools.PoolAlloc", "pool stats for chunk: get %d, put %d, diff %d, max %d\n", chunkStats.get, chunkStats.put, chunkStats.get-chunkStats.put, chunkStats.max)
debug.Log("pools.PoolAlloc", "pool stats for chunk: new %d, get %d, put %d, diff %d, max %d\n",
chunkStats.new, chunkStats.get, chunkStats.put, chunkStats.get-chunkStats.put, chunkStats.max)
for k, v := range chunkStats.mget {
debug.Log("pools.PoolAlloc", "pool stats for chunk[%s]: get %d, put %d, diff %d, max %d\n", k, v, chunkStats.mput[k], v-chunkStats.mput[k], chunkStats.mmax[k])
debug.Log("pools.PoolAlloc", "pool stats for chunk[%s]: get %d, put %d, diff %d, max %d\n",
k, v, chunkStats.mput[k], v-chunkStats.mput[k], chunkStats.mmax[k])
}
debug.Log("pools.PoolAlloc", "pool stats for node: get %d, put %d, diff %d, max %d\n", nodeStats.get, nodeStats.put, nodeStats.get-nodeStats.put, nodeStats.max)
debug.Log("pools.PoolAlloc", "pool stats for node: new %d, get %d, put %d, diff %d, max %d\n",
nodeStats.new, nodeStats.get, nodeStats.put, nodeStats.get-nodeStats.put, nodeStats.max)
for k, v := range nodeStats.mget {
debug.Log("pools.PoolAlloc", "pool stats for node[%s]: get %d, put %d, diff %d, max %d\n", k, v, nodeStats.mput[k], v-nodeStats.mput[k], nodeStats.mmax[k])
}
debug.Log("pools.PoolAlloc", "pool stats for chunker: new %d, get %d, put %d, diff %d, max %d\n",
chunkerStats.new, chunkerStats.get, chunkerStats.put, chunkerStats.get-chunkerStats.put, chunkerStats.max)
for k, v := range chunkerStats.mget {
debug.Log("pools.PoolAlloc", "pool stats for chunker[%s]: get %d, put %d, diff %d, max %d\n", k, v, chunkerStats.mput[k], v-chunkerStats.mput[k], chunkerStats.mmax[k])
}
}