octoleo/restic
2
2
Fork 0
mirror of https://github.com/octoleo/restic.git synced 2024-11-25 22:27:35 +00:00
Code Issues Releases Wiki Activity

chunker: move buffer pool to chunker package

Browse Source
This commit is contained in:
Alexander Neumann 2015-05-05 00:45:29 +02:00
parent bdbb3ab329
commit ea22b2dfb1
4 changed files with 26 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
archiver.go
View File

@ -170,12 +170,6 @@ func updateNodeContent(node *Node, results []saveResult) error {
return nil
}
const chunkerBufSize = 512 * chunker.KiB
var chunkerBufPool = sync.Pool{
New: func() interface{} { return make([]byte, chunkerBufSize) },
}
// SaveFile stores the content of the file on the backend as a Blob by calling
// Save for each chunk.
func (arch *Archiver) SaveFile(p *Progress, node *Node) error {
@ -190,9 +184,7 @@ func (arch *Archiver) SaveFile(p *Progress, node *Node) error {
return err
}
buf := chunkerBufPool.Get().([]byte)
chnker := chunker.New(file, arch.s.Config.ChunkerPolynomial, buf, sha256.New())
defer chunkerBufPool.Put(buf)
chnker := chunker.New(file, arch.s.Config.ChunkerPolynomial, sha256.New())
resultChannels := [](<-chan saveResult){}
for {

14
archiver_test.go
View File

@ -25,9 +25,9 @@ type Rdr interface {
io.ReaderAt
}
func benchmarkChunkEncrypt(b testing.TB, buf, buf2, chunkBuf []byte, rd Rdr, key *crypto.Key) {
func benchmarkChunkEncrypt(b testing.TB, buf, buf2 []byte, rd Rdr, key *crypto.Key) {
rd.Seek(0, 0)
ch := chunker.New(rd, testPol, chunkBuf, sha256.New())
ch := chunker.New(rd, testPol, sha256.New())
for {
chunk, err := ch.Next()
@ -58,18 +58,17 @@ func BenchmarkChunkEncrypt(b *testing.B) {
buf := make([]byte, chunker.MaxSize)
buf2 := make([]byte, chunker.MaxSize)
chunkBuf := make([]byte, chunkerBufSize)
b.ResetTimer()
b.SetBytes(int64(len(data)))
for i := 0; i < b.N; i++ {
benchmarkChunkEncrypt(b, buf, buf2, chunkBuf, rd, s.Key())
benchmarkChunkEncrypt(b, buf, buf2, rd, s.Key())
}
}
func benchmarkChunkEncryptP(b *testing.PB, buf, chunkBuf []byte, rd Rdr, key *crypto.Key) {
ch := chunker.New(rd, testPol, chunkBuf, sha256.New())
func benchmarkChunkEncryptP(b *testing.PB, buf []byte, rd Rdr, key *crypto.Key) {
ch := chunker.New(rd, testPol, sha256.New())
for {
chunk, err := ch.Next()
@ -91,7 +90,6 @@ func BenchmarkChunkEncryptParallel(b *testing.B) {
data := Random(23, 10<<20) // 10MiB
buf := make([]byte, chunker.MaxSize)
chunkBuf := make([]byte, chunkerBufSize)
b.ResetTimer()
b.SetBytes(int64(len(data)))
@ -99,7 +97,7 @@ func BenchmarkChunkEncryptParallel(b *testing.B) {
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
rd := bytes.NewReader(data)
benchmarkChunkEncryptP(pb, buf, chunkBuf, rd, s.Key())
benchmarkChunkEncryptP(pb, buf, rd, s.Key())
}
})
}

21
chunker/chunker.go
View File

@ -23,8 +23,14 @@ const (
MaxSize = 8 * MiB
splitmask = (1 << averageBits) - 1
chunkerBufSize = 512 * KiB
)
var bufPool = sync.Pool{
New: func() interface{} { return make([]byte, chunkerBufSize) },
}
type tables struct {
out [256]Pol
mod [256]Pol
@ -79,18 +85,12 @@ type Chunker struct {
h hash.Hash
}
const minBufSize = 32
// New returns a new Chunker based on polynomial p that reads from rd
// with bufsize and pass all data to hash along the way, using buf for
// buffering. Buf must at least hold 32 bytes.
func New(rd io.Reader, pol Pol, buf []byte, h hash.Hash) *Chunker {
if len(buf) < minBufSize {
buf = make([]byte, minBufSize)
}
// buffering.
func New(rd io.Reader, pol Pol, h hash.Hash) *Chunker {
c := &Chunker{
buf: buf,
buf: bufPool.Get().([]byte),
h: h,
pol: pol,
rd: rd,
@ -204,6 +204,9 @@ func (c *Chunker) Next() (*Chunk, error) {
if err == io.EOF && !c.closed {
c.closed = true
// return the buffer to the pool
bufPool.Put(c.buf)
// return current chunk, if any bytes have been processed
if c.count > 0 {
return &Chunk{

23
chunker/chunker_test.go
View File

@ -147,13 +147,10 @@ func getRandom(seed, count int) []byte {
return buf
}
const chunkerBufSize = 512 * chunker.KiB
func TestChunker(t *testing.T) {
// setup data source
buf := getRandom(23, 32*1024*1024)
chunkBuf := make([]byte, chunkerBufSize)
ch := chunker.New(bytes.NewReader(buf), testPol, chunkBuf, sha256.New())
ch := chunker.New(bytes.NewReader(buf), testPol, sha256.New())
chunks := testWithData(t, ch, chunks1)
// test reader
@ -180,7 +177,7 @@ func TestChunker(t *testing.T) {
// setup nullbyte data source
buf = bytes.Repeat([]byte{0}, len(chunks2)*chunker.MinSize)
ch = chunker.New(bytes.NewReader(buf), testPol, chunkBuf, sha256.New())
ch = chunker.New(bytes.NewReader(buf), testPol, sha256.New())
testWithData(t, ch, chunks2)
}
@ -188,7 +185,6 @@ func TestChunker(t *testing.T) {
func TestChunkerWithRandomPolynomial(t *testing.T) {
// setup data source
buf := getRandom(23, 32*1024*1024)
chunkBuf := make([]byte, chunkerBufSize)
// generate a new random polynomial
start := time.Now()
@ -197,7 +193,7 @@ func TestChunkerWithRandomPolynomial(t *testing.T) {
t.Logf("generating random polynomial took %v", time.Since(start))
start = time.Now()
ch := chunker.New(bytes.NewReader(buf), p, chunkBuf, sha256.New())
ch := chunker.New(bytes.NewReader(buf), p, sha256.New())
t.Logf("creating chunker took %v", time.Since(start))
// make sure that first chunk is different
@ -214,9 +210,8 @@ func TestChunkerWithRandomPolynomial(t *testing.T) {
func TestChunkerWithoutHash(t *testing.T) {
// setup data source
buf := getRandom(23, 32*1024*1024)
chunkBuf := make([]byte, chunkerBufSize)
ch := chunker.New(bytes.NewReader(buf), testPol, chunkBuf, nil)
ch := chunker.New(bytes.NewReader(buf), testPol, nil)
chunks := testWithData(t, ch, chunks1)
// test reader
@ -246,7 +241,7 @@ func TestChunkerWithoutHash(t *testing.T) {
// setup nullbyte data source
buf = bytes.Repeat([]byte{0}, len(chunks2)*chunker.MinSize)
ch = chunker.New(bytes.NewReader(buf), testPol, chunkBuf, sha256.New())
ch = chunker.New(bytes.NewReader(buf), testPol, sha256.New())
testWithData(t, ch, chunks2)
}
@ -276,8 +271,6 @@ func benchmarkChunker(b *testing.B, hash hash.Hash) {
rd = bytes.NewReader(getRandom(23, size))
}
chunkBuf := make([]byte, chunkerBufSize)
b.ResetTimer()
b.SetBytes(int64(size))
@ -286,7 +279,7 @@ func benchmarkChunker(b *testing.B, hash hash.Hash) {
chunks = 0
rd.Seek(0, 0)
ch := chunker.New(rd, testPol, chunkBuf, hash)
ch := chunker.New(rd, testPol, hash)
for {
_, err := ch.Next()
@ -322,11 +315,9 @@ func BenchmarkNewChunker(b *testing.B) {
p, err := chunker.RandomPolynomial()
OK(b, err)
chunkBuf := make([]byte, chunkerBufSize)
b.ResetTimer()
for i := 0; i < b.N; i++ {
chunker.New(bytes.NewBuffer(nil), p, chunkBuf, nil)
chunker.New(bytes.NewBuffer(nil), p, nil)
}
}