chunker: move buffer pool to chunker package

archiver.go

@@ -170,12 +170,6 @@ func updateNodeContent(node *Node, results []saveResult) error {
 	return nil
 }
 
-const chunkerBufSize = 512 * chunker.KiB
-
-var chunkerBufPool = sync.Pool{
-	New: func() interface{} { return make([]byte, chunkerBufSize) },
-}
-
 // SaveFile stores the content of the file on the backend as a Blob by calling
 // Save for each chunk.
 func (arch *Archiver) SaveFile(p *Progress, node *Node) error {
@@ -190,9 +184,7 @@ func (arch *Archiver) SaveFile(p *Progress, node *Node) error {
 		return err
 	}
 
-	buf := chunkerBufPool.Get().([]byte)
+	chnker := chunker.New(file, arch.s.Config.ChunkerPolynomial, sha256.New())
-	chnker := chunker.New(file, arch.s.Config.ChunkerPolynomial, buf, sha256.New())
-	defer chunkerBufPool.Put(buf)
 	resultChannels := [](<-chan saveResult){}
 
 	for {

archiver_test.go

@@ -25,9 +25,9 @@ type Rdr interface {
 	io.ReaderAt
 }
 
-func benchmarkChunkEncrypt(b testing.TB, buf, buf2, chunkBuf []byte, rd Rdr, key *crypto.Key) {
+func benchmarkChunkEncrypt(b testing.TB, buf, buf2 []byte, rd Rdr, key *crypto.Key) {
 	rd.Seek(0, 0)
-	ch := chunker.New(rd, testPol, chunkBuf, sha256.New())
+	ch := chunker.New(rd, testPol, sha256.New())
 
 	for {
 		chunk, err := ch.Next()
@@ -58,18 +58,17 @@ func BenchmarkChunkEncrypt(b *testing.B) {
 
 	buf := make([]byte, chunker.MaxSize)
 	buf2 := make([]byte, chunker.MaxSize)
-	chunkBuf := make([]byte, chunkerBufSize)
 
 	b.ResetTimer()
 	b.SetBytes(int64(len(data)))
 
 	for i := 0; i < b.N; i++ {
-		benchmarkChunkEncrypt(b, buf, buf2, chunkBuf, rd, s.Key())
+		benchmarkChunkEncrypt(b, buf, buf2, rd, s.Key())
 	}
 }
 
-func benchmarkChunkEncryptP(b *testing.PB, buf, chunkBuf []byte, rd Rdr, key *crypto.Key) {
+func benchmarkChunkEncryptP(b *testing.PB, buf []byte, rd Rdr, key *crypto.Key) {
-	ch := chunker.New(rd, testPol, chunkBuf, sha256.New())
+	ch := chunker.New(rd, testPol, sha256.New())
 
 	for {
 		chunk, err := ch.Next()
@@ -91,7 +90,6 @@ func BenchmarkChunkEncryptParallel(b *testing.B) {
 	data := Random(23, 10<<20) // 10MiB
 
 	buf := make([]byte, chunker.MaxSize)
-	chunkBuf := make([]byte, chunkerBufSize)
 
 	b.ResetTimer()
 	b.SetBytes(int64(len(data)))
@@ -99,7 +97,7 @@ func BenchmarkChunkEncryptParallel(b *testing.B) {
 	b.RunParallel(func(pb *testing.PB) {
 		for pb.Next() {
 			rd := bytes.NewReader(data)
-			benchmarkChunkEncryptP(pb, buf, chunkBuf, rd, s.Key())
+			benchmarkChunkEncryptP(pb, buf, rd, s.Key())
 		}
 	})
 }

chunker/chunker.go

@@ -23,8 +23,14 @@ const (
 	MaxSize = 8 * MiB
 
 	splitmask = (1 << averageBits) - 1
+
+	chunkerBufSize = 512 * KiB
 )
 
+var bufPool = sync.Pool{
+	New: func() interface{} { return make([]byte, chunkerBufSize) },
+}
+
 type tables struct {
 	out [256]Pol
 	mod [256]Pol
@@ -79,18 +85,12 @@ type Chunker struct {
 	h      hash.Hash
 }
 
-const minBufSize = 32
-
 // New returns a new Chunker based on polynomial p that reads from rd
 // with bufsize and pass all data to hash along the way, using buf for
-// buffering. Buf must at least hold 32 bytes.
+// buffering.
-func New(rd io.Reader, pol Pol, buf []byte, h hash.Hash) *Chunker {
+func New(rd io.Reader, pol Pol, h hash.Hash) *Chunker {
-	if len(buf) < minBufSize {
-		buf = make([]byte, minBufSize)
-	}
-
 	c := &Chunker{
-		buf: buf,
+		buf: bufPool.Get().([]byte),
 		h:   h,
 		pol: pol,
 		rd:  rd,
@@ -204,6 +204,9 @@ func (c *Chunker) Next() (*Chunk, error) {
 			if err == io.EOF && !c.closed {
 				c.closed = true
 
+				// return the buffer to the pool
+				bufPool.Put(c.buf)
+
 				// return current chunk, if any bytes have been processed
 				if c.count > 0 {
 					return &Chunk{

chunker/chunker_test.go

@@ -147,13 +147,10 @@ func getRandom(seed, count int) []byte {
 	return buf
 }
 
-const chunkerBufSize = 512 * chunker.KiB
-
 func TestChunker(t *testing.T) {
 	// setup data source
 	buf := getRandom(23, 32*1024*1024)
-	chunkBuf := make([]byte, chunkerBufSize)
+	ch := chunker.New(bytes.NewReader(buf), testPol, sha256.New())
-	ch := chunker.New(bytes.NewReader(buf), testPol, chunkBuf, sha256.New())
 	chunks := testWithData(t, ch, chunks1)
 
 	// test reader
@@ -180,7 +177,7 @@ func TestChunker(t *testing.T) {
 
 	// setup nullbyte data source
 	buf = bytes.Repeat([]byte{0}, len(chunks2)*chunker.MinSize)
-	ch = chunker.New(bytes.NewReader(buf), testPol, chunkBuf, sha256.New())
+	ch = chunker.New(bytes.NewReader(buf), testPol, sha256.New())
 
 	testWithData(t, ch, chunks2)
 }
@@ -188,7 +185,6 @@ func TestChunker(t *testing.T) {
 func TestChunkerWithRandomPolynomial(t *testing.T) {
 	// setup data source
 	buf := getRandom(23, 32*1024*1024)
-	chunkBuf := make([]byte, chunkerBufSize)
 
 	// generate a new random polynomial
 	start := time.Now()
@@ -197,7 +193,7 @@ func TestChunkerWithRandomPolynomial(t *testing.T) {
 	t.Logf("generating random polynomial took %v", time.Since(start))
 
 	start = time.Now()
-	ch := chunker.New(bytes.NewReader(buf), p, chunkBuf, sha256.New())
+	ch := chunker.New(bytes.NewReader(buf), p, sha256.New())
 	t.Logf("creating chunker took %v", time.Since(start))
 
 	// make sure that first chunk is different
@@ -214,9 +210,8 @@ func TestChunkerWithRandomPolynomial(t *testing.T) {
 func TestChunkerWithoutHash(t *testing.T) {
 	// setup data source
 	buf := getRandom(23, 32*1024*1024)
-	chunkBuf := make([]byte, chunkerBufSize)
 
-	ch := chunker.New(bytes.NewReader(buf), testPol, chunkBuf, nil)
+	ch := chunker.New(bytes.NewReader(buf), testPol, nil)
 	chunks := testWithData(t, ch, chunks1)
 
 	// test reader
@@ -246,7 +241,7 @@ func TestChunkerWithoutHash(t *testing.T) {
 
 	// setup nullbyte data source
 	buf = bytes.Repeat([]byte{0}, len(chunks2)*chunker.MinSize)
-	ch = chunker.New(bytes.NewReader(buf), testPol, chunkBuf, sha256.New())
+	ch = chunker.New(bytes.NewReader(buf), testPol, sha256.New())
 
 	testWithData(t, ch, chunks2)
 }
@@ -276,8 +271,6 @@ func benchmarkChunker(b *testing.B, hash hash.Hash) {
 		rd = bytes.NewReader(getRandom(23, size))
 	}
 
-	chunkBuf := make([]byte, chunkerBufSize)
-
 	b.ResetTimer()
 	b.SetBytes(int64(size))
 
@@ -286,7 +279,7 @@ func benchmarkChunker(b *testing.B, hash hash.Hash) {
 		chunks = 0
 
 		rd.Seek(0, 0)
-		ch := chunker.New(rd, testPol, chunkBuf, hash)
+		ch := chunker.New(rd, testPol, hash)
 
 		for {
 			_, err := ch.Next()
@@ -322,11 +315,9 @@ func BenchmarkNewChunker(b *testing.B) {
 	p, err := chunker.RandomPolynomial()
 	OK(b, err)
 
-	chunkBuf := make([]byte, chunkerBufSize)
-
 	b.ResetTimer()
 
 	for i := 0; i < b.N; i++ {
-		chunker.New(bytes.NewBuffer(nil), p, chunkBuf, nil)
+		chunker.New(bytes.NewBuffer(nil), p, nil)
 	}
 }