From bbb3145f11f7b8cdf40c44f1eda7461dfe7e620c Mon Sep 17 00:00:00 2001
From: Alexander Neumann <alexander@bumpern.de>
Date: Mon, 9 Feb 2015 23:38:50 +0100
Subject: [PATCH] Add chunker pool

---
 archiver.go | 10 ++++++----
 pools.go    | 55 ++++++++++++++++++++++++++++++++++++++++++++++-------
 2 files changed, 54 insertions(+), 11 deletions(-)

diff --git a/archiver.go b/archiver.go
index 42891e48d..db20195db 100644
--- a/archiver.go
+++ b/archiver.go
@@ -1,7 +1,6 @@
 package restic
 
 import (
-	"crypto/sha256"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -18,7 +17,9 @@ import (
 const (
 	maxConcurrentFiles = 16
 	maxConcurrentBlobs = 16
-	chunkerBufSize     = 512 * chunker.KiB
+
+	// chunkerBufSize is used in pool.go
+	chunkerBufSize = 512 * chunker.KiB
 )
 
 type Archiver struct {
@@ -161,9 +162,10 @@ func (arch *Archiver) SaveFile(node *Node) (Blobs, error) {
 	var blobs Blobs
 
 	// store all chunks
-	chnker := chunker.New(file, chunkerBufSize, sha256.New)
+	chnker := GetChunker("archiver.SaveFile")
+	chnker.Reset(file)
 	chans := [](<-chan Blob){}
-	//defer chnker.Free()
+	defer FreeChunker("archiver.SaveFile", chnker)
 
 	chunks := 0
 
diff --git a/pools.go b/pools.go
index cbdaa035f..782c6bacf 100644
--- a/pools.go
+++ b/pools.go
@@ -1,8 +1,10 @@
 package restic
 
 import (
+	"crypto/sha256"
 	"sync"
 
+	"github.com/restic/restic/chunker"
 	"github.com/restic/restic/debug"
 )
 
@@ -12,6 +14,7 @@ type poolStats struct {
 	mput map[string]int
 	mmax map[string]int
 
+	new int
 	get int
 	put int
 	max int
@@ -63,23 +66,42 @@ func newPoolStats() *poolStats {
 }
 
 var (
-	chunkPool = sync.Pool{New: newChunkBuf}
-	nodePool  = sync.Pool{New: newNode}
+	chunkPool   = sync.Pool{New: newChunkBuf}
+	nodePool    = sync.Pool{New: newNode}
+	chunkerPool = sync.Pool{New: newChunker}
 
-	chunkStats = newPoolStats()
-	nodeStats  = newPoolStats()
+	chunkStats   = newPoolStats()
+	nodeStats    = newPoolStats()
+	chunkerStats = newPoolStats()
 )
 
 func newChunkBuf() interface{} {
+	chunkStats.m.Lock()
+	defer chunkStats.m.Unlock()
+	chunkStats.new++
+
 	// create buffer for iv, data and hmac
 	return make([]byte, maxCiphertextSize)
 }
 
 func newNode() interface{} {
+	nodeStats.m.Lock()
+	defer nodeStats.m.Unlock()
+	nodeStats.new++
+
 	// create buffer for iv, data and hmac
 	return new(Node)
 }
 
+func newChunker() interface{} {
+	chunkStats.m.Lock()
+	defer chunkStats.m.Unlock()
+	chunkStats.new++
+
+	// create a new chunker with a nil reader
+	return chunker.New(nil, chunkerBufSize, sha256.New)
+}
+
 func GetChunkBuf(s string) []byte {
 	chunkStats.Get(s)
 	return chunkPool.Get().([]byte)
@@ -100,14 +122,33 @@ func FreeNode(n *Node) {
 	nodePool.Put(n)
 }
 
+func GetChunker(s string) *chunker.Chunker {
+	chunkerStats.Get(s)
+	return chunkerPool.Get().(*chunker.Chunker)
+}
+
+func FreeChunker(s string, ch *chunker.Chunker) {
+	chunkerStats.Put(s)
+	chunkerPool.Put(ch)
+}
+
 func PoolAlloc() {
-	debug.Log("pools.PoolAlloc", "pool stats for chunk: get %d, put %d, diff %d, max %d\n", chunkStats.get, chunkStats.put, chunkStats.get-chunkStats.put, chunkStats.max)
+	debug.Log("pools.PoolAlloc", "pool stats for chunk: new %d, get %d, put %d, diff %d, max %d\n",
+		chunkStats.new, chunkStats.get, chunkStats.put, chunkStats.get-chunkStats.put, chunkStats.max)
 	for k, v := range chunkStats.mget {
-		debug.Log("pools.PoolAlloc", "pool stats for chunk[%s]: get %d, put %d, diff %d, max %d\n", k, v, chunkStats.mput[k], v-chunkStats.mput[k], chunkStats.mmax[k])
+		debug.Log("pools.PoolAlloc", "pool stats for chunk[%s]: get %d, put %d, diff %d, max %d\n",
+			k, v, chunkStats.mput[k], v-chunkStats.mput[k], chunkStats.mmax[k])
 	}
 
-	debug.Log("pools.PoolAlloc", "pool stats for node: get %d, put %d, diff %d, max %d\n", nodeStats.get, nodeStats.put, nodeStats.get-nodeStats.put, nodeStats.max)
+	debug.Log("pools.PoolAlloc", "pool stats for node: new %d, get %d, put %d, diff %d, max %d\n",
+		nodeStats.new, nodeStats.get, nodeStats.put, nodeStats.get-nodeStats.put, nodeStats.max)
 	for k, v := range nodeStats.mget {
 		debug.Log("pools.PoolAlloc", "pool stats for node[%s]: get %d, put %d, diff %d, max %d\n", k, v, nodeStats.mput[k], v-nodeStats.mput[k], nodeStats.mmax[k])
 	}
+
+	debug.Log("pools.PoolAlloc", "pool stats for chunker: new %d, get %d, put %d, diff %d, max %d\n",
+		chunkerStats.new, chunkerStats.get, chunkerStats.put, chunkerStats.get-chunkerStats.put, chunkerStats.max)
+	for k, v := range chunkerStats.mget {
+		debug.Log("pools.PoolAlloc", "pool stats for chunker[%s]: get %d, put %d, diff %d, max %d\n", k, v, chunkerStats.mput[k], v-chunkerStats.mput[k], chunkerStats.mmax[k])
+	}
 }