Improve memory usage (and speed) of crypto reader/writer

benchcmp:

    benchmark                         old ns/op     new ns/op     delta
    BenchmarkChunkEncrypt             260811933     261294510     +0.19%
    BenchmarkChunkEncryptParallel     262787948     259394601     -1.29%
    BenchmarkEncryptWriter            89769328      87108804      -2.96%
    BenchmarkEncrypt                  88586645      87489151      -1.24%
    BenchmarkDecryptReader            99669425      90019414      -9.68%
    BenchmarkEncryptDecryptReader     189263157     185185733     -2.15%
    BenchmarkDecrypt                  89631109      89172417      -0.51%

    benchmark                         old MB/s     new MB/s     speedup
    BenchmarkChunkEncrypt             40.20        40.13        1.00x
    BenchmarkChunkEncryptParallel     39.90        40.42        1.01x
    BenchmarkEncryptWriter            93.45        96.30        1.03x
    BenchmarkEncrypt                  94.69        95.88        1.01x
    BenchmarkDecryptReader            84.16        93.19        1.11x
    BenchmarkEncryptDecryptReader     44.32        45.30        1.02x
    BenchmarkDecrypt                  93.59        94.07        1.01x

    benchmark                         old allocs     new allocs     delta
    BenchmarkChunkEncrypt             113            113            +0.00%
    BenchmarkChunkEncryptParallel     104            104            +0.00%
    BenchmarkEncryptWriter            20             20             +0.00%
    BenchmarkEncrypt                  14             14             +0.00%
    BenchmarkDecryptReader            38             18             -52.63%
    BenchmarkEncryptDecryptReader     61             55             -9.84%
    BenchmarkDecrypt                  17             17             +0.00%

    benchmark                         old bytes     new bytes     delta
    BenchmarkChunkEncrypt             8515750       8515750       +0.00%
    BenchmarkChunkEncryptParallel     8515766       8515766       +0.00%
    BenchmarkEncryptWriter            8391305       28927         -99.66%
    BenchmarkEncrypt                  2475          2475          +0.00%
    BenchmarkDecryptReader            33563550      527827        -98.43%
    BenchmarkEncryptDecryptReader     50348456      35814894      -28.87%
    BenchmarkDecrypt                  8391127       8391127       +0.00%

key.go

@@ -1,7 +1,6 @@
 package restic
 
 import (
-	"bytes"
 	"crypto/aes"
 	"crypto/cipher"
 	"crypto/hmac"
@@ -15,6 +14,7 @@ import (
 	"io/ioutil"
 	"os"
 	"os/user"
+	"sync"
 	"time"
 
 	"github.com/restic/restic/backend"
@@ -338,6 +338,7 @@ type encryptWriter struct {
 	iv      []byte
 	wroteIV bool
 	h       hash.Hash
+	s       cipher.Stream
 	w       io.Writer
 	origWr  io.Writer
 	err     error // remember error writing iv
@@ -353,6 +354,13 @@ func (e *encryptWriter) Close() error {
 	return nil
 }
 
+const encryptWriterChunkSize = 512 * 1024 // 512 KiB
+var encryptWriterBufPool = sync.Pool{
+	New: func() interface{} {
+		return make([]byte, encryptWriterChunkSize)
+	},
+}
+
 func (e *encryptWriter) Write(p []byte) (int, error) {
 	// write iv first
 	if !e.wroteIV {
@@ -364,13 +372,34 @@ func (e *encryptWriter) Write(p []byte) (int, error) {
 		return 0, e.err
 	}
 
-	n, err := e.w.Write(p)
-	if err != nil {
-		e.err = err
-		return n, err
+	buf := encryptWriterBufPool.Get().([]byte)
+	defer encryptWriterBufPool.Put(buf)
+
+	written := 0
+	for len(p) > 0 {
+		max := len(p)
+		if max > encryptWriterChunkSize {
+			max = encryptWriterChunkSize
+		}
+
+		e.s.XORKeyStream(buf, p[:max])
+		n, err := e.w.Write(buf[:max])
+		if n != max {
+			if err == nil { // should never happen
+				err = io.ErrShortWrite
+			}
+		}
+
+		written += n
+		p = p[n:]
+
+		if err != nil {
+			e.err = err
+			return written, err
+		}
 	}
 
-	return n, nil
+	return written, nil
 }
 
 func (k *Key) encryptTo(ks *keys, wr io.Writer) io.WriteCloser {
@@ -396,10 +425,8 @@ func (k *Key) encryptTo(ks *keys, wr io.Writer) io.WriteCloser {
 		panic(fmt.Sprintf("unable to create cipher: %v", err))
 	}
 
-	ew.w = cipher.StreamWriter{
-		S: cipher.NewCTR(c, ew.iv),
-		W: io.MultiWriter(ew.h, wr),
-	}
+	ew.s = cipher.NewCTR(c, ew.iv)
+	ew.w = io.MultiWriter(ew.h, wr)
 
 	return ew
 }
@@ -474,6 +501,34 @@ func (k *Key) DecryptUser(ciphertext []byte) ([]byte, error) {
 	return k.decrypt(k.user, ciphertext)
 }
 
+type decryptReader struct {
+	buf []byte
+	pos int
+}
+
+func (d *decryptReader) Read(dst []byte) (int, error) {
+	if d.buf == nil {
+		return 0, io.EOF
+	}
+
+	if len(dst) == 0 {
+		return 0, nil
+	}
+
+	remaining := len(d.buf) - d.pos
+	if len(dst) >= remaining {
+		n := copy(dst, d.buf[d.pos:])
+		FreeChunkBuf("decryptReader", d.buf)
+		d.buf = nil
+		return n, io.EOF
+	}
+
+	n := copy(dst, d.buf[d.pos:d.pos+len(dst)])
+	d.pos += n
+
+	return n, nil
+}
+
 // decryptFrom verifies and decrypts the ciphertext read from rd with ks and
 // makes it available on the returned Reader. Ciphertext must be in the form IV
 // || Ciphertext || HMAC. In order to correctly verify the ciphertext, rd is
@@ -481,14 +536,28 @@ func (k *Key) DecryptUser(ciphertext []byte) ([]byte, error) {
 // afterwards. If an HMAC verification failure is observed, it is returned
 // immediately.
 func (k *Key) decryptFrom(ks *keys, rd io.Reader) (io.Reader, error) {
-	ciphertext, err := ioutil.ReadAll(rd)
+	ciphertext := GetChunkBuf("decryptReader")
+	ciphertext = ciphertext[0:cap(ciphertext)]
+	n, err := io.ReadFull(rd, ciphertext)
+	if err != io.ErrUnexpectedEOF {
+		// read remaining data
+		buf, e := ioutil.ReadAll(rd)
+		ciphertext = append(ciphertext, buf...)
+		n += len(buf)
+		err = e
+	} else {
+		err = nil
+	}
+
 	if err != nil {
 		return nil, err
 	}
 
+	ciphertext = ciphertext[:n]
+
 	// check for plausible length
 	if len(ciphertext) < ivSize+hmacSize {
-		panic("trying to decryipt invalid data: ciphertext too small")
+		panic("trying to decrypt invalid data: ciphertext too small")
 	}
 
 	hm := hmac.New(sha256.New, ks.Sign)
@@ -498,7 +567,7 @@ func (k *Key) decryptFrom(ks *keys, rd io.Reader) (io.Reader, error) {
 	ciphertext, mac := ciphertext[:l], ciphertext[l:]
 
 	// calculate new hmac
-	n, err := hm.Write(ciphertext)
+	n, err = hm.Write(ciphertext)
 	if err != nil || n != len(ciphertext) {
 		panic(fmt.Sprintf("unable to calculate hmac of ciphertext, err %v", err))
 	}
@@ -519,12 +588,10 @@ func (k *Key) decryptFrom(ks *keys, rd io.Reader) (io.Reader, error) {
 		panic(fmt.Sprintf("unable to create cipher: %v", err))
 	}
 
-	r := cipher.StreamReader{
-		S: cipher.NewCTR(c, iv),
-		R: bytes.NewReader(ciphertext),
-	}
+	stream := cipher.NewCTR(c, iv)
+	stream.XORKeyStream(ciphertext, ciphertext)
 
-	return r, nil
+	return &decryptReader{buf: ciphertext}, nil
 }
 
 // DecryptFrom verifies and decrypts the ciphertext read from rd and makes it