mirror of
https://github.com/octoleo/restic.git
synced 2024-11-26 14:56:29 +00:00
Improve memory usage (and speed) of crypto reader/writer
benchcmp:
benchmark old ns/op new ns/op delta
BenchmarkChunkEncrypt 260811933 261294510 +0.19%
BenchmarkChunkEncryptParallel 262787948 259394601 -1.29%
BenchmarkEncryptWriter 89769328 87108804 -2.96%
BenchmarkEncrypt 88586645 87489151 -1.24%
BenchmarkDecryptReader 99669425 90019414 -9.68%
BenchmarkEncryptDecryptReader 189263157 185185733 -2.15%
BenchmarkDecrypt 89631109 89172417 -0.51%
benchmark old MB/s new MB/s speedup
BenchmarkChunkEncrypt 40.20 40.13 1.00x
BenchmarkChunkEncryptParallel 39.90 40.42 1.01x
BenchmarkEncryptWriter 93.45 96.30 1.03x
BenchmarkEncrypt 94.69 95.88 1.01x
BenchmarkDecryptReader 84.16 93.19 1.11x
BenchmarkEncryptDecryptReader 44.32 45.30 1.02x
BenchmarkDecrypt 93.59 94.07 1.01x
benchmark old allocs new allocs delta
BenchmarkChunkEncrypt 113 113 +0.00%
BenchmarkChunkEncryptParallel 104 104 +0.00%
BenchmarkEncryptWriter 20 20 +0.00%
BenchmarkEncrypt 14 14 +0.00%
BenchmarkDecryptReader 38 18 -52.63%
BenchmarkEncryptDecryptReader 61 55 -9.84%
BenchmarkDecrypt 17 17 +0.00%
benchmark old bytes new bytes delta
BenchmarkChunkEncrypt 8515750 8515750 +0.00%
BenchmarkChunkEncryptParallel 8515766 8515766 +0.00%
BenchmarkEncryptWriter 8391305 28927 -99.66%
BenchmarkEncrypt 2475 2475 +0.00%
BenchmarkDecryptReader 33563550 527827 -98.43%
BenchmarkEncryptDecryptReader 50348456 35814894 -28.87%
BenchmarkDecrypt 8391127 8391127 +0.00%
This commit is contained in:
Alexander Neumann
parent
d94d003165
commit
874b29b91f
103
key.go
103
key.go
@ -1,7 +1,6 @@
|
||||
package restic
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/aes"
|
||||
"crypto/cipher"
|
||||
"crypto/hmac"
|
||||
@ -15,6 +14,7 @@ import (
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"os/user"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/restic/restic/backend"
|
||||
@ -338,6 +338,7 @@ type encryptWriter struct {
|
||||
iv []byte
|
||||
wroteIV bool
|
||||
h hash.Hash
|
||||
s cipher.Stream
|
||||
w io.Writer
|
||||
origWr io.Writer
|
||||
err error // remember error writing iv
|
||||
@ -353,6 +354,13 @@ func (e *encryptWriter) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
const encryptWriterChunkSize = 512 * 1024 // 512 KiB
|
||||
var encryptWriterBufPool = sync.Pool{
|
||||
New: func() interface{} {
|
||||
return make([]byte, encryptWriterChunkSize)
|
||||
},
|
||||
}
|
||||
|
||||
func (e *encryptWriter) Write(p []byte) (int, error) {
|
||||
// write iv first
|
||||
if !e.wroteIV {
|
||||
@ -364,13 +372,34 @@ func (e *encryptWriter) Write(p []byte) (int, error) {
|
||||
return 0, e.err
|
||||
}
|
||||
|
||||
n, err := e.w.Write(p)
|
||||
if err != nil {
|
||||
e.err = err
|
||||
return n, err
|
||||
buf := encryptWriterBufPool.Get().([]byte)
|
||||
defer encryptWriterBufPool.Put(buf)
|
||||
|
||||
written := 0
|
||||
for len(p) > 0 {
|
||||
max := len(p)
|
||||
if max > encryptWriterChunkSize {
|
||||
max = encryptWriterChunkSize
|
||||
}
|
||||
|
||||
return n, nil
|
||||
e.s.XORKeyStream(buf, p[:max])
|
||||
n, err := e.w.Write(buf[:max])
|
||||
if n != max {
|
||||
if err == nil { // should never happen
|
||||
err = io.ErrShortWrite
|
||||
}
|
||||
}
|
||||
|
||||
written += n
|
||||
p = p[n:]
|
||||
|
||||
if err != nil {
|
||||
e.err = err
|
||||
return written, err
|
||||
}
|
||||
}
|
||||
|
||||
return written, nil
|
||||
}
|
||||
|
||||
func (k *Key) encryptTo(ks *keys, wr io.Writer) io.WriteCloser {
|
||||
@ -396,10 +425,8 @@ func (k *Key) encryptTo(ks *keys, wr io.Writer) io.WriteCloser {
|
||||
panic(fmt.Sprintf("unable to create cipher: %v", err))
|
||||
}
|
||||
|
||||
ew.w = cipher.StreamWriter{
|
||||
S: cipher.NewCTR(c, ew.iv),
|
||||
W: io.MultiWriter(ew.h, wr),
|
||||
}
|
||||
ew.s = cipher.NewCTR(c, ew.iv)
|
||||
ew.w = io.MultiWriter(ew.h, wr)
|
||||
|
||||
return ew
|
||||
}
|
||||
@ -474,6 +501,34 @@ func (k *Key) DecryptUser(ciphertext []byte) ([]byte, error) {
|
||||
return k.decrypt(k.user, ciphertext)
|
||||
}
|
||||
|
||||
type decryptReader struct {
|
||||
buf []byte
|
||||
pos int
|
||||
}
|
||||
|
||||
func (d *decryptReader) Read(dst []byte) (int, error) {
|
||||
if d.buf == nil {
|
||||
return 0, io.EOF
|
||||
}
|
||||
|
||||
if len(dst) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
remaining := len(d.buf) - d.pos
|
||||
if len(dst) >= remaining {
|
||||
n := copy(dst, d.buf[d.pos:])
|
||||
FreeChunkBuf("decryptReader", d.buf)
|
||||
d.buf = nil
|
||||
return n, io.EOF
|
||||
}
|
||||
|
||||
n := copy(dst, d.buf[d.pos:d.pos+len(dst)])
|
||||
d.pos += n
|
||||
|
||||
return n, nil
|
||||
}
|
||||
|
||||
// decryptFrom verifies and decrypts the ciphertext read from rd with ks and
|
||||
// makes it available on the returned Reader. Ciphertext must be in the form IV
|
||||
// || Ciphertext || HMAC. In order to correctly verify the ciphertext, rd is
|
||||
@ -481,14 +536,28 @@ func (k *Key) DecryptUser(ciphertext []byte) ([]byte, error) {
|
||||
// afterwards. If an HMAC verification failure is observed, it is returned
|
||||
// immediately.
|
||||
func (k *Key) decryptFrom(ks *keys, rd io.Reader) (io.Reader, error) {
|
||||
ciphertext, err := ioutil.ReadAll(rd)
|
||||
ciphertext := GetChunkBuf("decryptReader")
|
||||
ciphertext = ciphertext[0:cap(ciphertext)]
|
||||
n, err := io.ReadFull(rd, ciphertext)
|
||||
if err != io.ErrUnexpectedEOF {
|
||||
// read remaining data
|
||||
buf, e := ioutil.ReadAll(rd)
|
||||
ciphertext = append(ciphertext, buf...)
|
||||
n += len(buf)
|
||||
err = e
|
||||
} else {
|
||||
err = nil
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
ciphertext = ciphertext[:n]
|
||||
|
||||
// check for plausible length
|
||||
if len(ciphertext) < ivSize+hmacSize {
|
||||
panic("trying to decryipt invalid data: ciphertext too small")
|
||||
panic("trying to decrypt invalid data: ciphertext too small")
|
||||
}
|
||||
|
||||
hm := hmac.New(sha256.New, ks.Sign)
|
||||
@ -498,7 +567,7 @@ func (k *Key) decryptFrom(ks *keys, rd io.Reader) (io.Reader, error) {
|
||||
ciphertext, mac := ciphertext[:l], ciphertext[l:]
|
||||
|
||||
// calculate new hmac
|
||||
n, err := hm.Write(ciphertext)
|
||||
n, err = hm.Write(ciphertext)
|
||||
if err != nil || n != len(ciphertext) {
|
||||
panic(fmt.Sprintf("unable to calculate hmac of ciphertext, err %v", err))
|
||||
}
|
||||
@ -519,12 +588,10 @@ func (k *Key) decryptFrom(ks *keys, rd io.Reader) (io.Reader, error) {
|
||||
panic(fmt.Sprintf("unable to create cipher: %v", err))
|
||||
}
|
||||
|
||||
r := cipher.StreamReader{
|
||||
S: cipher.NewCTR(c, iv),
|
||||
R: bytes.NewReader(ciphertext),
|
||||
}
|
||||
stream := cipher.NewCTR(c, iv)
|
||||
stream.XORKeyStream(ciphertext, ciphertext)
|
||||
|
||||
return r, nil
|
||||
return &decryptReader{buf: ciphertext}, nil
|
||||
}
|
||||
|
||||
// DecryptFrom verifies and decrypts the ciphertext read from rd and makes it
|
||||
|
||||
Loading…
Reference in New Issue
Block a user