Merge pull request #731 from restic/improve-memory-usage

Improve memory usage

src/cmds/restic/cmd_prune.go

@@ -103,11 +103,13 @@ func runPrune(gopts GlobalOptions) error {
 		return err
 	}
 
+	blobs := 0
 	for _, pack := range idx.Packs {
 		stats.bytes += pack.Size
+		blobs += len(pack.Entries)
 	}
 	Verbosef("repository contains %v packs (%v blobs) with %v bytes\n",
-		len(idx.Packs), len(idx.Blobs), formatBytes(uint64(stats.bytes)))
+		len(idx.Packs), blobs, formatBytes(uint64(stats.bytes)))
 
 	blobCount := make(map[restic.BlobHandle]int)
 	duplicateBlobs := 0
@@ -164,14 +166,17 @@ func runPrune(gopts GlobalOptions) error {
 
 	// find packs that need a rewrite
 	rewritePacks := restic.NewIDSet()
-	for h, blob := range idx.Blobs {
+	for _, pack := range idx.Packs {
-		if !usedBlobs.Has(h) {
+		for _, blob := range pack.Entries {
-			rewritePacks.Merge(blob.Packs)
+			h := restic.BlobHandle{ID: blob.ID, Type: blob.Type}
-			continue
+			if !usedBlobs.Has(h) {
-		}
+				rewritePacks.Insert(pack.ID)
+				continue
+			}
 
-		if blobCount[h] > 1 {
+			if blobCount[h] > 1 {
-			rewritePacks.Merge(blob.Packs)
+				rewritePacks.Insert(pack.ID)
+			}
 		}
 	}
 

src/restic/archiver/archive_reader_test.go

@@ -44,7 +44,7 @@ func checkSavedFile(t *testing.T, repo restic.Repository, treeID restic.ID, name
 			t.Fatal(err)
 		}
 
-		buf := make([]byte, int(size))
+		buf := restic.NewBlobBuffer(int(size))
 		n := loadBlob(t, repo, id, buf)
 		if n != len(buf) {
 			t.Errorf("wrong number of bytes read, want %d, got %d", len(buf), n)

src/restic/buffer.go

@@ -0,0 +1,21 @@
+package restic
+
+import "restic/crypto"
+
+// NewBlobBuffer returns a buffer that is large enough to hold a blob of size
+// plaintext bytes, including the crypto overhead.
+func NewBlobBuffer(size int) []byte {
+	return make([]byte, size, size+crypto.Extension)
+}
+
+// PlaintextLength returns the plaintext length of a blob with ciphertextSize
+// bytes.
+func PlaintextLength(ciphertextSize int) int {
+	return ciphertextSize - crypto.Extension
+}
+
+// CiphertextLength returns the encrypted length of a blob with plaintextSize
+// bytes.
+func CiphertextLength(plaintextSize int) int {
+	return plaintextSize + crypto.Extension
+}

src/restic/index/index.go

@@ -14,27 +14,20 @@ import (
 
 // Pack contains information about the contents of a pack.
 type Pack struct {
+	ID      restic.ID
 	Size    int64
 	Entries []restic.Blob
 }
 
-// Blob contains information about a blob.
-type Blob struct {
-	Size  int64
-	Packs restic.IDSet
-}
-
 // Index contains information about blobs and packs stored in a repo.
 type Index struct {
 	Packs    map[restic.ID]Pack
-	Blobs    map[restic.BlobHandle]Blob
 	IndexIDs restic.IDSet
 }
 
 func newIndex() *Index {
 	return &Index{
 		Packs:    make(map[restic.ID]Pack),
-		Blobs:    make(map[restic.BlobHandle]Blob),
 		IndexIDs: restic.NewIDSet(),
 	}
 }
@@ -69,9 +62,6 @@ func New(repo restic.Repository, p *restic.Progress) (*Index, error) {
 		if err != nil {
 			return nil, err
 		}
-
-		p := Pack{Entries: j.Entries(), Size: j.Size()}
-		idx.Packs[packID] = p
 	}
 
 	return idx, nil
@@ -181,18 +171,6 @@ func (idx *Index) AddPack(id restic.ID, size int64, entries []restic.Blob) error
 
 	idx.Packs[id] = Pack{Size: size, Entries: entries}
 
-	for _, entry := range entries {
-		h := restic.BlobHandle{ID: entry.ID, Type: entry.Type}
-		if _, ok := idx.Blobs[h]; !ok {
-			idx.Blobs[h] = Blob{
-				Size:  int64(entry.Length),
-				Packs: restic.NewIDSet(),
-			}
-		}
-
-		idx.Blobs[h].Packs.Insert(id)
-	}
-
 	return nil
 }
 
@@ -202,15 +180,6 @@ func (idx *Index) RemovePack(id restic.ID) error {
 		return errors.Errorf("pack %v not found in the index", id.Str())
 	}
 
-	for _, blob := range idx.Packs[id].Entries {
-		h := restic.BlobHandle{ID: blob.ID, Type: blob.Type}
-		idx.Blobs[h].Packs.Delete(id)
-
-		if len(idx.Blobs[h].Packs) == 0 {
-			delete(idx.Blobs, h)
-		}
-	}
-
 	delete(idx.Packs, id)
 
 	return nil
@@ -239,14 +208,11 @@ func (idx *Index) DuplicateBlobs() (dups restic.BlobSet) {
 func (idx *Index) PacksForBlobs(blobs restic.BlobSet) (packs restic.IDSet) {
 	packs = restic.NewIDSet()
 
-	for h := range blobs {
+	for id, p := range idx.Packs {
-		blob, ok := idx.Blobs[h]
+		for _, entry := range p.Entries {
-		if !ok {
+			if blobs.Has(restic.BlobHandle{ID: entry.ID, Type: entry.Type}) {
-			continue
+				packs.Insert(id)
-		}
+			}
-
-		for id := range blob.Packs {
-			packs.Insert(id)
 		}
 	}
 
@@ -264,31 +230,20 @@ type Location struct {
 var ErrBlobNotFound = errors.New("blob not found in index")
 
 // FindBlob returns a list of packs and positions the blob can be found in.
-func (idx *Index) FindBlob(h restic.BlobHandle) ([]Location, error) {
+func (idx *Index) FindBlob(h restic.BlobHandle) (result []Location, err error) {
-	blob, ok := idx.Blobs[h]
+	for id, p := range idx.Packs {
-	if !ok {
+		for _, entry := range p.Entries {
-		return nil, ErrBlobNotFound
+			if entry.ID.Equal(h.ID) && entry.Type == h.Type {
+				result = append(result, Location{
+					PackID: id,
+					Blob:   entry,
+				})
+			}
+		}
 	}
 
-	result := make([]Location, 0, len(blob.Packs))
+	if len(result) == 0 {
-	for packID := range blob.Packs {
+		return nil, ErrBlobNotFound
-		pack, ok := idx.Packs[packID]
-		if !ok {
-			return nil, errors.Errorf("pack %v not found in index", packID.Str())
-		}
-
-		for _, entry := range pack.Entries {
-			if entry.Type != h.Type {
-				continue
-			}
-
-			if !entry.ID.Equal(h.ID) {
-				continue
-			}
-
-			loc := Location{PackID: packID, Blob: entry}
-			result = append(result, loc)
-		}
 	}
 
 	return result, nil

src/restic/index/index_test.go

@@ -3,7 +3,9 @@ package index
 import (
 	"math/rand"
 	"restic"
+	"restic/checker"
 	"restic/repository"
+	"restic/test"
 	"testing"
 	"time"
 )
@@ -135,6 +137,40 @@ func BenchmarkIndexNew(b *testing.B) {
 		if idx == nil {
 			b.Fatalf("New() returned nil index")
 		}
+		b.Logf("idx %v packs", len(idx.Packs))
+	}
+}
+
+func BenchmarkIndexSave(b *testing.B) {
+	repo, cleanup := repository.TestRepository(b)
+	defer cleanup()
+
+	idx, err := New(repo, nil)
+	test.OK(b, err)
+
+	for i := 0; i < 8000; i++ {
+		entries := make([]restic.Blob, 0, 200)
+		for j := 0; j < len(entries); j++ {
+			entries = append(entries, restic.Blob{
+				ID:     restic.NewRandomID(),
+				Length: 1000,
+				Offset: 5,
+				Type:   restic.DataBlob,
+			})
+		}
+
+		idx.AddPack(restic.NewRandomID(), 10000, entries)
+	}
+
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		id, err := idx.Save(repo, nil)
+		if err != nil {
+			b.Fatalf("New() returned error %v", err)
+		}
+
+		b.Logf("saved as %v", id.Str())
 	}
 }
 
@@ -151,7 +187,7 @@ func TestIndexDuplicateBlobs(t *testing.T) {
 	if len(dups) == 0 {
 		t.Errorf("no duplicate blobs found")
 	}
-	t.Logf("%d packs, %d unique blobs", len(idx.Packs), len(idx.Blobs))
+	t.Logf("%d packs, %d duplicate blobs", len(idx.Packs), len(dups))
 
 	packs := idx.PacksForBlobs(dups)
 	if len(packs) == 0 {
@@ -169,7 +205,7 @@ func loadIndex(t testing.TB, repo restic.Repository) *Index {
 	return idx
 }
 
-func TestIndexSave(t *testing.T) {
+func TestSave(t *testing.T) {
 	repo, cleanup := createFilledRepo(t, 3, 0)
 	defer cleanup()
 
@@ -219,6 +255,41 @@ func TestIndexSave(t *testing.T) {
 	}
 }
 
+func TestIndexSave(t *testing.T) {
+	repo, cleanup := createFilledRepo(t, 3, 0)
+	defer cleanup()
+
+	idx := loadIndex(t, repo)
+
+	id, err := idx.Save(repo, idx.IndexIDs.List())
+	if err != nil {
+		t.Fatalf("unable to save new index: %v", err)
+	}
+
+	t.Logf("new index saved as %v", id.Str())
+
+	for id := range idx.IndexIDs {
+		t.Logf("remove index %v", id.Str())
+		err = repo.Backend().Remove(restic.IndexFile, id.String())
+		if err != nil {
+			t.Errorf("error removing index %v: %v", id, err)
+		}
+	}
+
+	idx2 := loadIndex(t, repo)
+	t.Logf("load new index with %d packs", len(idx2.Packs))
+
+	checker := checker.New(repo)
+	hints, errs := checker.LoadIndex()
+	for _, h := range hints {
+		t.Logf("hint: %v\n", h)
+	}
+
+	for _, err := range errs {
+		t.Errorf("checker found error: %v", err)
+	}
+}
+
 func TestIndexAddRemovePack(t *testing.T) {
 	repo, cleanup := createFilledRepo(t, 3, 0)
 	defer cleanup()
@@ -249,12 +320,7 @@ func TestIndexAddRemovePack(t *testing.T) {
 		if err == nil {
 			t.Errorf("removed blob %v found in index", h)
 		}
-
-		if _, ok := idx.Blobs[h]; ok {
-			t.Errorf("removed blob %v found in index.Blobs", h)
-		}
 	}
-
 }
 
 // example index serialization from doc/Design.md

src/restic/node.go

@@ -208,7 +208,7 @@ func (node Node) createFileAt(path string, repo Repository) error {
 
 		buf = buf[:cap(buf)]
 		if uint(len(buf)) < size {
-			buf = make([]byte, size)
+			buf = NewBlobBuffer(int(size))
 		}
 
 		n, err := repo.LoadBlob(DataBlob, id, buf)

src/restic/pack/pack.go

@@ -85,15 +85,15 @@ func (p *Packer) Finalize() (uint, error) {
 		return 0, errors.Wrap(err, "Write")
 	}
 
-	hdrBytes := bytesHeader + crypto.Extension
+	hdrBytes := restic.CiphertextLength(int(bytesHeader))
-	if uint(n) != hdrBytes {
+	if n != hdrBytes {
 		return 0, errors.New("wrong number of bytes written")
 	}
 
-	bytesWritten += hdrBytes
+	bytesWritten += uint(hdrBytes)
 
 	// write length
-	err = binary.Write(p.wr, binary.LittleEndian, uint32(uint(len(p.blobs))*entrySize+crypto.Extension))
+	err = binary.Write(p.wr, binary.LittleEndian, uint32(restic.CiphertextLength(len(p.blobs)*int(entrySize))))
 	if err != nil {
 		return 0, errors.Wrap(err, "binary.Write")
 	}
@@ -233,6 +233,8 @@ func List(k *crypto.Key, rd io.ReaderAt, size int64) (entries []restic.Blob, err
 
 	hdrRd := bytes.NewReader(buf)
 
+	entries = make([]restic.Blob, 0, uint(n)/entrySize)
+
 	pos := uint(0)
 	for {
 		e := headerEntry{}

src/restic/pack/pack_test.go

@@ -54,10 +54,9 @@ func verifyBlobs(t testing.TB, bufs []Buf, k *crypto.Key, rd io.ReaderAt, packSi
 	}
 	// header length
 	written += binary.Size(uint32(0))
-	// header
+	// header + header crypto
-	written += len(bufs) * (binary.Size(restic.BlobType(0)) + binary.Size(uint32(0)) + len(restic.ID{}))
+	headerSize := len(bufs) * (binary.Size(restic.BlobType(0)) + binary.Size(uint32(0)) + len(restic.ID{}))
-	// header crypto
+	written += restic.CiphertextLength(headerSize)
-	written += crypto.Extension
 
 	// check length
 	Equals(t, uint(written), packSize)

src/restic/repository/index.go

@@ -1,7 +1,6 @@
 package repository
 
 import (
-	"bytes"
 	"encoding/json"
 	"io"
 	"restic"
@@ -10,7 +9,6 @@ import (
 
 	"restic/errors"
 
-	"restic/crypto"
 	"restic/debug"
 )
 
@@ -177,15 +175,15 @@ func (idx *Index) Has(id restic.ID, tpe restic.BlobType) bool {
 	return false
 }
 
-// LookupSize returns the length of the cleartext content behind the
+// LookupSize returns the length of the plaintext content of the blob with the
-// given id
+// given id.
-func (idx *Index) LookupSize(id restic.ID, tpe restic.BlobType) (cleartextLength uint, err error) {
+func (idx *Index) LookupSize(id restic.ID, tpe restic.BlobType) (plaintextLength uint, err error) {
 	blobs, err := idx.Lookup(id, tpe)
 	if err != nil {
 		return 0, err
 	}
 
-	return blobs[0].Length - crypto.Extension, nil
+	return uint(restic.PlaintextLength(int(blobs[0].Length))), nil
 }
 
 // Supersedes returns the list of indexes this index supersedes, if any.
@@ -452,12 +450,11 @@ func isErrOldIndex(err error) bool {
 var ErrOldIndexFormat = errors.New("index has old format")
 
 // DecodeIndex loads and unserializes an index from rd.
-func DecodeIndex(rd io.Reader) (idx *Index, err error) {
+func DecodeIndex(buf []byte) (idx *Index, err error) {
 	debug.Log("Start decoding index")
-	idxJSON := jsonIndex{}
+	idxJSON := &jsonIndex{}
 
-	dec := json.NewDecoder(rd)
+	err = json.Unmarshal(buf, idxJSON)
-	err = dec.Decode(&idxJSON)
 	if err != nil {
 		debug.Log("Error %v", err)
 
@@ -491,12 +488,11 @@ func DecodeIndex(rd io.Reader) (idx *Index, err error) {
 }
 
 // DecodeOldIndex loads and unserializes an index in the old format from rd.
-func DecodeOldIndex(rd io.Reader) (idx *Index, err error) {
+func DecodeOldIndex(buf []byte) (idx *Index, err error) {
 	debug.Log("Start decoding old index")
 	list := []*packJSON{}
 
-	dec := json.NewDecoder(rd)
+	err = json.Unmarshal(buf, &list)
-	err = dec.Decode(&list)
 	if err != nil {
 		debug.Log("Error %#v", err)
 		return nil, errors.Wrap(err, "Decode")
@@ -523,7 +519,7 @@ func DecodeOldIndex(rd io.Reader) (idx *Index, err error) {
 }
 
 // LoadIndexWithDecoder loads the index and decodes it with fn.
-func LoadIndexWithDecoder(repo restic.Repository, id restic.ID, fn func(io.Reader) (*Index, error)) (idx *Index, err error) {
+func LoadIndexWithDecoder(repo restic.Repository, id restic.ID, fn func([]byte) (*Index, error)) (idx *Index, err error) {
 	debug.Log("Loading index %v", id.Str())
 
 	buf, err := repo.LoadAndDecrypt(restic.IndexFile, id)
@@ -531,7 +527,7 @@ func LoadIndexWithDecoder(repo restic.Repository, id restic.ID, fn func(io.Reade
 		return nil, err
 	}
 
-	idx, err = fn(bytes.NewReader(buf))
+	idx, err = fn(buf)
 	if err != nil {
 		debug.Log("error while decoding index %v: %v", id, err)
 		return nil, err

src/restic/repository/index_test.go

@@ -54,7 +54,7 @@ func TestIndexSerialize(t *testing.T) {
 	err := idx.Encode(wr)
 	OK(t, err)
 
-	idx2, err := repository.DecodeIndex(wr)
+	idx2, err := repository.DecodeIndex(wr.Bytes())
 	OK(t, err)
 	Assert(t, idx2 != nil,
 		"nil returned for decoded index")
@@ -136,7 +136,7 @@ func TestIndexSerialize(t *testing.T) {
 	Assert(t, id2.Equal(id),
 		"wrong ID returned: want %v, got %v", id, id2)
 
-	idx3, err := repository.DecodeIndex(wr3)
+	idx3, err := repository.DecodeIndex(wr3.Bytes())
 	OK(t, err)
 	Assert(t, idx3 != nil,
 		"nil returned for decoded index")
@@ -288,7 +288,7 @@ var exampleLookupTest = struct {
 func TestIndexUnserialize(t *testing.T) {
 	oldIdx := restic.IDs{restic.TestParseID("ed54ae36197f4745ebc4b54d10e0f623eaaaedd03013eb7ae90df881b7781452")}
 
-	idx, err := repository.DecodeIndex(bytes.NewReader(docExample))
+	idx, err := repository.DecodeIndex(docExample)
 	OK(t, err)
 
 	for _, test := range exampleTests {
@@ -326,8 +326,17 @@ func TestIndexUnserialize(t *testing.T) {
 	}
 }
 
+func BenchmarkDecodeIndex(b *testing.B) {
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		_, err := repository.DecodeIndex(docExample)
+		OK(b, err)
+	}
+}
+
 func TestIndexUnserializeOld(t *testing.T) {
-	idx, err := repository.DecodeOldIndex(bytes.NewReader(docOldExample))
+	idx, err := repository.DecodeOldIndex(docOldExample)
 	OK(t, err)
 
 	for _, test := range exampleTests {

src/restic/repository/master_index.go

@@ -30,8 +30,7 @@ func (mi *MasterIndex) Lookup(id restic.ID, tpe restic.BlobType) (blobs []restic
 	for _, idx := range mi.idx {
 		blobs, err = idx.Lookup(id, tpe)
 		if err == nil {
-			debug.Log("MasterIndex.Lookup",
+			debug.Log("found id %v: %v", id.Str(), blobs)
-				"found id %v: %v", id.Str(), blobs)
 			return
 		}
 	}
@@ -46,9 +45,8 @@ func (mi *MasterIndex) LookupSize(id restic.ID, tpe restic.BlobType) (uint, erro
 	defer mi.idxMutex.RUnlock()
 
 	for _, idx := range mi.idx {
-		length, err := idx.LookupSize(id, tpe)
+		if idx.Has(id, tpe) {
-		if err == nil {
+			return idx.LookupSize(id, tpe)
-			return length, nil
 		}
 	}
 

src/restic/repository/repository.go

@@ -64,15 +64,13 @@ func (r *Repository) LoadAndDecrypt(t restic.FileType, id restic.ID) ([]byte, er
 		return nil, errors.New("invalid data returned")
 	}
 
-	plain := make([]byte, len(buf))
-
 	// decrypt
-	n, err := r.decryptTo(plain, buf)
+	n, err := r.decryptTo(buf, buf)
 	if err != nil {
 		return nil, err
 	}
 
-	return plain[:n], nil
+	return buf[:n], nil
 }
 
 // loadBlob tries to load and decrypt content identified by t and id from a
@@ -81,17 +79,6 @@ func (r *Repository) LoadAndDecrypt(t restic.FileType, id restic.ID) ([]byte, er
 func (r *Repository) loadBlob(id restic.ID, t restic.BlobType, plaintextBuf []byte) (int, error) {
 	debug.Log("load %v with id %v (buf %p, len %d)", t, id.Str(), plaintextBuf, len(plaintextBuf))
 
-	// lookup plaintext size of blob
-	size, err := r.idx.LookupSize(id, t)
-	if err != nil {
-		return 0, err
-	}
-
-	// make sure the plaintext buffer is large enough, extend otherwise
-	if len(plaintextBuf) < int(size) {
-		return 0, errors.Errorf("buffer is too small: %d < %d", len(plaintextBuf), size)
-	}
-
 	// lookup packs
 	blobs, err := r.idx.Lookup(id, t)
 	if err != nil {
@@ -109,8 +96,8 @@ func (r *Repository) loadBlob(id restic.ID, t restic.BlobType, plaintextBuf []by
 
 		// load blob from pack
 		h := restic.Handle{Type: restic.DataFile, Name: blob.PackID.String()}
-		ciphertextBuf := make([]byte, blob.Length)
+		plaintextBuf = plaintextBuf[:cap(plaintextBuf)]
-		n, err := r.be.Load(h, ciphertextBuf, int64(blob.Offset))
+		n, err := r.be.Load(h, plaintextBuf, int64(blob.Offset))
 		if err != nil {
 			debug.Log("error loading blob %v: %v", blob, err)
 			lastError = err
@@ -125,7 +112,7 @@ func (r *Repository) loadBlob(id restic.ID, t restic.BlobType, plaintextBuf []by
 		}
 
 		// decrypt
-		n, err = r.decryptTo(plaintextBuf, ciphertextBuf)
+		n, err = r.decryptTo(plaintextBuf, plaintextBuf)
 		if err != nil {
 			lastError = errors.Errorf("decrypting blob %v failed: %v", id, err)
 			continue
@@ -224,7 +211,7 @@ func (r *Repository) SaveJSONUnpacked(t restic.FileType, item interface{}) (rest
 // SaveUnpacked encrypts data and stores it in the backend. Returned is the
 // storage hash.
 func (r *Repository) SaveUnpacked(t restic.FileType, p []byte) (id restic.ID, err error) {
-	ciphertext := make([]byte, len(p)+crypto.Extension)
+	ciphertext := restic.NewBlobBuffer(len(p))
 	ciphertext, err = r.Encrypt(ciphertext, p)
 	if err != nil {
 		return restic.ID{}, err
@@ -528,7 +515,9 @@ func (r *Repository) Close() error {
 	return r.be.Close()
 }
 
-// LoadBlob loads a blob of type t from the repository to the buffer.
+// LoadBlob loads a blob of type t from the repository to the buffer. buf must
+// be large enough to hold the encrypted blob, since it is used as scratch
+// space.
 func (r *Repository) LoadBlob(t restic.BlobType, id restic.ID, buf []byte) (int, error) {
 	debug.Log("load blob %v into buf %p", id.Str(), buf)
 	size, err := r.idx.LookupSize(id, t)
@@ -536,8 +525,9 @@ func (r *Repository) LoadBlob(t restic.BlobType, id restic.ID, buf []byte) (int,
 		return 0, err
 	}
 
-	if len(buf) < int(size) {
+	buf = buf[:cap(buf)]
-		return 0, errors.Errorf("buffer is too small for data blob (%d < %d)", len(buf), size)
+	if len(buf) < restic.CiphertextLength(int(size)) {
+		return 0, errors.Errorf("buffer is too small for data blob (%d < %d)", len(buf), restic.CiphertextLength(int(size)))
 	}
 
 	n, err := r.loadBlob(id, t, buf)
@@ -571,7 +561,7 @@ func (r *Repository) LoadTree(id restic.ID) (*restic.Tree, error) {
 	}
 
 	debug.Log("size is %d, create buffer", size)
-	buf := make([]byte, size)
+	buf := restic.NewBlobBuffer(int(size))
 
 	n, err := r.loadBlob(id, restic.TreeBlob, buf)
 	if err != nil {

src/restic/repository/repository_test.go

@@ -2,12 +2,12 @@ package repository_test
 
 import (
 	"bytes"
-	"crypto/rand"
 	"crypto/sha256"
 	"io"
-	mrand "math/rand"
+	"math/rand"
 	"path/filepath"
 	"testing"
+	"time"
 
 	"restic"
 	"restic/archiver"
@@ -17,13 +17,15 @@ import (
 
 var testSizes = []int{5, 23, 2<<18 + 23, 1 << 20}
 
+var rnd = rand.New(rand.NewSource(time.Now().UnixNano()))
+
 func TestSave(t *testing.T) {
 	repo, cleanup := repository.TestRepository(t)
 	defer cleanup()
 
 	for _, size := range testSizes {
 		data := make([]byte, size)
-		_, err := io.ReadFull(rand.Reader, data)
+		_, err := io.ReadFull(rnd, data)
 		OK(t, err)
 
 		id := restic.Hash(data)
@@ -38,7 +40,7 @@ func TestSave(t *testing.T) {
 		// OK(t, repo.SaveIndex())
 
 		// read back
-		buf := make([]byte, size)
+		buf := restic.NewBlobBuffer(size)
 		n, err := repo.LoadBlob(restic.DataBlob, id, buf)
 		OK(t, err)
 		Equals(t, len(buf), n)
@@ -59,7 +61,7 @@ func TestSaveFrom(t *testing.T) {
 
 	for _, size := range testSizes {
 		data := make([]byte, size)
-		_, err := io.ReadFull(rand.Reader, data)
+		_, err := io.ReadFull(rnd, data)
 		OK(t, err)
 
 		id := restic.Hash(data)
@@ -72,7 +74,7 @@ func TestSaveFrom(t *testing.T) {
 		OK(t, repo.Flush())
 
 		// read back
-		buf := make([]byte, size)
+		buf := restic.NewBlobBuffer(size)
 		n, err := repo.LoadBlob(restic.DataBlob, id, buf)
 		OK(t, err)
 		Equals(t, len(buf), n)
@@ -94,7 +96,7 @@ func BenchmarkSaveAndEncrypt(t *testing.B) {
 	size := 4 << 20 // 4MiB
 
 	data := make([]byte, size)
-	_, err := io.ReadFull(rand.Reader, data)
+	_, err := io.ReadFull(rnd, data)
 	OK(t, err)
 
 	id := restic.ID(sha256.Sum256(data))
@@ -145,6 +147,68 @@ func BenchmarkLoadTree(t *testing.B) {
 	}
 }
 
+func BenchmarkLoadBlob(b *testing.B) {
+	repo, cleanup := repository.TestRepository(b)
+	defer cleanup()
+
+	length := 1000000
+	buf := restic.NewBlobBuffer(length)
+	_, err := io.ReadFull(rnd, buf)
+	OK(b, err)
+
+	id, err := repo.SaveBlob(restic.DataBlob, buf, restic.ID{})
+	OK(b, err)
+	OK(b, repo.Flush())
+
+	b.ResetTimer()
+	b.SetBytes(int64(length))
+
+	for i := 0; i < b.N; i++ {
+		n, err := repo.LoadBlob(restic.DataBlob, id, buf)
+		OK(b, err)
+		if n != length {
+			b.Errorf("wanted %d bytes, got %d", length, n)
+		}
+
+		id2 := restic.Hash(buf[:n])
+		if !id.Equal(id2) {
+			b.Errorf("wrong data returned, wanted %v, got %v", id.Str(), id2.Str())
+		}
+	}
+}
+
+func BenchmarkLoadAndDecrypt(b *testing.B) {
+	repo, cleanup := repository.TestRepository(b)
+	defer cleanup()
+
+	length := 1000000
+	buf := restic.NewBlobBuffer(length)
+	_, err := io.ReadFull(rnd, buf)
+	OK(b, err)
+
+	dataID := restic.Hash(buf)
+
+	storageID, err := repo.SaveUnpacked(restic.DataFile, buf)
+	OK(b, err)
+	// OK(b, repo.Flush())
+
+	b.ResetTimer()
+	b.SetBytes(int64(length))
+
+	for i := 0; i < b.N; i++ {
+		data, err := repo.LoadAndDecrypt(restic.DataFile, storageID)
+		OK(b, err)
+		if len(data) != length {
+			b.Errorf("wanted %d bytes, got %d", length, len(data))
+		}
+
+		id2 := restic.Hash(data)
+		if !dataID.Equal(id2) {
+			b.Errorf("wrong data returned, wanted %v, got %v", storageID.Str(), id2.Str())
+		}
+	}
+}
+
 func TestLoadJSONUnpacked(t *testing.T) {
 	repo, cleanup := repository.TestRepository(t)
 	defer cleanup()
@@ -182,25 +246,48 @@ func TestRepositoryLoadIndex(t *testing.T) {
 }
 
 func BenchmarkLoadIndex(b *testing.B) {
-	repodir, cleanup := Env(b, repoFixture)
+	repository.TestUseLowSecurityKDFParameters(b)
+
+	repo, cleanup := repository.TestRepository(b)
 	defer cleanup()
 
-	repo := repository.TestOpenLocal(b, repodir)
+	idx := repository.NewIndex()
+
+	for i := 0; i < 5000; i++ {
+		idx.Store(restic.PackedBlob{
+			Blob: restic.Blob{
+				Type:   restic.DataBlob,
+				Length: 1234,
+				ID:     restic.NewRandomID(),
+				Offset: 1235,
+			},
+			PackID: restic.NewRandomID(),
+		})
+	}
+
+	id, err := repository.SaveIndex(repo, idx)
+	OK(b, err)
+
+	b.Logf("index saved as %v (%v entries)", id.Str(), idx.Count(restic.DataBlob))
+	fi, err := repo.Backend().Stat(restic.Handle{Type: restic.IndexFile, Name: id.String()})
+	OK(b, err)
+	b.Logf("filesize is %v", fi.Size)
+
 	b.ResetTimer()
 
 	for i := 0; i < b.N; i++ {
-		repo.SetIndex(repository.NewMasterIndex())
+		_, err := repository.LoadIndex(repo, id)
-		OK(b, repo.LoadIndex())
+		OK(b, err)
 	}
 }
 
 // saveRandomDataBlobs generates random data blobs and saves them to the repository.
 func saveRandomDataBlobs(t testing.TB, repo restic.Repository, num int, sizeMax int) {
 	for i := 0; i < num; i++ {
-		size := mrand.Int() % sizeMax
+		size := rand.Int() % sizeMax
 
 		buf := make([]byte, size)
-		_, err := io.ReadFull(rand.Reader, buf)
+		_, err := io.ReadFull(rnd, buf)
 		OK(t, err)
 
 		_, err = repo.SaveBlob(restic.DataBlob, buf, restic.ID{})

src/restic/testing.go

@@ -23,16 +23,26 @@ type fakeFileSystem struct {
 	repo        Repository
 	knownBlobs  IDSet
 	duplication float32
+	buf         []byte
+	chunker     *chunker.Chunker
 }
 
 // saveFile reads from rd and saves the blobs in the repository. The list of
 // IDs is returned.
-func (fs fakeFileSystem) saveFile(rd io.Reader) (blobs IDs) {
+func (fs *fakeFileSystem) saveFile(rd io.Reader) (blobs IDs) {
-	blobs = IDs{}
+	if fs.buf == nil {
-	ch := chunker.New(rd, fs.repo.Config().ChunkerPolynomial)
+		fs.buf = make([]byte, chunker.MaxSize)
+	}
 
+	if fs.chunker == nil {
+		fs.chunker = chunker.New(rd, fs.repo.Config().ChunkerPolynomial)
+	} else {
+		fs.chunker.Reset(rd, fs.repo.Config().ChunkerPolynomial)
+	}
+
+	blobs = IDs{}
 	for {
-		chunk, err := ch.Next(getBuf())
+		chunk, err := fs.chunker.Next(fs.buf)
 		if errors.Cause(err) == io.EOF {
 			break
 		}
@@ -50,7 +60,6 @@ func (fs fakeFileSystem) saveFile(rd io.Reader) (blobs IDs) {
 
 			fs.knownBlobs.Insert(id)
 		}
-		freeBuf(chunk.Data)
 
 		blobs = append(blobs, id)
 	}
@@ -64,7 +73,7 @@ const (
 	maxNodes    = 32
 )
 
-func (fs fakeFileSystem) treeIsKnown(tree *Tree) (bool, []byte, ID) {
+func (fs *fakeFileSystem) treeIsKnown(tree *Tree) (bool, []byte, ID) {
 	data, err := json.Marshal(tree)
 	if err != nil {
 		fs.t.Fatalf("json.Marshal(tree) returned error: %v", err)
@@ -76,7 +85,7 @@ func (fs fakeFileSystem) treeIsKnown(tree *Tree) (bool, []byte, ID) {
 	return fs.blobIsKnown(id, TreeBlob), data, id
 }
 
-func (fs fakeFileSystem) blobIsKnown(id ID, t BlobType) bool {
+func (fs *fakeFileSystem) blobIsKnown(id ID, t BlobType) bool {
 	if rand.Float32() < fs.duplication {
 		return false
 	}
@@ -94,7 +103,7 @@ func (fs fakeFileSystem) blobIsKnown(id ID, t BlobType) bool {
 }
 
 // saveTree saves a tree of fake files in the repo and returns the ID.
-func (fs fakeFileSystem) saveTree(seed int64, depth int) ID {
+func (fs *fakeFileSystem) saveTree(seed int64, depth int) ID {
 	rnd := rand.NewSource(seed)
 	numNodes := int(rnd.Int63() % maxNodes)
 

src/restic/testing_test.go

@@ -47,3 +47,14 @@ func TestCreateSnapshot(t *testing.T) {
 
 	checker.TestCheckRepo(t, repo)
 }
+
+func BenchmarkTestCreateSnapshot(t *testing.B) {
+	repo, cleanup := repository.TestRepository(t)
+	defer cleanup()
+
+	t.ResetTimer()
+
+	for i := 0; i < t.N; i++ {
+		restic.TestCreateSnapshot(t, repo, testSnapshotTime.Add(time.Duration(i)*time.Second), testDepth, 0)
+	}
+}