Merge pull request #4177 from greatroar/loadunpacked

repository: Reuse buffers in Repository.LoadUnpacked

cmd/restic/cmd_cat.go

@@ -72,7 +72,7 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string) error {
 		Println(string(buf))
 		return nil
 	case "index":
-		buf, err := repo.LoadUnpacked(ctx, restic.IndexFile, id, nil)
+		buf, err := repo.LoadUnpacked(ctx, restic.IndexFile, id)
 		if err != nil {
 			return err
 		}

internal/index/index_parallel.go

@@ -24,7 +24,7 @@ func ForAllIndexes(ctx context.Context, repo restic.Repository,
 		var idx *Index
 		oldFormat := false
 
-		buf, err := repo.LoadUnpacked(ctx, restic.IndexFile, id, nil)
+		buf, err := repo.LoadUnpacked(ctx, restic.IndexFile, id)
 		if err == nil {
 			idx, oldFormat, err = DecodeIndex(buf, id)
 		}

internal/repository/repository.go

@@ -170,14 +170,8 @@ func (r *Repository) SetDryRun() {
 	r.be = dryrun.New(r.be)
 }
 
-// LoadUnpacked loads and decrypts the file with the given type and ID, using
+// LoadUnpacked loads and decrypts the file with the given type and ID.
-// the supplied buffer (which must be empty). If the buffer is nil, a new
+func (r *Repository) LoadUnpacked(ctx context.Context, t restic.FileType, id restic.ID) ([]byte, error) {
-// buffer will be allocated and returned.
-func (r *Repository) LoadUnpacked(ctx context.Context, t restic.FileType, id restic.ID, buf []byte) ([]byte, error) {
-	if len(buf) != 0 {
-		panic("buf is not empty")
-	}
-
 	debug.Log("load %v with id %v", t, id)
 
 	if t == restic.ConfigFile {
@@ -189,15 +183,17 @@ func (r *Repository) LoadUnpacked(ctx context.Context, t restic.FileType, id res
 	h := restic.Handle{Type: t, Name: id.String()}
 	retriedInvalidData := false
 	var dataErr error
+	wr := new(bytes.Buffer)
+
 	err := r.be.Load(ctx, h, 0, 0, func(rd io.Reader) error {
 		// make sure this call is idempotent, in case an error occurs
-		wr := bytes.NewBuffer(buf[:0])
+		wr.Reset()
 		_, cerr := io.Copy(wr, rd)
 		if cerr != nil {
 			return cerr
 		}
-		buf = wr.Bytes()
 
+		buf := wr.Bytes()
 		if t != restic.ConfigFile && !restic.Hash(buf).Equal(id) {
 			debug.Log("retry loading broken blob %v", h)
 			if !retriedInvalidData {
@@ -221,6 +217,7 @@ func (r *Repository) LoadUnpacked(ctx context.Context, t restic.FileType, id res
 		return nil, err
 	}
 
+	buf := wr.Bytes()
 	nonce, ciphertext := buf[:r.key.NonceSize()], buf[r.key.NonceSize():]
 	plaintext, err := r.key.Open(ciphertext[:0], nonce, ciphertext, nil)
 	if err != nil {

internal/repository/repository_test.go

@@ -232,7 +232,7 @@ func benchmarkLoadUnpacked(b *testing.B, version uint) {
 	b.SetBytes(int64(length))
 
 	for i := 0; i < b.N; i++ {
-		data, err := repo.LoadUnpacked(context.TODO(), restic.PackFile, storageID, nil)
+		data, err := repo.LoadUnpacked(context.TODO(), restic.PackFile, storageID)
 		rtest.OK(b, err)
 
 		// See comment in BenchmarkLoadBlob.
@@ -261,7 +261,7 @@ func TestRepositoryLoadIndex(t *testing.T) {
 
 // loadIndex loads the index id from backend and returns it.
 func loadIndex(ctx context.Context, repo restic.Repository, id restic.ID) (*index.Index, error) {
-	buf, err := repo.LoadUnpacked(ctx, restic.IndexFile, id, nil)
+	buf, err := repo.LoadUnpacked(ctx, restic.IndexFile, id)
 	if err != nil {
 		return nil, err
 	}
@@ -289,7 +289,7 @@ func TestRepositoryLoadUnpackedBroken(t *testing.T) {
 	rtest.OK(t, err)
 
 	// without a retry backend this will just return an error that the file is broken
-	_, err = repo.LoadUnpacked(context.TODO(), restic.IndexFile, id, nil)
+	_, err = repo.LoadUnpacked(context.TODO(), restic.IndexFile, id)
 	if err == nil {
 		t.Fatal("missing expected error")
 	}

internal/restic/config_test.go

@@ -21,11 +21,11 @@ func (s saver) Connections() uint {
 }
 
 type loader struct {
-	fn func(restic.FileType, restic.ID, []byte) ([]byte, error)
+	fn func(restic.FileType, restic.ID) ([]byte, error)
 }
 
-func (l loader) LoadUnpacked(ctx context.Context, t restic.FileType, id restic.ID, buf []byte) (data []byte, err error) {
+func (l loader) LoadUnpacked(ctx context.Context, t restic.FileType, id restic.ID) (data []byte, err error) {
-	return l.fn(t, id, buf)
+	return l.fn(t, id)
 }
 
 func (l loader) Connections() uint {
@@ -49,7 +49,7 @@ func TestConfig(t *testing.T) {
 	err = restic.SaveConfig(context.TODO(), saver{save}, cfg1)
 	rtest.OK(t, err)
 
-	load := func(tpe restic.FileType, id restic.ID, in []byte) ([]byte, error) {
+	load := func(tpe restic.FileType, id restic.ID) ([]byte, error) {
 		rtest.Assert(t, tpe == restic.ConfigFile,
 			"wrong backend type: got %v, wanted %v",
 			tpe, restic.ConfigFile)

internal/restic/json.go

@@ -11,7 +11,7 @@ import (
 // LoadJSONUnpacked decrypts the data and afterwards calls json.Unmarshal on
 // the item.
 func LoadJSONUnpacked(ctx context.Context, repo LoaderUnpacked, t FileType, id ID, item interface{}) (err error) {
-	buf, err := repo.LoadUnpacked(ctx, t, id, nil)
+	buf, err := repo.LoadUnpacked(ctx, t, id)
 	if err != nil {
 		return err
 	}

internal/restic/repository.go

@@ -51,10 +51,8 @@ type Repository interface {
 	StartPackUploader(ctx context.Context, wg *errgroup.Group)
 	Flush(context.Context) error
 
-	// LoadUnpacked loads and decrypts the file with the given type and ID,
+	// LoadUnpacked loads and decrypts the file with the given type and ID.
-	// using the supplied buffer (which must be empty). If the buffer is nil, a
+	LoadUnpacked(ctx context.Context, t FileType, id ID) (data []byte, err error)
-	// new buffer will be allocated and returned.
-	LoadUnpacked(ctx context.Context, t FileType, id ID, buf []byte) (data []byte, err error)
 	SaveUnpacked(context.Context, FileType, []byte) (ID, error)
 }
 
@@ -67,7 +65,7 @@ type Lister interface {
 type LoaderUnpacked interface {
 	// Connections returns the maximum number of concurrent backend operations
 	Connections() uint
-	LoadUnpacked(ctx context.Context, t FileType, id ID, buf []byte) (data []byte, err error)
+	LoadUnpacked(ctx context.Context, t FileType, id ID) (data []byte, err error)
 }
 
 // SaverUnpacked allows saving a blob not stored in a pack file