2
2
mirror of https://github.com/octoleo/restic.git synced 2024-11-22 12:55:18 +00:00

Add []byte to repo.LoadAndDecrypt and utils.LoadAll

This commit changes the signatures for repository.LoadAndDecrypt and
utils.LoadAll to allow passing in a []byte as the buffer to use. This
buffer is enlarged as needed, and returned back to the caller for
further use.

In later commits, this allows reducing allocations by reusing a buffer
for multiple calls, e.g. in a worker function.
This commit is contained in:
Alexander Neumann 2019-03-24 21:59:14 +01:00
parent e046428c94
commit d51e9d1b98
10 changed files with 112 additions and 71 deletions

View File

@ -74,7 +74,7 @@ func runCat(gopts GlobalOptions, args []string) error {
fmt.Println(string(buf)) fmt.Println(string(buf))
return nil return nil
case "index": case "index":
buf, err := repo.LoadAndDecrypt(gopts.ctx, restic.IndexFile, id) buf, err := repo.LoadAndDecrypt(gopts.ctx, nil, restic.IndexFile, id)
if err != nil { if err != nil {
return err return err
} }
@ -99,7 +99,7 @@ func runCat(gopts GlobalOptions, args []string) error {
return nil return nil
case "key": case "key":
h := restic.Handle{Type: restic.KeyFile, Name: id.String()} h := restic.Handle{Type: restic.KeyFile, Name: id.String()}
buf, err := backend.LoadAll(gopts.ctx, repo.Backend(), h) buf, err := backend.LoadAll(gopts.ctx, nil, repo.Backend(), h)
if err != nil { if err != nil {
return err return err
} }
@ -150,7 +150,7 @@ func runCat(gopts GlobalOptions, args []string) error {
switch tpe { switch tpe {
case "pack": case "pack":
h := restic.Handle{Type: restic.DataFile, Name: id.String()} h := restic.Handle{Type: restic.DataFile, Name: id.String()}
buf, err := backend.LoadAll(gopts.ctx, repo.Backend(), h) buf, err := backend.LoadAll(gopts.ctx, nil, repo.Backend(), h)
if err != nil { if err != nil {
return err return err
} }

View File

@ -79,7 +79,7 @@ func (s *Suite) TestConfig(t *testing.T) {
var testString = "Config" var testString = "Config"
// create config and read it back // create config and read it back
_, err := backend.LoadAll(context.TODO(), b, restic.Handle{Type: restic.ConfigFile}) _, err := backend.LoadAll(context.TODO(), nil, b, restic.Handle{Type: restic.ConfigFile})
if err == nil { if err == nil {
t.Fatalf("did not get expected error for non-existing config") t.Fatalf("did not get expected error for non-existing config")
} }
@ -93,7 +93,7 @@ func (s *Suite) TestConfig(t *testing.T) {
// same config // same config
for _, name := range []string{"", "foo", "bar", "0000000000000000000000000000000000000000000000000000000000000000"} { for _, name := range []string{"", "foo", "bar", "0000000000000000000000000000000000000000000000000000000000000000"} {
h := restic.Handle{Type: restic.ConfigFile, Name: name} h := restic.Handle{Type: restic.ConfigFile, Name: name}
buf, err := backend.LoadAll(context.TODO(), b, h) buf, err := backend.LoadAll(context.TODO(), nil, b, h)
if err != nil { if err != nil {
t.Fatalf("unable to read config with name %q: %+v", name, err) t.Fatalf("unable to read config with name %q: %+v", name, err)
} }
@ -491,7 +491,7 @@ func (s *Suite) TestSave(t *testing.T) {
err := b.Save(context.TODO(), h, restic.NewByteReader(data)) err := b.Save(context.TODO(), h, restic.NewByteReader(data))
test.OK(t, err) test.OK(t, err)
buf, err := backend.LoadAll(context.TODO(), b, h) buf, err := backend.LoadAll(context.TODO(), nil, b, h)
test.OK(t, err) test.OK(t, err)
if len(buf) != len(data) { if len(buf) != len(data) {
t.Fatalf("number of bytes does not match, want %v, got %v", len(data), len(buf)) t.Fatalf("number of bytes does not match, want %v, got %v", len(data), len(buf))
@ -584,7 +584,7 @@ func (s *Suite) TestSaveFilenames(t *testing.T) {
continue continue
} }
buf, err := backend.LoadAll(context.TODO(), b, h) buf, err := backend.LoadAll(context.TODO(), nil, b, h)
if err != nil { if err != nil {
t.Errorf("test %d failed: Load() returned %+v", i, err) t.Errorf("test %d failed: Load() returned %+v", i, err)
continue continue
@ -734,7 +734,7 @@ func (s *Suite) TestBackend(t *testing.T) {
// test Load() // test Load()
h := restic.Handle{Type: tpe, Name: ts.id} h := restic.Handle{Type: tpe, Name: ts.id}
buf, err := backend.LoadAll(context.TODO(), b, h) buf, err := backend.LoadAll(context.TODO(), nil, b, h)
test.OK(t, err) test.OK(t, err)
test.Equals(t, ts.data, string(buf)) test.Equals(t, ts.data, string(buf))

View File

@ -1,20 +1,33 @@
package backend package backend
import ( import (
"bytes"
"context" "context"
"io" "io"
"io/ioutil"
"github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/restic"
) )
// LoadAll reads all data stored in the backend for the handle. // LoadAll reads all data stored in the backend for the handle into the given
func LoadAll(ctx context.Context, be restic.Backend, h restic.Handle) (buf []byte, err error) { // buffer, which is truncated. If the buffer is not large enough or nil, a new
err = be.Load(ctx, h, 0, 0, func(rd io.Reader) (ierr error) { // one is allocated.
buf, ierr = ioutil.ReadAll(rd) func LoadAll(ctx context.Context, buf []byte, be restic.Backend, h restic.Handle) ([]byte, error) {
return ierr err := be.Load(ctx, h, 0, 0, func(rd io.Reader) error {
// make sure this is idempotent, in case an error occurs this function may be called multiple times!
wr := bytes.NewBuffer(buf[:0])
_, cerr := io.Copy(wr, rd)
if cerr != nil {
return cerr
}
buf = wr.Bytes()
return nil
}) })
return buf, err
if err != nil {
return nil, err
}
return buf, nil
} }
// LimitedReadCloser wraps io.LimitedReader and exposes the Close() method. // LimitedReadCloser wraps io.LimitedReader and exposes the Close() method.

View File

@ -19,6 +19,7 @@ const MiB = 1 << 20
func TestLoadAll(t *testing.T) { func TestLoadAll(t *testing.T) {
b := mem.New() b := mem.New()
var buf []byte
for i := 0; i < 20; i++ { for i := 0; i < 20; i++ {
data := rtest.Random(23+i, rand.Intn(MiB)+500*KiB) data := rtest.Random(23+i, rand.Intn(MiB)+500*KiB)
@ -28,7 +29,7 @@ func TestLoadAll(t *testing.T) {
err := b.Save(context.TODO(), h, restic.NewByteReader(data)) err := b.Save(context.TODO(), h, restic.NewByteReader(data))
rtest.OK(t, err) rtest.OK(t, err)
buf, err := backend.LoadAll(context.TODO(), b, restic.Handle{Type: restic.DataFile, Name: id.String()}) buf, err := backend.LoadAll(context.TODO(), buf, b, restic.Handle{Type: restic.DataFile, Name: id.String()})
rtest.OK(t, err) rtest.OK(t, err)
if len(buf) != len(data) { if len(buf) != len(data) {
@ -43,55 +44,66 @@ func TestLoadAll(t *testing.T) {
} }
} }
func TestLoadSmallBuffer(t *testing.T) { func save(t testing.TB, be restic.Backend, buf []byte) restic.Handle {
b := mem.New() id := restic.Hash(buf)
h := restic.Handle{Name: id.String(), Type: restic.DataFile}
for i := 0; i < 20; i++ { err := be.Save(context.TODO(), h, restic.NewByteReader(buf))
data := rtest.Random(23+i, rand.Intn(MiB)+500*KiB) if err != nil {
t.Fatal(err)
id := restic.Hash(data)
h := restic.Handle{Name: id.String(), Type: restic.DataFile}
err := b.Save(context.TODO(), h, restic.NewByteReader(data))
rtest.OK(t, err)
buf, err := backend.LoadAll(context.TODO(), b, restic.Handle{Type: restic.DataFile, Name: id.String()})
rtest.OK(t, err)
if len(buf) != len(data) {
t.Errorf("length of returned buffer does not match, want %d, got %d", len(data), len(buf))
continue
}
if !bytes.Equal(buf, data) {
t.Errorf("wrong data returned")
continue
}
} }
return h
} }
func TestLoadLargeBuffer(t *testing.T) { func TestLoadAllAppend(t *testing.T) {
b := mem.New() b := mem.New()
for i := 0; i < 20; i++ { h1 := save(t, b, []byte("foobar test string"))
data := rtest.Random(23+i, rand.Intn(MiB)+500*KiB) randomData := rtest.Random(23, rand.Intn(MiB)+500*KiB)
h2 := save(t, b, randomData)
id := restic.Hash(data) var tests = []struct {
h := restic.Handle{Name: id.String(), Type: restic.DataFile} handle restic.Handle
err := b.Save(context.TODO(), h, restic.NewByteReader(data)) buf []byte
rtest.OK(t, err) want []byte
}{
{
handle: h1,
buf: nil,
want: []byte("foobar test string"),
},
{
handle: h1,
buf: []byte("xxx"),
want: []byte("foobar test string"),
},
{
handle: h2,
buf: nil,
want: randomData,
},
{
handle: h2,
buf: make([]byte, 0, 200),
want: randomData,
},
{
handle: h2,
buf: []byte("foobarbaz"),
want: randomData,
},
}
buf, err := backend.LoadAll(context.TODO(), b, restic.Handle{Type: restic.DataFile, Name: id.String()}) for _, test := range tests {
rtest.OK(t, err) t.Run("", func(t *testing.T) {
buf, err := backend.LoadAll(context.TODO(), test.buf, b, test.handle)
if err != nil {
t.Fatal(err)
}
if len(buf) != len(data) { if !bytes.Equal(buf, test.want) {
t.Errorf("length of returned buffer does not match, want %d, got %d", len(data), len(buf)) t.Errorf("wrong data returned, want %q, got %q", test.want, buf)
continue }
} })
if !bytes.Equal(buf, data) {
t.Errorf("wrong data returned")
continue
}
} }
} }

View File

@ -17,7 +17,7 @@ import (
) )
func loadAndCompare(t testing.TB, be restic.Backend, h restic.Handle, data []byte) { func loadAndCompare(t testing.TB, be restic.Backend, h restic.Handle, data []byte) {
buf, err := backend.LoadAll(context.TODO(), be, h) buf, err := backend.LoadAll(context.TODO(), nil, be, h)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -147,7 +147,7 @@ func TestErrorBackend(t *testing.T) {
loadTest := func(wg *sync.WaitGroup, be restic.Backend) { loadTest := func(wg *sync.WaitGroup, be restic.Backend) {
defer wg.Done() defer wg.Done()
buf, err := backend.LoadAll(context.TODO(), be, h) buf, err := backend.LoadAll(context.TODO(), nil, be, h)
if err == testErr { if err == testErr {
return return
} }

View File

@ -552,7 +552,7 @@ func DecodeOldIndex(buf []byte) (idx *Index, err error) {
func LoadIndexWithDecoder(ctx context.Context, repo restic.Repository, id restic.ID, fn func([]byte) (*Index, error)) (idx *Index, err error) { func LoadIndexWithDecoder(ctx context.Context, repo restic.Repository, id restic.ID, fn func([]byte) (*Index, error)) (idx *Index, err error) {
debug.Log("Loading index %v", id) debug.Log("Loading index %v", id)
buf, err := repo.LoadAndDecrypt(ctx, restic.IndexFile, id) buf, err := repo.LoadAndDecrypt(ctx, nil, restic.IndexFile, id)
if err != nil { if err != nil {
return nil, err return nil, err
} }

View File

@ -184,7 +184,7 @@ func SearchKey(ctx context.Context, s *Repository, password string, maxKeys int,
// LoadKey loads a key from the backend. // LoadKey loads a key from the backend.
func LoadKey(ctx context.Context, s *Repository, name string) (k *Key, err error) { func LoadKey(ctx context.Context, s *Repository, name string) (k *Key, err error) {
h := restic.Handle{Type: restic.KeyFile, Name: name} h := restic.Handle{Type: restic.KeyFile, Name: name}
data, err := backend.LoadAll(ctx, s.be, h) data, err := backend.LoadAll(ctx, nil, s.be, h)
if err != nil { if err != nil {
return nil, err return nil, err
} }

View File

@ -9,7 +9,6 @@ import (
"io" "io"
"os" "os"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/cache" "github.com/restic/restic/internal/cache"
"github.com/restic/restic/internal/crypto" "github.com/restic/restic/internal/crypto"
"github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/debug"
@ -67,15 +66,29 @@ func (r *Repository) PrefixLength(t restic.FileType) (int, error) {
return restic.PrefixLength(r.be, t) return restic.PrefixLength(r.be, t)
} }
// LoadAndDecrypt loads and decrypts data identified by t and id from the // LoadAndDecrypt loads and decrypts the file with the given type and ID, using
// backend. // the supplied buffer (which must be empty). If the buffer is nil, a new
func (r *Repository) LoadAndDecrypt(ctx context.Context, t restic.FileType, id restic.ID) (buf []byte, err error) { // buffer will be allocated and returned.
func (r *Repository) LoadAndDecrypt(ctx context.Context, buf []byte, t restic.FileType, id restic.ID) ([]byte, error) {
if len(buf) != 0 {
panic("buf is not empty")
}
debug.Log("load %v with id %v", t, id) debug.Log("load %v with id %v", t, id)
h := restic.Handle{Type: t, Name: id.String()} h := restic.Handle{Type: t, Name: id.String()}
buf, err = backend.LoadAll(ctx, r.be, h) err := r.be.Load(ctx, h, 0, 0, func(rd io.Reader) error {
// make sure this call is idempotent, in case an error occurs
wr := bytes.NewBuffer(buf[:0])
_, cerr := io.Copy(wr, rd)
if cerr != nil {
return cerr
}
buf = wr.Bytes()
return nil
})
if err != nil { if err != nil {
debug.Log("error loading %v: %v", h, err)
return nil, err return nil, err
} }
@ -188,7 +201,7 @@ func (r *Repository) loadBlob(ctx context.Context, id restic.ID, t restic.BlobTy
// LoadJSONUnpacked decrypts the data and afterwards calls json.Unmarshal on // LoadJSONUnpacked decrypts the data and afterwards calls json.Unmarshal on
// the item. // the item.
func (r *Repository) LoadJSONUnpacked(ctx context.Context, t restic.FileType, id restic.ID, item interface{}) (err error) { func (r *Repository) LoadJSONUnpacked(ctx context.Context, t restic.FileType, id restic.ID, item interface{}) (err error) {
buf, err := r.LoadAndDecrypt(ctx, t, id) buf, err := r.LoadAndDecrypt(ctx, nil, t, id)
if err != nil { if err != nil {
return err return err
} }

View File

@ -244,7 +244,7 @@ func BenchmarkLoadAndDecrypt(b *testing.B) {
b.SetBytes(int64(length)) b.SetBytes(int64(length))
for i := 0; i < b.N; i++ { for i := 0; i < b.N; i++ {
data, err := repo.LoadAndDecrypt(context.TODO(), restic.DataFile, storageID) data, err := repo.LoadAndDecrypt(context.TODO(), nil, restic.DataFile, storageID)
rtest.OK(b, err) rtest.OK(b, err)
if len(data) != length { if len(data) != length {
b.Errorf("wanted %d bytes, got %d", length, len(data)) b.Errorf("wanted %d bytes, got %d", length, len(data))

View File

@ -39,8 +39,11 @@ type Repository interface {
SaveUnpacked(context.Context, FileType, []byte) (ID, error) SaveUnpacked(context.Context, FileType, []byte) (ID, error)
SaveJSONUnpacked(context.Context, FileType, interface{}) (ID, error) SaveJSONUnpacked(context.Context, FileType, interface{}) (ID, error)
LoadJSONUnpacked(context.Context, FileType, ID, interface{}) error LoadJSONUnpacked(ctx context.Context, t FileType, id ID, dest interface{}) error
LoadAndDecrypt(context.Context, FileType, ID) ([]byte, error) // LoadAndDecrypt loads and decrypts the file with the given type and ID,
// using the supplied buffer (which must be empty). If the buffer is nil, a
// new buffer will be allocated and returned.
LoadAndDecrypt(ctx context.Context, buf []byte, t FileType, id ID) (data []byte, err error)
LoadBlob(context.Context, BlobType, ID, []byte) (int, error) LoadBlob(context.Context, BlobType, ID, []byte) (int, error)
SaveBlob(context.Context, BlobType, []byte, ID) (ID, error) SaveBlob(context.Context, BlobType, []byte, ID) (ID, error)