octoleo/restic
2
2
Fork 0
mirror of https://github.com/octoleo/restic.git synced 2024-11-26 14:56:29 +00:00
Code Issues Releases Wiki Activity

Merge pull request #4177 from greatroar/loadunpacked

Browse Source 
repository: Reuse buffers in Repository.LoadUnpacked
This commit is contained in:
Michael Eischer 2023-01-30 22:12:13 +01:00 committed by GitHub
commit 56ad761b19
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 20 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/restic/cmd_cat.go
View File

@ -72,7 +72,7 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string) error {
Println(string(buf))
return nil
case "index":
buf, err := repo.LoadUnpacked(ctx, restic.IndexFile, id, nil)
buf, err := repo.LoadUnpacked(ctx, restic.IndexFile, id)
if err != nil {
return err
}

2
internal/index/index_parallel.go
View File

@ -24,7 +24,7 @@ func ForAllIndexes(ctx context.Context, repo restic.Repository,
var idx *Index
oldFormat := false
buf, err := repo.LoadUnpacked(ctx, restic.IndexFile, id, nil)
buf, err := repo.LoadUnpacked(ctx, restic.IndexFile, id)
if err == nil {
idx, oldFormat, err = DecodeIndex(buf, id)
}

17
internal/repository/repository.go
View File

@ -170,14 +170,8 @@ func (r *Repository) SetDryRun() {
r.be = dryrun.New(r.be)
}
// LoadUnpacked loads and decrypts the file with the given type and ID, using
// the supplied buffer (which must be empty). If the buffer is nil, a new
// buffer will be allocated and returned.
func (r *Repository) LoadUnpacked(ctx context.Context, t restic.FileType, id restic.ID, buf []byte) ([]byte, error) {
if len(buf) != 0 {
panic("buf is not empty")
}
// LoadUnpacked loads and decrypts the file with the given type and ID.
func (r *Repository) LoadUnpacked(ctx context.Context, t restic.FileType, id restic.ID) ([]byte, error) {
debug.Log("load %v with id %v", t, id)
if t == restic.ConfigFile {
@ -189,15 +183,17 @@ func (r *Repository) LoadUnpacked(ctx context.Context, t restic.FileType, id res
h := restic.Handle{Type: t, Name: id.String()}
retriedInvalidData := false
var dataErr error
wr := new(bytes.Buffer)
err := r.be.Load(ctx, h, 0, 0, func(rd io.Reader) error {
// make sure this call is idempotent, in case an error occurs
wr := bytes.NewBuffer(buf[:0])
wr.Reset()
_, cerr := io.Copy(wr, rd)
if cerr != nil {
return cerr
}
buf = wr.Bytes()
buf := wr.Bytes()
if t != restic.ConfigFile && !restic.Hash(buf).Equal(id) {
debug.Log("retry loading broken blob %v", h)
if !retriedInvalidData {
@ -221,6 +217,7 @@ func (r *Repository) LoadUnpacked(ctx context.Context, t restic.FileType, id res
return nil, err
}
buf := wr.Bytes()
nonce, ciphertext := buf[:r.key.NonceSize()], buf[r.key.NonceSize():]
plaintext, err := r.key.Open(ciphertext[:0], nonce, ciphertext, nil)
if err != nil {

6
internal/repository/repository_test.go
View File

@ -232,7 +232,7 @@ func benchmarkLoadUnpacked(b *testing.B, version uint) {
b.SetBytes(int64(length))
for i := 0; i < b.N; i++ {
data, err := repo.LoadUnpacked(context.TODO(), restic.PackFile, storageID, nil)
data, err := repo.LoadUnpacked(context.TODO(), restic.PackFile, storageID)
rtest.OK(b, err)
// See comment in BenchmarkLoadBlob.
@ -261,7 +261,7 @@ func TestRepositoryLoadIndex(t *testing.T) {
// loadIndex loads the index id from backend and returns it.
func loadIndex(ctx context.Context, repo restic.Repository, id restic.ID) (*index.Index, error) {
buf, err := repo.LoadUnpacked(ctx, restic.IndexFile, id, nil)
buf, err := repo.LoadUnpacked(ctx, restic.IndexFile, id)
if err != nil {
return nil, err
}
@ -289,7 +289,7 @@ func TestRepositoryLoadUnpackedBroken(t *testing.T) {
rtest.OK(t, err)
// without a retry backend this will just return an error that the file is broken
_, err = repo.LoadUnpacked(context.TODO(), restic.IndexFile, id, nil)
_, err = repo.LoadUnpacked(context.TODO(), restic.IndexFile, id)
if err == nil {
t.Fatal("missing expected error")
}

8
internal/restic/config_test.go
View File

@ -21,11 +21,11 @@ func (s saver) Connections() uint {
}
type loader struct {
fn func(restic.FileType, restic.ID, []byte) ([]byte, error)
fn func(restic.FileType, restic.ID) ([]byte, error)
}
func (l loader) LoadUnpacked(ctx context.Context, t restic.FileType, id restic.ID, buf []byte) (data []byte, err error) {
return l.fn(t, id, buf)
func (l loader) LoadUnpacked(ctx context.Context, t restic.FileType, id restic.ID) (data []byte, err error) {
return l.fn(t, id)
}
func (l loader) Connections() uint {
@ -49,7 +49,7 @@ func TestConfig(t *testing.T) {
err = restic.SaveConfig(context.TODO(), saver{save}, cfg1)
rtest.OK(t, err)
load := func(tpe restic.FileType, id restic.ID, in []byte) ([]byte, error) {
load := func(tpe restic.FileType, id restic.ID) ([]byte, error) {
rtest.Assert(t, tpe == restic.ConfigFile,
"wrong backend type: got %v, wanted %v",
tpe, restic.ConfigFile)

2
internal/restic/json.go
View File

@ -11,7 +11,7 @@ import (
// LoadJSONUnpacked decrypts the data and afterwards calls json.Unmarshal on
// the item.
func LoadJSONUnpacked(ctx context.Context, repo LoaderUnpacked, t FileType, id ID, item interface{}) (err error) {
buf, err := repo.LoadUnpacked(ctx, t, id, nil)
buf, err := repo.LoadUnpacked(ctx, t, id)
if err != nil {
return err
}

8
internal/restic/repository.go
View File

@ -51,10 +51,8 @@ type Repository interface {
StartPackUploader(ctx context.Context, wg *errgroup.Group)
Flush(context.Context) error
// LoadUnpacked loads and decrypts the file with the given type and ID,
// using the supplied buffer (which must be empty). If the buffer is nil, a
// new buffer will be allocated and returned.
LoadUnpacked(ctx context.Context, t FileType, id ID, buf []byte) (data []byte, err error)
// LoadUnpacked loads and decrypts the file with the given type and ID.
LoadUnpacked(ctx context.Context, t FileType, id ID) (data []byte, err error)
SaveUnpacked(context.Context, FileType, []byte) (ID, error)
}
@ -67,7 +65,7 @@ type Lister interface {
type LoaderUnpacked interface {
// Connections returns the maximum number of concurrent backend operations
Connections() uint
LoadUnpacked(ctx context.Context, t FileType, id ID, buf []byte) (data []byte, err error)
LoadUnpacked(ctx context.Context, t FileType, id ID) (data []byte, err error)
}
// SaverUnpacked allows saving a blob not stored in a pack file