From e7d7b85d5976723fd1563660e3761cf72257fde2 Mon Sep 17 00:00:00 2001 From: greatroar <@> Date: Tue, 10 Mar 2020 17:52:14 +0100 Subject: [PATCH] Merge Repository.{LoadBlob,loadBlob} MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pushing the allocation logic down into the former loadBlob body means that fewer allocations have to be performed: name old time/op new time/op delta LoadTree-8 478µs ± 1% 481µs ± 2% ~ (p=0.315 n=9+10) LoadBlob-8 11.6ms ± 1% 11.6ms ± 2% ~ (p=0.393 n=10+10) LoadAndDecrypt-8 13.3ms ± 3% 13.3ms ± 3% ~ (p=0.905 n=10+9) LoadIndex-8 33.6ms ± 2% 33.2ms ± 1% -1.15% (p=0.028 n=10+9) name old alloc/op new alloc/op delta LoadTree-8 41.2kB ± 0% 41.1kB ± 0% -0.23% (p=0.000 n=10+10) LoadBlob-8 2.28kB ± 0% 2.18kB ± 0% -4.21% (p=0.000 n=10+10) LoadAndDecrypt-8 2.10MB ± 0% 2.10MB ± 0% ~ (all equal) LoadIndex-8 5.22MB ± 0% 5.22MB ± 0% ~ (p=0.631 n=10+10) name old allocs/op new allocs/op delta LoadTree-8 652 ± 0% 651 ± 0% -0.15% (p=0.000 n=10+10) LoadBlob-8 24.0 ± 0% 23.0 ± 0% -4.17% (p=0.000 n=10+10) LoadAndDecrypt-8 30.0 ± 0% 30.0 ± 0% ~ (all equal) LoadIndex-8 30.2k ± 0% 30.2k ± 0% ~ (p=0.610 n=10+10) name old speed new speed delta LoadBlob-8 86.4MB/s ± 1% 85.9MB/s ± 2% ~ (p=0.393 n=10+10) LoadAndDecrypt-8 75.4MB/s ± 3% 75.4MB/s ± 3% ~ (p=0.858 n=10+9) --- internal/repository/repository.go | 69 ++++++++----------------------- 1 file changed, 18 insertions(+), 51 deletions(-) diff --git a/internal/repository/repository.go b/internal/repository/repository.go index 75dd78db2..08241f683 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -130,17 +130,16 @@ func (r *Repository) sortCachedPacks(blobs []restic.PackedBlob) []restic.PackedB return append(cached, noncached...) } -// loadBlob tries to load and decrypt content identified by t and id from a -// pack from the backend, the result is stored in plaintextBuf, which must be -// large enough to hold the complete blob. -func (r *Repository) loadBlob(ctx context.Context, id restic.ID, t restic.BlobType, plaintextBuf []byte) (int, error) { - debug.Log("load %v with id %v (buf len %v, cap %d)", t, id, len(plaintextBuf), cap(plaintextBuf)) +// LoadBlob loads a blob of type t from the repository. +// It may use all of buf[:cap(buf)] as scratch space. +func (r *Repository) LoadBlob(ctx context.Context, t restic.BlobType, id restic.ID, buf []byte) ([]byte, error) { + debug.Log("load %v with id %v (buf len %v, cap %d)", t, id, len(buf), cap(buf)) // lookup packs blobs, found := r.idx.Lookup(id, t) if !found { debug.Log("id %v not found in index", id) - return 0, errors.Errorf("id %v not found in repository", id) + return nil, errors.Errorf("id %v not found in repository", id) } // try cached pack files first @@ -157,13 +156,14 @@ func (r *Repository) loadBlob(ctx context.Context, id restic.ID, t restic.BlobTy // load blob from pack h := restic.Handle{Type: restic.DataFile, Name: blob.PackID.String()} - if uint(cap(plaintextBuf)) < blob.Length { - return 0, errors.Errorf("buffer is too small: %v < %v", cap(plaintextBuf), blob.Length) + switch { + case cap(buf) < int(blob.Length): + buf = make([]byte, blob.Length) + case len(buf) != int(blob.Length): + buf = buf[:blob.Length] } - plaintextBuf = plaintextBuf[:blob.Length] - - n, err := restic.ReadAt(ctx, r.be, h, int64(blob.Offset), plaintextBuf) + n, err := restic.ReadAt(ctx, r.be, h, int64(blob.Offset), buf) if err != nil { debug.Log("error loading blob %v: %v", blob, err) lastError = err @@ -178,7 +178,7 @@ func (r *Repository) loadBlob(ctx context.Context, id restic.ID, t restic.BlobTy } // decrypt - nonce, ciphertext := plaintextBuf[:r.key.NonceSize()], plaintextBuf[r.key.NonceSize():] + nonce, ciphertext := buf[:r.key.NonceSize()], buf[r.key.NonceSize():] plaintext, err := r.key.Open(ciphertext[:0], nonce, ciphertext, nil) if err != nil { lastError = errors.Errorf("decrypting blob %v failed: %v", id, err) @@ -191,16 +191,16 @@ func (r *Repository) loadBlob(ctx context.Context, id restic.ID, t restic.BlobTy continue } - // move decrypted data to the start of the provided buffer - copy(plaintextBuf[0:], plaintext) - return len(plaintext), nil + // move decrypted data to the start of the buffer + copy(buf, plaintext) + return buf[:len(plaintext)], nil } if lastError != nil { - return 0, lastError + return nil, lastError } - return 0, errors.Errorf("loading blob %v from %v packs failed", id.Str(), len(blobs)) + return nil, errors.Errorf("loading blob %v from %v packs failed", id.Str(), len(blobs)) } // LoadJSONUnpacked decrypts the data and afterwards calls json.Unmarshal on @@ -670,30 +670,6 @@ func (r *Repository) Close() error { return r.be.Close() } -// LoadBlob loads a blob of type t from the repository. -// It may use all of buf[:cap(buf)] as scratch space. -func (r *Repository) LoadBlob(ctx context.Context, t restic.BlobType, id restic.ID, buf []byte) ([]byte, error) { - debug.Log("load blob %v into buf (len %v, cap %v)", id, len(buf), cap(buf)) - size, found := r.idx.LookupSize(id, t) - if !found { - return nil, errors.Errorf("id %v not found in repository", id) - } - - if cap(buf) < restic.CiphertextLength(int(size)) { - buf = restic.NewBlobBuffer(int(size)) - } - - n, err := r.loadBlob(ctx, id, t, buf) - if err != nil { - return nil, err - } - buf = buf[:n] - - debug.Log("loaded %d bytes into buf %p", len(buf), buf) - - return buf, err -} - // SaveBlob saves a blob of type t into the repository. If id is the null id, it // will be computed and returned. func (r *Repository) SaveBlob(ctx context.Context, t restic.BlobType, buf []byte, id restic.ID) (restic.ID, error) { @@ -708,19 +684,10 @@ func (r *Repository) SaveBlob(ctx context.Context, t restic.BlobType, buf []byte func (r *Repository) LoadTree(ctx context.Context, id restic.ID) (*restic.Tree, error) { debug.Log("load tree %v", id) - size, found := r.idx.LookupSize(id, restic.TreeBlob) - if !found { - return nil, errors.Errorf("tree %v not found in repository", id) - } - - debug.Log("size is %d, create buffer", size) - buf := restic.NewBlobBuffer(int(size)) - - n, err := r.loadBlob(ctx, id, restic.TreeBlob, buf) + buf, err := r.LoadBlob(ctx, restic.TreeBlob, id, nil) if err != nil { return nil, err } - buf = buf[:n] t := &restic.Tree{} err = json.Unmarshal(buf, t)