octoleo/restic
2
2
Fork 0
mirror of https://github.com/octoleo/restic.git synced 2024-12-22 10:58:55 +00:00
Code Issues Releases Wiki Activity

Merge Repository.{LoadBlob,loadBlob}

Browse Source 
Pushing the allocation logic down into the former loadBlob body means
that fewer allocations have to be performed:

name              old time/op    new time/op    delta
LoadTree-8           478µs ± 1%     481µs ± 2%    ~     (p=0.315 n=9+10)
LoadBlob-8          11.6ms ± 1%    11.6ms ± 2%    ~     (p=0.393 n=10+10)
LoadAndDecrypt-8    13.3ms ± 3%    13.3ms ± 3%    ~     (p=0.905 n=10+9)
LoadIndex-8         33.6ms ± 2%    33.2ms ± 1%  -1.15%  (p=0.028 n=10+9)

name              old alloc/op   new alloc/op   delta
LoadTree-8          41.2kB ± 0%    41.1kB ± 0%  -0.23%  (p=0.000 n=10+10)
LoadBlob-8          2.28kB ± 0%    2.18kB ± 0%  -4.21%  (p=0.000 n=10+10)
LoadAndDecrypt-8    2.10MB ± 0%    2.10MB ± 0%    ~     (all equal)
LoadIndex-8         5.22MB ± 0%    5.22MB ± 0%    ~     (p=0.631 n=10+10)

name              old allocs/op  new allocs/op  delta
LoadTree-8             652 ± 0%       651 ± 0%  -0.15%  (p=0.000 n=10+10)
LoadBlob-8            24.0 ± 0%      23.0 ± 0%  -4.17%  (p=0.000 n=10+10)
LoadAndDecrypt-8      30.0 ± 0%      30.0 ± 0%    ~     (all equal)
LoadIndex-8          30.2k ± 0%     30.2k ± 0%    ~     (p=0.610 n=10+10)

name              old speed      new speed      delta
LoadBlob-8        86.4MB/s ± 1%  85.9MB/s ± 2%    ~     (p=0.393 n=10+10)
LoadAndDecrypt-8  75.4MB/s ± 3%  75.4MB/s ± 3%    ~     (p=0.858 n=10+9)
This commit is contained in:
greatroar 2020-03-10 17:52:14 +01:00
parent be5a0ff59f
commit e7d7b85d59
1 changed files with 18 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

69
internal/repository/repository.go
View File

@ -130,17 +130,16 @@ func (r *Repository) sortCachedPacks(blobs []restic.PackedBlob) []restic.PackedB
return append(cached, noncached...)
}
// loadBlob tries to load and decrypt content identified by t and id from a
// pack from the backend, the result is stored in plaintextBuf, which must be
// large enough to hold the complete blob.
func (r *Repository) loadBlob(ctx context.Context, id restic.ID, t restic.BlobType, plaintextBuf []byte) (int, error) {
debug.Log("load %v with id %v (buf len %v, cap %d)", t, id, len(plaintextBuf), cap(plaintextBuf))
// LoadBlob loads a blob of type t from the repository.
// It may use all of buf[:cap(buf)] as scratch space.
func (r *Repository) LoadBlob(ctx context.Context, t restic.BlobType, id restic.ID, buf []byte) ([]byte, error) {
debug.Log("load %v with id %v (buf len %v, cap %d)", t, id, len(buf), cap(buf))
// lookup packs
blobs, found := r.idx.Lookup(id, t)
if !found {
debug.Log("id %v not found in index", id)
return 0, errors.Errorf("id %v not found in repository", id)
return nil, errors.Errorf("id %v not found in repository", id)
}
// try cached pack files first
@ -157,13 +156,14 @@ func (r *Repository) loadBlob(ctx context.Context, id restic.ID, t restic.BlobTy
// load blob from pack
h := restic.Handle{Type: restic.DataFile, Name: blob.PackID.String()}
if uint(cap(plaintextBuf)) < blob.Length {
return 0, errors.Errorf("buffer is too small: %v < %v", cap(plaintextBuf), blob.Length)
switch {
case cap(buf) < int(blob.Length):
buf = make([]byte, blob.Length)
case len(buf) != int(blob.Length):
buf = buf[:blob.Length]
}
plaintextBuf = plaintextBuf[:blob.Length]
n, err := restic.ReadAt(ctx, r.be, h, int64(blob.Offset), plaintextBuf)
n, err := restic.ReadAt(ctx, r.be, h, int64(blob.Offset), buf)
if err != nil {
debug.Log("error loading blob %v: %v", blob, err)
lastError = err
@ -178,7 +178,7 @@ func (r *Repository) loadBlob(ctx context.Context, id restic.ID, t restic.BlobTy
}
// decrypt
nonce, ciphertext := plaintextBuf[:r.key.NonceSize()], plaintextBuf[r.key.NonceSize():]
nonce, ciphertext := buf[:r.key.NonceSize()], buf[r.key.NonceSize():]
plaintext, err := r.key.Open(ciphertext[:0], nonce, ciphertext, nil)
if err != nil {
lastError = errors.Errorf("decrypting blob %v failed: %v", id, err)
@ -191,16 +191,16 @@ func (r *Repository) loadBlob(ctx context.Context, id restic.ID, t restic.BlobTy
continue
}
// move decrypted data to the start of the provided buffer
copy(plaintextBuf[0:], plaintext)
return len(plaintext), nil
// move decrypted data to the start of the buffer
copy(buf, plaintext)
return buf[:len(plaintext)], nil
}
if lastError != nil {
return 0, lastError
return nil, lastError
}
return 0, errors.Errorf("loading blob %v from %v packs failed", id.Str(), len(blobs))
return nil, errors.Errorf("loading blob %v from %v packs failed", id.Str(), len(blobs))
}
// LoadJSONUnpacked decrypts the data and afterwards calls json.Unmarshal on
@ -670,30 +670,6 @@ func (r *Repository) Close() error {
return r.be.Close()
}
// LoadBlob loads a blob of type t from the repository.
// It may use all of buf[:cap(buf)] as scratch space.
func (r *Repository) LoadBlob(ctx context.Context, t restic.BlobType, id restic.ID, buf []byte) ([]byte, error) {
debug.Log("load blob %v into buf (len %v, cap %v)", id, len(buf), cap(buf))
size, found := r.idx.LookupSize(id, t)
if !found {
return nil, errors.Errorf("id %v not found in repository", id)
}
if cap(buf) < restic.CiphertextLength(int(size)) {
buf = restic.NewBlobBuffer(int(size))
}
n, err := r.loadBlob(ctx, id, t, buf)
if err != nil {
return nil, err
}
buf = buf[:n]
debug.Log("loaded %d bytes into buf %p", len(buf), buf)
return buf, err
}
// SaveBlob saves a blob of type t into the repository. If id is the null id, it
// will be computed and returned.
func (r *Repository) SaveBlob(ctx context.Context, t restic.BlobType, buf []byte, id restic.ID) (restic.ID, error) {
@ -708,19 +684,10 @@ func (r *Repository) SaveBlob(ctx context.Context, t restic.BlobType, buf []byte
func (r *Repository) LoadTree(ctx context.Context, id restic.ID) (*restic.Tree, error) {
debug.Log("load tree %v", id)
size, found := r.idx.LookupSize(id, restic.TreeBlob)
if !found {
return nil, errors.Errorf("tree %v not found in repository", id)
}
debug.Log("size is %d, create buffer", size)
buf := restic.NewBlobBuffer(int(size))
n, err := r.loadBlob(ctx, id, restic.TreeBlob, buf)
buf, err := r.LoadBlob(ctx, restic.TreeBlob, id, nil)
if err != nil {
return nil, err
}
buf = buf[:n]
t := &restic.Tree{}
err = json.Unmarshal(buf, t)