octoleo/restic
2
2
Fork 0
mirror of https://github.com/octoleo/restic.git synced 2024-11-21 20:35:12 +00:00
Code Issues Releases Wiki Activity

Centralize buffer allocation and size checking in Repository.LoadBlob

Browse Source 
Benchmark results for internal/repository:

name              old time/op    new time/op    delta
LoadTree-8           479µs ± 2%     478µs ± 1%   ~     (p=0.780 n=10+9)
LoadBlob-8          11.6ms ± 2%    11.6ms ± 1%   ~     (p=0.631 n=10+10)
LoadAndDecrypt-8    13.2ms ± 2%    13.3ms ± 3%   ~     (p=0.631 n=10+10)

name              old alloc/op   new alloc/op   delta
LoadTree-8          41.2kB ± 0%    41.2kB ± 0%   ~     (all equal)
LoadBlob-8          2.28kB ± 0%    2.28kB ± 0%   ~     (all equal)
LoadAndDecrypt-8    2.10MB ± 0%    2.10MB ± 0%   ~     (all equal)

name              old allocs/op  new allocs/op  delta
LoadTree-8             652 ± 0%       652 ± 0%   ~     (all equal)
LoadBlob-8            24.0 ± 0%      24.0 ± 0%   ~     (all equal)
LoadAndDecrypt-8      30.0 ± 0%      30.0 ± 0%   ~     (all equal)

name              old speed      new speed      delta
LoadBlob-8        86.2MB/s ± 2%  86.4MB/s ± 1%   ~     (p=0.594 n=10+10)
LoadAndDecrypt-8  75.7MB/s ± 2%  75.4MB/s ± 3%   ~     (p=0.617 n=10+10)
This commit is contained in:
greatroar 2020-03-10 16:41:22 +01:00
parent 956a1b0f96
commit be5a0ff59f
9 changed files with 37 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
cmd/restic/cmd_cat.go
View File

@ -170,18 +170,15 @@ func runCat(gopts GlobalOptions, args []string) error {
case "blob":
for _, t := range []restic.BlobType{restic.DataBlob, restic.TreeBlob} {
list, found := repo.Index().Lookup(id, t)
_, found := repo.Index().Lookup(id, t)
if !found {
continue
}
blob := list[0]
buf := make([]byte, blob.Length)
n, err := repo.LoadBlob(gopts.ctx, t, id, buf)
buf, err := repo.LoadBlob(gopts.ctx, t, id, nil)
if err != nil {
return err
}
buf = buf[:n]
_, err = os.Stdout.Write(buf)
return err

19
cmd/restic/cmd_dump.go
View File

@ -171,24 +171,15 @@ func runDump(opts DumpOptions, gopts GlobalOptions, args []string) error {
}
func getNodeData(ctx context.Context, output io.Writer, repo restic.Repository, node *restic.Node) error {
var buf []byte
var (
buf []byte
err error
)
for _, id := range node.Content {
size, found := repo.LookupBlobSize(id, restic.DataBlob)
if !found {
return errors.Errorf("id %v not found in repository", id)
}
buf = buf[:cap(buf)]
if len(buf) < restic.CiphertextLength(int(size)) {
buf = restic.NewBlobBuffer(int(size))
}
n, err := repo.LoadBlob(ctx, restic.DataBlob, id, buf)
buf, err = repo.LoadBlob(ctx, restic.DataBlob, id, buf)
if err != nil {
return err
}
buf = buf[:n]
_, err = output.Write(buf)
if err != nil {

4
internal/archiver/testing.go
View File

@ -228,13 +228,13 @@ func TestEnsureFileContent(ctx context.Context, t testing.TB, repo restic.Reposi
content := make([]byte, restic.CiphertextLength(len(file.Content)))
pos := 0
for _, id := range node.Content {
n, err := repo.LoadBlob(ctx, restic.DataBlob, id, content[pos:])
part, err := repo.LoadBlob(ctx, restic.DataBlob, id, content[pos:])
if err != nil {
t.Fatalf("error loading blob %v: %v", id.Str(), err)
return
}
pos += n
pos += len(part)
}
content = content[:pos]

7
internal/fuse/file.go
View File

@ -96,15 +96,14 @@ func (f *file) getBlobAt(ctx context.Context, i int) (blob []byte, err error) {
f.blobs[j] = nil
}
buf := restic.NewBlobBuffer(f.sizes[i])
n, err := f.root.repo.LoadBlob(ctx, restic.DataBlob, f.node.Content[i], buf)
blob, err = f.root.repo.LoadBlob(ctx, restic.DataBlob, f.node.Content[i], nil)
if err != nil {
debug.Log("LoadBlob(%v, %v) failed: %v", f.node.Name, f.node.Content[i], err)
return nil, err
}
f.blobs[i] = buf[:n]
f.blobs[i] = blob
return buf[:n], nil
return blob, nil
}
func (f *file) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.ReadResponse) error {

7
internal/fuse/fuse_test.go
View File

@ -94,12 +94,11 @@ func TestFuseFile(t *testing.T) {
rtest.Assert(t, found, "Expected to find blob id %v", id)
filesize += uint64(size)
buf := restic.NewBlobBuffer(int(size))
n, err := repo.LoadBlob(context.TODO(), restic.DataBlob, id, buf)
buf, err := repo.LoadBlob(context.TODO(), restic.DataBlob, id, nil)
rtest.OK(t, err)
if uint(n) != size {
t.Fatalf("not enough bytes read for id %v: want %v, got %v", id.Str(), size, n)
if len(buf) != int(size) {
t.Fatalf("not enough bytes read for id %v: want %v, got %v", id.Str(), size, len(buf))
}
if uint(len(buf)) != size {

15
internal/repository/repository.go
View File

@ -670,29 +670,28 @@ func (r *Repository) Close() error {
return r.be.Close()
}
// LoadBlob loads a blob of type t from the repository to the buffer. buf must
// be large enough to hold the encrypted blob, since it is used as scratch
// space.
func (r *Repository) LoadBlob(ctx context.Context, t restic.BlobType, id restic.ID, buf []byte) (int, error) {
// LoadBlob loads a blob of type t from the repository.
// It may use all of buf[:cap(buf)] as scratch space.
func (r *Repository) LoadBlob(ctx context.Context, t restic.BlobType, id restic.ID, buf []byte) ([]byte, error) {
debug.Log("load blob %v into buf (len %v, cap %v)", id, len(buf), cap(buf))
size, found := r.idx.LookupSize(id, t)
if !found {
return 0, errors.Errorf("id %v not found in repository", id)
return nil, errors.Errorf("id %v not found in repository", id)
}
if cap(buf) < restic.CiphertextLength(int(size)) {
return 0, errors.Errorf("buffer is too small for data blob (%d < %d)", cap(buf), restic.CiphertextLength(int(size)))
buf = restic.NewBlobBuffer(int(size))
}
n, err := r.loadBlob(ctx, id, t, buf)
if err != nil {
return 0, err
return nil, err
}
buf = buf[:n]
debug.Log("loaded %d bytes into buf %p", len(buf), buf)
return len(buf), err
return buf, err
}
// SaveBlob saves a blob of type t into the repository. If id is the null id, it

43
internal/repository/repository_test.go
View File

@ -43,10 +43,9 @@ func TestSave(t *testing.T) {
// rtest.OK(t, repo.SaveIndex())
// read back
buf := restic.NewBlobBuffer(size)
n, err := repo.LoadBlob(context.TODO(), restic.DataBlob, id, buf)
buf, err := repo.LoadBlob(context.TODO(), restic.DataBlob, id, nil)
rtest.OK(t, err)
rtest.Equals(t, len(buf), n)
rtest.Equals(t, size, len(buf))
rtest.Assert(t, len(buf) == len(data),
"number of bytes read back does not match: expected %d, got %d",
@ -77,10 +76,9 @@ func TestSaveFrom(t *testing.T) {
rtest.OK(t, repo.Flush(context.Background()))
// read back
buf := restic.NewBlobBuffer(size)
n, err := repo.LoadBlob(context.TODO(), restic.DataBlob, id, buf)
buf, err := repo.LoadBlob(context.TODO(), restic.DataBlob, id, nil)
rtest.OK(t, err)
rtest.Equals(t, len(buf), n)
rtest.Equals(t, size, len(buf))
rtest.Assert(t, len(buf) == len(data),
"number of bytes read back does not match: expected %d, got %d",
@ -164,33 +162,17 @@ func TestLoadBlob(t *testing.T) {
rtest.OK(t, err)
rtest.OK(t, repo.Flush(context.Background()))
// first, test with buffers that are too small
for _, testlength := range []int{length - 20, length, restic.CiphertextLength(length) - 1} {
buf = make([]byte, 0, testlength)
n, err := repo.LoadBlob(context.TODO(), restic.DataBlob, id, buf)
if err == nil {
t.Errorf("LoadBlob() did not return an error for a buffer that is too small to hold the blob")
continue
}
if n != 0 {
t.Errorf("LoadBlob() returned an error and n > 0")
continue
}
}
// then use buffers that are large enough
base := restic.CiphertextLength(length)
for _, testlength := range []int{base, base + 7, base + 15, base + 1000} {
for _, testlength := range []int{0, base - 20, base - 1, base, base + 7, base + 15, base + 1000} {
buf = make([]byte, 0, testlength)
n, err := repo.LoadBlob(context.TODO(), restic.DataBlob, id, buf)
buf, err := repo.LoadBlob(context.TODO(), restic.DataBlob, id, buf)
if err != nil {
t.Errorf("LoadBlob() returned an error for buffer size %v: %v", testlength, err)
continue
}
if n != length {
t.Errorf("LoadBlob() returned the wrong number of bytes: want %v, got %v", length, n)
if len(buf) != length {
t.Errorf("LoadBlob() returned the wrong number of bytes: want %v, got %v", length, len(buf))
continue
}
}
@ -213,13 +195,14 @@ func BenchmarkLoadBlob(b *testing.B) {
b.SetBytes(int64(length))
for i := 0; i < b.N; i++ {
n, err := repo.LoadBlob(context.TODO(), restic.DataBlob, id, buf)
var err error
buf, err = repo.LoadBlob(context.TODO(), restic.DataBlob, id, buf)
rtest.OK(b, err)
if n != length {
b.Errorf("wanted %d bytes, got %d", length, n)
if len(buf) != length {
b.Errorf("wanted %d bytes, got %d", length, len(buf))
}
id2 := restic.Hash(buf[:n])
id2 := restic.Hash(buf)
if !id.Equal(id2) {
b.Errorf("wrong data returned, wanted %v, got %v", id.Str(), id2.Str())
}

13
internal/restic/node.go
View File

@ -282,21 +282,10 @@ func (node Node) createFileAt(ctx context.Context, path string, repo Repository)
func (node Node) writeNodeContent(ctx context.Context, repo Repository, f *os.File) error {
var buf []byte
for _, id := range node.Content {
size, found := repo.LookupBlobSize(id, DataBlob)
if !found {
return errors.Errorf("id %v not found in repository", id)
}
buf = buf[:cap(buf)]
if len(buf) < CiphertextLength(int(size)) {
buf = NewBlobBuffer(int(size))
}
n, err := repo.LoadBlob(ctx, DataBlob, id, buf)
buf, err := repo.LoadBlob(ctx, DataBlob, id, buf)
if err != nil {
return err
}
buf = buf[:n]
_, err = f.Write(buf)
if err != nil {

2
internal/restic/repository.go
View File

@ -45,7 +45,7 @@ type Repository interface {
// new buffer will be allocated and returned.
LoadAndDecrypt(ctx context.Context, buf []byte, t FileType, id ID) (data []byte, err error)
LoadBlob(context.Context, BlobType, ID, []byte) (int, error)
LoadBlob(context.Context, BlobType, ID, []byte) ([]byte, error)
SaveBlob(context.Context, BlobType, []byte, ID) (ID, error)
LoadTree(context.Context, ID) (*Tree, error)