2
2
mirror of https://github.com/octoleo/restic.git synced 2024-12-23 19:38:57 +00:00

Merge pull request #779 from restic/benchmark-checker

checker: Reduce memory usage
This commit is contained in:
Alexander Neumann 2017-02-07 11:03:49 +01:00
commit ed1739acbd
3 changed files with 84 additions and 10 deletions

View File

@ -14,6 +14,11 @@ type Blob struct {
Offset uint Offset uint
} }
func (b Blob) String() string {
return fmt.Sprintf("<Blob (%v) %v, offset %v, length %v>",
b.Type, b.ID.Str(), b.Offset, b.Length)
}
// PackedBlob is a blob stored within a file. // PackedBlob is a blob stored within a file.
type PackedBlob struct { type PackedBlob struct {
Blob Blob

View File

@ -1,14 +1,17 @@
package checker package checker
import ( import (
"bytes" "crypto/sha256"
"fmt" "fmt"
"io"
"io/ioutil"
"os"
"sync" "sync"
"restic/errors" "restic/errors"
"restic/hashing"
"restic" "restic"
"restic/backend"
"restic/crypto" "restic/crypto"
"restic/debug" "restic/debug"
"restic/pack" "restic/pack"
@ -659,36 +662,77 @@ func (c *Checker) CountPacks() uint64 {
func checkPack(r restic.Repository, id restic.ID) error { func checkPack(r restic.Repository, id restic.ID) error {
debug.Log("checking pack %v", id.Str()) debug.Log("checking pack %v", id.Str())
h := restic.Handle{Type: restic.DataFile, Name: id.String()} h := restic.Handle{Type: restic.DataFile, Name: id.String()}
buf, err := backend.LoadAll(r.Backend(), h)
rd, err := r.Backend().Load(h, 0, 0)
if err != nil { if err != nil {
return err return err
} }
hash := restic.Hash(buf) packfile, err := ioutil.TempFile("", "restic-temp-check-")
if err != nil {
return errors.Wrap(err, "TempFile")
}
defer func() {
packfile.Close()
os.Remove(packfile.Name())
}()
hrd := hashing.NewReader(rd, sha256.New())
size, err := io.Copy(packfile, hrd)
if err != nil {
return errors.Wrap(err, "Copy")
}
if err = rd.Close(); err != nil {
return err
}
hash := restic.IDFromHash(hrd.Sum(nil))
debug.Log("hash for pack %v is %v", id.Str(), hash.Str())
if !hash.Equal(id) { if !hash.Equal(id) {
debug.Log("Pack ID does not match, want %v, got %v", id.Str(), hash.Str()) debug.Log("Pack ID does not match, want %v, got %v", id.Str(), hash.Str())
return errors.Errorf("Pack ID does not match, want %v, got %v", id.Str(), hash.Str()) return errors.Errorf("Pack ID does not match, want %v, got %v", id.Str(), hash.Str())
} }
blobs, err := pack.List(r.Key(), bytes.NewReader(buf), int64(len(buf))) blobs, err := pack.List(r.Key(), packfile, size)
if err != nil { if err != nil {
return err return err
} }
var errs []error var errs []error
var buf []byte
for i, blob := range blobs { for i, blob := range blobs {
debug.Log(" check blob %d: %v", i, blob.ID.Str()) debug.Log(" check blob %d: %v", i, blob)
plainBuf := make([]byte, blob.Length) buf = buf[:cap(buf)]
n, err := crypto.Decrypt(r.Key(), plainBuf, buf[blob.Offset:blob.Offset+blob.Length]) if uint(len(buf)) < blob.Length {
buf = make([]byte, blob.Length)
}
buf = buf[:blob.Length]
_, err := packfile.Seek(int64(blob.Offset), 0)
if err != nil {
return errors.Errorf("Seek(%v): %v", blob.Offset, err)
}
_, err = io.ReadFull(packfile, buf)
if err != nil {
debug.Log(" error loading blob %v: %v", blob.ID.Str(), err)
errs = append(errs, errors.Errorf("blob %v: %v", i, err))
continue
}
n, err := crypto.Decrypt(r.Key(), buf, buf)
if err != nil { if err != nil {
debug.Log(" error decrypting blob %v: %v", blob.ID.Str(), err) debug.Log(" error decrypting blob %v: %v", blob.ID.Str(), err)
errs = append(errs, errors.Errorf("blob %v: %v", i, err)) errs = append(errs, errors.Errorf("blob %v: %v", i, err))
continue continue
} }
plainBuf = plainBuf[:n] buf = buf[:n]
hash := restic.Hash(plainBuf) hash := restic.Hash(buf)
if !hash.Equal(blob.ID) { if !hash.Equal(blob.ID) {
debug.Log(" Blob ID does not match, want %v, got %v", blob.ID.Str(), hash.Str()) debug.Log(" Blob ID does not match, want %v, got %v", blob.ID.Str(), hash.Str())
errs = append(errs, errors.Errorf("Blob ID does not match, want %v, got %v", blob.ID.Str(), hash.Str())) errs = append(errs, errors.Errorf("Blob ID does not match, want %v, got %v", blob.ID.Str(), hash.Str()))

View File

@ -299,3 +299,28 @@ func TestCheckerModifiedData(t *testing.T) {
t.Fatal("no error found, checker is broken") t.Fatal("no error found, checker is broken")
} }
} }
func BenchmarkChecker(t *testing.B) {
repodir, cleanup := test.Env(t, checkerTestData)
defer cleanup()
repo := repository.TestOpenLocal(t, repodir)
chkr := checker.New(repo)
hints, errs := chkr.LoadIndex()
if len(errs) > 0 {
t.Fatalf("expected no errors, got %v: %v", len(errs), errs)
}
if len(hints) > 0 {
t.Errorf("expected no hints, got %v: %v", len(hints), hints)
}
t.ResetTimer()
for i := 0; i < t.N; i++ {
test.OKs(t, checkPacks(chkr))
test.OKs(t, checkStruct(chkr))
test.OKs(t, checkData(chkr))
}
}