From 31055d88a52635f73afbae8a50ad56754293aee5 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Tue, 24 Jan 2017 09:03:33 +0100 Subject: [PATCH 1/6] Add debug messages --- src/restic/readerat.go | 4 ++++ src/restic/repository/repository.go | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/restic/readerat.go b/src/restic/readerat.go index 320d032a4..a57974473 100644 --- a/src/restic/readerat.go +++ b/src/restic/readerat.go @@ -2,6 +2,7 @@ package restic import ( "io" + "restic/debug" ) type backendReaderAt struct { @@ -20,6 +21,7 @@ func ReaderAt(be Backend, h Handle) io.ReaderAt { // ReadAt reads from the backend handle h at the given position. func ReadAt(be Backend, h Handle, offset int64, p []byte) (n int, err error) { + debug.Log("ReadAt(%v) at %v, len %v", h, offset, len(p)) rd, err := be.Load(h, len(p), offset) if err != nil { return 0, err @@ -31,5 +33,7 @@ func ReadAt(be Backend, h Handle, offset int64, p []byte) (n int, err error) { err = e } + debug.Log("ReadAt(%v) ReadFull returned %v bytes", h, n) + return n, err } diff --git a/src/restic/repository/repository.go b/src/restic/repository/repository.go index 3545f42de..8344271d6 100644 --- a/src/restic/repository/repository.go +++ b/src/restic/repository/repository.go @@ -520,7 +520,7 @@ func (r *Repository) Close() error { // be large enough to hold the encrypted blob, since it is used as scratch // space. func (r *Repository) LoadBlob(t restic.BlobType, id restic.ID, buf []byte) (int, error) { - debug.Log("load blob %v into buf %p", id.Str(), buf) + debug.Log("load blob %v into buf (len %v, cap %v)", id.Str(), len(buf), cap(buf)) size, err := r.idx.LookupSize(id, t) if err != nil { return 0, err From 0e445ec0f5ac362ecaa9d7ca9266a7cff8dd92f3 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Tue, 24 Jan 2017 09:23:13 +0100 Subject: [PATCH 2/6] checker: Use TestRepository --- src/restic/checker/checker_test.go | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/src/restic/checker/checker_test.go b/src/restic/checker/checker_test.go index bdd611471..612fc6053 100644 --- a/src/restic/checker/checker_test.go +++ b/src/restic/checker/checker_test.go @@ -9,7 +9,6 @@ import ( "restic" "restic/archiver" - "restic/backend/mem" "restic/checker" "restic/repository" "restic/test" @@ -249,19 +248,15 @@ func induceError(data []byte) { } func TestCheckerModifiedData(t *testing.T) { - be := mem.New() - - repository.TestUseLowSecurityKDFParameters(t) - - repo := repository.New(be) - test.OK(t, repo.Init(test.TestPassword)) + repo, cleanup := repository.TestRepository(t) + defer cleanup() arch := archiver.New(repo) _, id, err := arch.Snapshot(nil, []string{"."}, nil, nil) test.OK(t, err) t.Logf("archived as %v", id.Str()) - beError := &errorBackend{Backend: be} + beError := &errorBackend{Backend: repo.Backend()} checkRepo := repository.New(beError) test.OK(t, checkRepo.SearchKey(test.TestPassword, 5)) From 3d2fbed55f9f8aec4ce4d26fce933f6f3f8d3280 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Tue, 24 Jan 2017 11:25:33 +0100 Subject: [PATCH 3/6] Add test for LoadBlob --- src/restic/repository/repository_test.go | 45 ++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/src/restic/repository/repository_test.go b/src/restic/repository/repository_test.go index efa8fc35d..6ee99f265 100644 --- a/src/restic/repository/repository_test.go +++ b/src/restic/repository/repository_test.go @@ -147,6 +147,51 @@ func BenchmarkLoadTree(t *testing.B) { } } +func TestLoadBlob(t *testing.T) { + repo, cleanup := repository.TestRepository(t) + defer cleanup() + + length := 1000000 + buf := restic.NewBlobBuffer(length) + _, err := io.ReadFull(rnd, buf) + OK(t, err) + + id, err := repo.SaveBlob(restic.DataBlob, buf, restic.ID{}) + OK(t, err) + OK(t, repo.Flush()) + + // first, test with buffers that are too small + for _, testlength := range []int{length - 20, length, restic.CiphertextLength(length) - 1} { + buf = make([]byte, 0, testlength) + n, err := repo.LoadBlob(restic.DataBlob, id, buf) + if err == nil { + t.Errorf("LoadBlob() did not return an error for a buffer that is too small to hold the blob") + continue + } + + if n != 0 { + t.Errorf("LoadBlob() returned an error and n > 0") + continue + } + } + + // then use buffers that are large enough + base := restic.CiphertextLength(length) + for _, testlength := range []int{base, base + 7, base + 15, base + 1000} { + buf = make([]byte, 0, testlength) + n, err := repo.LoadBlob(restic.DataBlob, id, buf) + if err != nil { + t.Errorf("LoadBlob() returned an error for buffer size %v: %v", testlength, err) + continue + } + + if n != length { + t.Errorf("LoadBlob() returned the wrong number of bytes: want %v, got %v", length, n) + continue + } + } +} + func BenchmarkLoadBlob(b *testing.B) { repo, cleanup := repository.TestRepository(b) defer cleanup() From 2de7e0369800546e1db4adce39a7aa072160876b Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Tue, 24 Jan 2017 11:27:36 +0100 Subject: [PATCH 4/6] repository.LoadBlob: Read correct number of bytes --- src/restic/repository/repository.go | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/restic/repository/repository.go b/src/restic/repository/repository.go index 8344271d6..6cf70d1a4 100644 --- a/src/restic/repository/repository.go +++ b/src/restic/repository/repository.go @@ -77,7 +77,7 @@ func (r *Repository) LoadAndDecrypt(t restic.FileType, id restic.ID) ([]byte, er // pack from the backend, the result is stored in plaintextBuf, which must be // large enough to hold the complete blob. func (r *Repository) loadBlob(id restic.ID, t restic.BlobType, plaintextBuf []byte) (int, error) { - debug.Log("load %v with id %v (buf %p, len %d)", t, id.Str(), plaintextBuf, len(plaintextBuf)) + debug.Log("load %v with id %v (buf len %v, cap %d)", t, id.Str(), len(plaintextBuf), cap(plaintextBuf)) // lookup packs blobs, err := r.idx.Lookup(id, t) @@ -96,7 +96,12 @@ func (r *Repository) loadBlob(id restic.ID, t restic.BlobType, plaintextBuf []by // load blob from pack h := restic.Handle{Type: restic.DataFile, Name: blob.PackID.String()} - plaintextBuf = plaintextBuf[:cap(plaintextBuf)] + + if uint(cap(plaintextBuf)) < blob.Length { + return 0, errors.Errorf("buffer is too small: %v < %v", cap(plaintextBuf), blob.Length) + } + + plaintextBuf = plaintextBuf[:blob.Length] n, err := restic.ReadAt(r.be, h, int64(blob.Offset), plaintextBuf) if err != nil { @@ -526,9 +531,8 @@ func (r *Repository) LoadBlob(t restic.BlobType, id restic.ID, buf []byte) (int, return 0, err } - buf = buf[:cap(buf)] - if len(buf) < restic.CiphertextLength(int(size)) { - return 0, errors.Errorf("buffer is too small for data blob (%d < %d)", len(buf), restic.CiphertextLength(int(size))) + if cap(buf) < restic.CiphertextLength(int(size)) { + return 0, errors.Errorf("buffer is too small for data blob (%d < %d)", cap(buf), restic.CiphertextLength(int(size))) } n, err := r.loadBlob(id, t, buf) From 26da14f315fbfad09bc9e0fa10f16298fa2c0225 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Tue, 24 Jan 2017 11:41:29 +0100 Subject: [PATCH 5/6] fuse: improve tests --- src/restic/fuse/file.go | 30 +----- src/restic/fuse/file_test.go | 175 ++++++++++++++++------------------- 2 files changed, 83 insertions(+), 122 deletions(-) diff --git a/src/restic/fuse/file.go b/src/restic/fuse/file.go index 032705942..be63e8e89 100644 --- a/src/restic/fuse/file.go +++ b/src/restic/fuse/file.go @@ -4,8 +4,6 @@ package fuse import ( - "sync" - "restic/errors" "restic" @@ -35,29 +33,23 @@ type file struct { node *restic.Node ownerIsRoot bool - sizes []uint + sizes []int blobs [][]byte } const defaultBlobSize = 128 * 1024 -var blobPool = sync.Pool{ - New: func() interface{} { - return make([]byte, defaultBlobSize) - }, -} - func newFile(repo BlobLoader, node *restic.Node, ownerIsRoot bool) (*file, error) { debug.Log("create new file for %v with %d blobs", node.Name, len(node.Content)) var bytes uint64 - sizes := make([]uint, len(node.Content)) + sizes := make([]int, len(node.Content)) for i, id := range node.Content { size, err := repo.LookupBlobSize(id, restic.DataBlob) if err != nil { return nil, err } - sizes[i] = size + sizes[i] = int(size) bytes += uint64(size) } @@ -99,16 +91,7 @@ func (f *file) getBlobAt(i int) (blob []byte, err error) { return f.blobs[i], nil } - buf := blobPool.Get().([]byte) - buf = buf[:cap(buf)] - - if uint(len(buf)) < f.sizes[i] { - if len(buf) > defaultBlobSize { - blobPool.Put(buf) - } - buf = make([]byte, f.sizes[i]) - } - + buf := restic.NewBlobBuffer(f.sizes[i]) n, err := f.repo.LoadBlob(restic.DataBlob, f.node.Content[i], buf) if err != nil { debug.Log("LoadBlob(%v, %v) failed: %v", f.node.Name, f.node.Content[i], err) @@ -169,10 +152,7 @@ func (f *file) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.ReadR func (f *file) Release(ctx context.Context, req *fuse.ReleaseRequest) error { for i := range f.blobs { - if f.blobs[i] != nil { - blobPool.Put(f.blobs[i]) - f.blobs[i] = nil - } + f.blobs[i] = nil } return nil } diff --git a/src/restic/fuse/file_test.go b/src/restic/fuse/file_test.go index 090e43200..9b2e0982d 100644 --- a/src/restic/fuse/file_test.go +++ b/src/restic/fuse/file_test.go @@ -9,7 +9,9 @@ import ( "testing" "time" - "restic/errors" + "golang.org/x/net/context" + + "restic/repository" "bazil.org/fuse" @@ -17,108 +19,96 @@ import ( . "restic/test" ) -type MockRepo struct { - blobs map[restic.ID][]byte -} - -func NewMockRepo(content map[restic.ID][]byte) *MockRepo { - return &MockRepo{blobs: content} -} - -func (m *MockRepo) LookupBlobSize(id restic.ID, t restic.BlobType) (uint, error) { - buf, ok := m.blobs[id] - if !ok { - return 0, errors.New("blob not found") - } - - return uint(len(buf)), nil -} - -func (m *MockRepo) LoadBlob(t restic.BlobType, id restic.ID, buf []byte) (int, error) { - size, err := m.LookupBlobSize(id, t) - if err != nil { - return 0, err - } - - if uint(len(buf)) < size { - return 0, errors.New("buffer too small") - } - - buf = buf[:size] - copy(buf, m.blobs[id]) - return int(size), nil -} - -type MockContext struct{} - -func (m MockContext) Deadline() (time.Time, bool) { return time.Now(), false } -func (m MockContext) Done() <-chan struct{} { return nil } -func (m MockContext) Err() error { return nil } -func (m MockContext) Value(key interface{}) interface{} { return nil } - -var testContent = genTestContent() -var testContentLengths = []uint{ - 4646 * 1024, - 655 * 1024, - 378 * 1024, - 8108 * 1024, - 558 * 1024, -} -var testMaxFileSize uint - -func genTestContent() map[restic.ID][]byte { - m := make(map[restic.ID][]byte) - - for _, length := range testContentLengths { - buf := Random(int(length), int(length)) - id := restic.Hash(buf) - m[id] = buf - testMaxFileSize += length - } - - return m -} - -const maxBufSize = 20 * 1024 * 1024 - -func testRead(t *testing.T, f *file, offset, length int, data []byte) { - ctx := MockContext{} +func testRead(t testing.TB, f *file, offset, length int, data []byte) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() req := &fuse.ReadRequest{ Offset: int64(offset), Size: length, } resp := &fuse.ReadResponse{ - Data: make([]byte, length), + Data: data, } OK(t, f.Read(ctx, req, resp)) } -var offsetReadsTests = []struct { - offset, length int -}{ - {0, 5 * 1024 * 1024}, - {4000 * 1024, 1000 * 1024}, +func firstSnapshotID(t testing.TB, repo restic.Repository) (first restic.ID) { + done := make(chan struct{}) + defer close(done) + for id := range repo.List(restic.SnapshotFile, done) { + if first.IsNull() { + first = id + } + } + return first +} + +func loadFirstSnapshot(t testing.TB, repo restic.Repository) *restic.Snapshot { + id := firstSnapshotID(t, repo) + sn, err := restic.LoadSnapshot(repo, id) + OK(t, err) + return sn +} + +func loadTree(t testing.TB, repo restic.Repository, id restic.ID) *restic.Tree { + tree, err := repo.LoadTree(id) + OK(t, err) + return tree } func TestFuseFile(t *testing.T) { - repo := NewMockRepo(testContent) - ctx := MockContext{} + repo, cleanup := repository.TestRepository(t) + defer cleanup() - memfile := make([]byte, 0, maxBufSize) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + timestamp, err := time.Parse(time.RFC3339, "2017-01-24T10:42:56+01:00") + OK(t, err) + restic.TestCreateSnapshot(t, repo, timestamp, 2, 0.1) + + sn := loadFirstSnapshot(t, repo) + tree := loadTree(t, repo, *sn.Tree) + + var content restic.IDs + for _, node := range tree.Nodes { + content = append(content, node.Content...) + } + t.Logf("tree loaded, content: %v", content) + + var ( + filesize uint64 + memfile []byte + ) + for _, id := range content { + size, err := repo.LookupBlobSize(id, restic.DataBlob) + OK(t, err) + filesize += uint64(size) + + buf := restic.NewBlobBuffer(int(size)) + n, err := repo.LoadBlob(restic.DataBlob, id, buf) + OK(t, err) + + if uint(n) != size { + t.Fatalf("not enough bytes read for id %v: want %v, got %v", id.Str(), size, n) + } + + if uint(len(buf)) != size { + t.Fatalf("buffer has wrong length for id %v: want %v, got %v", id.Str(), size, len(buf)) + } - var ids restic.IDs - for id, buf := range repo.blobs { - ids = append(ids, id) memfile = append(memfile, buf...) } + t.Logf("filesize is %v, memfile has size %v", filesize, len(memfile)) + node := &restic.Node{ Name: "foo", Inode: 23, Mode: 0742, - Size: 42, - Content: ids, + Size: filesize, + Content: content, } f, err := newFile(repo, node, false) OK(t, err) @@ -131,28 +121,19 @@ func TestFuseFile(t *testing.T) { Equals(t, node.Size, attr.Size) Equals(t, (node.Size/uint64(attr.BlockSize))+1, attr.Blocks) - for i, test := range offsetReadsTests { - b := memfile[test.offset : test.offset+test.length] - buf := make([]byte, test.length) - testRead(t, f, test.offset, test.length, buf) - if !bytes.Equal(b, buf) { - t.Errorf("test %d failed, wrong data returned", i) - } - } - for i := 0; i < 200; i++ { - length := rand.Intn(int(testMaxFileSize) / 2) - offset := rand.Intn(int(testMaxFileSize)) - if length+offset > int(testMaxFileSize) { - diff := length + offset - int(testMaxFileSize) - length -= diff - } + offset := rand.Intn(int(filesize)) + length := rand.Intn(int(filesize)-offset) + 100 b := memfile[offset : offset+length] + buf := make([]byte, length) + testRead(t, f, offset, length, buf) if !bytes.Equal(b, buf) { - t.Errorf("test %d failed (offset %d, length %d), wrong data returned", i, offset, length) + t.Errorf("test %d failed, wrong data returned (offset %v, length %v)", i, offset, length) } } + + OK(t, f.Release(ctx, nil)) } From 0b982d3316b2cb6152662c123c1dc57f580a8c19 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Tue, 24 Jan 2017 11:51:21 +0100 Subject: [PATCH 6/6] Add restore test with larger files --- src/cmds/restic/integration_test.go | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/cmds/restic/integration_test.go b/src/cmds/restic/integration_test.go index a3734bd02..426367724 100644 --- a/src/cmds/restic/integration_test.go +++ b/src/cmds/restic/integration_test.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "io/ioutil" + mrand "math/rand" "os" "path/filepath" "regexp" @@ -739,6 +740,30 @@ func TestRestoreFilter(t *testing.T) { }) } +func TestRestore(t *testing.T) { + withTestEnvironment(t, func(env *testEnvironment, gopts GlobalOptions) { + testRunInit(t, gopts) + + for i := 0; i < 10; i++ { + p := filepath.Join(env.testdata, fmt.Sprintf("foo/bar/testfile%v", i)) + OK(t, os.MkdirAll(filepath.Dir(p), 0755)) + OK(t, appendRandomData(p, uint(mrand.Intn(5<<21)))) + } + + opts := BackupOptions{} + + testRunBackup(t, []string{env.testdata}, opts, gopts) + testRunCheck(t, gopts) + + // Restore latest without any filters + restoredir := filepath.Join(env.base, "restore") + testRunRestoreLatest(t, gopts, restoredir, nil, "") + + Assert(t, directoriesEqualContents(env.testdata, filepath.Join(restoredir, filepath.Base(env.testdata))), + "directories are not equal") + }) +} + func TestRestoreLatest(t *testing.T) { withTestEnvironment(t, func(env *testEnvironment, gopts GlobalOptions) {