Merge pull request #745 from restic/fix-742

Fix restore/fuse with larger files

src/cmds/restic/integration_test.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	mrand "math/rand"
 	"os"
 	"path/filepath"
 	"regexp"
@@ -739,6 +740,30 @@ func TestRestoreFilter(t *testing.T) {
 	})
 }
 
+func TestRestore(t *testing.T) {
+	withTestEnvironment(t, func(env *testEnvironment, gopts GlobalOptions) {
+		testRunInit(t, gopts)
+
+		for i := 0; i < 10; i++ {
+			p := filepath.Join(env.testdata, fmt.Sprintf("foo/bar/testfile%v", i))
+			OK(t, os.MkdirAll(filepath.Dir(p), 0755))
+			OK(t, appendRandomData(p, uint(mrand.Intn(5<<21))))
+		}
+
+		opts := BackupOptions{}
+
+		testRunBackup(t, []string{env.testdata}, opts, gopts)
+		testRunCheck(t, gopts)
+
+		// Restore latest without any filters
+		restoredir := filepath.Join(env.base, "restore")
+		testRunRestoreLatest(t, gopts, restoredir, nil, "")
+
+		Assert(t, directoriesEqualContents(env.testdata, filepath.Join(restoredir, filepath.Base(env.testdata))),
+			"directories are not equal")
+	})
+}
+
 func TestRestoreLatest(t *testing.T) {
 
 	withTestEnvironment(t, func(env *testEnvironment, gopts GlobalOptions) {

src/restic/checker/checker_test.go

@@ -9,7 +9,6 @@ import (
 
 	"restic"
 	"restic/archiver"
-	"restic/backend/mem"
 	"restic/checker"
 	"restic/repository"
 	"restic/test"
@@ -249,19 +248,15 @@ func induceError(data []byte) {
 }
 
 func TestCheckerModifiedData(t *testing.T) {
-	be := mem.New()
+	repo, cleanup := repository.TestRepository(t)
-
+	defer cleanup()
-	repository.TestUseLowSecurityKDFParameters(t)
-
-	repo := repository.New(be)
-	test.OK(t, repo.Init(test.TestPassword))
 
 	arch := archiver.New(repo)
 	_, id, err := arch.Snapshot(nil, []string{"."}, nil, nil)
 	test.OK(t, err)
 	t.Logf("archived as %v", id.Str())
 
-	beError := &errorBackend{Backend: be}
+	beError := &errorBackend{Backend: repo.Backend()}
 	checkRepo := repository.New(beError)
 	test.OK(t, checkRepo.SearchKey(test.TestPassword, 5))
 

src/restic/fuse/file.go

@@ -4,8 +4,6 @@
 package fuse
 
 import (
-	"sync"
-
 	"restic/errors"
 
 	"restic"
@@ -35,29 +33,23 @@ type file struct {
 	node        *restic.Node
 	ownerIsRoot bool
 
-	sizes []uint
+	sizes []int
 	blobs [][]byte
 }
 
 const defaultBlobSize = 128 * 1024
 
-var blobPool = sync.Pool{
-	New: func() interface{} {
-		return make([]byte, defaultBlobSize)
-	},
-}
-
 func newFile(repo BlobLoader, node *restic.Node, ownerIsRoot bool) (*file, error) {
 	debug.Log("create new file for %v with %d blobs", node.Name, len(node.Content))
 	var bytes uint64
-	sizes := make([]uint, len(node.Content))
+	sizes := make([]int, len(node.Content))
 	for i, id := range node.Content {
 		size, err := repo.LookupBlobSize(id, restic.DataBlob)
 		if err != nil {
 			return nil, err
 		}
 
-		sizes[i] = size
+		sizes[i] = int(size)
 		bytes += uint64(size)
 	}
 
@@ -99,16 +91,7 @@ func (f *file) getBlobAt(i int) (blob []byte, err error) {
 		return f.blobs[i], nil
 	}
 
-	buf := blobPool.Get().([]byte)
+	buf := restic.NewBlobBuffer(f.sizes[i])
-	buf = buf[:cap(buf)]
-
-	if uint(len(buf)) < f.sizes[i] {
-		if len(buf) > defaultBlobSize {
-			blobPool.Put(buf)
-		}
-		buf = make([]byte, f.sizes[i])
-	}
-
 	n, err := f.repo.LoadBlob(restic.DataBlob, f.node.Content[i], buf)
 	if err != nil {
 		debug.Log("LoadBlob(%v, %v) failed: %v", f.node.Name, f.node.Content[i], err)
@@ -169,10 +152,7 @@ func (f *file) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.ReadR
 
 func (f *file) Release(ctx context.Context, req *fuse.ReleaseRequest) error {
 	for i := range f.blobs {
-		if f.blobs[i] != nil {
-			blobPool.Put(f.blobs[i])
 		f.blobs[i] = nil
 	}
-	}
 	return nil
 }

src/restic/fuse/file_test.go

@@ -9,7 +9,9 @@ import (
 	"testing"
 	"time"
 
-	"restic/errors"
+	"golang.org/x/net/context"
+
+	"restic/repository"
 
 	"bazil.org/fuse"
 
@@ -17,108 +19,96 @@ import (
 	. "restic/test"
 )
 
-type MockRepo struct {
+func testRead(t testing.TB, f *file, offset, length int, data []byte) {
-	blobs map[restic.ID][]byte
+	ctx, cancel := context.WithCancel(context.Background())
-}
+	defer cancel()
-
-func NewMockRepo(content map[restic.ID][]byte) *MockRepo {
-	return &MockRepo{blobs: content}
-}
-
-func (m *MockRepo) LookupBlobSize(id restic.ID, t restic.BlobType) (uint, error) {
-	buf, ok := m.blobs[id]
-	if !ok {
-		return 0, errors.New("blob not found")
-	}
-
-	return uint(len(buf)), nil
-}
-
-func (m *MockRepo) LoadBlob(t restic.BlobType, id restic.ID, buf []byte) (int, error) {
-	size, err := m.LookupBlobSize(id, t)
-	if err != nil {
-		return 0, err
-	}
-
-	if uint(len(buf)) < size {
-		return 0, errors.New("buffer too small")
-	}
-
-	buf = buf[:size]
-	copy(buf, m.blobs[id])
-	return int(size), nil
-}
-
-type MockContext struct{}
-
-func (m MockContext) Deadline() (time.Time, bool)       { return time.Now(), false }
-func (m MockContext) Done() <-chan struct{}             { return nil }
-func (m MockContext) Err() error                        { return nil }
-func (m MockContext) Value(key interface{}) interface{} { return nil }
-
-var testContent = genTestContent()
-var testContentLengths = []uint{
-	4646 * 1024,
-	655 * 1024,
-	378 * 1024,
-	8108 * 1024,
-	558 * 1024,
-}
-var testMaxFileSize uint
-
-func genTestContent() map[restic.ID][]byte {
-	m := make(map[restic.ID][]byte)
-
-	for _, length := range testContentLengths {
-		buf := Random(int(length), int(length))
-		id := restic.Hash(buf)
-		m[id] = buf
-		testMaxFileSize += length
-	}
-
-	return m
-}
-
-const maxBufSize = 20 * 1024 * 1024
-
-func testRead(t *testing.T, f *file, offset, length int, data []byte) {
-	ctx := MockContext{}
 
 	req := &fuse.ReadRequest{
 		Offset: int64(offset),
 		Size:   length,
 	}
 	resp := &fuse.ReadResponse{
-		Data: make([]byte, length),
+		Data: data,
 	}
 	OK(t, f.Read(ctx, req, resp))
 }
 
-var offsetReadsTests = []struct {
+func firstSnapshotID(t testing.TB, repo restic.Repository) (first restic.ID) {
-	offset, length int
+	done := make(chan struct{})
-}{
+	defer close(done)
-	{0, 5 * 1024 * 1024},
+	for id := range repo.List(restic.SnapshotFile, done) {
-	{4000 * 1024, 1000 * 1024},
+		if first.IsNull() {
+			first = id
+		}
+	}
+	return first
+}
+
+func loadFirstSnapshot(t testing.TB, repo restic.Repository) *restic.Snapshot {
+	id := firstSnapshotID(t, repo)
+	sn, err := restic.LoadSnapshot(repo, id)
+	OK(t, err)
+	return sn
+}
+
+func loadTree(t testing.TB, repo restic.Repository, id restic.ID) *restic.Tree {
+	tree, err := repo.LoadTree(id)
+	OK(t, err)
+	return tree
 }
 
 func TestFuseFile(t *testing.T) {
-	repo := NewMockRepo(testContent)
+	repo, cleanup := repository.TestRepository(t)
-	ctx := MockContext{}
+	defer cleanup()
 
-	memfile := make([]byte, 0, maxBufSize)
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	timestamp, err := time.Parse(time.RFC3339, "2017-01-24T10:42:56+01:00")
+	OK(t, err)
+	restic.TestCreateSnapshot(t, repo, timestamp, 2, 0.1)
+
+	sn := loadFirstSnapshot(t, repo)
+	tree := loadTree(t, repo, *sn.Tree)
+
+	var content restic.IDs
+	for _, node := range tree.Nodes {
+		content = append(content, node.Content...)
+	}
+	t.Logf("tree loaded, content: %v", content)
+
+	var (
+		filesize uint64
+		memfile  []byte
+	)
+	for _, id := range content {
+		size, err := repo.LookupBlobSize(id, restic.DataBlob)
+		OK(t, err)
+		filesize += uint64(size)
+
+		buf := restic.NewBlobBuffer(int(size))
+		n, err := repo.LoadBlob(restic.DataBlob, id, buf)
+		OK(t, err)
+
+		if uint(n) != size {
+			t.Fatalf("not enough bytes read for id %v: want %v, got %v", id.Str(), size, n)
+		}
+
+		if uint(len(buf)) != size {
+			t.Fatalf("buffer has wrong length for id %v: want %v, got %v", id.Str(), size, len(buf))
+		}
 
-	var ids restic.IDs
-	for id, buf := range repo.blobs {
-		ids = append(ids, id)
 		memfile = append(memfile, buf...)
 	}
 
+	t.Logf("filesize is %v, memfile has size %v", filesize, len(memfile))
+
 	node := &restic.Node{
 		Name:    "foo",
 		Inode:   23,
 		Mode:    0742,
-		Size:    42,
+		Size:    filesize,
-		Content: ids,
+		Content: content,
 	}
 	f, err := newFile(repo, node, false)
 	OK(t, err)
@@ -131,28 +121,19 @@ func TestFuseFile(t *testing.T) {
 	Equals(t, node.Size, attr.Size)
 	Equals(t, (node.Size/uint64(attr.BlockSize))+1, attr.Blocks)
 
-	for i, test := range offsetReadsTests {
-		b := memfile[test.offset : test.offset+test.length]
-		buf := make([]byte, test.length)
-		testRead(t, f, test.offset, test.length, buf)
-		if !bytes.Equal(b, buf) {
-			t.Errorf("test %d failed, wrong data returned", i)
-		}
-	}
-
 	for i := 0; i < 200; i++ {
-		length := rand.Intn(int(testMaxFileSize) / 2)
+		offset := rand.Intn(int(filesize))
-		offset := rand.Intn(int(testMaxFileSize))
+		length := rand.Intn(int(filesize)-offset) + 100
-		if length+offset > int(testMaxFileSize) {
-			diff := length + offset - int(testMaxFileSize)
-			length -= diff
-		}
 
 		b := memfile[offset : offset+length]
+
 		buf := make([]byte, length)
+
 		testRead(t, f, offset, length, buf)
 		if !bytes.Equal(b, buf) {
-			t.Errorf("test %d failed (offset %d, length %d), wrong data returned", i, offset, length)
+			t.Errorf("test %d failed, wrong data returned (offset %v, length %v)", i, offset, length)
 		}
 	}
+
+	OK(t, f.Release(ctx, nil))
 }

src/restic/readerat.go

@@ -2,6 +2,7 @@ package restic
 
 import (
 	"io"
+	"restic/debug"
 )
 
 type backendReaderAt struct {
@@ -20,6 +21,7 @@ func ReaderAt(be Backend, h Handle) io.ReaderAt {
 
 // ReadAt reads from the backend handle h at the given position.
 func ReadAt(be Backend, h Handle, offset int64, p []byte) (n int, err error) {
+	debug.Log("ReadAt(%v) at %v, len %v", h, offset, len(p))
 	rd, err := be.Load(h, len(p), offset)
 	if err != nil {
 		return 0, err
@@ -31,5 +33,7 @@ func ReadAt(be Backend, h Handle, offset int64, p []byte) (n int, err error) {
 		err = e
 	}
 
+	debug.Log("ReadAt(%v) ReadFull returned %v bytes", h, n)
+
 	return n, err
 }

src/restic/repository/repository.go

@@ -77,7 +77,7 @@ func (r *Repository) LoadAndDecrypt(t restic.FileType, id restic.ID) ([]byte, er
 // pack from the backend, the result is stored in plaintextBuf, which must be
 // large enough to hold the complete blob.
 func (r *Repository) loadBlob(id restic.ID, t restic.BlobType, plaintextBuf []byte) (int, error) {
-	debug.Log("load %v with id %v (buf %p, len %d)", t, id.Str(), plaintextBuf, len(plaintextBuf))
+	debug.Log("load %v with id %v (buf len %v, cap %d)", t, id.Str(), len(plaintextBuf), cap(plaintextBuf))
 
 	// lookup packs
 	blobs, err := r.idx.Lookup(id, t)
@@ -96,7 +96,12 @@ func (r *Repository) loadBlob(id restic.ID, t restic.BlobType, plaintextBuf []by
 
 		// load blob from pack
 		h := restic.Handle{Type: restic.DataFile, Name: blob.PackID.String()}
-		plaintextBuf = plaintextBuf[:cap(plaintextBuf)]
+
+		if uint(cap(plaintextBuf)) < blob.Length {
+			return 0, errors.Errorf("buffer is too small: %v < %v", cap(plaintextBuf), blob.Length)
+		}
+
+		plaintextBuf = plaintextBuf[:blob.Length]
 
 		n, err := restic.ReadAt(r.be, h, int64(blob.Offset), plaintextBuf)
 		if err != nil {
@@ -520,15 +525,14 @@ func (r *Repository) Close() error {
 // be large enough to hold the encrypted blob, since it is used as scratch
 // space.
 func (r *Repository) LoadBlob(t restic.BlobType, id restic.ID, buf []byte) (int, error) {
-	debug.Log("load blob %v into buf %p", id.Str(), buf)
+	debug.Log("load blob %v into buf (len %v, cap %v)", id.Str(), len(buf), cap(buf))
 	size, err := r.idx.LookupSize(id, t)
 	if err != nil {
 		return 0, err
 	}
 
-	buf = buf[:cap(buf)]
+	if cap(buf) < restic.CiphertextLength(int(size)) {
-	if len(buf) < restic.CiphertextLength(int(size)) {
+		return 0, errors.Errorf("buffer is too small for data blob (%d < %d)", cap(buf), restic.CiphertextLength(int(size)))
-		return 0, errors.Errorf("buffer is too small for data blob (%d < %d)", len(buf), restic.CiphertextLength(int(size)))
 	}
 
 	n, err := r.loadBlob(id, t, buf)

src/restic/repository/repository_test.go

@@ -147,6 +147,51 @@ func BenchmarkLoadTree(t *testing.B) {
 	}
 }
 
+func TestLoadBlob(t *testing.T) {
+	repo, cleanup := repository.TestRepository(t)
+	defer cleanup()
+
+	length := 1000000
+	buf := restic.NewBlobBuffer(length)
+	_, err := io.ReadFull(rnd, buf)
+	OK(t, err)
+
+	id, err := repo.SaveBlob(restic.DataBlob, buf, restic.ID{})
+	OK(t, err)
+	OK(t, repo.Flush())
+
+	// first, test with buffers that are too small
+	for _, testlength := range []int{length - 20, length, restic.CiphertextLength(length) - 1} {
+		buf = make([]byte, 0, testlength)
+		n, err := repo.LoadBlob(restic.DataBlob, id, buf)
+		if err == nil {
+			t.Errorf("LoadBlob() did not return an error for a buffer that is too small to hold the blob")
+			continue
+		}
+
+		if n != 0 {
+			t.Errorf("LoadBlob() returned an error and n > 0")
+			continue
+		}
+	}
+
+	// then use buffers that are large enough
+	base := restic.CiphertextLength(length)
+	for _, testlength := range []int{base, base + 7, base + 15, base + 1000} {
+		buf = make([]byte, 0, testlength)
+		n, err := repo.LoadBlob(restic.DataBlob, id, buf)
+		if err != nil {
+			t.Errorf("LoadBlob() returned an error for buffer size %v: %v", testlength, err)
+			continue
+		}
+
+		if n != length {
+			t.Errorf("LoadBlob() returned the wrong number of bytes: want %v, got %v", length, n)
+			continue
+		}
+	}
+}
+
 func BenchmarkLoadBlob(b *testing.B) {
 	repo, cleanup := repository.TestRepository(b)
 	defer cleanup()