mirror of
https://github.com/octoleo/restic.git
synced 2024-12-23 03:18:55 +00:00
Merge pull request #745 from restic/fix-742
Fix restore/fuse with larger files
This commit is contained in:
Alexander Neumann
commit
17d7af6ccc
@ -7,6 +7,7 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
mrand "math/rand"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
@ -739,6 +740,30 @@ func TestRestoreFilter(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
func TestRestore(t *testing.T) {
|
||||
withTestEnvironment(t, func(env *testEnvironment, gopts GlobalOptions) {
|
||||
testRunInit(t, gopts)
|
||||
|
||||
for i := 0; i < 10; i++ {
|
||||
p := filepath.Join(env.testdata, fmt.Sprintf("foo/bar/testfile%v", i))
|
||||
OK(t, os.MkdirAll(filepath.Dir(p), 0755))
|
||||
OK(t, appendRandomData(p, uint(mrand.Intn(5<<21))))
|
||||
}
|
||||
|
||||
opts := BackupOptions{}
|
||||
|
||||
testRunBackup(t, []string{env.testdata}, opts, gopts)
|
||||
testRunCheck(t, gopts)
|
||||
|
||||
// Restore latest without any filters
|
||||
restoredir := filepath.Join(env.base, "restore")
|
||||
testRunRestoreLatest(t, gopts, restoredir, nil, "")
|
||||
|
||||
Assert(t, directoriesEqualContents(env.testdata, filepath.Join(restoredir, filepath.Base(env.testdata))),
|
||||
"directories are not equal")
|
||||
})
|
||||
}
|
||||
|
||||
func TestRestoreLatest(t *testing.T) {
|
||||
|
||||
withTestEnvironment(t, func(env *testEnvironment, gopts GlobalOptions) {
|
||||
|
||||
@ -9,7 +9,6 @@ import (
|
||||
|
||||
"restic"
|
||||
"restic/archiver"
|
||||
"restic/backend/mem"
|
||||
"restic/checker"
|
||||
"restic/repository"
|
||||
"restic/test"
|
||||
@ -249,19 +248,15 @@ func induceError(data []byte) {
|
||||
}
|
||||
|
||||
func TestCheckerModifiedData(t *testing.T) {
|
||||
be := mem.New()
|
||||
|
||||
repository.TestUseLowSecurityKDFParameters(t)
|
||||
|
||||
repo := repository.New(be)
|
||||
test.OK(t, repo.Init(test.TestPassword))
|
||||
repo, cleanup := repository.TestRepository(t)
|
||||
defer cleanup()
|
||||
|
||||
arch := archiver.New(repo)
|
||||
_, id, err := arch.Snapshot(nil, []string{"."}, nil, nil)
|
||||
test.OK(t, err)
|
||||
t.Logf("archived as %v", id.Str())
|
||||
|
||||
beError := &errorBackend{Backend: be}
|
||||
beError := &errorBackend{Backend: repo.Backend()}
|
||||
checkRepo := repository.New(beError)
|
||||
test.OK(t, checkRepo.SearchKey(test.TestPassword, 5))
|
||||
|
||||
|
||||
@ -4,8 +4,6 @@
|
||||
package fuse
|
||||
|
||||
import (
|
||||
"sync"
|
||||
|
||||
"restic/errors"
|
||||
|
||||
"restic"
|
||||
@ -35,29 +33,23 @@ type file struct {
|
||||
node *restic.Node
|
||||
ownerIsRoot bool
|
||||
|
||||
sizes []uint
|
||||
sizes []int
|
||||
blobs [][]byte
|
||||
}
|
||||
|
||||
const defaultBlobSize = 128 * 1024
|
||||
|
||||
var blobPool = sync.Pool{
|
||||
New: func() interface{} {
|
||||
return make([]byte, defaultBlobSize)
|
||||
},
|
||||
}
|
||||
|
||||
func newFile(repo BlobLoader, node *restic.Node, ownerIsRoot bool) (*file, error) {
|
||||
debug.Log("create new file for %v with %d blobs", node.Name, len(node.Content))
|
||||
var bytes uint64
|
||||
sizes := make([]uint, len(node.Content))
|
||||
sizes := make([]int, len(node.Content))
|
||||
for i, id := range node.Content {
|
||||
size, err := repo.LookupBlobSize(id, restic.DataBlob)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
sizes[i] = size
|
||||
sizes[i] = int(size)
|
||||
bytes += uint64(size)
|
||||
}
|
||||
|
||||
@ -99,16 +91,7 @@ func (f *file) getBlobAt(i int) (blob []byte, err error) {
|
||||
return f.blobs[i], nil
|
||||
}
|
||||
|
||||
buf := blobPool.Get().([]byte)
|
||||
buf = buf[:cap(buf)]
|
||||
|
||||
if uint(len(buf)) < f.sizes[i] {
|
||||
if len(buf) > defaultBlobSize {
|
||||
blobPool.Put(buf)
|
||||
}
|
||||
buf = make([]byte, f.sizes[i])
|
||||
}
|
||||
|
||||
buf := restic.NewBlobBuffer(f.sizes[i])
|
||||
n, err := f.repo.LoadBlob(restic.DataBlob, f.node.Content[i], buf)
|
||||
if err != nil {
|
||||
debug.Log("LoadBlob(%v, %v) failed: %v", f.node.Name, f.node.Content[i], err)
|
||||
@ -169,10 +152,7 @@ func (f *file) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.ReadR
|
||||
|
||||
func (f *file) Release(ctx context.Context, req *fuse.ReleaseRequest) error {
|
||||
for i := range f.blobs {
|
||||
if f.blobs[i] != nil {
|
||||
blobPool.Put(f.blobs[i])
|
||||
f.blobs[i] = nil
|
||||
}
|
||||
f.blobs[i] = nil
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -9,7 +9,9 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"restic/errors"
|
||||
"golang.org/x/net/context"
|
||||
|
||||
"restic/repository"
|
||||
|
||||
"bazil.org/fuse"
|
||||
|
||||
@ -17,108 +19,96 @@ import (
|
||||
. "restic/test"
|
||||
)
|
||||
|
||||
type MockRepo struct {
|
||||
blobs map[restic.ID][]byte
|
||||
}
|
||||
|
||||
func NewMockRepo(content map[restic.ID][]byte) *MockRepo {
|
||||
return &MockRepo{blobs: content}
|
||||
}
|
||||
|
||||
func (m *MockRepo) LookupBlobSize(id restic.ID, t restic.BlobType) (uint, error) {
|
||||
buf, ok := m.blobs[id]
|
||||
if !ok {
|
||||
return 0, errors.New("blob not found")
|
||||
}
|
||||
|
||||
return uint(len(buf)), nil
|
||||
}
|
||||
|
||||
func (m *MockRepo) LoadBlob(t restic.BlobType, id restic.ID, buf []byte) (int, error) {
|
||||
size, err := m.LookupBlobSize(id, t)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
if uint(len(buf)) < size {
|
||||
return 0, errors.New("buffer too small")
|
||||
}
|
||||
|
||||
buf = buf[:size]
|
||||
copy(buf, m.blobs[id])
|
||||
return int(size), nil
|
||||
}
|
||||
|
||||
type MockContext struct{}
|
||||
|
||||
func (m MockContext) Deadline() (time.Time, bool) { return time.Now(), false }
|
||||
func (m MockContext) Done() <-chan struct{} { return nil }
|
||||
func (m MockContext) Err() error { return nil }
|
||||
func (m MockContext) Value(key interface{}) interface{} { return nil }
|
||||
|
||||
var testContent = genTestContent()
|
||||
var testContentLengths = []uint{
|
||||
4646 * 1024,
|
||||
655 * 1024,
|
||||
378 * 1024,
|
||||
8108 * 1024,
|
||||
558 * 1024,
|
||||
}
|
||||
var testMaxFileSize uint
|
||||
|
||||
func genTestContent() map[restic.ID][]byte {
|
||||
m := make(map[restic.ID][]byte)
|
||||
|
||||
for _, length := range testContentLengths {
|
||||
buf := Random(int(length), int(length))
|
||||
id := restic.Hash(buf)
|
||||
m[id] = buf
|
||||
testMaxFileSize += length
|
||||
}
|
||||
|
||||
return m
|
||||
}
|
||||
|
||||
const maxBufSize = 20 * 1024 * 1024
|
||||
|
||||
func testRead(t *testing.T, f *file, offset, length int, data []byte) {
|
||||
ctx := MockContext{}
|
||||
func testRead(t testing.TB, f *file, offset, length int, data []byte) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
req := &fuse.ReadRequest{
|
||||
Offset: int64(offset),
|
||||
Size: length,
|
||||
}
|
||||
resp := &fuse.ReadResponse{
|
||||
Data: make([]byte, length),
|
||||
Data: data,
|
||||
}
|
||||
OK(t, f.Read(ctx, req, resp))
|
||||
}
|
||||
|
||||
var offsetReadsTests = []struct {
|
||||
offset, length int
|
||||
}{
|
||||
{0, 5 * 1024 * 1024},
|
||||
{4000 * 1024, 1000 * 1024},
|
||||
func firstSnapshotID(t testing.TB, repo restic.Repository) (first restic.ID) {
|
||||
done := make(chan struct{})
|
||||
defer close(done)
|
||||
for id := range repo.List(restic.SnapshotFile, done) {
|
||||
if first.IsNull() {
|
||||
first = id
|
||||
}
|
||||
}
|
||||
return first
|
||||
}
|
||||
|
||||
func loadFirstSnapshot(t testing.TB, repo restic.Repository) *restic.Snapshot {
|
||||
id := firstSnapshotID(t, repo)
|
||||
sn, err := restic.LoadSnapshot(repo, id)
|
||||
OK(t, err)
|
||||
return sn
|
||||
}
|
||||
|
||||
func loadTree(t testing.TB, repo restic.Repository, id restic.ID) *restic.Tree {
|
||||
tree, err := repo.LoadTree(id)
|
||||
OK(t, err)
|
||||
return tree
|
||||
}
|
||||
|
||||
func TestFuseFile(t *testing.T) {
|
||||
repo := NewMockRepo(testContent)
|
||||
ctx := MockContext{}
|
||||
repo, cleanup := repository.TestRepository(t)
|
||||
defer cleanup()
|
||||
|
||||
memfile := make([]byte, 0, maxBufSize)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
timestamp, err := time.Parse(time.RFC3339, "2017-01-24T10:42:56+01:00")
|
||||
OK(t, err)
|
||||
restic.TestCreateSnapshot(t, repo, timestamp, 2, 0.1)
|
||||
|
||||
sn := loadFirstSnapshot(t, repo)
|
||||
tree := loadTree(t, repo, *sn.Tree)
|
||||
|
||||
var content restic.IDs
|
||||
for _, node := range tree.Nodes {
|
||||
content = append(content, node.Content...)
|
||||
}
|
||||
t.Logf("tree loaded, content: %v", content)
|
||||
|
||||
var (
|
||||
filesize uint64
|
||||
memfile []byte
|
||||
)
|
||||
for _, id := range content {
|
||||
size, err := repo.LookupBlobSize(id, restic.DataBlob)
|
||||
OK(t, err)
|
||||
filesize += uint64(size)
|
||||
|
||||
buf := restic.NewBlobBuffer(int(size))
|
||||
n, err := repo.LoadBlob(restic.DataBlob, id, buf)
|
||||
OK(t, err)
|
||||
|
||||
if uint(n) != size {
|
||||
t.Fatalf("not enough bytes read for id %v: want %v, got %v", id.Str(), size, n)
|
||||
}
|
||||
|
||||
if uint(len(buf)) != size {
|
||||
t.Fatalf("buffer has wrong length for id %v: want %v, got %v", id.Str(), size, len(buf))
|
||||
}
|
||||
|
||||
var ids restic.IDs
|
||||
for id, buf := range repo.blobs {
|
||||
ids = append(ids, id)
|
||||
memfile = append(memfile, buf...)
|
||||
}
|
||||
|
||||
t.Logf("filesize is %v, memfile has size %v", filesize, len(memfile))
|
||||
|
||||
node := &restic.Node{
|
||||
Name: "foo",
|
||||
Inode: 23,
|
||||
Mode: 0742,
|
||||
Size: 42,
|
||||
Content: ids,
|
||||
Size: filesize,
|
||||
Content: content,
|
||||
}
|
||||
f, err := newFile(repo, node, false)
|
||||
OK(t, err)
|
||||
@ -131,28 +121,19 @@ func TestFuseFile(t *testing.T) {
|
||||
Equals(t, node.Size, attr.Size)
|
||||
Equals(t, (node.Size/uint64(attr.BlockSize))+1, attr.Blocks)
|
||||
|
||||
for i, test := range offsetReadsTests {
|
||||
b := memfile[test.offset : test.offset+test.length]
|
||||
buf := make([]byte, test.length)
|
||||
testRead(t, f, test.offset, test.length, buf)
|
||||
if !bytes.Equal(b, buf) {
|
||||
t.Errorf("test %d failed, wrong data returned", i)
|
||||
}
|
||||
}
|
||||
|
||||
for i := 0; i < 200; i++ {
|
||||
length := rand.Intn(int(testMaxFileSize) / 2)
|
||||
offset := rand.Intn(int(testMaxFileSize))
|
||||
if length+offset > int(testMaxFileSize) {
|
||||
diff := length + offset - int(testMaxFileSize)
|
||||
length -= diff
|
||||
}
|
||||
offset := rand.Intn(int(filesize))
|
||||
length := rand.Intn(int(filesize)-offset) + 100
|
||||
|
||||
b := memfile[offset : offset+length]
|
||||
|
||||
buf := make([]byte, length)
|
||||
|
||||
testRead(t, f, offset, length, buf)
|
||||
if !bytes.Equal(b, buf) {
|
||||
t.Errorf("test %d failed (offset %d, length %d), wrong data returned", i, offset, length)
|
||||
t.Errorf("test %d failed, wrong data returned (offset %v, length %v)", i, offset, length)
|
||||
}
|
||||
}
|
||||
|
||||
OK(t, f.Release(ctx, nil))
|
||||
}
|
||||
|
||||
@ -2,6 +2,7 @@ package restic
|
||||
|
||||
import (
|
||||
"io"
|
||||
"restic/debug"
|
||||
)
|
||||
|
||||
type backendReaderAt struct {
|
||||
@ -20,6 +21,7 @@ func ReaderAt(be Backend, h Handle) io.ReaderAt {
|
||||
|
||||
// ReadAt reads from the backend handle h at the given position.
|
||||
func ReadAt(be Backend, h Handle, offset int64, p []byte) (n int, err error) {
|
||||
debug.Log("ReadAt(%v) at %v, len %v", h, offset, len(p))
|
||||
rd, err := be.Load(h, len(p), offset)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
@ -31,5 +33,7 @@ func ReadAt(be Backend, h Handle, offset int64, p []byte) (n int, err error) {
|
||||
err = e
|
||||
}
|
||||
|
||||
debug.Log("ReadAt(%v) ReadFull returned %v bytes", h, n)
|
||||
|
||||
return n, err
|
||||
}
|
||||
|
||||
@ -77,7 +77,7 @@ func (r *Repository) LoadAndDecrypt(t restic.FileType, id restic.ID) ([]byte, er
|
||||
// pack from the backend, the result is stored in plaintextBuf, which must be
|
||||
// large enough to hold the complete blob.
|
||||
func (r *Repository) loadBlob(id restic.ID, t restic.BlobType, plaintextBuf []byte) (int, error) {
|
||||
debug.Log("load %v with id %v (buf %p, len %d)", t, id.Str(), plaintextBuf, len(plaintextBuf))
|
||||
debug.Log("load %v with id %v (buf len %v, cap %d)", t, id.Str(), len(plaintextBuf), cap(plaintextBuf))
|
||||
|
||||
// lookup packs
|
||||
blobs, err := r.idx.Lookup(id, t)
|
||||
@ -96,7 +96,12 @@ func (r *Repository) loadBlob(id restic.ID, t restic.BlobType, plaintextBuf []by
|
||||
|
||||
// load blob from pack
|
||||
h := restic.Handle{Type: restic.DataFile, Name: blob.PackID.String()}
|
||||
plaintextBuf = plaintextBuf[:cap(plaintextBuf)]
|
||||
|
||||
if uint(cap(plaintextBuf)) < blob.Length {
|
||||
return 0, errors.Errorf("buffer is too small: %v < %v", cap(plaintextBuf), blob.Length)
|
||||
}
|
||||
|
||||
plaintextBuf = plaintextBuf[:blob.Length]
|
||||
|
||||
n, err := restic.ReadAt(r.be, h, int64(blob.Offset), plaintextBuf)
|
||||
if err != nil {
|
||||
@ -520,15 +525,14 @@ func (r *Repository) Close() error {
|
||||
// be large enough to hold the encrypted blob, since it is used as scratch
|
||||
// space.
|
||||
func (r *Repository) LoadBlob(t restic.BlobType, id restic.ID, buf []byte) (int, error) {
|
||||
debug.Log("load blob %v into buf %p", id.Str(), buf)
|
||||
debug.Log("load blob %v into buf (len %v, cap %v)", id.Str(), len(buf), cap(buf))
|
||||
size, err := r.idx.LookupSize(id, t)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
buf = buf[:cap(buf)]
|
||||
if len(buf) < restic.CiphertextLength(int(size)) {
|
||||
return 0, errors.Errorf("buffer is too small for data blob (%d < %d)", len(buf), restic.CiphertextLength(int(size)))
|
||||
if cap(buf) < restic.CiphertextLength(int(size)) {
|
||||
return 0, errors.Errorf("buffer is too small for data blob (%d < %d)", cap(buf), restic.CiphertextLength(int(size)))
|
||||
}
|
||||
|
||||
n, err := r.loadBlob(id, t, buf)
|
||||
|
||||
@ -147,6 +147,51 @@ func BenchmarkLoadTree(t *testing.B) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadBlob(t *testing.T) {
|
||||
repo, cleanup := repository.TestRepository(t)
|
||||
defer cleanup()
|
||||
|
||||
length := 1000000
|
||||
buf := restic.NewBlobBuffer(length)
|
||||
_, err := io.ReadFull(rnd, buf)
|
||||
OK(t, err)
|
||||
|
||||
id, err := repo.SaveBlob(restic.DataBlob, buf, restic.ID{})
|
||||
OK(t, err)
|
||||
OK(t, repo.Flush())
|
||||
|
||||
// first, test with buffers that are too small
|
||||
for _, testlength := range []int{length - 20, length, restic.CiphertextLength(length) - 1} {
|
||||
buf = make([]byte, 0, testlength)
|
||||
n, err := repo.LoadBlob(restic.DataBlob, id, buf)
|
||||
if err == nil {
|
||||
t.Errorf("LoadBlob() did not return an error for a buffer that is too small to hold the blob")
|
||||
continue
|
||||
}
|
||||
|
||||
if n != 0 {
|
||||
t.Errorf("LoadBlob() returned an error and n > 0")
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// then use buffers that are large enough
|
||||
base := restic.CiphertextLength(length)
|
||||
for _, testlength := range []int{base, base + 7, base + 15, base + 1000} {
|
||||
buf = make([]byte, 0, testlength)
|
||||
n, err := repo.LoadBlob(restic.DataBlob, id, buf)
|
||||
if err != nil {
|
||||
t.Errorf("LoadBlob() returned an error for buffer size %v: %v", testlength, err)
|
||||
continue
|
||||
}
|
||||
|
||||
if n != length {
|
||||
t.Errorf("LoadBlob() returned the wrong number of bytes: want %v, got %v", length, n)
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkLoadBlob(b *testing.B) {
|
||||
repo, cleanup := repository.TestRepository(b)
|
||||
defer cleanup()
|
||||
|
||||
Loading…
Reference in New Issue
Block a user