From bd746a0425b9a7fd2e0986bfcc904d09f3222c19 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 26 Jul 2015 16:43:42 +0200 Subject: [PATCH] fuse: refactor and add tests for fuse.file --- cmd/restic/fuse/file.go | 26 ++++-- cmd/restic/fuse/file_test.go | 157 +++++++++++++++++++++++++++++++++++ repository/repository.go | 5 ++ 3 files changed, 179 insertions(+), 9 deletions(-) create mode 100644 cmd/restic/fuse/file_test.go diff --git a/cmd/restic/fuse/file.go b/cmd/restic/fuse/file.go index 895c844aa..97e46543b 100644 --- a/cmd/restic/fuse/file.go +++ b/cmd/restic/fuse/file.go @@ -4,8 +4,8 @@ import ( "sync" "github.com/restic/restic" + "github.com/restic/restic/backend" "github.com/restic/restic/pack" - "github.com/restic/restic/repository" "bazil.org/fuse" "bazil.org/fuse/fs" @@ -16,11 +16,18 @@ import ( var _ = fs.HandleReader(&file{}) var _ = fs.HandleReleaser(&file{}) +// BlobLoader is an abstracted repository with a reduced set of methods used +// for fuse operations. +type BlobLoader interface { + LookupBlobSize(backend.ID) (uint, error) + LoadBlob(pack.BlobType, backend.ID, []byte) ([]byte, error) +} + type file struct { - repo *repository.Repository + repo BlobLoader node *restic.Node - sizes []uint32 + sizes []uint blobs [][]byte } @@ -32,14 +39,15 @@ var blobPool = sync.Pool{ }, } -func newFile(repo *repository.Repository, node *restic.Node) (*file, error) { - sizes := make([]uint32, len(node.Content)) - for i, blobID := range node.Content { - length, err := repo.Index().LookupSize(blobID) +func newFile(repo BlobLoader, node *restic.Node) (*file, error) { + sizes := make([]uint, len(node.Content)) + for i, id := range node.Content { + size, err := repo.LookupBlobSize(id) if err != nil { return nil, err } - sizes[i] = uint32(length) + + sizes[i] = size } return &file{ @@ -65,7 +73,7 @@ func (f *file) getBlobAt(i int) (blob []byte, err error) { buf := blobPool.Get().([]byte) buf = buf[:cap(buf)] - if uint32(len(buf)) < f.sizes[i] { + if uint(len(buf)) < f.sizes[i] { if len(buf) > defaultBlobSize { blobPool.Put(buf) } diff --git a/cmd/restic/fuse/file_test.go b/cmd/restic/fuse/file_test.go new file mode 100644 index 000000000..25ea574c3 --- /dev/null +++ b/cmd/restic/fuse/file_test.go @@ -0,0 +1,157 @@ +package fuse + +import ( + "bytes" + "errors" + "fmt" + "math/rand" + "testing" + "time" + + "bazil.org/fuse" + + "github.com/restic/restic" + "github.com/restic/restic/backend" + "github.com/restic/restic/pack" + . "github.com/restic/restic/test" +) + +type MockRepo struct { + blobs map[backend.ID][]byte +} + +func NewMockRepo(content map[backend.ID][]byte) *MockRepo { + return &MockRepo{blobs: content} +} + +func (m *MockRepo) LookupBlobSize(id backend.ID) (uint, error) { + buf, ok := m.blobs[id] + if !ok { + return 0, errors.New("blob not found") + } + + return uint(len(buf)), nil +} + +func (m *MockRepo) LoadBlob(t pack.BlobType, id backend.ID, buf []byte) ([]byte, error) { + size, err := m.LookupBlobSize(id) + if err != nil { + return nil, err + } + + if uint(cap(buf)) < size { + return nil, errors.New("buffer too small") + } + + buf = buf[:size] + copy(buf, m.blobs[id]) + return buf, nil +} + +type MockContext struct{} + +func (m MockContext) Deadline() (time.Time, bool) { return time.Now(), false } +func (m MockContext) Done() <-chan struct{} { return nil } +func (m MockContext) Err() error { return nil } +func (m MockContext) Value(key interface{}) interface{} { return nil } + +var testContent = genTestContent() +var testContentLengths = []uint{ + 4646 * 1024, + 655 * 1024, + 378 * 1024, + 8108 * 1024, + 558 * 1024, +} +var testMaxFileSize uint + +func genTestContent() map[backend.ID][]byte { + m := make(map[backend.ID][]byte) + + for _, length := range testContentLengths { + buf := Random(int(length), int(length)) + id := backend.Hash(buf) + m[id] = buf + testMaxFileSize += length + } + + return m +} + +const maxBufSize = 20 * 1024 * 1024 + +func testRead(t *testing.T, f *file, offset, length int, data []byte) []byte { + ctx := MockContext{} + + req := &fuse.ReadRequest{ + Offset: int64(offset), + Size: length, + } + resp := &fuse.ReadResponse{ + Data: make([]byte, length), + } + OK(t, f.Read(ctx, req, resp)) + + return resp.Data +} + +var offsetReadsTests = []struct { + offset, length int +}{ + {0, 5 * 1024 * 1024}, + {4000 * 1024, 1000 * 1024}, +} + +func TestFuseFile(t *testing.T) { + repo := NewMockRepo(testContent) + ctx := MockContext{} + + memfile := make([]byte, 0, maxBufSize) + + var ids backend.IDs + for id, buf := range repo.blobs { + ids = append(ids, id) + memfile = append(memfile, buf...) + } + + node := &restic.Node{ + Name: "foo", + Inode: 23, + Mode: 0742, + Size: 42, + Content: ids, + } + f, err := newFile(repo, node) + OK(t, err) + + attr := fuse.Attr{} + OK(t, f.Attr(ctx, &attr)) + + Equals(t, node.Inode, attr.Inode) + Equals(t, node.Mode, attr.Mode) + Equals(t, node.Size, attr.Size) + + for i, test := range offsetReadsTests { + b := memfile[test.offset : test.offset+test.length] + res := testRead(t, f, test.offset, test.length, b) + if !bytes.Equal(b, res) { + t.Errorf("test %d failed, wrong data returned", i) + } + } + + for i := 0; i < 200; i++ { + length := rand.Intn(int(testMaxFileSize) / 2) + offset := rand.Intn(int(testMaxFileSize)) + if length+offset > int(testMaxFileSize) { + diff := length + offset - int(testMaxFileSize) + length -= diff + } + + b := memfile[offset : offset+length] + fmt.Printf("test offset %d, length %d\n", offset, length) + res := testRead(t, f, offset, length, b) + if !bytes.Equal(b, res) { + t.Errorf("test %d failed (offset %d, length %d), wrong data returned", i, offset, length) + } + } +} diff --git a/repository/repository.go b/repository/repository.go index faef38f61..40f9cc7ec 100644 --- a/repository/repository.go +++ b/repository/repository.go @@ -201,6 +201,11 @@ func (r *Repository) LoadJSONPack(t pack.BlobType, id backend.ID, item interface return nil } +// LookupBlobSize returns the size of blob id. +func (r *Repository) LookupBlobSize(id backend.ID) (uint, error) { + return r.Index().LookupSize(id) +} + const minPackSize = 4 * chunker.MiB const maxPackSize = 16 * chunker.MiB const maxPackers = 200