From 067be2c5519a3553b4c4bc701f0588e9211b7511 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Mon, 5 Jun 2017 23:15:33 +0200 Subject: [PATCH] fuse: Add cache for blob sizes Closes: #820 --- src/restic/fuse/dir.go | 12 ++++++++---- src/restic/fuse/file.go | 11 +++++++---- src/restic/fuse/file_test.go | 2 +- src/restic/fuse/snapshot.go | 17 ++++++++++++++++- 4 files changed, 32 insertions(+), 10 deletions(-) diff --git a/src/restic/fuse/dir.go b/src/restic/fuse/dir.go index 49210bb90..439ea16e7 100644 --- a/src/restic/fuse/dir.go +++ b/src/restic/fuse/dir.go @@ -24,9 +24,11 @@ type dir struct { inode uint64 node *restic.Node ownerIsRoot bool + + sizes map[restic.ID]uint } -func newDir(ctx context.Context, repo restic.Repository, node *restic.Node, ownerIsRoot bool) (*dir, error) { +func newDir(ctx context.Context, repo restic.Repository, node *restic.Node, ownerIsRoot bool, sizes map[restic.ID]uint) (*dir, error) { debug.Log("new dir for %v (%v)", node.Name, node.Subtree.Str()) tree, err := repo.LoadTree(ctx, *node.Subtree) if err != nil { @@ -44,6 +46,7 @@ func newDir(ctx context.Context, repo restic.Repository, node *restic.Node, owne items: items, inode: node.Inode, ownerIsRoot: ownerIsRoot, + sizes: sizes, }, nil } @@ -66,7 +69,7 @@ func replaceSpecialNodes(ctx context.Context, repo restic.Repository, node *rest return tree.Nodes, nil } -func newDirFromSnapshot(ctx context.Context, repo restic.Repository, snapshot SnapshotWithId, ownerIsRoot bool) (*dir, error) { +func newDirFromSnapshot(ctx context.Context, repo restic.Repository, snapshot SnapshotWithId, ownerIsRoot bool, sizes map[restic.ID]uint) (*dir, error) { debug.Log("new dir for snapshot %v (%v)", snapshot.ID.Str(), snapshot.Tree.Str()) tree, err := repo.LoadTree(ctx, *snapshot.Tree) if err != nil { @@ -99,6 +102,7 @@ func newDirFromSnapshot(ctx context.Context, repo restic.Repository, snapshot Sn items: items, inode: inodeFromBackendID(snapshot.ID), ownerIsRoot: ownerIsRoot, + sizes: sizes, }, nil } @@ -167,9 +171,9 @@ func (d *dir) Lookup(ctx context.Context, name string) (fs.Node, error) { } switch node.Type { case "dir": - return newDir(ctx, d.repo, node, d.ownerIsRoot) + return newDir(ctx, d.repo, node, d.ownerIsRoot, d.sizes) case "file": - return newFile(d.repo, node, d.ownerIsRoot) + return newFile(d.repo, node, d.ownerIsRoot, d.sizes) case "symlink": return newLink(d.repo, node, d.ownerIsRoot) default: diff --git a/src/restic/fuse/file.go b/src/restic/fuse/file.go index ae3ba5a7d..679c5e010 100644 --- a/src/restic/fuse/file.go +++ b/src/restic/fuse/file.go @@ -41,14 +41,17 @@ type file struct { const defaultBlobSize = 128 * 1024 -func newFile(repo BlobLoader, node *restic.Node, ownerIsRoot bool) (*file, error) { +func newFile(repo BlobLoader, node *restic.Node, ownerIsRoot bool, sizecache map[restic.ID]uint) (fusefile *file, err error) { debug.Log("create new file for %v with %d blobs", node.Name, len(node.Content)) var bytes uint64 sizes := make([]int, len(node.Content)) for i, id := range node.Content { - size, err := repo.LookupBlobSize(id, restic.DataBlob) - if err != nil { - return nil, err + size, ok := sizecache[id] + if !ok { + size, err = repo.LookupBlobSize(id, restic.DataBlob) + if err != nil { + return nil, err + } } sizes[i] = int(size) diff --git a/src/restic/fuse/file_test.go b/src/restic/fuse/file_test.go index dcb959fec..87aaa052d 100644 --- a/src/restic/fuse/file_test.go +++ b/src/restic/fuse/file_test.go @@ -108,7 +108,7 @@ func TestFuseFile(t *testing.T) { Size: filesize, Content: content, } - f, err := newFile(repo, node, false) + f, err := newFile(repo, node, false, make(map[restic.ID]uint)) OK(t, err) attr := fuse.Attr{} diff --git a/src/restic/fuse/snapshot.go b/src/restic/fuse/snapshot.go index 4057301f8..2aa34e341 100644 --- a/src/restic/fuse/snapshot.go +++ b/src/restic/fuse/snapshot.go @@ -14,6 +14,7 @@ import ( "restic" "restic/debug" + "restic/repository" "golang.org/x/net/context" ) @@ -36,12 +37,25 @@ type SnapshotsDir struct { tags []string host string + // sizes caches the sizes of all blobs. + sizes map[restic.ID]uint + // knownSnapshots maps snapshot timestamp to the snapshot sync.RWMutex knownSnapshots map[string]SnapshotWithId processed restic.IDSet } +func sizeCache(midx *repository.MasterIndex) map[restic.ID]uint { + c := make(map[restic.ID]uint, 1000) + for _, idx := range midx.All() { + for pb := range idx.Each(nil) { + c[pb.ID] = pb.Length + } + } + return c +} + // NewSnapshotsDir returns a new dir object for the snapshots. func NewSnapshotsDir(repo restic.Repository, ownerIsRoot bool, paths []string, tags []string, host string) *SnapshotsDir { debug.Log("fuse mount initiated") @@ -53,6 +67,7 @@ func NewSnapshotsDir(repo restic.Repository, ownerIsRoot bool, paths []string, t host: host, knownSnapshots: make(map[string]SnapshotWithId), processed: restic.NewIDSet(), + sizes: sizeCache(repo.Index().(*repository.MasterIndex)), } } @@ -158,5 +173,5 @@ func (sn *SnapshotsDir) Lookup(ctx context.Context, name string) (fs.Node, error } } - return newDirFromSnapshot(ctx, sn.repo, snapshot, sn.ownerIsRoot) + return newDirFromSnapshot(ctx, sn.repo, snapshot, sn.ownerIsRoot, sn.sizes) }