Merge pull request #5048 from MichaelEischer/fix-macos-fuse

Fix unusable `mount` on macOS Sonoma

changelog/unreleased/issue-4971

@@ -0,0 +1,9 @@
+Bugfix: Fix unusable `mount` on macOS Sonoma
+
+On macOS Sonoma when using fuse-t, it was not possible to access files in
+a mounted repository.
+
+This issue has been resolved.
+
+https://github.com/restic/restic/issues/4971
+https://github.com/restic/restic/pull/5048

internal/fuse/dir.go

@@ -20,29 +20,36 @@ import (
 
 // Statically ensure that *dir implement those interface
 var _ = fs.HandleReadDirAller(&dir{})
+var _ = fs.NodeForgetter(&dir{})
+var _ = fs.NodeGetxattrer(&dir{})
+var _ = fs.NodeListxattrer(&dir{})
 var _ = fs.NodeStringLookuper(&dir{})
 
 type dir struct {
 	root        *Root
+	forget      forgetFn
 	items       map[string]*restic.Node
 	inode       uint64
 	parentInode uint64
 	node        *restic.Node
 	m           sync.Mutex
+	cache       treeCache
 }
 
 func cleanupNodeName(name string) string {
 	return filepath.Base(name)
 }
 
-func newDir(root *Root, inode, parentInode uint64, node *restic.Node) (*dir, error) {
+func newDir(root *Root, forget forgetFn, inode, parentInode uint64, node *restic.Node) (*dir, error) {
 	debug.Log("new dir for %v (%v)", node.Name, node.Subtree)
 
 	return &dir{
 		root:        root,
+		forget:      forget,
 		node:        node,
 		inode:       inode,
 		parentInode: parentInode,
+		cache:       *newTreeCache(),
 	}, nil
 }
 
@@ -75,10 +82,11 @@ func replaceSpecialNodes(ctx context.Context, repo restic.BlobLoader, node *rest
 	return tree.Nodes, nil
 }
 
-func newDirFromSnapshot(root *Root, inode uint64, snapshot *restic.Snapshot) (*dir, error) {
+func newDirFromSnapshot(root *Root, forget forgetFn, inode uint64, snapshot *restic.Snapshot) (*dir, error) {
 	debug.Log("new dir for snapshot %v (%v)", snapshot.ID(), snapshot.Tree)
 	return &dir{
 		root:   root,
+		forget: forget,
 		node: &restic.Node{
 			AccessTime: snapshot.Time,
 			ModTime:    snapshot.Time,
@@ -87,6 +95,7 @@ func newDirFromSnapshot(root *Root, inode uint64, snapshot *restic.Snapshot) (*d
 			Subtree:    snapshot.Tree,
 		},
 		inode: inode,
+		cache: *newTreeCache(),
 	}, nil
 }
 
@@ -208,6 +217,7 @@ func (d *dir) Lookup(ctx context.Context, name string) (fs.Node, error) {
 		return nil, err
 	}
 
+	return d.cache.lookupOrCreate(name, func(forget forgetFn) (fs.Node, error) {
 		node, ok := d.items[name]
 		if !ok {
 			debug.Log("  Lookup(%v) -> not found", name)
@@ -216,17 +226,18 @@ func (d *dir) Lookup(ctx context.Context, name string) (fs.Node, error) {
 		inode := inodeFromNode(d.inode, node)
 		switch node.Type {
 		case restic.NodeTypeDir:
-		return newDir(d.root, inode, d.inode, node)
+			return newDir(d.root, forget, inode, d.inode, node)
 		case restic.NodeTypeFile:
-		return newFile(d.root, inode, node)
+			return newFile(d.root, forget, inode, node)
 		case restic.NodeTypeSymlink:
-		return newLink(d.root, inode, node)
+			return newLink(d.root, forget, inode, node)
 		case restic.NodeTypeDev, restic.NodeTypeCharDev, restic.NodeTypeFifo, restic.NodeTypeSocket:
-		return newOther(d.root, inode, node)
+			return newOther(d.root, forget, inode, node)
 		default:
 			debug.Log("  node %v has unknown type %v", name, node.Type)
 			return nil, syscall.ENOENT
 		}
+	})
 }
 
 func (d *dir) Listxattr(_ context.Context, req *fuse.ListxattrRequest, resp *fuse.ListxattrResponse) error {
@@ -237,3 +248,7 @@ func (d *dir) Listxattr(_ context.Context, req *fuse.ListxattrRequest, resp *fus
 func (d *dir) Getxattr(_ context.Context, req *fuse.GetxattrRequest, resp *fuse.GetxattrResponse) error {
 	return nodeGetXattr(d.node, req, resp)
 }
+
+func (d *dir) Forget() {
+	d.forget()
+}

internal/fuse/file.go

@@ -20,12 +20,14 @@ const blockSize = 512
 
 // Statically ensure that *file and *openFile implement the given interfaces
 var _ = fs.HandleReader(&openFile{})
-var _ = fs.NodeListxattrer(&file{})
+var _ = fs.NodeForgetter(&file{})
 var _ = fs.NodeGetxattrer(&file{})
+var _ = fs.NodeListxattrer(&file{})
 var _ = fs.NodeOpener(&file{})
 
 type file struct {
 	root   *Root
+	forget forgetFn
 	node   *restic.Node
 	inode  uint64
 }
@@ -36,10 +38,11 @@ type openFile struct {
 	cumsize []uint64
 }
 
-func newFile(root *Root, inode uint64, node *restic.Node) (fusefile *file, err error) {
+func newFile(root *Root, forget forgetFn, inode uint64, node *restic.Node) (fusefile *file, err error) {
 	debug.Log("create new file for %v with %d blobs", node.Name, len(node.Content))
 	return &file{
 		inode:  inode,
+		forget: forget,
 		root:   root,
 		node:   node,
 	}, nil
@@ -172,3 +175,7 @@ func (f *file) Listxattr(_ context.Context, req *fuse.ListxattrRequest, resp *fu
 func (f *file) Getxattr(_ context.Context, req *fuse.GetxattrRequest, resp *fuse.GetxattrResponse) error {
 	return nodeGetXattr(f.node, req, resp)
 }
+
+func (f *file) Forget() {
+	f.forget()
+}

internal/fuse/fuse_test.go

@@ -119,7 +119,7 @@ func TestFuseFile(t *testing.T) {
 	root := &Root{repo: repo, blobCache: bloblru.New(blobCacheSize)}
 
 	inode := inodeFromNode(1, node)
-	f, err := newFile(root, inode, node)
+	f, err := newFile(root, func() {}, inode, node)
 	rtest.OK(t, err)
 	of, err := f.Open(context.TODO(), nil, nil)
 	rtest.OK(t, err)
@@ -162,7 +162,7 @@ func TestFuseDir(t *testing.T) {
 	}
 	parentInode := inodeFromName(0, "parent")
 	inode := inodeFromName(1, "foo")
-	d, err := newDir(root, inode, parentInode, node)
+	d, err := newDir(root, func() {}, inode, parentInode, node)
 	rtest.OK(t, err)
 
 	// don't open the directory as that would require setting up a proper tree blob
@@ -217,6 +217,34 @@ func testTopUIDGID(t *testing.T, cfg Config, repo restic.Repository, uid, gid ui
 	rtest.Equals(t, uint32(0), attr.Gid)
 }
 
+// The Lookup method must return the same Node object unless it was forgotten in the meantime
+func testStableLookup(t *testing.T, node fs.Node, path string) fs.Node {
+	t.Helper()
+	result, err := node.(fs.NodeStringLookuper).Lookup(context.TODO(), path)
+	rtest.OK(t, err)
+	result2, err := node.(fs.NodeStringLookuper).Lookup(context.TODO(), path)
+	rtest.OK(t, err)
+	rtest.Assert(t, result == result2, "%v are not the same object", path)
+
+	result2.(fs.NodeForgetter).Forget()
+	result2, err = node.(fs.NodeStringLookuper).Lookup(context.TODO(), path)
+	rtest.OK(t, err)
+	rtest.Assert(t, result != result2, "object for %v should change after forget", path)
+	return result
+}
+
+func TestStableNodeObjects(t *testing.T) {
+	repo := repository.TestRepository(t)
+	restic.TestCreateSnapshot(t, repo, time.Unix(1460289341, 207401672), 2)
+	root := NewRoot(repo, Config{})
+
+	idsdir := testStableLookup(t, root, "ids")
+	snapID := loadFirstSnapshot(t, repo).ID().Str()
+	snapshotdir := testStableLookup(t, idsdir, snapID)
+	dir := testStableLookup(t, snapshotdir, "dir-0")
+	testStableLookup(t, dir, "file-2")
+}
+
 // Test reporting of fuse.Attr.Blocks in multiples of 512.
 func TestBlocks(t *testing.T) {
 	root := &Root{}
@@ -276,7 +304,7 @@ func TestLink(t *testing.T) {
 		{Name: "foo", Value: []byte("bar")},
 	}}
 
-	lnk, err := newLink(&Root{}, 42, node)
+	lnk, err := newLink(&Root{}, func() {}, 42, node)
 	rtest.OK(t, err)
 	target, err := lnk.Readlink(context.TODO(), nil)
 	rtest.OK(t, err)

internal/fuse/link.go

@@ -12,16 +12,20 @@ import (
 )
 
 // Statically ensure that *link implements the given interface
+var _ = fs.NodeForgetter(&link{})
+var _ = fs.NodeGetxattrer(&link{})
+var _ = fs.NodeListxattrer(&link{})
 var _ = fs.NodeReadlinker(&link{})
 
 type link struct {
 	root   *Root
+	forget forgetFn
 	node   *restic.Node
 	inode  uint64
 }
 
-func newLink(root *Root, inode uint64, node *restic.Node) (*link, error) {
+func newLink(root *Root, forget forgetFn, inode uint64, node *restic.Node) (*link, error) {
-	return &link{root: root, inode: inode, node: node}, nil
+	return &link{root: root, forget: forget, inode: inode, node: node}, nil
 }
 
 func (l *link) Readlink(_ context.Context, _ *fuse.ReadlinkRequest) (string, error) {
@@ -55,3 +59,7 @@ func (l *link) Listxattr(_ context.Context, req *fuse.ListxattrRequest, resp *fu
 func (l *link) Getxattr(_ context.Context, req *fuse.GetxattrRequest, resp *fuse.GetxattrResponse) error {
 	return nodeGetXattr(l.node, req, resp)
 }
+
+func (l *link) Forget() {
+	l.forget()
+}

internal/fuse/other.go

@@ -7,17 +7,23 @@ import (
 	"context"
 
 	"github.com/anacrolix/fuse"
+	"github.com/anacrolix/fuse/fs"
 	"github.com/restic/restic/internal/restic"
 )
 
+// Statically ensure that *other implements the given interface
+var _ = fs.NodeForgetter(&other{})
+var _ = fs.NodeReadlinker(&other{})
+
 type other struct {
 	root   *Root
+	forget forgetFn
 	node   *restic.Node
 	inode  uint64
 }
 
-func newOther(root *Root, inode uint64, node *restic.Node) (*other, error) {
+func newOther(root *Root, forget forgetFn, inode uint64, node *restic.Node) (*other, error) {
-	return &other{root: root, inode: inode, node: node}, nil
+	return &other{root: root, forget: forget, inode: inode, node: node}, nil
 }
 
 func (l *other) Readlink(_ context.Context, _ *fuse.ReadlinkRequest) (string, error) {
@@ -40,3 +46,7 @@ func (l *other) Attr(_ context.Context, a *fuse.Attr) error {
 
 	return nil
 }
+
+func (l *other) Forget() {
+	l.forget()
+}

internal/fuse/root.go

@@ -66,7 +66,7 @@ func NewRoot(repo restic.Repository, cfg Config) *Root {
 		}
 	}
 
-	root.SnapshotsDir = NewSnapshotsDir(root, rootInode, rootInode, NewSnapshotsDirStructure(root, cfg.PathTemplates, cfg.TimeTemplate), "")
+	root.SnapshotsDir = NewSnapshotsDir(root, func() {}, rootInode, rootInode, NewSnapshotsDirStructure(root, cfg.PathTemplates, cfg.TimeTemplate), "")
 
 	return root
 }

internal/fuse/snapshots_dir.go

@@ -19,25 +19,30 @@ import (
 // It uses the saved prefix to select the corresponding MetaDirData.
 type SnapshotsDir struct {
 	root        *Root
+	forget      forgetFn
 	inode       uint64
 	parentInode uint64
 	dirStruct   *SnapshotsDirStructure
 	prefix      string
+	cache       treeCache
 }
 
 // ensure that *SnapshotsDir implements these interfaces
 var _ = fs.HandleReadDirAller(&SnapshotsDir{})
+var _ = fs.NodeForgetter(&SnapshotsDir{})
 var _ = fs.NodeStringLookuper(&SnapshotsDir{})
 
 // NewSnapshotsDir returns a new directory structure containing snapshots and "latest" links
-func NewSnapshotsDir(root *Root, inode, parentInode uint64, dirStruct *SnapshotsDirStructure, prefix string) *SnapshotsDir {
+func NewSnapshotsDir(root *Root, forget forgetFn, inode, parentInode uint64, dirStruct *SnapshotsDirStructure, prefix string) *SnapshotsDir {
 	debug.Log("create snapshots dir, inode %d", inode)
 	return &SnapshotsDir{
 		root:        root,
+		forget:      forget,
 		inode:       inode,
 		parentInode: parentInode,
 		dirStruct:   dirStruct,
 		prefix:      prefix,
+		cache:       *newTreeCache(),
 	}
 }
 
@@ -107,33 +112,41 @@ func (d *SnapshotsDir) Lookup(ctx context.Context, name string) (fs.Node, error)
 		return nil, syscall.ENOENT
 	}
 
+	return d.cache.lookupOrCreate(name, func(forget forgetFn) (fs.Node, error) {
 		entry := meta.names[name]
-	if entry != nil {
+		if entry == nil {
-		inode := inodeFromName(d.inode, name)
+			return nil, syscall.ENOENT
-		if entry.linkTarget != "" {
-			return newSnapshotLink(d.root, inode, entry.linkTarget, entry.snapshot)
-		} else if entry.snapshot != nil {
-			return newDirFromSnapshot(d.root, inode, entry.snapshot)
-		}
-		return NewSnapshotsDir(d.root, inode, d.inode, d.dirStruct, d.prefix+"/"+name), nil
 		}
 
-	return nil, syscall.ENOENT
+		inode := inodeFromName(d.inode, name)
+		if entry.linkTarget != "" {
+			return newSnapshotLink(d.root, forget, inode, entry.linkTarget, entry.snapshot)
+		} else if entry.snapshot != nil {
+			return newDirFromSnapshot(d.root, forget, inode, entry.snapshot)
+		}
+		return NewSnapshotsDir(d.root, forget, inode, d.inode, d.dirStruct, d.prefix+"/"+name), nil
+	})
+}
+
+func (d *SnapshotsDir) Forget() {
+	d.forget()
 }
 
 // SnapshotLink
 type snapshotLink struct {
 	root     *Root
+	forget   forgetFn
 	inode    uint64
 	target   string
 	snapshot *restic.Snapshot
 }
 
+var _ = fs.NodeForgetter(&snapshotLink{})
 var _ = fs.NodeReadlinker(&snapshotLink{})
 
 // newSnapshotLink
-func newSnapshotLink(root *Root, inode uint64, target string, snapshot *restic.Snapshot) (*snapshotLink, error) {
+func newSnapshotLink(root *Root, forget forgetFn, inode uint64, target string, snapshot *restic.Snapshot) (*snapshotLink, error) {
-	return &snapshotLink{root: root, inode: inode, target: target, snapshot: snapshot}, nil
+	return &snapshotLink{root: root, forget: forget, inode: inode, target: target, snapshot: snapshot}, nil
 }
 
 // Readlink
@@ -157,3 +170,7 @@ func (l *snapshotLink) Attr(_ context.Context, a *fuse.Attr) error {
 
 	return nil
 }
+
+func (l *snapshotLink) Forget() {
+	l.forget()
+}

internal/fuse/tree_cache.go

@@ -0,0 +1,45 @@
+//go:build darwin || freebsd || linux
+// +build darwin freebsd linux
+
+package fuse
+
+import (
+	"sync"
+
+	"github.com/anacrolix/fuse/fs"
+)
+
+type treeCache struct {
+	nodes map[string]fs.Node
+	m     sync.Mutex
+}
+
+type forgetFn func()
+
+func newTreeCache() *treeCache {
+	return &treeCache{
+		nodes: map[string]fs.Node{},
+	}
+}
+
+func (t *treeCache) lookupOrCreate(name string, create func(forget forgetFn) (fs.Node, error)) (fs.Node, error) {
+	t.m.Lock()
+	defer t.m.Unlock()
+
+	if node, ok := t.nodes[name]; ok {
+		return node, nil
+	}
+
+	node, err := create(func() {
+		t.m.Lock()
+		defer t.m.Unlock()
+
+		delete(t.nodes, name)
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	t.nodes[name] = node
+	return node, nil
+}