Merge pull request #1337 from mungomat/fuse_updateRepo

fuse: mount and backup in parallel (#1330)

cmd/restic/cmd_mount.go

@@ -67,6 +67,12 @@ func mount(opts MountOptions, gopts GlobalOptions, mountpoint string) error {
 		return err
 	}
 
+	lock, err := lockRepo(repo)
+	defer unlockRepo(lock)
+	if err != nil {
+		return err
+	}
+
 	err = repo.LoadIndex(context.TODO())
 	if err != nil {
 		return err

internal/fuse/root.go

@@ -41,22 +41,18 @@ const rootInode = 1
 func NewRoot(ctx context.Context, repo restic.Repository, cfg Config) (*Root, error) {
 	debug.Log("NewRoot(), config %v", cfg)
 
-	snapshots := restic.FindFilteredSnapshots(ctx, repo, cfg.Host, cfg.Tags, cfg.Paths)
-	debug.Log("found %d matching snapshots", len(snapshots))
-
 	root := &Root{
 		repo:          repo,
 		inode:         rootInode,
 		cfg:           cfg,
-		snapshots:     snapshots,
 		blobSizeCache: NewBlobSizeCache(ctx, repo.Index()),
 	}
 
 	entries := map[string]fs.Node{
-		"snapshots": NewSnapshotsDir(root, fs.GenerateDynamicInode(root.inode, "snapshots"), snapshots),
+		"snapshots": NewSnapshotsDir(root, fs.GenerateDynamicInode(root.inode, "snapshots"), "", ""),
-		"tags":      NewTagsDir(root, fs.GenerateDynamicInode(root.inode, "tags"), snapshots),
+		"tags":      NewTagsDir(root, fs.GenerateDynamicInode(root.inode, "tags")),
-		"hosts":     NewHostsDir(root, fs.GenerateDynamicInode(root.inode, "hosts"), snapshots),
+		"hosts":     NewHostsDir(root, fs.GenerateDynamicInode(root.inode, "hosts")),
-		"ids":       NewSnapshotsIDSDir(root, fs.GenerateDynamicInode(root.inode, "ids"), snapshots),
+		"ids":       NewSnapshotsIDSDir(root, fs.GenerateDynamicInode(root.inode, "ids")),
 	}
 
 	root.MetaDir = NewMetaDir(root, rootInode, entries)
@@ -64,46 +60,6 @@ func NewRoot(ctx context.Context, repo restic.Repository, cfg Config) (*Root, er
 	return root, nil
 }
 
-// NewTagsDir returns a new directory containing entries, which in turn contains
-// snapshots with this tag set.
-func NewTagsDir(root *Root, inode uint64, snapshots restic.Snapshots) fs.Node {
-	tags := make(map[string]restic.Snapshots)
-	for _, sn := range snapshots {
-		for _, tag := range sn.Tags {
-			tags[tag] = append(tags[tag], sn)
-		}
-	}
-
-	debug.Log("create tags dir with %d tags, inode %d", len(tags), inode)
-
-	entries := make(map[string]fs.Node)
-	for name, snapshots := range tags {
-		debug.Log("  tag %v has %v snapshots", name, len(snapshots))
-		entries[name] = NewSnapshotsDir(root, fs.GenerateDynamicInode(inode, name), snapshots)
-	}
-
-	return NewMetaDir(root, inode, entries)
-}
-
-// NewHostsDir returns a new directory containing hostnames, which in
-// turn contains snapshots of a single host each.
-func NewHostsDir(root *Root, inode uint64, snapshots restic.Snapshots) fs.Node {
-	hosts := make(map[string]restic.Snapshots)
-	for _, sn := range snapshots {
-		hosts[sn.Hostname] = append(hosts[sn.Hostname], sn)
-	}
-
-	debug.Log("create hosts dir with %d snapshots, inode %d", len(hosts), inode)
-
-	entries := make(map[string]fs.Node)
-	for name, snapshots := range hosts {
-		debug.Log("  host %v has %v snapshots", name, len(snapshots))
-		entries[name] = NewSnapshotsDir(root, fs.GenerateDynamicInode(inode, name), snapshots)
-	}
-
-	return NewMetaDir(root, inode, entries)
-}
-
 // Root is just there to satisfy fs.Root, it returns itself.
 func (r *Root) Root() (fs.Node, error) {
 	debug.Log("Root()")

internal/fuse/snapshots_dir.go

@@ -17,70 +17,128 @@ import (
 	"bazil.org/fuse/fs"
 )
 
-// SnapshotsDir is a fuse directory which contains snapshots.
+// SnapshotsDir is a fuse directory which contains snapshots named by timestamp.
 type SnapshotsDir struct {
 	inode  uint64
 	root   *Root
-	snapshots restic.Snapshots
 	names  map[string]*restic.Snapshot
 	latest string
+	tag    string
+	host   string
+}
+
+// SnapshotsIDSDir is a fuse directory which contains snapshots named by ids.
+type SnapshotsIDSDir struct {
+	inode uint64
+	root  *Root
+	names map[string]*restic.Snapshot
+}
+
+// HostsDir is a fuse directory which contains hosts.
+type HostsDir struct {
+	inode uint64
+	root  *Root
+	hosts map[string]bool
+}
+
+// TagsDir is a fuse directory which contains tags.
+type TagsDir struct {
+	inode uint64
+	root  *Root
+	tags  map[string]bool
+}
+
+// SnapshotLink
+type snapshotLink struct {
+	root     *Root
+	inode    uint64
+	target   string
+	snapshot *restic.Snapshot
 }
 
 // ensure that *SnapshotsDir implements these interfaces
 var _ = fs.HandleReadDirAller(&SnapshotsDir{})
 var _ = fs.NodeStringLookuper(&SnapshotsDir{})
+var _ = fs.HandleReadDirAller(&SnapshotsIDSDir{})
+var _ = fs.NodeStringLookuper(&SnapshotsIDSDir{})
+var _ = fs.HandleReadDirAller(&TagsDir{})
+var _ = fs.NodeStringLookuper(&TagsDir{})
+var _ = fs.HandleReadDirAller(&HostsDir{})
+var _ = fs.NodeStringLookuper(&HostsDir{})
 var _ = fs.NodeReadlinker(&snapshotLink{})
 
+// read tag names from the current repository-state.
+func getTagNames(d *TagsDir) {
+	d.tags = make(map[string]bool, len(d.root.snapshots))
+	for _, snapshot := range d.root.snapshots {
+		for _, tag := range snapshot.Tags {
+			d.tags[tag] = true
+		}
+	}
+}
+
+// read host names from the current repository-state.
+func getHostsNames(d *HostsDir) {
+	d.hosts = make(map[string]bool, len(d.root.snapshots))
+	for _, snapshot := range d.root.snapshots {
+		d.hosts[snapshot.Hostname] = true
+	}
+}
+
+// read snapshot id names from the current repository-state.
+func getSnapshotIDSNames(d *SnapshotsIDSDir) {
+	for _, sn := range d.root.snapshots {
+		name := sn.ID().Str()
+		d.names[name] = sn
+	}
+}
+
 // NewSnapshotsDir returns a new directory containing snapshots.
-func NewSnapshotsDir(root *Root, inode uint64, snapshots restic.Snapshots) *SnapshotsDir {
+func NewSnapshotsDir(root *Root, inode uint64, tag string, host string) *SnapshotsDir {
-	debug.Log("create snapshots dir with %d snapshots, inode %d", len(snapshots), inode)
+	debug.Log("create snapshots dir, inode %d", inode)
 	d := &SnapshotsDir{
 		root:   root,
 		inode:  inode,
-		snapshots: snapshots,
+		names:  make(map[string]*restic.Snapshot),
-		names:     make(map[string]*restic.Snapshot, len(snapshots)),
+		latest: "",
-	}
+		tag:    tag,
-
+		host:   host,
-	// Track latest Snapshot
-	var latestTime time.Time
-	d.latest = ""
-
-	for _, sn := range snapshots {
-		name := sn.Time.Format(time.RFC3339)
-		if d.latest == "" || !sn.Time.Before(latestTime) {
-			latestTime = sn.Time
-			d.latest = name
-		}
-		for i := 1; ; i++ {
-			if _, ok := d.names[name]; !ok {
-				break
-			}
-
-			name = fmt.Sprintf("%s-%d", sn.Time.Format(time.RFC3339), i)
-		}
-
-		d.names[name] = sn
-		debug.Log("  add snapshot %v as dir %v", sn.ID().Str(), name)
 	}
 
 	return d
 }
 
 // NewSnapshotsIDSDir returns a new directory containing snapshots named by ids.
-func NewSnapshotsIDSDir(root *Root, inode uint64, snapshots restic.Snapshots) *SnapshotsDir {
+func NewSnapshotsIDSDir(root *Root, inode uint64) *SnapshotsIDSDir {
-	debug.Log("create snapshots ids dir with %d snapshots, inode %d", len(snapshots), inode)
+	debug.Log("create snapshots ids dir, inode %d", inode)
-	d := &SnapshotsDir{
+	d := &SnapshotsIDSDir{
 		root:  root,
 		inode: inode,
-		snapshots: snapshots,
+		names: make(map[string]*restic.Snapshot),
-		names:     make(map[string]*restic.Snapshot, len(snapshots)),
 	}
 
-	for _, sn := range snapshots {
+	return d
-		name := sn.ID().Str()
+}
 
-		d.names[name] = sn
+// NewHostsDir returns a new directory containing host names
-		debug.Log("  add snapshot %v", name)
+func NewHostsDir(root *Root, inode uint64) *HostsDir {
+	debug.Log("create hosts dir, inode %d", inode)
+	d := &HostsDir{
+		root:  root,
+		inode: inode,
+		hosts: make(map[string]bool),
+	}
+
+	return d
+}
+
+// NewTagsDir returns a new directory containing tag names
+func NewTagsDir(root *Root, inode uint64) *TagsDir {
+	debug.Log("create tags dir, inode %d", inode)
+	d := &TagsDir{
+		root:  root,
+		inode: inode,
+		tags:  make(map[string]bool),
 	}
 
 	return d
@@ -99,9 +157,91 @@ func (d *SnapshotsDir) Attr(ctx context.Context, attr *fuse.Attr) error {
 	return nil
 }
 
-// ReadDirAll returns all entries of the root node.
+// Attr returns the attributes for the SnapshotsDir.
+func (d *SnapshotsIDSDir) Attr(ctx context.Context, attr *fuse.Attr) error {
+	attr.Inode = d.inode
+	attr.Mode = os.ModeDir | 0555
+
+	if !d.root.cfg.OwnerIsRoot {
+		attr.Uid = uint32(os.Getuid())
+		attr.Gid = uint32(os.Getgid())
+	}
+	debug.Log("attr: %v", attr)
+	return nil
+}
+
+// Attr returns the attributes for the HostsDir.
+func (d *HostsDir) Attr(ctx context.Context, attr *fuse.Attr) error {
+	attr.Inode = d.inode
+	attr.Mode = os.ModeDir | 0555
+
+	if !d.root.cfg.OwnerIsRoot {
+		attr.Uid = uint32(os.Getuid())
+		attr.Gid = uint32(os.Getgid())
+	}
+	debug.Log("attr: %v", attr)
+	return nil
+}
+
+// Attr returns the attributes for the TagsDir.
+func (d *TagsDir) Attr(ctx context.Context, attr *fuse.Attr) error {
+	attr.Inode = d.inode
+	attr.Mode = os.ModeDir | 0555
+
+	if !d.root.cfg.OwnerIsRoot {
+		attr.Uid = uint32(os.Getuid())
+		attr.Gid = uint32(os.Getgid())
+	}
+	debug.Log("attr: %v", attr)
+	return nil
+}
+
+// search element in string list.
+func isElem(e string, list []string) bool {
+	for _, x := range list {
+		if e == x {
+			return true
+		}
+	}
+	return false
+}
+
+// read snapshot timestamps from the current repository-state.
+func getSnapshotNames(d *SnapshotsDir) {
+	var latestTime time.Time
+	d.latest = ""
+	d.names = make(map[string]*restic.Snapshot, len(d.root.snapshots))
+	for _, sn := range d.root.snapshots {
+		if d.tag == "" || isElem(d.tag, sn.Tags) {
+			if d.host == "" || d.host == sn.Hostname {
+				name := sn.Time.Format(time.RFC3339)
+				if d.latest == "" || !sn.Time.Before(latestTime) {
+					latestTime = sn.Time
+					d.latest = name
+				}
+				for i := 1; ; i++ {
+					if _, ok := d.names[name]; !ok {
+						break
+					}
+
+					name = fmt.Sprintf("%s-%d", sn.Time.Format(time.RFC3339), i)
+				}
+
+				d.names[name] = sn
+			}
+		}
+	}
+}
+
+// ReadDirAll returns all entries of the SnapshotsDir.
 func (d *SnapshotsDir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) {
 	debug.Log("ReadDirAll()")
+
+	d.root.repo.LoadIndex(ctx)
+	d.root.snapshots = restic.FindFilteredSnapshots(ctx, d.root.repo, d.root.cfg.Host, d.root.cfg.Tags, d.root.cfg.Paths)
+
+	getSnapshotNames(d)
+
 	items := []fuse.Dirent{
 		{
 			Inode: d.inode,
@@ -134,21 +274,116 @@ func (d *SnapshotsDir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) {
 	return items, nil
 }
 
-type snapshotLink struct {
+// ReadDirAll returns all entries of the SnapshotsIDSDir.
-	root     *Root
+func (d *SnapshotsIDSDir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) {
-	inode    uint64
+	debug.Log("ReadDirAll()")
-	target   string
+
-	snapshot *restic.Snapshot
+	d.root.repo.LoadIndex(ctx)
+	d.root.snapshots = restic.FindFilteredSnapshots(ctx, d.root.repo, d.root.cfg.Host, d.root.cfg.Tags, d.root.cfg.Paths)
+
+	getSnapshotIDSNames(d)
+
+	items := []fuse.Dirent{
+		{
+			Inode: d.inode,
+			Name:  ".",
+			Type:  fuse.DT_Dir,
+		},
+		{
+			Inode: d.root.inode,
+			Name:  "..",
+			Type:  fuse.DT_Dir,
+		},
+	}
+
+	for name := range d.names {
+		items = append(items, fuse.Dirent{
+			Inode: fs.GenerateDynamicInode(d.inode, name),
+			Name:  name,
+			Type:  fuse.DT_Dir,
+		})
+	}
+
+	return items, nil
 }
 
+// ReadDirAll returns all entries of the HostsDir.
+func (d *HostsDir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) {
+	debug.Log("ReadDirAll()")
+
+	d.root.repo.LoadIndex(ctx)
+	d.root.snapshots = restic.FindFilteredSnapshots(ctx, d.root.repo, d.root.cfg.Host, d.root.cfg.Tags, d.root.cfg.Paths)
+
+	getHostsNames(d)
+
+	items := []fuse.Dirent{
+		{
+			Inode: d.inode,
+			Name:  ".",
+			Type:  fuse.DT_Dir,
+		},
+		{
+			Inode: d.root.inode,
+			Name:  "..",
+			Type:  fuse.DT_Dir,
+		},
+	}
+
+	for host := range d.hosts {
+		items = append(items, fuse.Dirent{
+			Inode: fs.GenerateDynamicInode(d.inode, host),
+			Name:  host,
+			Type:  fuse.DT_Dir,
+		})
+	}
+
+	return items, nil
+}
+
+// ReadDirAll returns all entries of the TagsDir.
+func (d *TagsDir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) {
+	debug.Log("ReadDirAll()")
+
+	d.root.repo.LoadIndex(ctx)
+	d.root.snapshots = restic.FindFilteredSnapshots(ctx, d.root.repo, d.root.cfg.Host, d.root.cfg.Tags, d.root.cfg.Paths)
+
+	getTagNames(d)
+
+	items := []fuse.Dirent{
+		{
+			Inode: d.inode,
+			Name:  ".",
+			Type:  fuse.DT_Dir,
+		},
+		{
+			Inode: d.root.inode,
+			Name:  "..",
+			Type:  fuse.DT_Dir,
+		},
+	}
+
+	for tag := range d.tags {
+		items = append(items, fuse.Dirent{
+			Inode: fs.GenerateDynamicInode(d.inode, tag),
+			Name:  tag,
+			Type:  fuse.DT_Dir,
+		})
+	}
+
+	return items, nil
+}
+
+// newSnapshotLink
 func newSnapshotLink(ctx context.Context, root *Root, inode uint64, target string, snapshot *restic.Snapshot) (*snapshotLink, error) {
 	return &snapshotLink{root: root, inode: inode, target: target, snapshot: snapshot}, nil
 }
 
+// Readlink
 func (l *snapshotLink) Readlink(ctx context.Context, req *fuse.ReadlinkRequest) (string, error) {
 	return l.target, nil
 }
 
+// Attr
 func (l *snapshotLink) Attr(ctx context.Context, a *fuse.Attr) error {
 	a.Inode = l.inode
 	a.Mode = os.ModeSymlink | 0777
@@ -166,24 +401,104 @@ func (l *snapshotLink) Attr(ctx context.Context, a *fuse.Attr) error {
 	return nil
 }
 
-// Lookup returns a specific entry from the root node.
+// Lookup returns a specific entry from the SnapshotsDir.
 func (d *SnapshotsDir) Lookup(ctx context.Context, name string) (fs.Node, error) {
 	debug.Log("Lookup(%s)", name)
 
 	sn, ok := d.names[name]
 	if !ok {
+		// could not find entry. Updating repository-state
+		d.root.repo.LoadIndex(ctx)
+		d.root.snapshots = restic.FindFilteredSnapshots(ctx, d.root.repo, d.root.cfg.Host, d.root.cfg.Tags, d.root.cfg.Paths)
+
+		getSnapshotNames(d)
+
+		sn, ok := d.names[name]
+		if ok {
+			return newDirFromSnapshot(ctx, d.root, fs.GenerateDynamicInode(d.inode, name), sn)
+		}
+
 		if name == "latest" && d.latest != "" {
-			sn2, ok2 := d.names[d.latest]
+			sn, ok := d.names[d.latest]
 
 			// internal error
-			if !ok2 {
+			if !ok {
 				return nil, fuse.ENOENT
 			}
 
-			return newSnapshotLink(ctx, d.root, fs.GenerateDynamicInode(d.inode, name), d.latest, sn2)
+			return newSnapshotLink(ctx, d.root, fs.GenerateDynamicInode(d.inode, name), d.latest, sn)
 		}
 		return nil, fuse.ENOENT
 	}
 
 	return newDirFromSnapshot(ctx, d.root, fs.GenerateDynamicInode(d.inode, name), sn)
 }
+
+// Lookup returns a specific entry from the SnapshotsIDSDir.
+func (d *SnapshotsIDSDir) Lookup(ctx context.Context, name string) (fs.Node, error) {
+	debug.Log("Lookup(%s)", name)
+
+	sn, ok := d.names[name]
+	if !ok {
+		// could not find entry. Updating repository-state
+		d.root.repo.LoadIndex(ctx)
+		d.root.snapshots = restic.FindFilteredSnapshots(ctx, d.root.repo, d.root.cfg.Host, d.root.cfg.Tags, d.root.cfg.Paths)
+
+		getSnapshotIDSNames(d)
+
+		sn, ok := d.names[name]
+		if ok {
+			return newDirFromSnapshot(ctx, d.root, fs.GenerateDynamicInode(d.inode, name), sn)
+		}
+
+		return nil, fuse.ENOENT
+	}
+
+	return newDirFromSnapshot(ctx, d.root, fs.GenerateDynamicInode(d.inode, name), sn)
+}
+
+// Lookup returns a specific entry from the HostsDir.
+func (d *HostsDir) Lookup(ctx context.Context, name string) (fs.Node, error) {
+	debug.Log("Lookup(%s)", name)
+
+	_, ok := d.hosts[name]
+	if !ok {
+		// could not find entry. Updating repository-state
+		d.root.repo.LoadIndex(ctx)
+		d.root.snapshots = restic.FindFilteredSnapshots(ctx, d.root.repo, d.root.cfg.Host, d.root.cfg.Tags, d.root.cfg.Paths)
+
+		getHostsNames(d)
+
+		_, ok := d.hosts[name]
+		if ok {
+			return NewSnapshotsDir(d.root, fs.GenerateDynamicInode(d.root.inode, name), "", name), nil
+		}
+
+		return nil, fuse.ENOENT
+	}
+
+	return NewSnapshotsDir(d.root, fs.GenerateDynamicInode(d.root.inode, name), "", name), nil
+}
+
+// Lookup returns a specific entry from the TagsDir.
+func (d *TagsDir) Lookup(ctx context.Context, name string) (fs.Node, error) {
+	debug.Log("Lookup(%s)", name)
+
+	_, ok := d.tags[name]
+	if !ok {
+		// could not find entry. Updating repository-state
+		d.root.repo.LoadIndex(ctx)
+		d.root.snapshots = restic.FindFilteredSnapshots(ctx, d.root.repo, d.root.cfg.Host, d.root.cfg.Tags, d.root.cfg.Paths)
+
+		getTagNames(d)
+
+		_, ok := d.tags[name]
+		if ok {
+			return NewSnapshotsDir(d.root, fs.GenerateDynamicInode(d.root.inode, name), name, ""), nil
+		}
+
+		return nil, fuse.ENOENT
+	}
+
+	return NewSnapshotsDir(d.root, fs.GenerateDynamicInode(d.root.inode, name), name, ""), nil
+}