From 696c18e031c96128da027b71a0f054c9e93a5014 Mon Sep 17 00:00:00 2001 From: Alexander Weiss Date: Mon, 28 Dec 2020 07:46:56 +0100 Subject: [PATCH 1/7] Add possibility to set snapshot ID (used in test) --- internal/restic/snapshot.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/internal/restic/snapshot.go b/internal/restic/snapshot.go index b12548459..2e8fa8a15 100644 --- a/internal/restic/snapshot.go +++ b/internal/restic/snapshot.go @@ -146,6 +146,11 @@ func (sn Snapshot) ID() *ID { return sn.id } +// SetID sets the snapshot's ID. +func (sn *Snapshot) SetID(id ID) { + sn.id = &id +} + func (sn *Snapshot) fillUserInfo() error { usr, err := user.Current() if err != nil { From 57f4003f2f92cd9bffb3a5875c1c9ea4020155e3 Mon Sep 17 00:00:00 2001 From: Alexander Weiss Date: Wed, 2 Sep 2020 21:27:24 +0200 Subject: [PATCH 2/7] Generalize fuse snapshot dirs implemetation + allow "/" in tags and snapshot template --- cmd/restic/cmd_mount.go | 4 - internal/fuse/root.go | 19 +- internal/fuse/snapshots_dir.go | 522 +++------------------- internal/fuse/snapshots_dirstruct.go | 202 +++++++++ internal/fuse/snapshots_dirstruct_test.go | 157 +++++++ 5 files changed, 434 insertions(+), 470 deletions(-) create mode 100644 internal/fuse/snapshots_dirstruct.go create mode 100644 internal/fuse/snapshots_dirstruct_test.go diff --git a/cmd/restic/cmd_mount.go b/cmd/restic/cmd_mount.go index 905cee1d3..b5112f181 100644 --- a/cmd/restic/cmd_mount.go +++ b/cmd/restic/cmd_mount.go @@ -5,7 +5,6 @@ package main import ( "os" - "strings" "time" "github.com/spf13/cobra" @@ -86,9 +85,6 @@ func runMount(opts MountOptions, gopts GlobalOptions, args []string) error { if opts.SnapshotTemplate == "" { return errors.Fatal("snapshot template string cannot be empty") } - if strings.ContainsAny(opts.SnapshotTemplate, `\/`) { - return errors.Fatal("snapshot template string contains a slash (/) or backslash (\\) character") - } if len(args) == 0 { return errors.Fatal("wrong number of parameters") } diff --git a/internal/fuse/root.go b/internal/fuse/root.go index bed760f02..8165ddafa 100644 --- a/internal/fuse/root.go +++ b/internal/fuse/root.go @@ -5,7 +5,6 @@ package fuse import ( "os" - "time" "github.com/restic/restic/internal/bloblru" "github.com/restic/restic/internal/debug" @@ -28,13 +27,9 @@ type Root struct { repo restic.Repository cfg Config inode uint64 - snapshots restic.Snapshots blobCache *bloblru.Cache - snCount int - lastCheck time.Time - - *MetaDir + *SnapshotsDir uid, gid uint32 } @@ -64,14 +59,14 @@ func NewRoot(repo restic.Repository, cfg Config) *Root { root.gid = uint32(os.Getgid()) } - entries := map[string]fs.Node{ - "snapshots": NewSnapshotsDir(root, fs.GenerateDynamicInode(root.inode, "snapshots"), "", ""), - "tags": NewTagsDir(root, fs.GenerateDynamicInode(root.inode, "tags")), - "hosts": NewHostsDir(root, fs.GenerateDynamicInode(root.inode, "hosts")), - "ids": NewSnapshotsIDSDir(root, fs.GenerateDynamicInode(root.inode, "ids")), + paths := []string{ + "ids/%i", + "snapshots/%T", + "hosts/%h/%T", + "tags/%t/%T", } - root.MetaDir = NewMetaDir(root, rootInode, entries) + root.SnapshotsDir = NewSnapshotsDir(root, rootInode, NewSnapshotsDirStructure(root, paths, cfg.SnapshotTemplate), "") return root } diff --git a/internal/fuse/snapshots_dir.go b/internal/fuse/snapshots_dir.go index 88638d50c..2924919da 100644 --- a/internal/fuse/snapshots_dir.go +++ b/internal/fuse/snapshots_dir.go @@ -5,9 +5,8 @@ package fuse import ( "context" - "fmt" "os" - "time" + "strings" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/restic" @@ -16,152 +15,32 @@ import ( "bazil.org/fuse/fs" ) -// SnapshotsDir is a fuse directory which contains snapshots named by timestamp. +// SnapshotsDir is a actual fuse directory generated from SnapshotsDirStructure +// It uses the saved prefix to filter out the relevant subtrees or entries +// from SnapshotsDirStructure.names and .latest, respectively. type SnapshotsDir struct { - inode uint64 - root *Root - names map[string]*restic.Snapshot - latest string - tag string - host string - snCount int - - template string -} - -// SnapshotsIDSDir is a fuse directory which contains snapshots named by ids. -type SnapshotsIDSDir struct { - inode uint64 - root *Root - names map[string]*restic.Snapshot - snCount int -} - -// HostsDir is a fuse directory which contains hosts. -type HostsDir struct { - inode uint64 - root *Root - hosts map[string]bool - snCount int -} - -// TagsDir is a fuse directory which contains tags. -type TagsDir struct { - inode uint64 - root *Root - tags map[string]bool - snCount int -} - -// SnapshotLink -type snapshotLink struct { - root *Root - inode uint64 - target string - snapshot *restic.Snapshot + root *Root + inode uint64 + dirStruct *SnapshotsDirStructure + prefix string } // ensure that *SnapshotsDir implements these interfaces var _ = fs.HandleReadDirAller(&SnapshotsDir{}) var _ = fs.NodeStringLookuper(&SnapshotsDir{}) -var _ = fs.HandleReadDirAller(&SnapshotsIDSDir{}) -var _ = fs.NodeStringLookuper(&SnapshotsIDSDir{}) -var _ = fs.HandleReadDirAller(&TagsDir{}) -var _ = fs.NodeStringLookuper(&TagsDir{}) -var _ = fs.HandleReadDirAller(&HostsDir{}) -var _ = fs.NodeStringLookuper(&HostsDir{}) -var _ = fs.NodeReadlinker(&snapshotLink{}) -// read tag names from the current repository-state. -func updateTagNames(d *TagsDir) { - if d.snCount != d.root.snCount { - d.snCount = d.root.snCount - d.tags = make(map[string]bool, len(d.root.snapshots)) - for _, snapshot := range d.root.snapshots { - for _, tag := range snapshot.Tags { - if tag != "" { - d.tags[tag] = true - } - } - } - } -} - -// read host names from the current repository-state. -func updateHostsNames(d *HostsDir) { - if d.snCount != d.root.snCount { - d.snCount = d.root.snCount - d.hosts = make(map[string]bool, len(d.root.snapshots)) - for _, snapshot := range d.root.snapshots { - d.hosts[snapshot.Hostname] = true - } - } -} - -// read snapshot id names from the current repository-state. -func updateSnapshotIDSNames(d *SnapshotsIDSDir) { - if d.snCount != d.root.snCount { - d.snCount = d.root.snCount - for _, sn := range d.root.snapshots { - name := sn.ID().Str() - d.names[name] = sn - } - } -} - -// NewSnapshotsDir returns a new directory containing snapshots. -func NewSnapshotsDir(root *Root, inode uint64, tag string, host string) *SnapshotsDir { +// NewSnapshotsDir returns a new directory structure containing snapshots and "latest" links +func NewSnapshotsDir(root *Root, inode uint64, dirStruct *SnapshotsDirStructure, prefix string) *SnapshotsDir { debug.Log("create snapshots dir, inode %d", inode) - d := &SnapshotsDir{ - root: root, - inode: inode, - names: make(map[string]*restic.Snapshot), - latest: "", - tag: tag, - host: host, - template: root.cfg.SnapshotTemplate, + return &SnapshotsDir{ + root: root, + inode: inode, + dirStruct: dirStruct, + prefix: prefix, } - - return d } -// NewSnapshotsIDSDir returns a new directory containing snapshots named by ids. -func NewSnapshotsIDSDir(root *Root, inode uint64) *SnapshotsIDSDir { - debug.Log("create snapshots ids dir, inode %d", inode) - d := &SnapshotsIDSDir{ - root: root, - inode: inode, - names: make(map[string]*restic.Snapshot), - } - - return d -} - -// NewHostsDir returns a new directory containing host names -func NewHostsDir(root *Root, inode uint64) *HostsDir { - debug.Log("create hosts dir, inode %d", inode) - d := &HostsDir{ - root: root, - inode: inode, - hosts: make(map[string]bool), - } - - return d -} - -// NewTagsDir returns a new directory containing tag names -func NewTagsDir(root *Root, inode uint64) *TagsDir { - debug.Log("create tags dir, inode %d", inode) - d := &TagsDir{ - root: root, - inode: inode, - tags: make(map[string]bool), - } - - return d -} - -// Attr returns the attributes for the root node. +// Attr returns the attributes for any dir in the snapshots directory structure func (d *SnapshotsDir) Attr(ctx context.Context, attr *fuse.Attr) error { attr.Inode = d.inode attr.Mode = os.ModeDir | 0555 @@ -172,118 +51,16 @@ func (d *SnapshotsDir) Attr(ctx context.Context, attr *fuse.Attr) error { return nil } -// Attr returns the attributes for the SnapshotsDir. -func (d *SnapshotsIDSDir) Attr(ctx context.Context, attr *fuse.Attr) error { - attr.Inode = d.inode - attr.Mode = os.ModeDir | 0555 - attr.Uid = d.root.uid - attr.Gid = d.root.gid - - debug.Log("attr: %v", attr) - return nil -} - -// Attr returns the attributes for the HostsDir. -func (d *HostsDir) Attr(ctx context.Context, attr *fuse.Attr) error { - attr.Inode = d.inode - attr.Mode = os.ModeDir | 0555 - attr.Uid = d.root.uid - attr.Gid = d.root.gid - - debug.Log("attr: %v", attr) - return nil -} - -// Attr returns the attributes for the TagsDir. -func (d *TagsDir) Attr(ctx context.Context, attr *fuse.Attr) error { - attr.Inode = d.inode - attr.Mode = os.ModeDir | 0555 - attr.Uid = d.root.uid - attr.Gid = d.root.gid - - debug.Log("attr: %v", attr) - return nil -} - -// search element in string list. -func isElem(e string, list []string) bool { - for _, x := range list { - if e == x { - return true - } - } - return false -} - -const minSnapshotsReloadTime = 60 * time.Second - -// update snapshots if repository has changed -func updateSnapshots(ctx context.Context, root *Root) error { - if time.Since(root.lastCheck) < minSnapshotsReloadTime { - return nil - } - - snapshots, err := restic.FindFilteredSnapshots(ctx, root.repo.Backend(), root.repo, root.cfg.Hosts, root.cfg.Tags, root.cfg.Paths) - if err != nil { - return err - } - - if root.snCount != len(snapshots) { - root.snCount = len(snapshots) - err := root.repo.LoadIndex(ctx) - if err != nil { - return err - } - root.snapshots = snapshots - } - root.lastCheck = time.Now() - - return nil -} - -// read snapshot timestamps from the current repository-state. -func updateSnapshotNames(d *SnapshotsDir, template string) { - if d.snCount != d.root.snCount { - d.snCount = d.root.snCount - var latestTime time.Time - d.latest = "" - d.names = make(map[string]*restic.Snapshot, len(d.root.snapshots)) - for _, sn := range d.root.snapshots { - if d.tag == "" || isElem(d.tag, sn.Tags) { - if d.host == "" || d.host == sn.Hostname { - name := sn.Time.Format(template) - if d.latest == "" || !sn.Time.Before(latestTime) { - latestTime = sn.Time - d.latest = name - } - for i := 1; ; i++ { - if _, ok := d.names[name]; !ok { - break - } - - name = fmt.Sprintf("%s-%d", sn.Time.Format(template), i) - } - - d.names[name] = sn - } - } - } - } -} - // ReadDirAll returns all entries of the SnapshotsDir. func (d *SnapshotsDir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) { debug.Log("ReadDirAll()") // update snapshots - err := updateSnapshots(ctx, d.root) + err := d.dirStruct.updateSnapshots(ctx) if err != nil { return nil, err } - // update snapshot names - updateSnapshotNames(d, d.root.cfg.SnapshotTemplate) - items := []fuse.Dirent{ { Inode: d.inode, @@ -297,135 +74,86 @@ func (d *SnapshotsDir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) { }, } - for name := range d.names { + // map to ensure that all names are only listed once + hasName := make(map[string]struct{}) + + for name := range d.dirStruct.names { + if !strings.HasPrefix(name, d.prefix) { + continue + } + shortname := strings.Split(name[len(d.prefix):], "/")[0] + if shortname == "" { + continue + } + if _, ok := hasName[shortname]; ok { + continue + } + hasName[shortname] = struct{}{} items = append(items, fuse.Dirent{ - Inode: fs.GenerateDynamicInode(d.inode, name), - Name: name, + Inode: fs.GenerateDynamicInode(d.inode, shortname), + Name: shortname, Type: fuse.DT_Dir, }) } // Latest - if d.latest != "" { + if _, ok := d.dirStruct.latest[d.prefix]; ok { items = append(items, fuse.Dirent{ Inode: fs.GenerateDynamicInode(d.inode, "latest"), Name: "latest", Type: fuse.DT_Link, }) } + return items, nil } -// ReadDirAll returns all entries of the SnapshotsIDSDir. -func (d *SnapshotsIDSDir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) { - debug.Log("ReadDirAll()") +// Lookup returns a specific entry from the SnapshotsDir. +func (d *SnapshotsDir) Lookup(ctx context.Context, name string) (fs.Node, error) { + debug.Log("Lookup(%s)", name) - // update snapshots - err := updateSnapshots(ctx, d.root) + err := d.dirStruct.updateSnapshots(ctx) if err != nil { return nil, err } - // update snapshot ids - updateSnapshotIDSNames(d) + fullname := d.prefix + name - items := []fuse.Dirent{ - { - Inode: d.inode, - Name: ".", - Type: fuse.DT_Dir, - }, - { - Inode: d.root.inode, - Name: "..", - Type: fuse.DT_Dir, - }, + // check if this is already a complete snapshot path + sn := d.dirStruct.names[fullname] + if sn != nil { + return newDirFromSnapshot(ctx, d.root, fs.GenerateDynamicInode(d.inode, name), sn) } - for name := range d.names { - items = append(items, fuse.Dirent{ - Inode: fs.GenerateDynamicInode(d.inode, name), - Name: name, - Type: fuse.DT_Dir, - }) + // handle latest case + if name == "latest" { + link := d.dirStruct.latest[d.prefix] + sn := d.dirStruct.names[d.prefix+link] + if sn != nil { + return newSnapshotLink(ctx, d.root, fs.GenerateDynamicInode(d.inode, name), link, sn) + } } - return items, nil + // check if this is a valid subdir + fullname = fullname + "/" + for name := range d.dirStruct.names { + if strings.HasPrefix(name, fullname) { + return NewSnapshotsDir(d.root, fs.GenerateDynamicInode(d.inode, name), d.dirStruct, fullname), nil + } + } + + return nil, fuse.ENOENT } -// ReadDirAll returns all entries of the HostsDir. -func (d *HostsDir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) { - debug.Log("ReadDirAll()") - - // update snapshots - err := updateSnapshots(ctx, d.root) - if err != nil { - return nil, err - } - - // update host names - updateHostsNames(d) - - items := []fuse.Dirent{ - { - Inode: d.inode, - Name: ".", - Type: fuse.DT_Dir, - }, - { - Inode: d.root.inode, - Name: "..", - Type: fuse.DT_Dir, - }, - } - - for host := range d.hosts { - items = append(items, fuse.Dirent{ - Inode: fs.GenerateDynamicInode(d.inode, host), - Name: host, - Type: fuse.DT_Dir, - }) - } - - return items, nil +// SnapshotLink +type snapshotLink struct { + root *Root + inode uint64 + target string + snapshot *restic.Snapshot } -// ReadDirAll returns all entries of the TagsDir. -func (d *TagsDir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) { - debug.Log("ReadDirAll()") - - // update snapshots - err := updateSnapshots(ctx, d.root) - if err != nil { - return nil, err - } - - // update tag names - updateTagNames(d) - - items := []fuse.Dirent{ - { - Inode: d.inode, - Name: ".", - Type: fuse.DT_Dir, - }, - { - Inode: d.root.inode, - Name: "..", - Type: fuse.DT_Dir, - }, - } - - for tag := range d.tags { - items = append(items, fuse.Dirent{ - Inode: fs.GenerateDynamicInode(d.inode, tag), - Name: tag, - Type: fuse.DT_Dir, - }) - } - - return items, nil -} +var _ = fs.NodeReadlinker(&snapshotLink{}) // newSnapshotLink func newSnapshotLink(ctx context.Context, root *Root, inode uint64, target string, snapshot *restic.Snapshot) (*snapshotLink, error) { @@ -453,117 +181,3 @@ func (l *snapshotLink) Attr(ctx context.Context, a *fuse.Attr) error { return nil } - -// Lookup returns a specific entry from the SnapshotsDir. -func (d *SnapshotsDir) Lookup(ctx context.Context, name string) (fs.Node, error) { - debug.Log("Lookup(%s)", name) - - sn, ok := d.names[name] - if !ok { - // could not find entry. Updating repository-state - err := updateSnapshots(ctx, d.root) - if err != nil { - return nil, err - } - - // update snapshot names - updateSnapshotNames(d, d.root.cfg.SnapshotTemplate) - - sn, ok := d.names[name] - if ok { - return newDirFromSnapshot(ctx, d.root, fs.GenerateDynamicInode(d.inode, name), sn) - } - - if name == "latest" && d.latest != "" { - sn, ok := d.names[d.latest] - - // internal error - if !ok { - return nil, fuse.ENOENT - } - - return newSnapshotLink(ctx, d.root, fs.GenerateDynamicInode(d.inode, name), d.latest, sn) - } - return nil, fuse.ENOENT - } - - return newDirFromSnapshot(ctx, d.root, fs.GenerateDynamicInode(d.inode, name), sn) -} - -// Lookup returns a specific entry from the SnapshotsIDSDir. -func (d *SnapshotsIDSDir) Lookup(ctx context.Context, name string) (fs.Node, error) { - debug.Log("Lookup(%s)", name) - - sn, ok := d.names[name] - if !ok { - // could not find entry. Updating repository-state - err := updateSnapshots(ctx, d.root) - if err != nil { - return nil, err - } - - // update snapshot ids - updateSnapshotIDSNames(d) - - sn, ok := d.names[name] - if ok { - return newDirFromSnapshot(ctx, d.root, fs.GenerateDynamicInode(d.inode, name), sn) - } - - return nil, fuse.ENOENT - } - - return newDirFromSnapshot(ctx, d.root, fs.GenerateDynamicInode(d.inode, name), sn) -} - -// Lookup returns a specific entry from the HostsDir. -func (d *HostsDir) Lookup(ctx context.Context, name string) (fs.Node, error) { - debug.Log("Lookup(%s)", name) - - _, ok := d.hosts[name] - if !ok { - // could not find entry. Updating repository-state - err := updateSnapshots(ctx, d.root) - if err != nil { - return nil, err - } - - // update host names - updateHostsNames(d) - - _, ok := d.hosts[name] - if ok { - return NewSnapshotsDir(d.root, fs.GenerateDynamicInode(d.root.inode, name), "", name), nil - } - - return nil, fuse.ENOENT - } - - return NewSnapshotsDir(d.root, fs.GenerateDynamicInode(d.root.inode, name), "", name), nil -} - -// Lookup returns a specific entry from the TagsDir. -func (d *TagsDir) Lookup(ctx context.Context, name string) (fs.Node, error) { - debug.Log("Lookup(%s)", name) - - _, ok := d.tags[name] - if !ok { - // could not find entry. Updating repository-state - err := updateSnapshots(ctx, d.root) - if err != nil { - return nil, err - } - - // update tag names - updateTagNames(d) - - _, ok := d.tags[name] - if ok { - return NewSnapshotsDir(d.root, fs.GenerateDynamicInode(d.root.inode, name), name, ""), nil - } - - return nil, fuse.ENOENT - } - - return NewSnapshotsDir(d.root, fs.GenerateDynamicInode(d.root.inode, name), name, ""), nil -} diff --git a/internal/fuse/snapshots_dirstruct.go b/internal/fuse/snapshots_dirstruct.go new file mode 100644 index 000000000..9bf2f5a7e --- /dev/null +++ b/internal/fuse/snapshots_dirstruct.go @@ -0,0 +1,202 @@ +//go:build darwin || freebsd || linux +// +build darwin freebsd linux + +package fuse + +import ( + "context" + "fmt" + "path" + "sort" + "strings" + "time" + + "github.com/restic/restic/internal/debug" + "github.com/restic/restic/internal/restic" +) + +// SnapshotsDirStructure contains the directory structure for snapshots. +// It uses a paths and time template to generate a map of pathnames +// pointing to the actual snapshots. For templates that end with a time, +// also "latest" links are generated. +type SnapshotsDirStructure struct { + root *Root + pathTemplates []string + timeTemplate string + + names map[string]*restic.Snapshot + latest map[string]string + snCount int + lastCheck time.Time +} + +// NewSnapshotsDirStructure returns a new directory structure for snapshots. +func NewSnapshotsDirStructure(root *Root, pathTemplates []string, timeTemplate string) *SnapshotsDirStructure { + return &SnapshotsDirStructure{ + root: root, + pathTemplates: pathTemplates, + timeTemplate: timeTemplate, + snCount: -1, + } +} + +// uniqueName returns a unique name to be used for prefix+name. +// It appends -number to make the name unique. +func (d *SnapshotsDirStructure) uniqueName(prefix, name string) (newname string) { + newname = name + for i := 1; ; i++ { + if _, ok := d.names[prefix+newname]; !ok { + break + } + newname = fmt.Sprintf("%s-%d", name, i) + } + return newname +} + +// pathsFromSn generates the paths from pathTemplate and timeTemplate +// where the variables are replaced by the snapshot data. +// The time is given as suffix if the pathTemplate ends with "%T". +func pathsFromSn(pathTemplate string, timeTemplate string, sn *restic.Snapshot) (paths []string, timeSuffix string) { + timeformat := sn.Time.Format(timeTemplate) + + inVerb := false + writeTime := false + out := make([]strings.Builder, 1) + for _, c := range pathTemplate { + if writeTime { + for i := range out { + out[i].WriteString(timeformat) + } + writeTime = false + } + + if !inVerb { + if c == '%' { + inVerb = true + } else { + for i := range out { + out[i].WriteRune(c) + } + } + continue + } + + var repl string + inVerb = false + switch c { + case 'T': + // lazy write; time might be returned as suffix + writeTime = true + continue + + case 't': + if len(sn.Tags) != 1 { + // needs special treatment: Rebuild the string builders + newout := make([]strings.Builder, len(out)*len(sn.Tags)) + for i, tag := range sn.Tags { + for j := range out { + newout[i*len(out)+j].WriteString(out[j].String() + tag) + } + } + out = newout + continue + } + repl = sn.Tags[0] + + case 'i': + repl = sn.ID().Str() + + case 'I': + repl = sn.ID().String() + + case 'u': + repl = sn.Username + + case 'h': + repl = sn.Hostname + } + + // write replacement string to all string builders + for i := range out { + out[i].WriteString(repl) + } + } + + for i := range out { + paths = append(paths, out[i].String()) + } + + if writeTime { + timeSuffix = timeformat + } + + return paths, timeSuffix +} + +// makeDirs inserts all paths generated from pathTemplates and +// TimeTemplate for all given snapshots into d.names. +// Also adds d.latest links if "%T" is at end of a path template +func (d *SnapshotsDirStructure) makeDirs(snapshots restic.Snapshots) { + d.names = make(map[string]*restic.Snapshot) + d.latest = make(map[string]string) + + // insert pure directories; needed to get empty structure even if there + // are no snapshots in these dirs + for _, p := range d.pathTemplates { + for _, pattern := range []string{"%i", "%I", "%u", "%h", "%t", "%T"} { + p = strings.ReplaceAll(p, pattern, "") + } + d.names[path.Clean(p)+"/"] = nil + } + + latestTime := make(map[string]time.Time) + for _, sn := range snapshots { + for _, templ := range d.pathTemplates { + paths, timeSuffix := pathsFromSn(templ, d.timeTemplate, sn) + for _, p := range paths { + suffix := d.uniqueName(p, timeSuffix) + d.names[path.Clean(p+suffix)] = sn + if timeSuffix != "" { + lt, ok := latestTime[p] + if !ok || !sn.Time.Before(lt) { + debug.Log("link (update) %v -> %v\n", p, suffix) + d.latest[p] = suffix + latestTime[p] = sn.Time + } + } + } + } + } +} + +const minSnapshotsReloadTime = 60 * time.Second + +// update snapshots if repository has changed +func (d *SnapshotsDirStructure) updateSnapshots(ctx context.Context) error { + if time.Since(d.lastCheck) < minSnapshotsReloadTime { + return nil + } + + snapshots, err := restic.FindFilteredSnapshots(ctx, d.root.repo.Backend(), d.root.repo, d.root.cfg.Hosts, d.root.cfg.Tags, d.root.cfg.Paths) + if err != nil { + return err + } + // sort snapshots ascending by time (default order is descending) + sort.Sort(sort.Reverse(snapshots)) + + d.lastCheck = time.Now() + + if d.snCount == len(snapshots) { + return nil + } + + err = d.root.repo.LoadIndex(ctx) + if err != nil { + return err + } + + d.snCount = len(snapshots) + + d.makeDirs(snapshots) + return nil +} diff --git a/internal/fuse/snapshots_dirstruct_test.go b/internal/fuse/snapshots_dirstruct_test.go new file mode 100644 index 000000000..5b0d88c1b --- /dev/null +++ b/internal/fuse/snapshots_dirstruct_test.go @@ -0,0 +1,157 @@ +//go:build darwin || freebsd || linux +// +build darwin freebsd linux + +package fuse + +import ( + "testing" + "time" + + "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/test" +) + +func TestPathsFromSn(t *testing.T) { + id1, _ := restic.ParseID("1234567812345678123456781234567812345678123456781234567812345678") + time1, _ := time.Parse("2006-01-02T15:04:05", "2021-01-01T00:00:01") + sn1 := &restic.Snapshot{Hostname: "host", Username: "user", Tags: []string{"tag1", "tag2"}, Time: time1} + sn1.SetID(id1) + + var p []string + var s string + + p, s = pathsFromSn("ids/%i", "2006-01-02T15:04:05", sn1) + test.Equals(t, []string{"ids/12345678"}, p) + test.Equals(t, "", s) + + p, s = pathsFromSn("snapshots/%T", "2006-01-02T15:04:05", sn1) + test.Equals(t, []string{"snapshots/"}, p) + test.Equals(t, "2021-01-01T00:00:01", s) + + p, s = pathsFromSn("hosts/%h/%T", "2006-01-02T15:04:05", sn1) + test.Equals(t, []string{"hosts/host/"}, p) + test.Equals(t, "2021-01-01T00:00:01", s) + + p, s = pathsFromSn("tags/%t/%T", "2006-01-02T15:04:05", sn1) + test.Equals(t, []string{"tags/tag1/", "tags/tag2/"}, p) + test.Equals(t, "2021-01-01T00:00:01", s) + + p, s = pathsFromSn("users/%u/%T", "2006-01-02T15:04:05", sn1) + test.Equals(t, []string{"users/user/"}, p) + test.Equals(t, "2021-01-01T00:00:01", s) + + p, s = pathsFromSn("longids/%I", "2006-01-02T15:04:05", sn1) + test.Equals(t, []string{"longids/1234567812345678123456781234567812345678123456781234567812345678"}, p) + test.Equals(t, "", s) + + p, s = pathsFromSn("%T/%h", "2006/01/02", sn1) + test.Equals(t, []string{"2021/01/01/host"}, p) + test.Equals(t, "", s) + + p, s = pathsFromSn("%T/%i", "2006/01", sn1) + test.Equals(t, []string{"2021/01/12345678"}, p) + test.Equals(t, "", s) +} + +func TestMakeDirs(t *testing.T) { + pathTemplates := []string{"ids/%i", "snapshots/%T", "hosts/%h/%T", + "tags/%t/%T", "users/%u/%T", "longids/%I", "%T/%h", "%T/%i", + } + timeTemplate := "2006/01/02" + + sds := &SnapshotsDirStructure{ + pathTemplates: pathTemplates, + timeTemplate: timeTemplate, + } + + id0, _ := restic.ParseID("0000000012345678123456781234567812345678123456781234567812345678") + time0, _ := time.Parse("2006-01-02T15:04:05", "2020-12-31T00:00:01") + sn0 := &restic.Snapshot{Hostname: "host", Username: "user", Tags: []string{"tag1", "tag2"}, Time: time0} + sn0.SetID(id0) + + id1, _ := restic.ParseID("1234567812345678123456781234567812345678123456781234567812345678") + time1, _ := time.Parse("2006-01-02T15:04:05", "2021-01-01T00:00:01") + sn1 := &restic.Snapshot{Hostname: "host", Username: "user", Tags: []string{"tag1", "tag2"}, Time: time1} + sn1.SetID(id1) + + id2, _ := restic.ParseID("8765432112345678123456781234567812345678123456781234567812345678") + time2, _ := time.Parse("2006-01-02T15:04:05", "2021-01-01T01:02:03") + sn2 := &restic.Snapshot{Hostname: "host2", Username: "user2", Tags: []string{"tag2", "tag3", "tag4"}, Time: time2} + sn2.SetID(id2) + + id3, _ := restic.ParseID("aaaaaaaa12345678123456781234567812345678123456781234567812345678") + time3, _ := time.Parse("2006-01-02T15:04:05", "2021-01-01T01:02:03") + sn3 := &restic.Snapshot{Hostname: "host", Username: "user2", Tags: []string{}, Time: time3} + sn3.SetID(id3) + + sds.makeDirs(restic.Snapshots{sn0, sn1, sn2, sn3}) + + expNames := make(map[string]*restic.Snapshot) + expLatest := make(map[string]string) + + // empty entries for dir structure + expNames["ids/"] = nil + expNames["snapshots/"] = nil + expNames["hosts/"] = nil + expNames["tags/"] = nil + expNames["users/"] = nil + expNames["longids/"] = nil + expNames["//"] = nil + + // entries for sn0 + expNames["ids/00000000"] = sn0 + expNames["snapshots/2020/12/31"] = sn0 + expNames["hosts/host/2020/12/31"] = sn0 + expNames["tags/tag1/2020/12/31"] = sn0 + expNames["tags/tag2/2020/12/31"] = sn0 + expNames["users/user/2020/12/31"] = sn0 + expNames["longids/0000000012345678123456781234567812345678123456781234567812345678"] = sn0 + expNames["2020/12/31/host"] = sn0 + expNames["2020/12/31/00000000"] = sn0 + + // entries for sn1 + expNames["ids/12345678"] = sn1 + expNames["snapshots/2021/01/01"] = sn1 + expNames["hosts/host/2021/01/01"] = sn1 + expNames["tags/tag1/2021/01/01"] = sn1 + expNames["tags/tag2/2021/01/01"] = sn1 + expNames["users/user/2021/01/01"] = sn1 + expNames["longids/1234567812345678123456781234567812345678123456781234567812345678"] = sn1 + expNames["2021/01/01/host"] = sn1 + expNames["2021/01/01/12345678"] = sn1 + + // entries for sn2 + expNames["ids/87654321"] = sn2 + expNames["snapshots/2021/01/01-1"] = sn2 // sn1 and sn2 have same time string + expNames["hosts/host2/2021/01/01"] = sn2 + expNames["tags/tag2/2021/01/01-1"] = sn2 // sn1 and sn2 have same time string + expNames["tags/tag3/2021/01/01"] = sn2 + expNames["tags/tag4/2021/01/01"] = sn2 + expNames["users/user2/2021/01/01"] = sn2 + expNames["longids/8765432112345678123456781234567812345678123456781234567812345678"] = sn2 + expNames["2021/01/01/host2"] = sn2 + expNames["2021/01/01/87654321"] = sn2 + + // entries for sn3 + expNames["ids/aaaaaaaa"] = sn3 + expNames["snapshots/2021/01/01-2"] = sn3 // sn1 - sn3 have same time string + expNames["hosts/host/2021/01/01-1"] = sn3 // sn1 and sn3 have same time string + expNames["users/user2/2021/01/01-1"] = sn3 // sn2 and sn3 have same time string + expNames["longids/aaaaaaaa12345678123456781234567812345678123456781234567812345678"] = sn3 + expNames["2021/01/01/host-1"] = sn3 // sn1 and sn3 have same time string and identical host + expNames["2021/01/01/aaaaaaaa"] = sn3 + + // latest links + expLatest["snapshots/"] = "2021/01/01-2" // sn1 - sn3 have same time string + expLatest["hosts/host/"] = "2021/01/01-1" + expLatest["hosts/host2/"] = "2021/01/01" + expLatest["tags/tag1/"] = "2021/01/01" + expLatest["tags/tag2/"] = "2021/01/01-1" // sn1 and sn2 have same time string + expLatest["tags/tag3/"] = "2021/01/01" + expLatest["tags/tag4/"] = "2021/01/01" + expLatest["users/user/"] = "2021/01/01" + expLatest["users/user2/"] = "2021/01/01-1" // sn2 and sn3 have same time string + + test.Equals(t, expNames, sds.names) + test.Equals(t, expLatest, sds.latest) +} From 1751afae26975dcbb7700b84eac263633f32b5a8 Mon Sep 17 00:00:00 2001 From: Alexander Weiss Date: Thu, 3 Sep 2020 20:31:57 +0200 Subject: [PATCH 3/7] Make snapshots dirs in mount command customizable --- changelog/unreleased/issue-2907 | 7 ++++ cmd/restic/cmd_mount.go | 52 ++++++++++++++++++++++------- cmd/restic/integration_fuse_test.go | 2 +- internal/fuse/root.go | 26 +++++++++------ 4 files changed, 63 insertions(+), 24 deletions(-) create mode 100644 changelog/unreleased/issue-2907 diff --git a/changelog/unreleased/issue-2907 b/changelog/unreleased/issue-2907 new file mode 100644 index 000000000..5d5c9d8e8 --- /dev/null +++ b/changelog/unreleased/issue-2907 @@ -0,0 +1,7 @@ +Enhancement: Make snapshot directory structure of mount command custimizable + +We've added the possibility to customize the snapshot directory structure of the mount command. +This includes using subdirectories which is now also possible within time template and tags. + +https://github.com/restic/restic/issues/2907 +https://github.com/restic/restic/pull/2913 diff --git a/cmd/restic/cmd_mount.go b/cmd/restic/cmd_mount.go index b5112f181..747316f9f 100644 --- a/cmd/restic/cmd_mount.go +++ b/cmd/restic/cmd_mount.go @@ -5,6 +5,7 @@ package main import ( "os" + "strings" "time" "github.com/spf13/cobra" @@ -30,10 +31,13 @@ read-only mount. Snapshot Directories ==================== -If you need a different template for all directories that contain snapshots, -you can pass a template via --snapshot-template. Example without colons: +If you need a different template for directories that contain snapshots, +you can pass a time template via --time-template and path templates via +--path-template. - --snapshot-template "2006-01-02_15-04-05" +Example time template without colons: + + --time-template "2006-01-02_15-04-05" You need to specify a sample format for exactly the following timestamp: @@ -42,6 +46,20 @@ You need to specify a sample format for exactly the following timestamp: For details please see the documentation for time.Format() at: https://godoc.org/time#Time.Format +For path templates, you can use the following patterns which will be replaced: + %i by short snapshot ID + %I by long snapshot ID + %u by username + %h by hostname + %t by tags + %T by timestamp as specified by --time-template + +The default path templates are: + "ids/%i" + "snapshots/%T" + "hosts/%h/%T" + "tags/%t/%T" + EXIT STATUS =========== @@ -61,7 +79,8 @@ type MountOptions struct { Hosts []string Tags restic.TagLists Paths []string - SnapshotTemplate string + TimeTemplate string + PathTemplates []string } var mountOptions MountOptions @@ -78,13 +97,21 @@ func init() { mountFlags.Var(&mountOptions.Tags, "tag", "only consider snapshots which include this `taglist`") mountFlags.StringArrayVar(&mountOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`") - mountFlags.StringVar(&mountOptions.SnapshotTemplate, "snapshot-template", time.RFC3339, "set `template` to use for snapshot dirs") + mountFlags.StringArrayVar(&mountOptions.PathTemplates, "path-template", nil, "set `template` for path names (can be specified multiple times)") + mountFlags.StringVar(&mountOptions.TimeTemplate, "snapshot-template", time.RFC3339, "set `template` to use for snapshot dirs") + mountFlags.StringVar(&mountOptions.TimeTemplate, "time-template", time.RFC3339, "set `template` to use for times") + _ = mountFlags.MarkDeprecated("snapshot-template", "use --time-template") } func runMount(opts MountOptions, gopts GlobalOptions, args []string) error { - if opts.SnapshotTemplate == "" { - return errors.Fatal("snapshot template string cannot be empty") + if opts.TimeTemplate == "" { + return errors.Fatal("time template string cannot be empty") } + + if strings.HasPrefix(opts.TimeTemplate, "/") || strings.HasSuffix(opts.TimeTemplate, "/") { + return errors.Fatal("time template string cannot start or end with '/'") + } + if len(args) == 0 { return errors.Fatal("wrong number of parameters") } @@ -150,11 +177,12 @@ func runMount(opts MountOptions, gopts GlobalOptions, args []string) error { } cfg := fuse.Config{ - OwnerIsRoot: opts.OwnerRoot, - Hosts: opts.Hosts, - Tags: opts.Tags, - Paths: opts.Paths, - SnapshotTemplate: opts.SnapshotTemplate, + OwnerIsRoot: opts.OwnerRoot, + Hosts: opts.Hosts, + Tags: opts.Tags, + Paths: opts.Paths, + TimeTemplate: opts.TimeTemplate, + PathTemplates: opts.PathTemplates, } root := fuse.NewRoot(repo, cfg) diff --git a/cmd/restic/integration_fuse_test.go b/cmd/restic/integration_fuse_test.go index 156a8abae..6a95ac87d 100644 --- a/cmd/restic/integration_fuse_test.go +++ b/cmd/restic/integration_fuse_test.go @@ -55,7 +55,7 @@ func waitForMount(t testing.TB, dir string) { func testRunMount(t testing.TB, gopts GlobalOptions, dir string) { opts := MountOptions{ - SnapshotTemplate: time.RFC3339, + TimeTemplate: time.RFC3339, } rtest.OK(t, runMount(opts, gopts, []string{dir})) } diff --git a/internal/fuse/root.go b/internal/fuse/root.go index 8165ddafa..5a14b374f 100644 --- a/internal/fuse/root.go +++ b/internal/fuse/root.go @@ -15,11 +15,12 @@ import ( // Config holds settings for the fuse mount. type Config struct { - OwnerIsRoot bool - Hosts []string - Tags []restic.TagList - Paths []string - SnapshotTemplate string + OwnerIsRoot bool + Hosts []string + Tags []restic.TagList + Paths []string + TimeTemplate string + PathTemplates []string } // Root is the root node of the fuse mount of a repository. @@ -59,14 +60,17 @@ func NewRoot(repo restic.Repository, cfg Config) *Root { root.gid = uint32(os.Getgid()) } - paths := []string{ - "ids/%i", - "snapshots/%T", - "hosts/%h/%T", - "tags/%t/%T", + // set defaults, if PathTemplates is not set + if len(cfg.PathTemplates) == 0 { + cfg.PathTemplates = []string{ + "ids/%i", + "snapshots/%T", + "hosts/%h/%T", + "tags/%t/%T", + } } - root.SnapshotsDir = NewSnapshotsDir(root, rootInode, NewSnapshotsDirStructure(root, paths, cfg.SnapshotTemplate), "") + root.SnapshotsDir = NewSnapshotsDir(root, rootInode, NewSnapshotsDirStructure(root, cfg.PathTemplates, cfg.TimeTemplate), "") return root } From f678f7cb0447c6b56c895ede04d2f9d26560c9cb Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 24 Jul 2022 11:22:57 +0200 Subject: [PATCH 4/7] fuse: cleanup test --- internal/fuse/snapshots_dirstruct_test.go | 10 +++++----- internal/restic/snapshot.go | 5 ----- internal/restic/testing.go | 5 +++++ 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/internal/fuse/snapshots_dirstruct_test.go b/internal/fuse/snapshots_dirstruct_test.go index 5b0d88c1b..2e96d2093 100644 --- a/internal/fuse/snapshots_dirstruct_test.go +++ b/internal/fuse/snapshots_dirstruct_test.go @@ -15,7 +15,7 @@ func TestPathsFromSn(t *testing.T) { id1, _ := restic.ParseID("1234567812345678123456781234567812345678123456781234567812345678") time1, _ := time.Parse("2006-01-02T15:04:05", "2021-01-01T00:00:01") sn1 := &restic.Snapshot{Hostname: "host", Username: "user", Tags: []string{"tag1", "tag2"}, Time: time1} - sn1.SetID(id1) + restic.TestSetSnapshotID(t, sn1, id1) var p []string var s string @@ -67,22 +67,22 @@ func TestMakeDirs(t *testing.T) { id0, _ := restic.ParseID("0000000012345678123456781234567812345678123456781234567812345678") time0, _ := time.Parse("2006-01-02T15:04:05", "2020-12-31T00:00:01") sn0 := &restic.Snapshot{Hostname: "host", Username: "user", Tags: []string{"tag1", "tag2"}, Time: time0} - sn0.SetID(id0) + restic.TestSetSnapshotID(t, sn0, id0) id1, _ := restic.ParseID("1234567812345678123456781234567812345678123456781234567812345678") time1, _ := time.Parse("2006-01-02T15:04:05", "2021-01-01T00:00:01") sn1 := &restic.Snapshot{Hostname: "host", Username: "user", Tags: []string{"tag1", "tag2"}, Time: time1} - sn1.SetID(id1) + restic.TestSetSnapshotID(t, sn1, id1) id2, _ := restic.ParseID("8765432112345678123456781234567812345678123456781234567812345678") time2, _ := time.Parse("2006-01-02T15:04:05", "2021-01-01T01:02:03") sn2 := &restic.Snapshot{Hostname: "host2", Username: "user2", Tags: []string{"tag2", "tag3", "tag4"}, Time: time2} - sn2.SetID(id2) + restic.TestSetSnapshotID(t, sn2, id2) id3, _ := restic.ParseID("aaaaaaaa12345678123456781234567812345678123456781234567812345678") time3, _ := time.Parse("2006-01-02T15:04:05", "2021-01-01T01:02:03") sn3 := &restic.Snapshot{Hostname: "host", Username: "user2", Tags: []string{}, Time: time3} - sn3.SetID(id3) + restic.TestSetSnapshotID(t, sn3, id3) sds.makeDirs(restic.Snapshots{sn0, sn1, sn2, sn3}) diff --git a/internal/restic/snapshot.go b/internal/restic/snapshot.go index 2e8fa8a15..b12548459 100644 --- a/internal/restic/snapshot.go +++ b/internal/restic/snapshot.go @@ -146,11 +146,6 @@ func (sn Snapshot) ID() *ID { return sn.id } -// SetID sets the snapshot's ID. -func (sn *Snapshot) SetID(id ID) { - sn.id = &id -} - func (sn *Snapshot) fillUserInfo() error { usr, err := user.Current() if err != nil { diff --git a/internal/restic/testing.go b/internal/restic/testing.go index 79bc0813a..ebafdf651 100644 --- a/internal/restic/testing.go +++ b/internal/restic/testing.go @@ -207,3 +207,8 @@ func TestParseID(s string) ID { func TestParseHandle(s string, t BlobType) BlobHandle { return BlobHandle{ID: TestParseID(s), Type: t} } + +// TestSetSnapshotID sets the snapshot's ID. +func TestSetSnapshotID(t testing.TB, sn *Snapshot, id ID) { + sn.id = &id +} From 2db7733ee325f98e737fea00c2f3a8859c30ae5e Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 30 Jul 2022 19:10:39 +0200 Subject: [PATCH 5/7] fuse: remove unused MetaDir --- internal/fuse/meta_dir.go | 85 --------------------------------------- 1 file changed, 85 deletions(-) delete mode 100644 internal/fuse/meta_dir.go diff --git a/internal/fuse/meta_dir.go b/internal/fuse/meta_dir.go deleted file mode 100644 index b102965f9..000000000 --- a/internal/fuse/meta_dir.go +++ /dev/null @@ -1,85 +0,0 @@ -//go:build darwin || freebsd || linux -// +build darwin freebsd linux - -package fuse - -import ( - "context" - "os" - - "github.com/restic/restic/internal/debug" - - "bazil.org/fuse" - "bazil.org/fuse/fs" -) - -// ensure that *DirSnapshots implements these interfaces -var _ = fs.HandleReadDirAller(&MetaDir{}) -var _ = fs.NodeStringLookuper(&MetaDir{}) - -// MetaDir is a fuse directory which contains other directories. -type MetaDir struct { - inode uint64 - root *Root - entries map[string]fs.Node -} - -// NewMetaDir returns a new meta dir. -func NewMetaDir(root *Root, inode uint64, entries map[string]fs.Node) *MetaDir { - debug.Log("new meta dir with %d entries, inode %d", len(entries), inode) - - return &MetaDir{ - root: root, - inode: inode, - entries: entries, - } -} - -// Attr returns the attributes for the root node. -func (d *MetaDir) Attr(ctx context.Context, attr *fuse.Attr) error { - attr.Inode = d.inode - attr.Mode = os.ModeDir | 0555 - attr.Uid = d.root.uid - attr.Gid = d.root.gid - - debug.Log("attr: %v", attr) - return nil -} - -// ReadDirAll returns all entries of the root node. -func (d *MetaDir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) { - debug.Log("ReadDirAll()") - items := []fuse.Dirent{ - { - Inode: d.inode, - Name: ".", - Type: fuse.DT_Dir, - }, - { - Inode: d.root.inode, - Name: "..", - Type: fuse.DT_Dir, - }, - } - - for name := range d.entries { - items = append(items, fuse.Dirent{ - Inode: fs.GenerateDynamicInode(d.inode, name), - Name: name, - Type: fuse.DT_Dir, - }) - } - - return items, nil -} - -// Lookup returns a specific entry from the root node. -func (d *MetaDir) Lookup(ctx context.Context, name string) (fs.Node, error) { - debug.Log("Lookup(%s)", name) - - if dir, ok := d.entries[name]; ok { - return dir, nil - } - - return nil, fuse.ENOENT -} From caa17988a3cfcefed27798c4856c32c92168f4c1 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 30 Jul 2022 20:45:51 +0200 Subject: [PATCH 6/7] fuse: Redesign snapshot dirstruct Cleanly separate the directory presentation and the snapshot directory structure. SnapshotsDir now translates the dirStruct into a format usable by the fuse library and contains only minimal special case rules. All decisions have moved into SnapshotsDirStructure which now creates a fully preassembled tree data structure. --- internal/fuse/snapshots_dir.go | 79 +++----- internal/fuse/snapshots_dirstruct.go | 153 ++++++++++++--- internal/fuse/snapshots_dirstruct_test.go | 228 ++++++++++++++++------ 3 files changed, 327 insertions(+), 133 deletions(-) diff --git a/internal/fuse/snapshots_dir.go b/internal/fuse/snapshots_dir.go index 2924919da..e5b18b20f 100644 --- a/internal/fuse/snapshots_dir.go +++ b/internal/fuse/snapshots_dir.go @@ -6,7 +6,6 @@ package fuse import ( "context" "os" - "strings" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/restic" @@ -16,8 +15,7 @@ import ( ) // SnapshotsDir is a actual fuse directory generated from SnapshotsDirStructure -// It uses the saved prefix to filter out the relevant subtrees or entries -// from SnapshotsDirStructure.names and .latest, respectively. +// It uses the saved prefix to select the corresponding MetaDirData. type SnapshotsDir struct { root *Root inode uint64 @@ -56,9 +54,11 @@ func (d *SnapshotsDir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) { debug.Log("ReadDirAll()") // update snapshots - err := d.dirStruct.updateSnapshots(ctx) + meta, err := d.dirStruct.UpdatePrefix(ctx, d.prefix) if err != nil { return nil, err + } else if meta == nil { + return nil, fuse.ENOENT } items := []fuse.Dirent{ @@ -74,35 +74,16 @@ func (d *SnapshotsDir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) { }, } - // map to ensure that all names are only listed once - hasName := make(map[string]struct{}) - - for name := range d.dirStruct.names { - if !strings.HasPrefix(name, d.prefix) { - continue - } - shortname := strings.Split(name[len(d.prefix):], "/")[0] - if shortname == "" { - continue - } - if _, ok := hasName[shortname]; ok { - continue - } - hasName[shortname] = struct{}{} - items = append(items, fuse.Dirent{ - Inode: fs.GenerateDynamicInode(d.inode, shortname), - Name: shortname, + for name, entry := range meta.names { + d := fuse.Dirent{ + Inode: fs.GenerateDynamicInode(d.inode, name), + Name: name, Type: fuse.DT_Dir, - }) - } - - // Latest - if _, ok := d.dirStruct.latest[d.prefix]; ok { - items = append(items, fuse.Dirent{ - Inode: fs.GenerateDynamicInode(d.inode, "latest"), - Name: "latest", - Type: fuse.DT_Link, - }) + } + if entry.linkTarget != "" { + d.Type = fuse.DT_Link + } + items = append(items, d) } return items, nil @@ -112,33 +93,21 @@ func (d *SnapshotsDir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) { func (d *SnapshotsDir) Lookup(ctx context.Context, name string) (fs.Node, error) { debug.Log("Lookup(%s)", name) - err := d.dirStruct.updateSnapshots(ctx) + meta, err := d.dirStruct.UpdatePrefix(ctx, d.prefix) if err != nil { return nil, err + } else if meta == nil { + return nil, fuse.ENOENT } - fullname := d.prefix + name - - // check if this is already a complete snapshot path - sn := d.dirStruct.names[fullname] - if sn != nil { - return newDirFromSnapshot(ctx, d.root, fs.GenerateDynamicInode(d.inode, name), sn) - } - - // handle latest case - if name == "latest" { - link := d.dirStruct.latest[d.prefix] - sn := d.dirStruct.names[d.prefix+link] - if sn != nil { - return newSnapshotLink(ctx, d.root, fs.GenerateDynamicInode(d.inode, name), link, sn) - } - } - - // check if this is a valid subdir - fullname = fullname + "/" - for name := range d.dirStruct.names { - if strings.HasPrefix(name, fullname) { - return NewSnapshotsDir(d.root, fs.GenerateDynamicInode(d.inode, name), d.dirStruct, fullname), nil + entry := meta.names[name] + if entry != nil { + if entry.linkTarget != "" { + return newSnapshotLink(ctx, d.root, fs.GenerateDynamicInode(d.inode, name), entry.linkTarget, entry.snapshot) + } else if entry.snapshot != nil { + return newDirFromSnapshot(ctx, d.root, fs.GenerateDynamicInode(d.inode, name), entry.snapshot) + } else { + return NewSnapshotsDir(d.root, fs.GenerateDynamicInode(d.inode, name), d.dirStruct, d.prefix+"/"+name), nil } } diff --git a/internal/fuse/snapshots_dirstruct.go b/internal/fuse/snapshots_dirstruct.go index 9bf2f5a7e..18831dcb1 100644 --- a/internal/fuse/snapshots_dirstruct.go +++ b/internal/fuse/snapshots_dirstruct.go @@ -9,12 +9,21 @@ import ( "path" "sort" "strings" + "sync" "time" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/restic" ) +type MetaDirData struct { + // set if this is a symlink or a snapshot mount point + linkTarget string + snapshot *restic.Snapshot + // names is set if this is a pseudo directory + names map[string]*MetaDirData +} + // SnapshotsDirStructure contains the directory structure for snapshots. // It uses a paths and time template to generate a map of pathnames // pointing to the actual snapshots. For templates that end with a time, @@ -24,8 +33,12 @@ type SnapshotsDirStructure struct { pathTemplates []string timeTemplate string - names map[string]*restic.Snapshot - latest map[string]string + mutex sync.Mutex + // "" is the root path, subdirectory paths are assembled as parent+"/"+childFn + // thus all subdirectories are prefixed with a slash as the root is "" + // that way we don't need path processing special cases when using the entries tree + entries map[string]*MetaDirData + snCount int lastCheck time.Time } @@ -40,19 +53,6 @@ func NewSnapshotsDirStructure(root *Root, pathTemplates []string, timeTemplate s } } -// uniqueName returns a unique name to be used for prefix+name. -// It appends -number to make the name unique. -func (d *SnapshotsDirStructure) uniqueName(prefix, name string) (newname string) { - newname = name - for i := 1; ; i++ { - if _, ok := d.names[prefix+newname]; !ok { - break - } - newname = fmt.Sprintf("%s-%d", name, i) - } - return newname -} - // pathsFromSn generates the paths from pathTemplate and timeTemplate // where the variables are replaced by the snapshot data. // The time is given as suffix if the pathTemplate ends with "%T". @@ -90,6 +90,9 @@ func pathsFromSn(pathTemplate string, timeTemplate string, sn *restic.Snapshot) continue case 't': + if len(sn.Tags) == 0 { + return nil, "" + } if len(sn.Tags) != 1 { // needs special treatment: Rebuild the string builders newout := make([]strings.Builder, len(out)*len(sn.Tags)) @@ -114,6 +117,9 @@ func pathsFromSn(pathTemplate string, timeTemplate string, sn *restic.Snapshot) case 'h': repl = sn.Hostname + + default: + repl = string(c) } // write replacement string to all string builders @@ -133,20 +139,102 @@ func pathsFromSn(pathTemplate string, timeTemplate string, sn *restic.Snapshot) return paths, timeSuffix } +// determine static path prefix +func staticPrefix(pathTemplate string) (prefix string) { + inVerb := false + patternStart := -1 +outer: + for i, c := range pathTemplate { + if !inVerb { + if c == '%' { + inVerb = true + } + continue + } + inVerb = false + switch c { + case 'i', 'I', 'u', 'h', 't', 'T': + patternStart = i + break outer + } + } + if patternStart < 0 { + // ignore patterns without template variable + return "" + } + + p := pathTemplate[:patternStart] + idx := strings.LastIndex(p, "/") + if idx < 0 { + return "" + } + return p[:idx] +} + +// uniqueName returns a unique name to be used for prefix+name. +// It appends -number to make the name unique. +func uniqueName(entries map[string]*MetaDirData, prefix, name string) string { + newname := name + for i := 1; ; i++ { + if _, ok := entries[prefix+newname]; !ok { + break + } + newname = fmt.Sprintf("%s-%d", name, i) + } + return newname +} + // makeDirs inserts all paths generated from pathTemplates and // TimeTemplate for all given snapshots into d.names. // Also adds d.latest links if "%T" is at end of a path template func (d *SnapshotsDirStructure) makeDirs(snapshots restic.Snapshots) { - d.names = make(map[string]*restic.Snapshot) - d.latest = make(map[string]string) + entries := make(map[string]*MetaDirData) + + type mountData struct { + sn *restic.Snapshot + linkTarget string // if linkTarget!= "", this is a symlink + childFn string + child *MetaDirData + } + + // recursively build tree structure + var mount func(path string, data mountData) + mount = func(path string, data mountData) { + e := entries[path] + if e == nil { + e = &MetaDirData{} + } + if data.sn != nil { + e.snapshot = data.sn + e.linkTarget = data.linkTarget + } else { + // intermediate directory, register as a child directory + if e.names == nil { + e.names = make(map[string]*MetaDirData) + } + if data.child != nil { + e.names[data.childFn] = data.child + } + } + entries[path] = e + + slashIdx := strings.LastIndex(path, "/") + if slashIdx >= 0 { + // add to parent dir, but without snapshot + mount(path[:slashIdx], mountData{childFn: path[slashIdx+1:], child: e}) + } + } + + // root directory + mount("", mountData{}) // insert pure directories; needed to get empty structure even if there // are no snapshots in these dirs for _, p := range d.pathTemplates { - for _, pattern := range []string{"%i", "%I", "%u", "%h", "%t", "%T"} { - p = strings.ReplaceAll(p, pattern, "") + p = staticPrefix(p) + if p != "" { + mount(path.Clean("/"+p), mountData{}) } - d.names[path.Clean(p)+"/"] = nil } latestTime := make(map[string]time.Time) @@ -154,25 +242,33 @@ func (d *SnapshotsDirStructure) makeDirs(snapshots restic.Snapshots) { for _, templ := range d.pathTemplates { paths, timeSuffix := pathsFromSn(templ, d.timeTemplate, sn) for _, p := range paths { - suffix := d.uniqueName(p, timeSuffix) - d.names[path.Clean(p+suffix)] = sn + if p != "" { + p = "/" + p + } + suffix := uniqueName(entries, p, timeSuffix) + mount(path.Clean(p+suffix), mountData{sn: sn}) if timeSuffix != "" { lt, ok := latestTime[p] if !ok || !sn.Time.Before(lt) { debug.Log("link (update) %v -> %v\n", p, suffix) - d.latest[p] = suffix + // inject symlink + mount(path.Clean(p+"/latest"), mountData{sn: sn, linkTarget: suffix}) latestTime[p] = sn.Time } } } } } + + d.entries = entries } const minSnapshotsReloadTime = 60 * time.Second // update snapshots if repository has changed func (d *SnapshotsDirStructure) updateSnapshots(ctx context.Context) error { + d.mutex.Lock() + defer d.mutex.Unlock() if time.Since(d.lastCheck) < minSnapshotsReloadTime { return nil } @@ -200,3 +296,14 @@ func (d *SnapshotsDirStructure) updateSnapshots(ctx context.Context) error { d.makeDirs(snapshots) return nil } + +func (d *SnapshotsDirStructure) UpdatePrefix(ctx context.Context, prefix string) (*MetaDirData, error) { + err := d.updateSnapshots(ctx) + if err != nil { + return nil, err + } + + d.mutex.Lock() + defer d.mutex.Unlock() + return d.entries[prefix], nil +} diff --git a/internal/fuse/snapshots_dirstruct_test.go b/internal/fuse/snapshots_dirstruct_test.go index 2e96d2093..3a6d397be 100644 --- a/internal/fuse/snapshots_dirstruct_test.go +++ b/internal/fuse/snapshots_dirstruct_test.go @@ -4,6 +4,7 @@ package fuse import ( + "strings" "testing" "time" @@ -89,69 +90,186 @@ func TestMakeDirs(t *testing.T) { expNames := make(map[string]*restic.Snapshot) expLatest := make(map[string]string) - // empty entries for dir structure - expNames["ids/"] = nil - expNames["snapshots/"] = nil - expNames["hosts/"] = nil - expNames["tags/"] = nil - expNames["users/"] = nil - expNames["longids/"] = nil - expNames["//"] = nil - // entries for sn0 - expNames["ids/00000000"] = sn0 - expNames["snapshots/2020/12/31"] = sn0 - expNames["hosts/host/2020/12/31"] = sn0 - expNames["tags/tag1/2020/12/31"] = sn0 - expNames["tags/tag2/2020/12/31"] = sn0 - expNames["users/user/2020/12/31"] = sn0 - expNames["longids/0000000012345678123456781234567812345678123456781234567812345678"] = sn0 - expNames["2020/12/31/host"] = sn0 - expNames["2020/12/31/00000000"] = sn0 + expNames["/ids/00000000"] = sn0 + expNames["/snapshots/2020/12/31"] = sn0 + expNames["/hosts/host/2020/12/31"] = sn0 + expNames["/tags/tag1/2020/12/31"] = sn0 + expNames["/tags/tag2/2020/12/31"] = sn0 + expNames["/users/user/2020/12/31"] = sn0 + expNames["/longids/0000000012345678123456781234567812345678123456781234567812345678"] = sn0 + expNames["/2020/12/31/host"] = sn0 + expNames["/2020/12/31/00000000"] = sn0 // entries for sn1 - expNames["ids/12345678"] = sn1 - expNames["snapshots/2021/01/01"] = sn1 - expNames["hosts/host/2021/01/01"] = sn1 - expNames["tags/tag1/2021/01/01"] = sn1 - expNames["tags/tag2/2021/01/01"] = sn1 - expNames["users/user/2021/01/01"] = sn1 - expNames["longids/1234567812345678123456781234567812345678123456781234567812345678"] = sn1 - expNames["2021/01/01/host"] = sn1 - expNames["2021/01/01/12345678"] = sn1 + expNames["/ids/12345678"] = sn1 + expNames["/snapshots/2021/01/01"] = sn1 + expNames["/hosts/host/2021/01/01"] = sn1 + expNames["/tags/tag1/2021/01/01"] = sn1 + expNames["/tags/tag2/2021/01/01"] = sn1 + expNames["/users/user/2021/01/01"] = sn1 + expNames["/longids/1234567812345678123456781234567812345678123456781234567812345678"] = sn1 + expNames["/2021/01/01/host"] = sn1 + expNames["/2021/01/01/12345678"] = sn1 // entries for sn2 - expNames["ids/87654321"] = sn2 - expNames["snapshots/2021/01/01-1"] = sn2 // sn1 and sn2 have same time string - expNames["hosts/host2/2021/01/01"] = sn2 - expNames["tags/tag2/2021/01/01-1"] = sn2 // sn1 and sn2 have same time string - expNames["tags/tag3/2021/01/01"] = sn2 - expNames["tags/tag4/2021/01/01"] = sn2 - expNames["users/user2/2021/01/01"] = sn2 - expNames["longids/8765432112345678123456781234567812345678123456781234567812345678"] = sn2 - expNames["2021/01/01/host2"] = sn2 - expNames["2021/01/01/87654321"] = sn2 + expNames["/ids/87654321"] = sn2 + expNames["/snapshots/2021/01/01-1"] = sn2 // sn1 and sn2 have same time string + expNames["/hosts/host2/2021/01/01"] = sn2 + expNames["/tags/tag2/2021/01/01-1"] = sn2 // sn1 and sn2 have same time string + expNames["/tags/tag3/2021/01/01"] = sn2 + expNames["/tags/tag4/2021/01/01"] = sn2 + expNames["/users/user2/2021/01/01"] = sn2 + expNames["/longids/8765432112345678123456781234567812345678123456781234567812345678"] = sn2 + expNames["/2021/01/01/host2"] = sn2 + expNames["/2021/01/01/87654321"] = sn2 // entries for sn3 - expNames["ids/aaaaaaaa"] = sn3 - expNames["snapshots/2021/01/01-2"] = sn3 // sn1 - sn3 have same time string - expNames["hosts/host/2021/01/01-1"] = sn3 // sn1 and sn3 have same time string - expNames["users/user2/2021/01/01-1"] = sn3 // sn2 and sn3 have same time string - expNames["longids/aaaaaaaa12345678123456781234567812345678123456781234567812345678"] = sn3 - expNames["2021/01/01/host-1"] = sn3 // sn1 and sn3 have same time string and identical host - expNames["2021/01/01/aaaaaaaa"] = sn3 + expNames["/ids/aaaaaaaa"] = sn3 + expNames["/snapshots/2021/01/01-2"] = sn3 // sn1 - sn3 have same time string + expNames["/hosts/host/2021/01/01-1"] = sn3 // sn1 and sn3 have same time string + expNames["/users/user2/2021/01/01-1"] = sn3 // sn2 and sn3 have same time string + expNames["/longids/aaaaaaaa12345678123456781234567812345678123456781234567812345678"] = sn3 + expNames["/2021/01/01/host-1"] = sn3 // sn1 and sn3 have same time string and identical host + expNames["/2021/01/01/aaaaaaaa"] = sn3 + + // intermediate directories + // sn0 + expNames["/ids"] = nil + expNames[""] = nil + expNames["/snapshots/2020/12"] = nil + expNames["/snapshots/2020"] = nil + expNames["/snapshots"] = nil + expNames["/hosts/host/2020/12"] = nil + expNames["/hosts/host/2020"] = nil + expNames["/hosts/host"] = nil + expNames["/hosts"] = nil + expNames["/tags/tag1/2020/12"] = nil + expNames["/tags/tag1/2020"] = nil + expNames["/tags/tag1"] = nil + expNames["/tags"] = nil + expNames["/tags/tag2/2020/12"] = nil + expNames["/tags/tag2/2020"] = nil + expNames["/tags/tag2"] = nil + expNames["/users/user/2020/12"] = nil + expNames["/users/user/2020"] = nil + expNames["/users/user"] = nil + expNames["/users"] = nil + expNames["/longids"] = nil + expNames["/2020/12/31"] = nil + expNames["/2020/12"] = nil + expNames["/2020"] = nil + + // sn1 + expNames["/snapshots/2021/01"] = nil + expNames["/snapshots/2021"] = nil + expNames["/hosts/host/2021/01"] = nil + expNames["/hosts/host/2021"] = nil + expNames["/tags/tag1/2021/01"] = nil + expNames["/tags/tag1/2021"] = nil + expNames["/tags/tag2/2021/01"] = nil + expNames["/tags/tag2/2021"] = nil + expNames["/users/user/2021/01"] = nil + expNames["/users/user/2021"] = nil + expNames["/2021/01/01"] = nil + expNames["/2021/01"] = nil + expNames["/2021"] = nil + + // sn2 + expNames["/hosts/host2/2021/01"] = nil + expNames["/hosts/host2/2021"] = nil + expNames["/hosts/host2"] = nil + expNames["/tags/tag3/2021/01"] = nil + expNames["/tags/tag3/2021"] = nil + expNames["/tags/tag3"] = nil + expNames["/tags/tag4/2021/01"] = nil + expNames["/tags/tag4/2021"] = nil + expNames["/tags/tag4"] = nil + expNames["/users/user2/2021/01"] = nil + expNames["/users/user2/2021"] = nil + expNames["/users/user2"] = nil + + // target snapshots for links + expNames["/snapshots/latest"] = sn3 // sn1 - sn3 have same time string + expNames["/hosts/host/latest"] = sn3 + expNames["/hosts/host2/latest"] = sn2 + expNames["/tags/tag1/latest"] = sn1 + expNames["/tags/tag2/latest"] = sn2 // sn1 and sn2 have same time string + expNames["/tags/tag3/latest"] = sn2 + expNames["/tags/tag4/latest"] = sn2 + expNames["/users/user/latest"] = sn1 + expNames["/users/user2/latest"] = sn3 // sn2 and sn3 have same time string // latest links - expLatest["snapshots/"] = "2021/01/01-2" // sn1 - sn3 have same time string - expLatest["hosts/host/"] = "2021/01/01-1" - expLatest["hosts/host2/"] = "2021/01/01" - expLatest["tags/tag1/"] = "2021/01/01" - expLatest["tags/tag2/"] = "2021/01/01-1" // sn1 and sn2 have same time string - expLatest["tags/tag3/"] = "2021/01/01" - expLatest["tags/tag4/"] = "2021/01/01" - expLatest["users/user/"] = "2021/01/01" - expLatest["users/user2/"] = "2021/01/01-1" // sn2 and sn3 have same time string + expLatest["/snapshots/latest"] = "2021/01/01-2" // sn1 - sn3 have same time string + expLatest["/hosts/host/latest"] = "2021/01/01-1" + expLatest["/hosts/host2/latest"] = "2021/01/01" + expLatest["/tags/tag1/latest"] = "2021/01/01" + expLatest["/tags/tag2/latest"] = "2021/01/01-1" // sn1 and sn2 have same time string + expLatest["/tags/tag3/latest"] = "2021/01/01" + expLatest["/tags/tag4/latest"] = "2021/01/01" + expLatest["/users/user/latest"] = "2021/01/01" + expLatest["/users/user2/latest"] = "2021/01/01-1" // sn2 and sn3 have same time string - test.Equals(t, expNames, sds.names) - test.Equals(t, expLatest, sds.latest) + verifyEntries(t, expNames, expLatest, sds.entries) +} + +func verifyEntries(t *testing.T, expNames map[string]*restic.Snapshot, expLatest map[string]string, entries map[string]*MetaDirData) { + actNames := make(map[string]*restic.Snapshot) + actLatest := make(map[string]string) + for path, entry := range entries { + actNames[path] = entry.snapshot + if entry.linkTarget != "" { + actLatest[path] = entry.linkTarget + } + } + + test.Equals(t, expNames, actNames) + test.Equals(t, expLatest, actLatest) + + // verify tree integrity + for path, entry := range entries { + // check that all children are actually contained in entry.names + for otherPath := range entries { + if strings.HasPrefix(otherPath, path+"/") { + sub := otherPath[len(path)+1:] + // remaining path does not contain a directory + test.Assert(t, strings.Contains(sub, "/") || (entry.names != nil && entry.names[sub] != nil), "missing entry %v in %v", sub, path) + } + } + if entry.names == nil { + continue + } + // child entries reference the correct MetaDirData + for elem, subentry := range entry.names { + test.Equals(t, entries[path+"/"+elem], subentry) + } + } +} + +func TestMakeEmptyDirs(t *testing.T) { + pathTemplates := []string{"ids/%i", "snapshots/%T", "hosts/%h/%T", + "tags/%t/%T", "users/%u/%T", "longids/id-%I", "%T/%h", "%T/%i", "id-%i", + } + timeTemplate := "2006/01/02" + + sds := &SnapshotsDirStructure{ + pathTemplates: pathTemplates, + timeTemplate: timeTemplate, + } + sds.makeDirs(restic.Snapshots{}) + + expNames := make(map[string]*restic.Snapshot) + expLatest := make(map[string]string) + + // empty entries for dir structure + expNames["/ids"] = nil + expNames["/snapshots"] = nil + expNames["/hosts"] = nil + expNames["/tags"] = nil + expNames["/users"] = nil + expNames["/longids"] = nil + expNames[""] = nil + + verifyEntries(t, expNames, expLatest, sds.entries) } From 83b4c50ee3c336bf7c188bd1ecf799f9d88e0777 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Fri, 5 Aug 2022 23:46:13 +0200 Subject: [PATCH 7/7] Mention --snapshot-template and --time-template in changelog --- changelog/unreleased/issue-2907 | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/changelog/unreleased/issue-2907 b/changelog/unreleased/issue-2907 index 5d5c9d8e8..ca21d0ef6 100644 --- a/changelog/unreleased/issue-2907 +++ b/changelog/unreleased/issue-2907 @@ -1,7 +1,10 @@ Enhancement: Make snapshot directory structure of mount command custimizable -We've added the possibility to customize the snapshot directory structure of the mount command. -This includes using subdirectories which is now also possible within time template and tags. +We've added the possibility to customize the snapshot directory structure of +the mount command using templates passed to `--snapshot-template`. The +formatting of the time for a snapshot is now controlled using `--time-template` +and supports subdirectories to for example group snapshots by year. Please +refer to the help output of the `mount` command for further details. https://github.com/restic/restic/issues/2907 https://github.com/restic/restic/pull/2913