From caa17988a3cfcefed27798c4856c32c92168f4c1 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 30 Jul 2022 20:45:51 +0200 Subject: [PATCH] fuse: Redesign snapshot dirstruct Cleanly separate the directory presentation and the snapshot directory structure. SnapshotsDir now translates the dirStruct into a format usable by the fuse library and contains only minimal special case rules. All decisions have moved into SnapshotsDirStructure which now creates a fully preassembled tree data structure. --- internal/fuse/snapshots_dir.go | 79 +++----- internal/fuse/snapshots_dirstruct.go | 153 ++++++++++++--- internal/fuse/snapshots_dirstruct_test.go | 228 ++++++++++++++++------ 3 files changed, 327 insertions(+), 133 deletions(-) diff --git a/internal/fuse/snapshots_dir.go b/internal/fuse/snapshots_dir.go index 2924919da..e5b18b20f 100644 --- a/internal/fuse/snapshots_dir.go +++ b/internal/fuse/snapshots_dir.go @@ -6,7 +6,6 @@ package fuse import ( "context" "os" - "strings" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/restic" @@ -16,8 +15,7 @@ import ( ) // SnapshotsDir is a actual fuse directory generated from SnapshotsDirStructure -// It uses the saved prefix to filter out the relevant subtrees or entries -// from SnapshotsDirStructure.names and .latest, respectively. +// It uses the saved prefix to select the corresponding MetaDirData. type SnapshotsDir struct { root *Root inode uint64 @@ -56,9 +54,11 @@ func (d *SnapshotsDir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) { debug.Log("ReadDirAll()") // update snapshots - err := d.dirStruct.updateSnapshots(ctx) + meta, err := d.dirStruct.UpdatePrefix(ctx, d.prefix) if err != nil { return nil, err + } else if meta == nil { + return nil, fuse.ENOENT } items := []fuse.Dirent{ @@ -74,35 +74,16 @@ func (d *SnapshotsDir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) { }, } - // map to ensure that all names are only listed once - hasName := make(map[string]struct{}) - - for name := range d.dirStruct.names { - if !strings.HasPrefix(name, d.prefix) { - continue - } - shortname := strings.Split(name[len(d.prefix):], "/")[0] - if shortname == "" { - continue - } - if _, ok := hasName[shortname]; ok { - continue - } - hasName[shortname] = struct{}{} - items = append(items, fuse.Dirent{ - Inode: fs.GenerateDynamicInode(d.inode, shortname), - Name: shortname, + for name, entry := range meta.names { + d := fuse.Dirent{ + Inode: fs.GenerateDynamicInode(d.inode, name), + Name: name, Type: fuse.DT_Dir, - }) - } - - // Latest - if _, ok := d.dirStruct.latest[d.prefix]; ok { - items = append(items, fuse.Dirent{ - Inode: fs.GenerateDynamicInode(d.inode, "latest"), - Name: "latest", - Type: fuse.DT_Link, - }) + } + if entry.linkTarget != "" { + d.Type = fuse.DT_Link + } + items = append(items, d) } return items, nil @@ -112,33 +93,21 @@ func (d *SnapshotsDir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) { func (d *SnapshotsDir) Lookup(ctx context.Context, name string) (fs.Node, error) { debug.Log("Lookup(%s)", name) - err := d.dirStruct.updateSnapshots(ctx) + meta, err := d.dirStruct.UpdatePrefix(ctx, d.prefix) if err != nil { return nil, err + } else if meta == nil { + return nil, fuse.ENOENT } - fullname := d.prefix + name - - // check if this is already a complete snapshot path - sn := d.dirStruct.names[fullname] - if sn != nil { - return newDirFromSnapshot(ctx, d.root, fs.GenerateDynamicInode(d.inode, name), sn) - } - - // handle latest case - if name == "latest" { - link := d.dirStruct.latest[d.prefix] - sn := d.dirStruct.names[d.prefix+link] - if sn != nil { - return newSnapshotLink(ctx, d.root, fs.GenerateDynamicInode(d.inode, name), link, sn) - } - } - - // check if this is a valid subdir - fullname = fullname + "/" - for name := range d.dirStruct.names { - if strings.HasPrefix(name, fullname) { - return NewSnapshotsDir(d.root, fs.GenerateDynamicInode(d.inode, name), d.dirStruct, fullname), nil + entry := meta.names[name] + if entry != nil { + if entry.linkTarget != "" { + return newSnapshotLink(ctx, d.root, fs.GenerateDynamicInode(d.inode, name), entry.linkTarget, entry.snapshot) + } else if entry.snapshot != nil { + return newDirFromSnapshot(ctx, d.root, fs.GenerateDynamicInode(d.inode, name), entry.snapshot) + } else { + return NewSnapshotsDir(d.root, fs.GenerateDynamicInode(d.inode, name), d.dirStruct, d.prefix+"/"+name), nil } } diff --git a/internal/fuse/snapshots_dirstruct.go b/internal/fuse/snapshots_dirstruct.go index 9bf2f5a7e..18831dcb1 100644 --- a/internal/fuse/snapshots_dirstruct.go +++ b/internal/fuse/snapshots_dirstruct.go @@ -9,12 +9,21 @@ import ( "path" "sort" "strings" + "sync" "time" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/restic" ) +type MetaDirData struct { + // set if this is a symlink or a snapshot mount point + linkTarget string + snapshot *restic.Snapshot + // names is set if this is a pseudo directory + names map[string]*MetaDirData +} + // SnapshotsDirStructure contains the directory structure for snapshots. // It uses a paths and time template to generate a map of pathnames // pointing to the actual snapshots. For templates that end with a time, @@ -24,8 +33,12 @@ type SnapshotsDirStructure struct { pathTemplates []string timeTemplate string - names map[string]*restic.Snapshot - latest map[string]string + mutex sync.Mutex + // "" is the root path, subdirectory paths are assembled as parent+"/"+childFn + // thus all subdirectories are prefixed with a slash as the root is "" + // that way we don't need path processing special cases when using the entries tree + entries map[string]*MetaDirData + snCount int lastCheck time.Time } @@ -40,19 +53,6 @@ func NewSnapshotsDirStructure(root *Root, pathTemplates []string, timeTemplate s } } -// uniqueName returns a unique name to be used for prefix+name. -// It appends -number to make the name unique. -func (d *SnapshotsDirStructure) uniqueName(prefix, name string) (newname string) { - newname = name - for i := 1; ; i++ { - if _, ok := d.names[prefix+newname]; !ok { - break - } - newname = fmt.Sprintf("%s-%d", name, i) - } - return newname -} - // pathsFromSn generates the paths from pathTemplate and timeTemplate // where the variables are replaced by the snapshot data. // The time is given as suffix if the pathTemplate ends with "%T". @@ -90,6 +90,9 @@ func pathsFromSn(pathTemplate string, timeTemplate string, sn *restic.Snapshot) continue case 't': + if len(sn.Tags) == 0 { + return nil, "" + } if len(sn.Tags) != 1 { // needs special treatment: Rebuild the string builders newout := make([]strings.Builder, len(out)*len(sn.Tags)) @@ -114,6 +117,9 @@ func pathsFromSn(pathTemplate string, timeTemplate string, sn *restic.Snapshot) case 'h': repl = sn.Hostname + + default: + repl = string(c) } // write replacement string to all string builders @@ -133,20 +139,102 @@ func pathsFromSn(pathTemplate string, timeTemplate string, sn *restic.Snapshot) return paths, timeSuffix } +// determine static path prefix +func staticPrefix(pathTemplate string) (prefix string) { + inVerb := false + patternStart := -1 +outer: + for i, c := range pathTemplate { + if !inVerb { + if c == '%' { + inVerb = true + } + continue + } + inVerb = false + switch c { + case 'i', 'I', 'u', 'h', 't', 'T': + patternStart = i + break outer + } + } + if patternStart < 0 { + // ignore patterns without template variable + return "" + } + + p := pathTemplate[:patternStart] + idx := strings.LastIndex(p, "/") + if idx < 0 { + return "" + } + return p[:idx] +} + +// uniqueName returns a unique name to be used for prefix+name. +// It appends -number to make the name unique. +func uniqueName(entries map[string]*MetaDirData, prefix, name string) string { + newname := name + for i := 1; ; i++ { + if _, ok := entries[prefix+newname]; !ok { + break + } + newname = fmt.Sprintf("%s-%d", name, i) + } + return newname +} + // makeDirs inserts all paths generated from pathTemplates and // TimeTemplate for all given snapshots into d.names. // Also adds d.latest links if "%T" is at end of a path template func (d *SnapshotsDirStructure) makeDirs(snapshots restic.Snapshots) { - d.names = make(map[string]*restic.Snapshot) - d.latest = make(map[string]string) + entries := make(map[string]*MetaDirData) + + type mountData struct { + sn *restic.Snapshot + linkTarget string // if linkTarget!= "", this is a symlink + childFn string + child *MetaDirData + } + + // recursively build tree structure + var mount func(path string, data mountData) + mount = func(path string, data mountData) { + e := entries[path] + if e == nil { + e = &MetaDirData{} + } + if data.sn != nil { + e.snapshot = data.sn + e.linkTarget = data.linkTarget + } else { + // intermediate directory, register as a child directory + if e.names == nil { + e.names = make(map[string]*MetaDirData) + } + if data.child != nil { + e.names[data.childFn] = data.child + } + } + entries[path] = e + + slashIdx := strings.LastIndex(path, "/") + if slashIdx >= 0 { + // add to parent dir, but without snapshot + mount(path[:slashIdx], mountData{childFn: path[slashIdx+1:], child: e}) + } + } + + // root directory + mount("", mountData{}) // insert pure directories; needed to get empty structure even if there // are no snapshots in these dirs for _, p := range d.pathTemplates { - for _, pattern := range []string{"%i", "%I", "%u", "%h", "%t", "%T"} { - p = strings.ReplaceAll(p, pattern, "") + p = staticPrefix(p) + if p != "" { + mount(path.Clean("/"+p), mountData{}) } - d.names[path.Clean(p)+"/"] = nil } latestTime := make(map[string]time.Time) @@ -154,25 +242,33 @@ func (d *SnapshotsDirStructure) makeDirs(snapshots restic.Snapshots) { for _, templ := range d.pathTemplates { paths, timeSuffix := pathsFromSn(templ, d.timeTemplate, sn) for _, p := range paths { - suffix := d.uniqueName(p, timeSuffix) - d.names[path.Clean(p+suffix)] = sn + if p != "" { + p = "/" + p + } + suffix := uniqueName(entries, p, timeSuffix) + mount(path.Clean(p+suffix), mountData{sn: sn}) if timeSuffix != "" { lt, ok := latestTime[p] if !ok || !sn.Time.Before(lt) { debug.Log("link (update) %v -> %v\n", p, suffix) - d.latest[p] = suffix + // inject symlink + mount(path.Clean(p+"/latest"), mountData{sn: sn, linkTarget: suffix}) latestTime[p] = sn.Time } } } } } + + d.entries = entries } const minSnapshotsReloadTime = 60 * time.Second // update snapshots if repository has changed func (d *SnapshotsDirStructure) updateSnapshots(ctx context.Context) error { + d.mutex.Lock() + defer d.mutex.Unlock() if time.Since(d.lastCheck) < minSnapshotsReloadTime { return nil } @@ -200,3 +296,14 @@ func (d *SnapshotsDirStructure) updateSnapshots(ctx context.Context) error { d.makeDirs(snapshots) return nil } + +func (d *SnapshotsDirStructure) UpdatePrefix(ctx context.Context, prefix string) (*MetaDirData, error) { + err := d.updateSnapshots(ctx) + if err != nil { + return nil, err + } + + d.mutex.Lock() + defer d.mutex.Unlock() + return d.entries[prefix], nil +} diff --git a/internal/fuse/snapshots_dirstruct_test.go b/internal/fuse/snapshots_dirstruct_test.go index 2e96d2093..3a6d397be 100644 --- a/internal/fuse/snapshots_dirstruct_test.go +++ b/internal/fuse/snapshots_dirstruct_test.go @@ -4,6 +4,7 @@ package fuse import ( + "strings" "testing" "time" @@ -89,69 +90,186 @@ func TestMakeDirs(t *testing.T) { expNames := make(map[string]*restic.Snapshot) expLatest := make(map[string]string) - // empty entries for dir structure - expNames["ids/"] = nil - expNames["snapshots/"] = nil - expNames["hosts/"] = nil - expNames["tags/"] = nil - expNames["users/"] = nil - expNames["longids/"] = nil - expNames["//"] = nil - // entries for sn0 - expNames["ids/00000000"] = sn0 - expNames["snapshots/2020/12/31"] = sn0 - expNames["hosts/host/2020/12/31"] = sn0 - expNames["tags/tag1/2020/12/31"] = sn0 - expNames["tags/tag2/2020/12/31"] = sn0 - expNames["users/user/2020/12/31"] = sn0 - expNames["longids/0000000012345678123456781234567812345678123456781234567812345678"] = sn0 - expNames["2020/12/31/host"] = sn0 - expNames["2020/12/31/00000000"] = sn0 + expNames["/ids/00000000"] = sn0 + expNames["/snapshots/2020/12/31"] = sn0 + expNames["/hosts/host/2020/12/31"] = sn0 + expNames["/tags/tag1/2020/12/31"] = sn0 + expNames["/tags/tag2/2020/12/31"] = sn0 + expNames["/users/user/2020/12/31"] = sn0 + expNames["/longids/0000000012345678123456781234567812345678123456781234567812345678"] = sn0 + expNames["/2020/12/31/host"] = sn0 + expNames["/2020/12/31/00000000"] = sn0 // entries for sn1 - expNames["ids/12345678"] = sn1 - expNames["snapshots/2021/01/01"] = sn1 - expNames["hosts/host/2021/01/01"] = sn1 - expNames["tags/tag1/2021/01/01"] = sn1 - expNames["tags/tag2/2021/01/01"] = sn1 - expNames["users/user/2021/01/01"] = sn1 - expNames["longids/1234567812345678123456781234567812345678123456781234567812345678"] = sn1 - expNames["2021/01/01/host"] = sn1 - expNames["2021/01/01/12345678"] = sn1 + expNames["/ids/12345678"] = sn1 + expNames["/snapshots/2021/01/01"] = sn1 + expNames["/hosts/host/2021/01/01"] = sn1 + expNames["/tags/tag1/2021/01/01"] = sn1 + expNames["/tags/tag2/2021/01/01"] = sn1 + expNames["/users/user/2021/01/01"] = sn1 + expNames["/longids/1234567812345678123456781234567812345678123456781234567812345678"] = sn1 + expNames["/2021/01/01/host"] = sn1 + expNames["/2021/01/01/12345678"] = sn1 // entries for sn2 - expNames["ids/87654321"] = sn2 - expNames["snapshots/2021/01/01-1"] = sn2 // sn1 and sn2 have same time string - expNames["hosts/host2/2021/01/01"] = sn2 - expNames["tags/tag2/2021/01/01-1"] = sn2 // sn1 and sn2 have same time string - expNames["tags/tag3/2021/01/01"] = sn2 - expNames["tags/tag4/2021/01/01"] = sn2 - expNames["users/user2/2021/01/01"] = sn2 - expNames["longids/8765432112345678123456781234567812345678123456781234567812345678"] = sn2 - expNames["2021/01/01/host2"] = sn2 - expNames["2021/01/01/87654321"] = sn2 + expNames["/ids/87654321"] = sn2 + expNames["/snapshots/2021/01/01-1"] = sn2 // sn1 and sn2 have same time string + expNames["/hosts/host2/2021/01/01"] = sn2 + expNames["/tags/tag2/2021/01/01-1"] = sn2 // sn1 and sn2 have same time string + expNames["/tags/tag3/2021/01/01"] = sn2 + expNames["/tags/tag4/2021/01/01"] = sn2 + expNames["/users/user2/2021/01/01"] = sn2 + expNames["/longids/8765432112345678123456781234567812345678123456781234567812345678"] = sn2 + expNames["/2021/01/01/host2"] = sn2 + expNames["/2021/01/01/87654321"] = sn2 // entries for sn3 - expNames["ids/aaaaaaaa"] = sn3 - expNames["snapshots/2021/01/01-2"] = sn3 // sn1 - sn3 have same time string - expNames["hosts/host/2021/01/01-1"] = sn3 // sn1 and sn3 have same time string - expNames["users/user2/2021/01/01-1"] = sn3 // sn2 and sn3 have same time string - expNames["longids/aaaaaaaa12345678123456781234567812345678123456781234567812345678"] = sn3 - expNames["2021/01/01/host-1"] = sn3 // sn1 and sn3 have same time string and identical host - expNames["2021/01/01/aaaaaaaa"] = sn3 + expNames["/ids/aaaaaaaa"] = sn3 + expNames["/snapshots/2021/01/01-2"] = sn3 // sn1 - sn3 have same time string + expNames["/hosts/host/2021/01/01-1"] = sn3 // sn1 and sn3 have same time string + expNames["/users/user2/2021/01/01-1"] = sn3 // sn2 and sn3 have same time string + expNames["/longids/aaaaaaaa12345678123456781234567812345678123456781234567812345678"] = sn3 + expNames["/2021/01/01/host-1"] = sn3 // sn1 and sn3 have same time string and identical host + expNames["/2021/01/01/aaaaaaaa"] = sn3 + + // intermediate directories + // sn0 + expNames["/ids"] = nil + expNames[""] = nil + expNames["/snapshots/2020/12"] = nil + expNames["/snapshots/2020"] = nil + expNames["/snapshots"] = nil + expNames["/hosts/host/2020/12"] = nil + expNames["/hosts/host/2020"] = nil + expNames["/hosts/host"] = nil + expNames["/hosts"] = nil + expNames["/tags/tag1/2020/12"] = nil + expNames["/tags/tag1/2020"] = nil + expNames["/tags/tag1"] = nil + expNames["/tags"] = nil + expNames["/tags/tag2/2020/12"] = nil + expNames["/tags/tag2/2020"] = nil + expNames["/tags/tag2"] = nil + expNames["/users/user/2020/12"] = nil + expNames["/users/user/2020"] = nil + expNames["/users/user"] = nil + expNames["/users"] = nil + expNames["/longids"] = nil + expNames["/2020/12/31"] = nil + expNames["/2020/12"] = nil + expNames["/2020"] = nil + + // sn1 + expNames["/snapshots/2021/01"] = nil + expNames["/snapshots/2021"] = nil + expNames["/hosts/host/2021/01"] = nil + expNames["/hosts/host/2021"] = nil + expNames["/tags/tag1/2021/01"] = nil + expNames["/tags/tag1/2021"] = nil + expNames["/tags/tag2/2021/01"] = nil + expNames["/tags/tag2/2021"] = nil + expNames["/users/user/2021/01"] = nil + expNames["/users/user/2021"] = nil + expNames["/2021/01/01"] = nil + expNames["/2021/01"] = nil + expNames["/2021"] = nil + + // sn2 + expNames["/hosts/host2/2021/01"] = nil + expNames["/hosts/host2/2021"] = nil + expNames["/hosts/host2"] = nil + expNames["/tags/tag3/2021/01"] = nil + expNames["/tags/tag3/2021"] = nil + expNames["/tags/tag3"] = nil + expNames["/tags/tag4/2021/01"] = nil + expNames["/tags/tag4/2021"] = nil + expNames["/tags/tag4"] = nil + expNames["/users/user2/2021/01"] = nil + expNames["/users/user2/2021"] = nil + expNames["/users/user2"] = nil + + // target snapshots for links + expNames["/snapshots/latest"] = sn3 // sn1 - sn3 have same time string + expNames["/hosts/host/latest"] = sn3 + expNames["/hosts/host2/latest"] = sn2 + expNames["/tags/tag1/latest"] = sn1 + expNames["/tags/tag2/latest"] = sn2 // sn1 and sn2 have same time string + expNames["/tags/tag3/latest"] = sn2 + expNames["/tags/tag4/latest"] = sn2 + expNames["/users/user/latest"] = sn1 + expNames["/users/user2/latest"] = sn3 // sn2 and sn3 have same time string // latest links - expLatest["snapshots/"] = "2021/01/01-2" // sn1 - sn3 have same time string - expLatest["hosts/host/"] = "2021/01/01-1" - expLatest["hosts/host2/"] = "2021/01/01" - expLatest["tags/tag1/"] = "2021/01/01" - expLatest["tags/tag2/"] = "2021/01/01-1" // sn1 and sn2 have same time string - expLatest["tags/tag3/"] = "2021/01/01" - expLatest["tags/tag4/"] = "2021/01/01" - expLatest["users/user/"] = "2021/01/01" - expLatest["users/user2/"] = "2021/01/01-1" // sn2 and sn3 have same time string + expLatest["/snapshots/latest"] = "2021/01/01-2" // sn1 - sn3 have same time string + expLatest["/hosts/host/latest"] = "2021/01/01-1" + expLatest["/hosts/host2/latest"] = "2021/01/01" + expLatest["/tags/tag1/latest"] = "2021/01/01" + expLatest["/tags/tag2/latest"] = "2021/01/01-1" // sn1 and sn2 have same time string + expLatest["/tags/tag3/latest"] = "2021/01/01" + expLatest["/tags/tag4/latest"] = "2021/01/01" + expLatest["/users/user/latest"] = "2021/01/01" + expLatest["/users/user2/latest"] = "2021/01/01-1" // sn2 and sn3 have same time string - test.Equals(t, expNames, sds.names) - test.Equals(t, expLatest, sds.latest) + verifyEntries(t, expNames, expLatest, sds.entries) +} + +func verifyEntries(t *testing.T, expNames map[string]*restic.Snapshot, expLatest map[string]string, entries map[string]*MetaDirData) { + actNames := make(map[string]*restic.Snapshot) + actLatest := make(map[string]string) + for path, entry := range entries { + actNames[path] = entry.snapshot + if entry.linkTarget != "" { + actLatest[path] = entry.linkTarget + } + } + + test.Equals(t, expNames, actNames) + test.Equals(t, expLatest, actLatest) + + // verify tree integrity + for path, entry := range entries { + // check that all children are actually contained in entry.names + for otherPath := range entries { + if strings.HasPrefix(otherPath, path+"/") { + sub := otherPath[len(path)+1:] + // remaining path does not contain a directory + test.Assert(t, strings.Contains(sub, "/") || (entry.names != nil && entry.names[sub] != nil), "missing entry %v in %v", sub, path) + } + } + if entry.names == nil { + continue + } + // child entries reference the correct MetaDirData + for elem, subentry := range entry.names { + test.Equals(t, entries[path+"/"+elem], subentry) + } + } +} + +func TestMakeEmptyDirs(t *testing.T) { + pathTemplates := []string{"ids/%i", "snapshots/%T", "hosts/%h/%T", + "tags/%t/%T", "users/%u/%T", "longids/id-%I", "%T/%h", "%T/%i", "id-%i", + } + timeTemplate := "2006/01/02" + + sds := &SnapshotsDirStructure{ + pathTemplates: pathTemplates, + timeTemplate: timeTemplate, + } + sds.makeDirs(restic.Snapshots{}) + + expNames := make(map[string]*restic.Snapshot) + expLatest := make(map[string]string) + + // empty entries for dir structure + expNames["/ids"] = nil + expNames["/snapshots"] = nil + expNames["/hosts"] = nil + expNames["/tags"] = nil + expNames["/users"] = nil + expNames["/longids"] = nil + expNames[""] = nil + + verifyEntries(t, expNames, expLatest, sds.entries) }