octoleo/restic
2
2
Fork 0
mirror of https://github.com/octoleo/restic.git synced 2024-11-26 06:46:34 +00:00
Code Issues Releases Wiki Activity

fuse: mount and backup in parallel (#1330)

Browse Source
This commit is contained in:
Tobias Klein 2017-10-05 14:55:55 +02:00
parent a5c003acb0
commit 02200acad0
3 changed files with 378 additions and 101 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
cmd/restic/cmd_mount.go
View File

@ -67,6 +67,12 @@ func mount(opts MountOptions, gopts GlobalOptions, mountpoint string) error {
return err
}
lock, err := lockRepo(repo)
defer unlockRepo(lock)
if err != nil {
return err
}
err = repo.LoadIndex(context.TODO())
if err != nil {
return err

52
internal/fuse/root.go
View File

@ -41,22 +41,18 @@ const rootInode = 1
func NewRoot(ctx context.Context, repo restic.Repository, cfg Config) (*Root, error) {
debug.Log("NewRoot(), config %v", cfg)
snapshots := restic.FindFilteredSnapshots(ctx, repo, cfg.Host, cfg.Tags, cfg.Paths)
debug.Log("found %d matching snapshots", len(snapshots))
root := &Root{
repo: repo,
inode: rootInode,
cfg: cfg,
snapshots: snapshots,
blobSizeCache: NewBlobSizeCache(ctx, repo.Index()),
}
entries := map[string]fs.Node{
"snapshots": NewSnapshotsDir(root, fs.GenerateDynamicInode(root.inode, "snapshots"), snapshots),
"tags": NewTagsDir(root, fs.GenerateDynamicInode(root.inode, "tags"), snapshots),
"hosts": NewHostsDir(root, fs.GenerateDynamicInode(root.inode, "hosts"), snapshots),
"ids": NewSnapshotsIDSDir(root, fs.GenerateDynamicInode(root.inode, "ids"), snapshots),
"snapshots": NewSnapshotsDir(root, fs.GenerateDynamicInode(root.inode, "snapshots"), "", ""),
"tags": NewTagsDir(root, fs.GenerateDynamicInode(root.inode, "tags")),
"hosts": NewHostsDir(root, fs.GenerateDynamicInode(root.inode, "hosts")),
"ids": NewSnapshotsIDSDir(root, fs.GenerateDynamicInode(root.inode, "ids")),
}
root.MetaDir = NewMetaDir(root, rootInode, entries)
@ -64,46 +60,6 @@ func NewRoot(ctx context.Context, repo restic.Repository, cfg Config) (*Root, er
return root, nil
}
// NewTagsDir returns a new directory containing entries, which in turn contains
// snapshots with this tag set.
func NewTagsDir(root *Root, inode uint64, snapshots restic.Snapshots) fs.Node {
tags := make(map[string]restic.Snapshots)
for _, sn := range snapshots {
for _, tag := range sn.Tags {
tags[tag] = append(tags[tag], sn)
}
}
debug.Log("create tags dir with %d tags, inode %d", len(tags), inode)
entries := make(map[string]fs.Node)
for name, snapshots := range tags {
debug.Log(" tag %v has %v snapshots", name, len(snapshots))
entries[name] = NewSnapshotsDir(root, fs.GenerateDynamicInode(inode, name), snapshots)
}
return NewMetaDir(root, inode, entries)
}
// NewHostsDir returns a new directory containing hostnames, which in
// turn contains snapshots of a single host each.
func NewHostsDir(root *Root, inode uint64, snapshots restic.Snapshots) fs.Node {
hosts := make(map[string]restic.Snapshots)
for _, sn := range snapshots {
hosts[sn.Hostname] = append(hosts[sn.Hostname], sn)
}
debug.Log("create hosts dir with %d snapshots, inode %d", len(hosts), inode)
entries := make(map[string]fs.Node)
for name, snapshots := range hosts {
debug.Log(" host %v has %v snapshots", name, len(snapshots))
entries[name] = NewSnapshotsDir(root, fs.GenerateDynamicInode(inode, name), snapshots)
}
return NewMetaDir(root, inode, entries)
}
// Root is just there to satisfy fs.Root, it returns itself.
func (r *Root) Root() (fs.Node, error) {
debug.Log("Root()")

421
internal/fuse/snapshots_dir.go
View File

@ -17,70 +17,128 @@ import (
"bazil.org/fuse/fs"
)
// SnapshotsDir is a fuse directory which contains snapshots.
// SnapshotsDir is a fuse directory which contains snapshots named by timestamp.
type SnapshotsDir struct {
inode uint64
root *Root
snapshots restic.Snapshots
names map[string]*restic.Snapshot
latest string
inode uint64
root *Root
names map[string]*restic.Snapshot
latest string
tag string
host string
}
// SnapshotsIDSDir is a fuse directory which contains snapshots named by ids.
type SnapshotsIDSDir struct {
inode uint64
root *Root
names map[string]*restic.Snapshot
}
// HostsDir is a fuse directory which contains hosts.
type HostsDir struct {
inode uint64
root *Root
hosts map[string]bool
}
// TagsDir is a fuse directory which contains tags.
type TagsDir struct {
inode uint64
root *Root
tags map[string]bool
}
// SnapshotLink
type snapshotLink struct {
root *Root
inode uint64
target string
snapshot *restic.Snapshot
}
// ensure that *SnapshotsDir implements these interfaces
var _ = fs.HandleReadDirAller(&SnapshotsDir{})
var _ = fs.NodeStringLookuper(&SnapshotsDir{})
var _ = fs.HandleReadDirAller(&SnapshotsIDSDir{})
var _ = fs.NodeStringLookuper(&SnapshotsIDSDir{})
var _ = fs.HandleReadDirAller(&TagsDir{})
var _ = fs.NodeStringLookuper(&TagsDir{})
var _ = fs.HandleReadDirAller(&HostsDir{})
var _ = fs.NodeStringLookuper(&HostsDir{})
var _ = fs.NodeReadlinker(&snapshotLink{})
// NewSnapshotsDir returns a new directory containing snapshots.
func NewSnapshotsDir(root *Root, inode uint64, snapshots restic.Snapshots) *SnapshotsDir {
debug.Log("create snapshots dir with %d snapshots, inode %d", len(snapshots), inode)
d := &SnapshotsDir{
root: root,
inode: inode,
snapshots: snapshots,
names: make(map[string]*restic.Snapshot, len(snapshots)),
// read tag names from the current repository-state.
func getTagNames(d *TagsDir) {
d.tags = make(map[string]bool, len(d.root.snapshots))
for _, snapshot := range d.root.snapshots {
for _, tag := range snapshot.Tags {
d.tags[tag] = true
}
}
}
// Track latest Snapshot
var latestTime time.Time
d.latest = ""
for _, sn := range snapshots {
name := sn.Time.Format(time.RFC3339)
if d.latest == "" || !sn.Time.Before(latestTime) {
latestTime = sn.Time
d.latest = name
}
for i := 1; ; i++ {
if _, ok := d.names[name]; !ok {
break
}
name = fmt.Sprintf("%s-%d", sn.Time.Format(time.RFC3339), i)
}
// read host names from the current repository-state.
func getHostsNames(d *HostsDir) {
d.hosts = make(map[string]bool, len(d.root.snapshots))
for _, snapshot := range d.root.snapshots {
d.hosts[snapshot.Hostname] = true
}
}
// read snapshot id names from the current repository-state.
func getSnapshotIDSNames(d *SnapshotsIDSDir) {
for _, sn := range d.root.snapshots {
name := sn.ID().Str()
d.names[name] = sn
debug.Log(" add snapshot %v as dir %v", sn.ID().Str(), name)
}
}
// NewSnapshotsDir returns a new directory containing snapshots.
func NewSnapshotsDir(root *Root, inode uint64, tag string, host string) *SnapshotsDir {
debug.Log("create snapshots dir, inode %d", inode)
d := &SnapshotsDir{
root: root,
inode: inode,
names: make(map[string]*restic.Snapshot),
latest: "",
tag: tag,
host: host,
}
return d
}
// NewSnapshotsIDSDir returns a new directory containing snapshots named by ids.
func NewSnapshotsIDSDir(root *Root, inode uint64, snapshots restic.Snapshots) *SnapshotsDir {
debug.Log("create snapshots ids dir with %d snapshots, inode %d", len(snapshots), inode)
d := &SnapshotsDir{
root: root,
inode: inode,
snapshots: snapshots,
names: make(map[string]*restic.Snapshot, len(snapshots)),
func NewSnapshotsIDSDir(root *Root, inode uint64) *SnapshotsIDSDir {
debug.Log("create snapshots ids dir, inode %d", inode)
d := &SnapshotsIDSDir{
root: root,
inode: inode,
names: make(map[string]*restic.Snapshot),
}
for _, sn := range snapshots {
name := sn.ID().Str()
return d
}
d.names[name] = sn
debug.Log(" add snapshot %v", name)
// NewHostsDir returns a new directory containing host names
func NewHostsDir(root *Root, inode uint64) *HostsDir {
debug.Log("create hosts dir, inode %d", inode)
d := &HostsDir{
root: root,
inode: inode,
hosts: make(map[string]bool),
}
return d
}
// NewTagsDir returns a new directory containing tag names
func NewTagsDir(root *Root, inode uint64) *TagsDir {
debug.Log("create tags dir, inode %d", inode)
d := &TagsDir{
root: root,
inode: inode,
tags: make(map[string]bool),
}
return d
@ -99,9 +157,91 @@ func (d *SnapshotsDir) Attr(ctx context.Context, attr *fuse.Attr) error {
return nil
}
// ReadDirAll returns all entries of the root node.
// Attr returns the attributes for the SnapshotsDir.
func (d *SnapshotsIDSDir) Attr(ctx context.Context, attr *fuse.Attr) error {
attr.Inode = d.inode
attr.Mode = os.ModeDir | 0555
if !d.root.cfg.OwnerIsRoot {
attr.Uid = uint32(os.Getuid())
attr.Gid = uint32(os.Getgid())
}
debug.Log("attr: %v", attr)
return nil
}
// Attr returns the attributes for the HostsDir.
func (d *HostsDir) Attr(ctx context.Context, attr *fuse.Attr) error {
attr.Inode = d.inode
attr.Mode = os.ModeDir | 0555
if !d.root.cfg.OwnerIsRoot {
attr.Uid = uint32(os.Getuid())
attr.Gid = uint32(os.Getgid())
}
debug.Log("attr: %v", attr)
return nil
}
// Attr returns the attributes for the TagsDir.
func (d *TagsDir) Attr(ctx context.Context, attr *fuse.Attr) error {
attr.Inode = d.inode
attr.Mode = os.ModeDir | 0555
if !d.root.cfg.OwnerIsRoot {
attr.Uid = uint32(os.Getuid())
attr.Gid = uint32(os.Getgid())
}
debug.Log("attr: %v", attr)
return nil
}
// search element in string list.
func isElem(e string, list []string) bool {
for _, x := range list {
if e == x {
return true
}
}
return false
}
// read snapshot timestamps from the current repository-state.
func getSnapshotNames(d *SnapshotsDir) {
var latestTime time.Time
d.latest = ""
d.names = make(map[string]*restic.Snapshot, len(d.root.snapshots))
for _, sn := range d.root.snapshots {
if d.tag == "" || isElem(d.tag, sn.Tags) {
if d.host == "" || d.host == sn.Hostname {
name := sn.Time.Format(time.RFC3339)
if d.latest == "" || !sn.Time.Before(latestTime) {
latestTime = sn.Time
d.latest = name
}
for i := 1; ; i++ {
if _, ok := d.names[name]; !ok {
break
}
name = fmt.Sprintf("%s-%d", sn.Time.Format(time.RFC3339), i)
}
d.names[name] = sn
}
}
}
}
// ReadDirAll returns all entries of the SnapshotsDir.
func (d *SnapshotsDir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) {
debug.Log("ReadDirAll()")
d.root.repo.LoadIndex(ctx)
d.root.snapshots = restic.FindFilteredSnapshots(ctx, d.root.repo, d.root.cfg.Host, d.root.cfg.Tags, d.root.cfg.Paths)
getSnapshotNames(d)
items := []fuse.Dirent{
{
Inode: d.inode,
@ -134,21 +274,116 @@ func (d *SnapshotsDir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) {
return items, nil
}
type snapshotLink struct {
root *Root
inode uint64
target string
snapshot *restic.Snapshot
// ReadDirAll returns all entries of the SnapshotsIDSDir.
func (d *SnapshotsIDSDir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) {
debug.Log("ReadDirAll()")
d.root.repo.LoadIndex(ctx)
d.root.snapshots = restic.FindFilteredSnapshots(ctx, d.root.repo, d.root.cfg.Host, d.root.cfg.Tags, d.root.cfg.Paths)
getSnapshotIDSNames(d)
items := []fuse.Dirent{
{
Inode: d.inode,
Name: ".",
Type: fuse.DT_Dir,
},
{
Inode: d.root.inode,
Name: "..",
Type: fuse.DT_Dir,
},
}
for name := range d.names {
items = append(items, fuse.Dirent{
Inode: fs.GenerateDynamicInode(d.inode, name),
Name: name,
Type: fuse.DT_Dir,
})
}
return items, nil
}
// ReadDirAll returns all entries of the HostsDir.
func (d *HostsDir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) {
debug.Log("ReadDirAll()")
d.root.repo.LoadIndex(ctx)
d.root.snapshots = restic.FindFilteredSnapshots(ctx, d.root.repo, d.root.cfg.Host, d.root.cfg.Tags, d.root.cfg.Paths)
getHostsNames(d)
items := []fuse.Dirent{
{
Inode: d.inode,
Name: ".",
Type: fuse.DT_Dir,
},
{
Inode: d.root.inode,
Name: "..",
Type: fuse.DT_Dir,
},
}
for host := range d.hosts {
items = append(items, fuse.Dirent{
Inode: fs.GenerateDynamicInode(d.inode, host),
Name: host,
Type: fuse.DT_Dir,
})
}
return items, nil
}
// ReadDirAll returns all entries of the TagsDir.
func (d *TagsDir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) {
debug.Log("ReadDirAll()")
d.root.repo.LoadIndex(ctx)
d.root.snapshots = restic.FindFilteredSnapshots(ctx, d.root.repo, d.root.cfg.Host, d.root.cfg.Tags, d.root.cfg.Paths)
getTagNames(d)
items := []fuse.Dirent{
{
Inode: d.inode,
Name: ".",
Type: fuse.DT_Dir,
},
{
Inode: d.root.inode,
Name: "..",
Type: fuse.DT_Dir,
},
}
for tag := range d.tags {
items = append(items, fuse.Dirent{
Inode: fs.GenerateDynamicInode(d.inode, tag),
Name: tag,
Type: fuse.DT_Dir,
})
}
return items, nil
}
// newSnapshotLink
func newSnapshotLink(ctx context.Context, root *Root, inode uint64, target string, snapshot *restic.Snapshot) (*snapshotLink, error) {
return &snapshotLink{root: root, inode: inode, target: target, snapshot: snapshot}, nil
}
// Readlink
func (l *snapshotLink) Readlink(ctx context.Context, req *fuse.ReadlinkRequest) (string, error) {
return l.target, nil
}
// Attr
func (l *snapshotLink) Attr(ctx context.Context, a *fuse.Attr) error {
a.Inode = l.inode
a.Mode = os.ModeSymlink | 0777
@ -166,24 +401,104 @@ func (l *snapshotLink) Attr(ctx context.Context, a *fuse.Attr) error {
return nil
}
// Lookup returns a specific entry from the root node.
// Lookup returns a specific entry from the SnapshotsDir.
func (d *SnapshotsDir) Lookup(ctx context.Context, name string) (fs.Node, error) {
debug.Log("Lookup(%s)", name)
sn, ok := d.names[name]
if !ok {
// could not find entry. Updating repository-state
d.root.repo.LoadIndex(ctx)
d.root.snapshots = restic.FindFilteredSnapshots(ctx, d.root.repo, d.root.cfg.Host, d.root.cfg.Tags, d.root.cfg.Paths)
getSnapshotNames(d)
sn, ok := d.names[name]
if ok {
return newDirFromSnapshot(ctx, d.root, fs.GenerateDynamicInode(d.inode, name), sn)
}
if name == "latest" && d.latest != "" {
sn2, ok2 := d.names[d.latest]
sn, ok := d.names[d.latest]
// internal error
if !ok2 {
if !ok {
return nil, fuse.ENOENT
}
return newSnapshotLink(ctx, d.root, fs.GenerateDynamicInode(d.inode, name), d.latest, sn2)
return newSnapshotLink(ctx, d.root, fs.GenerateDynamicInode(d.inode, name), d.latest, sn)
}
return nil, fuse.ENOENT
}
return newDirFromSnapshot(ctx, d.root, fs.GenerateDynamicInode(d.inode, name), sn)
}
// Lookup returns a specific entry from the SnapshotsIDSDir.
func (d *SnapshotsIDSDir) Lookup(ctx context.Context, name string) (fs.Node, error) {
debug.Log("Lookup(%s)", name)
sn, ok := d.names[name]
if !ok {
// could not find entry. Updating repository-state
d.root.repo.LoadIndex(ctx)
d.root.snapshots = restic.FindFilteredSnapshots(ctx, d.root.repo, d.root.cfg.Host, d.root.cfg.Tags, d.root.cfg.Paths)
getSnapshotIDSNames(d)
sn, ok := d.names[name]
if ok {
return newDirFromSnapshot(ctx, d.root, fs.GenerateDynamicInode(d.inode, name), sn)
}
return nil, fuse.ENOENT
}
return newDirFromSnapshot(ctx, d.root, fs.GenerateDynamicInode(d.inode, name), sn)
}
// Lookup returns a specific entry from the HostsDir.
func (d *HostsDir) Lookup(ctx context.Context, name string) (fs.Node, error) {
debug.Log("Lookup(%s)", name)
_, ok := d.hosts[name]
if !ok {
// could not find entry. Updating repository-state
d.root.repo.LoadIndex(ctx)
d.root.snapshots = restic.FindFilteredSnapshots(ctx, d.root.repo, d.root.cfg.Host, d.root.cfg.Tags, d.root.cfg.Paths)
getHostsNames(d)
_, ok := d.hosts[name]
if ok {
return NewSnapshotsDir(d.root, fs.GenerateDynamicInode(d.root.inode, name), "", name), nil
}
return nil, fuse.ENOENT
}
return NewSnapshotsDir(d.root, fs.GenerateDynamicInode(d.root.inode, name), "", name), nil
}
// Lookup returns a specific entry from the TagsDir.
func (d *TagsDir) Lookup(ctx context.Context, name string) (fs.Node, error) {
debug.Log("Lookup(%s)", name)
_, ok := d.tags[name]
if !ok {
// could not find entry. Updating repository-state
d.root.repo.LoadIndex(ctx)
d.root.snapshots = restic.FindFilteredSnapshots(ctx, d.root.repo, d.root.cfg.Host, d.root.cfg.Tags, d.root.cfg.Paths)
getTagNames(d)
_, ok := d.tags[name]
if ok {
return NewSnapshotsDir(d.root, fs.GenerateDynamicInode(d.root.inode, name), name, ""), nil
}
return nil, fuse.ENOENT
}
return NewSnapshotsDir(d.root, fs.GenerateDynamicInode(d.root.inode, name), name, ""), nil
}