restic/internal/archiver/archiver.go

881 lines
24 KiB
Go
Raw Normal View History

2018-03-30 20:43:18 +00:00
package archiver
import (
"context"
"os"
"path"
"runtime"
"sort"
"time"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/fs"
"github.com/restic/restic/internal/restic"
2022-05-27 17:08:50 +00:00
"golang.org/x/sync/errgroup"
2018-03-30 20:43:18 +00:00
)
// SelectByNameFunc returns true for all items that should be included (files and
// dirs). If false is returned, files are ignored and dirs are not even walked.
type SelectByNameFunc func(item string) bool
2018-03-30 20:43:18 +00:00
// SelectFunc returns true for all items that should be included (files and
// dirs). If false is returned, files are ignored and dirs are not even walked.
type SelectFunc func(item string, fi os.FileInfo) bool
// ErrorFunc is called when an error during archiving occurs. When nil is
// returned, the archiver continues, otherwise it aborts and passes the error
// up the call stack.
type ErrorFunc func(file string, err error) error
2018-03-30 20:43:18 +00:00
// ItemStats collects some statistics about a particular file or directory.
type ItemStats struct {
DataBlobs int // number of new data blobs added for this item
DataSize uint64 // sum of the sizes of all new data blobs
DataSizeInRepo uint64 // sum of the bytes added to the repo (including compression and crypto overhead)
TreeBlobs int // number of new tree blobs added for this item
TreeSize uint64 // sum of the sizes of all new tree blobs
TreeSizeInRepo uint64 // sum of the bytes added to the repo (including compression and crypto overhead)
2018-03-30 20:43:18 +00:00
}
// Add adds other to the current ItemStats.
func (s *ItemStats) Add(other ItemStats) {
s.DataBlobs += other.DataBlobs
s.DataSize += other.DataSize
s.DataSizeInRepo += other.DataSizeInRepo
2018-03-30 20:43:18 +00:00
s.TreeBlobs += other.TreeBlobs
s.TreeSize += other.TreeSize
s.TreeSizeInRepo += other.TreeSizeInRepo
2018-03-30 20:43:18 +00:00
}
// Archiver saves a directory structure to the repo.
type Archiver struct {
Repo restic.Repository
SelectByName SelectByNameFunc
Select SelectFunc
FS fs.FS
Options Options
2018-03-30 20:43:18 +00:00
blobSaver *BlobSaver
fileSaver *FileSaver
2018-04-30 13:13:03 +00:00
treeSaver *TreeSaver
2018-03-30 20:43:18 +00:00
// Error is called for all errors that occur during backup.
Error ErrorFunc
// CompleteItem is called for all files and dirs once they have been
// processed successfully. The parameter item contains the path as it will
// be in the snapshot after saving. s contains some statistics about this
// particular file/dir.
//
// CompleteItem may be called asynchronously from several different
// goroutines!
CompleteItem func(item string, previous, current *restic.Node, s ItemStats, d time.Duration)
// StartFile is called when a file is being processed by a worker.
StartFile func(filename string)
// CompleteBlob is called for all saved blobs for files.
CompleteBlob func(filename string, bytes uint64)
// WithAtime configures if the access time for files and directories should
// be saved. Enabling it may result in much metadata, so it's off by
// default.
WithAtime bool
// Flags controlling change detection. See doc/040_backup.rst for details.
ChangeIgnoreFlags uint
2018-03-30 20:43:18 +00:00
}
// Flags for the ChangeIgnoreFlags bitfield.
const (
ChangeIgnoreCtime = 1 << iota
ChangeIgnoreInode
)
2018-03-30 20:43:18 +00:00
// Options is used to configure the archiver.
type Options struct {
// FileReadConcurrency sets how many files are read in concurrently. If
// it's set to zero, at most two files are read in concurrently (which
// turned out to be a good default for most situations).
FileReadConcurrency uint
// SaveBlobConcurrency sets how many blobs are hashed and saved
// concurrently. If it's set to zero, the default is the number of CPUs
// available in the system.
SaveBlobConcurrency uint
2018-04-30 13:13:03 +00:00
// SaveTreeConcurrency sets how many trees are marshalled and saved to the
// repo concurrently.
SaveTreeConcurrency uint
2018-03-30 20:43:18 +00:00
}
// ApplyDefaults returns a copy of o with the default options set for all unset
// fields.
func (o Options) ApplyDefaults() Options {
if o.FileReadConcurrency == 0 {
// two is a sweet spot for almost all situations. We've done some
// experiments documented here:
// https://github.com/borgbackup/borg/issues/3500
o.FileReadConcurrency = 2
}
if o.SaveBlobConcurrency == 0 {
// blob saving is CPU bound due to hash checking and encryption
// the actual upload is handled by the repository itself
o.SaveBlobConcurrency = uint(runtime.GOMAXPROCS(0))
2018-03-30 20:43:18 +00:00
}
2018-04-30 13:13:03 +00:00
if o.SaveTreeConcurrency == 0 {
// can either wait for a file, wait for a tree, serialize a tree or wait for saveblob
// the last two are cpu-bound and thus mutually exclusive.
// Also allow waiting for FileReadConcurrency files, this is the maximum of FutureFiles
// which currently can be in progress. The main backup loop blocks when trying to queue
// more files to read.
o.SaveTreeConcurrency = uint(runtime.GOMAXPROCS(0)) + o.FileReadConcurrency
2018-04-30 13:13:03 +00:00
}
2018-03-30 20:43:18 +00:00
return o
}
// New initializes a new archiver.
func New(repo restic.Repository, fs fs.FS, opts Options) *Archiver {
arch := &Archiver{
Repo: repo,
SelectByName: func(item string) bool { return true },
Select: func(item string, fi os.FileInfo) bool { return true },
FS: fs,
Options: opts.ApplyDefaults(),
2018-03-30 20:43:18 +00:00
CompleteItem: func(string, *restic.Node, *restic.Node, ItemStats, time.Duration) {},
StartFile: func(string) {},
CompleteBlob: func(string, uint64) {},
}
return arch
}
// error calls arch.Error if it is set and the error is different from context.Canceled.
func (arch *Archiver) error(item string, err error) error {
2018-03-30 20:43:18 +00:00
if arch.Error == nil || err == nil {
return err
}
if err == context.Canceled {
return err
}
errf := arch.Error(item, err)
2018-03-30 20:43:18 +00:00
if err != errf {
debug.Log("item %v: error was filtered by handler, before: %q, after: %v", item, err, errf)
}
return errf
}
// saveTree stores a tree in the repo. It checks the index and the known blobs
// before saving anything.
func (arch *Archiver) saveTree(ctx context.Context, t *restic.TreeJSONBuilder) (restic.ID, ItemStats, error) {
2018-03-30 20:43:18 +00:00
var s ItemStats
buf, err := t.Finalize()
2018-03-30 20:43:18 +00:00
if err != nil {
return restic.ID{}, s, err
2018-03-30 20:43:18 +00:00
}
2018-04-29 13:34:41 +00:00
b := &Buffer{Data: buf}
2018-03-30 20:43:18 +00:00
res := arch.blobSaver.Save(ctx, restic.TreeBlob, b)
sbr := res.Take(ctx)
if !sbr.known {
2018-03-30 20:43:18 +00:00
s.TreeBlobs++
s.TreeSize += uint64(sbr.length)
s.TreeSizeInRepo += uint64(sbr.sizeInRepo)
2018-03-30 20:43:18 +00:00
}
// The context was canceled in the meantime, id might be invalid
if ctx.Err() != nil {
return restic.ID{}, s, ctx.Err()
}
return sbr.id, s, nil
2018-03-30 20:43:18 +00:00
}
2020-05-16 06:05:26 +00:00
// nodeFromFileInfo returns the restic node from an os.FileInfo.
2018-03-30 20:43:18 +00:00
func (arch *Archiver) nodeFromFileInfo(filename string, fi os.FileInfo) (*restic.Node, error) {
node, err := restic.NodeFromFileInfo(filename, fi)
if !arch.WithAtime {
node.AccessTime = node.ModTime
}
return node, errors.Wrap(err, "NodeFromFileInfo")
}
// loadSubtree tries to load the subtree referenced by node. In case of an error, nil is returned.
// If there is no node to load, then nil is returned without an error.
func (arch *Archiver) loadSubtree(ctx context.Context, node *restic.Node) (*restic.Tree, error) {
2018-03-30 20:43:18 +00:00
if node == nil || node.Type != "dir" || node.Subtree == nil {
return nil, nil
2018-03-30 20:43:18 +00:00
}
tree, err := restic.LoadTree(ctx, arch.Repo, *node.Subtree)
2018-03-30 20:43:18 +00:00
if err != nil {
debug.Log("unable to load tree %v: %v", node.Subtree.Str(), err)
// a tree in the repository is not readable -> warn the user
return nil, arch.wrapLoadTreeError(*node.Subtree, err)
2018-03-30 20:43:18 +00:00
}
return tree, nil
}
func (arch *Archiver) wrapLoadTreeError(id restic.ID, err error) error {
2020-11-05 21:18:00 +00:00
if arch.Repo.Index().Has(restic.BlobHandle{ID: id, Type: restic.TreeBlob}) {
err = errors.Errorf("tree %v could not be loaded; the repository could be damaged: %v", id, err)
} else {
err = errors.Errorf("tree %v is not known; the repository could be damaged, run `rebuild-index` to try to repair it", id)
}
return err
2018-03-30 20:43:18 +00:00
}
// SaveDir stores a directory in the repo and returns the node. snPath is the
// path within the current snapshot.
func (arch *Archiver) SaveDir(ctx context.Context, snPath string, dir string, fi os.FileInfo, previous *restic.Tree, complete CompleteFunc) (d FutureNode, err error) {
2018-03-30 20:43:18 +00:00
debug.Log("%v %v", snPath, dir)
treeNode, err := arch.nodeFromFileInfo(dir, fi)
if err != nil {
return FutureNode{}, err
2018-03-30 20:43:18 +00:00
}
names, err := readdirnames(arch.FS, dir, fs.O_NOFOLLOW)
2018-03-30 20:43:18 +00:00
if err != nil {
return FutureNode{}, err
2018-03-30 20:43:18 +00:00
}
sort.Strings(names)
2018-03-30 20:43:18 +00:00
2018-04-30 13:13:03 +00:00
nodes := make([]FutureNode, 0, len(names))
2018-03-30 20:43:18 +00:00
for _, name := range names {
// test if context has been cancelled
if ctx.Err() != nil {
debug.Log("context has been cancelled, aborting")
return FutureNode{}, ctx.Err()
}
2018-03-30 20:43:18 +00:00
pathname := arch.FS.Join(dir, name)
oldNode := previous.Find(name)
snItem := join(snPath, name)
fn, excluded, err := arch.Save(ctx, snItem, pathname, oldNode)
// return error early if possible
if err != nil {
err = arch.error(pathname, err)
2018-03-30 20:43:18 +00:00
if err == nil {
// ignore error
continue
}
return FutureNode{}, err
2018-03-30 20:43:18 +00:00
}
if excluded {
continue
}
2018-04-30 13:13:03 +00:00
nodes = append(nodes, fn)
2018-03-30 20:43:18 +00:00
}
fn := arch.treeSaver.Save(ctx, snPath, dir, treeNode, nodes, complete)
2018-03-30 20:43:18 +00:00
return fn, nil
2018-03-30 20:43:18 +00:00
}
// FutureNode holds a reference to a channel that returns a FutureNodeResult
// or a reference to an already existing result. If the result is available
// immediatelly, then storing a reference directly requires less memory than
// using the indirection via a channel.
2018-03-30 20:43:18 +00:00
type FutureNode struct {
ch <-chan futureNodeResult
res *futureNodeResult
}
type futureNodeResult struct {
2018-03-30 20:43:18 +00:00
snPath, target string
node *restic.Node
stats ItemStats
err error
}
2018-03-30 20:43:18 +00:00
func newFutureNode() (FutureNode, chan<- futureNodeResult) {
ch := make(chan futureNodeResult, 1)
return FutureNode{ch: ch}, ch
2018-03-30 20:43:18 +00:00
}
func newFutureNodeWithResult(res futureNodeResult) FutureNode {
return FutureNode{
res: &res,
2018-03-30 20:43:18 +00:00
}
}
func (fn *FutureNode) take(ctx context.Context) futureNodeResult {
if fn.res != nil {
res := fn.res
// free result
fn.res = nil
return *res
}
select {
case res, ok := <-fn.ch:
if ok {
// free channel
fn.ch = nil
return res
}
case <-ctx.Done():
}
return futureNodeResult{}
}
// allBlobsPresent checks if all blobs (contents) of the given node are
// present in the index.
func (arch *Archiver) allBlobsPresent(previous *restic.Node) bool {
// check if all blobs are contained in index
for _, id := range previous.Content {
2020-11-05 21:18:00 +00:00
if !arch.Repo.Index().Has(restic.BlobHandle{ID: id, Type: restic.DataBlob}) {
return false
}
}
return true
}
2018-03-30 20:43:18 +00:00
// Save saves a target (file or directory) to the repo. If the item is
// excluded, this function returns a nil node and error, with excluded set to
2018-05-12 19:59:38 +00:00
// true.
2018-03-30 20:43:18 +00:00
//
// Errors and completion needs to be handled by the caller.
2018-03-30 20:43:18 +00:00
//
// snPath is the path within the current snapshot.
func (arch *Archiver) Save(ctx context.Context, snPath, target string, previous *restic.Node) (fn FutureNode, excluded bool, err error) {
2018-04-30 13:13:03 +00:00
start := time.Now()
2018-03-30 20:43:18 +00:00
debug.Log("%v target %q, previous %v", snPath, target, previous)
abstarget, err := arch.FS.Abs(target)
if err != nil {
return FutureNode{}, false, err
}
// exclude files by path before running Lstat to reduce number of lstat calls
if !arch.SelectByName(abstarget) {
debug.Log("%v is excluded by path", target)
return FutureNode{}, true, nil
}
// get file info and run remaining select functions that require file information
fi, err := arch.FS.Lstat(target)
if err != nil {
debug.Log("lstat() for %v returned error: %v", target, err)
err = arch.error(abstarget, err)
if err != nil {
return FutureNode{}, false, errors.Wrap(err, "Lstat")
2018-03-30 20:43:18 +00:00
}
return FutureNode{}, true, nil
2018-03-30 20:43:18 +00:00
}
if !arch.Select(abstarget, fi) {
debug.Log("%v is excluded", target)
return FutureNode{}, true, nil
}
2018-03-30 20:43:18 +00:00
switch {
case fs.IsRegularFile(fi):
debug.Log(" %v regular file", target)
start := time.Now()
// check if the file has not changed before performing a fopen operation (more expensive, specially
// in network filesystems)
if previous != nil && !fileChanged(fi, previous, arch.ChangeIgnoreFlags) {
if arch.allBlobsPresent(previous) {
debug.Log("%v hasn't changed, using old list of blobs", target)
arch.CompleteItem(snPath, previous, previous, ItemStats{}, time.Since(start))
arch.CompleteBlob(snPath, previous.Size)
node, err := arch.nodeFromFileInfo(target, fi)
if err != nil {
return FutureNode{}, false, err
}
// copy list of blobs
node.Content = previous.Content
fn = newFutureNodeWithResult(futureNodeResult{
snPath: snPath,
target: target,
node: node,
})
return fn, false, nil
}
debug.Log("%v hasn't changed, but contents are missing!", target)
// There are contents missing - inform user!
err := errors.Errorf("parts of %v not found in the repository index; storing the file again", target)
err = arch.error(abstarget, err)
2021-01-30 16:25:10 +00:00
if err != nil {
return FutureNode{}, false, err
}
}
// reopen file and do an fstat() on the open file to check it is still
// a file (and has not been exchanged for e.g. a symlink)
file, err := arch.FS.OpenFile(target, fs.O_RDONLY|fs.O_NOFOLLOW, 0)
if err != nil {
debug.Log("Openfile() for %v returned error: %v", target, err)
err = arch.error(abstarget, err)
if err != nil {
return FutureNode{}, false, errors.Wrap(err, "Lstat")
}
return FutureNode{}, true, nil
}
fi, err = file.Stat()
if err != nil {
debug.Log("stat() on opened file %v returned error: %v", target, err)
_ = file.Close()
err = arch.error(abstarget, err)
if err != nil {
return FutureNode{}, false, errors.Wrap(err, "Lstat")
}
return FutureNode{}, true, nil
}
// make sure it's still a file
if !fs.IsRegularFile(fi) {
err = errors.Errorf("file %v changed type, refusing to archive")
_ = file.Close()
err = arch.error(abstarget, err)
if err != nil {
return FutureNode{}, false, err
}
return FutureNode{}, true, nil
}
2018-03-30 20:43:18 +00:00
// Save will close the file, we don't need to do that
fn = arch.fileSaver.Save(ctx, snPath, target, file, fi, func() {
2018-03-30 20:43:18 +00:00
arch.StartFile(snPath)
}, func(node *restic.Node, stats ItemStats) {
arch.CompleteItem(snPath, previous, node, stats, time.Since(start))
})
case fi.IsDir():
debug.Log(" %v dir", target)
snItem := snPath + "/"
start := time.Now()
oldSubtree, err := arch.loadSubtree(ctx, previous)
if err != nil {
err = arch.error(abstarget, err)
2021-01-30 16:25:10 +00:00
}
if err != nil {
return FutureNode{}, false, err
}
2018-04-30 13:13:03 +00:00
fn, err = arch.SaveDir(ctx, snPath, target, fi, oldSubtree,
func(node *restic.Node, stats ItemStats) {
arch.CompleteItem(snItem, previous, node, stats, time.Since(start))
})
if err != nil {
debug.Log("SaveDir for %v returned error: %v", snPath, err)
2018-03-30 20:43:18 +00:00
return FutureNode{}, false, err
}
case fi.Mode()&os.ModeSocket > 0:
debug.Log(" %v is a socket, ignoring", target)
return FutureNode{}, true, nil
default:
debug.Log(" %v other", target)
node, err := arch.nodeFromFileInfo(target, fi)
2018-03-30 20:43:18 +00:00
if err != nil {
return FutureNode{}, false, err
}
fn = newFutureNodeWithResult(futureNodeResult{
snPath: snPath,
target: target,
node: node,
})
2018-03-30 20:43:18 +00:00
}
2018-04-30 13:13:03 +00:00
debug.Log("return after %.3f", time.Since(start).Seconds())
2018-03-30 20:43:18 +00:00
return fn, false, nil
}
// fileChanged tries to detect whether a file's content has changed compared
// to the contents of node, which describes the same path in the parent backup.
// It should only be run for regular files.
func fileChanged(fi os.FileInfo, node *restic.Node, ignoreFlags uint) bool {
switch {
case node == nil:
2018-03-30 20:43:18 +00:00
return true
case node.Type != "file":
// We're only called for regular files, so this is a type change.
2018-03-30 20:43:18 +00:00
return true
case uint64(fi.Size()) != node.Size:
2018-03-30 20:43:18 +00:00
return true
case !fi.ModTime().Equal(node.ModTime):
return true
}
checkCtime := ignoreFlags&ChangeIgnoreCtime == 0
checkInode := ignoreFlags&ChangeIgnoreInode == 0
2018-03-30 20:43:18 +00:00
extFI := fs.ExtendedStat(fi)
switch {
case checkCtime && !extFI.ChangeTime.Equal(node.ChangeTime):
return true
case checkInode && node.Inode != extFI.Inode:
2018-03-30 20:43:18 +00:00
return true
}
return false
}
// join returns all elements separated with a forward slash.
func join(elem ...string) string {
return path.Join(elem...)
}
// statDir returns the file info for the directory. Symbolic links are
// resolved. If the target directory is not a directory, an error is returned.
func (arch *Archiver) statDir(dir string) (os.FileInfo, error) {
fi, err := arch.FS.Stat(dir)
if err != nil {
return nil, errors.Wrap(err, "Lstat")
}
tpe := fi.Mode() & (os.ModeType | os.ModeCharDevice)
if tpe != os.ModeDir {
return fi, errors.Errorf("path is not a directory: %v", dir)
}
return fi, nil
}
// SaveTree stores a Tree in the repo, returned is the tree. snPath is the path
// within the current snapshot.
func (arch *Archiver) SaveTree(ctx context.Context, snPath string, atree *Tree, previous *restic.Tree) (*restic.Tree, error) {
debug.Log("%v (%v nodes), parent %v", snPath, len(atree.Nodes), previous)
nodeNames := atree.NodeNames()
tree := restic.NewTree(len(nodeNames))
2018-03-30 20:43:18 +00:00
futureNodes := make(map[string]FutureNode)
2018-05-12 21:07:16 +00:00
// iterate over the nodes of atree in lexicographic (=deterministic) order
for _, name := range nodeNames {
2018-05-12 21:07:16 +00:00
subatree := atree.Nodes[name]
// test if context has been cancelled
if ctx.Err() != nil {
return nil, ctx.Err()
}
2018-03-30 20:43:18 +00:00
// this is a leaf node
if subatree.Leaf() {
2018-03-30 20:43:18 +00:00
fn, excluded, err := arch.Save(ctx, join(snPath, name), subatree.Path, previous.Find(name))
if err != nil {
err = arch.error(subatree.Path, err)
2018-03-30 20:43:18 +00:00
if err == nil {
// ignore error
continue
}
return nil, err
}
if err != nil {
return nil, err
}
if !excluded {
futureNodes[name] = fn
}
continue
}
snItem := join(snPath, name) + "/"
start := time.Now()
oldNode := previous.Find(name)
oldSubtree, err := arch.loadSubtree(ctx, oldNode)
if err != nil {
err = arch.error(join(snPath, name), err)
2021-01-30 16:25:10 +00:00
}
if err != nil {
return nil, err
}
2018-03-30 20:43:18 +00:00
// not a leaf node, archive subtree
subtree, err := arch.SaveTree(ctx, join(snPath, name), &subatree, oldSubtree)
if err != nil {
return nil, err
}
tb, err := restic.TreeToBuilder(subtree)
if err != nil {
return nil, err
}
id, nodeStats, err := arch.saveTree(ctx, tb)
2018-03-30 20:43:18 +00:00
if err != nil {
return nil, err
}
if subatree.FileInfoPath == "" {
return nil, errors.Errorf("FileInfoPath for %v/%v is empty", snPath, name)
}
debug.Log("%v, saved subtree %v as %v", snPath, subtree, id.Str())
fi, err := arch.statDir(subatree.FileInfoPath)
if err != nil {
return nil, err
}
debug.Log("%v, dir node data loaded from %v", snPath, subatree.FileInfoPath)
node, err := arch.nodeFromFileInfo(subatree.FileInfoPath, fi)
if err != nil {
return nil, err
}
node.Name = name
node.Subtree = &id
err = tree.Insert(node)
if err != nil {
return nil, err
}
arch.CompleteItem(snItem, oldNode, node, nodeStats, time.Since(start))
}
2018-04-30 13:13:03 +00:00
debug.Log("waiting on %d nodes", len(futureNodes))
2018-03-30 20:43:18 +00:00
// process all futures
for name, fn := range futureNodes {
fnr := fn.take(ctx)
2018-03-30 20:43:18 +00:00
// return the error, or ignore it
if fnr.err != nil {
fnr.err = arch.error(fnr.target, fnr.err)
if fnr.err == nil {
2018-03-30 20:43:18 +00:00
// ignore error
continue
}
return nil, fnr.err
2018-03-30 20:43:18 +00:00
}
// when the error is ignored, the node could not be saved, so ignore it
if fnr.node == nil {
debug.Log("%v excluded: %v", fnr.snPath, fnr.target)
2018-03-30 20:43:18 +00:00
continue
}
fnr.node.Name = name
2018-03-30 20:43:18 +00:00
err := tree.Insert(fnr.node)
2018-03-30 20:43:18 +00:00
if err != nil {
return nil, err
}
}
return tree, nil
}
// flags are passed to fs.OpenFile. O_RDONLY is implied.
func readdirnames(filesystem fs.FS, dir string, flags int) ([]string, error) {
f, err := filesystem.OpenFile(dir, fs.O_RDONLY|flags, 0)
2018-03-30 20:43:18 +00:00
if err != nil {
return nil, errors.Wrap(err, "Open")
}
entries, err := f.Readdirnames(-1)
if err != nil {
_ = f.Close()
2018-05-15 09:03:33 +00:00
return nil, errors.Wrapf(err, "Readdirnames %v failed", dir)
2018-03-30 20:43:18 +00:00
}
err = f.Close()
if err != nil {
return nil, err
}
return entries, nil
}
// resolveRelativeTargets replaces targets that only contain relative
// directories ("." or "../../") with the contents of the directory. Each
// element of target is processed with fs.Clean().
func resolveRelativeTargets(filesys fs.FS, targets []string) ([]string, error) {
2018-03-30 20:43:18 +00:00
debug.Log("targets before resolving: %v", targets)
result := make([]string, 0, len(targets))
for _, target := range targets {
target = filesys.Clean(target)
pc, _ := pathComponents(filesys, target, false)
2018-03-30 20:43:18 +00:00
if len(pc) > 0 {
result = append(result, target)
continue
}
debug.Log("replacing %q with readdir(%q)", target, target)
entries, err := readdirnames(filesys, target, fs.O_NOFOLLOW)
2018-03-30 20:43:18 +00:00
if err != nil {
return nil, err
}
sort.Strings(entries)
2018-03-30 20:43:18 +00:00
for _, name := range entries {
result = append(result, filesys.Join(target, name))
2018-03-30 20:43:18 +00:00
}
}
debug.Log("targets after resolving: %v", result)
return result, nil
}
// SnapshotOptions collect attributes for a new snapshot.
type SnapshotOptions struct {
Tags restic.TagList
2018-03-30 20:43:18 +00:00
Hostname string
Excludes []string
Time time.Time
ParentSnapshot restic.ID
}
// loadParentTree loads a tree referenced by snapshot id. If id is null, nil is returned.
func (arch *Archiver) loadParentTree(ctx context.Context, snapshotID restic.ID) *restic.Tree {
if snapshotID.IsNull() {
return nil
}
debug.Log("load parent snapshot %v", snapshotID)
sn, err := restic.LoadSnapshot(ctx, arch.Repo, snapshotID)
if err != nil {
debug.Log("unable to load snapshot %v: %v", snapshotID, err)
return nil
}
if sn.Tree == nil {
debug.Log("snapshot %v has empty tree %v", snapshotID)
return nil
}
debug.Log("load parent tree %v", *sn.Tree)
tree, err := restic.LoadTree(ctx, arch.Repo, *sn.Tree)
2018-03-30 20:43:18 +00:00
if err != nil {
debug.Log("unable to load tree %v: %v", *sn.Tree, err)
_ = arch.error("/", arch.wrapLoadTreeError(*sn.Tree, err))
2018-03-30 20:43:18 +00:00
return nil
}
return tree
}
// runWorkers starts the worker pools, which are stopped when the context is cancelled.
2022-05-27 17:08:50 +00:00
func (arch *Archiver) runWorkers(ctx context.Context, wg *errgroup.Group) {
arch.blobSaver = NewBlobSaver(ctx, wg, arch.Repo, arch.Options.SaveBlobConcurrency)
2018-04-30 13:13:03 +00:00
2022-05-27 17:08:50 +00:00
arch.fileSaver = NewFileSaver(ctx, wg,
arch.blobSaver.Save,
arch.Repo.Config().ChunkerPolynomial,
arch.Options.FileReadConcurrency, arch.Options.SaveBlobConcurrency)
2018-03-30 20:43:18 +00:00
arch.fileSaver.CompleteBlob = arch.CompleteBlob
arch.fileSaver.NodeFromFileInfo = arch.nodeFromFileInfo
2018-04-30 13:13:03 +00:00
2022-05-27 17:08:50 +00:00
arch.treeSaver = NewTreeSaver(ctx, wg, arch.Options.SaveTreeConcurrency, arch.saveTree, arch.Error)
}
func (arch *Archiver) stopWorkers() {
arch.blobSaver.TriggerShutdown()
arch.fileSaver.TriggerShutdown()
arch.treeSaver.TriggerShutdown()
2022-06-05 13:48:10 +00:00
arch.blobSaver = nil
arch.fileSaver = nil
arch.treeSaver = nil
2018-03-30 20:43:18 +00:00
}
// Snapshot saves several targets and returns a snapshot.
func (arch *Archiver) Snapshot(ctx context.Context, targets []string, opts SnapshotOptions) (*restic.Snapshot, restic.ID, error) {
cleanTargets, err := resolveRelativeTargets(arch.FS, targets)
if err != nil {
return nil, restic.ID{}, err
}
atree, err := NewTree(arch.FS, cleanTargets)
if err != nil {
return nil, restic.ID{}, err
}
var rootTreeID restic.ID
wgUp, wgUpCtx := errgroup.WithContext(ctx)
arch.Repo.StartPackUploader(wgUpCtx, wgUp)
wgUp.Go(func() error {
wg, wgCtx := errgroup.WithContext(wgUpCtx)
start := time.Now()
var stats ItemStats
wg.Go(func() error {
arch.runWorkers(wgCtx, wg)
debug.Log("starting snapshot")
tree, err := arch.SaveTree(wgCtx, "/", atree, arch.loadParentTree(wgCtx, opts.ParentSnapshot))
if err != nil {
return err
}
2018-03-30 20:43:18 +00:00
if len(tree.Nodes) == 0 {
return errors.New("snapshot is empty")
}
tb, err := restic.TreeToBuilder(tree)
if err != nil {
return err
}
rootTreeID, stats, err = arch.saveTree(wgCtx, tb)
arch.stopWorkers()
return err
})
2018-03-30 20:43:18 +00:00
err = wg.Wait()
debug.Log("err is %v", err)
2018-03-30 20:43:18 +00:00
if err != nil {
debug.Log("error while saving tree: %v", err)
return err
}
arch.CompleteItem("/", nil, nil, stats, time.Since(start))
return arch.Repo.Flush(ctx)
})
err = wgUp.Wait()
2018-03-30 20:43:18 +00:00
if err != nil {
return nil, restic.ID{}, err
}
sn, err := restic.NewSnapshot(targets, opts.Tags, opts.Hostname, opts.Time)
2020-02-12 21:37:37 +00:00
if err != nil {
return nil, restic.ID{}, err
}
2018-03-30 20:43:18 +00:00
sn.Excludes = opts.Excludes
if !opts.ParentSnapshot.IsNull() {
id := opts.ParentSnapshot
sn.Parent = &id
}
sn.Tree = &rootTreeID
id, err := restic.SaveSnapshot(ctx, arch.Repo, sn)
2018-03-30 20:43:18 +00:00
if err != nil {
return nil, restic.ID{}, err
}
return sn, id, nil
}