mirror of
https://github.com/octoleo/restic.git
synced 2024-12-23 03:18:55 +00:00
Skip fopen for file change check to avoid network penalty
This commit is contained in:
parent
4df8861e09
commit
958dc6aafc
9
changelog/unreleased/issue-2969
Normal file
9
changelog/unreleased/issue-2969
Normal file
@ -0,0 +1,9 @@
|
||||
Enhancement: Optimize check for unchanged files during backup
|
||||
|
||||
During a backup restic skips processing files which have not changed since the last backup run.
|
||||
Previously this required opening each file once which can be slow on network filesystems. The
|
||||
backup command now checks for file changes before opening a file. This considerably reduces
|
||||
the time to create a backup on network filesystems.
|
||||
|
||||
https://github.com/restic/restic/issues/2969
|
||||
https://github.com/restic/restic/pull/2970
|
@ -364,6 +364,30 @@ func (arch *Archiver) Save(ctx context.Context, snPath, target string, previous
|
||||
debug.Log(" %v regular file", target)
|
||||
start := time.Now()
|
||||
|
||||
// check if the file has not changed before performing a fopen operation (more expensive, specially
|
||||
// in network filesystems)
|
||||
if previous != nil && !fileChanged(fi, previous, arch.IgnoreInode) {
|
||||
if arch.allBlobsPresent(previous) {
|
||||
debug.Log("%v hasn't changed, using old list of blobs", target)
|
||||
arch.CompleteItem(snPath, previous, previous, ItemStats{}, time.Since(start))
|
||||
arch.CompleteBlob(snPath, previous.Size)
|
||||
fn.node, err = arch.nodeFromFileInfo(target, fi)
|
||||
if err != nil {
|
||||
return FutureNode{}, false, err
|
||||
}
|
||||
|
||||
// copy list of blobs
|
||||
fn.node.Content = previous.Content
|
||||
|
||||
return fn, false, nil
|
||||
}
|
||||
|
||||
debug.Log("%v hasn't changed, but contents are missing!", target)
|
||||
// There are contents missing - inform user!
|
||||
err := errors.Errorf("parts of %v not found in the repository index; storing the file again", target)
|
||||
arch.error(abstarget, fi, err)
|
||||
}
|
||||
|
||||
// reopen file and do an fstat() on the open file to check it is still
|
||||
// a file (and has not been exchanged for e.g. a symlink)
|
||||
file, err := arch.FS.OpenFile(target, fs.O_RDONLY|fs.O_NOFOLLOW, 0)
|
||||
@ -398,30 +422,6 @@ func (arch *Archiver) Save(ctx context.Context, snPath, target string, previous
|
||||
return FutureNode{}, true, nil
|
||||
}
|
||||
|
||||
// use previous list of blobs if the file hasn't changed
|
||||
if previous != nil && !fileChanged(fi, previous, arch.IgnoreInode) {
|
||||
if arch.allBlobsPresent(previous) {
|
||||
debug.Log("%v hasn't changed, using old list of blobs", target)
|
||||
arch.CompleteItem(snPath, previous, previous, ItemStats{}, time.Since(start))
|
||||
arch.CompleteBlob(snPath, previous.Size)
|
||||
fn.node, err = arch.nodeFromFileInfo(target, fi)
|
||||
if err != nil {
|
||||
return FutureNode{}, false, err
|
||||
}
|
||||
|
||||
// copy list of blobs
|
||||
fn.node.Content = previous.Content
|
||||
|
||||
_ = file.Close()
|
||||
return fn, false, nil
|
||||
}
|
||||
|
||||
debug.Log("%v hasn't changed, but contents are missing!", target)
|
||||
// There are contents missing - inform user!
|
||||
err := errors.Errorf("parts of %v not found in the repository index; storing the file again", target)
|
||||
arch.error(abstarget, fi, err)
|
||||
}
|
||||
|
||||
fn.isFile = true
|
||||
// Save will close the file, we don't need to do that
|
||||
fn.file = arch.fileSaver.Save(ctx, snPath, file, fi, func() {
|
||||
|
Loading…
Reference in New Issue
Block a user