octoleo/restic
2
2
Fork 0
mirror of https://github.com/octoleo/restic.git synced 2024-12-22 10:58:55 +00:00
Code Issues Releases Wiki Activity

sftp: Implement atomic uploads

Browse Source 
Create a temporary file with a sufficiently random name to essentially
avoid any chance of conflicts. Once the upload has finished remove the
temporary suffix. Interrupted upload thus will be ignored by restic.
This commit is contained in:
Michael Eischer 2021-09-22 22:28:57 +02:00
parent cc90f2ba6b
commit 5ec312ca06
2 changed files with 33 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
changelog/unreleased/issue-3003 Normal file
View File

@ -0,0 +1,10 @@
Enhancement: Atomic uploads for SFTP
The SFTP backend did not upload files atomically. An interrupted upload could
leave an incomplete file behind which could prevent restic from accessing the
repository.
Uploads in the SFTP backend are now done atomically.
https://github.com/restic/restic/issues/3003
https://github.com/restic/restic/pull/3524

27
internal/backend/sftp/sftp.go
View File

@ -3,6 +3,8 @@ package sftp
import (
"bufio"
"context"
"crypto/rand"
"encoding/hex"
"fmt"
"hash"
"io"
@ -252,6 +254,17 @@ func Join(parts ...string) string {
return path.Clean(path.Join(parts...))
}
// tempSuffix generates a random string suffix that should be sufficiently long
// to avoid accidential conflicts
func tempSuffix() string {
var nonce [16]byte
_, err := rand.Read(nonce[:])
if err != nil {
panic(err)
}
return hex.EncodeToString(nonce[:])
}
// Save stores data in the backend at the handle.
func (r *SFTP) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
debug.Log("Save %v", h)
@ -264,10 +277,11 @@ func (r *SFTP) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader
}
filename := r.Filename(h)
tmpFilename := filename + "-restic-temp-" + tempSuffix()
dirname := r.Dirname(h)
// create new file
f, err := r.c.OpenFile(filename, os.O_CREATE|os.O_EXCL|os.O_WRONLY)
f, err := r.c.OpenFile(tmpFilename, os.O_CREATE|os.O_EXCL|os.O_WRONLY)
if r.IsNotExist(err) {
// error is caused by a missing directory, try to create it
@ -276,7 +290,7 @@ func (r *SFTP) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader
debug.Log("error creating dir %v: %v", r.Dirname(h), mkdirErr)
} else {
// try again
f, err = r.c.OpenFile(filename, os.O_CREATE|os.O_EXCL|os.O_WRONLY)
f, err = r.c.OpenFile(tmpFilename, os.O_CREATE|os.O_EXCL|os.O_WRONLY)
}
}
@ -298,7 +312,7 @@ func (r *SFTP) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader
rmErr := r.c.Remove(f.Name())
if rmErr != nil {
debug.Log("sftp: failed to remove broken file %v: %v",
filename, rmErr)
f.Name(), rmErr)
}
err = r.checkNoSpace(dirname, rd.Length(), err)
@ -318,7 +332,12 @@ func (r *SFTP) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader
}
err = f.Close()
return errors.Wrap(err, "Close")
if err != nil {
return errors.Wrap(err, "Close")
}
err = r.c.Rename(tmpFilename, filename)
return errors.Wrap(err, "Rename")
}
// checkNoSpace checks if err was likely caused by lack of available space