mirror of
https://github.com/octoleo/restic.git
synced 2024-12-25 20:11:06 +00:00
4032af0b78
The sftp library introduced a change so that the error returned for (among others) Lstat() can be used with os.IsNotExist() to test whether the target file does not exist.
462 lines
10 KiB
Go
462 lines
10 KiB
Go
package sftp
|
|
|
|
import (
|
|
"crypto/rand"
|
|
"encoding/hex"
|
|
"fmt"
|
|
"io"
|
|
"log"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"strings"
|
|
|
|
"github.com/juju/errors"
|
|
"github.com/pkg/sftp"
|
|
"github.com/restic/restic/backend"
|
|
"github.com/restic/restic/debug"
|
|
)
|
|
|
|
const (
|
|
tempfileRandomSuffixLength = 10
|
|
)
|
|
|
|
// SFTP is a backend in a directory accessed via SFTP.
|
|
type SFTP struct {
|
|
c *sftp.Client
|
|
p string
|
|
|
|
cmd *exec.Cmd
|
|
}
|
|
|
|
func startClient(program string, args ...string) (*SFTP, error) {
|
|
// Connect to a remote host and request the sftp subsystem via the 'ssh'
|
|
// command. This assumes that passwordless login is correctly configured.
|
|
cmd := exec.Command(program, args...)
|
|
|
|
// send errors from ssh to stderr
|
|
cmd.Stderr = os.Stderr
|
|
|
|
// ignore signals sent to the parent (e.g. SIGINT)
|
|
cmd.SysProcAttr = ignoreSigIntProcAttr()
|
|
|
|
// get stdin and stdout
|
|
wr, err := cmd.StdinPipe()
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
rd, err := cmd.StdoutPipe()
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
|
|
// start the process
|
|
if err := cmd.Start(); err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
|
|
// open the SFTP session
|
|
client, err := sftp.NewClientPipe(rd, wr)
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
|
|
return &SFTP{c: client, cmd: cmd}, nil
|
|
}
|
|
|
|
func paths(dir string) []string {
|
|
return []string{
|
|
dir,
|
|
Join(dir, backend.Paths.Data),
|
|
Join(dir, backend.Paths.Snapshots),
|
|
Join(dir, backend.Paths.Index),
|
|
Join(dir, backend.Paths.Locks),
|
|
Join(dir, backend.Paths.Keys),
|
|
Join(dir, backend.Paths.Temp),
|
|
}
|
|
}
|
|
|
|
// Open opens an sftp backend. When the command is started via
|
|
// exec.Command, it is expected to speak sftp on stdin/stdout. The backend
|
|
// is expected at the given path.
|
|
func Open(dir string, program string, args ...string) (*SFTP, error) {
|
|
sftp, err := startClient(program, args...)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// test if all necessary dirs and files are there
|
|
for _, d := range paths(dir) {
|
|
if _, err := sftp.c.Lstat(d); err != nil {
|
|
return nil, fmt.Errorf("%s does not exist", d)
|
|
}
|
|
}
|
|
|
|
sftp.p = dir
|
|
return sftp, nil
|
|
}
|
|
|
|
func buildSSHCommand(cfg Config) []string {
|
|
args := []string{cfg.Host}
|
|
if cfg.User != "" {
|
|
args = append(args, "-l")
|
|
args = append(args, cfg.User)
|
|
}
|
|
args = append(args, "-s")
|
|
args = append(args, "sftp")
|
|
return args
|
|
}
|
|
|
|
// OpenWithConfig opens an sftp backend as described by the config by running
|
|
// "ssh" with the appropiate arguments.
|
|
func OpenWithConfig(cfg Config) (*SFTP, error) {
|
|
return Open(cfg.Dir, "ssh", buildSSHCommand(cfg)...)
|
|
}
|
|
|
|
// Create creates all the necessary files and directories for a new sftp
|
|
// backend at dir. Afterwards a new config blob should be created.
|
|
func Create(dir string, program string, args ...string) (*SFTP, error) {
|
|
sftp, err := startClient(program, args...)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// test if config file already exists
|
|
_, err = sftp.c.Lstat(Join(dir, backend.Paths.Config))
|
|
if err == nil {
|
|
return nil, errors.New("config file already exists")
|
|
}
|
|
|
|
// create paths for data, refs and temp blobs
|
|
for _, d := range paths(dir) {
|
|
err = sftp.mkdirAll(d, backend.Modes.Dir)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
|
|
err = sftp.c.Close()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
err = sftp.cmd.Wait()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// open backend
|
|
return Open(dir, program, args...)
|
|
}
|
|
|
|
// CreateWithConfig creates an sftp backend as described by the config by running
|
|
// "ssh" with the appropiate arguments.
|
|
func CreateWithConfig(cfg Config) (*SFTP, error) {
|
|
return Create(cfg.Dir, "ssh", buildSSHCommand(cfg)...)
|
|
}
|
|
|
|
// Location returns this backend's location (the directory name).
|
|
func (r *SFTP) Location() string {
|
|
return r.p
|
|
}
|
|
|
|
// Return temp directory in correct directory for this backend.
|
|
func (r *SFTP) tempFile() (string, *sftp.File, error) {
|
|
// choose random suffix
|
|
buf := make([]byte, tempfileRandomSuffixLength)
|
|
_, err := io.ReadFull(rand.Reader, buf)
|
|
if err != nil {
|
|
return "", nil, errors.Annotatef(err,
|
|
"unable to read %d random bytes for tempfile name",
|
|
tempfileRandomSuffixLength)
|
|
}
|
|
|
|
// construct tempfile name
|
|
name := Join(r.p, backend.Paths.Temp, "temp-"+hex.EncodeToString(buf))
|
|
|
|
// create file in temp dir
|
|
f, err := r.c.Create(name)
|
|
if err != nil {
|
|
return "", nil, errors.Annotatef(err, "creating tempfile %q failed", name)
|
|
}
|
|
|
|
return name, f, nil
|
|
}
|
|
|
|
func (r *SFTP) mkdirAll(dir string, mode os.FileMode) error {
|
|
// check if directory already exists
|
|
fi, err := r.c.Lstat(dir)
|
|
if err == nil {
|
|
if fi.IsDir() {
|
|
return nil
|
|
}
|
|
|
|
return fmt.Errorf("mkdirAll(%s): entry exists but is not a directory", dir)
|
|
}
|
|
|
|
// create parent directories
|
|
errMkdirAll := r.mkdirAll(filepath.Dir(dir), backend.Modes.Dir)
|
|
|
|
// create directory
|
|
errMkdir := r.c.Mkdir(dir)
|
|
|
|
// test if directory was created successfully
|
|
fi, err = r.c.Lstat(dir)
|
|
if err != nil {
|
|
// return previous errors
|
|
return fmt.Errorf("mkdirAll(%s): unable to create directories: %v, %v", dir, errMkdirAll, errMkdir)
|
|
}
|
|
|
|
if !fi.IsDir() {
|
|
return fmt.Errorf("mkdirAll(%s): entry exists but is not a directory", dir)
|
|
}
|
|
|
|
// set mode
|
|
return r.c.Chmod(dir, mode)
|
|
}
|
|
|
|
// Rename temp file to final name according to type and name.
|
|
func (r *SFTP) renameFile(oldname string, t backend.Type, name string) error {
|
|
filename := r.filename(t, name)
|
|
|
|
// create directories if necessary
|
|
if t == backend.Data {
|
|
err := r.mkdirAll(filepath.Dir(filename), backend.Modes.Dir)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
// test if new file exists
|
|
if _, err := r.c.Lstat(filename); err == nil {
|
|
return fmt.Errorf("Close(): file %v already exists", filename)
|
|
}
|
|
|
|
err := r.c.Rename(oldname, filename)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// set mode to read-only
|
|
fi, err := r.c.Lstat(filename)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
return r.c.Chmod(filename, fi.Mode()&os.FileMode(^uint32(0222)))
|
|
}
|
|
|
|
// Join joins the given paths and cleans them afterwards.
|
|
func Join(parts ...string) string {
|
|
return filepath.Clean(strings.Join(parts, "/"))
|
|
}
|
|
|
|
// Construct path for given backend.Type and name.
|
|
func (r *SFTP) filename(t backend.Type, name string) string {
|
|
if t == backend.Config {
|
|
return Join(r.p, "config")
|
|
}
|
|
|
|
return Join(r.dirname(t, name), name)
|
|
}
|
|
|
|
// Construct directory for given backend.Type.
|
|
func (r *SFTP) dirname(t backend.Type, name string) string {
|
|
var n string
|
|
switch t {
|
|
case backend.Data:
|
|
n = backend.Paths.Data
|
|
if len(name) > 2 {
|
|
n = Join(n, name[:2])
|
|
}
|
|
case backend.Snapshot:
|
|
n = backend.Paths.Snapshots
|
|
case backend.Index:
|
|
n = backend.Paths.Index
|
|
case backend.Lock:
|
|
n = backend.Paths.Locks
|
|
case backend.Key:
|
|
n = backend.Paths.Keys
|
|
}
|
|
return Join(r.p, n)
|
|
}
|
|
|
|
// Load returns the data stored in the backend for h at the given offset
|
|
// and saves it in p. Load has the same semantics as io.ReaderAt.
|
|
func (r *SFTP) Load(h backend.Handle, p []byte, off int64) (n int, err error) {
|
|
if err := h.Valid(); err != nil {
|
|
return 0, err
|
|
}
|
|
|
|
f, err := r.c.Open(r.filename(h.Type, h.Name))
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
|
|
defer func() {
|
|
e := f.Close()
|
|
if err == nil && e != nil {
|
|
err = e
|
|
}
|
|
}()
|
|
|
|
if off > 0 {
|
|
_, err = f.Seek(off, 0)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
}
|
|
|
|
return io.ReadFull(f, p)
|
|
}
|
|
|
|
// Save stores data in the backend at the handle.
|
|
func (r *SFTP) Save(h backend.Handle, p []byte) (err error) {
|
|
if err := h.Valid(); err != nil {
|
|
return err
|
|
}
|
|
|
|
filename, tmpfile, err := r.tempFile()
|
|
debug.Log("sftp.Save", "save %v (%d bytes) to %v", h, len(p), filename)
|
|
|
|
n, err := tmpfile.Write(p)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if n != len(p) {
|
|
return errors.New("not all bytes writen")
|
|
}
|
|
|
|
err = tmpfile.Close()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
err = r.renameFile(filename, h.Type, h.Name)
|
|
debug.Log("sftp.Save", "save %v: rename %v: %v",
|
|
h, filepath.Base(filename), err)
|
|
if err != nil {
|
|
return fmt.Errorf("sftp: renameFile: %v", err)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// Stat returns information about a blob.
|
|
func (r *SFTP) Stat(h backend.Handle) (backend.BlobInfo, error) {
|
|
if err := h.Valid(); err != nil {
|
|
return backend.BlobInfo{}, err
|
|
}
|
|
|
|
fi, err := r.c.Lstat(r.filename(h.Type, h.Name))
|
|
if err != nil {
|
|
return backend.BlobInfo{}, err
|
|
}
|
|
|
|
return backend.BlobInfo{Size: fi.Size()}, nil
|
|
}
|
|
|
|
// Test returns true if a blob of the given type and name exists in the backend.
|
|
func (r *SFTP) Test(t backend.Type, name string) (bool, error) {
|
|
_, err := r.c.Lstat(r.filename(t, name))
|
|
if os.IsNotExist(err) {
|
|
return false, nil
|
|
}
|
|
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
|
|
return true, nil
|
|
}
|
|
|
|
// Remove removes the content stored at name.
|
|
func (r *SFTP) Remove(t backend.Type, name string) error {
|
|
return r.c.Remove(r.filename(t, name))
|
|
}
|
|
|
|
// List returns a channel that yields all names of blobs of type t. A
|
|
// goroutine is started for this. If the channel done is closed, sending
|
|
// stops.
|
|
func (r *SFTP) List(t backend.Type, done <-chan struct{}) <-chan string {
|
|
ch := make(chan string)
|
|
|
|
go func() {
|
|
defer close(ch)
|
|
|
|
if t == backend.Data {
|
|
// read first level
|
|
basedir := r.dirname(t, "")
|
|
|
|
list1, err := r.c.ReadDir(basedir)
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
dirs := make([]string, 0, len(list1))
|
|
for _, d := range list1 {
|
|
dirs = append(dirs, d.Name())
|
|
}
|
|
|
|
// read files
|
|
for _, dir := range dirs {
|
|
entries, err := r.c.ReadDir(Join(basedir, dir))
|
|
if err != nil {
|
|
continue
|
|
}
|
|
|
|
items := make([]string, 0, len(entries))
|
|
for _, entry := range entries {
|
|
items = append(items, entry.Name())
|
|
}
|
|
|
|
for _, file := range items {
|
|
select {
|
|
case ch <- file:
|
|
case <-done:
|
|
return
|
|
}
|
|
}
|
|
}
|
|
} else {
|
|
entries, err := r.c.ReadDir(r.dirname(t, ""))
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
items := make([]string, 0, len(entries))
|
|
for _, entry := range entries {
|
|
items = append(items, entry.Name())
|
|
}
|
|
|
|
for _, file := range items {
|
|
select {
|
|
case ch <- file:
|
|
case <-done:
|
|
return
|
|
}
|
|
}
|
|
}
|
|
}()
|
|
|
|
return ch
|
|
|
|
}
|
|
|
|
// Close closes the sftp connection and terminates the underlying command.
|
|
func (r *SFTP) Close() error {
|
|
if r == nil {
|
|
return nil
|
|
}
|
|
|
|
err := r.c.Close()
|
|
debug.Log("sftp.Close", "Close returned error %v", err)
|
|
|
|
if err := r.cmd.Process.Kill(); err != nil {
|
|
return err
|
|
}
|
|
|
|
return r.cmd.Wait()
|
|
}
|