mirror of
https://github.com/octoleo/restic.git
synced 2024-12-25 12:09:59 +00:00
488 lines
10 KiB
Go
488 lines
10 KiB
Go
package sftp
|
|
|
|
import (
|
|
"crypto/rand"
|
|
"encoding/hex"
|
|
"fmt"
|
|
"io"
|
|
"log"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"sort"
|
|
"strings"
|
|
|
|
"github.com/juju/errors"
|
|
"github.com/pkg/sftp"
|
|
"github.com/restic/restic/backend"
|
|
)
|
|
|
|
const (
|
|
tempfileRandomSuffixLength = 10
|
|
)
|
|
|
|
type SFTP struct {
|
|
c *sftp.Client
|
|
p string
|
|
|
|
cmd *exec.Cmd
|
|
}
|
|
|
|
func startClient(program string, args ...string) (*SFTP, error) {
|
|
// Connect to a remote host and request the sftp subsystem via the 'ssh'
|
|
// command. This assumes that passwordless login is correctly configured.
|
|
cmd := exec.Command(program, args...)
|
|
|
|
// send errors from ssh to stderr
|
|
cmd.Stderr = os.Stderr
|
|
|
|
// ignore signals sent to the parent (e.g. SIGINT)
|
|
cmd.SysProcAttr = ignoreSigIntProcAttr()
|
|
|
|
// get stdin and stdout
|
|
wr, err := cmd.StdinPipe()
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
rd, err := cmd.StdoutPipe()
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
|
|
// start the process
|
|
if err := cmd.Start(); err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
|
|
// open the SFTP session
|
|
client, err := sftp.NewClientPipe(rd, wr)
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
|
|
return &SFTP{c: client, cmd: cmd}, nil
|
|
}
|
|
|
|
// Open opens an sftp backend. When the command is started via
|
|
// exec.Command, it is expected to speak sftp on stdin/stdout. The backend
|
|
// is expected at the given path.
|
|
func Open(dir string, program string, args ...string) (*SFTP, error) {
|
|
sftp, err := startClient(program, args...)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// test if all necessary dirs and files are there
|
|
items := []string{
|
|
dir,
|
|
Join(dir, backend.Paths.Data),
|
|
Join(dir, backend.Paths.Snapshots),
|
|
Join(dir, backend.Paths.Index),
|
|
Join(dir, backend.Paths.Locks),
|
|
Join(dir, backend.Paths.Keys),
|
|
Join(dir, backend.Paths.Temp),
|
|
}
|
|
for _, d := range items {
|
|
if _, err := sftp.c.Lstat(d); err != nil {
|
|
return nil, fmt.Errorf("%s does not exist", d)
|
|
}
|
|
}
|
|
|
|
sftp.p = dir
|
|
return sftp, nil
|
|
}
|
|
|
|
func buildSSHCommand(cfg Config) []string {
|
|
args := []string{cfg.Host}
|
|
if cfg.User != "" {
|
|
args = append(args, "-l")
|
|
args = append(args, cfg.User)
|
|
}
|
|
args = append(args, "-s")
|
|
args = append(args, "sftp")
|
|
return args
|
|
}
|
|
|
|
// OpenWithConfig opens an sftp backend as described by the config by running
|
|
// "ssh" with the appropiate arguments.
|
|
func OpenWithConfig(cfg Config) (*SFTP, error) {
|
|
return Open(cfg.Dir, "ssh", buildSSHCommand(cfg)...)
|
|
}
|
|
|
|
// Create creates all the necessary files and directories for a new sftp
|
|
// backend at dir. Afterwards a new config blob should be created.
|
|
func Create(dir string, program string, args ...string) (*SFTP, error) {
|
|
sftp, err := startClient(program, args...)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
dirs := []string{
|
|
dir,
|
|
Join(dir, backend.Paths.Data),
|
|
Join(dir, backend.Paths.Snapshots),
|
|
Join(dir, backend.Paths.Index),
|
|
Join(dir, backend.Paths.Locks),
|
|
Join(dir, backend.Paths.Keys),
|
|
Join(dir, backend.Paths.Temp),
|
|
}
|
|
|
|
// test if config file already exists
|
|
_, err = sftp.c.Lstat(Join(dir, backend.Paths.Config))
|
|
if err == nil {
|
|
return nil, errors.New("config file already exists")
|
|
}
|
|
|
|
// create paths for data, refs and temp blobs
|
|
for _, d := range dirs {
|
|
err = sftp.mkdirAll(d, backend.Modes.Dir)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
|
|
err = sftp.c.Close()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
err = sftp.cmd.Wait()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// open backend
|
|
return Open(dir, program, args...)
|
|
}
|
|
|
|
// CreateWithConfig creates an sftp backend as described by the config by running
|
|
// "ssh" with the appropiate arguments.
|
|
func CreateWithConfig(cfg Config) (*SFTP, error) {
|
|
return Create(cfg.Dir, "ssh", buildSSHCommand(cfg)...)
|
|
}
|
|
|
|
// Location returns this backend's location (the directory name).
|
|
func (r *SFTP) Location() string {
|
|
return r.p
|
|
}
|
|
|
|
// Return temp directory in correct directory for this backend.
|
|
func (r *SFTP) tempFile() (string, *sftp.File, error) {
|
|
// choose random suffix
|
|
buf := make([]byte, tempfileRandomSuffixLength)
|
|
_, err := io.ReadFull(rand.Reader, buf)
|
|
if err != nil {
|
|
return "", nil, errors.Annotatef(err,
|
|
"unable to read %d random bytes for tempfile name",
|
|
tempfileRandomSuffixLength)
|
|
}
|
|
|
|
// construct tempfile name
|
|
name := Join(r.p, backend.Paths.Temp, "temp-"+hex.EncodeToString(buf))
|
|
|
|
// create file in temp dir
|
|
f, err := r.c.Create(name)
|
|
if err != nil {
|
|
return "", nil, errors.Annotatef(err, "creating tempfile %q failed", name)
|
|
}
|
|
|
|
return name, f, nil
|
|
}
|
|
|
|
func (r *SFTP) mkdirAll(dir string, mode os.FileMode) error {
|
|
// check if directory already exists
|
|
fi, err := r.c.Lstat(dir)
|
|
if err == nil {
|
|
if fi.IsDir() {
|
|
return nil
|
|
}
|
|
|
|
return fmt.Errorf("mkdirAll(%s): entry exists but is not a directory", dir)
|
|
}
|
|
|
|
// create parent directories
|
|
errMkdirAll := r.mkdirAll(filepath.Dir(dir), backend.Modes.Dir)
|
|
|
|
// create directory
|
|
errMkdir := r.c.Mkdir(dir)
|
|
|
|
// test if directory was created successfully
|
|
fi, err = r.c.Lstat(dir)
|
|
if err != nil {
|
|
// return previous errors
|
|
return fmt.Errorf("mkdirAll(%s): unable to create directories: %v, %v", dir, errMkdirAll, errMkdir)
|
|
}
|
|
|
|
if !fi.IsDir() {
|
|
return fmt.Errorf("mkdirAll(%s): entry exists but is not a directory", dir)
|
|
}
|
|
|
|
// set mode
|
|
return r.c.Chmod(dir, mode)
|
|
}
|
|
|
|
// Rename temp file to final name according to type and name.
|
|
func (r *SFTP) renameFile(oldname string, t backend.Type, name string) error {
|
|
filename := r.filename(t, name)
|
|
|
|
// create directories if necessary
|
|
if t == backend.Data {
|
|
err := r.mkdirAll(filepath.Dir(filename), backend.Modes.Dir)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
// test if new file exists
|
|
if _, err := r.c.Lstat(filename); err == nil {
|
|
return fmt.Errorf("Close(): file %v already exists", filename)
|
|
}
|
|
|
|
err := r.c.Rename(oldname, filename)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// set mode to read-only
|
|
fi, err := r.c.Lstat(filename)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
return r.c.Chmod(filename, fi.Mode()&os.FileMode(^uint32(0222)))
|
|
}
|
|
|
|
type sftpBlob struct {
|
|
f *sftp.File
|
|
tempname string
|
|
size uint
|
|
closed bool
|
|
backend *SFTP
|
|
}
|
|
|
|
func (sb *sftpBlob) Finalize(t backend.Type, name string) error {
|
|
if sb.closed {
|
|
return errors.New("Close() called on closed file")
|
|
}
|
|
sb.closed = true
|
|
|
|
err := sb.f.Close()
|
|
if err != nil {
|
|
return fmt.Errorf("sftp: file.Close: %v", err)
|
|
}
|
|
|
|
// rename file
|
|
err = sb.backend.renameFile(sb.tempname, t, name)
|
|
if err != nil {
|
|
return fmt.Errorf("sftp: renameFile: %v", err)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (sb *sftpBlob) Write(p []byte) (int, error) {
|
|
n, err := sb.f.Write(p)
|
|
sb.size += uint(n)
|
|
return n, err
|
|
}
|
|
|
|
func (sb *sftpBlob) Size() uint {
|
|
return sb.size
|
|
}
|
|
|
|
// Create creates a new Blob. The data is available only after Finalize()
|
|
// has been called on the returned Blob.
|
|
func (r *SFTP) Create() (backend.Blob, error) {
|
|
// TODO: make sure that tempfile is removed upon error
|
|
|
|
// create tempfile in backend
|
|
filename, file, err := r.tempFile()
|
|
if err != nil {
|
|
return nil, errors.Annotate(err, "create tempfile")
|
|
}
|
|
|
|
blob := sftpBlob{
|
|
f: file,
|
|
tempname: filename,
|
|
backend: r,
|
|
}
|
|
|
|
return &blob, nil
|
|
}
|
|
|
|
func Join(parts ...string) string {
|
|
return filepath.Clean(strings.Join(parts, "/"))
|
|
}
|
|
|
|
// Construct path for given backend.Type and name.
|
|
func (r *SFTP) filename(t backend.Type, name string) string {
|
|
if t == backend.Config {
|
|
return Join(r.p, "config")
|
|
}
|
|
|
|
return Join(r.dirname(t, name), name)
|
|
}
|
|
|
|
// Construct directory for given backend.Type.
|
|
func (r *SFTP) dirname(t backend.Type, name string) string {
|
|
var n string
|
|
switch t {
|
|
case backend.Data:
|
|
n = backend.Paths.Data
|
|
if len(name) > 2 {
|
|
n = Join(n, name[:2])
|
|
}
|
|
case backend.Snapshot:
|
|
n = backend.Paths.Snapshots
|
|
case backend.Index:
|
|
n = backend.Paths.Index
|
|
case backend.Lock:
|
|
n = backend.Paths.Locks
|
|
case backend.Key:
|
|
n = backend.Paths.Keys
|
|
}
|
|
return Join(r.p, n)
|
|
}
|
|
|
|
// Get returns a reader that yields the content stored under the given
|
|
// name. The reader should be closed after draining it.
|
|
func (r *SFTP) Get(t backend.Type, name string) (io.ReadCloser, error) {
|
|
// try to open file
|
|
file, err := r.c.Open(r.filename(t, name))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return file, nil
|
|
}
|
|
|
|
// GetReader returns an io.ReadCloser for the Blob with the given name of
|
|
// type t at offset and length. If length is 0, the reader reads until EOF.
|
|
func (r *SFTP) GetReader(t backend.Type, name string, offset, length uint) (io.ReadCloser, error) {
|
|
f, err := r.c.Open(r.filename(t, name))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
if offset > 0 {
|
|
_, err = f.Seek(int64(offset), 0)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
|
|
if length == 0 {
|
|
return f, nil
|
|
}
|
|
|
|
return backend.LimitReadCloser(f, int64(length)), nil
|
|
}
|
|
|
|
// Test returns true if a blob of the given type and name exists in the backend.
|
|
func (r *SFTP) Test(t backend.Type, name string) (bool, error) {
|
|
_, err := r.c.Lstat(r.filename(t, name))
|
|
if err != nil {
|
|
if _, ok := err.(*sftp.StatusError); ok {
|
|
return false, nil
|
|
}
|
|
|
|
return false, err
|
|
}
|
|
|
|
return true, nil
|
|
}
|
|
|
|
// Remove removes the content stored at name.
|
|
func (r *SFTP) Remove(t backend.Type, name string) error {
|
|
return r.c.Remove(r.filename(t, name))
|
|
}
|
|
|
|
// List returns a channel that yields all names of blobs of type t. A
|
|
// goroutine is started for this. If the channel done is closed, sending
|
|
// stops.
|
|
func (r *SFTP) List(t backend.Type, done <-chan struct{}) <-chan string {
|
|
ch := make(chan string)
|
|
|
|
go func() {
|
|
defer close(ch)
|
|
|
|
if t == backend.Data {
|
|
// read first level
|
|
basedir := r.dirname(t, "")
|
|
|
|
list1, err := r.c.ReadDir(basedir)
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
dirs := make([]string, 0, len(list1))
|
|
for _, d := range list1 {
|
|
dirs = append(dirs, d.Name())
|
|
}
|
|
|
|
sort.Strings(dirs)
|
|
|
|
// read files
|
|
for _, dir := range dirs {
|
|
entries, err := r.c.ReadDir(Join(basedir, dir))
|
|
if err != nil {
|
|
continue
|
|
}
|
|
|
|
items := make([]string, 0, len(entries))
|
|
for _, entry := range entries {
|
|
items = append(items, entry.Name())
|
|
}
|
|
|
|
sort.Strings(items)
|
|
|
|
for _, file := range items {
|
|
select {
|
|
case ch <- file:
|
|
case <-done:
|
|
return
|
|
}
|
|
}
|
|
}
|
|
} else {
|
|
entries, err := r.c.ReadDir(r.dirname(t, ""))
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
items := make([]string, 0, len(entries))
|
|
for _, entry := range entries {
|
|
items = append(items, entry.Name())
|
|
}
|
|
|
|
sort.Strings(items)
|
|
|
|
for _, file := range items {
|
|
select {
|
|
case ch <- file:
|
|
case <-done:
|
|
return
|
|
}
|
|
}
|
|
}
|
|
}()
|
|
|
|
return ch
|
|
|
|
}
|
|
|
|
// Close closes the sftp connection and terminates the underlying command.
|
|
func (s *SFTP) Close() error {
|
|
if s == nil {
|
|
return nil
|
|
}
|
|
|
|
s.c.Close()
|
|
|
|
if err := s.cmd.Process.Kill(); err != nil {
|
|
return err
|
|
}
|
|
|
|
return s.cmd.Wait()
|
|
}
|