From f848afed276a99d80c28383564b9d20e131381cf Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 4 Oct 2014 19:20:15 +0200 Subject: [PATCH] Add SFTP backend --- backend/interface.go | 2 + backend/local.go | 12 +- backend/sftp.go | 369 +++++++++++++++++++++++++++++++++++++++ cmd/khepri/cmd_backup.go | 2 +- cmd/khepri/cmd_init.go | 34 ---- cmd/khepri/main.go | 74 +++++++- 6 files changed, 456 insertions(+), 37 deletions(-) create mode 100644 backend/sftp.go delete mode 100644 cmd/khepri/cmd_init.go diff --git a/backend/interface.go b/backend/interface.go index efe7dcbb1..b6ca94d70 100644 --- a/backend/interface.go +++ b/backend/interface.go @@ -22,5 +22,7 @@ type Server interface { Remove(Type, ID) error Version() uint + Close() error + Location() string } diff --git a/backend/local.go b/backend/local.go index cd0079127..71e28549b 100644 --- a/backend/local.go +++ b/backend/local.go @@ -186,7 +186,12 @@ func (b *Local) Create(t Type, data []byte) (ID, error) { return nil, err } - // close tempfile, return id + err = file.Close() + if err != nil { + return nil, err + } + + // return id id := IDFromData(data) err = b.renameFile(file, t, id) if err != nil { @@ -276,3 +281,8 @@ func (b *Local) List(t Type) (IDs, error) { func (b *Local) Version() uint { return b.ver } + +// Close closes the repository +func (b *Local) Close() error { + return nil +} diff --git a/backend/sftp.go b/backend/sftp.go new file mode 100644 index 000000000..09290fe69 --- /dev/null +++ b/backend/sftp.go @@ -0,0 +1,369 @@ +package backend + +import ( + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "io" + "io/ioutil" + "log" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + + "github.com/pkg/sftp" +) + +const ( + // dirMode = 0700 + // blobPath = "blobs" + // snapshotPath = "snapshots" + // treePath = "trees" + // lockPath = "locks" + // keyPath = "keys" + // tempPath = "tmp" + // versionFileName = "version" + + tempfileRandomSuffixLength = 10 +) + +type SFTP struct { + c *sftp.Client + p string + ver uint + + cmd *exec.Cmd +} + +func start_client(program string, args ...string) (*SFTP, error) { + // Connect to a remote host and request the sftp subsystem via the 'ssh' + // command. This assumes that passwordless login is correctly configured. + cmd := exec.Command(program, args...) + + // send errors from ssh to stderr + cmd.Stderr = os.Stderr + + // get stdin and stdout + wr, err := cmd.StdinPipe() + if err != nil { + log.Fatal(err) + } + rd, err := cmd.StdoutPipe() + if err != nil { + log.Fatal(err) + } + + // start the process + if err := cmd.Start(); err != nil { + log.Fatal(err) + } + + // open the SFTP session + client, err := sftp.NewClientPipe(rd, wr) + if err != nil { + log.Fatal(err) + } + + return &SFTP{c: client, cmd: cmd}, nil +} + +// OpenSFTP opens an sftp backend. When the command is started via +// exec.Command, it is expected to speak sftp on stdin/stdout. The repository +// is expected at the given path. +func OpenSFTP(dir string, program string, args ...string) (*SFTP, error) { + sftp, err := start_client(program, args...) + if err != nil { + return nil, err + } + + // test if all necessary dirs and files are there + items := []string{ + dir, + filepath.Join(dir, blobPath), + filepath.Join(dir, snapshotPath), + filepath.Join(dir, treePath), + filepath.Join(dir, lockPath), + filepath.Join(dir, keyPath), + filepath.Join(dir, tempPath), + } + for _, d := range items { + if _, err := sftp.c.Lstat(d); err != nil { + return nil, fmt.Errorf("%s does not exist", d) + } + } + + // read version file + f, err := sftp.c.Open(filepath.Join(dir, versionFileName)) + if err != nil { + return nil, fmt.Errorf("unable to read version file: %v\n", err) + } + + buf := make([]byte, 100) + n, err := f.Read(buf) + if err != nil && err != io.EOF { + return nil, err + } + + err = f.Close() + if err != nil { + return nil, err + } + + version, err := strconv.Atoi(strings.TrimSpace(string(buf[:n]))) + if err != nil { + return nil, fmt.Errorf("unable to convert version to integer: %v\n", err) + } + + if version != BackendVersion { + return nil, fmt.Errorf("wrong version %d", version) + } + + // check version + if version != BackendVersion { + return nil, fmt.Errorf("wrong version %d", version) + } + + sftp.p = dir + + return sftp, nil +} + +// CreateSFTP creates all the necessary files and directories for a new sftp +// backend at dir. +func CreateSFTP(dir string, program string, args ...string) (*SFTP, error) { + sftp, err := start_client(program, args...) + if err != nil { + return nil, err + } + + versionFile := filepath.Join(dir, versionFileName) + dirs := []string{ + dir, + filepath.Join(dir, blobPath), + filepath.Join(dir, snapshotPath), + filepath.Join(dir, treePath), + filepath.Join(dir, lockPath), + filepath.Join(dir, keyPath), + filepath.Join(dir, tempPath), + } + + // test if version file already exists + _, err = sftp.c.Lstat(versionFile) + if err == nil { + return nil, errors.New("version file already exists") + } + + // test if directories already exist + for _, d := range dirs[1:] { + if _, err := sftp.c.Lstat(d); err == nil { + return nil, fmt.Errorf("dir %s already exists", d) + } + } + + // create paths for blobs, refs and temp + for _, d := range dirs { + // TODO: implement client.MkdirAll() and set mode to dirMode + _, err = sftp.c.Lstat(d) + if err != nil { + err = sftp.c.Mkdir(d) + if err != nil { + return nil, err + } + } + } + + // create version file + f, err := sftp.c.Create(versionFile) + if err != nil { + return nil, err + } + + _, err = f.Write([]byte(strconv.Itoa(BackendVersion))) + if err != nil { + return nil, err + } + + err = f.Close() + if err != nil { + return nil, err + } + + err = sftp.c.Close() + if err != nil { + return nil, err + } + + err = sftp.cmd.Wait() + if err != nil { + return nil, err + } + + // open repository + return OpenSFTP(dir, program, args...) +} + +// Location returns this backend's location (the directory name). +func (r *SFTP) Location() string { + return r.p +} + +// Return temp directory in correct directory for this backend. +func (r *SFTP) tempFile() (string, *sftp.File, error) { + // choose random suffix + buf := make([]byte, tempfileRandomSuffixLength) + n, err := io.ReadFull(rand.Reader, buf) + if err != nil { + return "", nil, err + } + + if n != len(buf) { + return "", nil, errors.New("unable to generate enough random bytes for temp file") + } + + // construct tempfile name + name := filepath.Join(r.p, tempPath, fmt.Sprintf("temp-%s", hex.EncodeToString(buf))) + + // create file in temp dir + f, err := r.c.Create(name) + if err != nil { + return "", nil, err + } + + return name, f, nil +} + +// Rename temp file to final name according to type and ID. +func (r *SFTP) renameFile(filename string, t Type, id ID) error { + return r.c.Rename(filename, filepath.Join(r.dir(t), id.String())) +} + +// Construct directory for given Type. +func (r *SFTP) dir(t Type) string { + var n string + switch t { + case Blob: + n = blobPath + case Snapshot: + n = snapshotPath + case Tree: + n = treePath + case Lock: + n = lockPath + case Key: + n = keyPath + } + return filepath.Join(r.p, n) +} + +// Create stores new content of type t and data and returns the ID. +func (r *SFTP) Create(t Type, data []byte) (ID, error) { + // TODO: make sure that tempfile is removed upon error + + // create tempfile in repository + var err error + filename, file, err := r.tempFile() + if err != nil { + return nil, err + } + + // write data to tempfile + _, err = file.Write(data) + if err != nil { + return nil, err + } + + err = file.Close() + if err != nil { + return nil, err + } + + // return id + id := IDFromData(data) + err = r.renameFile(filename, t, id) + if err != nil { + return nil, err + } + + return id, nil +} + +// Construct path for given Type and ID. +func (r *SFTP) filename(t Type, id ID) string { + return filepath.Join(r.dir(t), id.String()) +} + +// Get returns the content stored under the given ID. +func (r *SFTP) Get(t Type, id ID) ([]byte, error) { + // try to open file + file, err := r.c.Open(r.filename(t, id)) + defer file.Close() + if err != nil { + return nil, err + } + + // read all + buf, err := ioutil.ReadAll(file) + if err != nil { + return nil, err + } + + return buf, nil +} + +// Test returns true if a blob of the given type and ID exists in the backend. +func (r *SFTP) Test(t Type, id ID) (bool, error) { + // try to open file + file, err := r.c.Open(r.filename(t, id)) + defer func() { + file.Close() + }() + + if err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, err + } + + return true, nil +} + +// Remove removes the content stored at ID. +func (r *SFTP) Remove(t Type, id ID) error { + return r.c.Remove(r.filename(t, id)) +} + +// List lists all objects of a given type. +func (r *SFTP) List(t Type) (IDs, error) { + list, err := r.c.ReadDir(r.dir(t)) + if err != nil { + return nil, err + } + + ids := make(IDs, 0, len(list)) + for _, item := range list { + id, err := ParseID(item.Name()) + // ignore everything that does not parse as an ID + if err != nil { + continue + } + ids = append(ids, id) + } + + return ids, nil +} + +// Version returns the version of this local backend. +func (r *SFTP) Version() uint { + return r.ver +} + +// Close closes the sftp connection and terminates the underlying command. +func (s *SFTP) Close() error { + s.c.Close() + // TODO: add timeout after which the process is killed + return s.cmd.Wait() +} diff --git a/cmd/khepri/cmd_backup.go b/cmd/khepri/cmd_backup.go index 97fe80b0b..6fdc6c761 100644 --- a/cmd/khepri/cmd_backup.go +++ b/cmd/khepri/cmd_backup.go @@ -21,7 +21,7 @@ func commandBackup(be backend.Server, key *khepri.Key, args []string) error { fmt.Fprintf(os.Stderr, "err: %v\n", err) } arch.Error = func(dir string, fi os.FileInfo, err error) error { - fmt.Fprintf(os.Stderr, "error for %s: %v\n%s\n", dir, err, fi) + fmt.Fprintf(os.Stderr, "error for %s: %v\n%v\n", dir, err, fi) return err } diff --git a/cmd/khepri/cmd_init.go b/cmd/khepri/cmd_init.go deleted file mode 100644 index b7992bcd7..000000000 --- a/cmd/khepri/cmd_init.go +++ /dev/null @@ -1,34 +0,0 @@ -package main - -import ( - "fmt" - "os" - - "github.com/fd0/khepri" - "github.com/fd0/khepri/backend" -) - -func commandInit(path string) error { - pw := read_password("enter password for new backend: ") - pw2 := read_password("enter password again: ") - - if pw != pw2 { - errx(1, "passwords do not match") - } - - be, err := backend.CreateLocal(path) - if err != nil { - fmt.Fprintf(os.Stderr, "creating local backend at %s failed: %v\n", path, err) - os.Exit(1) - } - - _, err = khepri.CreateKey(be, pw) - if err != nil { - fmt.Fprintf(os.Stderr, "creating key in local backend at %s failed: %v\n", path, err) - os.Exit(1) - } - - fmt.Printf("created khepri backend at %s\n", be.Location()) - - return nil -} diff --git a/cmd/khepri/main.go b/cmd/khepri/main.go index 0ef5c4635..ce8b81531 100644 --- a/cmd/khepri/main.go +++ b/cmd/khepri/main.go @@ -3,6 +3,7 @@ package main import ( "fmt" "log" + "net/url" "os" "sort" "strings" @@ -46,6 +47,77 @@ func read_password(prompt string) string { return string(pw) } +func commandInit(repo string) error { + pw := read_password("enter password for new backend: ") + pw2 := read_password("enter password again: ") + + if pw != pw2 { + errx(1, "passwords do not match") + } + + be, err := create(repo) + if err != nil { + fmt.Fprintf(os.Stderr, "creating backend at %s failed: %v\n", repo, err) + os.Exit(1) + } + + _, err = khepri.CreateKey(be, pw) + if err != nil { + fmt.Fprintf(os.Stderr, "creating key in backend at %s failed: %v\n", repo, err) + os.Exit(1) + } + + fmt.Printf("created khepri backend at %s\n", be.Location()) + + return nil +} + +// Open the backend specified by URI. +// Valid formats are: +// * /foo/bar -> local repository at /foo/bar +// * sftp://user@host/foo/bar -> remote sftp repository on host for user at path foo/bar +// * sftp://host//tmp/backup -> remote sftp repository on host at path /tmp/backup +func open(u string) (backend.Server, error) { + url, err := url.Parse(u) + if err != nil { + return nil, err + } + + if url.Scheme == "" { + return backend.OpenLocal(url.Path) + } else { + args := []string{url.Host} + if url.User != nil && url.User.Username() != "" { + args = append(args, "-l") + args = append(args, url.User.Username()) + } + args = append(args, "-s") + args = append(args, "sftp") + return backend.OpenSFTP(url.Path[1:], "ssh", args...) + } +} + +// Create the backend specified by URI. +func create(u string) (backend.Server, error) { + url, err := url.Parse(u) + if err != nil { + return nil, err + } + + if url.Scheme == "" { + return backend.CreateLocal(url.Path) + } else { + args := []string{url.Host} + if url.User != nil && url.User.Username() != "" { + args = append(args, "-l") + args = append(args, url.User.Username()) + } + args = append(args, "-s") + args = append(args, "sftp") + return backend.CreateSFTP(url.Path[1:], "ssh", args...) + } +} + func init() { commands = make(map[string]commandFunc) commands["backup"] = commandBackup @@ -94,7 +166,7 @@ func main() { } // read_password("enter password: ") - repo, err := backend.OpenLocal(Opts.Repo) + repo, err := open(Opts.Repo) if err != nil { errx(1, "unable to open repo: %v", err) }