From 2bb55f017d98893f5669033be6ead2ddc2401769 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 13 Feb 2016 19:11:35 +0100 Subject: [PATCH 1/2] Update pkg/sftp library --- Godeps/Godeps.json | 4 +- .../src/github.com/pkg/sftp/.gitignore | 8 + .../src/github.com/pkg/sftp/.travis.yml | 21 + .../src/github.com/pkg/sftp/README.md | 8 +- .../src/github.com/pkg/sftp/attrs.go | 107 +- .../src/github.com/pkg/sftp/attrs_stubs.go | 11 + .../src/github.com/pkg/sftp/attrs_test.go | 45 - .../src/github.com/pkg/sftp/attrs_unix.go | 17 + .../src/github.com/pkg/sftp/client.go | 299 +++-- .../pkg/sftp/client_integration_test.go | 1175 ----------------- .../src/github.com/pkg/sftp/client_test.go | 75 -- .../src/github.com/pkg/sftp/debug.go | 2 +- .../src/github.com/pkg/sftp/example_test.go | 91 -- .../pkg/sftp/examples/sftp-server/README.md | 12 + .../pkg/sftp/examples/sftp-server/main.go | 134 ++ .../src/github.com/pkg/sftp/packet.go | 555 +++++++- .../src/github.com/pkg/sftp/packet_test.go | 261 ---- .../src/github.com/pkg/sftp/release.go | 2 +- .../src/github.com/pkg/sftp/server.go | 648 +++++++++ .../pkg/sftp/server_standalone/main.go | 40 + .../src/github.com/pkg/sftp/server_stubs.go | 12 + .../src/github.com/pkg/sftp/server_unix.go | 143 ++ .../src/github.com/pkg/sftp/sftp.go | 32 +- .../src/github.com/pkg/sftp/wercker.yml | 1 - 24 files changed, 1890 insertions(+), 1813 deletions(-) create mode 100644 Godeps/_workspace/src/github.com/pkg/sftp/.gitignore create mode 100644 Godeps/_workspace/src/github.com/pkg/sftp/.travis.yml create mode 100644 Godeps/_workspace/src/github.com/pkg/sftp/attrs_stubs.go delete mode 100644 Godeps/_workspace/src/github.com/pkg/sftp/attrs_test.go create mode 100644 Godeps/_workspace/src/github.com/pkg/sftp/attrs_unix.go delete mode 100644 Godeps/_workspace/src/github.com/pkg/sftp/client_integration_test.go delete mode 100644 Godeps/_workspace/src/github.com/pkg/sftp/client_test.go delete mode 100644 Godeps/_workspace/src/github.com/pkg/sftp/example_test.go create mode 100644 Godeps/_workspace/src/github.com/pkg/sftp/examples/sftp-server/README.md create mode 100644 Godeps/_workspace/src/github.com/pkg/sftp/examples/sftp-server/main.go delete mode 100644 Godeps/_workspace/src/github.com/pkg/sftp/packet_test.go create mode 100644 Godeps/_workspace/src/github.com/pkg/sftp/server.go create mode 100644 Godeps/_workspace/src/github.com/pkg/sftp/server_standalone/main.go create mode 100644 Godeps/_workspace/src/github.com/pkg/sftp/server_stubs.go create mode 100644 Godeps/_workspace/src/github.com/pkg/sftp/server_unix.go delete mode 100644 Godeps/_workspace/src/github.com/pkg/sftp/wercker.yml diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index de55309e1..9b36036ab 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -1,6 +1,6 @@ { "ImportPath": "github.com/restic/restic", - "GoVersion": "go1.4.2", + "GoVersion": "go1.5", "Packages": [ "./..." ], @@ -29,7 +29,7 @@ }, { "ImportPath": "github.com/pkg/sftp", - "Rev": "518aed2757a65cfa64d4b1b2baf08410f8b7a6bc" + "Rev": "e84cc8c755ca39b7b64f510fe1fffc1b51f210a5" }, { "ImportPath": "github.com/restic/chunker", diff --git a/Godeps/_workspace/src/github.com/pkg/sftp/.gitignore b/Godeps/_workspace/src/github.com/pkg/sftp/.gitignore new file mode 100644 index 000000000..9fc1e3d24 --- /dev/null +++ b/Godeps/_workspace/src/github.com/pkg/sftp/.gitignore @@ -0,0 +1,8 @@ +.*.swo +.*.swp + +server_standalone/server_standalone + +examples/sftp-server/id_rsa +examples/sftp-server/id_rsa.pub +examples/sftp-server/sftp-server diff --git a/Godeps/_workspace/src/github.com/pkg/sftp/.travis.yml b/Godeps/_workspace/src/github.com/pkg/sftp/.travis.yml new file mode 100644 index 000000000..9ef70d869 --- /dev/null +++ b/Godeps/_workspace/src/github.com/pkg/sftp/.travis.yml @@ -0,0 +1,21 @@ +language: go +go_import_path: github.com/pkg/sftp +go: + - 1.5.2 + - 1.4.3 + - tip + +sudo: false + +addons: + ssh_known_hosts: + - bitbucket.org + +install: + - go get -t -v ./... + - ssh-keygen -t rsa -q -P "" -f /home/travis/.ssh/id_rsa + +script: + - go test -integration -v ./... + - go test -testserver -v ./... + - go test -integration -testserver -v ./... diff --git a/Godeps/_workspace/src/github.com/pkg/sftp/README.md b/Godeps/_workspace/src/github.com/pkg/sftp/README.md index d7058eb61..875bf2747 100644 --- a/Godeps/_workspace/src/github.com/pkg/sftp/README.md +++ b/Godeps/_workspace/src/github.com/pkg/sftp/README.md @@ -3,14 +3,14 @@ sftp The `sftp` package provides support for file system operations on remote ssh servers using the SFTP subsystem. -[![Build Status](https://drone.io/github.com/pkg/sftp/status.png)](https://drone.io/github.com/pkg/sftp/latest) +[![UNIX Build Status](https://travis-ci.org/pkg/sftp.svg?branch=master)](https://travis-ci.org/pkg/sftp) [![GoDoc](http://godoc.org/github.com/pkg/sftp?status.svg)](http://godoc.org/github.com/pkg/sftp) usage and examples ------------------ See [godoc.org/github.com/pkg/sftp](http://godoc.org/github.com/pkg/sftp) for examples and usage. -The basic operation of the package mirrors the facilities of the [os](http://golang.org/pkg/os) package. +The basic operation of the package mirrors the facilities of the [os](http://golang.org/pkg/os) package. The Walker interface for directory traversal is heavily inspired by Keith Rarick's [fs](http://godoc.org/github.com/kr/fs) package. @@ -24,4 +24,6 @@ roadmap contributing ------------ -Features, Issues, and Pull Requests are always welcome. +We welcome pull requests, bug fixes and issue reports. + +Before proposing a large change, first please discuss your change by raising an issue. diff --git a/Godeps/_workspace/src/github.com/pkg/sftp/attrs.go b/Godeps/_workspace/src/github.com/pkg/sftp/attrs.go index 0d37db080..3e4c2912d 100644 --- a/Godeps/_workspace/src/github.com/pkg/sftp/attrs.go +++ b/Godeps/_workspace/src/github.com/pkg/sftp/attrs.go @@ -50,11 +50,12 @@ type FileStat struct { Mode uint32 Mtime uint32 Atime uint32 - Uid uint32 - Gid uint32 + UID uint32 + GID uint32 Extended []StatExtended } +// StatExtended contains additional, extended information for a FileStat. type StatExtended struct { ExtType string ExtData string @@ -71,6 +72,26 @@ func fileInfoFromStat(st *FileStat, name string) os.FileInfo { return fs } +func fileStatFromInfo(fi os.FileInfo) (uint32, FileStat) { + mtime := fi.ModTime().Unix() + atime := mtime + var flags uint32 = ssh_FILEXFER_ATTR_SIZE | + ssh_FILEXFER_ATTR_PERMISSIONS | + ssh_FILEXFER_ATTR_ACMODTIME + + fileStat := FileStat{ + Size: uint64(fi.Size()), + Mode: fromFileMode(fi.Mode()), + Mtime: uint32(mtime), + Atime: uint32(atime), + } + + // os specific file stat decoding + fileStatFromInfoOs(fi, &flags, &fileStat) + + return flags, fileStat +} + func unmarshalAttrs(b []byte) (*FileStat, []byte) { flags, b := unmarshalUint32(b) var fs FileStat @@ -78,10 +99,10 @@ func unmarshalAttrs(b []byte) (*FileStat, []byte) { fs.Size, b = unmarshalUint64(b) } if flags&ssh_FILEXFER_ATTR_UIDGID == ssh_FILEXFER_ATTR_UIDGID { - fs.Uid, b = unmarshalUint32(b) + fs.UID, b = unmarshalUint32(b) } if flags&ssh_FILEXFER_ATTR_UIDGID == ssh_FILEXFER_ATTR_UIDGID { - fs.Gid, b = unmarshalUint32(b) + fs.GID, b = unmarshalUint32(b) } if flags&ssh_FILEXFER_ATTR_PERMISSIONS == ssh_FILEXFER_ATTR_PERMISSIONS { fs.Mode, b = unmarshalUint32(b) @@ -106,6 +127,43 @@ func unmarshalAttrs(b []byte) (*FileStat, []byte) { return &fs, b } +func marshalFileInfo(b []byte, fi os.FileInfo) []byte { + // attributes variable struct, and also variable per protocol version + // spec version 3 attributes: + // uint32 flags + // uint64 size present only if flag SSH_FILEXFER_ATTR_SIZE + // uint32 uid present only if flag SSH_FILEXFER_ATTR_UIDGID + // uint32 gid present only if flag SSH_FILEXFER_ATTR_UIDGID + // uint32 permissions present only if flag SSH_FILEXFER_ATTR_PERMISSIONS + // uint32 atime present only if flag SSH_FILEXFER_ACMODTIME + // uint32 mtime present only if flag SSH_FILEXFER_ACMODTIME + // uint32 extended_count present only if flag SSH_FILEXFER_ATTR_EXTENDED + // string extended_type + // string extended_data + // ... more extended data (extended_type - extended_data pairs), + // so that number of pairs equals extended_count + + flags, fileStat := fileStatFromInfo(fi) + + b = marshalUint32(b, flags) + if flags&ssh_FILEXFER_ATTR_SIZE != 0 { + b = marshalUint64(b, fileStat.Size) + } + if flags&ssh_FILEXFER_ATTR_UIDGID != 0 { + b = marshalUint32(b, fileStat.UID) + b = marshalUint32(b, fileStat.GID) + } + if flags&ssh_FILEXFER_ATTR_PERMISSIONS != 0 { + b = marshalUint32(b, fileStat.Mode) + } + if flags&ssh_FILEXFER_ATTR_ACMODTIME != 0 { + b = marshalUint32(b, fileStat.Atime) + b = marshalUint32(b, fileStat.Mtime) + } + + return b +} + // toFileMode converts sftp filemode bits to the os.FileMode specification func toFileMode(mode uint32) os.FileMode { var fm = os.FileMode(mode & 0777) @@ -136,3 +194,44 @@ func toFileMode(mode uint32) os.FileMode { } return fm } + +// fromFileMode converts from the os.FileMode specification to sftp filemode bits +func fromFileMode(mode os.FileMode) uint32 { + ret := uint32(0) + + if mode&os.ModeDevice != 0 { + if mode&os.ModeCharDevice != 0 { + ret |= syscall.S_IFCHR + } else { + ret |= syscall.S_IFBLK + } + } + if mode&os.ModeDir != 0 { + ret |= syscall.S_IFDIR + } + if mode&os.ModeSymlink != 0 { + ret |= syscall.S_IFLNK + } + if mode&os.ModeNamedPipe != 0 { + ret |= syscall.S_IFIFO + } + if mode&os.ModeSetgid != 0 { + ret |= syscall.S_ISGID + } + if mode&os.ModeSetuid != 0 { + ret |= syscall.S_ISUID + } + if mode&os.ModeSticky != 0 { + ret |= syscall.S_ISVTX + } + if mode&os.ModeSocket != 0 { + ret |= syscall.S_IFSOCK + } + + if mode&os.ModeType == 0 { + ret |= syscall.S_IFREG + } + ret |= uint32(mode & os.ModePerm) + + return ret +} diff --git a/Godeps/_workspace/src/github.com/pkg/sftp/attrs_stubs.go b/Godeps/_workspace/src/github.com/pkg/sftp/attrs_stubs.go new file mode 100644 index 000000000..81cf3eac2 --- /dev/null +++ b/Godeps/_workspace/src/github.com/pkg/sftp/attrs_stubs.go @@ -0,0 +1,11 @@ +// +build !cgo,!plan9 windows android + +package sftp + +import ( + "os" +) + +func fileStatFromInfoOs(fi os.FileInfo, flags *uint32, fileStat *FileStat) { + // todo +} diff --git a/Godeps/_workspace/src/github.com/pkg/sftp/attrs_test.go b/Godeps/_workspace/src/github.com/pkg/sftp/attrs_test.go deleted file mode 100644 index d55290599..000000000 --- a/Godeps/_workspace/src/github.com/pkg/sftp/attrs_test.go +++ /dev/null @@ -1,45 +0,0 @@ -package sftp - -import ( - "bytes" - "os" - "reflect" - "testing" - "time" -) - -// ensure that attrs implemenst os.FileInfo -var _ os.FileInfo = new(fileInfo) - -var unmarshalAttrsTests = []struct { - b []byte - want *fileInfo - rest []byte -}{ - {marshal(nil, struct{ Flags uint32 }{}), &fileInfo{mtime: time.Unix(int64(0), 0)}, nil}, - {marshal(nil, struct { - Flags uint32 - Size uint64 - }{ssh_FILEXFER_ATTR_SIZE, 20}), &fileInfo{size: 20, mtime: time.Unix(int64(0), 0)}, nil}, - {marshal(nil, struct { - Flags uint32 - Size uint64 - Permissions uint32 - }{ssh_FILEXFER_ATTR_SIZE | ssh_FILEXFER_ATTR_PERMISSIONS, 20, 0644}), &fileInfo{size: 20, mode: os.FileMode(0644), mtime: time.Unix(int64(0), 0)}, nil}, - {marshal(nil, struct { - Flags uint32 - Size uint64 - Uid, Gid, Permissions uint32 - }{ssh_FILEXFER_ATTR_SIZE | ssh_FILEXFER_ATTR_UIDGID | ssh_FILEXFER_ATTR_UIDGID | ssh_FILEXFER_ATTR_PERMISSIONS, 20, 1000, 1000, 0644}), &fileInfo{size: 20, mode: os.FileMode(0644), mtime: time.Unix(int64(0), 0)}, nil}, -} - -func TestUnmarshalAttrs(t *testing.T) { - for _, tt := range unmarshalAttrsTests { - stat, rest := unmarshalAttrs(tt.b) - got := fileInfoFromStat(stat, "") - tt.want.sys = got.Sys() - if !reflect.DeepEqual(got, tt.want) || !bytes.Equal(tt.rest, rest) { - t.Errorf("unmarshalAttrs(%#v): want %#v, %#v, got: %#v, %#v", tt.b, tt.want, tt.rest, got, rest) - } - } -} diff --git a/Godeps/_workspace/src/github.com/pkg/sftp/attrs_unix.go b/Godeps/_workspace/src/github.com/pkg/sftp/attrs_unix.go new file mode 100644 index 000000000..ab6ecdea9 --- /dev/null +++ b/Godeps/_workspace/src/github.com/pkg/sftp/attrs_unix.go @@ -0,0 +1,17 @@ +// +build darwin dragonfly freebsd !android,linux netbsd openbsd solaris +// +build cgo + +package sftp + +import ( + "os" + "syscall" +) + +func fileStatFromInfoOs(fi os.FileInfo, flags *uint32, fileStat *FileStat) { + if statt, ok := fi.Sys().(*syscall.Stat_t); ok { + *flags |= ssh_FILEXFER_ATTR_UIDGID + fileStat.UID = statt.Uid + fileStat.GID = statt.Gid + } +} diff --git a/Godeps/_workspace/src/github.com/pkg/sftp/client.go b/Godeps/_workspace/src/github.com/pkg/sftp/client.go index 9bb26c65d..f0d67e801 100644 --- a/Godeps/_workspace/src/github.com/pkg/sftp/client.go +++ b/Godeps/_workspace/src/github.com/pkg/sftp/client.go @@ -29,7 +29,8 @@ func MaxPacket(size int) func(*Client) error { } } -// New creates a new SFTP client on conn. +// NewClient creates a new SFTP client on conn, using zero or more option +// functions. func NewClient(conn *ssh.Client, opts ...func(*Client) error) (*Client, error) { s, err := conn.NewSession() if err != nil { @@ -117,7 +118,7 @@ func (c *Client) sendInit() error { } // returns the next value of c.nextid -func (c *Client) nextId() uint32 { +func (c *Client) nextID() uint32 { return atomic.AddUint32(&c.nextid, 1) } @@ -194,9 +195,9 @@ func (c *Client) ReadDir(p string) ([]os.FileInfo, error) { var attrs []os.FileInfo var done = false for !done { - id := c.nextId() + id := c.nextID() typ, data, err1 := c.sendRequest(sshFxpReaddirPacket{ - Id: id, + ID: id, Handle: handle, }) if err1 != nil { @@ -208,7 +209,7 @@ func (c *Client) ReadDir(p string) ([]os.FileInfo, error) { case ssh_FXP_NAME: sid, data := unmarshalUint32(data) if sid != id { - return nil, &unexpectedIdErr{id, sid} + return nil, &unexpectedIDErr{id, sid} } count, data := unmarshalUint32(data) for i := uint32(0); i < count; i++ { @@ -224,7 +225,7 @@ func (c *Client) ReadDir(p string) ([]os.FileInfo, error) { } case ssh_FXP_STATUS: // TODO(dfc) scope warning! - err = eofOrErr(unmarshalStatus(id, data)) + err = normaliseError(unmarshalStatus(id, data)) done = true default: return nil, unimplementedPacketErr(typ) @@ -235,10 +236,11 @@ func (c *Client) ReadDir(p string) ([]os.FileInfo, error) { } return attrs, err } + func (c *Client) opendir(path string) (string, error) { - id := c.nextId() + id := c.nextID() typ, data, err := c.sendRequest(sshFxpOpendirPacket{ - Id: id, + ID: id, Path: path, }) if err != nil { @@ -248,7 +250,7 @@ func (c *Client) opendir(path string) (string, error) { case ssh_FXP_HANDLE: sid, data := unmarshalUint32(data) if sid != id { - return "", &unexpectedIdErr{id, sid} + return "", &unexpectedIDErr{id, sid} } handle, _ := unmarshalString(data) return handle, nil @@ -259,10 +261,12 @@ func (c *Client) opendir(path string) (string, error) { } } -func (c *Client) Lstat(p string) (os.FileInfo, error) { - id := c.nextId() - typ, data, err := c.sendRequest(sshFxpLstatPacket{ - Id: id, +// Stat returns a FileInfo structure describing the file specified by path 'p'. +// If 'p' is a symbolic link, the returned FileInfo structure describes the referent file. +func (c *Client) Stat(p string) (os.FileInfo, error) { + id := c.nextID() + typ, data, err := c.sendRequest(sshFxpStatPacket{ + ID: id, Path: p, }) if err != nil { @@ -272,12 +276,38 @@ func (c *Client) Lstat(p string) (os.FileInfo, error) { case ssh_FXP_ATTRS: sid, data := unmarshalUint32(data) if sid != id { - return nil, &unexpectedIdErr{id, sid} + return nil, &unexpectedIDErr{id, sid} } attr, _ := unmarshalAttrs(data) return fileInfoFromStat(attr, path.Base(p)), nil case ssh_FXP_STATUS: - return nil, unmarshalStatus(id, data) + return nil, normaliseError(unmarshalStatus(id, data)) + default: + return nil, unimplementedPacketErr(typ) + } +} + +// Lstat returns a FileInfo structure describing the file specified by path 'p'. +// If 'p' is a symbolic link, the returned FileInfo structure describes the symbolic link. +func (c *Client) Lstat(p string) (os.FileInfo, error) { + id := c.nextID() + typ, data, err := c.sendRequest(sshFxpLstatPacket{ + ID: id, + Path: p, + }) + if err != nil { + return nil, err + } + switch typ { + case ssh_FXP_ATTRS: + sid, data := unmarshalUint32(data) + if sid != id { + return nil, &unexpectedIDErr{id, sid} + } + attr, _ := unmarshalAttrs(data) + return fileInfoFromStat(attr, path.Base(p)), nil + case ssh_FXP_STATUS: + return nil, normaliseError(unmarshalStatus(id, data)) default: return nil, unimplementedPacketErr(typ) } @@ -285,9 +315,9 @@ func (c *Client) Lstat(p string) (os.FileInfo, error) { // ReadLink reads the target of a symbolic link. func (c *Client) ReadLink(p string) (string, error) { - id := c.nextId() + id := c.nextID() typ, data, err := c.sendRequest(sshFxpReadlinkPacket{ - Id: id, + ID: id, Path: p, }) if err != nil { @@ -297,7 +327,7 @@ func (c *Client) ReadLink(p string) (string, error) { case ssh_FXP_NAME: sid, data := unmarshalUint32(data) if sid != id { - return "", &unexpectedIdErr{id, sid} + return "", &unexpectedIDErr{id, sid} } count, data := unmarshalUint32(data) if count != 1 { @@ -312,11 +342,30 @@ func (c *Client) ReadLink(p string) (string, error) { } } +// Symlink creates a symbolic link at 'newname', pointing at target 'oldname' +func (c *Client) Symlink(oldname, newname string) error { + id := c.nextID() + typ, data, err := c.sendRequest(sshFxpSymlinkPacket{ + ID: id, + Linkpath: newname, + Targetpath: oldname, + }) + if err != nil { + return err + } + switch typ { + case ssh_FXP_STATUS: + return normaliseError(unmarshalStatus(id, data)) + default: + return unimplementedPacketErr(typ) + } +} + // setstat is a convience wrapper to allow for changing of various parts of the file descriptor. func (c *Client) setstat(path string, flags uint32, attrs interface{}) error { - id := c.nextId() + id := c.nextID() typ, data, err := c.sendRequest(sshFxpSetstatPacket{ - Id: id, + ID: id, Path: path, Flags: flags, Attrs: attrs, @@ -326,7 +375,7 @@ func (c *Client) setstat(path string, flags uint32, attrs interface{}) error { } switch typ { case ssh_FXP_STATUS: - return okOrErr(unmarshalStatus(id, data)) + return normaliseError(unmarshalStatus(id, data)) default: return unimplementedPacketErr(typ) } @@ -345,8 +394,8 @@ func (c *Client) Chtimes(path string, atime time.Time, mtime time.Time) error { // Chown changes the user and group owners of the named file. func (c *Client) Chown(path string, uid, gid int) error { type owner struct { - Uid uint32 - Gid uint32 + UID uint32 + GID uint32 } attrs := owner{uint32(uid), uint32(gid)} return c.setstat(path, ssh_FILEXFER_ATTR_UIDGID, attrs) @@ -380,9 +429,9 @@ func (c *Client) OpenFile(path string, f int) (*File, error) { } func (c *Client) open(path string, pflags uint32) (*File, error) { - id := c.nextId() + id := c.nextID() typ, data, err := c.sendRequest(sshFxpOpenPacket{ - Id: id, + ID: id, Path: path, Pflags: pflags, }) @@ -393,12 +442,12 @@ func (c *Client) open(path string, pflags uint32) (*File, error) { case ssh_FXP_HANDLE: sid, data := unmarshalUint32(data) if sid != id { - return nil, &unexpectedIdErr{id, sid} + return nil, &unexpectedIDErr{id, sid} } handle, _ := unmarshalString(data) return &File{c: c, path: path, handle: handle}, nil case ssh_FXP_STATUS: - return nil, unmarshalStatus(id, data) + return nil, normaliseError(unmarshalStatus(id, data)) default: return nil, unimplementedPacketErr(typ) } @@ -408,9 +457,9 @@ func (c *Client) open(path string, pflags uint32) (*File, error) { // to SSH_FXP_OPEN or SSH_FXP_OPENDIR. The handle becomes invalid // immediately after this request has been sent. func (c *Client) close(handle string) error { - id := c.nextId() + id := c.nextID() typ, data, err := c.sendRequest(sshFxpClosePacket{ - Id: id, + ID: id, Handle: handle, }) if err != nil { @@ -418,16 +467,16 @@ func (c *Client) close(handle string) error { } switch typ { case ssh_FXP_STATUS: - return okOrErr(unmarshalStatus(id, data)) + return normaliseError(unmarshalStatus(id, data)) default: return unimplementedPacketErr(typ) } } func (c *Client) fstat(handle string) (*FileStat, error) { - id := c.nextId() + id := c.nextID() typ, data, err := c.sendRequest(sshFxpFstatPacket{ - Id: id, + ID: id, Handle: handle, }) if err != nil { @@ -437,7 +486,7 @@ func (c *Client) fstat(handle string) (*FileStat, error) { case ssh_FXP_ATTRS: sid, data := unmarshalUint32(data) if sid != id { - return nil, &unexpectedIdErr{id, sid} + return nil, &unexpectedIDErr{id, sid} } attr, _ := unmarshalAttrs(data) return attr, nil @@ -448,14 +497,15 @@ func (c *Client) fstat(handle string) (*FileStat, error) { } } -// Get vfs stats from remote host. -// Implementing statvfs@openssh.com SSH_FXP_EXTENDED feature -// from http://www.opensource.apple.com/source/OpenSSH/OpenSSH-175/openssh/PROTOCOL?txt +// StatVFS retrieves VFS statistics from a remote host. +// +// It implements the statvfs@openssh.com SSH_FXP_EXTENDED feature +// from http://www.opensource.apple.com/source/OpenSSH/OpenSSH-175/openssh/PROTOCOL?txt. func (c *Client) StatVFS(path string) (*StatVFS, error) { // send the StatVFS packet to the server - id := c.nextId() + id := c.nextID() typ, data, err := c.sendRequest(sshFxpStatvfsPacket{ - Id: id, + ID: id, Path: path, }) if err != nil { @@ -492,16 +542,26 @@ func (c *Client) Join(elem ...string) string { return path.Join(elem...) } // is not empty. func (c *Client) Remove(path string) error { err := c.removeFile(path) - if status, ok := err.(*StatusError); ok && status.Code == ssh_FX_FAILURE { - err = c.removeDirectory(path) + switch err := err.(type) { + case *StatusError: + switch err.Code { + // some servers, *cough* osx *cough*, return EPERM, not ENODIR. + // serv-u returns ssh_FX_FILE_IS_A_DIRECTORY + case ssh_FX_PERMISSION_DENIED, ssh_FX_FAILURE, ssh_FX_FILE_IS_A_DIRECTORY: + return c.removeDirectory(path) + default: + return err + } + default: + return err } return err } func (c *Client) removeFile(path string) error { - id := c.nextId() + id := c.nextID() typ, data, err := c.sendRequest(sshFxpRemovePacket{ - Id: id, + ID: id, Filename: path, }) if err != nil { @@ -509,16 +569,16 @@ func (c *Client) removeFile(path string) error { } switch typ { case ssh_FXP_STATUS: - return okOrErr(unmarshalStatus(id, data)) + return normaliseError(unmarshalStatus(id, data)) default: return unimplementedPacketErr(typ) } } func (c *Client) removeDirectory(path string) error { - id := c.nextId() + id := c.nextID() typ, data, err := c.sendRequest(sshFxpRmdirPacket{ - Id: id, + ID: id, Path: path, }) if err != nil { @@ -526,7 +586,7 @@ func (c *Client) removeDirectory(path string) error { } switch typ { case ssh_FXP_STATUS: - return okOrErr(unmarshalStatus(id, data)) + return normaliseError(unmarshalStatus(id, data)) default: return unimplementedPacketErr(typ) } @@ -534,9 +594,9 @@ func (c *Client) removeDirectory(path string) error { // Rename renames a file. func (c *Client) Rename(oldname, newname string) error { - id := c.nextId() + id := c.nextID() typ, data, err := c.sendRequest(sshFxpRenamePacket{ - Id: id, + ID: id, Oldpath: oldname, Newpath: newname, }) @@ -545,12 +605,46 @@ func (c *Client) Rename(oldname, newname string) error { } switch typ { case ssh_FXP_STATUS: - return okOrErr(unmarshalStatus(id, data)) + return normaliseError(unmarshalStatus(id, data)) default: return unimplementedPacketErr(typ) } } +func (c *Client) realpath(path string) (string, error) { + id := c.nextID() + typ, data, err := c.sendRequest(sshFxpRealpathPacket{ + ID: id, + Path: path, + }) + if err != nil { + return "", err + } + switch typ { + case ssh_FXP_NAME: + sid, data := unmarshalUint32(data) + if sid != id { + return "", &unexpectedIDErr{id, sid} + } + count, data := unmarshalUint32(data) + if count != 1 { + return "", unexpectedCount(1, count) + } + filename, _ := unmarshalString(data) // ignore attributes + return filename, nil + case ssh_FXP_STATUS: + return "", normaliseError(unmarshalStatus(id, data)) + default: + return "", unimplementedPacketErr(typ) + } +} + +// Getwd returns the current working directory of the server. Operations +// involving relative paths will be based at this location. +func (c *Client) Getwd() (string, error) { + return c.realpath(".") +} + // result captures the result of receiving the a packet from the server type result struct { typ byte @@ -575,20 +669,18 @@ func (c *Client) dispatchRequest(ch chan<- result, p idmarshaler) { c.inflight[p.id()] = ch if err := sendPacket(c.w, p); err != nil { delete(c.inflight, p.id()) - c.mu.Unlock() ch <- result{err: err} - return } c.mu.Unlock() } -// Creates the specified directory. An error will be returned if a file or +// Mkdir creates the specified directory. An error will be returned if a file or // directory with the specified path already exists, or if the directory's // parent folder does not exist (the method cannot create complete paths). func (c *Client) Mkdir(path string) error { - id := c.nextId() + id := c.nextID() typ, data, err := c.sendRequest(sshFxpMkdirPacket{ - Id: id, + ID: id, Path: path, }) if err != nil { @@ -596,7 +688,7 @@ func (c *Client) Mkdir(path string) error { } switch typ { case ssh_FXP_STATUS: - return okOrErr(unmarshalStatus(id, data)) + return normaliseError(unmarshalStatus(id, data)) default: return unimplementedPacketErr(typ) } @@ -627,6 +719,11 @@ func (f *File) Close() error { return f.c.close(f.handle) } +// Name returns the name of the file as presented to Open or Create. +func (f *File) Name() string { + return f.path +} + const maxConcurrentRequests = 64 // Read reads up to len(b) bytes from the File. It returns the number of @@ -640,7 +737,7 @@ func (f *File) Read(b []byte) (int, error) { inFlight := 0 desiredInFlight := 1 offset := f.offset - ch := make(chan result) + ch := make(chan result, 1) type inflightRead struct { b []byte offset uint64 @@ -653,15 +750,15 @@ func (f *File) Read(b []byte) (int, error) { var firstErr offsetErr sendReq := func(b []byte, offset uint64) { - reqId := f.c.nextId() + reqID := f.c.nextID() f.c.dispatchRequest(ch, sshFxpReadPacket{ - Id: reqId, + ID: reqID, Handle: f.handle, Offset: offset, Len: uint32(len(b)), }) inFlight++ - reqs[reqId] = inflightRead{b: b, offset: offset} + reqs[reqID] = inflightRead{b: b, offset: offset} } var read int @@ -684,17 +781,20 @@ func (f *File) Read(b []byte) (int, error) { firstErr = offsetErr{offset: 0, err: res.err} break } - reqId, data := unmarshalUint32(res.data) - req, ok := reqs[reqId] + reqID, data := unmarshalUint32(res.data) + req, ok := reqs[reqID] if !ok { - firstErr = offsetErr{offset: 0, err: fmt.Errorf("sid: %v not found", reqId)} + firstErr = offsetErr{offset: 0, err: fmt.Errorf("sid: %v not found", reqID)} break } - delete(reqs, reqId) + delete(reqs, reqID) switch res.typ { case ssh_FXP_STATUS: if firstErr.err == nil || req.offset < firstErr.offset { - firstErr = offsetErr{offset: req.offset, err: eofOrErr(unmarshalStatus(reqId, res.data))} + firstErr = offsetErr{ + offset: req.offset, + err: normaliseError(unmarshalStatus(reqID, res.data)), + } break } case ssh_FXP_DATA: @@ -736,7 +836,7 @@ func (f *File) WriteTo(w io.Writer) (int64, error) { offset := f.offset writeOffset := offset fileSize := uint64(fi.Size()) - ch := make(chan result) + ch := make(chan result, 1) type inflightRead struct { b []byte offset uint64 @@ -750,15 +850,15 @@ func (f *File) WriteTo(w io.Writer) (int64, error) { var firstErr offsetErr sendReq := func(b []byte, offset uint64) { - reqId := f.c.nextId() + reqID := f.c.nextID() f.c.dispatchRequest(ch, sshFxpReadPacket{ - Id: reqId, + ID: reqID, Handle: f.handle, Offset: offset, Len: uint32(len(b)), }) inFlight++ - reqs[reqId] = inflightRead{b: b, offset: offset} + reqs[reqID] = inflightRead{b: b, offset: offset} } var copied int64 @@ -782,17 +882,17 @@ func (f *File) WriteTo(w io.Writer) (int64, error) { firstErr = offsetErr{offset: 0, err: res.err} break } - reqId, data := unmarshalUint32(res.data) - req, ok := reqs[reqId] + reqID, data := unmarshalUint32(res.data) + req, ok := reqs[reqID] if !ok { - firstErr = offsetErr{offset: 0, err: fmt.Errorf("sid: %v not found", reqId)} + firstErr = offsetErr{offset: 0, err: fmt.Errorf("sid: %v not found", reqID)} break } - delete(reqs, reqId) + delete(reqs, reqID) switch res.typ { case ssh_FXP_STATUS: if firstErr.err == nil || req.offset < firstErr.offset { - firstErr = offsetErr{offset: req.offset, err: eofOrErr(unmarshalStatus(reqId, res.data))} + firstErr = offsetErr{offset: req.offset, err: normaliseError(unmarshalStatus(reqID, res.data))} break } case ssh_FXP_DATA: @@ -846,7 +946,6 @@ func (f *File) WriteTo(w io.Writer) (int64, error) { return copied, firstErr.err } return copied, nil - } // Stat returns the FileInfo structure describing file. If there is an @@ -870,7 +969,7 @@ func (f *File) Write(b []byte) (int, error) { inFlight := 0 desiredInFlight := 1 offset := f.offset - ch := make(chan result) + ch := make(chan result, 1) var firstErr error written := len(b) for len(b) > 0 || inFlight > 0 { @@ -878,7 +977,7 @@ func (f *File) Write(b []byte) (int, error) { l := min(len(b), f.c.maxPacket) rb := b[:l] f.c.dispatchRequest(ch, sshFxpWritePacket{ - Id: f.c.nextId(), + ID: f.c.nextID(), Handle: f.handle, Offset: offset, Length: uint32(len(rb)), @@ -902,7 +1001,7 @@ func (f *File) Write(b []byte) (int, error) { switch res.typ { case ssh_FXP_STATUS: id, _ := unmarshalUint32(res.data) - err := okOrErr(unmarshalStatus(id, res.data)) + err := normaliseError(unmarshalStatus(id, res.data)) if err != nil && firstErr == nil { firstErr = err break @@ -933,7 +1032,7 @@ func (f *File) ReadFrom(r io.Reader) (int64, error) { inFlight := 0 desiredInFlight := 1 offset := f.offset - ch := make(chan result) + ch := make(chan result, 1) var firstErr error read := int64(0) b := make([]byte, f.c.maxPacket) @@ -944,7 +1043,7 @@ func (f *File) ReadFrom(r io.Reader) (int64, error) { firstErr = err } f.c.dispatchRequest(ch, sshFxpWritePacket{ - Id: f.c.nextId(), + ID: f.c.nextID(), Handle: f.handle, Offset: offset, Length: uint32(n), @@ -968,7 +1067,7 @@ func (f *File) ReadFrom(r io.Reader) (int64, error) { switch res.typ { case ssh_FXP_STATUS: id, _ := unmarshalUint32(res.data) - err := okOrErr(unmarshalStatus(id, res.data)) + err := normaliseError(unmarshalStatus(id, res.data)) if err != nil && firstErr == nil { firstErr = err break @@ -1041,29 +1140,34 @@ func min(a, b int) int { return a } -// okOrErr returns nil if Err.Code is SSH_FX_OK, otherwise it returns the error. -func okOrErr(err error) error { - if err, ok := err.(*StatusError); ok && err.Code == ssh_FX_OK { - return nil +// normaliseError normalises an error into a more standard form that can be +// checked against stdlib errors like io.EOF or os.ErrNotExist. +func normaliseError(err error) error { + switch err := err.(type) { + case *StatusError: + switch err.Code { + case ssh_FX_EOF: + return io.EOF + case ssh_FX_NO_SUCH_FILE: + return os.ErrNotExist + case ssh_FX_OK: + return nil + default: + return err + } + default: + return err } - return err -} - -func eofOrErr(err error) error { - if err, ok := err.(*StatusError); ok && err.Code == ssh_FX_EOF { - return io.EOF - } - return err } func unmarshalStatus(id uint32, data []byte) error { sid, data := unmarshalUint32(data) if sid != id { - return &unexpectedIdErr{id, sid} + return &unexpectedIDErr{id, sid} } code, data := unmarshalUint32(data) msg, data := unmarshalString(data) - lang, _ := unmarshalString(data) + lang, _, _ := unmarshalStringSafe(data) return &StatusError{ Code: code, msg: msg, @@ -1071,6 +1175,13 @@ func unmarshalStatus(id uint32, data []byte) error { } } +func marshalStatus(b []byte, err StatusError) []byte { + b = marshalUint32(b, err.Code) + b = marshalString(b, err.msg) + b = marshalString(b, err.lang) + return b +} + // flags converts the flags passed to OpenFile into ssh flags. // Unsupported flags are ignored. func flags(f int) uint32 { diff --git a/Godeps/_workspace/src/github.com/pkg/sftp/client_integration_test.go b/Godeps/_workspace/src/github.com/pkg/sftp/client_integration_test.go deleted file mode 100644 index ff8cc8de0..000000000 --- a/Godeps/_workspace/src/github.com/pkg/sftp/client_integration_test.go +++ /dev/null @@ -1,1175 +0,0 @@ -package sftp - -// sftp integration tests -// enable with -integration - -import ( - "crypto/sha1" - "flag" - "io" - "io/ioutil" - "math/rand" - "os" - "os/exec" - "path" - "path/filepath" - "reflect" - "syscall" - "testing" - "testing/quick" - "time" - - "github.com/kr/fs" -) - -const ( - READONLY = true - READWRITE = false - NO_DELAY time.Duration = 0 - - debuglevel = "ERROR" // set to "DEBUG" for debugging -) - -var testIntegration = flag.Bool("integration", false, "perform integration tests against sftp server process") -var testSftp = flag.String("sftp", "/usr/lib/openssh/sftp-server", "location of the sftp server binary") - -type delayedWrite struct { - t time.Time - b []byte -} - -// delayedWriter wraps a writer and artificially delays the write. This is -// meant to mimic connections with various latencies. Error's returned from the -// underlying writer will panic so this should only be used over reliable -// connections. -type delayedWriter struct { - w io.WriteCloser - ch chan delayedWrite - closed chan struct{} -} - -func newDelayedWriter(w io.WriteCloser, delay time.Duration) io.WriteCloser { - ch := make(chan delayedWrite, 128) - closed := make(chan struct{}) - go func() { - for writeMsg := range ch { - time.Sleep(writeMsg.t.Add(delay).Sub(time.Now())) - n, err := w.Write(writeMsg.b) - if err != nil { - panic("write error") - } - if n < len(writeMsg.b) { - panic("showrt write") - } - } - w.Close() - close(closed) - }() - return delayedWriter{w: w, ch: ch, closed: closed} -} - -func (w delayedWriter) Write(b []byte) (int, error) { - bcopy := make([]byte, len(b)) - copy(bcopy, b) - w.ch <- delayedWrite{t: time.Now(), b: bcopy} - return len(b), nil -} - -func (w delayedWriter) Close() error { - close(w.ch) - <-w.closed - return nil -} - -// testClient returns a *Client connected to a localy running sftp-server -// the *exec.Cmd returned must be defer Wait'd. -func testClient(t testing.TB, readonly bool, delay time.Duration) (*Client, *exec.Cmd) { - if !*testIntegration { - t.Skip("skipping intergration test") - } - cmd := exec.Command(*testSftp, "-e", "-R", "-l", debuglevel) // log to stderr, read only - if !readonly { - cmd = exec.Command(*testSftp, "-e", "-l", debuglevel) // log to stderr - } - cmd.Stderr = os.Stdout - pw, err := cmd.StdinPipe() - if err != nil { - t.Fatal(err) - } - if delay > NO_DELAY { - pw = newDelayedWriter(pw, delay) - } - pr, err := cmd.StdoutPipe() - if err != nil { - t.Fatal(err) - } - if err := cmd.Start(); err != nil { - t.Skipf("could not start sftp-server process: %v", err) - } - - sftp, err := NewClientPipe(pr, pw) - if err != nil { - t.Fatal(err) - } - - return sftp, cmd -} - -func TestNewClient(t *testing.T) { - sftp, cmd := testClient(t, READONLY, NO_DELAY) - defer cmd.Wait() - - if err := sftp.Close(); err != nil { - t.Fatal(err) - } -} - -func TestClientLstat(t *testing.T) { - sftp, cmd := testClient(t, READONLY, NO_DELAY) - defer cmd.Wait() - defer sftp.Close() - - f, err := ioutil.TempFile("", "sftptest") - if err != nil { - t.Fatal(err) - } - defer os.Remove(f.Name()) - - want, err := os.Lstat(f.Name()) - if err != nil { - t.Fatal(err) - } - - got, err := sftp.Lstat(f.Name()) - if err != nil { - t.Fatal(err) - } - - if !sameFile(want, got) { - t.Fatalf("Lstat(%q): want %#v, got %#v", f.Name(), want, got) - } -} - -func TestClientLstatMissing(t *testing.T) { - sftp, cmd := testClient(t, READONLY, NO_DELAY) - defer cmd.Wait() - defer sftp.Close() - - f, err := ioutil.TempFile("", "sftptest") - if err != nil { - t.Fatal(err) - } - os.Remove(f.Name()) - - _, err = sftp.Lstat(f.Name()) - if err1, ok := err.(*StatusError); !ok || err1.Code != ssh_FX_NO_SUCH_FILE { - t.Fatalf("Lstat: want: %v, got %#v", ssh_FX_NO_SUCH_FILE, err) - } -} - -func TestClientMkdir(t *testing.T) { - sftp, cmd := testClient(t, READWRITE, NO_DELAY) - defer cmd.Wait() - defer sftp.Close() - - dir, err := ioutil.TempDir("", "sftptest") - if err != nil { - t.Fatal(err) - } - sub := path.Join(dir, "mkdir1") - if err := sftp.Mkdir(sub); err != nil { - t.Fatal(err) - } - if _, err := os.Lstat(sub); err != nil { - t.Fatal(err) - } -} - -func TestClientOpen(t *testing.T) { - sftp, cmd := testClient(t, READONLY, NO_DELAY) - defer cmd.Wait() - defer sftp.Close() - - f, err := ioutil.TempFile("", "sftptest") - if err != nil { - t.Fatal(err) - } - defer os.Remove(f.Name()) - - got, err := sftp.Open(f.Name()) - if err != nil { - t.Fatal(err) - } - if err := got.Close(); err != nil { - t.Fatal(err) - } -} - -const seekBytes = 128 * 1024 - -type seek struct { - offset int64 -} - -func (s seek) Generate(r *rand.Rand, _ int) reflect.Value { - s.offset = int64(r.Int31n(seekBytes)) - return reflect.ValueOf(s) -} - -func (s seek) set(t *testing.T, r io.ReadSeeker) { - if _, err := r.Seek(s.offset, os.SEEK_SET); err != nil { - t.Fatalf("error while seeking with %+v: %v", s, err) - } -} - -func (s seek) current(t *testing.T, r io.ReadSeeker) { - const mid = seekBytes / 2 - - skip := s.offset / 2 - if s.offset > mid { - skip = -skip - } - - if _, err := r.Seek(mid, os.SEEK_SET); err != nil { - t.Fatalf("error seeking to midpoint with %+v: %v", s, err) - } - if _, err := r.Seek(skip, os.SEEK_CUR); err != nil { - t.Fatalf("error seeking from %d with %+v: %v", mid, s, err) - } -} - -func (s seek) end(t *testing.T, r io.ReadSeeker) { - if _, err := r.Seek(-s.offset, os.SEEK_END); err != nil { - t.Fatalf("error seeking from end with %+v: %v", s, err) - } -} - -func TestClientSeek(t *testing.T) { - sftp, cmd := testClient(t, READONLY, NO_DELAY) - defer cmd.Wait() - defer sftp.Close() - - fOS, err := ioutil.TempFile("", "seek-test") - if err != nil { - t.Fatal(err) - } - defer fOS.Close() - - fSFTP, err := sftp.Open(fOS.Name()) - if err != nil { - t.Fatal(err) - } - defer fSFTP.Close() - - writeN(t, fOS, seekBytes) - - if err := quick.CheckEqual( - func(s seek) (string, int64) { s.set(t, fOS); return readHash(t, fOS) }, - func(s seek) (string, int64) { s.set(t, fSFTP); return readHash(t, fSFTP) }, - nil, - ); err != nil { - t.Errorf("Seek: expected equal absolute seeks: %v", err) - } - - if err := quick.CheckEqual( - func(s seek) (string, int64) { s.current(t, fOS); return readHash(t, fOS) }, - func(s seek) (string, int64) { s.current(t, fSFTP); return readHash(t, fSFTP) }, - nil, - ); err != nil { - t.Errorf("Seek: expected equal seeks from middle: %v", err) - } - - if err := quick.CheckEqual( - func(s seek) (string, int64) { s.end(t, fOS); return readHash(t, fOS) }, - func(s seek) (string, int64) { s.end(t, fSFTP); return readHash(t, fSFTP) }, - nil, - ); err != nil { - t.Errorf("Seek: expected equal seeks from end: %v", err) - } -} - -func TestClientCreate(t *testing.T) { - sftp, cmd := testClient(t, READWRITE, NO_DELAY) - defer cmd.Wait() - defer sftp.Close() - - f, err := ioutil.TempFile("", "sftptest") - if err != nil { - t.Fatal(err) - } - defer f.Close() - defer os.Remove(f.Name()) - - f2, err := sftp.Create(f.Name()) - if err != nil { - t.Fatal(err) - } - defer f2.Close() -} - -func TestClientAppend(t *testing.T) { - sftp, cmd := testClient(t, READWRITE, NO_DELAY) - defer cmd.Wait() - defer sftp.Close() - - f, err := ioutil.TempFile("", "sftptest") - if err != nil { - t.Fatal(err) - } - defer f.Close() - defer os.Remove(f.Name()) - - f2, err := sftp.OpenFile(f.Name(), os.O_RDWR|os.O_APPEND) - if err != nil { - t.Fatal(err) - } - defer f2.Close() -} - -func TestClientCreateFailed(t *testing.T) { - sftp, cmd := testClient(t, READONLY, NO_DELAY) - defer cmd.Wait() - defer sftp.Close() - - f, err := ioutil.TempFile("", "sftptest") - if err != nil { - t.Fatal(err) - } - defer f.Close() - defer os.Remove(f.Name()) - - f2, err := sftp.Create(f.Name()) - if err1, ok := err.(*StatusError); !ok || err1.Code != ssh_FX_PERMISSION_DENIED { - t.Fatalf("Create: want: %v, got %#v", ssh_FX_PERMISSION_DENIED, err) - } - if err == nil { - f2.Close() - } -} - -func TestClientFileStat(t *testing.T) { - sftp, cmd := testClient(t, READONLY, NO_DELAY) - defer cmd.Wait() - defer sftp.Close() - - f, err := ioutil.TempFile("", "sftptest") - if err != nil { - t.Fatal(err) - } - defer os.Remove(f.Name()) - - want, err := os.Lstat(f.Name()) - if err != nil { - t.Fatal(err) - } - - f2, err := sftp.Open(f.Name()) - if err != nil { - t.Fatal(err) - } - - got, err := f2.Stat() - if err != nil { - t.Fatal(err) - } - - if !sameFile(want, got) { - t.Fatalf("Lstat(%q): want %#v, got %#v", f.Name(), want, got) - } -} - -func TestClientRemove(t *testing.T) { - sftp, cmd := testClient(t, READWRITE, NO_DELAY) - defer cmd.Wait() - defer sftp.Close() - - f, err := ioutil.TempFile("", "sftptest") - if err != nil { - t.Fatal(err) - } - if err := sftp.Remove(f.Name()); err != nil { - t.Fatal(err) - } - if _, err := os.Lstat(f.Name()); !os.IsNotExist(err) { - t.Fatal(err) - } -} - -func TestClientRemoveDir(t *testing.T) { - sftp, cmd := testClient(t, READWRITE, NO_DELAY) - defer cmd.Wait() - defer sftp.Close() - - dir, err := ioutil.TempDir("", "sftptest") - if err != nil { - t.Fatal(err) - } - if err := sftp.Remove(dir); err != nil { - t.Fatal(err) - } - if _, err := os.Lstat(dir); !os.IsNotExist(err) { - t.Fatal(err) - } -} - -func TestClientRemoveFailed(t *testing.T) { - sftp, cmd := testClient(t, READONLY, NO_DELAY) - defer cmd.Wait() - defer sftp.Close() - - f, err := ioutil.TempFile("", "sftptest") - if err != nil { - t.Fatal(err) - } - if err := sftp.Remove(f.Name()); err == nil { - t.Fatalf("Remove(%v): want: permission denied, got %v", f.Name(), err) - } - if _, err := os.Lstat(f.Name()); err != nil { - t.Fatal(err) - } -} - -func TestClientRename(t *testing.T) { - sftp, cmd := testClient(t, READWRITE, NO_DELAY) - defer cmd.Wait() - defer sftp.Close() - - f, err := ioutil.TempFile("", "sftptest") - if err != nil { - t.Fatal(err) - } - f2 := f.Name() + ".new" - if err := sftp.Rename(f.Name(), f2); err != nil { - t.Fatal(err) - } - if _, err := os.Lstat(f.Name()); !os.IsNotExist(err) { - t.Fatal(err) - } - if _, err := os.Lstat(f2); err != nil { - t.Fatal(err) - } -} - -func TestClientReadLine(t *testing.T) { - sftp, cmd := testClient(t, READWRITE, NO_DELAY) - defer cmd.Wait() - defer sftp.Close() - - f, err := ioutil.TempFile("", "sftptest") - if err != nil { - t.Fatal(err) - } - f2 := f.Name() + ".sym" - if err := os.Symlink(f.Name(), f2); err != nil { - t.Fatal(err) - } - if _, err := sftp.ReadLink(f2); err != nil { - t.Fatal(err) - } -} - -func sameFile(want, got os.FileInfo) bool { - return want.Name() == got.Name() && - want.Size() == got.Size() -} - -var clientReadTests = []struct { - n int64 -}{ - {0}, - {1}, - {1000}, - {1024}, - {1025}, - {2048}, - {4096}, - {1 << 12}, - {1 << 13}, - {1 << 14}, - {1 << 15}, - {1 << 16}, - {1 << 17}, - {1 << 18}, - {1 << 19}, - {1 << 20}, -} - -func TestClientRead(t *testing.T) { - sftp, cmd := testClient(t, READONLY, NO_DELAY) - defer cmd.Wait() - defer sftp.Close() - - d, err := ioutil.TempDir("", "sftptest") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(d) - - for _, tt := range clientReadTests { - f, err := ioutil.TempFile(d, "read-test") - if err != nil { - t.Fatal(err) - } - defer f.Close() - hash := writeN(t, f, tt.n) - f2, err := sftp.Open(f.Name()) - if err != nil { - t.Fatal(err) - } - defer f2.Close() - hash2, n := readHash(t, f2) - if hash != hash2 || tt.n != n { - t.Errorf("Read: hash: want: %q, got %q, read: want: %v, got %v", hash, hash2, tt.n, n) - } - } -} - -// readHash reads r until EOF returning the number of bytes read -// and the hash of the contents. -func readHash(t *testing.T, r io.Reader) (string, int64) { - h := sha1.New() - tr := io.TeeReader(r, h) - read, err := io.Copy(ioutil.Discard, tr) - if err != nil { - t.Fatal(err) - } - return string(h.Sum(nil)), read -} - -// writeN writes n bytes of random data to w and returns the -// hash of that data. -func writeN(t *testing.T, w io.Writer, n int64) string { - rand, err := os.Open("/dev/urandom") - if err != nil { - t.Fatal(err) - } - defer rand.Close() - - h := sha1.New() - - mw := io.MultiWriter(w, h) - - written, err := io.CopyN(mw, rand, n) - if err != nil { - t.Fatal(err) - } - if written != n { - t.Fatalf("CopyN(%v): wrote: %v", n, written) - } - return string(h.Sum(nil)) -} - -var clientWriteTests = []struct { - n int - total int64 // cumulative file size -}{ - {0, 0}, - {1, 1}, - {0, 1}, - {999, 1000}, - {24, 1024}, - {1023, 2047}, - {2048, 4095}, - {1 << 12, 8191}, - {1 << 13, 16383}, - {1 << 14, 32767}, - {1 << 15, 65535}, - {1 << 16, 131071}, - {1 << 17, 262143}, - {1 << 18, 524287}, - {1 << 19, 1048575}, - {1 << 20, 2097151}, - {1 << 21, 4194303}, -} - -func TestClientWrite(t *testing.T) { - sftp, cmd := testClient(t, READWRITE, NO_DELAY) - defer cmd.Wait() - defer sftp.Close() - - d, err := ioutil.TempDir("", "sftptest") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(d) - - f := path.Join(d, "writeTest") - w, err := sftp.Create(f) - if err != nil { - t.Fatal(err) - } - defer w.Close() - - for _, tt := range clientWriteTests { - got, err := w.Write(make([]byte, tt.n)) - if err != nil { - t.Fatal(err) - } - if got != tt.n { - t.Errorf("Write(%v): wrote: want: %v, got %v", tt.n, tt.n, got) - } - fi, err := os.Stat(f) - if err != nil { - t.Fatal(err) - } - if total := fi.Size(); total != tt.total { - t.Errorf("Write(%v): size: want: %v, got %v", tt.n, tt.total, total) - } - } -} - -// taken from github.com/kr/fs/walk_test.go - -type PathTest struct { - path, result string -} - -type Node struct { - name string - entries []*Node // nil if the entry is a file - mark int -} - -var tree = &Node{ - "testdata", - []*Node{ - {"a", nil, 0}, - {"b", []*Node{}, 0}, - {"c", nil, 0}, - { - "d", - []*Node{ - {"x", nil, 0}, - {"y", []*Node{}, 0}, - { - "z", - []*Node{ - {"u", nil, 0}, - {"v", nil, 0}, - }, - 0, - }, - }, - 0, - }, - }, - 0, -} - -func walkTree(n *Node, path string, f func(path string, n *Node)) { - f(path, n) - for _, e := range n.entries { - walkTree(e, filepath.Join(path, e.name), f) - } -} - -func makeTree(t *testing.T) { - walkTree(tree, tree.name, func(path string, n *Node) { - if n.entries == nil { - fd, err := os.Create(path) - if err != nil { - t.Errorf("makeTree: %v", err) - return - } - fd.Close() - } else { - os.Mkdir(path, 0770) - } - }) -} - -func markTree(n *Node) { walkTree(n, "", func(path string, n *Node) { n.mark++ }) } - -func checkMarks(t *testing.T, report bool) { - walkTree(tree, tree.name, func(path string, n *Node) { - if n.mark != 1 && report { - t.Errorf("node %s mark = %d; expected 1", path, n.mark) - } - n.mark = 0 - }) -} - -// Assumes that each node name is unique. Good enough for a test. -// If clear is true, any incoming error is cleared before return. The errors -// are always accumulated, though. -func mark(path string, info os.FileInfo, err error, errors *[]error, clear bool) error { - if err != nil { - *errors = append(*errors, err) - if clear { - return nil - } - return err - } - name := info.Name() - walkTree(tree, tree.name, func(path string, n *Node) { - if n.name == name { - n.mark++ - } - }) - return nil -} - -func TestClientWalk(t *testing.T) { - sftp, cmd := testClient(t, READONLY, NO_DELAY) - defer cmd.Wait() - defer sftp.Close() - - makeTree(t) - errors := make([]error, 0, 10) - clear := true - markFn := func(walker *fs.Walker) (err error) { - for walker.Step() { - err = mark(walker.Path(), walker.Stat(), walker.Err(), &errors, clear) - if err != nil { - break - } - } - return err - } - // Expect no errors. - err := markFn(sftp.Walk(tree.name)) - if err != nil { - t.Fatalf("no error expected, found: %s", err) - } - if len(errors) != 0 { - t.Fatalf("unexpected errors: %s", errors) - } - checkMarks(t, true) - errors = errors[0:0] - - // Test permission errors. Only possible if we're not root - // and only on some file systems (AFS, FAT). To avoid errors during - // all.bash on those file systems, skip during go test -short. - if os.Getuid() > 0 && !testing.Short() { - // introduce 2 errors: chmod top-level directories to 0 - os.Chmod(filepath.Join(tree.name, tree.entries[1].name), 0) - os.Chmod(filepath.Join(tree.name, tree.entries[3].name), 0) - - // 3) capture errors, expect two. - // mark respective subtrees manually - markTree(tree.entries[1]) - markTree(tree.entries[3]) - // correct double-marking of directory itself - tree.entries[1].mark-- - tree.entries[3].mark-- - err := markFn(sftp.Walk(tree.name)) - if err != nil { - t.Fatalf("expected no error return from Walk, got %s", err) - } - if len(errors) != 2 { - t.Errorf("expected 2 errors, got %d: %s", len(errors), errors) - } - // the inaccessible subtrees were marked manually - checkMarks(t, true) - errors = errors[0:0] - - // 4) capture errors, stop after first error. - // mark respective subtrees manually - markTree(tree.entries[1]) - markTree(tree.entries[3]) - // correct double-marking of directory itself - tree.entries[1].mark-- - tree.entries[3].mark-- - clear = false // error will stop processing - err = markFn(sftp.Walk(tree.name)) - if err == nil { - t.Fatalf("expected error return from Walk") - } - if len(errors) != 1 { - t.Errorf("expected 1 error, got %d: %s", len(errors), errors) - } - // the inaccessible subtrees were marked manually - checkMarks(t, false) - errors = errors[0:0] - - // restore permissions - os.Chmod(filepath.Join(tree.name, tree.entries[1].name), 0770) - os.Chmod(filepath.Join(tree.name, tree.entries[3].name), 0770) - } - - // cleanup - if err := os.RemoveAll(tree.name); err != nil { - t.Errorf("removeTree: %v", err) - } -} - -// sftp/issue/42, abrupt server hangup would result in client hangs. -func TestServerRoughDisconnect(t *testing.T) { - sftp, cmd := testClient(t, READONLY, NO_DELAY) - - f, err := sftp.Open("/dev/zero") - if err != nil { - t.Fatal(err) - } - defer f.Close() - go func() { - time.Sleep(100 * time.Millisecond) - cmd.Process.Kill() - }() - - io.Copy(ioutil.Discard, f) - sftp.Close() -} - -func benchmarkRead(b *testing.B, bufsize int, delay time.Duration) { - size := 10*1024*1024 + 123 // ~10MiB - - // open sftp client - sftp, cmd := testClient(b, READONLY, delay) - defer cmd.Wait() - defer sftp.Close() - - buf := make([]byte, bufsize) - - b.ResetTimer() - b.SetBytes(int64(size)) - - for i := 0; i < b.N; i++ { - offset := 0 - - f2, err := sftp.Open("/dev/zero") - if err != nil { - b.Fatal(err) - } - defer f2.Close() - - for offset < size { - n, err := io.ReadFull(f2, buf) - offset += n - if err == io.ErrUnexpectedEOF && offset != size { - b.Fatalf("read too few bytes! want: %d, got: %d", size, n) - } - - if err != nil { - b.Fatal(err) - } - - offset += n - } - } -} - -func BenchmarkRead1k(b *testing.B) { - benchmarkRead(b, 1*1024, NO_DELAY) -} - -func BenchmarkRead16k(b *testing.B) { - benchmarkRead(b, 16*1024, NO_DELAY) -} - -func BenchmarkRead32k(b *testing.B) { - benchmarkRead(b, 32*1024, NO_DELAY) -} - -func BenchmarkRead128k(b *testing.B) { - benchmarkRead(b, 128*1024, NO_DELAY) -} - -func BenchmarkRead512k(b *testing.B) { - benchmarkRead(b, 512*1024, NO_DELAY) -} - -func BenchmarkRead1MiB(b *testing.B) { - benchmarkRead(b, 1024*1024, NO_DELAY) -} - -func BenchmarkRead4MiB(b *testing.B) { - benchmarkRead(b, 4*1024*1024, NO_DELAY) -} - -func BenchmarkRead4MiBDelay10Msec(b *testing.B) { - benchmarkRead(b, 4*1024*1024, 10*time.Millisecond) -} - -func BenchmarkRead4MiBDelay50Msec(b *testing.B) { - benchmarkRead(b, 4*1024*1024, 50*time.Millisecond) -} - -func BenchmarkRead4MiBDelay150Msec(b *testing.B) { - benchmarkRead(b, 4*1024*1024, 150*time.Millisecond) -} - -func benchmarkWrite(b *testing.B, bufsize int, delay time.Duration) { - size := 10*1024*1024 + 123 // ~10MiB - - // open sftp client - sftp, cmd := testClient(b, false, delay) - defer cmd.Wait() - defer sftp.Close() - - data := make([]byte, size) - - b.ResetTimer() - b.SetBytes(int64(size)) - - for i := 0; i < b.N; i++ { - offset := 0 - - f, err := ioutil.TempFile("", "sftptest") - if err != nil { - b.Fatal(err) - } - defer os.Remove(f.Name()) - - f2, err := sftp.Create(f.Name()) - if err != nil { - b.Fatal(err) - } - defer f2.Close() - - for offset < size { - n, err := f2.Write(data[offset:min(len(data), offset+bufsize)]) - if err != nil { - b.Fatal(err) - } - - if offset+n < size && n != bufsize { - b.Fatalf("wrote too few bytes! want: %d, got: %d", size, n) - } - - offset += n - } - - f2.Close() - - fi, err := os.Stat(f.Name()) - if err != nil { - b.Fatal(err) - } - - if fi.Size() != int64(size) { - b.Fatalf("wrong file size: want %d, got %d", size, fi.Size()) - } - - os.Remove(f.Name()) - } -} - -func BenchmarkWrite1k(b *testing.B) { - benchmarkWrite(b, 1*1024, NO_DELAY) -} - -func BenchmarkWrite16k(b *testing.B) { - benchmarkWrite(b, 16*1024, NO_DELAY) -} - -func BenchmarkWrite32k(b *testing.B) { - benchmarkWrite(b, 32*1024, NO_DELAY) -} - -func BenchmarkWrite128k(b *testing.B) { - benchmarkWrite(b, 128*1024, NO_DELAY) -} - -func BenchmarkWrite512k(b *testing.B) { - benchmarkWrite(b, 512*1024, NO_DELAY) -} - -func BenchmarkWrite1MiB(b *testing.B) { - benchmarkWrite(b, 1024*1024, NO_DELAY) -} - -func BenchmarkWrite4MiB(b *testing.B) { - benchmarkWrite(b, 4*1024*1024, NO_DELAY) -} - -func BenchmarkWrite4MiBDelay10Msec(b *testing.B) { - benchmarkWrite(b, 4*1024*1024, 10*time.Millisecond) -} - -func BenchmarkWrite4MiBDelay50Msec(b *testing.B) { - benchmarkWrite(b, 4*1024*1024, 50*time.Millisecond) -} - -func BenchmarkWrite4MiBDelay150Msec(b *testing.B) { - benchmarkWrite(b, 4*1024*1024, 150*time.Millisecond) -} - -func TestClientStatVFS(t *testing.T) { - sftp, cmd := testClient(t, READWRITE, NO_DELAY) - defer cmd.Wait() - defer sftp.Close() - - vfs, err := sftp.StatVFS("/") - if err != nil { - t.Fatal(err) - } - - // get system stats - s := syscall.Statfs_t{} - err = syscall.Statfs("/", &s) - if err != nil { - t.Fatal(err) - } - - // check some stats - if vfs.Frsize != uint64(s.Frsize) { - t.Fatal("fr_size does not match") - } - - if vfs.Bsize != uint64(s.Bsize) { - t.Fatal("f_bsize does not match") - } - - if vfs.Namemax != uint64(s.Namelen) { - t.Fatal("f_namemax does not match") - } - - if vfs.Bavail != s.Bavail { - t.Fatal("f_bavail does not match") - } - -} - -func benchmarkCopyDown(b *testing.B, fileSize int64, delay time.Duration) { - // Create a temp file and fill it with zero's. - src, err := ioutil.TempFile("", "sftptest") - if err != nil { - b.Fatal(err) - } - defer src.Close() - srcFilename := src.Name() - defer os.Remove(srcFilename) - zero, err := os.Open("/dev/zero") - if err != nil { - b.Fatal(err) - } - n, err := io.Copy(src, io.LimitReader(zero, fileSize)) - if err != nil { - b.Fatal(err) - } - if n < fileSize { - b.Fatal("short copy") - } - zero.Close() - src.Close() - - sftp, cmd := testClient(b, READONLY, delay) - defer cmd.Wait() - defer sftp.Close() - b.ResetTimer() - b.SetBytes(fileSize) - - for i := 0; i < b.N; i++ { - dst, err := ioutil.TempFile("", "sftptest") - if err != nil { - b.Fatal(err) - } - defer os.Remove(dst.Name()) - - src, err := sftp.Open(srcFilename) - if err != nil { - b.Fatal(err) - } - defer src.Close() - n, err := io.Copy(dst, src) - if err != nil { - b.Fatalf("copy error: %v", err) - } - if n < fileSize { - b.Fatal("unable to copy all bytes") - } - dst.Close() - fi, err := os.Stat(dst.Name()) - if err != nil { - b.Fatal(err) - } - - if fi.Size() != fileSize { - b.Fatalf("wrong file size: want %d, got %d", fileSize, fi.Size()) - } - os.Remove(dst.Name()) - } -} - -func BenchmarkCopyDown10MiBDelay10Msec(b *testing.B) { - benchmarkCopyDown(b, 10*1024*1024, 10*time.Millisecond) -} - -func BenchmarkCopyDown10MiBDelay50Msec(b *testing.B) { - benchmarkCopyDown(b, 10*1024*1024, 50*time.Millisecond) -} - -func BenchmarkCopyDown10MiBDelay150Msec(b *testing.B) { - benchmarkCopyDown(b, 10*1024*1024, 150*time.Millisecond) -} - -func benchmarkCopyUp(b *testing.B, fileSize int64, delay time.Duration) { - // Create a temp file and fill it with zero's. - src, err := ioutil.TempFile("", "sftptest") - if err != nil { - b.Fatal(err) - } - defer src.Close() - srcFilename := src.Name() - defer os.Remove(srcFilename) - zero, err := os.Open("/dev/zero") - if err != nil { - b.Fatal(err) - } - n, err := io.Copy(src, io.LimitReader(zero, fileSize)) - if err != nil { - b.Fatal(err) - } - if n < fileSize { - b.Fatal("short copy") - } - zero.Close() - src.Close() - - sftp, cmd := testClient(b, false, delay) - defer cmd.Wait() - defer sftp.Close() - - b.ResetTimer() - b.SetBytes(fileSize) - - for i := 0; i < b.N; i++ { - tmp, err := ioutil.TempFile("", "sftptest") - if err != nil { - b.Fatal(err) - } - tmp.Close() - defer os.Remove(tmp.Name()) - - dst, err := sftp.Create(tmp.Name()) - if err != nil { - b.Fatal(err) - } - defer dst.Close() - src, err := os.Open(srcFilename) - if err != nil { - b.Fatal(err) - } - defer src.Close() - n, err := io.Copy(dst, src) - if err != nil { - b.Fatalf("copy error: %v", err) - } - if n < fileSize { - b.Fatal("unable to copy all bytes") - } - - fi, err := os.Stat(tmp.Name()) - if err != nil { - b.Fatal(err) - } - - if fi.Size() != fileSize { - b.Fatalf("wrong file size: want %d, got %d", fileSize, fi.Size()) - } - os.Remove(tmp.Name()) - } -} - -func BenchmarkCopyUp10MiBDelay10Msec(b *testing.B) { - benchmarkCopyUp(b, 10*1024*1024, 10*time.Millisecond) -} - -func BenchmarkCopyUp10MiBDelay50Msec(b *testing.B) { - benchmarkCopyUp(b, 10*1024*1024, 50*time.Millisecond) -} - -func BenchmarkCopyUp10MiBDelay150Msec(b *testing.B) { - benchmarkCopyUp(b, 10*1024*1024, 150*time.Millisecond) -} diff --git a/Godeps/_workspace/src/github.com/pkg/sftp/client_test.go b/Godeps/_workspace/src/github.com/pkg/sftp/client_test.go deleted file mode 100644 index 9ade6d1af..000000000 --- a/Godeps/_workspace/src/github.com/pkg/sftp/client_test.go +++ /dev/null @@ -1,75 +0,0 @@ -package sftp - -import ( - "io" - "os" - "testing" - - "github.com/kr/fs" -) - -// assert that *Client implements fs.FileSystem -var _ fs.FileSystem = new(Client) - -// assert that *File implements io.ReadWriteCloser -var _ io.ReadWriteCloser = new(File) - -var ok = &StatusError{Code: ssh_FX_OK} -var eof = &StatusError{Code: ssh_FX_EOF} -var fail = &StatusError{Code: ssh_FX_FAILURE} - -var eofOrErrTests = []struct { - err, want error -}{ - {nil, nil}, - {eof, io.EOF}, - {ok, ok}, - {io.EOF, io.EOF}, -} - -func TestEofOrErr(t *testing.T) { - for _, tt := range eofOrErrTests { - got := eofOrErr(tt.err) - if got != tt.want { - t.Errorf("eofOrErr(%#v): want: %#v, got: %#v", tt.err, tt.want, got) - } - } -} - -var okOrErrTests = []struct { - err, want error -}{ - {nil, nil}, - {eof, eof}, - {ok, nil}, - {io.EOF, io.EOF}, -} - -func TestOkOrErr(t *testing.T) { - for _, tt := range okOrErrTests { - got := okOrErr(tt.err) - if got != tt.want { - t.Errorf("okOrErr(%#v): want: %#v, got: %#v", tt.err, tt.want, got) - } - } -} - -var flagsTests = []struct { - flags int - want uint32 -}{ - {os.O_RDONLY, ssh_FXF_READ}, - {os.O_WRONLY, ssh_FXF_WRITE}, - {os.O_RDWR, ssh_FXF_READ | ssh_FXF_WRITE}, - {os.O_RDWR | os.O_CREATE | os.O_TRUNC, ssh_FXF_READ | ssh_FXF_WRITE | ssh_FXF_CREAT | ssh_FXF_TRUNC}, - {os.O_WRONLY | os.O_APPEND, ssh_FXF_WRITE | ssh_FXF_APPEND}, -} - -func TestFlags(t *testing.T) { - for i, tt := range flagsTests { - got := flags(tt.flags) - if got != tt.want { - t.Errorf("test %v: flags(%x): want: %x, got: %x", i, tt.flags, tt.want, got) - } - } -} diff --git a/Godeps/_workspace/src/github.com/pkg/sftp/debug.go b/Godeps/_workspace/src/github.com/pkg/sftp/debug.go index 87c1fe8e9..3e264abe3 100644 --- a/Godeps/_workspace/src/github.com/pkg/sftp/debug.go +++ b/Godeps/_workspace/src/github.com/pkg/sftp/debug.go @@ -1,4 +1,4 @@ -// +build debug_sftp +// +build debug package sftp diff --git a/Godeps/_workspace/src/github.com/pkg/sftp/example_test.go b/Godeps/_workspace/src/github.com/pkg/sftp/example_test.go deleted file mode 100644 index 3f73726de..000000000 --- a/Godeps/_workspace/src/github.com/pkg/sftp/example_test.go +++ /dev/null @@ -1,91 +0,0 @@ -package sftp_test - -import ( - "fmt" - "log" - "os" - "os/exec" - - "golang.org/x/crypto/ssh" - - "github.com/pkg/sftp" -) - -func Example(conn *ssh.Client) { - // open an SFTP session over an existing ssh connection. - sftp, err := sftp.NewClient(conn) - if err != nil { - log.Fatal(err) - } - defer sftp.Close() - - // walk a directory - w := sftp.Walk("/home/user") - for w.Step() { - if w.Err() != nil { - continue - } - log.Println(w.Path()) - } - - // leave your mark - f, err := sftp.Create("hello.txt") - if err != nil { - log.Fatal(err) - } - if _, err := f.Write([]byte("Hello world!")); err != nil { - log.Fatal(err) - } - - // check it's there - fi, err := sftp.Lstat("hello.txt") - if err != nil { - log.Fatal(err) - } - log.Println(fi) -} - -func ExampleNewClientPipe() { - // Connect to a remote host and request the sftp subsystem via the 'ssh' - // command. This assumes that passwordless login is correctly configured. - cmd := exec.Command("ssh", "example.com", "-s", "sftp") - - // send errors from ssh to stderr - cmd.Stderr = os.Stderr - - // get stdin and stdout - wr, err := cmd.StdinPipe() - if err != nil { - log.Fatal(err) - } - rd, err := cmd.StdoutPipe() - if err != nil { - log.Fatal(err) - } - - // start the process - if err := cmd.Start(); err != nil { - log.Fatal(err) - } - defer cmd.Wait() - - // open the SFTP session - client, err := sftp.NewClientPipe(rd, wr) - if err != nil { - log.Fatal(err) - } - - // read a directory - list, err := client.ReadDir("/") - if err != nil { - log.Fatal(err) - } - - // print contents - for _, item := range list { - fmt.Println(item.Name()) - } - - // close the connection - client.Close() -} diff --git a/Godeps/_workspace/src/github.com/pkg/sftp/examples/sftp-server/README.md b/Godeps/_workspace/src/github.com/pkg/sftp/examples/sftp-server/README.md new file mode 100644 index 000000000..bd96f2d8a --- /dev/null +++ b/Godeps/_workspace/src/github.com/pkg/sftp/examples/sftp-server/README.md @@ -0,0 +1,12 @@ +Example SFTP server implementation +=== + +In order to use this example you will need an RSA key. + +On linux-like systems with openssh installed, you can use the command: + +``` +ssh-keygen -t rsa -f id_rsa +``` + +Then you will be able to run the sftp-server command in the current directory. diff --git a/Godeps/_workspace/src/github.com/pkg/sftp/examples/sftp-server/main.go b/Godeps/_workspace/src/github.com/pkg/sftp/examples/sftp-server/main.go new file mode 100644 index 000000000..115d35c35 --- /dev/null +++ b/Godeps/_workspace/src/github.com/pkg/sftp/examples/sftp-server/main.go @@ -0,0 +1,134 @@ +// An example SFTP server implementation using the golang SSH package. +// Serves the whole filesystem visible to the user, and has a hard-coded username and password, +// so not for real use! +package main + +import ( + "flag" + "fmt" + "io/ioutil" + "log" + "net" + "os" + + "github.com/pkg/sftp" + "golang.org/x/crypto/ssh" +) + +// Based on example server code from golang.org/x/crypto/ssh and server_standalone +func main() { + + var ( + readOnly bool + debugStderr bool + ) + + flag.BoolVar(&readOnly, "R", false, "read-only server") + flag.BoolVar(&debugStderr, "e", false, "debug to stderr") + flag.Parse() + + debugStream := ioutil.Discard + if debugStderr { + debugStream = os.Stderr + } + + // An SSH server is represented by a ServerConfig, which holds + // certificate details and handles authentication of ServerConns. + config := &ssh.ServerConfig{ + PasswordCallback: func(c ssh.ConnMetadata, pass []byte) (*ssh.Permissions, error) { + // Should use constant-time compare (or better, salt+hash) in + // a production setting. + fmt.Fprintf(debugStream, "Login: %s\n", c.User()) + if c.User() == "testuser" && string(pass) == "tiger" { + return nil, nil + } + return nil, fmt.Errorf("password rejected for %q", c.User()) + }, + } + + privateBytes, err := ioutil.ReadFile("id_rsa") + if err != nil { + log.Fatal("Failed to load private key", err) + } + + private, err := ssh.ParsePrivateKey(privateBytes) + if err != nil { + log.Fatal("Failed to parse private key", err) + } + + config.AddHostKey(private) + + // Once a ServerConfig has been configured, connections can be + // accepted. + listener, err := net.Listen("tcp", "0.0.0.0:2022") + if err != nil { + log.Fatal("failed to listen for connection", err) + } + fmt.Printf("Listening on %v\n", listener.Addr()) + + nConn, err := listener.Accept() + if err != nil { + log.Fatal("failed to accept incoming connection", err) + } + + // Before use, a handshake must be performed on the incoming + // net.Conn. + _, chans, reqs, err := ssh.NewServerConn(nConn, config) + if err != nil { + log.Fatal("failed to handshake", err) + } + fmt.Fprintf(debugStream, "SSH server established\n") + + // The incoming Request channel must be serviced. + go ssh.DiscardRequests(reqs) + + // Service the incoming Channel channel. + for newChannel := range chans { + // Channels have a type, depending on the application level + // protocol intended. In the case of an SFTP session, this is "subsystem" + // with a payload string of "sftp" + fmt.Fprintf(debugStream, "Incoming channel: %s\n", newChannel.ChannelType()) + if newChannel.ChannelType() != "session" { + newChannel.Reject(ssh.UnknownChannelType, "unknown channel type") + fmt.Fprintf(debugStream, "Unknown channel type: %s\n", newChannel.ChannelType()) + continue + } + channel, requests, err := newChannel.Accept() + if err != nil { + log.Fatal("could not accept channel.", err) + } + fmt.Fprintf(debugStream, "Channel accepted\n") + + // Sessions have out-of-band requests such as "shell", + // "pty-req" and "env". Here we handle only the + // "subsystem" request. + go func(in <-chan *ssh.Request) { + for req := range in { + fmt.Fprintf(debugStream, "Request: %v\n", req.Type) + ok := false + switch req.Type { + case "subsystem": + fmt.Fprintf(debugStream, "Subsystem: %s\n", req.Payload[4:]) + if string(req.Payload[4:]) == "sftp" { + ok = true + } + } + fmt.Fprintf(debugStream, " - accepted: %v\n", ok) + req.Reply(ok, nil) + } + }(requests) + + server, err := sftp.NewServer( + channel, + channel, + sftp.WithDebug(debugStream), + sftp.ReadOnly(), + ) + if err != nil { + log.Fatal(err) + } + if err := server.Serve(); err != nil { + log.Fatal("sftp server completed with error:", err) + } + } +} diff --git a/Godeps/_workspace/src/github.com/pkg/sftp/packet.go b/Godeps/_workspace/src/github.com/pkg/sftp/packet.go index 45c9d7bdc..cd95c7246 100644 --- a/Godeps/_workspace/src/github.com/pkg/sftp/packet.go +++ b/Godeps/_workspace/src/github.com/pkg/sftp/packet.go @@ -2,11 +2,24 @@ package sftp import ( "encoding" + "errors" "fmt" "io" + "os" "reflect" ) +var ( + errShortPacket = errors.New("packet too short") +) + +const ( + debugDumpTxPacket = false + debugDumpRxPacket = false + debugDumpTxPacketBytes = false + debugDumpRxPacketBytes = false +) + func marshalUint32(b []byte, v uint32) []byte { return append(b, byte(v>>24), byte(v>>16), byte(v>>8), byte(v)) } @@ -20,6 +33,9 @@ func marshalString(b []byte, v string) []byte { } func marshal(b []byte, v interface{}) []byte { + if v == nil { + return b + } switch v := v.(type) { case uint8: return append(b, v) @@ -29,6 +45,8 @@ func marshal(b []byte, v interface{}) []byte { return marshalUint64(b, v) case string: return marshalString(b, v) + case os.FileInfo: + return marshalFileInfo(b, v) default: switch d := reflect.ValueOf(v); d.Kind() { case reflect.Struct: @@ -52,26 +70,59 @@ func unmarshalUint32(b []byte) (uint32, []byte) { return v, b[4:] } +func unmarshalUint32Safe(b []byte) (uint32, []byte, error) { + var v uint32 + if len(b) < 4 { + return 0, nil, errShortPacket + } + v, b = unmarshalUint32(b) + return v, b, nil +} + func unmarshalUint64(b []byte) (uint64, []byte) { h, b := unmarshalUint32(b) l, b := unmarshalUint32(b) return uint64(h)<<32 | uint64(l), b } +func unmarshalUint64Safe(b []byte) (uint64, []byte, error) { + var v uint64 + if len(b) < 8 { + return 0, nil, errShortPacket + } + v, b = unmarshalUint64(b) + return v, b, nil +} + func unmarshalString(b []byte) (string, []byte) { n, b := unmarshalUint32(b) return string(b[:n]), b[n:] } +func unmarshalStringSafe(b []byte) (string, []byte, error) { + n, b, err := unmarshalUint32Safe(b) + if err != nil { + return "", nil, err + } + if int64(n) > int64(len(b)) { + return "", nil, errShortPacket + } + return string(b[:n]), b[n:], nil +} + // sendPacket marshals p according to RFC 4234. func sendPacket(w io.Writer, m encoding.BinaryMarshaler) error { bb, err := m.MarshalBinary() if err != nil { return fmt.Errorf("marshal2(%#v): binary marshaller failed", err) } + if debugDumpTxPacketBytes { + debug("send packet: %s %d bytes %x", fxp(bb[0]), len(bb), bb[1:]) + } else if debugDumpTxPacket { + debug("send packet: %s %d bytes", fxp(bb[0]), len(bb)) + } l := uint32(len(bb)) hdr := []byte{byte(l >> 24), byte(l >> 16), byte(l >> 8), byte(l)} - debug("send packet %T, len: %v", m, l) _, err = w.Write(hdr) if err != nil { return err @@ -80,6 +131,13 @@ func sendPacket(w io.Writer, m encoding.BinaryMarshaler) error { return err } +func (svr *Server) sendPacket(m encoding.BinaryMarshaler) error { + // any responder can call sendPacket(); actual socket access must be serialized + svr.outMutex.Lock() + defer svr.outMutex.Unlock() + return sendPacket(svr.out, m) +} + func recvPacket(r io.Reader) (uint8, []byte, error) { var b = []byte{0, 0, 0, 0} if _, err := io.ReadFull(r, b); err != nil { @@ -88,11 +146,36 @@ func recvPacket(r io.Reader) (uint8, []byte, error) { l, _ := unmarshalUint32(b) b = make([]byte, l) if _, err := io.ReadFull(r, b); err != nil { + debug("recv packet %d bytes: err %v", l, err) return 0, nil, err } + if debugDumpRxPacketBytes { + debug("recv packet: %s %d bytes %x", fxp(b[0]), l, b[1:]) + } else if debugDumpRxPacket { + debug("recv packet: %s %d bytes", fxp(b[0]), l) + } return b[0], b[1:], nil } +type extensionPair struct { + Name string + Data string +} + +func unmarshalExtensionPair(b []byte) (extensionPair, []byte, error) { + var ep extensionPair + var err error + ep.Name, b, err = unmarshalStringSafe(b) + if err != nil { + return ep, b, err + } + ep.Data, b, err = unmarshalStringSafe(b) + if err != nil { + return ep, b, err + } + return ep, b, err +} + // Here starts the definition of packets along with their MarshalBinary // implementations. // Manually writing the marshalling logic wins us a lot of time and @@ -100,9 +183,7 @@ func recvPacket(r io.Reader) (uint8, []byte, error) { type sshFxInitPacket struct { Version uint32 - Extensions []struct { - Name, Data string - } + Extensions []extensionPair } func (p sshFxInitPacket) MarshalBinary() ([]byte, error) { @@ -121,7 +202,46 @@ func (p sshFxInitPacket) MarshalBinary() ([]byte, error) { return b, nil } -func marshalIdString(packetType byte, id uint32, str string) ([]byte, error) { +func (p *sshFxInitPacket) UnmarshalBinary(b []byte) error { + var err error + if p.Version, b, err = unmarshalUint32Safe(b); err != nil { + return err + } + for len(b) > 0 { + var ep extensionPair + ep, b, err = unmarshalExtensionPair(b) + if err != nil { + return err + } + p.Extensions = append(p.Extensions, ep) + } + return nil +} + +type sshFxVersionPacket struct { + Version uint32 + Extensions []struct { + Name, Data string + } +} + +func (p sshFxVersionPacket) MarshalBinary() ([]byte, error) { + l := 1 + 4 // byte + uint32 + for _, e := range p.Extensions { + l += 4 + len(e.Name) + 4 + len(e.Data) + } + + b := make([]byte, 0, l) + b = append(b, ssh_FXP_VERSION) + b = marshalUint32(b, p.Version) + for _, e := range p.Extensions { + b = marshalString(b, e.Name) + b = marshalString(b, e.Data) + } + return b, nil +} + +func marshalIDString(packetType byte, id uint32, str string) ([]byte, error) { l := 1 + 4 + // type(byte) + uint32 4 + len(str) @@ -132,102 +252,247 @@ func marshalIdString(packetType byte, id uint32, str string) ([]byte, error) { return b, nil } +func unmarshalIDString(b []byte, id *uint32, str *string) error { + var err error + *id, b, err = unmarshalUint32Safe(b) + if err != nil { + return err + } + *str, b, err = unmarshalStringSafe(b) + if err != nil { + return err + } + return nil +} + type sshFxpReaddirPacket struct { - Id uint32 + ID uint32 Handle string } +func (p sshFxpReaddirPacket) id() uint32 { return p.ID } + func (p sshFxpReaddirPacket) MarshalBinary() ([]byte, error) { - return marshalIdString(ssh_FXP_READDIR, p.Id, p.Handle) + return marshalIDString(ssh_FXP_READDIR, p.ID, p.Handle) } -func (p sshFxpReaddirPacket) id() uint32 { return p.Id } +func (p *sshFxpReaddirPacket) UnmarshalBinary(b []byte) error { + return unmarshalIDString(b, &p.ID, &p.Handle) +} type sshFxpOpendirPacket struct { - Id uint32 + ID uint32 Path string } +func (p sshFxpOpendirPacket) id() uint32 { return p.ID } + func (p sshFxpOpendirPacket) MarshalBinary() ([]byte, error) { - return marshalIdString(ssh_FXP_OPENDIR, p.Id, p.Path) + return marshalIDString(ssh_FXP_OPENDIR, p.ID, p.Path) } -func (p sshFxpOpendirPacket) id() uint32 { return p.Id } +func (p *sshFxpOpendirPacket) UnmarshalBinary(b []byte) error { + return unmarshalIDString(b, &p.ID, &p.Path) +} type sshFxpLstatPacket struct { - Id uint32 + ID uint32 Path string } -func (p sshFxpLstatPacket) id() uint32 { return p.Id } +func (p sshFxpLstatPacket) id() uint32 { return p.ID } func (p sshFxpLstatPacket) MarshalBinary() ([]byte, error) { - return marshalIdString(ssh_FXP_LSTAT, p.Id, p.Path) + return marshalIDString(ssh_FXP_LSTAT, p.ID, p.Path) +} + +func (p *sshFxpLstatPacket) UnmarshalBinary(b []byte) error { + return unmarshalIDString(b, &p.ID, &p.Path) +} + +type sshFxpStatPacket struct { + ID uint32 + Path string +} + +func (p sshFxpStatPacket) id() uint32 { return p.ID } + +func (p sshFxpStatPacket) MarshalBinary() ([]byte, error) { + return marshalIDString(ssh_FXP_LSTAT, p.ID, p.Path) +} + +func (p *sshFxpStatPacket) UnmarshalBinary(b []byte) error { + return unmarshalIDString(b, &p.ID, &p.Path) } type sshFxpFstatPacket struct { - Id uint32 + ID uint32 Handle string } -func (p sshFxpFstatPacket) id() uint32 { return p.Id } +func (p sshFxpFstatPacket) id() uint32 { return p.ID } func (p sshFxpFstatPacket) MarshalBinary() ([]byte, error) { - return marshalIdString(ssh_FXP_FSTAT, p.Id, p.Handle) + return marshalIDString(ssh_FXP_FSTAT, p.ID, p.Handle) +} + +func (p *sshFxpFstatPacket) UnmarshalBinary(b []byte) error { + return unmarshalIDString(b, &p.ID, &p.Handle) } type sshFxpClosePacket struct { - Id uint32 + ID uint32 Handle string } +func (p sshFxpClosePacket) id() uint32 { return p.ID } + func (p sshFxpClosePacket) MarshalBinary() ([]byte, error) { - return marshalIdString(ssh_FXP_CLOSE, p.Id, p.Handle) + return marshalIDString(ssh_FXP_CLOSE, p.ID, p.Handle) } -func (p sshFxpClosePacket) id() uint32 { return p.Id } +func (p *sshFxpClosePacket) UnmarshalBinary(b []byte) error { + return unmarshalIDString(b, &p.ID, &p.Handle) +} type sshFxpRemovePacket struct { - Id uint32 + ID uint32 Filename string } -func (p sshFxpRemovePacket) id() uint32 { return p.Id } +func (p sshFxpRemovePacket) id() uint32 { return p.ID } func (p sshFxpRemovePacket) MarshalBinary() ([]byte, error) { - return marshalIdString(ssh_FXP_REMOVE, p.Id, p.Filename) + return marshalIDString(ssh_FXP_REMOVE, p.ID, p.Filename) +} + +func (p *sshFxpRemovePacket) UnmarshalBinary(b []byte) error { + return unmarshalIDString(b, &p.ID, &p.Filename) } type sshFxpRmdirPacket struct { - Id uint32 + ID uint32 Path string } -func (p sshFxpRmdirPacket) id() uint32 { return p.Id } +func (p sshFxpRmdirPacket) id() uint32 { return p.ID } func (p sshFxpRmdirPacket) MarshalBinary() ([]byte, error) { - return marshalIdString(ssh_FXP_RMDIR, p.Id, p.Path) + return marshalIDString(ssh_FXP_RMDIR, p.ID, p.Path) +} + +func (p *sshFxpRmdirPacket) UnmarshalBinary(b []byte) error { + return unmarshalIDString(b, &p.ID, &p.Path) +} + +type sshFxpSymlinkPacket struct { + ID uint32 + Targetpath string + Linkpath string +} + +func (p sshFxpSymlinkPacket) id() uint32 { return p.ID } + +func (p sshFxpSymlinkPacket) MarshalBinary() ([]byte, error) { + l := 1 + 4 + // type(byte) + uint32 + 4 + len(p.Targetpath) + + 4 + len(p.Linkpath) + + b := make([]byte, 0, l) + b = append(b, ssh_FXP_SYMLINK) + b = marshalUint32(b, p.ID) + b = marshalString(b, p.Targetpath) + b = marshalString(b, p.Linkpath) + return b, nil +} + +func (p *sshFxpSymlinkPacket) UnmarshalBinary(b []byte) error { + var err error + if p.ID, b, err = unmarshalUint32Safe(b); err != nil { + return err + } else if p.Targetpath, b, err = unmarshalStringSafe(b); err != nil { + return err + } else if p.Linkpath, b, err = unmarshalStringSafe(b); err != nil { + return err + } + return nil } type sshFxpReadlinkPacket struct { - Id uint32 + ID uint32 Path string } -func (p sshFxpReadlinkPacket) id() uint32 { return p.Id } +func (p sshFxpReadlinkPacket) id() uint32 { return p.ID } func (p sshFxpReadlinkPacket) MarshalBinary() ([]byte, error) { - return marshalIdString(ssh_FXP_READLINK, p.Id, p.Path) + return marshalIDString(ssh_FXP_READLINK, p.ID, p.Path) +} + +func (p *sshFxpReadlinkPacket) UnmarshalBinary(b []byte) error { + return unmarshalIDString(b, &p.ID, &p.Path) +} + +type sshFxpRealpathPacket struct { + ID uint32 + Path string +} + +func (p sshFxpRealpathPacket) id() uint32 { return p.ID } + +func (p sshFxpRealpathPacket) MarshalBinary() ([]byte, error) { + return marshalIDString(ssh_FXP_REALPATH, p.ID, p.Path) +} + +func (p *sshFxpRealpathPacket) UnmarshalBinary(b []byte) error { + return unmarshalIDString(b, &p.ID, &p.Path) +} + +type sshFxpNameAttr struct { + Name string + LongName string + Attrs []interface{} +} + +func (p sshFxpNameAttr) MarshalBinary() ([]byte, error) { + b := []byte{} + b = marshalString(b, p.Name) + b = marshalString(b, p.LongName) + for _, attr := range p.Attrs { + b = marshal(b, attr) + } + return b, nil +} + +type sshFxpNamePacket struct { + ID uint32 + NameAttrs []sshFxpNameAttr +} + +func (p sshFxpNamePacket) MarshalBinary() ([]byte, error) { + b := []byte{} + b = append(b, ssh_FXP_NAME) + b = marshalUint32(b, p.ID) + b = marshalUint32(b, uint32(len(p.NameAttrs))) + for _, na := range p.NameAttrs { + ab, err := na.MarshalBinary() + if err != nil { + return nil, err + } + + b = append(b, ab...) + } + return b, nil } type sshFxpOpenPacket struct { - Id uint32 + ID uint32 Path string Pflags uint32 Flags uint32 // ignored } -func (p sshFxpOpenPacket) id() uint32 { return p.Id } +func (p sshFxpOpenPacket) id() uint32 { return p.ID } func (p sshFxpOpenPacket) MarshalBinary() ([]byte, error) { l := 1 + 4 + @@ -236,21 +501,35 @@ func (p sshFxpOpenPacket) MarshalBinary() ([]byte, error) { b := make([]byte, 0, l) b = append(b, ssh_FXP_OPEN) - b = marshalUint32(b, p.Id) + b = marshalUint32(b, p.ID) b = marshalString(b, p.Path) b = marshalUint32(b, p.Pflags) b = marshalUint32(b, p.Flags) return b, nil } +func (p *sshFxpOpenPacket) UnmarshalBinary(b []byte) error { + var err error + if p.ID, b, err = unmarshalUint32Safe(b); err != nil { + return err + } else if p.Path, b, err = unmarshalStringSafe(b); err != nil { + return err + } else if p.Pflags, b, err = unmarshalUint32Safe(b); err != nil { + return err + } else if p.Flags, b, err = unmarshalUint32Safe(b); err != nil { + return err + } + return nil +} + type sshFxpReadPacket struct { - Id uint32 + ID uint32 Handle string Offset uint64 Len uint32 } -func (p sshFxpReadPacket) id() uint32 { return p.Id } +func (p sshFxpReadPacket) id() uint32 { return p.ID } func (p sshFxpReadPacket) MarshalBinary() ([]byte, error) { l := 1 + 4 + // type(byte) + uint32 @@ -259,20 +538,34 @@ func (p sshFxpReadPacket) MarshalBinary() ([]byte, error) { b := make([]byte, 0, l) b = append(b, ssh_FXP_READ) - b = marshalUint32(b, p.Id) + b = marshalUint32(b, p.ID) b = marshalString(b, p.Handle) b = marshalUint64(b, p.Offset) b = marshalUint32(b, p.Len) return b, nil } +func (p *sshFxpReadPacket) UnmarshalBinary(b []byte) error { + var err error + if p.ID, b, err = unmarshalUint32Safe(b); err != nil { + return err + } else if p.Handle, b, err = unmarshalStringSafe(b); err != nil { + return err + } else if p.Offset, b, err = unmarshalUint64Safe(b); err != nil { + return err + } else if p.Len, b, err = unmarshalUint32Safe(b); err != nil { + return err + } + return nil +} + type sshFxpRenamePacket struct { - Id uint32 + ID uint32 Oldpath string Newpath string } -func (p sshFxpRenamePacket) id() uint32 { return p.Id } +func (p sshFxpRenamePacket) id() uint32 { return p.ID } func (p sshFxpRenamePacket) MarshalBinary() ([]byte, error) { l := 1 + 4 + // type(byte) + uint32 @@ -281,45 +574,75 @@ func (p sshFxpRenamePacket) MarshalBinary() ([]byte, error) { b := make([]byte, 0, l) b = append(b, ssh_FXP_RENAME) - b = marshalUint32(b, p.Id) + b = marshalUint32(b, p.ID) b = marshalString(b, p.Oldpath) b = marshalString(b, p.Newpath) return b, nil } +func (p *sshFxpRenamePacket) UnmarshalBinary(b []byte) error { + var err error + if p.ID, b, err = unmarshalUint32Safe(b); err != nil { + return err + } else if p.Oldpath, b, err = unmarshalStringSafe(b); err != nil { + return err + } else if p.Newpath, b, err = unmarshalStringSafe(b); err != nil { + return err + } + return nil +} + type sshFxpWritePacket struct { - Id uint32 + ID uint32 Handle string Offset uint64 Length uint32 Data []byte } -func (s sshFxpWritePacket) id() uint32 { return s.Id } +func (p sshFxpWritePacket) id() uint32 { return p.ID } -func (s sshFxpWritePacket) MarshalBinary() ([]byte, error) { +func (p sshFxpWritePacket) MarshalBinary() ([]byte, error) { l := 1 + 4 + // type(byte) + uint32 - 4 + len(s.Handle) + + 4 + len(p.Handle) + 8 + 4 + // uint64 + uint32 - len(s.Data) + len(p.Data) b := make([]byte, 0, l) b = append(b, ssh_FXP_WRITE) - b = marshalUint32(b, s.Id) - b = marshalString(b, s.Handle) - b = marshalUint64(b, s.Offset) - b = marshalUint32(b, s.Length) - b = append(b, s.Data...) + b = marshalUint32(b, p.ID) + b = marshalString(b, p.Handle) + b = marshalUint64(b, p.Offset) + b = marshalUint32(b, p.Length) + b = append(b, p.Data...) return b, nil } +func (p *sshFxpWritePacket) UnmarshalBinary(b []byte) error { + var err error + if p.ID, b, err = unmarshalUint32Safe(b); err != nil { + return err + } else if p.Handle, b, err = unmarshalStringSafe(b); err != nil { + return err + } else if p.Offset, b, err = unmarshalUint64Safe(b); err != nil { + return err + } else if p.Length, b, err = unmarshalUint32Safe(b); err != nil { + return err + } else if uint32(len(b)) < p.Length { + return errShortPacket + } + + p.Data = append([]byte{}, b[:p.Length]...) + return nil +} + type sshFxpMkdirPacket struct { - Id uint32 + ID uint32 Path string Flags uint32 // ignored } -func (p sshFxpMkdirPacket) id() uint32 { return p.Id } +func (p sshFxpMkdirPacket) id() uint32 { return p.ID } func (p sshFxpMkdirPacket) MarshalBinary() ([]byte, error) { l := 1 + 4 + // type(byte) + uint32 @@ -328,20 +651,40 @@ func (p sshFxpMkdirPacket) MarshalBinary() ([]byte, error) { b := make([]byte, 0, l) b = append(b, ssh_FXP_MKDIR) - b = marshalUint32(b, p.Id) + b = marshalUint32(b, p.ID) b = marshalString(b, p.Path) b = marshalUint32(b, p.Flags) return b, nil } +func (p *sshFxpMkdirPacket) UnmarshalBinary(b []byte) error { + var err error + if p.ID, b, err = unmarshalUint32Safe(b); err != nil { + return err + } else if p.Path, b, err = unmarshalStringSafe(b); err != nil { + return err + } else if p.Flags, b, err = unmarshalUint32Safe(b); err != nil { + return err + } + return nil +} + type sshFxpSetstatPacket struct { - Id uint32 + ID uint32 Path string Flags uint32 Attrs interface{} } -func (p sshFxpSetstatPacket) id() uint32 { return p.Id } +type sshFxpFsetstatPacket struct { + ID uint32 + Handle string + Flags uint32 + Attrs interface{} +} + +func (p sshFxpSetstatPacket) id() uint32 { return p.ID } +func (p sshFxpFsetstatPacket) id() uint32 { return p.ID } func (p sshFxpSetstatPacket) MarshalBinary() ([]byte, error) { l := 1 + 4 + // type(byte) + uint32 @@ -350,19 +693,112 @@ func (p sshFxpSetstatPacket) MarshalBinary() ([]byte, error) { b := make([]byte, 0, l) b = append(b, ssh_FXP_SETSTAT) - b = marshalUint32(b, p.Id) + b = marshalUint32(b, p.ID) b = marshalString(b, p.Path) b = marshalUint32(b, p.Flags) b = marshal(b, p.Attrs) return b, nil } +func (p sshFxpFsetstatPacket) MarshalBinary() ([]byte, error) { + l := 1 + 4 + // type(byte) + uint32 + 4 + len(p.Handle) + + 4 // uint32 + uint64 + + b := make([]byte, 0, l) + b = append(b, ssh_FXP_FSETSTAT) + b = marshalUint32(b, p.ID) + b = marshalString(b, p.Handle) + b = marshalUint32(b, p.Flags) + b = marshal(b, p.Attrs) + return b, nil +} + +func (p *sshFxpSetstatPacket) UnmarshalBinary(b []byte) error { + var err error + if p.ID, b, err = unmarshalUint32Safe(b); err != nil { + return err + } else if p.Path, b, err = unmarshalStringSafe(b); err != nil { + return err + } else if p.Flags, b, err = unmarshalUint32Safe(b); err != nil { + return err + } + p.Attrs = b + return nil +} + +func (p *sshFxpFsetstatPacket) UnmarshalBinary(b []byte) error { + var err error + if p.ID, b, err = unmarshalUint32Safe(b); err != nil { + return err + } else if p.Handle, b, err = unmarshalStringSafe(b); err != nil { + return err + } else if p.Flags, b, err = unmarshalUint32Safe(b); err != nil { + return err + } + p.Attrs = b + return nil +} + +type sshFxpHandlePacket struct { + ID uint32 + Handle string +} + +func (p sshFxpHandlePacket) MarshalBinary() ([]byte, error) { + b := []byte{ssh_FXP_HANDLE} + b = marshalUint32(b, p.ID) + b = marshalString(b, p.Handle) + return b, nil +} + +type sshFxpStatusPacket struct { + ID uint32 + StatusError +} + +func (p sshFxpStatusPacket) MarshalBinary() ([]byte, error) { + b := []byte{ssh_FXP_STATUS} + b = marshalUint32(b, p.ID) + b = marshalStatus(b, p.StatusError) + return b, nil +} + +type sshFxpDataPacket struct { + ID uint32 + Length uint32 + Data []byte +} + +func (p sshFxpDataPacket) MarshalBinary() ([]byte, error) { + b := []byte{ssh_FXP_DATA} + b = marshalUint32(b, p.ID) + b = marshalUint32(b, p.Length) + b = append(b, p.Data[:p.Length]...) + return b, nil +} + +func (p *sshFxpDataPacket) UnmarshalBinary(b []byte) error { + var err error + if p.ID, b, err = unmarshalUint32Safe(b); err != nil { + return err + } else if p.Length, b, err = unmarshalUint32Safe(b); err != nil { + return err + } else if uint32(len(b)) < p.Length { + return errors.New("truncated packet") + } + + p.Data = make([]byte, p.Length) + copy(p.Data, b) + return nil +} + type sshFxpStatvfsPacket struct { - Id uint32 + ID uint32 Path string } -func (p sshFxpStatvfsPacket) id() uint32 { return p.Id } +func (p sshFxpStatvfsPacket) id() uint32 { return p.ID } func (p sshFxpStatvfsPacket) MarshalBinary() ([]byte, error) { l := 1 + 4 + // type(byte) + uint32 @@ -371,14 +807,15 @@ func (p sshFxpStatvfsPacket) MarshalBinary() ([]byte, error) { b := make([]byte, 0, l) b = append(b, ssh_FXP_EXTENDED) - b = marshalUint32(b, p.Id) + b = marshalUint32(b, p.ID) b = marshalString(b, "statvfs@openssh.com") b = marshalString(b, p.Path) return b, nil } +// A StatVFS contains statistics about a filesystem. type StatVFS struct { - Id uint32 + ID uint32 Bsize uint64 /* file system block size */ Frsize uint64 /* fundamental fs block size */ Blocks uint64 /* number of blocks (unit f_frsize) */ @@ -392,10 +829,12 @@ type StatVFS struct { Namemax uint64 /* maximum filename length */ } +// TotalSpace calculates the amount of total space in a filesystem. func (p *StatVFS) TotalSpace() uint64 { return p.Frsize * p.Blocks } +// FreeSpace calculates the amount of free space in a filesystem. func (p *StatVFS) FreeSpace() uint64 { return p.Frsize * p.Bfree } diff --git a/Godeps/_workspace/src/github.com/pkg/sftp/packet_test.go b/Godeps/_workspace/src/github.com/pkg/sftp/packet_test.go deleted file mode 100644 index 80a1ebf02..000000000 --- a/Godeps/_workspace/src/github.com/pkg/sftp/packet_test.go +++ /dev/null @@ -1,261 +0,0 @@ -package sftp - -import ( - "bytes" - "encoding" - "os" - "testing" -) - -var marshalUint32Tests = []struct { - v uint32 - want []byte -}{ - {1, []byte{0, 0, 0, 1}}, - {256, []byte{0, 0, 1, 0}}, - {^uint32(0), []byte{255, 255, 255, 255}}, -} - -func TestMarshalUint32(t *testing.T) { - for _, tt := range marshalUint32Tests { - got := marshalUint32(nil, tt.v) - if !bytes.Equal(tt.want, got) { - t.Errorf("marshalUint32(%d): want %v, got %v", tt.v, tt.want, got) - } - } -} - -var marshalUint64Tests = []struct { - v uint64 - want []byte -}{ - {1, []byte{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1}}, - {256, []byte{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0}}, - {^uint64(0), []byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}}, - {1 << 32, []byte{0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0}}, -} - -func TestMarshalUint64(t *testing.T) { - for _, tt := range marshalUint64Tests { - got := marshalUint64(nil, tt.v) - if !bytes.Equal(tt.want, got) { - t.Errorf("marshalUint64(%d): want %#v, got %#v", tt.v, tt.want, got) - } - } -} - -var marshalStringTests = []struct { - v string - want []byte -}{ - {"", []byte{0, 0, 0, 0}}, - {"/foo", []byte{0x0, 0x0, 0x0, 0x4, 0x2f, 0x66, 0x6f, 0x6f}}, -} - -func TestMarshalString(t *testing.T) { - for _, tt := range marshalStringTests { - got := marshalString(nil, tt.v) - if !bytes.Equal(tt.want, got) { - t.Errorf("marshalString(%q): want %#v, got %#v", tt.v, tt.want, got) - } - } -} - -var marshalTests = []struct { - v interface{} - want []byte -}{ - {uint8(1), []byte{1}}, - {byte(1), []byte{1}}, - {uint32(1), []byte{0, 0, 0, 1}}, - {uint64(1), []byte{0, 0, 0, 0, 0, 0, 0, 1}}, - {"foo", []byte{0x0, 0x0, 0x0, 0x3, 0x66, 0x6f, 0x6f}}, - {[]uint32{1, 2, 3, 4}, []byte{0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x4}}, -} - -func TestMarshal(t *testing.T) { - for _, tt := range marshalTests { - got := marshal(nil, tt.v) - if !bytes.Equal(tt.want, got) { - t.Errorf("marshal(%v): want %#v, got %#v", tt.v, tt.want, got) - } - } -} - -var unmarshalUint32Tests = []struct { - b []byte - want uint32 - rest []byte -}{ - {[]byte{0, 0, 0, 0}, 0, nil}, - {[]byte{0, 0, 1, 0}, 256, nil}, - {[]byte{255, 0, 0, 255}, 4278190335, nil}, -} - -func TestUnmarshalUint32(t *testing.T) { - for _, tt := range unmarshalUint32Tests { - got, rest := unmarshalUint32(tt.b) - if got != tt.want || !bytes.Equal(rest, tt.rest) { - t.Errorf("unmarshalUint32(%v): want %v, %#v, got %v, %#v", tt.b, tt.want, tt.rest, got, rest) - } - } -} - -var unmarshalUint64Tests = []struct { - b []byte - want uint64 - rest []byte -}{ - {[]byte{0, 0, 0, 0, 0, 0, 0, 0}, 0, nil}, - {[]byte{0, 0, 0, 0, 0, 0, 1, 0}, 256, nil}, - {[]byte{255, 0, 0, 0, 0, 0, 0, 255}, 18374686479671623935, nil}, -} - -func TestUnmarshalUint64(t *testing.T) { - for _, tt := range unmarshalUint64Tests { - got, rest := unmarshalUint64(tt.b) - if got != tt.want || !bytes.Equal(rest, tt.rest) { - t.Errorf("unmarshalUint64(%v): want %v, %#v, got %v, %#v", tt.b, tt.want, tt.rest, got, rest) - } - } -} - -var unmarshalStringTests = []struct { - b []byte - want string - rest []byte -}{ - {marshalString(nil, ""), "", nil}, - {marshalString(nil, "blah"), "blah", nil}, -} - -func TestUnmarshalString(t *testing.T) { - for _, tt := range unmarshalStringTests { - got, rest := unmarshalString(tt.b) - if got != tt.want || !bytes.Equal(rest, tt.rest) { - t.Errorf("unmarshalUint64(%v): want %q, %#v, got %q, %#v", tt.b, tt.want, tt.rest, got, rest) - } - } -} - -var sendPacketTests = []struct { - p encoding.BinaryMarshaler - want []byte -}{ - {sshFxInitPacket{ - Version: 3, - Extensions: []struct{ Name, Data string }{ - {"posix-rename@openssh.com", "1"}, - }, - }, []byte{0x0, 0x0, 0x0, 0x26, 0x1, 0x0, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x18, 0x70, 0x6f, 0x73, 0x69, 0x78, 0x2d, 0x72, 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x40, 0x6f, 0x70, 0x65, 0x6e, 0x73, 0x73, 0x68, 0x2e, 0x63, 0x6f, 0x6d, 0x0, 0x0, 0x0, 0x1, 0x31}}, - - {sshFxpOpenPacket{ - Id: 1, - Path: "/foo", - Pflags: flags(os.O_RDONLY), - }, []byte{0x0, 0x0, 0x0, 0x15, 0x3, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x4, 0x2f, 0x66, 0x6f, 0x6f, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0}}, - - {sshFxpWritePacket{ - Id: 124, - Handle: "foo", - Offset: 13, - Length: uint32(len([]byte("bar"))), - Data: []byte("bar"), - }, []byte{0x0, 0x0, 0x0, 0x1b, 0x6, 0x0, 0x0, 0x0, 0x7c, 0x0, 0x0, 0x0, 0x3, 0x66, 0x6f, 0x6f, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xd, 0x0, 0x0, 0x0, 0x3, 0x62, 0x61, 0x72}}, - - {sshFxpSetstatPacket{ - Id: 31, - Path: "/bar", - Flags: flags(os.O_WRONLY), - Attrs: struct { - Uid uint32 - Gid uint32 - }{1000, 100}, - }, []byte{0x0, 0x0, 0x0, 0x19, 0x9, 0x0, 0x0, 0x0, 0x1f, 0x0, 0x0, 0x0, 0x4, 0x2f, 0x62, 0x61, 0x72, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x3, 0xe8, 0x0, 0x0, 0x0, 0x64}}, -} - -func TestSendPacket(t *testing.T) { - for _, tt := range sendPacketTests { - var w bytes.Buffer - sendPacket(&w, tt.p) - if got := w.Bytes(); !bytes.Equal(tt.want, got) { - t.Errorf("sendPacket(%v): want %#v, got %#v", tt.p, tt.want, got) - } - } -} - -func sp(p encoding.BinaryMarshaler) []byte { - var w bytes.Buffer - sendPacket(&w, p) - return w.Bytes() -} - -var recvPacketTests = []struct { - b []byte - want uint8 - rest []byte -}{ - {sp(sshFxInitPacket{ - Version: 3, - Extensions: []struct{ Name, Data string }{ - {"posix-rename@openssh.com", "1"}, - }, - }), ssh_FXP_INIT, []byte{0x0, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x18, 0x70, 0x6f, 0x73, 0x69, 0x78, 0x2d, 0x72, 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x40, 0x6f, 0x70, 0x65, 0x6e, 0x73, 0x73, 0x68, 0x2e, 0x63, 0x6f, 0x6d, 0x0, 0x0, 0x0, 0x1, 0x31}}, -} - -func TestRecvPacket(t *testing.T) { - for _, tt := range recvPacketTests { - r := bytes.NewReader(tt.b) - got, rest, _ := recvPacket(r) - if got != tt.want || !bytes.Equal(rest, tt.rest) { - t.Errorf("recvPacket(%#v): want %v, %#v, got %v, %#v", tt.b, tt.want, tt.rest, got, rest) - } - } -} - -func BenchmarkMarshalInit(b *testing.B) { - for i := 0; i < b.N; i++ { - sp(sshFxInitPacket{ - Version: 3, - Extensions: []struct{ Name, Data string }{ - {"posix-rename@openssh.com", "1"}, - }, - }) - } -} - -func BenchmarkMarshalOpen(b *testing.B) { - for i := 0; i < b.N; i++ { - sp(sshFxpOpenPacket{ - Id: 1, - Path: "/home/test/some/random/path", - Pflags: flags(os.O_RDONLY), - }) - } -} - -func BenchmarkMarshalWriteWorstCase(b *testing.B) { - data := make([]byte, 32*1024) - for i := 0; i < b.N; i++ { - sp(sshFxpWritePacket{ - Id: 1, - Handle: "someopaquehandle", - Offset: 0, - Length: uint32(len(data)), - Data: data, - }) - } -} - -func BenchmarkMarshalWrite1k(b *testing.B) { - data := make([]byte, 1024) - for i := 0; i < b.N; i++ { - sp(sshFxpWritePacket{ - Id: 1, - Handle: "someopaquehandle", - Offset: 0, - Length: uint32(len(data)), - Data: data, - }) - } -} diff --git a/Godeps/_workspace/src/github.com/pkg/sftp/release.go b/Godeps/_workspace/src/github.com/pkg/sftp/release.go index aa8262cda..b695528fd 100644 --- a/Godeps/_workspace/src/github.com/pkg/sftp/release.go +++ b/Godeps/_workspace/src/github.com/pkg/sftp/release.go @@ -1,4 +1,4 @@ -// +build !debug_sftp +// +build !debug package sftp diff --git a/Godeps/_workspace/src/github.com/pkg/sftp/server.go b/Godeps/_workspace/src/github.com/pkg/sftp/server.go new file mode 100644 index 000000000..81b165917 --- /dev/null +++ b/Godeps/_workspace/src/github.com/pkg/sftp/server.go @@ -0,0 +1,648 @@ +package sftp + +// sftp server counterpart + +import ( + "encoding" + "fmt" + "io" + "io/ioutil" + "os" + "path/filepath" + "strconv" + "sync" + "syscall" + "time" +) + +const ( + sftpServerWorkerCount = 8 +) + +// Server is an SSH File Transfer Protocol (sftp) server. +// This is intended to provide the sftp subsystem to an ssh server daemon. +// This implementation currently supports most of sftp server protocol version 3, +// as specified at http://tools.ietf.org/html/draft-ietf-secsh-filexfer-02 +type Server struct { + in io.Reader + out io.WriteCloser + outMutex *sync.Mutex + debugStream io.Writer + readOnly bool + lastID uint32 + pktChan chan rxPacket + openFiles map[string]*os.File + openFilesLock *sync.RWMutex + handleCount int + maxTxPacket uint32 + workerCount int +} + +func (svr *Server) nextHandle(f *os.File) string { + svr.openFilesLock.Lock() + defer svr.openFilesLock.Unlock() + svr.handleCount++ + handle := strconv.Itoa(svr.handleCount) + svr.openFiles[handle] = f + return handle +} + +func (svr *Server) closeHandle(handle string) error { + svr.openFilesLock.Lock() + defer svr.openFilesLock.Unlock() + if f, ok := svr.openFiles[handle]; ok { + delete(svr.openFiles, handle) + return f.Close() + } + + return syscall.EBADF +} + +func (svr *Server) getHandle(handle string) (*os.File, bool) { + svr.openFilesLock.RLock() + defer svr.openFilesLock.RUnlock() + f, ok := svr.openFiles[handle] + return f, ok +} + +type serverRespondablePacket interface { + encoding.BinaryUnmarshaler + id() uint32 + respond(svr *Server) error + readonly() bool +} + +// NewServer creates a new Server instance around the provided streams, serving +// content from the root of the filesystem. Optionally, ServerOption +// functions may be specified to further configure the Server. +// +// A subsequent call to Serve() is required to begin serving files over SFTP. +func NewServer(in io.Reader, out io.WriteCloser, options ...ServerOption) (*Server, error) { + s := &Server{ + in: in, + out: out, + outMutex: &sync.Mutex{}, + debugStream: ioutil.Discard, + pktChan: make(chan rxPacket, sftpServerWorkerCount), + openFiles: map[string]*os.File{}, + openFilesLock: &sync.RWMutex{}, + maxTxPacket: 1 << 15, + workerCount: sftpServerWorkerCount, + } + + for _, o := range options { + if err := o(s); err != nil { + return nil, err + } + } + + return s, nil +} + +// A ServerOption is a function which applies configuration to a Server. +type ServerOption func(*Server) error + +// WithDebug enables Server debugging output to the supplied io.Writer. +func WithDebug(w io.Writer) ServerOption { + return func(s *Server) error { + s.debugStream = w + return nil + } +} + +// ReadOnly configures a Server to serve files in read-only mode. +func ReadOnly() ServerOption { + return func(s *Server) error { + s.readOnly = true + return nil + } +} + +type rxPacket struct { + pktType fxp + pktBytes []byte +} + +// Unmarshal a single logical packet from the secure channel +func (svr *Server) rxPackets() error { + defer close(svr.pktChan) + + for { + pktType, pktBytes, err := recvPacket(svr.in) + switch err { + case nil: + svr.pktChan <- rxPacket{fxp(pktType), pktBytes} + case io.EOF: + return nil + default: + fmt.Fprintf(svr.debugStream, "recvPacket error: %v\n", err) + return err + } + } +} + +// Up to N parallel servers +func (svr *Server) sftpServerWorker(doneChan chan error) { + for pkt := range svr.pktChan { + dPkt, err := svr.decodePacket(pkt.pktType, pkt.pktBytes) + if err != nil { + fmt.Fprintf(svr.debugStream, "decodePacket error: %v\n", err) + doneChan <- err + return + } + + // If server is operating read-only and a write operation is requested, + // return permission denied + if !dPkt.readonly() && svr.readOnly { + _ = svr.sendPacket(statusFromError(dPkt.id(), syscall.EPERM)) + continue + } + + _ = dPkt.respond(svr) + } + doneChan <- nil +} + +// Serve serves SFTP connections until the streams stop or the SFTP subsystem +// is stopped. +func (svr *Server) Serve() error { + go svr.rxPackets() + doneChan := make(chan error) + for i := 0; i < svr.workerCount; i++ { + go svr.sftpServerWorker(doneChan) + } + for i := 0; i < svr.workerCount; i++ { + if err := <-doneChan; err != nil { + // abort early and shut down the session on un-decodable packets + break + } + } + // close any still-open files + for handle, file := range svr.openFiles { + fmt.Fprintf(svr.debugStream, "sftp server file with handle '%v' left open: %v\n", handle, file.Name()) + file.Close() + } + return svr.out.Close() +} + +func (svr *Server) decodePacket(pktType fxp, pktBytes []byte) (serverRespondablePacket, error) { + var pkt serverRespondablePacket + switch pktType { + case ssh_FXP_INIT: + pkt = &sshFxInitPacket{} + case ssh_FXP_LSTAT: + pkt = &sshFxpLstatPacket{} + case ssh_FXP_OPEN: + pkt = &sshFxpOpenPacket{} + case ssh_FXP_CLOSE: + pkt = &sshFxpClosePacket{} + case ssh_FXP_READ: + pkt = &sshFxpReadPacket{} + case ssh_FXP_WRITE: + pkt = &sshFxpWritePacket{} + case ssh_FXP_FSTAT: + pkt = &sshFxpFstatPacket{} + case ssh_FXP_SETSTAT: + pkt = &sshFxpSetstatPacket{} + case ssh_FXP_FSETSTAT: + pkt = &sshFxpFsetstatPacket{} + case ssh_FXP_OPENDIR: + pkt = &sshFxpOpendirPacket{} + case ssh_FXP_READDIR: + pkt = &sshFxpReaddirPacket{} + case ssh_FXP_REMOVE: + pkt = &sshFxpRemovePacket{} + case ssh_FXP_MKDIR: + pkt = &sshFxpMkdirPacket{} + case ssh_FXP_RMDIR: + pkt = &sshFxpRmdirPacket{} + case ssh_FXP_REALPATH: + pkt = &sshFxpRealpathPacket{} + case ssh_FXP_STAT: + pkt = &sshFxpStatPacket{} + case ssh_FXP_RENAME: + pkt = &sshFxpRenamePacket{} + case ssh_FXP_READLINK: + pkt = &sshFxpReadlinkPacket{} + case ssh_FXP_SYMLINK: + pkt = &sshFxpSymlinkPacket{} + default: + return nil, fmt.Errorf("unhandled packet type: %s", pktType) + } + err := pkt.UnmarshalBinary(pktBytes) + return pkt, err +} + +func (p sshFxInitPacket) respond(svr *Server) error { + return svr.sendPacket(sshFxVersionPacket{sftpProtocolVersion, nil}) +} + +// The init packet has no ID, so we just return a zero-value ID +func (p sshFxInitPacket) id() uint32 { return 0 } +func (p sshFxInitPacket) readonly() bool { return true } + +type sshFxpStatResponse struct { + ID uint32 + info os.FileInfo +} + +func (p sshFxpStatResponse) MarshalBinary() ([]byte, error) { + b := []byte{ssh_FXP_ATTRS} + b = marshalUint32(b, p.ID) + b = marshalFileInfo(b, p.info) + return b, nil +} + +func (p sshFxpLstatPacket) readonly() bool { return true } + +func (p sshFxpLstatPacket) respond(svr *Server) error { + // stat the requested file + info, err := os.Lstat(p.Path) + if err != nil { + return svr.sendPacket(statusFromError(p.ID, err)) + } + + return svr.sendPacket(sshFxpStatResponse{ + ID: p.ID, + info: info, + }) +} + +func (p sshFxpStatPacket) readonly() bool { return true } + +func (p sshFxpStatPacket) respond(svr *Server) error { + // stat the requested file + info, err := os.Stat(p.Path) + if err != nil { + return svr.sendPacket(statusFromError(p.ID, err)) + } + + return svr.sendPacket(sshFxpStatResponse{ + ID: p.ID, + info: info, + }) +} + +func (p sshFxpFstatPacket) readonly() bool { return true } + +func (p sshFxpFstatPacket) respond(svr *Server) error { + f, ok := svr.getHandle(p.Handle) + if !ok { + return svr.sendPacket(statusFromError(p.ID, syscall.EBADF)) + } + + info, err := f.Stat() + if err != nil { + return svr.sendPacket(statusFromError(p.ID, err)) + } + + return svr.sendPacket(sshFxpStatResponse{ + ID: p.ID, + info: info, + }) +} + +func (p sshFxpMkdirPacket) readonly() bool { return false } + +func (p sshFxpMkdirPacket) respond(svr *Server) error { + // TODO FIXME: ignore flags field + err := os.Mkdir(p.Path, 0755) + return svr.sendPacket(statusFromError(p.ID, err)) +} + +func (p sshFxpRmdirPacket) readonly() bool { return false } + +func (p sshFxpRmdirPacket) respond(svr *Server) error { + err := os.Remove(p.Path) + return svr.sendPacket(statusFromError(p.ID, err)) +} + +func (p sshFxpRemovePacket) readonly() bool { return false } + +func (p sshFxpRemovePacket) respond(svr *Server) error { + err := os.Remove(p.Filename) + return svr.sendPacket(statusFromError(p.ID, err)) +} + +func (p sshFxpRenamePacket) readonly() bool { return false } + +func (p sshFxpRenamePacket) respond(svr *Server) error { + err := os.Rename(p.Oldpath, p.Newpath) + return svr.sendPacket(statusFromError(p.ID, err)) +} + +func (p sshFxpSymlinkPacket) readonly() bool { return false } + +func (p sshFxpSymlinkPacket) respond(svr *Server) error { + err := os.Symlink(p.Targetpath, p.Linkpath) + return svr.sendPacket(statusFromError(p.ID, err)) +} + +var emptyFileStat = []interface{}{uint32(0)} + +func (p sshFxpReadlinkPacket) readonly() bool { return true } + +func (p sshFxpReadlinkPacket) respond(svr *Server) error { + f, err := os.Readlink(p.Path) + if err != nil { + return svr.sendPacket(statusFromError(p.ID, err)) + } + + return svr.sendPacket(sshFxpNamePacket{ + ID: p.ID, + NameAttrs: []sshFxpNameAttr{{ + Name: f, + LongName: f, + Attrs: emptyFileStat, + }}, + }) +} + +func (p sshFxpRealpathPacket) readonly() bool { return true } + +func (p sshFxpRealpathPacket) respond(svr *Server) error { + f, err := filepath.Abs(p.Path) + if err != nil { + return svr.sendPacket(statusFromError(p.ID, err)) + } + + f = filepath.Clean(f) + + return svr.sendPacket(sshFxpNamePacket{ + ID: p.ID, + NameAttrs: []sshFxpNameAttr{{ + Name: f, + LongName: f, + Attrs: emptyFileStat, + }}, + }) +} + +func (p sshFxpOpendirPacket) readonly() bool { return true } + +func (p sshFxpOpendirPacket) respond(svr *Server) error { + return sshFxpOpenPacket{ + ID: p.ID, + Path: p.Path, + Pflags: ssh_FXF_READ, + }.respond(svr) +} + +func (p sshFxpOpenPacket) readonly() bool { + return !p.hasPflags(ssh_FXF_WRITE) +} + +func (p sshFxpOpenPacket) hasPflags(flags ...uint32) bool { + for _, f := range flags { + if p.Pflags&f == 0 { + return false + } + } + + return true +} + +func (p sshFxpOpenPacket) respond(svr *Server) error { + var osFlags int + if p.hasPflags(ssh_FXF_READ, ssh_FXF_WRITE) { + osFlags |= os.O_RDWR + } else if p.hasPflags(ssh_FXF_WRITE) { + osFlags |= os.O_WRONLY + } else if p.hasPflags(ssh_FXF_READ) { + osFlags |= os.O_RDONLY + } else { + // how are they opening? + return svr.sendPacket(statusFromError(p.ID, syscall.EINVAL)) + } + + if p.hasPflags(ssh_FXF_APPEND) { + osFlags |= os.O_APPEND + } + if p.hasPflags(ssh_FXF_CREAT) { + osFlags |= os.O_CREATE + } + if p.hasPflags(ssh_FXF_TRUNC) { + osFlags |= os.O_TRUNC + } + if p.hasPflags(ssh_FXF_EXCL) { + osFlags |= os.O_EXCL + } + + f, err := os.OpenFile(p.Path, osFlags, 0644) + if err != nil { + return svr.sendPacket(statusFromError(p.ID, err)) + } + + handle := svr.nextHandle(f) + return svr.sendPacket(sshFxpHandlePacket{p.ID, handle}) +} + +func (p sshFxpClosePacket) readonly() bool { return true } + +func (p sshFxpClosePacket) respond(svr *Server) error { + return svr.sendPacket(statusFromError(p.ID, svr.closeHandle(p.Handle))) +} + +func (p sshFxpReadPacket) readonly() bool { return true } + +func (p sshFxpReadPacket) respond(svr *Server) error { + f, ok := svr.getHandle(p.Handle) + if !ok { + return svr.sendPacket(statusFromError(p.ID, syscall.EBADF)) + } + + if p.Len > svr.maxTxPacket { + p.Len = svr.maxTxPacket + } + ret := sshFxpDataPacket{ + ID: p.ID, + Length: p.Len, + Data: make([]byte, p.Len), + } + + n, err := f.ReadAt(ret.Data, int64(p.Offset)) + if err != nil && (err != io.EOF || n == 0) { + return svr.sendPacket(statusFromError(p.ID, err)) + } + + ret.Length = uint32(n) + return svr.sendPacket(ret) +} + +func (p sshFxpWritePacket) readonly() bool { return false } + +func (p sshFxpWritePacket) respond(svr *Server) error { + f, ok := svr.getHandle(p.Handle) + if !ok { + return svr.sendPacket(statusFromError(p.ID, syscall.EBADF)) + } + + _, err := f.WriteAt(p.Data, int64(p.Offset)) + return svr.sendPacket(statusFromError(p.ID, err)) +} + +func (p sshFxpReaddirPacket) readonly() bool { return true } + +func (p sshFxpReaddirPacket) respond(svr *Server) error { + f, ok := svr.getHandle(p.Handle) + if !ok { + return svr.sendPacket(statusFromError(p.ID, syscall.EBADF)) + } + + dirname := f.Name() + dirents, err := f.Readdir(128) + if err != nil { + return svr.sendPacket(statusFromError(p.ID, err)) + } + + ret := sshFxpNamePacket{ID: p.ID} + for _, dirent := range dirents { + ret.NameAttrs = append(ret.NameAttrs, sshFxpNameAttr{ + Name: dirent.Name(), + LongName: runLs(dirname, dirent), + Attrs: []interface{}{dirent}, + }) + } + return svr.sendPacket(ret) +} + +func (p sshFxpSetstatPacket) readonly() bool { return false } + +func (p sshFxpSetstatPacket) respond(svr *Server) error { + // additional unmarshalling is required for each possibility here + b := p.Attrs.([]byte) + var err error + + debug("setstat name \"%s\"", p.Path) + if (p.Flags & ssh_FILEXFER_ATTR_SIZE) != 0 { + var size uint64 + if size, b, err = unmarshalUint64Safe(b); err == nil { + err = os.Truncate(p.Path, int64(size)) + } + } + if (p.Flags & ssh_FILEXFER_ATTR_PERMISSIONS) != 0 { + var mode uint32 + if mode, b, err = unmarshalUint32Safe(b); err == nil { + err = os.Chmod(p.Path, os.FileMode(mode)) + } + } + if (p.Flags & ssh_FILEXFER_ATTR_ACMODTIME) != 0 { + var atime uint32 + var mtime uint32 + if atime, b, err = unmarshalUint32Safe(b); err != nil { + } else if mtime, b, err = unmarshalUint32Safe(b); err != nil { + } else { + atimeT := time.Unix(int64(atime), 0) + mtimeT := time.Unix(int64(mtime), 0) + err = os.Chtimes(p.Path, atimeT, mtimeT) + } + } + if (p.Flags & ssh_FILEXFER_ATTR_UIDGID) != 0 { + var uid uint32 + var gid uint32 + if uid, b, err = unmarshalUint32Safe(b); err != nil { + } else if gid, b, err = unmarshalUint32Safe(b); err != nil { + } else { + err = os.Chown(p.Path, int(uid), int(gid)) + } + } + + return svr.sendPacket(statusFromError(p.ID, err)) +} + +func (p sshFxpFsetstatPacket) readonly() bool { return false } + +func (p sshFxpFsetstatPacket) respond(svr *Server) error { + f, ok := svr.getHandle(p.Handle) + if !ok { + return svr.sendPacket(statusFromError(p.ID, syscall.EBADF)) + } + + // additional unmarshalling is required for each possibility here + b := p.Attrs.([]byte) + var err error + + debug("fsetstat name \"%s\"", f.Name()) + if (p.Flags & ssh_FILEXFER_ATTR_SIZE) != 0 { + var size uint64 + if size, b, err = unmarshalUint64Safe(b); err == nil { + err = f.Truncate(int64(size)) + } + } + if (p.Flags & ssh_FILEXFER_ATTR_PERMISSIONS) != 0 { + var mode uint32 + if mode, b, err = unmarshalUint32Safe(b); err == nil { + err = f.Chmod(os.FileMode(mode)) + } + } + if (p.Flags & ssh_FILEXFER_ATTR_ACMODTIME) != 0 { + var atime uint32 + var mtime uint32 + if atime, b, err = unmarshalUint32Safe(b); err != nil { + } else if mtime, b, err = unmarshalUint32Safe(b); err != nil { + } else { + atimeT := time.Unix(int64(atime), 0) + mtimeT := time.Unix(int64(mtime), 0) + err = os.Chtimes(f.Name(), atimeT, mtimeT) + } + } + if (p.Flags & ssh_FILEXFER_ATTR_UIDGID) != 0 { + var uid uint32 + var gid uint32 + if uid, b, err = unmarshalUint32Safe(b); err != nil { + } else if gid, b, err = unmarshalUint32Safe(b); err != nil { + } else { + err = f.Chown(int(uid), int(gid)) + } + } + + return svr.sendPacket(statusFromError(p.ID, err)) +} + +// translateErrno translates a syscall error number to a SFTP error code. +func translateErrno(errno syscall.Errno) uint32 { + switch errno { + case 0: + return ssh_FX_OK + case syscall.ENOENT: + return ssh_FX_NO_SUCH_FILE + case syscall.EPERM: + return ssh_FX_PERMISSION_DENIED + } + + return ssh_FX_FAILURE +} + +func statusFromError(id uint32, err error) sshFxpStatusPacket { + ret := sshFxpStatusPacket{ + ID: id, + StatusError: StatusError{ + // ssh_FX_OK = 0 + // ssh_FX_EOF = 1 + // ssh_FX_NO_SUCH_FILE = 2 ENOENT + // ssh_FX_PERMISSION_DENIED = 3 + // ssh_FX_FAILURE = 4 + // ssh_FX_BAD_MESSAGE = 5 + // ssh_FX_NO_CONNECTION = 6 + // ssh_FX_CONNECTION_LOST = 7 + // ssh_FX_OP_UNSUPPORTED = 8 + Code: ssh_FX_OK, + }, + } + if err != nil { + debug("statusFromError: error is %T %#v", err, err) + ret.StatusError.Code = ssh_FX_FAILURE + ret.StatusError.msg = err.Error() + if err == io.EOF { + ret.StatusError.Code = ssh_FX_EOF + } else if errno, ok := err.(syscall.Errno); ok { + ret.StatusError.Code = translateErrno(errno) + } else if pathError, ok := err.(*os.PathError); ok { + debug("statusFromError: error is %T %#v", pathError.Err, pathError.Err) + if errno, ok := pathError.Err.(syscall.Errno); ok { + ret.StatusError.Code = translateErrno(errno) + } + } + } + return ret +} diff --git a/Godeps/_workspace/src/github.com/pkg/sftp/server_standalone/main.go b/Godeps/_workspace/src/github.com/pkg/sftp/server_standalone/main.go new file mode 100644 index 000000000..f7fcccdee --- /dev/null +++ b/Godeps/_workspace/src/github.com/pkg/sftp/server_standalone/main.go @@ -0,0 +1,40 @@ +package main + +// small wrapper around sftp server that allows it to be used as a separate process subsystem call by the ssh server. +// in practice this will statically link; however this allows unit testing from the sftp client. + +import ( + "flag" + "fmt" + "io/ioutil" + "os" + + "github.com/pkg/sftp" +) + +func main() { + var ( + readOnly bool + debugStderr bool + ) + + flag.BoolVar(&readOnly, "R", false, "read-only server") + flag.BoolVar(&debugStderr, "e", false, "debug to stderr") + flag.Parse() + + debugStream := ioutil.Discard + if debugStderr { + debugStream = os.Stderr + } + + svr, _ := sftp.NewServer( + os.Stdin, + os.Stdout, + sftp.WithDebug(debugStream), + sftp.ReadOnly(), + ) + if err := svr.Serve(); err != nil { + fmt.Fprintf(debugStream, "sftp server completed with error: %v", err) + os.Exit(1) + } +} diff --git a/Godeps/_workspace/src/github.com/pkg/sftp/server_stubs.go b/Godeps/_workspace/src/github.com/pkg/sftp/server_stubs.go new file mode 100644 index 000000000..3b1ddbdbb --- /dev/null +++ b/Godeps/_workspace/src/github.com/pkg/sftp/server_stubs.go @@ -0,0 +1,12 @@ +// +build !cgo,!plan9 windows android + +package sftp + +import ( + "os" + "path" +) + +func runLs(dirname string, dirent os.FileInfo) string { + return path.Join(dirname, dirent.Name()) +} diff --git a/Godeps/_workspace/src/github.com/pkg/sftp/server_unix.go b/Godeps/_workspace/src/github.com/pkg/sftp/server_unix.go new file mode 100644 index 000000000..8c3f0b44e --- /dev/null +++ b/Godeps/_workspace/src/github.com/pkg/sftp/server_unix.go @@ -0,0 +1,143 @@ +// +build darwin dragonfly freebsd !android,linux netbsd openbsd solaris +// +build cgo + +package sftp + +import ( + "fmt" + "os" + "path" + "syscall" + "time" +) + +func runLsTypeWord(dirent os.FileInfo) string { + // find first character, the type char + // b Block special file. + // c Character special file. + // d Directory. + // l Symbolic link. + // s Socket link. + // p FIFO. + // - Regular file. + tc := '-' + mode := dirent.Mode() + if (mode & os.ModeDir) != 0 { + tc = 'd' + } else if (mode & os.ModeDevice) != 0 { + tc = 'b' + if (mode & os.ModeCharDevice) != 0 { + tc = 'c' + } + } else if (mode & os.ModeSymlink) != 0 { + tc = 'l' + } else if (mode & os.ModeSocket) != 0 { + tc = 's' + } else if (mode & os.ModeNamedPipe) != 0 { + tc = 'p' + } + + // owner + orc := '-' + if (mode & 0400) != 0 { + orc = 'r' + } + owc := '-' + if (mode & 0200) != 0 { + owc = 'w' + } + oxc := '-' + ox := (mode & 0100) != 0 + setuid := (mode & os.ModeSetuid) != 0 + if ox && setuid { + oxc = 's' + } else if setuid { + oxc = 'S' + } else if ox { + oxc = 'x' + } + + // group + grc := '-' + if (mode & 040) != 0 { + grc = 'r' + } + gwc := '-' + if (mode & 020) != 0 { + gwc = 'w' + } + gxc := '-' + gx := (mode & 010) != 0 + setgid := (mode & os.ModeSetgid) != 0 + if gx && setgid { + gxc = 's' + } else if setgid { + gxc = 'S' + } else if gx { + gxc = 'x' + } + + // all / others + arc := '-' + if (mode & 04) != 0 { + arc = 'r' + } + awc := '-' + if (mode & 02) != 0 { + awc = 'w' + } + axc := '-' + ax := (mode & 01) != 0 + sticky := (mode & os.ModeSticky) != 0 + if ax && sticky { + axc = 't' + } else if sticky { + axc = 'T' + } else if ax { + axc = 'x' + } + + return fmt.Sprintf("%c%c%c%c%c%c%c%c%c%c", tc, orc, owc, oxc, grc, gwc, gxc, arc, awc, axc) +} + +func runLsStatt(dirname string, dirent os.FileInfo, statt *syscall.Stat_t) string { + // example from openssh sftp server: + // crw-rw-rw- 1 root wheel 0 Jul 31 20:52 ttyvd + // format: + // {directory / char device / etc}{rwxrwxrwx} {number of links} owner group size month day [time (this year) | year (otherwise)] name + + typeword := runLsTypeWord(dirent) + numLinks := statt.Nlink + uid := statt.Uid + gid := statt.Gid + username := fmt.Sprintf("%d", uid) + groupname := fmt.Sprintf("%d", gid) + // TODO FIXME: uid -> username, gid -> groupname lookup for ls -l format output + + mtime := dirent.ModTime() + monthStr := mtime.Month().String()[0:3] + day := mtime.Day() + year := mtime.Year() + now := time.Now() + isOld := mtime.Before(now.Add(-time.Hour * 24 * 365 / 2)) + + yearOrTime := fmt.Sprintf("%02d:%02d", mtime.Hour(), mtime.Minute()) + if isOld { + yearOrTime = fmt.Sprintf("%d", year) + } + + return fmt.Sprintf("%s %4d %-8s %-8s %8d %s %2d %5s %s", typeword, numLinks, username, groupname, dirent.Size(), monthStr, day, yearOrTime, dirent.Name()) +} + +// ls -l style output for a file, which is in the 'long output' section of a readdir response packet +// this is a very simple (lazy) implementation, just enough to look almost like openssh in a few basic cases +func runLs(dirname string, dirent os.FileInfo) string { + dsys := dirent.Sys() + if dsys == nil { + } else if statt, ok := dsys.(*syscall.Stat_t); !ok { + } else { + return runLsStatt(dirname, dirent, statt) + } + + return path.Join(dirname, dirent.Name()) +} diff --git a/Godeps/_workspace/src/github.com/pkg/sftp/sftp.go b/Godeps/_workspace/src/github.com/pkg/sftp/sftp.go index 934684c23..7d96660f8 100644 --- a/Godeps/_workspace/src/github.com/pkg/sftp/sftp.go +++ b/Godeps/_workspace/src/github.com/pkg/sftp/sftp.go @@ -46,6 +46,32 @@ const ( ssh_FX_NO_CONNECTION = 6 ssh_FX_CONNECTION_LOST = 7 ssh_FX_OP_UNSUPPORTED = 8 + + // see draft-ietf-secsh-filexfer-13 + // https://tools.ietf.org/html/draft-ietf-secsh-filexfer-13#section-9.1 + ssh_FX_INVALID_HANDLE = 9 + ssh_FX_NO_SUCH_PATH = 10 + ssh_FX_FILE_ALREADY_EXISTS = 11 + ssh_FX_WRITE_PROTECT = 12 + ssh_FX_NO_MEDIA = 13 + ssh_FX_NO_SPACE_ON_FILESYSTEM = 14 + ssh_FX_QUOTA_EXCEEDED = 15 + ssh_FX_UNKNOWN_PRINCIPAL = 16 + ssh_FX_LOCK_CONFLICT = 17 + ssh_FX_DIR_NOT_EMPTY = 18 + ssh_FX_NOT_A_DIRECTORY = 19 + ssh_FX_INVALID_FILENAME = 20 + ssh_FX_LINK_LOOP = 21 + ssh_FX_CANNOT_DELETE = 22 + ssh_FX_INVALID_PARAMETER = 23 + ssh_FX_FILE_IS_A_DIRECTORY = 24 + ssh_FX_BYTE_RANGE_LOCK_CONFLICT = 25 + ssh_FX_BYTE_RANGE_LOCK_REFUSED = 26 + ssh_FX_DELETE_PENDING = 27 + ssh_FX_FILE_CORRUPT = 28 + ssh_FX_OWNER_INVALID = 29 + ssh_FX_GROUP_INVALID = 30 + ssh_FX_NO_MATCHING_BYTE_RANGE_LOCK = 31 ) const ( @@ -159,9 +185,9 @@ func unimplementedPacketErr(u uint8) error { return fmt.Errorf("sftp: unimplemented packet type: got %v", fxp(u)) } -type unexpectedIdErr struct{ want, got uint32 } +type unexpectedIDErr struct{ want, got uint32 } -func (u *unexpectedIdErr) Error() string { +func (u *unexpectedIDErr) Error() string { return fmt.Sprintf("sftp: unexpected id: want %v, got %v", u.want, u.got) } @@ -179,6 +205,8 @@ func (u *unexpectedVersionErr) Error() string { return fmt.Sprintf("sftp: unexpected server version: want %v, got %v", u.want, u.got) } +// A StatusError is returned when an SFTP operation fails, and provides +// additional information about the failure. type StatusError struct { Code uint32 msg, lang string diff --git a/Godeps/_workspace/src/github.com/pkg/sftp/wercker.yml b/Godeps/_workspace/src/github.com/pkg/sftp/wercker.yml deleted file mode 100644 index 41d2c5280..000000000 --- a/Godeps/_workspace/src/github.com/pkg/sftp/wercker.yml +++ /dev/null @@ -1 +0,0 @@ -box: wercker/golang From 4032af0b787858c2e7458f4755a9cbe0431e8d46 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 13 Feb 2016 19:11:41 +0100 Subject: [PATCH 2/2] sftp: Use os.IsNotExist() for Test() The sftp library introduced a change so that the error returned for (among others) Lstat() can be used with os.IsNotExist() to test whether the target file does not exist. --- backend/sftp/sftp.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/backend/sftp/sftp.go b/backend/sftp/sftp.go index fb10c3854..a4b3ac73a 100644 --- a/backend/sftp/sftp.go +++ b/backend/sftp/sftp.go @@ -360,11 +360,11 @@ func (r *SFTP) Stat(h backend.Handle) (backend.BlobInfo, error) { // Test returns true if a blob of the given type and name exists in the backend. func (r *SFTP) Test(t backend.Type, name string) (bool, error) { _, err := r.c.Lstat(r.filename(t, name)) - if err != nil { - if _, ok := err.(*sftp.StatusError); ok { - return false, nil - } + if os.IsNotExist(err) { + return false, nil + } + if err != nil { return false, err }