From 41f70f1f4ff25901a0b7baae0298d0a6063e4ec3 Mon Sep 17 00:00:00 2001 From: Michael Manganiello Date: Tue, 17 Oct 2023 22:21:06 -0300 Subject: [PATCH 1/3] backend/sftp: Add sftp.args option Allow setting custom arguments for the `sftp` backend, by using the `sftp.args` option. This is similar to the approach already implemented in the `rclone` backend, to support new arguments without requiring future code changes for each different SSH argument. Closes #4241 --- changelog/unreleased/pull-4519 | 16 ++++++++++++++++ doc/030_preparing_a_new_repo.rst | 10 +++++++--- internal/backend/sftp/config.go | 3 ++- internal/backend/sftp/sftp.go | 19 +++++++++++++++---- internal/backend/sftp/sshcmd_test.go | 5 +++++ 5 files changed, 45 insertions(+), 8 deletions(-) create mode 100644 changelog/unreleased/pull-4519 diff --git a/changelog/unreleased/pull-4519 b/changelog/unreleased/pull-4519 new file mode 100644 index 000000000..c5982df1b --- /dev/null +++ b/changelog/unreleased/pull-4519 @@ -0,0 +1,16 @@ +Enhancement: Add config option to set SFTP command arguments + +The `sftp.args` option can be passed to restic (using `-o`) to specify +custom arguments to be used by the SSH command executed by the SFTP +backend. + +Before this change, a common scenario where a custom identity file was +needed for the SSH connection, required the full command to be +specified: +`-o sftp.command='ssh user@host:port -i /ssh/my_private_key -s sftp'` + +With this new configuration option: +`-o sftp.args='-i /ssh/my_private_key'` + +https://github.com/restic/restic/pull/4519 +https://github.com/restic/restic/issues/4241 diff --git a/doc/030_preparing_a_new_repo.rst b/doc/030_preparing_a_new_repo.rst index 04c189d07..3e7d9ff9e 100644 --- a/doc/030_preparing_a_new_repo.rst +++ b/doc/030_preparing_a_new_repo.rst @@ -119,10 +119,10 @@ user's home directory. Also, if the SFTP server is enforcing domain-confined users, you can specify the user this way: ``user@domain@host``. -.. note:: Please be aware that sftp servers do not expand the tilde character +.. note:: Please be aware that SFTP servers do not expand the tilde character (``~``) normally used as an alias for a user's home directory. If you want to specify a path relative to the user's home directory, pass a - relative path to the sftp backend. + relative path to the SFTP backend. If you need to specify a port number or IPv6 address, you'll need to use URL syntax. E.g., the repository ``/srv/restic-repo`` on ``[::1]`` (localhost) @@ -174,7 +174,11 @@ Last, if you'd like to use an entirely different program to create the SFTP connection, you can specify the command to be run with the option ``-o sftp.command="foobar"``. -.. note:: Please be aware that sftp servers close connections when no data is +The SFTP backend has the following additional option: + + * ``-o sftp.args`` allows setting the arguments passed to the default SSH command (ignored when ``sftp.command`` is set) + +.. note:: Please be aware that SFTP servers close connections when no data is received by the client. This can happen when restic is processing huge amounts of unchanged data. To avoid this issue add the following lines to the client's .ssh/config file: diff --git a/internal/backend/sftp/config.go b/internal/backend/sftp/config.go index ed7c2cafa..65af50d19 100644 --- a/internal/backend/sftp/config.go +++ b/internal/backend/sftp/config.go @@ -13,8 +13,9 @@ import ( type Config struct { User, Host, Port, Path string - Layout string `option:"layout" help:"use this backend directory layout (default: auto-detect)"` + Layout string `option:"layout" help:"use this backend directory layout (default: auto-detect)"` Command string `option:"command" help:"specify command to create sftp connection"` + Args string `option:"args" help:"specify arguments for ssh"` Connections uint `option:"connections" help:"set a limit for the number of concurrent connections (default: 5)"` } diff --git a/internal/backend/sftp/sftp.go b/internal/backend/sftp/sftp.go index 3e127ef05..735991eb4 100644 --- a/internal/backend/sftp/sftp.go +++ b/internal/backend/sftp/sftp.go @@ -213,6 +213,9 @@ func buildSSHCommand(cfg Config) (cmd string, args []string, err error) { if err != nil { return "", nil, err } + if cfg.Args != "" { + return "", nil, errors.New("cannot specify both sftp.command and sftp.args options") + } return args[0], args[1:], nil } @@ -226,11 +229,19 @@ func buildSSHCommand(cfg Config) (cmd string, args []string, err error) { args = append(args, "-p", port) } if cfg.User != "" { - args = append(args, "-l") - args = append(args, cfg.User) + args = append(args, "-l", cfg.User) } - args = append(args, "-s") - args = append(args, "sftp") + + if cfg.Args != "" { + a, err := backend.SplitShellStrings(cfg.Args) + if err != nil { + return "", nil, err + } + + args = append(args, a...) + } + + args = append(args, "-s", "sftp") return cmd, args, nil } diff --git a/internal/backend/sftp/sshcmd_test.go b/internal/backend/sftp/sshcmd_test.go index 822f28b5d..62738d2cc 100644 --- a/internal/backend/sftp/sshcmd_test.go +++ b/internal/backend/sftp/sshcmd_test.go @@ -30,6 +30,11 @@ var sshcmdTests = []struct { "ssh", []string{"host", "-p", "10022", "-l", "user", "-s", "sftp"}, }, + { + Config{User: "user", Host: "host", Port: "10022", Path: "/dir/subdir", Args: "-i /path/to/id_rsa"}, + "ssh", + []string{"host", "-p", "10022", "-l", "user", "-i", "/path/to/id_rsa", "-s", "sftp"}, + }, { // IPv6 address. Config{User: "user", Host: "::1", Path: "dir"}, From 6a4d6d5da4e5023a6761133a4c928a76f3d1af90 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 21 Oct 2023 19:26:39 +0200 Subject: [PATCH 2/3] sftp: test that Args and Command options cannot be set at the same time --- internal/backend/sftp/sshcmd_test.go | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/internal/backend/sftp/sshcmd_test.go b/internal/backend/sftp/sshcmd_test.go index 62738d2cc..601c94bd2 100644 --- a/internal/backend/sftp/sshcmd_test.go +++ b/internal/backend/sftp/sshcmd_test.go @@ -9,43 +9,57 @@ var sshcmdTests = []struct { cfg Config cmd string args []string + err string }{ { Config{User: "user", Host: "host", Path: "dir/subdir"}, "ssh", []string{"host", "-l", "user", "-s", "sftp"}, + "", }, { Config{Host: "host", Path: "dir/subdir"}, "ssh", []string{"host", "-s", "sftp"}, + "", }, { Config{Host: "host", Port: "10022", Path: "/dir/subdir"}, "ssh", []string{"host", "-p", "10022", "-s", "sftp"}, + "", }, { Config{User: "user", Host: "host", Port: "10022", Path: "/dir/subdir"}, "ssh", []string{"host", "-p", "10022", "-l", "user", "-s", "sftp"}, + "", }, { Config{User: "user", Host: "host", Port: "10022", Path: "/dir/subdir", Args: "-i /path/to/id_rsa"}, "ssh", []string{"host", "-p", "10022", "-l", "user", "-i", "/path/to/id_rsa", "-s", "sftp"}, + "", + }, + { + Config{Command: "ssh something", Args: "-i /path/to/id_rsa"}, + "", + nil, + "cannot specify both sftp.command and sftp.args options", }, { // IPv6 address. Config{User: "user", Host: "::1", Path: "dir"}, "ssh", []string{"::1", "-l", "user", "-s", "sftp"}, + "", }, { // IPv6 address with zone and port. Config{User: "user", Host: "::1%lo0", Port: "22", Path: "dir"}, "ssh", []string{"::1%lo0", "-p", "22", "-l", "user", "-s", "sftp"}, + "", }, } @@ -53,8 +67,14 @@ func TestBuildSSHCommand(t *testing.T) { for i, test := range sshcmdTests { t.Run("", func(t *testing.T) { cmd, args, err := buildSSHCommand(test.cfg) - if err != nil { - t.Fatalf("%v in test %d", err, i) + if test.err != "" { + if err.Error() != test.err { + t.Fatalf("expected error %v got %v", test.err, err.Error()) + } + } else { + if err != nil { + t.Fatalf("%v in test %d", err, i) + } } if cmd != test.cmd { From baf9b54891df5e79d765047b8c392d7aa5be113f Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 21 Oct 2023 19:28:27 +0200 Subject: [PATCH 3/3] sftp: simplify documentation for sftp.args --- doc/030_preparing_a_new_repo.rst | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/doc/030_preparing_a_new_repo.rst b/doc/030_preparing_a_new_repo.rst index 3e7d9ff9e..6197cad66 100644 --- a/doc/030_preparing_a_new_repo.rst +++ b/doc/030_preparing_a_new_repo.rst @@ -172,11 +172,9 @@ Then use it in the backend specification: Last, if you'd like to use an entirely different program to create the SFTP connection, you can specify the command to be run with the option -``-o sftp.command="foobar"``. - -The SFTP backend has the following additional option: - - * ``-o sftp.args`` allows setting the arguments passed to the default SSH command (ignored when ``sftp.command`` is set) +``-o sftp.command="foobar"``. Alternatively, ``-o sftp.args`` allows +setting the arguments passed to the default SSH command (ignored when +``sftp.command`` is set) .. note:: Please be aware that SFTP servers close connections when no data is received by the client. This can happen when restic is processing huge