From 669a66960330cd2d822b4c83f7d3f1a8ae536013 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 28 Apr 2024 11:48:26 +0200 Subject: [PATCH 1/3] sftp: Fix upload performance issue Since pkg/sftp 1.13.0 files were uploaded sequentially using 32kb chunks instead of sending 64 chunks in parallel. --- internal/backend/sftp/sftp.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/internal/backend/sftp/sftp.go b/internal/backend/sftp/sftp.go index 0a94e4aa3..389abab67 100644 --- a/internal/backend/sftp/sftp.go +++ b/internal/backend/sftp/sftp.go @@ -102,7 +102,10 @@ func startClient(cfg Config) (*SFTP, error) { }() // open the SFTP session - client, err := sftp.NewClientPipe(rd, wr) + client, err := sftp.NewClientPipe(rd, wr, + // write multiple packets (32kb) in parallel per file + // not strictly necessary as we use ReadFromWithConcurrency + sftp.UseConcurrentWrites(true)) if err != nil { return nil, errors.Errorf("unable to start the sftp session, error: %v", err) } @@ -359,7 +362,7 @@ func (r *SFTP) Save(_ context.Context, h backend.Handle, rd backend.RewindReader }() // save data, make sure to use the optimized sftp upload method - wbytes, err := f.ReadFrom(rd) + wbytes, err := f.ReadFromWithConcurrency(rd, 0) if err != nil { _ = f.Close() err = r.checkNoSpace(dirname, rd.Length(), err) From 935327d480507d4bf88d548555cb3e2ac07677a5 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 28 Apr 2024 11:50:09 +0200 Subject: [PATCH 2/3] sftp: slightly increase write concurrency This should increase upload throughput for high latency links a bit. --- internal/backend/sftp/sftp.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/internal/backend/sftp/sftp.go b/internal/backend/sftp/sftp.go index 389abab67..efd66f76c 100644 --- a/internal/backend/sftp/sftp.go +++ b/internal/backend/sftp/sftp.go @@ -105,7 +105,9 @@ func startClient(cfg Config) (*SFTP, error) { client, err := sftp.NewClientPipe(rd, wr, // write multiple packets (32kb) in parallel per file // not strictly necessary as we use ReadFromWithConcurrency - sftp.UseConcurrentWrites(true)) + sftp.UseConcurrentWrites(true), + // increase send buffer per file to 4MB + sftp.MaxConcurrentRequestsPerFile(128)) if err != nil { return nil, errors.Errorf("unable to start the sftp session, error: %v", err) } From a1d682ce0e31375e39478a374328987f7e256eea Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 28 Apr 2024 11:58:08 +0200 Subject: [PATCH 3/3] add changelog for sftp performance fix --- changelog/unreleased/issue-4209 | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 changelog/unreleased/issue-4209 diff --git a/changelog/unreleased/issue-4209 b/changelog/unreleased/issue-4209 new file mode 100644 index 000000000..2e49191c1 --- /dev/null +++ b/changelog/unreleased/issue-4209 @@ -0,0 +1,7 @@ +Bugfix: Fix slow sftp upload performance + +Since restic 0.12.1, the upload speed of the sftp backend to a remote server +has regressed significantly. This has been fixed. + +https://github.com/restic/restic/issues/4209 +https://github.com/restic/restic/pull/4782