Merge pull request #3840 from greatroar/sftp-init

Speed up restic init over slow SFTP links

changelog/unreleased/issue-3837

@@ -0,0 +1,8 @@
+Enhancement: SFTP backend initialization is faster on slow links
+
+Restic init on an SFTP backend now sends multiple mkdir commands to the
+backend concurrently, to reduce the wait when creating a repository
+over a very slow link.
+
+https://github.com/restic/restic/issues/3837
+https://github.com/restic/restic/pull/3840

go.mod

@@ -38,7 +38,7 @@ require (
 	golang.org/x/crypto v0.0.0-20220321153916-2c7772ba3064
 	golang.org/x/net v0.0.0-20220325170049-de3da57026de
 	golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a
-	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
+	golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4
 	golang.org/x/sys v0.0.0-20220325203850-36772127a21f
 	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211
 	golang.org/x/text v0.3.7

go.sum

@@ -439,8 +439,9 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

internal/backend/sftp/sftp.go

@@ -21,6 +21,7 @@ import (
 
 	"github.com/cenkalti/backoff/v4"
 	"github.com/pkg/sftp"
+	"golang.org/x/sync/errgroup"
 )
 
 // SFTP is a backend in a directory accessed via SFTP.
@@ -116,8 +117,7 @@ func (r *SFTP) clientError() error {
 }
 
 // Open opens an sftp backend as described by the config by running
-// "ssh" with the appropriate arguments (or cfg.Command, if set). The function
+// "ssh" with the appropriate arguments (or cfg.Command, if set).
-// preExec is run just before, postExec just after starting a program.
 func Open(ctx context.Context, cfg Config) (*SFTP, error) {
 	debug.Log("open backend with config %#v", cfg)
 
@@ -155,15 +155,29 @@ func Open(ctx context.Context, cfg Config) (*SFTP, error) {
 	return sftp, nil
 }
 
-func (r *SFTP) mkdirAllDataSubdirs() error {
+func (r *SFTP) mkdirAllDataSubdirs(ctx context.Context, nconn uint) error {
+	// Run multiple MkdirAll calls concurrently. These involve multiple
+	// round-trips and we do a lot of them, so this whole operation can be slow
+	// on high-latency links.
+	g, _ := errgroup.WithContext(ctx)
+	// Use errgroup's built-in semaphore, because r.sem is not initialized yet.
+	g.SetLimit(int(nconn))
+
 	for _, d := range r.Paths() {
-		err := r.c.MkdirAll(d)
+		d := d
-		if err != nil {
+		g.Go(func() error {
-			return err
+			// First try Mkdir. For most directories in Paths, this takes one
-		}
+			// round trip, not counting duplicate parent creations causes by
+			// concurrency. MkdirAll first does Stat, then recursive MkdirAll
+			// on the parent, so calls typically take three round trips.
+			if err := r.c.Mkdir(d); err == nil {
+				return nil
+			}
+			return r.c.MkdirAll(d)
+		})
 	}
 
-	return nil
+	return g.Wait()
 }
 
 // Join combines path components with slashes (according to the sftp spec).
@@ -214,8 +228,7 @@ func buildSSHCommand(cfg Config) (cmd string, args []string, err error) {
 }
 
 // Create creates an sftp backend as described by the config by running "ssh"
-// with the appropriate arguments (or cfg.Command, if set). The function
+// with the appropriate arguments (or cfg.Command, if set).
-// preExec is run just before, postExec just after starting a program.
 func Create(ctx context.Context, cfg Config) (*SFTP, error) {
 	cmd, args, err := buildSSHCommand(cfg)
 	if err != nil {
@@ -242,7 +255,7 @@ func Create(ctx context.Context, cfg Config) (*SFTP, error) {
 	}
 
 	// create paths for data and refs
-	if err = sftp.mkdirAllDataSubdirs(); err != nil {
+	if err = sftp.mkdirAllDataSubdirs(ctx, cfg.Connections); err != nil {
 		return nil, err
 	}