Merge pull request #1276 from fawick/supply_ca_cert

Add REST backend option to use CA root certificate

cmd/restic/global.go

@@ -10,6 +10,7 @@ import (
 	"strings"
 	"syscall"
 
+	"github.com/restic/restic/internal/backend"
 	"github.com/restic/restic/internal/backend/azure"
 	"github.com/restic/restic/internal/backend/b2"
 	"github.com/restic/restic/internal/backend/gs"
@@ -41,6 +42,7 @@ type GlobalOptions struct {
 	JSON         bool
 	CacheDir     string
 	NoCache      bool
+	CACerts      []string
 
 	ctx      context.Context
 	password string
@@ -73,6 +75,7 @@ func init() {
 	f.BoolVarP(&globalOptions.JSON, "json", "", false, "set output mode to JSON for commands that support it")
 	f.StringVar(&globalOptions.CacheDir, "cache-dir", "", "set the cache directory")
 	f.BoolVar(&globalOptions.NoCache, "no-cache", false, "do not use a local cache")
+	f.StringSliceVar(&globalOptions.CACerts, "cacert", nil, "path to load root certificates from (default: use system certificates)")
 	f.StringSliceVarP(&globalOptions.Options, "option", "o", []string{}, "set extended option (`key=value`, can be specified multiple times)")
 
 	restoreTerminal()
@@ -485,23 +488,28 @@ func open(s string, opts options.Options) (restic.Backend, error) {
 		return nil, err
 	}
 
+	rt, err := backend.Transport(globalOptions.CACerts)
+	if err != nil {
+		return nil, err
+	}
+
 	switch loc.Scheme {
 	case "local":
 		be, err = local.Open(cfg.(local.Config))
 	case "sftp":
 		be, err = sftp.Open(cfg.(sftp.Config), SuspendSignalHandler, InstallSignalHandler)
 	case "s3":
-		be, err = s3.Open(cfg.(s3.Config))
+		be, err = s3.Open(cfg.(s3.Config), rt)
 	case "gs":
 		be, err = gs.Open(cfg.(gs.Config))
 	case "azure":
-		be, err = azure.Open(cfg.(azure.Config))
+		be, err = azure.Open(cfg.(azure.Config), rt)
 	case "swift":
-		be, err = swift.Open(cfg.(swift.Config))
+		be, err = swift.Open(cfg.(swift.Config), rt)
 	case "b2":
-		be, err = b2.Open(cfg.(b2.Config))
+		be, err = b2.Open(cfg.(b2.Config), rt)
 	case "rest":
-		be, err = rest.Open(cfg.(rest.Config))
+		be, err = rest.Open(cfg.(rest.Config), rt)
 
 	default:
 		return nil, errors.Fatalf("invalid backend: %q", loc.Scheme)
@@ -537,23 +545,28 @@ func create(s string, opts options.Options) (restic.Backend, error) {
 		return nil, err
 	}
 
+	rt, err := backend.Transport(globalOptions.CACerts)
+	if err != nil {
+		return nil, err
+	}
+
 	switch loc.Scheme {
 	case "local":
 		return local.Create(cfg.(local.Config))
 	case "sftp":
 		return sftp.Create(cfg.(sftp.Config), SuspendSignalHandler, InstallSignalHandler)
 	case "s3":
-		return s3.Create(cfg.(s3.Config))
+		return s3.Create(cfg.(s3.Config), rt)
 	case "gs":
 		return gs.Create(cfg.(gs.Config))
 	case "azure":
-		return azure.Create(cfg.(azure.Config))
+		return azure.Create(cfg.(azure.Config), rt)
 	case "swift":
-		return swift.Open(cfg.(swift.Config))
+		return swift.Open(cfg.(swift.Config), rt)
 	case "b2":
-		return b2.Create(cfg.(b2.Config))
+		return b2.Create(cfg.(b2.Config), rt)
 	case "rest":
-		return rest.Create(cfg.(rest.Config))
+		return rest.Create(cfg.(rest.Config), rt)
 	}
 
 	debug.Log("invalid repository scheme: %v", s)

internal/backend/azure/azure.go

@@ -30,7 +30,7 @@ const defaultListMaxItems = 5000
 // make sure that *Backend implements backend.Backend
 var _ restic.Backend = &Backend{}
 
-func open(cfg Config) (*Backend, error) {
+func open(cfg Config, rt http.RoundTripper) (*Backend, error) {
 	debug.Log("open, config %#v", cfg)
 
 	client, err := storage.NewBasicClient(cfg.AccountName, cfg.AccountKey)
@@ -38,7 +38,7 @@ func open(cfg Config) (*Backend, error) {
 		return nil, errors.Wrap(err, "NewBasicClient")
 	}
 
-	client.HTTPClient = &http.Client{Transport: backend.Transport()}
+	client.HTTPClient = &http.Client{Transport: rt}
 
 	service := client.GetBlobService()
 
@@ -63,14 +63,14 @@ func open(cfg Config) (*Backend, error) {
 }
 
 // Open opens the Azure backend at specified container.
-func Open(cfg Config) (restic.Backend, error) {
+func Open(cfg Config, rt http.RoundTripper) (restic.Backend, error) {
-	return open(cfg)
+	return open(cfg, rt)
 }
 
 // Create opens the Azure backend at specified container and creates the container if
 // it does not exist yet.
-func Create(cfg Config) (restic.Backend, error) {
+func Create(cfg Config, rt http.RoundTripper) (restic.Backend, error) {
-	be, err := open(cfg)
+	be, err := open(cfg, rt)
 
 	if err != nil {
 		return nil, errors.Wrap(err, "open")

internal/backend/azure/azure_test.go

@@ -7,6 +7,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/restic/restic/internal/backend"
 	"github.com/restic/restic/internal/backend/azure"
 	"github.com/restic/restic/internal/backend/test"
 	"github.com/restic/restic/internal/errors"
@@ -15,6 +16,11 @@ import (
 )
 
 func newAzureTestSuite(t testing.TB) *test.Suite {
+	tr, err := backend.Transport(nil)
+	if err != nil {
+		t.Fatalf("cannot create transport for tests: %v", err)
+	}
+
 	return &test.Suite{
 		// do not use excessive data
 		MinimalData: true,
@@ -37,7 +43,7 @@ func newAzureTestSuite(t testing.TB) *test.Suite {
 		Create: func(config interface{}) (restic.Backend, error) {
 			cfg := config.(azure.Config)
 
-			be, err := azure.Create(cfg)
+			be, err := azure.Create(cfg, tr)
 			if err != nil {
 				return nil, err
 			}
@@ -57,14 +63,15 @@ func newAzureTestSuite(t testing.TB) *test.Suite {
 		// OpenFn is a function that opens a previously created temporary repository.
 		Open: func(config interface{}) (restic.Backend, error) {
 			cfg := config.(azure.Config)
-			return azure.Open(cfg)
+
+			return azure.Open(cfg, tr)
 		},
 
 		// CleanupFn removes data created during the tests.
 		Cleanup: func(config interface{}) error {
 			cfg := config.(azure.Config)
 
-			be, err := azure.Open(cfg)
+			be, err := azure.Open(cfg, tr)
 			if err != nil {
 				return err
 			}

internal/backend/b2/b2.go

@@ -3,6 +3,7 @@ package b2
 import (
 	"context"
 	"io"
+	"net/http"
 	"path"
 	"strings"
 
@@ -29,8 +30,8 @@ const defaultListMaxItems = 1000
 // ensure statically that *b2Backend implements restic.Backend.
 var _ restic.Backend = &b2Backend{}
 
-func newClient(ctx context.Context, cfg Config) (*b2.Client, error) {
+func newClient(ctx context.Context, cfg Config, rt http.RoundTripper) (*b2.Client, error) {
-	opts := []b2.ClientOption{b2.Transport(backend.Transport())}
+	opts := []b2.ClientOption{b2.Transport(rt)}
 
 	c, err := b2.NewClient(ctx, cfg.AccountID, cfg.Key, opts...)
 	if err != nil {
@@ -40,13 +41,13 @@ func newClient(ctx context.Context, cfg Config) (*b2.Client, error) {
 }
 
 // Open opens a connection to the B2 service.
-func Open(cfg Config) (restic.Backend, error) {
+func Open(cfg Config, rt http.RoundTripper) (restic.Backend, error) {
 	debug.Log("cfg %#v", cfg)
 
 	ctx, cancel := context.WithCancel(context.TODO())
 	defer cancel()
 
-	client, err := newClient(ctx, cfg)
+	client, err := newClient(ctx, cfg, rt)
 	if err != nil {
 		return nil, err
 	}
@@ -77,13 +78,13 @@ func Open(cfg Config) (restic.Backend, error) {
 
 // Create opens a connection to the B2 service. If the bucket does not exist yet,
 // it is created.
-func Create(cfg Config) (restic.Backend, error) {
+func Create(cfg Config, rt http.RoundTripper) (restic.Backend, error) {
 	debug.Log("cfg %#v", cfg)
 
 	ctx, cancel := context.WithCancel(context.TODO())
 	defer cancel()
 
-	client, err := newClient(ctx, cfg)
+	client, err := newClient(ctx, cfg, rt)
 	if err != nil {
 		return nil, err
 	}

internal/backend/b2/b2_test.go

@@ -7,6 +7,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/restic/restic/internal/backend"
 	"github.com/restic/restic/internal/backend/b2"
 	"github.com/restic/restic/internal/backend/test"
 	"github.com/restic/restic/internal/restic"
@@ -15,6 +16,11 @@ import (
 )
 
 func newB2TestSuite(t testing.TB) *test.Suite {
+	tr, err := backend.Transport(nil)
+	if err != nil {
+		t.Fatalf("cannot create transport for tests: %v", err)
+	}
+
 	return &test.Suite{
 		// do not use excessive data
 		MinimalData: true,
@@ -39,19 +45,19 @@ func newB2TestSuite(t testing.TB) *test.Suite {
 		// CreateFn is a function that creates a temporary repository for the tests.
 		Create: func(config interface{}) (restic.Backend, error) {
 			cfg := config.(b2.Config)
-			return b2.Create(cfg)
+			return b2.Create(cfg, tr)
 		},
 
 		// OpenFn is a function that opens a previously created temporary repository.
 		Open: func(config interface{}) (restic.Backend, error) {
 			cfg := config.(b2.Config)
-			return b2.Open(cfg)
+			return b2.Open(cfg, tr)
 		},
 
 		// CleanupFn removes data created during the tests.
 		Cleanup: func(config interface{}) error {
 			cfg := config.(b2.Config)
-			be, err := b2.Open(cfg)
+			be, err := b2.Open(cfg, tr)
 			if err != nil {
 				return err
 			}

internal/backend/http_transport.go

@@ -1,6 +1,10 @@
 package backend
 
 import (
+	"crypto/tls"
+	"crypto/x509"
+	"fmt"
+	"io/ioutil"
 	"net"
 	"net/http"
 	"time"
@@ -8,8 +12,10 @@ import (
 	"github.com/restic/restic/internal/debug"
 )
 
-// Transport returns a new http.RoundTripper with default settings applied.
+// Transport returns a new http.RoundTripper with default settings applied. If
-func Transport() http.RoundTripper {
+// a custom rootCertFilename is non-empty, it must point to a valid PEM file,
+// otherwise the function will return an error.
+func Transport(rootCertFilenames []string) (http.RoundTripper, error) {
 	// copied from net/http
 	tr := &http.Transport{
 		Proxy: http.ProxyFromEnvironment,
@@ -25,6 +31,28 @@ func Transport() http.RoundTripper {
 		ExpectContinueTimeout: 1 * time.Second,
 	}
 
+	if rootCertFilenames == nil {
+		return debug.RoundTripper(tr), nil
+	}
+
+	p := x509.NewCertPool()
+	for _, filename := range rootCertFilenames {
+		if filename == "" {
+			return nil, fmt.Errorf("empty filename for root certificate supplied")
+		}
+		b, err := ioutil.ReadFile(filename)
+		if err != nil {
+			return nil, fmt.Errorf("unable to read root certificate: %v", err)
+		}
+		if ok := p.AppendCertsFromPEM(b); !ok {
+			return nil, fmt.Errorf("cannot parse root certificate from %q", filename)
+		}
+	}
+
+	tr.TLSClientConfig = &tls.Config{
+		RootCAs: p,
+	}
+
 	// wrap in the debug round tripper
-	return debug.RoundTripper(tr)
+	return debug.RoundTripper(tr), nil
 }

internal/backend/rest/rest.go

@@ -31,8 +31,8 @@ type restBackend struct {
 }
 
 // Open opens the REST backend with the given config.
-func Open(cfg Config) (restic.Backend, error) {
+func Open(cfg Config, rt http.RoundTripper) (restic.Backend, error) {
-	client := &http.Client{Transport: backend.Transport()}
+	client := &http.Client{Transport: rt}
 
 	sem, err := backend.NewSemaphore(cfg.Connections)
 	if err != nil {
@@ -56,8 +56,8 @@ func Open(cfg Config) (restic.Backend, error) {
 }
 
 // Create creates a new REST on server configured in config.
-func Create(cfg Config) (restic.Backend, error) {
+func Create(cfg Config, rt http.RoundTripper) (restic.Backend, error) {
-	be, err := Open(cfg)
+	be, err := Open(cfg, rt)
 	if err != nil {
 		return nil, err
 	}

internal/backend/rest/rest_test.go

@@ -10,6 +10,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/restic/restic/internal/backend"
 	"github.com/restic/restic/internal/backend/rest"
 	"github.com/restic/restic/internal/backend/test"
 	"github.com/restic/restic/internal/restic"
@@ -61,6 +62,11 @@ func runRESTServer(ctx context.Context, t testing.TB, dir string) func() {
 }
 
 func newTestSuite(ctx context.Context, t testing.TB) *test.Suite {
+	tr, err := backend.Transport(nil)
+	if err != nil {
+		t.Fatalf("cannot create transport for tests: %v", err)
+	}
+
 	return &test.Suite{
 		// NewConfig returns a config for a new temporary backend that will be used in tests.
 		NewConfig: func() (interface{}, error) {
@@ -84,13 +90,13 @@ func newTestSuite(ctx context.Context, t testing.TB) *test.Suite {
 		// CreateFn is a function that creates a temporary repository for the tests.
 		Create: func(config interface{}) (restic.Backend, error) {
 			cfg := config.(rest.Config)
-			return rest.Create(cfg)
+			return rest.Create(cfg, tr)
 		},
 
 		// OpenFn is a function that opens a previously created temporary repository.
 		Open: func(config interface{}) (restic.Backend, error) {
 			cfg := config.(rest.Config)
-			return rest.Open(cfg)
+			return rest.Open(cfg, tr)
 		},
 
 		// CleanupFn removes data created during the tests.

internal/backend/s3/s3.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"net/http"
 	"os"
 	"path"
 	"strings"
@@ -32,7 +33,7 @@ var _ restic.Backend = &Backend{}
 
 const defaultLayout = "default"
 
-func open(cfg Config) (*Backend, error) {
+func open(cfg Config, rt http.RoundTripper) (*Backend, error) {
 	debug.Log("open, config %#v", cfg)
 
 	if cfg.MaxRetries > 0 {
@@ -70,7 +71,7 @@ func open(cfg Config) (*Backend, error) {
 		cfg:    cfg,
 	}
 
-	client.SetCustomTransport(backend.Transport())
+	client.SetCustomTransport(rt)
 
 	l, err := backend.ParseLayout(be, cfg.Layout, defaultLayout, cfg.Prefix)
 	if err != nil {
@@ -84,14 +85,14 @@ func open(cfg Config) (*Backend, error) {
 
 // Open opens the S3 backend at bucket and region. The bucket is created if it
 // does not exist yet.
-func Open(cfg Config) (restic.Backend, error) {
+func Open(cfg Config, rt http.RoundTripper) (restic.Backend, error) {
-	return open(cfg)
+	return open(cfg, rt)
 }
 
 // Create opens the S3 backend at bucket and region and creates the bucket if
 // it does not exist yet.
-func Create(cfg Config) (restic.Backend, error) {
+func Create(cfg Config, rt http.RoundTripper) (restic.Backend, error) {
-	be, err := open(cfg)
+	be, err := open(cfg, rt)
 	if err != nil {
 		return nil, errors.Wrap(err, "open")
 	}

internal/backend/s3/s3_test.go

@@ -8,12 +8,14 @@ import (
 	"fmt"
 	"io"
 	"net"
+	"net/http"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"testing"
 	"time"
 
+	"github.com/restic/restic/internal/backend"
 	"github.com/restic/restic/internal/backend/s3"
 	"github.com/restic/restic/internal/backend/test"
 	"github.com/restic/restic/internal/restic"
@@ -103,9 +105,9 @@ type MinioTestConfig struct {
 	stopServer    func()
 }
 
-func createS3(t testing.TB, cfg MinioTestConfig) (be restic.Backend, err error) {
+func createS3(t testing.TB, cfg MinioTestConfig, tr http.RoundTripper) (be restic.Backend, err error) {
 	for i := 0; i < 10; i++ {
-		be, err = s3.Create(cfg.Config)
+		be, err = s3.Create(cfg.Config, tr)
 		if err != nil {
 			t.Logf("s3 open: try %d: error %v", i, err)
 			time.Sleep(500 * time.Millisecond)
@@ -119,6 +121,11 @@ func createS3(t testing.TB, cfg MinioTestConfig) (be restic.Backend, err error)
 }
 
 func newMinioTestSuite(ctx context.Context, t testing.TB) *test.Suite {
+	tr, err := backend.Transport(nil)
+	if err != nil {
+		t.Fatalf("cannot create transport for tests: %v", err)
+	}
+
 	return &test.Suite{
 		// NewConfig returns a config for a new temporary backend that will be used in tests.
 		NewConfig: func() (interface{}, error) {
@@ -142,7 +149,7 @@ func newMinioTestSuite(ctx context.Context, t testing.TB) *test.Suite {
 		Create: func(config interface{}) (restic.Backend, error) {
 			cfg := config.(MinioTestConfig)
 
-			be, err := createS3(t, cfg)
+			be, err := createS3(t, cfg, tr)
 			if err != nil {
 				return nil, err
 			}
@@ -162,7 +169,7 @@ func newMinioTestSuite(ctx context.Context, t testing.TB) *test.Suite {
 		// OpenFn is a function that opens a previously created temporary repository.
 		Open: func(config interface{}) (restic.Backend, error) {
 			cfg := config.(MinioTestConfig)
-			return s3.Open(cfg.Config)
+			return s3.Open(cfg.Config, tr)
 		},
 
 		// CleanupFn removes data created during the tests.
@@ -214,6 +221,11 @@ func BenchmarkBackendMinio(t *testing.B) {
 }
 
 func newS3TestSuite(t testing.TB) *test.Suite {
+	tr, err := backend.Transport(nil)
+	if err != nil {
+		t.Fatalf("cannot create transport for tests: %v", err)
+	}
+
 	return &test.Suite{
 		// do not use excessive data
 		MinimalData: true,
@@ -236,7 +248,7 @@ func newS3TestSuite(t testing.TB) *test.Suite {
 		Create: func(config interface{}) (restic.Backend, error) {
 			cfg := config.(s3.Config)
 
-			be, err := s3.Create(cfg)
+			be, err := s3.Create(cfg, tr)
 			if err != nil {
 				return nil, err
 			}
@@ -256,14 +268,14 @@ func newS3TestSuite(t testing.TB) *test.Suite {
 		// OpenFn is a function that opens a previously created temporary repository.
 		Open: func(config interface{}) (restic.Backend, error) {
 			cfg := config.(s3.Config)
-			return s3.Open(cfg)
+			return s3.Open(cfg, tr)
 		},
 
 		// CleanupFn removes data created during the tests.
 		Cleanup: func(config interface{}) error {
 			cfg := config.(s3.Config)
 
-			be, err := s3.Open(cfg)
+			be, err := s3.Open(cfg, tr)
 			if err != nil {
 				return err
 			}

internal/backend/swift/swift.go

@@ -34,7 +34,7 @@ var _ restic.Backend = &beSwift{}
 
 // Open opens the swift backend at a container in region. The container is
 // created if it does not exist yet.
-func Open(cfg Config) (restic.Backend, error) {
+func Open(cfg Config, rt http.RoundTripper) (restic.Backend, error) {
 	debug.Log("config %#v", cfg)
 
 	sem, err := backend.NewSemaphore(cfg.Connections)
@@ -58,7 +58,7 @@ func Open(cfg Config) (restic.Backend, error) {
 			ConnectTimeout: time.Minute,
 			Timeout:        time.Minute,
 
-			Transport: backend.Transport(),
+			Transport: rt,
 		},
 		sem:       sem,
 		container: cfg.Container,

internal/backend/swift/swift_test.go

@@ -7,6 +7,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/restic/restic/internal/backend"
 	"github.com/restic/restic/internal/backend/swift"
 	"github.com/restic/restic/internal/backend/test"
 	"github.com/restic/restic/internal/errors"
@@ -15,6 +16,11 @@ import (
 )
 
 func newSwiftTestSuite(t testing.TB) *test.Suite {
+	tr, err := backend.Transport(nil)
+	if err != nil {
+		t.Fatalf("cannot create transport for tests: %v", err)
+	}
+
 	return &test.Suite{
 		// do not use excessive data
 		MinimalData: true,
@@ -55,7 +61,7 @@ func newSwiftTestSuite(t testing.TB) *test.Suite {
 		Create: func(config interface{}) (restic.Backend, error) {
 			cfg := config.(swift.Config)
 
-			be, err := swift.Open(cfg)
+			be, err := swift.Open(cfg, tr)
 			if err != nil {
 				return nil, err
 			}
@@ -75,14 +81,14 @@ func newSwiftTestSuite(t testing.TB) *test.Suite {
 		// OpenFn is a function that opens a previously created temporary repository.
 		Open: func(config interface{}) (restic.Backend, error) {
 			cfg := config.(swift.Config)
-			return swift.Open(cfg)
+			return swift.Open(cfg, tr)
 		},
 
 		// CleanupFn removes data created during the tests.
 		Cleanup: func(config interface{}) error {
 			cfg := config.(swift.Config)
 
-			be, err := swift.Open(cfg)
+			be, err := swift.Open(cfg, tr)
 			if err != nil {
 				return err
 			}