From 69a6e622d0b44f6f5845e296f39d000b88bb2394 Mon Sep 17 00:00:00 2001 From: Fabian Wickborn Date: Sun, 24 Sep 2017 20:04:23 +0200 Subject: [PATCH] Add REST backend option to use CA root certificate Closes #1114. --- cmd/restic/global.go | 33 +++++++++++++++++++-------- internal/backend/azure/azure.go | 12 +++++----- internal/backend/azure/azure_test.go | 13 ++++++++--- internal/backend/b2/b2.go | 13 ++++++----- internal/backend/b2/b2_test.go | 12 +++++++--- internal/backend/http_transport.go | 34 +++++++++++++++++++++++++--- internal/backend/rest/rest.go | 8 +++---- internal/backend/rest/rest_test.go | 10 ++++++-- internal/backend/s3/s3.go | 13 ++++++----- internal/backend/s3/s3_test.go | 26 +++++++++++++++------ internal/backend/swift/swift.go | 4 ++-- internal/backend/swift/swift_test.go | 12 +++++++--- 12 files changed, 135 insertions(+), 55 deletions(-) diff --git a/cmd/restic/global.go b/cmd/restic/global.go index f63052011..ac064814b 100644 --- a/cmd/restic/global.go +++ b/cmd/restic/global.go @@ -10,6 +10,7 @@ import ( "strings" "syscall" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/azure" "github.com/restic/restic/internal/backend/b2" "github.com/restic/restic/internal/backend/gs" @@ -41,6 +42,7 @@ type GlobalOptions struct { JSON bool CacheDir string NoCache bool + CACerts []string ctx context.Context password string @@ -73,6 +75,7 @@ func init() { f.BoolVarP(&globalOptions.JSON, "json", "", false, "set output mode to JSON for commands that support it") f.StringVar(&globalOptions.CacheDir, "cache-dir", "", "set the cache directory") f.BoolVar(&globalOptions.NoCache, "no-cache", false, "do not use a local cache") + f.StringSliceVar(&globalOptions.CACerts, "cacert", nil, "path to load root certificates from (default: use system certificates)") f.StringSliceVarP(&globalOptions.Options, "option", "o", []string{}, "set extended option (`key=value`, can be specified multiple times)") restoreTerminal() @@ -485,23 +488,28 @@ func open(s string, opts options.Options) (restic.Backend, error) { return nil, err } + rt, err := backend.Transport(globalOptions.CACerts) + if err != nil { + return nil, err + } + switch loc.Scheme { case "local": be, err = local.Open(cfg.(local.Config)) case "sftp": be, err = sftp.Open(cfg.(sftp.Config), SuspendSignalHandler, InstallSignalHandler) case "s3": - be, err = s3.Open(cfg.(s3.Config)) + be, err = s3.Open(cfg.(s3.Config), rt) case "gs": be, err = gs.Open(cfg.(gs.Config)) case "azure": - be, err = azure.Open(cfg.(azure.Config)) + be, err = azure.Open(cfg.(azure.Config), rt) case "swift": - be, err = swift.Open(cfg.(swift.Config)) + be, err = swift.Open(cfg.(swift.Config), rt) case "b2": - be, err = b2.Open(cfg.(b2.Config)) + be, err = b2.Open(cfg.(b2.Config), rt) case "rest": - be, err = rest.Open(cfg.(rest.Config)) + be, err = rest.Open(cfg.(rest.Config), rt) default: return nil, errors.Fatalf("invalid backend: %q", loc.Scheme) @@ -537,23 +545,28 @@ func create(s string, opts options.Options) (restic.Backend, error) { return nil, err } + rt, err := backend.Transport(globalOptions.CACerts) + if err != nil { + return nil, err + } + switch loc.Scheme { case "local": return local.Create(cfg.(local.Config)) case "sftp": return sftp.Create(cfg.(sftp.Config), SuspendSignalHandler, InstallSignalHandler) case "s3": - return s3.Create(cfg.(s3.Config)) + return s3.Create(cfg.(s3.Config), rt) case "gs": return gs.Create(cfg.(gs.Config)) case "azure": - return azure.Create(cfg.(azure.Config)) + return azure.Create(cfg.(azure.Config), rt) case "swift": - return swift.Open(cfg.(swift.Config)) + return swift.Open(cfg.(swift.Config), rt) case "b2": - return b2.Create(cfg.(b2.Config)) + return b2.Create(cfg.(b2.Config), rt) case "rest": - return rest.Create(cfg.(rest.Config)) + return rest.Create(cfg.(rest.Config), rt) } debug.Log("invalid repository scheme: %v", s) diff --git a/internal/backend/azure/azure.go b/internal/backend/azure/azure.go index 6fad216bf..44ed20048 100644 --- a/internal/backend/azure/azure.go +++ b/internal/backend/azure/azure.go @@ -30,7 +30,7 @@ const defaultListMaxItems = 5000 // make sure that *Backend implements backend.Backend var _ restic.Backend = &Backend{} -func open(cfg Config) (*Backend, error) { +func open(cfg Config, rt http.RoundTripper) (*Backend, error) { debug.Log("open, config %#v", cfg) client, err := storage.NewBasicClient(cfg.AccountName, cfg.AccountKey) @@ -38,7 +38,7 @@ func open(cfg Config) (*Backend, error) { return nil, errors.Wrap(err, "NewBasicClient") } - client.HTTPClient = &http.Client{Transport: backend.Transport()} + client.HTTPClient = &http.Client{Transport: rt} service := client.GetBlobService() @@ -63,14 +63,14 @@ func open(cfg Config) (*Backend, error) { } // Open opens the Azure backend at specified container. -func Open(cfg Config) (restic.Backend, error) { - return open(cfg) +func Open(cfg Config, rt http.RoundTripper) (restic.Backend, error) { + return open(cfg, rt) } // Create opens the Azure backend at specified container and creates the container if // it does not exist yet. -func Create(cfg Config) (restic.Backend, error) { - be, err := open(cfg) +func Create(cfg Config, rt http.RoundTripper) (restic.Backend, error) { + be, err := open(cfg, rt) if err != nil { return nil, errors.Wrap(err, "open") diff --git a/internal/backend/azure/azure_test.go b/internal/backend/azure/azure_test.go index 326b9152f..326e20183 100644 --- a/internal/backend/azure/azure_test.go +++ b/internal/backend/azure/azure_test.go @@ -7,6 +7,7 @@ import ( "testing" "time" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/azure" "github.com/restic/restic/internal/backend/test" "github.com/restic/restic/internal/errors" @@ -15,6 +16,11 @@ import ( ) func newAzureTestSuite(t testing.TB) *test.Suite { + tr, err := backend.Transport(nil) + if err != nil { + t.Fatalf("cannot create transport for tests: %v", err) + } + return &test.Suite{ // do not use excessive data MinimalData: true, @@ -37,7 +43,7 @@ func newAzureTestSuite(t testing.TB) *test.Suite { Create: func(config interface{}) (restic.Backend, error) { cfg := config.(azure.Config) - be, err := azure.Create(cfg) + be, err := azure.Create(cfg, tr) if err != nil { return nil, err } @@ -57,14 +63,15 @@ func newAzureTestSuite(t testing.TB) *test.Suite { // OpenFn is a function that opens a previously created temporary repository. Open: func(config interface{}) (restic.Backend, error) { cfg := config.(azure.Config) - return azure.Open(cfg) + + return azure.Open(cfg, tr) }, // CleanupFn removes data created during the tests. Cleanup: func(config interface{}) error { cfg := config.(azure.Config) - be, err := azure.Open(cfg) + be, err := azure.Open(cfg, tr) if err != nil { return err } diff --git a/internal/backend/b2/b2.go b/internal/backend/b2/b2.go index 8fd148769..4c68953b4 100644 --- a/internal/backend/b2/b2.go +++ b/internal/backend/b2/b2.go @@ -3,6 +3,7 @@ package b2 import ( "context" "io" + "net/http" "path" "strings" @@ -29,8 +30,8 @@ const defaultListMaxItems = 1000 // ensure statically that *b2Backend implements restic.Backend. var _ restic.Backend = &b2Backend{} -func newClient(ctx context.Context, cfg Config) (*b2.Client, error) { - opts := []b2.ClientOption{b2.Transport(backend.Transport())} +func newClient(ctx context.Context, cfg Config, rt http.RoundTripper) (*b2.Client, error) { + opts := []b2.ClientOption{b2.Transport(rt)} c, err := b2.NewClient(ctx, cfg.AccountID, cfg.Key, opts...) if err != nil { @@ -40,13 +41,13 @@ func newClient(ctx context.Context, cfg Config) (*b2.Client, error) { } // Open opens a connection to the B2 service. -func Open(cfg Config) (restic.Backend, error) { +func Open(cfg Config, rt http.RoundTripper) (restic.Backend, error) { debug.Log("cfg %#v", cfg) ctx, cancel := context.WithCancel(context.TODO()) defer cancel() - client, err := newClient(ctx, cfg) + client, err := newClient(ctx, cfg, rt) if err != nil { return nil, err } @@ -77,13 +78,13 @@ func Open(cfg Config) (restic.Backend, error) { // Create opens a connection to the B2 service. If the bucket does not exist yet, // it is created. -func Create(cfg Config) (restic.Backend, error) { +func Create(cfg Config, rt http.RoundTripper) (restic.Backend, error) { debug.Log("cfg %#v", cfg) ctx, cancel := context.WithCancel(context.TODO()) defer cancel() - client, err := newClient(ctx, cfg) + client, err := newClient(ctx, cfg, rt) if err != nil { return nil, err } diff --git a/internal/backend/b2/b2_test.go b/internal/backend/b2/b2_test.go index 9c5c57455..675a5c864 100644 --- a/internal/backend/b2/b2_test.go +++ b/internal/backend/b2/b2_test.go @@ -7,6 +7,7 @@ import ( "testing" "time" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/b2" "github.com/restic/restic/internal/backend/test" "github.com/restic/restic/internal/restic" @@ -15,6 +16,11 @@ import ( ) func newB2TestSuite(t testing.TB) *test.Suite { + tr, err := backend.Transport(nil) + if err != nil { + t.Fatalf("cannot create transport for tests: %v", err) + } + return &test.Suite{ // do not use excessive data MinimalData: true, @@ -39,19 +45,19 @@ func newB2TestSuite(t testing.TB) *test.Suite { // CreateFn is a function that creates a temporary repository for the tests. Create: func(config interface{}) (restic.Backend, error) { cfg := config.(b2.Config) - return b2.Create(cfg) + return b2.Create(cfg, tr) }, // OpenFn is a function that opens a previously created temporary repository. Open: func(config interface{}) (restic.Backend, error) { cfg := config.(b2.Config) - return b2.Open(cfg) + return b2.Open(cfg, tr) }, // CleanupFn removes data created during the tests. Cleanup: func(config interface{}) error { cfg := config.(b2.Config) - be, err := b2.Open(cfg) + be, err := b2.Open(cfg, tr) if err != nil { return err } diff --git a/internal/backend/http_transport.go b/internal/backend/http_transport.go index f3034f794..040c673d0 100644 --- a/internal/backend/http_transport.go +++ b/internal/backend/http_transport.go @@ -1,6 +1,10 @@ package backend import ( + "crypto/tls" + "crypto/x509" + "fmt" + "io/ioutil" "net" "net/http" "time" @@ -8,8 +12,10 @@ import ( "github.com/restic/restic/internal/debug" ) -// Transport returns a new http.RoundTripper with default settings applied. -func Transport() http.RoundTripper { +// Transport returns a new http.RoundTripper with default settings applied. If +// a custom rootCertFilename is non-empty, it must point to a valid PEM file, +// otherwise the function will return an error. +func Transport(rootCertFilenames []string) (http.RoundTripper, error) { // copied from net/http tr := &http.Transport{ Proxy: http.ProxyFromEnvironment, @@ -25,6 +31,28 @@ func Transport() http.RoundTripper { ExpectContinueTimeout: 1 * time.Second, } + if rootCertFilenames == nil { + return debug.RoundTripper(tr), nil + } + + p := x509.NewCertPool() + for _, filename := range rootCertFilenames { + if filename == "" { + return nil, fmt.Errorf("empty filename for root certificate supplied") + } + b, err := ioutil.ReadFile(filename) + if err != nil { + return nil, fmt.Errorf("unable to read root certificate: %v", err) + } + if ok := p.AppendCertsFromPEM(b); !ok { + return nil, fmt.Errorf("cannot parse root certificate from %q", filename) + } + } + + tr.TLSClientConfig = &tls.Config{ + RootCAs: p, + } + // wrap in the debug round tripper - return debug.RoundTripper(tr) + return debug.RoundTripper(tr), nil } diff --git a/internal/backend/rest/rest.go b/internal/backend/rest/rest.go index d0e0868af..c5d1140c2 100644 --- a/internal/backend/rest/rest.go +++ b/internal/backend/rest/rest.go @@ -31,8 +31,8 @@ type restBackend struct { } // Open opens the REST backend with the given config. -func Open(cfg Config) (restic.Backend, error) { - client := &http.Client{Transport: backend.Transport()} +func Open(cfg Config, rt http.RoundTripper) (restic.Backend, error) { + client := &http.Client{Transport: rt} sem, err := backend.NewSemaphore(cfg.Connections) if err != nil { @@ -56,8 +56,8 @@ func Open(cfg Config) (restic.Backend, error) { } // Create creates a new REST on server configured in config. -func Create(cfg Config) (restic.Backend, error) { - be, err := Open(cfg) +func Create(cfg Config, rt http.RoundTripper) (restic.Backend, error) { + be, err := Open(cfg, rt) if err != nil { return nil, err } diff --git a/internal/backend/rest/rest_test.go b/internal/backend/rest/rest_test.go index f6fbafbd6..b37c3663b 100644 --- a/internal/backend/rest/rest_test.go +++ b/internal/backend/rest/rest_test.go @@ -10,6 +10,7 @@ import ( "testing" "time" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/rest" "github.com/restic/restic/internal/backend/test" "github.com/restic/restic/internal/restic" @@ -61,6 +62,11 @@ func runRESTServer(ctx context.Context, t testing.TB, dir string) func() { } func newTestSuite(ctx context.Context, t testing.TB) *test.Suite { + tr, err := backend.Transport(nil) + if err != nil { + t.Fatalf("cannot create transport for tests: %v", err) + } + return &test.Suite{ // NewConfig returns a config for a new temporary backend that will be used in tests. NewConfig: func() (interface{}, error) { @@ -84,13 +90,13 @@ func newTestSuite(ctx context.Context, t testing.TB) *test.Suite { // CreateFn is a function that creates a temporary repository for the tests. Create: func(config interface{}) (restic.Backend, error) { cfg := config.(rest.Config) - return rest.Create(cfg) + return rest.Create(cfg, tr) }, // OpenFn is a function that opens a previously created temporary repository. Open: func(config interface{}) (restic.Backend, error) { cfg := config.(rest.Config) - return rest.Open(cfg) + return rest.Open(cfg, tr) }, // CleanupFn removes data created during the tests. diff --git a/internal/backend/s3/s3.go b/internal/backend/s3/s3.go index d3103acdc..416f290ee 100644 --- a/internal/backend/s3/s3.go +++ b/internal/backend/s3/s3.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "io" + "net/http" "os" "path" "strings" @@ -32,7 +33,7 @@ var _ restic.Backend = &Backend{} const defaultLayout = "default" -func open(cfg Config) (*Backend, error) { +func open(cfg Config, rt http.RoundTripper) (*Backend, error) { debug.Log("open, config %#v", cfg) if cfg.MaxRetries > 0 { @@ -70,7 +71,7 @@ func open(cfg Config) (*Backend, error) { cfg: cfg, } - client.SetCustomTransport(backend.Transport()) + client.SetCustomTransport(rt) l, err := backend.ParseLayout(be, cfg.Layout, defaultLayout, cfg.Prefix) if err != nil { @@ -84,14 +85,14 @@ func open(cfg Config) (*Backend, error) { // Open opens the S3 backend at bucket and region. The bucket is created if it // does not exist yet. -func Open(cfg Config) (restic.Backend, error) { - return open(cfg) +func Open(cfg Config, rt http.RoundTripper) (restic.Backend, error) { + return open(cfg, rt) } // Create opens the S3 backend at bucket and region and creates the bucket if // it does not exist yet. -func Create(cfg Config) (restic.Backend, error) { - be, err := open(cfg) +func Create(cfg Config, rt http.RoundTripper) (restic.Backend, error) { + be, err := open(cfg, rt) if err != nil { return nil, errors.Wrap(err, "open") } diff --git a/internal/backend/s3/s3_test.go b/internal/backend/s3/s3_test.go index 3cc20cc72..b9cbc4ef3 100644 --- a/internal/backend/s3/s3_test.go +++ b/internal/backend/s3/s3_test.go @@ -8,12 +8,14 @@ import ( "fmt" "io" "net" + "net/http" "os" "os/exec" "path/filepath" "testing" "time" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/s3" "github.com/restic/restic/internal/backend/test" "github.com/restic/restic/internal/restic" @@ -103,9 +105,9 @@ type MinioTestConfig struct { stopServer func() } -func createS3(t testing.TB, cfg MinioTestConfig) (be restic.Backend, err error) { +func createS3(t testing.TB, cfg MinioTestConfig, tr http.RoundTripper) (be restic.Backend, err error) { for i := 0; i < 10; i++ { - be, err = s3.Create(cfg.Config) + be, err = s3.Create(cfg.Config, tr) if err != nil { t.Logf("s3 open: try %d: error %v", i, err) time.Sleep(500 * time.Millisecond) @@ -119,6 +121,11 @@ func createS3(t testing.TB, cfg MinioTestConfig) (be restic.Backend, err error) } func newMinioTestSuite(ctx context.Context, t testing.TB) *test.Suite { + tr, err := backend.Transport(nil) + if err != nil { + t.Fatalf("cannot create transport for tests: %v", err) + } + return &test.Suite{ // NewConfig returns a config for a new temporary backend that will be used in tests. NewConfig: func() (interface{}, error) { @@ -142,7 +149,7 @@ func newMinioTestSuite(ctx context.Context, t testing.TB) *test.Suite { Create: func(config interface{}) (restic.Backend, error) { cfg := config.(MinioTestConfig) - be, err := createS3(t, cfg) + be, err := createS3(t, cfg, tr) if err != nil { return nil, err } @@ -162,7 +169,7 @@ func newMinioTestSuite(ctx context.Context, t testing.TB) *test.Suite { // OpenFn is a function that opens a previously created temporary repository. Open: func(config interface{}) (restic.Backend, error) { cfg := config.(MinioTestConfig) - return s3.Open(cfg.Config) + return s3.Open(cfg.Config, tr) }, // CleanupFn removes data created during the tests. @@ -214,6 +221,11 @@ func BenchmarkBackendMinio(t *testing.B) { } func newS3TestSuite(t testing.TB) *test.Suite { + tr, err := backend.Transport(nil) + if err != nil { + t.Fatalf("cannot create transport for tests: %v", err) + } + return &test.Suite{ // do not use excessive data MinimalData: true, @@ -236,7 +248,7 @@ func newS3TestSuite(t testing.TB) *test.Suite { Create: func(config interface{}) (restic.Backend, error) { cfg := config.(s3.Config) - be, err := s3.Create(cfg) + be, err := s3.Create(cfg, tr) if err != nil { return nil, err } @@ -256,14 +268,14 @@ func newS3TestSuite(t testing.TB) *test.Suite { // OpenFn is a function that opens a previously created temporary repository. Open: func(config interface{}) (restic.Backend, error) { cfg := config.(s3.Config) - return s3.Open(cfg) + return s3.Open(cfg, tr) }, // CleanupFn removes data created during the tests. Cleanup: func(config interface{}) error { cfg := config.(s3.Config) - be, err := s3.Open(cfg) + be, err := s3.Open(cfg, tr) if err != nil { return err } diff --git a/internal/backend/swift/swift.go b/internal/backend/swift/swift.go index d66b347ae..8c69caffa 100644 --- a/internal/backend/swift/swift.go +++ b/internal/backend/swift/swift.go @@ -34,7 +34,7 @@ var _ restic.Backend = &beSwift{} // Open opens the swift backend at a container in region. The container is // created if it does not exist yet. -func Open(cfg Config) (restic.Backend, error) { +func Open(cfg Config, rt http.RoundTripper) (restic.Backend, error) { debug.Log("config %#v", cfg) sem, err := backend.NewSemaphore(cfg.Connections) @@ -58,7 +58,7 @@ func Open(cfg Config) (restic.Backend, error) { ConnectTimeout: time.Minute, Timeout: time.Minute, - Transport: backend.Transport(), + Transport: rt, }, sem: sem, container: cfg.Container, diff --git a/internal/backend/swift/swift_test.go b/internal/backend/swift/swift_test.go index ce8693c33..f4cf28aaf 100644 --- a/internal/backend/swift/swift_test.go +++ b/internal/backend/swift/swift_test.go @@ -7,6 +7,7 @@ import ( "testing" "time" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/swift" "github.com/restic/restic/internal/backend/test" "github.com/restic/restic/internal/errors" @@ -15,6 +16,11 @@ import ( ) func newSwiftTestSuite(t testing.TB) *test.Suite { + tr, err := backend.Transport(nil) + if err != nil { + t.Fatalf("cannot create transport for tests: %v", err) + } + return &test.Suite{ // do not use excessive data MinimalData: true, @@ -55,7 +61,7 @@ func newSwiftTestSuite(t testing.TB) *test.Suite { Create: func(config interface{}) (restic.Backend, error) { cfg := config.(swift.Config) - be, err := swift.Open(cfg) + be, err := swift.Open(cfg, tr) if err != nil { return nil, err } @@ -75,14 +81,14 @@ func newSwiftTestSuite(t testing.TB) *test.Suite { // OpenFn is a function that opens a previously created temporary repository. Open: func(config interface{}) (restic.Backend, error) { cfg := config.(swift.Config) - return swift.Open(cfg) + return swift.Open(cfg, tr) }, // CleanupFn removes data created during the tests. Cleanup: func(config interface{}) error { cfg := config.(swift.Config) - be, err := swift.Open(cfg) + be, err := swift.Open(cfg, tr) if err != nil { return err }