octoleo
/
restic
mirror of https://github.com/octoleo/restic.git
2
2
Fork 0
Code Issues Releases Wiki Activity

Add REST backend option to use CA root certificate 

Closes #1114.
This commit is contained in:
Fabian Wickborn 2017-09-24 20:04:23 +02:00
parent 1dcfd64028
commit 69a6e622d0
12 changed files with 135 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
cmd/restic/global.go
View File

@ -10,6 +10,7 @@ import (
"strings"
"syscall"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/azure"
"github.com/restic/restic/internal/backend/b2"
"github.com/restic/restic/internal/backend/gs"
@ -41,6 +42,7 @@ type GlobalOptions struct {
JSON bool
CacheDir string
NoCache bool
CACerts []string
ctx context.Context
password string
@ -73,6 +75,7 @@ func init() {
f.BoolVarP(&globalOptions.JSON, "json", "", false, "set output mode to JSON for commands that support it")
f.StringVar(&globalOptions.CacheDir, "cache-dir", "", "set the cache directory")
f.BoolVar(&globalOptions.NoCache, "no-cache", false, "do not use a local cache")
f.StringSliceVar(&globalOptions.CACerts, "cacert", nil, "path to load root certificates from (default: use system certificates)")
f.StringSliceVarP(&globalOptions.Options, "option", "o", []string{}, "set extended option (`key=value`, can be specified multiple times)")
restoreTerminal()
@ -485,23 +488,28 @@ func open(s string, opts options.Options) (restic.Backend, error) {
return nil, err
}
rt, err := backend.Transport(globalOptions.CACerts)
if err != nil {
return nil, err
}
switch loc.Scheme {
case "local":
be, err = local.Open(cfg.(local.Config))
case "sftp":
be, err = sftp.Open(cfg.(sftp.Config), SuspendSignalHandler, InstallSignalHandler)
case "s3":
be, err = s3.Open(cfg.(s3.Config))
be, err = s3.Open(cfg.(s3.Config), rt)
case "gs":
be, err = gs.Open(cfg.(gs.Config))
case "azure":
be, err = azure.Open(cfg.(azure.Config))
be, err = azure.Open(cfg.(azure.Config), rt)
case "swift":
be, err = swift.Open(cfg.(swift.Config))
be, err = swift.Open(cfg.(swift.Config), rt)
case "b2":
be, err = b2.Open(cfg.(b2.Config))
be, err = b2.Open(cfg.(b2.Config), rt)
case "rest":
be, err = rest.Open(cfg.(rest.Config))
be, err = rest.Open(cfg.(rest.Config), rt)
default:
return nil, errors.Fatalf("invalid backend: %q", loc.Scheme)
@ -537,23 +545,28 @@ func create(s string, opts options.Options) (restic.Backend, error) {
return nil, err
}
rt, err := backend.Transport(globalOptions.CACerts)
if err != nil {
return nil, err
}
switch loc.Scheme {
case "local":
return local.Create(cfg.(local.Config))
case "sftp":
return sftp.Create(cfg.(sftp.Config), SuspendSignalHandler, InstallSignalHandler)
case "s3":
return s3.Create(cfg.(s3.Config))
return s3.Create(cfg.(s3.Config), rt)
case "gs":
return gs.Create(cfg.(gs.Config))
case "azure":
return azure.Create(cfg.(azure.Config))
return azure.Create(cfg.(azure.Config), rt)
case "swift":
return swift.Open(cfg.(swift.Config))
return swift.Open(cfg.(swift.Config), rt)
case "b2":
return b2.Create(cfg.(b2.Config))
return b2.Create(cfg.(b2.Config), rt)
case "rest":
return rest.Create(cfg.(rest.Config))
return rest.Create(cfg.(rest.Config), rt)
}
debug.Log("invalid repository scheme: %v", s)

12
internal/backend/azure/azure.go
View File

@ -30,7 +30,7 @@ const defaultListMaxItems = 5000
// make sure that *Backend implements backend.Backend
var _ restic.Backend = &Backend{}
func open(cfg Config) (*Backend, error) {
func open(cfg Config, rt http.RoundTripper) (*Backend, error) {
debug.Log("open, config %#v", cfg)
client, err := storage.NewBasicClient(cfg.AccountName, cfg.AccountKey)
@ -38,7 +38,7 @@ func open(cfg Config) (*Backend, error) {
return nil, errors.Wrap(err, "NewBasicClient")
}
client.HTTPClient = &http.Client{Transport: backend.Transport()}
client.HTTPClient = &http.Client{Transport: rt}
service := client.GetBlobService()
@ -63,14 +63,14 @@ func open(cfg Config) (*Backend, error) {
}
// Open opens the Azure backend at specified container.
func Open(cfg Config) (restic.Backend, error) {
return open(cfg)
func Open(cfg Config, rt http.RoundTripper) (restic.Backend, error) {
return open(cfg, rt)
}
// Create opens the Azure backend at specified container and creates the container if
// it does not exist yet.
func Create(cfg Config) (restic.Backend, error) {
be, err := open(cfg)
func Create(cfg Config, rt http.RoundTripper) (restic.Backend, error) {
be, err := open(cfg, rt)
if err != nil {
return nil, errors.Wrap(err, "open")

13
internal/backend/azure/azure_test.go
View File

@ -7,6 +7,7 @@ import (
"testing"
"time"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/azure"
"github.com/restic/restic/internal/backend/test"
"github.com/restic/restic/internal/errors"
@ -15,6 +16,11 @@ import (
)
func newAzureTestSuite(t testing.TB) *test.Suite {
tr, err := backend.Transport(nil)
if err != nil {
t.Fatalf("cannot create transport for tests: %v", err)
}
return &test.Suite{
// do not use excessive data
MinimalData: true,
@ -37,7 +43,7 @@ func newAzureTestSuite(t testing.TB) *test.Suite {
Create: func(config interface{}) (restic.Backend, error) {
cfg := config.(azure.Config)
be, err := azure.Create(cfg)
be, err := azure.Create(cfg, tr)
if err != nil {
return nil, err
}
@ -57,14 +63,15 @@ func newAzureTestSuite(t testing.TB) *test.Suite {
// OpenFn is a function that opens a previously created temporary repository.
Open: func(config interface{}) (restic.Backend, error) {
cfg := config.(azure.Config)
return azure.Open(cfg)
return azure.Open(cfg, tr)
},
// CleanupFn removes data created during the tests.
Cleanup: func(config interface{}) error {
cfg := config.(azure.Config)
be, err := azure.Open(cfg)
be, err := azure.Open(cfg, tr)
if err != nil {
return err
}

13
internal/backend/b2/b2.go
View File

@ -3,6 +3,7 @@ package b2
import (
"context"
"io"
"net/http"
"path"
"strings"
@ -29,8 +30,8 @@ const defaultListMaxItems = 1000
// ensure statically that *b2Backend implements restic.Backend.
var _ restic.Backend = &b2Backend{}
func newClient(ctx context.Context, cfg Config) (*b2.Client, error) {
opts := []b2.ClientOption{b2.Transport(backend.Transport())}
func newClient(ctx context.Context, cfg Config, rt http.RoundTripper) (*b2.Client, error) {
opts := []b2.ClientOption{b2.Transport(rt)}
c, err := b2.NewClient(ctx, cfg.AccountID, cfg.Key, opts...)
if err != nil {
@ -40,13 +41,13 @@ func newClient(ctx context.Context, cfg Config) (*b2.Client, error) {
}
// Open opens a connection to the B2 service.
func Open(cfg Config) (restic.Backend, error) {
func Open(cfg Config, rt http.RoundTripper) (restic.Backend, error) {
debug.Log("cfg %#v", cfg)
ctx, cancel := context.WithCancel(context.TODO())
defer cancel()
client, err := newClient(ctx, cfg)
client, err := newClient(ctx, cfg, rt)
if err != nil {
return nil, err
}
@ -77,13 +78,13 @@ func Open(cfg Config) (restic.Backend, error) {
// Create opens a connection to the B2 service. If the bucket does not exist yet,
// it is created.
func Create(cfg Config) (restic.Backend, error) {
func Create(cfg Config, rt http.RoundTripper) (restic.Backend, error) {
debug.Log("cfg %#v", cfg)
ctx, cancel := context.WithCancel(context.TODO())
defer cancel()
client, err := newClient(ctx, cfg)
client, err := newClient(ctx, cfg, rt)
if err != nil {
return nil, err
}

12
internal/backend/b2/b2_test.go
View File

@ -7,6 +7,7 @@ import (
"testing"
"time"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/b2"
"github.com/restic/restic/internal/backend/test"
"github.com/restic/restic/internal/restic"
@ -15,6 +16,11 @@ import (
)
func newB2TestSuite(t testing.TB) *test.Suite {
tr, err := backend.Transport(nil)
if err != nil {
t.Fatalf("cannot create transport for tests: %v", err)
}
return &test.Suite{
// do not use excessive data
MinimalData: true,
@ -39,19 +45,19 @@ func newB2TestSuite(t testing.TB) *test.Suite {
// CreateFn is a function that creates a temporary repository for the tests.
Create: func(config interface{}) (restic.Backend, error) {
cfg := config.(b2.Config)
return b2.Create(cfg)
return b2.Create(cfg, tr)
},
// OpenFn is a function that opens a previously created temporary repository.
Open: func(config interface{}) (restic.Backend, error) {
cfg := config.(b2.Config)
return b2.Open(cfg)
return b2.Open(cfg, tr)
},
// CleanupFn removes data created during the tests.
Cleanup: func(config interface{}) error {
cfg := config.(b2.Config)
be, err := b2.Open(cfg)
be, err := b2.Open(cfg, tr)
if err != nil {
return err
}

34
internal/backend/http_transport.go
View File

@ -1,6 +1,10 @@
package backend
import (
"crypto/tls"
"crypto/x509"
"fmt"
"io/ioutil"
"net"
"net/http"
"time"
@ -8,8 +12,10 @@ import (
"github.com/restic/restic/internal/debug"
)
// Transport returns a new http.RoundTripper with default settings applied.
func Transport() http.RoundTripper {
// Transport returns a new http.RoundTripper with default settings applied. If
// a custom rootCertFilename is non-empty, it must point to a valid PEM file,
// otherwise the function will return an error.
func Transport(rootCertFilenames []string) (http.RoundTripper, error) {
// copied from net/http
tr := &http.Transport{
Proxy: http.ProxyFromEnvironment,
@ -25,6 +31,28 @@ func Transport() http.RoundTripper {
ExpectContinueTimeout: 1 * time.Second,
}
if rootCertFilenames == nil {
return debug.RoundTripper(tr), nil
}
p := x509.NewCertPool()
for _, filename := range rootCertFilenames {
if filename == "" {
return nil, fmt.Errorf("empty filename for root certificate supplied")
}
b, err := ioutil.ReadFile(filename)
if err != nil {
return nil, fmt.Errorf("unable to read root certificate: %v", err)
}
if ok := p.AppendCertsFromPEM(b); !ok {
return nil, fmt.Errorf("cannot parse root certificate from %q", filename)
}
}
tr.TLSClientConfig = &tls.Config{
RootCAs: p,
}
// wrap in the debug round tripper
return debug.RoundTripper(tr)
return debug.RoundTripper(tr), nil
}

8
internal/backend/rest/rest.go
View File

@ -31,8 +31,8 @@ type restBackend struct {
}
// Open opens the REST backend with the given config.
func Open(cfg Config) (restic.Backend, error) {
client := &http.Client{Transport: backend.Transport()}
func Open(cfg Config, rt http.RoundTripper) (restic.Backend, error) {
client := &http.Client{Transport: rt}
sem, err := backend.NewSemaphore(cfg.Connections)
if err != nil {
@ -56,8 +56,8 @@ func Open(cfg Config) (restic.Backend, error) {
}
// Create creates a new REST on server configured in config.
func Create(cfg Config) (restic.Backend, error) {
be, err := Open(cfg)
func Create(cfg Config, rt http.RoundTripper) (restic.Backend, error) {
be, err := Open(cfg, rt)
if err != nil {
return nil, err
}

10
internal/backend/rest/rest_test.go
View File

@ -10,6 +10,7 @@ import (
"testing"
"time"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/rest"
"github.com/restic/restic/internal/backend/test"
"github.com/restic/restic/internal/restic"
@ -61,6 +62,11 @@ func runRESTServer(ctx context.Context, t testing.TB, dir string) func() {
}
func newTestSuite(ctx context.Context, t testing.TB) *test.Suite {
tr, err := backend.Transport(nil)
if err != nil {
t.Fatalf("cannot create transport for tests: %v", err)
}
return &test.Suite{
// NewConfig returns a config for a new temporary backend that will be used in tests.
NewConfig: func() (interface{}, error) {
@ -84,13 +90,13 @@ func newTestSuite(ctx context.Context, t testing.TB) *test.Suite {
// CreateFn is a function that creates a temporary repository for the tests.
Create: func(config interface{}) (restic.Backend, error) {
cfg := config.(rest.Config)
return rest.Create(cfg)
return rest.Create(cfg, tr)
},
// OpenFn is a function that opens a previously created temporary repository.
Open: func(config interface{}) (restic.Backend, error) {
cfg := config.(rest.Config)
return rest.Open(cfg)
return rest.Open(cfg, tr)
},
// CleanupFn removes data created during the tests.

13
internal/backend/s3/s3.go
View File

@ -4,6 +4,7 @@ import (
"context"
"fmt"
"io"
"net/http"
"os"
"path"
"strings"
@ -32,7 +33,7 @@ var _ restic.Backend = &Backend{}
const defaultLayout = "default"
func open(cfg Config) (*Backend, error) {
func open(cfg Config, rt http.RoundTripper) (*Backend, error) {
debug.Log("open, config %#v", cfg)
if cfg.MaxRetries > 0 {
@ -70,7 +71,7 @@ func open(cfg Config) (*Backend, error) {
cfg: cfg,
}
client.SetCustomTransport(backend.Transport())
client.SetCustomTransport(rt)
l, err := backend.ParseLayout(be, cfg.Layout, defaultLayout, cfg.Prefix)
if err != nil {
@ -84,14 +85,14 @@ func open(cfg Config) (*Backend, error) {
// Open opens the S3 backend at bucket and region. The bucket is created if it
// does not exist yet.
func Open(cfg Config) (restic.Backend, error) {
return open(cfg)
func Open(cfg Config, rt http.RoundTripper) (restic.Backend, error) {
return open(cfg, rt)
}
// Create opens the S3 backend at bucket and region and creates the bucket if
// it does not exist yet.
func Create(cfg Config) (restic.Backend, error) {
be, err := open(cfg)
func Create(cfg Config, rt http.RoundTripper) (restic.Backend, error) {
be, err := open(cfg, rt)
if err != nil {
return nil, errors.Wrap(err, "open")
}

26
internal/backend/s3/s3_test.go
View File

@ -8,12 +8,14 @@ import (
"fmt"
"io"
"net"
"net/http"
"os"
"os/exec"
"path/filepath"
"testing"
"time"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/s3"
"github.com/restic/restic/internal/backend/test"
"github.com/restic/restic/internal/restic"
@ -103,9 +105,9 @@ type MinioTestConfig struct {
stopServer func()
}
func createS3(t testing.TB, cfg MinioTestConfig) (be restic.Backend, err error) {
func createS3(t testing.TB, cfg MinioTestConfig, tr http.RoundTripper) (be restic.Backend, err error) {
for i := 0; i < 10; i++ {
be, err = s3.Create(cfg.Config)
be, err = s3.Create(cfg.Config, tr)
if err != nil {
t.Logf("s3 open: try %d: error %v", i, err)
time.Sleep(500 * time.Millisecond)
@ -119,6 +121,11 @@ func createS3(t testing.TB, cfg MinioTestConfig) (be restic.Backend, err error)
}
func newMinioTestSuite(ctx context.Context, t testing.TB) *test.Suite {
tr, err := backend.Transport(nil)
if err != nil {
t.Fatalf("cannot create transport for tests: %v", err)
}
return &test.Suite{
// NewConfig returns a config for a new temporary backend that will be used in tests.
NewConfig: func() (interface{}, error) {
@ -142,7 +149,7 @@ func newMinioTestSuite(ctx context.Context, t testing.TB) *test.Suite {
Create: func(config interface{}) (restic.Backend, error) {
cfg := config.(MinioTestConfig)
be, err := createS3(t, cfg)
be, err := createS3(t, cfg, tr)
if err != nil {
return nil, err
}
@ -162,7 +169,7 @@ func newMinioTestSuite(ctx context.Context, t testing.TB) *test.Suite {
// OpenFn is a function that opens a previously created temporary repository.
Open: func(config interface{}) (restic.Backend, error) {
cfg := config.(MinioTestConfig)
return s3.Open(cfg.Config)
return s3.Open(cfg.Config, tr)
},
// CleanupFn removes data created during the tests.
@ -214,6 +221,11 @@ func BenchmarkBackendMinio(t *testing.B) {
}
func newS3TestSuite(t testing.TB) *test.Suite {
tr, err := backend.Transport(nil)
if err != nil {
t.Fatalf("cannot create transport for tests: %v", err)
}
return &test.Suite{
// do not use excessive data
MinimalData: true,
@ -236,7 +248,7 @@ func newS3TestSuite(t testing.TB) *test.Suite {
Create: func(config interface{}) (restic.Backend, error) {
cfg := config.(s3.Config)
be, err := s3.Create(cfg)
be, err := s3.Create(cfg, tr)
if err != nil {
return nil, err
}
@ -256,14 +268,14 @@ func newS3TestSuite(t testing.TB) *test.Suite {
// OpenFn is a function that opens a previously created temporary repository.
Open: func(config interface{}) (restic.Backend, error) {
cfg := config.(s3.Config)
return s3.Open(cfg)
return s3.Open(cfg, tr)
},
// CleanupFn removes data created during the tests.
Cleanup: func(config interface{}) error {
cfg := config.(s3.Config)
be, err := s3.Open(cfg)
be, err := s3.Open(cfg, tr)
if err != nil {
return err
}

4
internal/backend/swift/swift.go
View File

@ -34,7 +34,7 @@ var _ restic.Backend = &beSwift{}
// Open opens the swift backend at a container in region. The container is
// created if it does not exist yet.
func Open(cfg Config) (restic.Backend, error) {
func Open(cfg Config, rt http.RoundTripper) (restic.Backend, error) {
debug.Log("config %#v", cfg)
sem, err := backend.NewSemaphore(cfg.Connections)
@ -58,7 +58,7 @@ func Open(cfg Config) (restic.Backend, error) {
ConnectTimeout: time.Minute,
Timeout: time.Minute,
Transport: backend.Transport(),
Transport: rt,
},
sem: sem,
container: cfg.Container,

12
internal/backend/swift/swift_test.go
View File

@ -7,6 +7,7 @@ import (
"testing"
"time"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/swift"
"github.com/restic/restic/internal/backend/test"
"github.com/restic/restic/internal/errors"
@ -15,6 +16,11 @@ import (
)
func newSwiftTestSuite(t testing.TB) *test.Suite {
tr, err := backend.Transport(nil)
if err != nil {
t.Fatalf("cannot create transport for tests: %v", err)
}
return &test.Suite{
// do not use excessive data
MinimalData: true,
@ -55,7 +61,7 @@ func newSwiftTestSuite(t testing.TB) *test.Suite {
Create: func(config interface{}) (restic.Backend, error) {
cfg := config.(swift.Config)
be, err := swift.Open(cfg)
be, err := swift.Open(cfg, tr)
if err != nil {
return nil, err
}
@ -75,14 +81,14 @@ func newSwiftTestSuite(t testing.TB) *test.Suite {
// OpenFn is a function that opens a previously created temporary repository.
Open: func(config interface{}) (restic.Backend, error) {
cfg := config.(swift.Config)
return swift.Open(cfg)
return swift.Open(cfg, tr)
},
// CleanupFn removes data created during the tests.
Cleanup: func(config interface{}) error {
cfg := config.(swift.Config)
be, err := swift.Open(cfg)
be, err := swift.Open(cfg, tr)
if err != nil {
return err
}