Merge pull request #3470 from MichaelEischer/sanitize-debug-log

Sanitize debug log

cmd/restic/global.go

@@ -555,13 +555,13 @@ func parseConfig(loc location.Location, opts options.Options) (interface{}, erro
 			cfg.KeyID = os.Getenv("AWS_ACCESS_KEY_ID")
 		}
 
-		if cfg.Secret == "" {
+		if cfg.Secret.String() == "" {
-			cfg.Secret = os.Getenv("AWS_SECRET_ACCESS_KEY")
+			cfg.Secret = options.NewSecretString(os.Getenv("AWS_SECRET_ACCESS_KEY"))
 		}
 
-		if cfg.KeyID == "" && cfg.Secret != "" {
+		if cfg.KeyID == "" && cfg.Secret.String() != "" {
 			return nil, errors.Fatalf("unable to open S3 backend: Key ID ($AWS_ACCESS_KEY_ID) is empty")
-		} else if cfg.KeyID != "" && cfg.Secret == "" {
+		} else if cfg.KeyID != "" && cfg.Secret.String() == "" {
 			return nil, errors.Fatalf("unable to open S3 backend: Secret ($AWS_SECRET_ACCESS_KEY) is empty")
 		}
 
@@ -595,8 +595,8 @@ func parseConfig(loc location.Location, opts options.Options) (interface{}, erro
 			cfg.AccountName = os.Getenv("AZURE_ACCOUNT_NAME")
 		}
 
-		if cfg.AccountKey == "" {
+		if cfg.AccountKey.String() == "" {
-			cfg.AccountKey = os.Getenv("AZURE_ACCOUNT_KEY")
+			cfg.AccountKey = options.NewSecretString(os.Getenv("AZURE_ACCOUNT_KEY"))
 		}
 
 		if err := opts.Apply(loc.Scheme, &cfg); err != nil {
@@ -631,11 +631,11 @@ func parseConfig(loc location.Location, opts options.Options) (interface{}, erro
 			return nil, errors.Fatalf("unable to open B2 backend: Account ID ($B2_ACCOUNT_ID) is empty")
 		}
 
-		if cfg.Key == "" {
+		if cfg.Key.String() == "" {
-			cfg.Key = os.Getenv("B2_ACCOUNT_KEY")
+			cfg.Key = options.NewSecretString(os.Getenv("B2_ACCOUNT_KEY"))
 		}
 
-		if cfg.Key == "" {
+		if cfg.Key.String() == "" {
 			return nil, errors.Fatalf("unable to open B2 backend: Key ($B2_ACCOUNT_KEY) is empty")
 		}
 

internal/backend/azure/azure.go

@@ -40,7 +40,7 @@ var _ restic.Backend = &Backend{}
 func open(cfg Config, rt http.RoundTripper) (*Backend, error) {
 	debug.Log("open, config %#v", cfg)
 
-	client, err := storage.NewBasicClient(cfg.AccountName, cfg.AccountKey)
+	client, err := storage.NewBasicClient(cfg.AccountName, cfg.AccountKey.Unwrap())
 	if err != nil {
 		return nil, errors.Wrap(err, "NewBasicClient")
 	}

internal/backend/azure/azure_test.go

@@ -13,6 +13,7 @@ import (
 	"github.com/restic/restic/internal/backend/azure"
 	"github.com/restic/restic/internal/backend/test"
 	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/options"
 	"github.com/restic/restic/internal/restic"
 	rtest "github.com/restic/restic/internal/test"
 )
@@ -36,7 +37,7 @@ func newAzureTestSuite(t testing.TB) *test.Suite {
 
 			cfg := azcfg.(azure.Config)
 			cfg.AccountName = os.Getenv("RESTIC_TEST_AZURE_ACCOUNT_NAME")
-			cfg.AccountKey = os.Getenv("RESTIC_TEST_AZURE_ACCOUNT_KEY")
+			cfg.AccountKey = options.NewSecretString(os.Getenv("RESTIC_TEST_AZURE_ACCOUNT_KEY"))
 			cfg.Prefix = fmt.Sprintf("test-%d", time.Now().UnixNano())
 			return cfg, nil
 		},
@@ -146,7 +147,7 @@ func TestUploadLargeFile(t *testing.T) {
 
 	cfg := azcfg.(azure.Config)
 	cfg.AccountName = os.Getenv("RESTIC_TEST_AZURE_ACCOUNT_NAME")
-	cfg.AccountKey = os.Getenv("RESTIC_TEST_AZURE_ACCOUNT_KEY")
+	cfg.AccountKey = options.NewSecretString(os.Getenv("RESTIC_TEST_AZURE_ACCOUNT_KEY"))
 	cfg.Prefix = fmt.Sprintf("test-upload-large-%d", time.Now().UnixNano())
 
 	tr, err := backend.Transport(backend.TransportOptions{})

internal/backend/azure/config.go

@@ -12,7 +12,7 @@ import (
 // server.
 type Config struct {
 	AccountName string
-	AccountKey  string
+	AccountKey  options.SecretString
 	Container   string
 	Prefix      string
 

internal/backend/b2/b2.go

@@ -35,7 +35,7 @@ var _ restic.Backend = &b2Backend{}
 func newClient(ctx context.Context, cfg Config, rt http.RoundTripper) (*b2.Client, error) {
 	opts := []b2.ClientOption{b2.Transport(rt)}
 
-	c, err := b2.NewClient(ctx, cfg.AccountID, cfg.Key, opts...)
+	c, err := b2.NewClient(ctx, cfg.AccountID, cfg.Key.Unwrap(), opts...)
 	if err != nil {
 		return nil, errors.Wrap(err, "b2.NewClient")
 	}

internal/backend/b2/b2_test.go

@@ -10,6 +10,7 @@ import (
 	"github.com/restic/restic/internal/backend"
 	"github.com/restic/restic/internal/backend/b2"
 	"github.com/restic/restic/internal/backend/test"
+	"github.com/restic/restic/internal/options"
 	"github.com/restic/restic/internal/restic"
 
 	rtest "github.com/restic/restic/internal/test"
@@ -37,7 +38,7 @@ func newB2TestSuite(t testing.TB) *test.Suite {
 
 			cfg := b2cfg.(b2.Config)
 			cfg.AccountID = os.Getenv("RESTIC_TEST_B2_ACCOUNT_ID")
-			cfg.Key = os.Getenv("RESTIC_TEST_B2_ACCOUNT_KEY")
+			cfg.Key = options.NewSecretString(os.Getenv("RESTIC_TEST_B2_ACCOUNT_KEY"))
 			cfg.Prefix = fmt.Sprintf("test-%d", time.Now().UnixNano())
 			return cfg, nil
 		},

internal/backend/b2/config.go

@@ -13,7 +13,7 @@ import (
 // server.
 type Config struct {
 	AccountID string
-	Key       string
+	Key       options.SecretString
 	Bucket    string
 	Prefix    string
 

internal/backend/s3/config.go

@@ -12,13 +12,14 @@ import (
 // Config contains all configuration necessary to connect to an s3 compatible
 // server.
 type Config struct {
-	Endpoint      string
+	Endpoint     string
-	UseHTTP       bool
+	UseHTTP      bool
-	KeyID, Secret string
+	KeyID        string
-	Bucket        string
+	Secret       options.SecretString
-	Prefix        string
+	Bucket       string
-	Layout        string `option:"layout" help:"use this backend layout (default: auto-detect)"`
+	Prefix       string
-	StorageClass  string `option:"storage-class" help:"set S3 storage class (STANDARD, STANDARD_IA, ONEZONE_IA, INTELLIGENT_TIERING or REDUCED_REDUNDANCY)"`
+	Layout       string `option:"layout" help:"use this backend layout (default: auto-detect)"`
+	StorageClass string `option:"storage-class" help:"set S3 storage class (STANDARD, STANDARD_IA, ONEZONE_IA, INTELLIGENT_TIERING or REDUCED_REDUNDANCY)"`
 
 	Connections   uint   `option:"connections" help:"set a limit for the number of concurrent connections (default: 5)"`
 	MaxRetries    uint   `option:"retries" help:"set the number of retries attempted"`

internal/backend/s3/s3.go

@@ -57,7 +57,7 @@ func open(ctx context.Context, cfg Config, rt http.RoundTripper) (*Backend, erro
 		&credentials.Static{
 			Value: credentials.Value{
 				AccessKeyID:     cfg.KeyID,
-				SecretAccessKey: cfg.Secret,
+				SecretAccessKey: cfg.Secret.Unwrap(),
 			},
 		},
 		&credentials.EnvMinio{},

internal/backend/s3/s3_test.go

@@ -18,6 +18,7 @@ import (
 	"github.com/restic/restic/internal/backend"
 	"github.com/restic/restic/internal/backend/s3"
 	"github.com/restic/restic/internal/backend/test"
+	"github.com/restic/restic/internal/options"
 	"github.com/restic/restic/internal/restic"
 	rtest "github.com/restic/restic/internal/test"
 )
@@ -141,7 +142,7 @@ func newMinioTestSuite(ctx context.Context, t testing.TB) *test.Suite {
 			cfg.Config.Prefix = fmt.Sprintf("test-%d", time.Now().UnixNano())
 			cfg.Config.UseHTTP = true
 			cfg.Config.KeyID = key
-			cfg.Config.Secret = secret
+			cfg.Config.Secret = options.NewSecretString(secret)
 			return cfg, nil
 		},
 
@@ -239,7 +240,7 @@ func newS3TestSuite(t testing.TB) *test.Suite {
 
 			cfg := s3cfg.(s3.Config)
 			cfg.KeyID = os.Getenv("RESTIC_TEST_S3_KEY")
-			cfg.Secret = os.Getenv("RESTIC_TEST_S3_SECRET")
+			cfg.Secret = options.NewSecretString(os.Getenv("RESTIC_TEST_S3_SECRET"))
 			cfg.Prefix = fmt.Sprintf("test-%d", time.Now().UnixNano())
 			return cfg, nil
 		},

internal/backend/swift/config.go

@@ -24,12 +24,12 @@ type Config struct {
 	TrustID        string
 
 	StorageURL string
-	AuthToken  string
+	AuthToken  options.SecretString
 
 	// auth v3 only
 	ApplicationCredentialID     string
 	ApplicationCredentialName   string
-	ApplicationCredentialSecret string
+	ApplicationCredentialSecret options.SecretString
 
 	Container              string
 	Prefix                 string
@@ -111,11 +111,9 @@ func ApplyEnvironment(prefix string, cfg interface{}) error {
 		// Application Credential auth
 		{&c.ApplicationCredentialID, prefix + "OS_APPLICATION_CREDENTIAL_ID"},
 		{&c.ApplicationCredentialName, prefix + "OS_APPLICATION_CREDENTIAL_NAME"},
-		{&c.ApplicationCredentialSecret, prefix + "OS_APPLICATION_CREDENTIAL_SECRET"},
 
 		// Manual authentication
 		{&c.StorageURL, prefix + "OS_STORAGE_URL"},
-		{&c.AuthToken, prefix + "OS_AUTH_TOKEN"},
 
 		{&c.DefaultContainerPolicy, prefix + "SWIFT_DEFAULT_CONTAINER_POLICY"},
 	} {
@@ -123,5 +121,16 @@ func ApplyEnvironment(prefix string, cfg interface{}) error {
 			*val.s = os.Getenv(val.env)
 		}
 	}
+	for _, val := range []struct {
+		s   *options.SecretString
+		env string
+	}{
+		{&c.ApplicationCredentialSecret, prefix + "OS_APPLICATION_CREDENTIAL_SECRET"},
+		{&c.AuthToken, prefix + "OS_AUTH_TOKEN"},
+	} {
+		if val.s.String() == "" {
+			*val.s = options.NewSecretString(os.Getenv(val.env))
+		}
+	}
 	return nil
 }

internal/backend/swift/swift.go

@@ -61,10 +61,10 @@ func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend
 			TenantDomainId:              cfg.TenantDomainID,
 			TrustId:                     cfg.TrustID,
 			StorageUrl:                  cfg.StorageURL,
-			AuthToken:                   cfg.AuthToken,
+			AuthToken:                   cfg.AuthToken.Unwrap(),
 			ApplicationCredentialId:     cfg.ApplicationCredentialID,
 			ApplicationCredentialName:   cfg.ApplicationCredentialName,
-			ApplicationCredentialSecret: cfg.ApplicationCredentialSecret,
+			ApplicationCredentialSecret: cfg.ApplicationCredentialSecret.Unwrap(),
 			ConnectTimeout:              time.Minute,
 			Timeout:                     time.Minute,
 

internal/debug/round_tripper_debug.go

@@ -75,7 +75,32 @@ func RoundTripper(upstream http.RoundTripper) http.RoundTripper {
 	return eofRoundTripper
 }
 
+func redactHeader(header http.Header) map[string][]string {
+	removedHeaders := make(map[string][]string)
+	for _, hdr := range []string{
+		"Authorization",
+		"X-Auth-Token", // Swift headers
+		"X-Auth-Key",
+	} {
+		origHeader, hasHeader := header[hdr]
+		if hasHeader {
+			removedHeaders[hdr] = origHeader
+			header[hdr] = []string{"**redacted**"}
+		}
+	}
+	return removedHeaders
+}
+
+func restoreHeader(header http.Header, origHeaders map[string][]string) {
+	for hdr, val := range origHeaders {
+		header[hdr] = val
+	}
+}
+
 func (tr loggingRoundTripper) RoundTrip(req *http.Request) (res *http.Response, err error) {
+	// save original auth and redact it
+	origHeaders := redactHeader(req.Header)
+
 	trace, err := httputil.DumpRequestOut(req, false)
 	if err != nil {
 		Log("DumpRequestOut() error: %v\n", err)
@@ -83,13 +108,17 @@ func (tr loggingRoundTripper) RoundTrip(req *http.Request) (res *http.Response,
 		Log("------------  HTTP REQUEST -----------\n%s", trace)
 	}
 
+	restoreHeader(req.Header, origHeaders)
+
 	res, err = tr.RoundTripper.RoundTrip(req)
 	if err != nil {
 		Log("RoundTrip() returned error: %v", err)
 	}
 
 	if res != nil {
+		origHeaders := redactHeader(res.Header)
 		trace, err := httputil.DumpResponse(res, false)
+		restoreHeader(res.Header, origHeaders)
 		if err != nil {
 			Log("DumpResponse() error: %v\n", err)
 		} else {

internal/debug/round_tripper_debug_test.go

@@ -0,0 +1,46 @@
+// +build debug
+
+package debug
+
+import (
+	"net/http"
+	"testing"
+
+	"github.com/restic/restic/internal/test"
+)
+
+func TestRedactHeader(t *testing.T) {
+	secretHeaders := []string{
+		"Authorization",
+		"X-Auth-Token",
+		"X-Auth-Key",
+	}
+
+	header := make(http.Header)
+	header["Authorization"] = []string{"123"}
+	header["X-Auth-Token"] = []string{"1234"}
+	header["X-Auth-Key"] = []string{"12345"}
+	header["Host"] = []string{"my.host"}
+
+	origHeaders := redactHeader(header)
+
+	for _, hdr := range secretHeaders {
+		test.Equals(t, "**redacted**", header[hdr][0])
+	}
+	test.Equals(t, "my.host", header["Host"][0])
+
+	restoreHeader(header, origHeaders)
+	test.Equals(t, "123", header["Authorization"][0])
+	test.Equals(t, "1234", header["X-Auth-Token"][0])
+	test.Equals(t, "12345", header["X-Auth-Key"][0])
+	test.Equals(t, "my.host", header["Host"][0])
+
+	delete(header, "X-Auth-Key")
+	origHeaders = redactHeader(header)
+	_, hasHeader := header["X-Auth-Key"]
+	test.Assert(t, !hasHeader, "Unexpected header: %v", header["X-Auth-Key"])
+
+	restoreHeader(header, origHeaders)
+	_, hasHeader = header["X-Auth-Key"]
+	test.Assert(t, !hasHeader, "Unexpected header: %v", header["X-Auth-Key"])
+}

internal/options/secret_string.go

@@ -0,0 +1,24 @@
+package options
+
+type SecretString struct {
+	s *string
+}
+
+func NewSecretString(s string) SecretString {
+	return SecretString{s: &s}
+}
+
+func (s SecretString) GoString() string {
+	return `"` + s.String() + `"`
+}
+
+func (s SecretString) String() string {
+	if len(*s.s) == 0 {
+		return ``
+	}
+	return `**redacted**`
+}
+
+func (s *SecretString) Unwrap() string {
+	return *s.s
+}

internal/options/secret_string_test.go

@@ -0,0 +1,55 @@
+package options_test
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/restic/restic/internal/options"
+	"github.com/restic/restic/internal/test"
+)
+
+type secretTest struct {
+	str options.SecretString
+}
+
+func assertNotIn(t *testing.T, str string, substr string) {
+	if strings.Contains(str, substr) {
+		t.Fatalf("'%s' should not contain '%s'", str, substr)
+	}
+}
+
+func TestSecretString(t *testing.T) {
+	keyStr := "secret-key"
+	secret := options.NewSecretString(keyStr)
+
+	test.Equals(t, "**redacted**", secret.String())
+	test.Equals(t, `"**redacted**"`, secret.GoString())
+	test.Equals(t, "**redacted**", fmt.Sprint(secret))
+	test.Equals(t, "**redacted**", fmt.Sprintf("%v", secret))
+	test.Equals(t, `"**redacted**"`, fmt.Sprintf("%#v", secret))
+	test.Equals(t, keyStr, secret.Unwrap())
+}
+
+func TestSecretStringStruct(t *testing.T) {
+	keyStr := "secret-key"
+	secretStruct := &secretTest{
+		str: options.NewSecretString(keyStr),
+	}
+
+	assertNotIn(t, fmt.Sprint(secretStruct), keyStr)
+	assertNotIn(t, fmt.Sprintf("%v", secretStruct), keyStr)
+	assertNotIn(t, fmt.Sprintf("%#v", secretStruct), keyStr)
+}
+
+func TestSecretStringEmpty(t *testing.T) {
+	keyStr := ""
+	secret := options.NewSecretString(keyStr)
+
+	test.Equals(t, "", secret.String())
+	test.Equals(t, `""`, secret.GoString())
+	test.Equals(t, "", fmt.Sprint(secret))
+	test.Equals(t, "", fmt.Sprintf("%v", secret))
+	test.Equals(t, `""`, fmt.Sprintf("%#v", secret))
+	test.Equals(t, keyStr, secret.Unwrap())
+}