octoleo/restic
2
2
Fork 0
mirror of https://github.com/octoleo/restic.git synced 2024-11-22 12:55:18 +00:00
Code Issues Releases Wiki Activity

Merge pull request #3470 from MichaelEischer/sanitize-debug-log

Browse Source 
Sanitize debug log
This commit is contained in:
MichaelEischer 2022-07-02 19:00:54 +02:00 committed by GitHub
commit c16f989d4a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 199 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
cmd/restic/global.go
View File

@ -555,13 +555,13 @@ func parseConfig(loc location.Location, opts options.Options) (interface{}, erro
cfg.KeyID = os.Getenv("AWS_ACCESS_KEY_ID")
}
if cfg.Secret == "" {
cfg.Secret = os.Getenv("AWS_SECRET_ACCESS_KEY")
if cfg.Secret.String() == "" {
cfg.Secret = options.NewSecretString(os.Getenv("AWS_SECRET_ACCESS_KEY"))
}
if cfg.KeyID == "" && cfg.Secret != "" {
if cfg.KeyID == "" && cfg.Secret.String() != "" {
return nil, errors.Fatalf("unable to open S3 backend: Key ID ($AWS_ACCESS_KEY_ID) is empty")
} else if cfg.KeyID != "" && cfg.Secret == "" {
} else if cfg.KeyID != "" && cfg.Secret.String() == "" {
return nil, errors.Fatalf("unable to open S3 backend: Secret ($AWS_SECRET_ACCESS_KEY) is empty")
}
@ -595,8 +595,8 @@ func parseConfig(loc location.Location, opts options.Options) (interface{}, erro
cfg.AccountName = os.Getenv("AZURE_ACCOUNT_NAME")
}
if cfg.AccountKey == "" {
cfg.AccountKey = os.Getenv("AZURE_ACCOUNT_KEY")
if cfg.AccountKey.String() == "" {
cfg.AccountKey = options.NewSecretString(os.Getenv("AZURE_ACCOUNT_KEY"))
}
if err := opts.Apply(loc.Scheme, &cfg); err != nil {
@ -631,11 +631,11 @@ func parseConfig(loc location.Location, opts options.Options) (interface{}, erro
return nil, errors.Fatalf("unable to open B2 backend: Account ID ($B2_ACCOUNT_ID) is empty")
}
if cfg.Key == "" {
cfg.Key = os.Getenv("B2_ACCOUNT_KEY")
if cfg.Key.String() == "" {
cfg.Key = options.NewSecretString(os.Getenv("B2_ACCOUNT_KEY"))
}
if cfg.Key == "" {
if cfg.Key.String() == "" {
return nil, errors.Fatalf("unable to open B2 backend: Key ($B2_ACCOUNT_KEY) is empty")
}

2
internal/backend/azure/azure.go
View File

@ -40,7 +40,7 @@ var _ restic.Backend = &Backend{}
func open(cfg Config, rt http.RoundTripper) (*Backend, error) {
debug.Log("open, config %#v", cfg)
client, err := storage.NewBasicClient(cfg.AccountName, cfg.AccountKey)
client, err := storage.NewBasicClient(cfg.AccountName, cfg.AccountKey.Unwrap())
if err != nil {
return nil, errors.Wrap(err, "NewBasicClient")
}

5
internal/backend/azure/azure_test.go
View File

@ -13,6 +13,7 @@ import (
"github.com/restic/restic/internal/backend/azure"
"github.com/restic/restic/internal/backend/test"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/options"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
)
@ -36,7 +37,7 @@ func newAzureTestSuite(t testing.TB) *test.Suite {
cfg := azcfg.(azure.Config)
cfg.AccountName = os.Getenv("RESTIC_TEST_AZURE_ACCOUNT_NAME")
cfg.AccountKey = os.Getenv("RESTIC_TEST_AZURE_ACCOUNT_KEY")
cfg.AccountKey = options.NewSecretString(os.Getenv("RESTIC_TEST_AZURE_ACCOUNT_KEY"))
cfg.Prefix = fmt.Sprintf("test-%d", time.Now().UnixNano())
return cfg, nil
},
@ -146,7 +147,7 @@ func TestUploadLargeFile(t *testing.T) {
cfg := azcfg.(azure.Config)
cfg.AccountName = os.Getenv("RESTIC_TEST_AZURE_ACCOUNT_NAME")
cfg.AccountKey = os.Getenv("RESTIC_TEST_AZURE_ACCOUNT_KEY")
cfg.AccountKey = options.NewSecretString(os.Getenv("RESTIC_TEST_AZURE_ACCOUNT_KEY"))
cfg.Prefix = fmt.Sprintf("test-upload-large-%d", time.Now().UnixNano())
tr, err := backend.Transport(backend.TransportOptions{})

2
internal/backend/azure/config.go
View File

@ -12,7 +12,7 @@ import (
// server.
type Config struct {
AccountName string
AccountKey string
AccountKey options.SecretString
Container string
Prefix string

2
internal/backend/b2/b2.go
View File

@ -35,7 +35,7 @@ var _ restic.Backend = &b2Backend{}
func newClient(ctx context.Context, cfg Config, rt http.RoundTripper) (*b2.Client, error) {
opts := []b2.ClientOption{b2.Transport(rt)}
c, err := b2.NewClient(ctx, cfg.AccountID, cfg.Key, opts...)
c, err := b2.NewClient(ctx, cfg.AccountID, cfg.Key.Unwrap(), opts...)
if err != nil {
return nil, errors.Wrap(err, "b2.NewClient")
}

3
internal/backend/b2/b2_test.go
View File

@ -10,6 +10,7 @@ import (
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/b2"
"github.com/restic/restic/internal/backend/test"
"github.com/restic/restic/internal/options"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
@ -37,7 +38,7 @@ func newB2TestSuite(t testing.TB) *test.Suite {
cfg := b2cfg.(b2.Config)
cfg.AccountID = os.Getenv("RESTIC_TEST_B2_ACCOUNT_ID")
cfg.Key = os.Getenv("RESTIC_TEST_B2_ACCOUNT_KEY")
cfg.Key = options.NewSecretString(os.Getenv("RESTIC_TEST_B2_ACCOUNT_KEY"))
cfg.Prefix = fmt.Sprintf("test-%d", time.Now().UnixNano())
return cfg, nil
},

2
internal/backend/b2/config.go
View File

@ -13,7 +13,7 @@ import (
// server.
type Config struct {
AccountID string
Key string
Key options.SecretString
Bucket string
Prefix string

15
internal/backend/s3/config.go
View File

@ -12,13 +12,14 @@ import (
// Config contains all configuration necessary to connect to an s3 compatible
// server.
type Config struct {
Endpoint string
UseHTTP bool
KeyID, Secret string
Bucket string
Prefix string
Layout string `option:"layout" help:"use this backend layout (default: auto-detect)"`
StorageClass string `option:"storage-class" help:"set S3 storage class (STANDARD, STANDARD_IA, ONEZONE_IA, INTELLIGENT_TIERING or REDUCED_REDUNDANCY)"`
Endpoint string
UseHTTP bool
KeyID string
Secret options.SecretString
Bucket string
Prefix string
Layout string `option:"layout" help:"use this backend layout (default: auto-detect)"`
StorageClass string `option:"storage-class" help:"set S3 storage class (STANDARD, STANDARD_IA, ONEZONE_IA, INTELLIGENT_TIERING or REDUCED_REDUNDANCY)"`
Connections uint `option:"connections" help:"set a limit for the number of concurrent connections (default: 5)"`
MaxRetries uint `option:"retries" help:"set the number of retries attempted"`

2
internal/backend/s3/s3.go
View File

@ -57,7 +57,7 @@ func open(ctx context.Context, cfg Config, rt http.RoundTripper) (*Backend, erro
&credentials.Static{
Value: credentials.Value{
AccessKeyID: cfg.KeyID,
SecretAccessKey: cfg.Secret,
SecretAccessKey: cfg.Secret.Unwrap(),
},
},
&credentials.EnvMinio{},

5
internal/backend/s3/s3_test.go
View File

@ -18,6 +18,7 @@ import (
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/s3"
"github.com/restic/restic/internal/backend/test"
"github.com/restic/restic/internal/options"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
)
@ -141,7 +142,7 @@ func newMinioTestSuite(ctx context.Context, t testing.TB) *test.Suite {
cfg.Config.Prefix = fmt.Sprintf("test-%d", time.Now().UnixNano())
cfg.Config.UseHTTP = true
cfg.Config.KeyID = key
cfg.Config.Secret = secret
cfg.Config.Secret = options.NewSecretString(secret)
return cfg, nil
},
@ -239,7 +240,7 @@ func newS3TestSuite(t testing.TB) *test.Suite {
cfg := s3cfg.(s3.Config)
cfg.KeyID = os.Getenv("RESTIC_TEST_S3_KEY")
cfg.Secret = os.Getenv("RESTIC_TEST_S3_SECRET")
cfg.Secret = options.NewSecretString(os.Getenv("RESTIC_TEST_S3_SECRET"))
cfg.Prefix = fmt.Sprintf("test-%d", time.Now().UnixNano())
return cfg, nil
},

17
internal/backend/swift/config.go
View File

@ -24,12 +24,12 @@ type Config struct {
TrustID string
StorageURL string
AuthToken string
AuthToken options.SecretString
// auth v3 only
ApplicationCredentialID string
ApplicationCredentialName string
ApplicationCredentialSecret string
ApplicationCredentialSecret options.SecretString
Container string
Prefix string
@ -111,11 +111,9 @@ func ApplyEnvironment(prefix string, cfg interface{}) error {
// Application Credential auth
{&c.ApplicationCredentialID, prefix + "OS_APPLICATION_CREDENTIAL_ID"},
{&c.ApplicationCredentialName, prefix + "OS_APPLICATION_CREDENTIAL_NAME"},
{&c.ApplicationCredentialSecret, prefix + "OS_APPLICATION_CREDENTIAL_SECRET"},
// Manual authentication
{&c.StorageURL, prefix + "OS_STORAGE_URL"},
{&c.AuthToken, prefix + "OS_AUTH_TOKEN"},
{&c.DefaultContainerPolicy, prefix + "SWIFT_DEFAULT_CONTAINER_POLICY"},
} {
@ -123,5 +121,16 @@ func ApplyEnvironment(prefix string, cfg interface{}) error {
*val.s = os.Getenv(val.env)
}
}
for _, val := range []struct {
s *options.SecretString
env string
}{
{&c.ApplicationCredentialSecret, prefix + "OS_APPLICATION_CREDENTIAL_SECRET"},
{&c.AuthToken, prefix + "OS_AUTH_TOKEN"},
} {
if val.s.String() == "" {
*val.s = options.NewSecretString(os.Getenv(val.env))
}
}
return nil
}

4
internal/backend/swift/swift.go
View File

@ -61,10 +61,10 @@ func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend
TenantDomainId: cfg.TenantDomainID,
TrustId: cfg.TrustID,
StorageUrl: cfg.StorageURL,
AuthToken: cfg.AuthToken,
AuthToken: cfg.AuthToken.Unwrap(),
ApplicationCredentialId: cfg.ApplicationCredentialID,
ApplicationCredentialName: cfg.ApplicationCredentialName,
ApplicationCredentialSecret: cfg.ApplicationCredentialSecret,
ApplicationCredentialSecret: cfg.ApplicationCredentialSecret.Unwrap(),
ConnectTimeout: time.Minute,
Timeout: time.Minute,

29
internal/debug/round_tripper_debug.go
View File

@ -75,7 +75,32 @@ func RoundTripper(upstream http.RoundTripper) http.RoundTripper {
return eofRoundTripper
}
func redactHeader(header http.Header) map[string][]string {
removedHeaders := make(map[string][]string)
for _, hdr := range []string{
"Authorization",
"X-Auth-Token", // Swift headers
"X-Auth-Key",
} {
origHeader, hasHeader := header[hdr]
if hasHeader {
removedHeaders[hdr] = origHeader
header[hdr] = []string{"**redacted**"}
}
}
return removedHeaders
}
func restoreHeader(header http.Header, origHeaders map[string][]string) {
for hdr, val := range origHeaders {
header[hdr] = val
}
}
func (tr loggingRoundTripper) RoundTrip(req *http.Request) (res *http.Response, err error) {
// save original auth and redact it
origHeaders := redactHeader(req.Header)
trace, err := httputil.DumpRequestOut(req, false)
if err != nil {
Log("DumpRequestOut() error: %v\n", err)
@ -83,13 +108,17 @@ func (tr loggingRoundTripper) RoundTrip(req *http.Request) (res *http.Response,
Log("------------ HTTP REQUEST -----------\n%s", trace)
}
restoreHeader(req.Header, origHeaders)
res, err = tr.RoundTripper.RoundTrip(req)
if err != nil {
Log("RoundTrip() returned error: %v", err)
}
if res != nil {
origHeaders := redactHeader(res.Header)
trace, err := httputil.DumpResponse(res, false)
restoreHeader(res.Header, origHeaders)
if err != nil {
Log("DumpResponse() error: %v\n", err)
} else {

46
internal/debug/round_tripper_debug_test.go Normal file
View File

@ -0,0 +1,46 @@
// +build debug
package debug
import (
"net/http"
"testing"
"github.com/restic/restic/internal/test"
)
func TestRedactHeader(t *testing.T) {
secretHeaders := []string{
"Authorization",
"X-Auth-Token",
"X-Auth-Key",
}
header := make(http.Header)
header["Authorization"] = []string{"123"}
header["X-Auth-Token"] = []string{"1234"}
header["X-Auth-Key"] = []string{"12345"}
header["Host"] = []string{"my.host"}
origHeaders := redactHeader(header)
for _, hdr := range secretHeaders {
test.Equals(t, "**redacted**", header[hdr][0])
}
test.Equals(t, "my.host", header["Host"][0])
restoreHeader(header, origHeaders)
test.Equals(t, "123", header["Authorization"][0])
test.Equals(t, "1234", header["X-Auth-Token"][0])
test.Equals(t, "12345", header["X-Auth-Key"][0])
test.Equals(t, "my.host", header["Host"][0])
delete(header, "X-Auth-Key")
origHeaders = redactHeader(header)
_, hasHeader := header["X-Auth-Key"]
test.Assert(t, !hasHeader, "Unexpected header: %v", header["X-Auth-Key"])
restoreHeader(header, origHeaders)
_, hasHeader = header["X-Auth-Key"]
test.Assert(t, !hasHeader, "Unexpected header: %v", header["X-Auth-Key"])
}

24
internal/options/secret_string.go Normal file
View File

@ -0,0 +1,24 @@
package options
type SecretString struct {
s *string
}
func NewSecretString(s string) SecretString {
return SecretString{s: &s}
}
func (s SecretString) GoString() string {
return `"` + s.String() + `"`
}
func (s SecretString) String() string {
if len(*s.s) == 0 {
return ``
}
return `**redacted**`
}
func (s *SecretString) Unwrap() string {
return *s.s
}

55
internal/options/secret_string_test.go Normal file
View File

@ -0,0 +1,55 @@
package options_test
import (
"fmt"
"strings"
"testing"
"github.com/restic/restic/internal/options"
"github.com/restic/restic/internal/test"
)
type secretTest struct {
str options.SecretString
}
func assertNotIn(t *testing.T, str string, substr string) {
if strings.Contains(str, substr) {
t.Fatalf("'%s' should not contain '%s'", str, substr)
}
}
func TestSecretString(t *testing.T) {
keyStr := "secret-key"
secret := options.NewSecretString(keyStr)
test.Equals(t, "**redacted**", secret.String())
test.Equals(t, `"**redacted**"`, secret.GoString())
test.Equals(t, "**redacted**", fmt.Sprint(secret))
test.Equals(t, "**redacted**", fmt.Sprintf("%v", secret))
test.Equals(t, `"**redacted**"`, fmt.Sprintf("%#v", secret))
test.Equals(t, keyStr, secret.Unwrap())
}
func TestSecretStringStruct(t *testing.T) {
keyStr := "secret-key"
secretStruct := &secretTest{
str: options.NewSecretString(keyStr),
}
assertNotIn(t, fmt.Sprint(secretStruct), keyStr)
assertNotIn(t, fmt.Sprintf("%v", secretStruct), keyStr)
assertNotIn(t, fmt.Sprintf("%#v", secretStruct), keyStr)
}
func TestSecretStringEmpty(t *testing.T) {
keyStr := ""
secret := options.NewSecretString(keyStr)
test.Equals(t, "", secret.String())
test.Equals(t, `""`, secret.GoString())
test.Equals(t, "", fmt.Sprint(secret))
test.Equals(t, "", fmt.Sprintf("%v", secret))
test.Equals(t, `""`, fmt.Sprintf("%#v", secret))
test.Equals(t, keyStr, secret.Unwrap())
}