From 82c268c917d22fc9f0542bf098a877d203b682be Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 16 Jul 2022 21:35:03 +0200 Subject: [PATCH 1/5] Remove unused hooks mechanism --- internal/debug/hooks.go | 29 ----------------------------- internal/debug/hooks_release.go | 10 ---------- 2 files changed, 39 deletions(-) delete mode 100644 internal/debug/hooks.go delete mode 100644 internal/debug/hooks_release.go diff --git a/internal/debug/hooks.go b/internal/debug/hooks.go deleted file mode 100644 index f17b02701..000000000 --- a/internal/debug/hooks.go +++ /dev/null @@ -1,29 +0,0 @@ -//go:build debug -// +build debug - -package debug - -var ( - hooks map[string]func(interface{}) -) - -func init() { - hooks = make(map[string]func(interface{})) -} - -func Hook(name string, f func(interface{})) { - hooks[name] = f -} - -func RunHook(name string, context interface{}) { - f, ok := hooks[name] - if !ok { - return - } - - f(context) -} - -func RemoveHook(name string) { - delete(hooks, name) -} diff --git a/internal/debug/hooks_release.go b/internal/debug/hooks_release.go deleted file mode 100644 index fb7a5bdcd..000000000 --- a/internal/debug/hooks_release.go +++ /dev/null @@ -1,10 +0,0 @@ -//go:build !debug -// +build !debug - -package debug - -func Hook(name string, f func(interface{})) {} - -func RunHook(name string, context interface{}) {} - -func RemoveHook(name string) {} From 38becfc43606b0060227be125bc9d08e09898f8d Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 16 Jul 2022 21:36:19 +0200 Subject: [PATCH 2/5] debug: enable debug support for release builds --- internal/debug/debug.go | 3 --- internal/debug/debug_release.go | 7 ------- 2 files changed, 10 deletions(-) delete mode 100644 internal/debug/debug_release.go diff --git a/internal/debug/debug.go b/internal/debug/debug.go index 8464ba008..62c145e1a 100644 --- a/internal/debug/debug.go +++ b/internal/debug/debug.go @@ -1,6 +1,3 @@ -//go:build debug -// +build debug - package debug import ( diff --git a/internal/debug/debug_release.go b/internal/debug/debug_release.go deleted file mode 100644 index 29e2fb066..000000000 --- a/internal/debug/debug_release.go +++ /dev/null @@ -1,7 +0,0 @@ -//go:build !debug -// +build !debug - -package debug - -// Log prints a message to the debug log (if debug is enabled). -func Log(fmt string, args ...interface{}) {} From 1ed775e3a8029258c6ab468a7d36f10a7cecce5a Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 16 Jul 2022 21:41:37 +0200 Subject: [PATCH 3/5] debug: support roundtripper logging also for release builds Different from debug builds do not use the eofDetectRoundTripper if logging is disabled. --- internal/debug/round_tripper.go | 116 ++++++++++++++++++ internal/debug/round_tripper_debug.go | 115 +---------------- internal/debug/round_tripper_release.go | 4 + ...er_debug_test.go => round_tripper_test.go} | 2 - 4 files changed, 121 insertions(+), 116 deletions(-) create mode 100644 internal/debug/round_tripper.go rename internal/debug/{round_tripper_debug_test.go => round_tripper_test.go} (98%) diff --git a/internal/debug/round_tripper.go b/internal/debug/round_tripper.go new file mode 100644 index 000000000..6795d43d0 --- /dev/null +++ b/internal/debug/round_tripper.go @@ -0,0 +1,116 @@ +package debug + +import ( + "fmt" + "io" + "io/ioutil" + "net/http" + "net/http/httputil" + "os" + + "github.com/restic/restic/internal/errors" +) + +type eofDetectRoundTripper struct { + http.RoundTripper +} + +type eofDetectReader struct { + eofSeen bool + rd io.ReadCloser +} + +func (rd *eofDetectReader) Read(p []byte) (n int, err error) { + n, err = rd.rd.Read(p) + if err == io.EOF { + rd.eofSeen = true + } + return n, err +} + +func (rd *eofDetectReader) Close() error { + if !rd.eofSeen { + buf, err := ioutil.ReadAll(rd) + msg := fmt.Sprintf("body not drained, %d bytes not read", len(buf)) + if err != nil { + msg += fmt.Sprintf(", error: %v", err) + } + + if len(buf) > 0 { + if len(buf) > 20 { + buf = append(buf[:20], []byte("...")...) + } + msg += fmt.Sprintf(", body: %q", buf) + } + + fmt.Fprintln(os.Stderr, msg) + Log("%s: %+v", msg, errors.New("Close()")) + } + return rd.rd.Close() +} + +func (tr eofDetectRoundTripper) RoundTrip(req *http.Request) (res *http.Response, err error) { + res, err = tr.RoundTripper.RoundTrip(req) + if res != nil && res.Body != nil { + res.Body = &eofDetectReader{rd: res.Body} + } + return res, err +} + +type loggingRoundTripper struct { + http.RoundTripper +} + +func redactHeader(header http.Header) map[string][]string { + removedHeaders := make(map[string][]string) + for _, hdr := range []string{ + "Authorization", + "X-Auth-Token", // Swift headers + "X-Auth-Key", + } { + origHeader, hasHeader := header[hdr] + if hasHeader { + removedHeaders[hdr] = origHeader + header[hdr] = []string{"**redacted**"} + } + } + return removedHeaders +} + +func restoreHeader(header http.Header, origHeaders map[string][]string) { + for hdr, val := range origHeaders { + header[hdr] = val + } +} + +func (tr loggingRoundTripper) RoundTrip(req *http.Request) (res *http.Response, err error) { + // save original auth and redact it + origHeaders := redactHeader(req.Header) + + trace, err := httputil.DumpRequestOut(req, false) + if err != nil { + Log("DumpRequestOut() error: %v\n", err) + } else { + Log("------------ HTTP REQUEST -----------\n%s", trace) + } + + restoreHeader(req.Header, origHeaders) + + res, err = tr.RoundTripper.RoundTrip(req) + if err != nil { + Log("RoundTrip() returned error: %v", err) + } + + if res != nil { + origHeaders := redactHeader(res.Header) + trace, err := httputil.DumpResponse(res, false) + restoreHeader(res.Header, origHeaders) + if err != nil { + Log("DumpResponse() error: %v\n", err) + } else { + Log("------------ HTTP RESPONSE ----------\n%s", trace) + } + } + + return res, err +} diff --git a/internal/debug/round_tripper_debug.go b/internal/debug/round_tripper_debug.go index 020e798f0..df207207b 100644 --- a/internal/debug/round_tripper_debug.go +++ b/internal/debug/round_tripper_debug.go @@ -3,66 +3,7 @@ package debug -import ( - "fmt" - "io" - "io/ioutil" - "net/http" - "net/http/httputil" - "os" - - "github.com/restic/restic/internal/errors" -) - -type eofDetectRoundTripper struct { - http.RoundTripper -} - -type eofDetectReader struct { - eofSeen bool - rd io.ReadCloser -} - -func (rd *eofDetectReader) Read(p []byte) (n int, err error) { - n, err = rd.rd.Read(p) - if err == io.EOF { - rd.eofSeen = true - } - return n, err -} - -func (rd *eofDetectReader) Close() error { - if !rd.eofSeen { - buf, err := ioutil.ReadAll(rd) - msg := fmt.Sprintf("body not drained, %d bytes not read", len(buf)) - if err != nil { - msg += fmt.Sprintf(", error: %v", err) - } - - if len(buf) > 0 { - if len(buf) > 20 { - buf = append(buf[:20], []byte("...")...) - } - msg += fmt.Sprintf(", body: %q", buf) - } - - fmt.Fprintln(os.Stderr, msg) - Log("%s: %+v", msg, errors.New("Close()")) - } - return rd.rd.Close() -} - -func (tr eofDetectRoundTripper) RoundTrip(req *http.Request) (res *http.Response, err error) { - res, err = tr.RoundTripper.RoundTrip(req) - if res != nil && res.Body != nil { - res.Body = &eofDetectReader{rd: res.Body} - } - return res, err -} - -type loggingRoundTripper struct { - http.RoundTripper -} +import "net/http" // RoundTripper returns a new http.RoundTripper which logs all requests (if // debug is enabled). When debug is not enabled, upstream is returned. @@ -74,57 +15,3 @@ func RoundTripper(upstream http.RoundTripper) http.RoundTripper { } return eofRoundTripper } - -func redactHeader(header http.Header) map[string][]string { - removedHeaders := make(map[string][]string) - for _, hdr := range []string{ - "Authorization", - "X-Auth-Token", // Swift headers - "X-Auth-Key", - } { - origHeader, hasHeader := header[hdr] - if hasHeader { - removedHeaders[hdr] = origHeader - header[hdr] = []string{"**redacted**"} - } - } - return removedHeaders -} - -func restoreHeader(header http.Header, origHeaders map[string][]string) { - for hdr, val := range origHeaders { - header[hdr] = val - } -} - -func (tr loggingRoundTripper) RoundTrip(req *http.Request) (res *http.Response, err error) { - // save original auth and redact it - origHeaders := redactHeader(req.Header) - - trace, err := httputil.DumpRequestOut(req, false) - if err != nil { - Log("DumpRequestOut() error: %v\n", err) - } else { - Log("------------ HTTP REQUEST -----------\n%s", trace) - } - - restoreHeader(req.Header, origHeaders) - - res, err = tr.RoundTripper.RoundTrip(req) - if err != nil { - Log("RoundTrip() returned error: %v", err) - } - - if res != nil { - origHeaders := redactHeader(res.Header) - trace, err := httputil.DumpResponse(res, false) - restoreHeader(res.Header, origHeaders) - if err != nil { - Log("DumpResponse() error: %v\n", err) - } else { - Log("------------ HTTP RESPONSE ----------\n%s", trace) - } - } - - return res, err -} diff --git a/internal/debug/round_tripper_release.go b/internal/debug/round_tripper_release.go index 924c5c61e..6edadb479 100644 --- a/internal/debug/round_tripper_release.go +++ b/internal/debug/round_tripper_release.go @@ -8,5 +8,9 @@ import "net/http" // RoundTripper returns a new http.RoundTripper which logs all requests (if // debug is enabled). When debug is not enabled, upstream is returned. func RoundTripper(upstream http.RoundTripper) http.RoundTripper { + if opts.isEnabled { + // only use loggingRoundTripper if the debug log is configured + return loggingRoundTripper{eofDetectRoundTripper{upstream}} + } return upstream } diff --git a/internal/debug/round_tripper_debug_test.go b/internal/debug/round_tripper_test.go similarity index 98% rename from internal/debug/round_tripper_debug_test.go rename to internal/debug/round_tripper_test.go index 2095bbc6e..cc42a87d1 100644 --- a/internal/debug/round_tripper_debug_test.go +++ b/internal/debug/round_tripper_test.go @@ -1,5 +1,3 @@ -// +build debug - package debug import ( From b3cdee66a98eef61fa6abf4e9cf3caefcf76bed2 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 16 Jul 2022 22:45:41 +0200 Subject: [PATCH 4/5] update documentation to reflect DEBUG_LOG for release builds --- CONTRIBUTING.md | 5 ++--- doc/090_participating.rst | 31 ++++++++++++++++++++----------- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 02b6fa5a5..cf1f1e739 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -48,9 +48,8 @@ environment was used and so on. Please tell us at least the following things: Remember, the easier it is for us to reproduce the bug, the earlier it will be corrected! -In addition, you can compile restic with debug support by running -`go run build.go -tags debug` and instructing it to create a debug -log by setting the environment variable `DEBUG_LOG` to a file, e.g. like this: +In addition, you can instruct restic to create a debug log by setting the +environment variable `DEBUG_LOG` to a file, e.g. like this: $ export DEBUG_LOG=/tmp/restic-debug.log $ restic backup ~/work diff --git a/doc/090_participating.rst b/doc/090_participating.rst index 7c7e0e72b..00a387974 100644 --- a/doc/090_participating.rst +++ b/doc/090_participating.rst @@ -14,18 +14,12 @@ Participating ############# -********* -Debugging -********* +********** +Debug Logs +********** -The program can be built with debug support like this: - -.. code-block:: console - - $ go run build.go -tags debug - -Afterwards, extensive debug messages are written to the file in -environment variable ``DEBUG_LOG``, e.g.: +Set the environment variable ``DEBUG_LOG`` to let restic write extensive debug +messages to the specified filed, e.g.: .. code-block:: console @@ -66,6 +60,21 @@ statements originating in functions that match the pattern ``*unlock*`` $ DEBUG_FUNCS=*unlock* restic check +********* +Debugging +********* + +The program can be built with debug support like this: + +.. code-block:: console + + $ go run build.go -tags debug + +This will make the ``restic debug `` available which can be used to +inspect internal data structures. In addition, this enables profiling support +which can help with investigation performance and memory usage issues. + + ************ Contributing ************ From a3e48da3a3a26f34d95ee322c794b88eeb11f9e2 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 16 Jul 2022 22:46:04 +0200 Subject: [PATCH 5/5] Add changelog for DEBUG_LOG available in release builds --- changelog/unreleased/issue-1842 | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 changelog/unreleased/issue-1842 diff --git a/changelog/unreleased/issue-1842 b/changelog/unreleased/issue-1842 new file mode 100644 index 000000000..7c2c32098 --- /dev/null +++ b/changelog/unreleased/issue-1842 @@ -0,0 +1,8 @@ +Change: Support debug log creation in release builds + +Creating a debug log was only possible in debug builds which required users to +manually build restic. We changed the release builds to allow creating debug +logs by setting the environment variable `DEBUG_LOG=logname.log`. + +https://github.com/restic/restic/issues/1842 +https://github.com/restic/restic/pull/3826