From 2f281799c16ac9f32563e6fba513e3a0b846bad6 Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Mon, 3 Jun 2024 07:16:56 +0200 Subject: [PATCH] cmd/stcrashreceiver: Ignore patterns, improve metrics --- cmd/stcrashreceiver/main.go | 90 ++++++++++++++++--- cmd/stcrashreceiver/sentry.go | 117 ++++--------------------- cmd/stcrashreceiver/sentry_test.go | 60 ------------- cmd/stcrashreceiver/stcrashreceiver.go | 31 +++++++ lib/build/parse.go | 93 ++++++++++++++++++++ lib/build/parse_test.go | 72 +++++++++++++++ 6 files changed, 294 insertions(+), 169 deletions(-) create mode 100644 lib/build/parse.go create mode 100644 lib/build/parse_test.go diff --git a/cmd/stcrashreceiver/main.go b/cmd/stcrashreceiver/main.go index 657d19e91..1a22313ab 100644 --- a/cmd/stcrashreceiver/main.go +++ b/cmd/stcrashreceiver/main.go @@ -21,11 +21,14 @@ import ( "net/http" "os" "path/filepath" + "regexp" + "strings" "github.com/alecthomas/kong" raven "github.com/getsentry/raven-go" "github.com/prometheus/client_golang/prometheus/promhttp" _ "github.com/syncthing/syncthing/lib/automaxprocs" + "github.com/syncthing/syncthing/lib/build" "github.com/syncthing/syncthing/lib/sha256" "github.com/syncthing/syncthing/lib/ur" ) @@ -33,13 +36,15 @@ import ( const maxRequestSize = 1 << 20 // 1 MiB type cli struct { - Dir string `help:"Parent directory to store crash and failure reports in" env:"REPORTS_DIR" default:"."` - DSN string `help:"Sentry DSN" env:"SENTRY_DSN"` - Listen string `help:"HTTP listen address" default:":8080" env:"LISTEN_ADDRESS"` - MaxDiskFiles int `help:"Maximum number of reports on disk" default:"100000" env:"MAX_DISK_FILES"` - MaxDiskSizeMB int64 `help:"Maximum disk space to use for reports" default:"1024" env:"MAX_DISK_SIZE_MB"` - SentryQueue int `help:"Maximum number of reports to queue for sending to Sentry" default:"64" env:"SENTRY_QUEUE"` - DiskQueue int `help:"Maximum number of reports to queue for writing to disk" default:"64" env:"DISK_QUEUE"` + Dir string `help:"Parent directory to store crash and failure reports in" env:"REPORTS_DIR" default:"."` + DSN string `help:"Sentry DSN" env:"SENTRY_DSN"` + Listen string `help:"HTTP listen address" default:":8080" env:"LISTEN_ADDRESS"` + MaxDiskFiles int `help:"Maximum number of reports on disk" default:"100000" env:"MAX_DISK_FILES"` + MaxDiskSizeMB int64 `help:"Maximum disk space to use for reports" default:"1024" env:"MAX_DISK_SIZE_MB"` + SentryQueue int `help:"Maximum number of reports to queue for sending to Sentry" default:"64" env:"SENTRY_QUEUE"` + DiskQueue int `help:"Maximum number of reports to queue for writing to disk" default:"64" env:"DISK_QUEUE"` + MetricsListen string `help:"HTTP listen address for metrics" default:":8081" env:"METRICS_LISTEN_ADDRESS"` + IngorePatterns string `help:"File containing ignore patterns (regexp)" env:"IGNORE_PATTERNS" type:"existingfile"` } func main() { @@ -62,19 +67,38 @@ func main() { } go ss.Serve(context.Background()) + var ip *ignorePatterns + if params.IngorePatterns != "" { + var err error + ip, err = loadIgnorePatterns(params.IngorePatterns) + if err != nil { + log.Fatalf("Failed to load ignore patterns: %v", err) + } + } + cr := &crashReceiver{ store: ds, sentry: ss, + ignore: ip, } mux.Handle("/", cr) mux.HandleFunc("/ping", func(w http.ResponseWriter, req *http.Request) { w.Write([]byte("OK")) }) - mux.Handle("/metrics", promhttp.Handler()) + + if params.MetricsListen != "" { + mmux := http.NewServeMux() + mmux.Handle("/metrics", promhttp.Handler()) + go func() { + if err := http.ListenAndServe(params.MetricsListen, mmux); err != nil { + log.Fatalln("HTTP serve metrics:", err) + } + }() + } if params.DSN != "" { - mux.HandleFunc("/newcrash/failure", handleFailureFn(params.DSN, filepath.Join(params.Dir, "failure_reports"))) + mux.HandleFunc("/newcrash/failure", handleFailureFn(params.DSN, filepath.Join(params.Dir, "failure_reports"), ip)) } log.SetOutput(os.Stdout) @@ -83,7 +107,7 @@ func main() { } } -func handleFailureFn(dsn, failureDir string) func(w http.ResponseWriter, req *http.Request) { +func handleFailureFn(dsn, failureDir string, ignore *ignorePatterns) func(w http.ResponseWriter, req *http.Request) { return func(w http.ResponseWriter, req *http.Request) { result := "failure" defer func() { @@ -98,6 +122,11 @@ func handleFailureFn(dsn, failureDir string) func(w http.ResponseWriter, req *ht return } + if ignore.match(bs) { + result = "ignored" + return + } + var reports []ur.FailureReport err = json.Unmarshal(bs, &reports) if err != nil { @@ -110,7 +139,7 @@ func handleFailureFn(dsn, failureDir string) func(w http.ResponseWriter, req *ht return } - version, err := parseVersion(reports[0].Version) + version, err := build.ParseVersion(reports[0].Version) if err != nil { http.Error(w, err.Error(), 400) return @@ -158,3 +187,42 @@ func saveFailureWithGoroutines(data ur.FailureData, failureDir string) (string, } return reportServer + path, nil } + +type ignorePatterns struct { + patterns []*regexp.Regexp +} + +func loadIgnorePatterns(path string) (*ignorePatterns, error) { + bs, err := os.ReadFile(path) + if err != nil { + return nil, err + } + + var patterns []*regexp.Regexp + for _, line := range strings.Split(string(bs), "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + re, err := regexp.Compile(line) + if err != nil { + return nil, err + } + patterns = append(patterns, re) + } + + log.Printf("Loaded %d ignore patterns", len(patterns)) + return &ignorePatterns{patterns: patterns}, nil +} + +func (i *ignorePatterns) match(report []byte) bool { + if i == nil { + return false + } + for _, re := range i.patterns { + if re.Match(report) { + return true + } + } + return false +} diff --git a/cmd/stcrashreceiver/sentry.go b/cmd/stcrashreceiver/sentry.go index dadd9f333..f8bc22741 100644 --- a/cmd/stcrashreceiver/sentry.go +++ b/cmd/stcrashreceiver/sentry.go @@ -18,6 +18,7 @@ import ( raven "github.com/getsentry/raven-go" "github.com/maruel/panicparse/v2/stack" + "github.com/syncthing/syncthing/lib/build" ) const reportServer = "https://crash.syncthing.net/report/" @@ -105,7 +106,7 @@ func parseCrashReport(path string, report []byte) (*raven.Packet, error) { return nil, errors.New("no first line") } - version, err := parseVersion(string(parts[0])) + version, err := build.ParseVersion(string(parts[0])) if err != nil { return nil, err } @@ -143,12 +144,12 @@ func parseCrashReport(path string, report []byte) (*raven.Packet, error) { } // Lock the source code loader to the version we are processing here. - if version.commit != "" { + if version.Commit != "" { // We have a commit hash, so we know exactly which source to use - loader.LockWithVersion(version.commit) - } else if strings.HasPrefix(version.tag, "v") { + loader.LockWithVersion(version.Commit) + } else if strings.HasPrefix(version.Tag, "v") { // Lets hope the tag is close enough - loader.LockWithVersion(version.tag) + loader.LockWithVersion(version.Tag) } else { // Last resort loader.LockWithVersion("main") @@ -215,106 +216,26 @@ func crashReportFingerprint(message string) []string { return []string{"{{ default }}", message} } -// syncthing v1.1.4-rc.1+30-g6aaae618-dirty-crashrep "Erbium Earthworm" (go1.12.5 darwin-amd64) jb@kvin.kastelo.net 2019-05-23 16:08:14 UTC [foo, bar] -// or, somewhere along the way the "+" in the version tag disappeared: -// syncthing v1.23.7-dev.26.gdf7b56ae.dirty-stversionextra "Fermium Flea" (go1.20.5 darwin-arm64) jb@ok.kastelo.net 2023-07-12 06:55:26 UTC [Some Wrapper, purego, stnoupgrade] -var ( - longVersionRE = regexp.MustCompile(`syncthing\s+(v[^\s]+)\s+"([^"]+)"\s\(([^\s]+)\s+([^-]+)-([^)]+)\)\s+([^\s]+)[^\[]*(?:\[(.+)\])?$`) - gitExtraRE = regexp.MustCompile(`\.\d+\.g[0-9a-f]+`) // ".1.g6aaae618" - gitExtraSepRE = regexp.MustCompile(`[.-]`) // dot or dash -) - -type version struct { - version string // "v1.1.4-rc.1+30-g6aaae618-dirty-crashrep" - tag string // "v1.1.4-rc.1" - commit string // "6aaae618", blank when absent - codename string // "Erbium Earthworm" - runtime string // "go1.12.5" - goos string // "darwin" - goarch string // "amd64" - builder string // "jb@kvin.kastelo.net" - extra []string // "foo", "bar" -} - -func (v version) environment() string { - if v.commit != "" { - return "Development" - } - if strings.Contains(v.tag, "-rc.") { - return "Candidate" - } - if strings.Contains(v.tag, "-") { - return "Beta" - } - return "Stable" -} - -func parseVersion(line string) (version, error) { - m := longVersionRE.FindStringSubmatch(line) - if len(m) == 0 { - return version{}, errors.New("unintelligeble version string") - } - - v := version{ - version: m[1], - codename: m[2], - runtime: m[3], - goos: m[4], - goarch: m[5], - builder: m[6], - } - - // Split the version tag into tag and commit. This is old style - // v1.2.3-something.4+11-g12345678 or newer with just dots - // v1.2.3-something.4.11.g12345678 or v1.2.3-dev.11.g12345678. - parts := []string{v.version} - if strings.Contains(v.version, "+") { - parts = strings.Split(v.version, "+") - } else { - idxs := gitExtraRE.FindStringIndex(v.version) - if len(idxs) > 0 { - parts = []string{v.version[:idxs[0]], v.version[idxs[0]+1:]} - } - } - v.tag = parts[0] - if len(parts) > 1 { - fields := gitExtraSepRE.Split(parts[1], -1) - if len(fields) >= 2 && strings.HasPrefix(fields[1], "g") { - v.commit = fields[1][1:] - } - } - - if len(m) >= 8 && m[7] != "" { - tags := strings.Split(m[7], ",") - for i := range tags { - tags[i] = strings.TrimSpace(tags[i]) - } - v.extra = tags - } - - return v, nil -} - -func packet(version version, reportType string) *raven.Packet { +func packet(version build.VersionParts, reportType string) *raven.Packet { pkt := &raven.Packet{ Platform: "go", - Release: version.tag, - Environment: version.environment(), + Release: version.Tag, + Environment: version.Environment(), Tags: raven.Tags{ - raven.Tag{Key: "version", Value: version.version}, - raven.Tag{Key: "tag", Value: version.tag}, - raven.Tag{Key: "codename", Value: version.codename}, - raven.Tag{Key: "runtime", Value: version.runtime}, - raven.Tag{Key: "goos", Value: version.goos}, - raven.Tag{Key: "goarch", Value: version.goarch}, - raven.Tag{Key: "builder", Value: version.builder}, + raven.Tag{Key: "version", Value: version.Version}, + raven.Tag{Key: "tag", Value: version.Tag}, + raven.Tag{Key: "codename", Value: version.Codename}, + raven.Tag{Key: "runtime", Value: version.Runtime}, + raven.Tag{Key: "goos", Value: version.GOOS}, + raven.Tag{Key: "goarch", Value: version.GOARCH}, + raven.Tag{Key: "builder", Value: version.Builder}, raven.Tag{Key: "report_type", Value: reportType}, }, } - if version.commit != "" { - pkt.Tags = append(pkt.Tags, raven.Tag{Key: "commit", Value: version.commit}) + if version.Commit != "" { + pkt.Tags = append(pkt.Tags, raven.Tag{Key: "commit", Value: version.Commit}) } - for _, tag := range version.extra { + for _, tag := range version.Extra { pkt.Tags = append(pkt.Tags, raven.Tag{Key: tag, Value: "1"}) } return pkt diff --git a/cmd/stcrashreceiver/sentry_test.go b/cmd/stcrashreceiver/sentry_test.go index 9fa30f262..f087641e4 100644 --- a/cmd/stcrashreceiver/sentry_test.go +++ b/cmd/stcrashreceiver/sentry_test.go @@ -12,66 +12,6 @@ import ( "testing" ) -func TestParseVersion(t *testing.T) { - cases := []struct { - longVersion string - parsed version - }{ - { - longVersion: `syncthing v1.1.4-rc.1+30-g6aaae618-dirty-crashrep "Erbium Earthworm" (go1.12.5 darwin-amd64) jb@kvin.kastelo.net 2019-05-23 16:08:14 UTC`, - parsed: version{ - version: "v1.1.4-rc.1+30-g6aaae618-dirty-crashrep", - tag: "v1.1.4-rc.1", - commit: "6aaae618", - codename: "Erbium Earthworm", - runtime: "go1.12.5", - goos: "darwin", - goarch: "amd64", - builder: "jb@kvin.kastelo.net", - }, - }, - { - longVersion: `syncthing v1.1.4-rc.1+30-g6aaae618-dirty-crashrep "Erbium Earthworm" (go1.12.5 darwin-amd64) jb@kvin.kastelo.net 2019-05-23 16:08:14 UTC [foo, bar]`, - parsed: version{ - version: "v1.1.4-rc.1+30-g6aaae618-dirty-crashrep", - tag: "v1.1.4-rc.1", - commit: "6aaae618", - codename: "Erbium Earthworm", - runtime: "go1.12.5", - goos: "darwin", - goarch: "amd64", - builder: "jb@kvin.kastelo.net", - extra: []string{"foo", "bar"}, - }, - }, - { - longVersion: `syncthing v1.23.7-dev.26.gdf7b56ae-stversionextra "Fermium Flea" (go1.20.5 darwin-arm64) jb@ok.kastelo.net 2023-07-12 06:55:26 UTC [Some Wrapper, purego, stnoupgrade]`, - parsed: version{ - version: "v1.23.7-dev.26.gdf7b56ae-stversionextra", - tag: "v1.23.7-dev", - commit: "df7b56ae", - codename: "Fermium Flea", - runtime: "go1.20.5", - goos: "darwin", - goarch: "arm64", - builder: "jb@ok.kastelo.net", - extra: []string{"Some Wrapper", "purego", "stnoupgrade"}, - }, - }, - } - - for _, tc := range cases { - v, err := parseVersion(tc.longVersion) - if err != nil { - t.Errorf("%s\nerror: %v\n", tc.longVersion, err) - continue - } - if fmt.Sprint(v) != fmt.Sprint(tc.parsed) { - t.Errorf("%s\nA: %v\nE: %v\n", tc.longVersion, v, tc.parsed) - } - } -} - func TestParseReport(t *testing.T) { bs, err := os.ReadFile("_testdata/panic.log") if err != nil { diff --git a/cmd/stcrashreceiver/stcrashreceiver.go b/cmd/stcrashreceiver/stcrashreceiver.go index d8c0d2965..d45e057ed 100644 --- a/cmd/stcrashreceiver/stcrashreceiver.go +++ b/cmd/stcrashreceiver/stcrashreceiver.go @@ -12,11 +12,16 @@ import ( "net/http" "path" "strings" + "sync" ) type crashReceiver struct { store *diskStore sentry *sentryService + ignore *ignorePatterns + + ignoredMut sync.RWMutex + ignored map[string]struct{} } func (r *crashReceiver) ServeHTTP(w http.ResponseWriter, req *http.Request) { @@ -64,6 +69,12 @@ func (r *crashReceiver) serveGet(reportID string, w http.ResponseWriter, _ *http // serveHead responds to HEAD requests by checking if the named report // already exists in the system. func (r *crashReceiver) serveHead(reportID string, w http.ResponseWriter, _ *http.Request) { + r.ignoredMut.RLock() + _, ignored := r.ignored[reportID] + r.ignoredMut.RUnlock() + if ignored { + return // found + } if !r.store.Exists(reportID) { http.Error(w, "Not found", http.StatusNotFound) } @@ -76,6 +87,15 @@ func (r *crashReceiver) servePut(reportID string, w http.ResponseWriter, req *ht metricCrashReportsTotal.WithLabelValues(result).Inc() }() + r.ignoredMut.RLock() + _, ignored := r.ignored[reportID] + r.ignoredMut.RUnlock() + if ignored { + result = "ignored_cached" + io.Copy(io.Discard, req.Body) + return // found + } + // Read at most maxRequestSize of report data. log.Println("Receiving report", reportID) lr := io.LimitReader(req.Body, maxRequestSize) @@ -86,6 +106,17 @@ func (r *crashReceiver) servePut(reportID string, w http.ResponseWriter, req *ht return } + if r.ignore.match(bs) { + r.ignoredMut.Lock() + if r.ignored == nil { + r.ignored = make(map[string]struct{}) + } + r.ignored[reportID] = struct{}{} + r.ignoredMut.Unlock() + result = "ignored" + return + } + result = "success" // Store the report diff --git a/lib/build/parse.go b/lib/build/parse.go new file mode 100644 index 000000000..d130b099b --- /dev/null +++ b/lib/build/parse.go @@ -0,0 +1,93 @@ +// Copyright (C) 2019 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at https://mozilla.org/MPL/2.0/. + +package build + +import ( + "errors" + "regexp" + "strings" +) + +// syncthing v1.1.4-rc.1+30-g6aaae618-dirty-crashrep "Erbium Earthworm" (go1.12.5 darwin-amd64) jb@kvin.kastelo.net 2019-05-23 16:08:14 UTC [foo, bar] +// or, somewhere along the way the "+" in the version tag disappeared: +// syncthing v1.23.7-dev.26.gdf7b56ae.dirty-stversionextra "Fermium Flea" (go1.20.5 darwin-arm64) jb@ok.kastelo.net 2023-07-12 06:55:26 UTC [Some Wrapper, purego, stnoupgrade] +var ( + longVersionRE = regexp.MustCompile(`syncthing\s+(v[^\s]+)\s+"([^"]+)"\s\(([^\s]+)\s+([^-]+)-([^)]+)\)\s+([^\s]+)[^\[]*(?:\[(.+)\])?$`) + gitExtraRE = regexp.MustCompile(`\.\d+\.g[0-9a-f]+`) // ".1.g6aaae618" + gitExtraSepRE = regexp.MustCompile(`[.-]`) // dot or dash +) + +type VersionParts struct { + Version string // "v1.1.4-rc.1+30-g6aaae618-dirty-crashrep" + Tag string // "v1.1.4-rc.1" + Commit string // "6aaae618", blank when absent + Codename string // "Erbium Earthworm" + Runtime string // "go1.12.5" + GOOS string // "darwin" + GOARCH string // "amd64" + Builder string // "jb@kvin.kastelo.net" + Extra []string // "foo", "bar" +} + +func (v VersionParts) Environment() string { + if v.Commit != "" { + return "Development" + } + if strings.Contains(v.Tag, "-rc.") { + return "Candidate" + } + if strings.Contains(v.Tag, "-") { + return "Beta" + } + return "Stable" +} + +func ParseVersion(line string) (VersionParts, error) { + m := longVersionRE.FindStringSubmatch(line) + if len(m) == 0 { + return VersionParts{}, errors.New("unintelligeble version string") + } + + v := VersionParts{ + Version: m[1], + Codename: m[2], + Runtime: m[3], + GOOS: m[4], + GOARCH: m[5], + Builder: m[6], + } + + // Split the version tag into tag and commit. This is old style + // v1.2.3-something.4+11-g12345678 or newer with just dots + // v1.2.3-something.4.11.g12345678 or v1.2.3-dev.11.g12345678. + parts := []string{v.Version} + if strings.Contains(v.Version, "+") { + parts = strings.Split(v.Version, "+") + } else { + idxs := gitExtraRE.FindStringIndex(v.Version) + if len(idxs) > 0 { + parts = []string{v.Version[:idxs[0]], v.Version[idxs[0]+1:]} + } + } + v.Tag = parts[0] + if len(parts) > 1 { + fields := gitExtraSepRE.Split(parts[1], -1) + if len(fields) >= 2 && strings.HasPrefix(fields[1], "g") { + v.Commit = fields[1][1:] + } + } + + if len(m) >= 8 && m[7] != "" { + tags := strings.Split(m[7], ",") + for i := range tags { + tags[i] = strings.TrimSpace(tags[i]) + } + v.Extra = tags + } + + return v, nil +} diff --git a/lib/build/parse_test.go b/lib/build/parse_test.go new file mode 100644 index 000000000..5cab988dd --- /dev/null +++ b/lib/build/parse_test.go @@ -0,0 +1,72 @@ +// Copyright (C) 2019 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at https://mozilla.org/MPL/2.0/. + +package build + +import ( + "fmt" + "testing" +) + +func TestParseVersion(t *testing.T) { + cases := []struct { + longVersion string + parsed VersionParts + }{ + { + longVersion: `syncthing v1.1.4-rc.1+30-g6aaae618-dirty-crashrep "Erbium Earthworm" (go1.12.5 darwin-amd64) jb@kvin.kastelo.net 2019-05-23 16:08:14 UTC`, + parsed: VersionParts{ + Version: "v1.1.4-rc.1+30-g6aaae618-dirty-crashrep", + Tag: "v1.1.4-rc.1", + Commit: "6aaae618", + Codename: "Erbium Earthworm", + Runtime: "go1.12.5", + GOOS: "darwin", + GOARCH: "amd64", + Builder: "jb@kvin.kastelo.net", + }, + }, + { + longVersion: `syncthing v1.1.4-rc.1+30-g6aaae618-dirty-crashrep "Erbium Earthworm" (go1.12.5 darwin-amd64) jb@kvin.kastelo.net 2019-05-23 16:08:14 UTC [foo, bar]`, + parsed: VersionParts{ + Version: "v1.1.4-rc.1+30-g6aaae618-dirty-crashrep", + Tag: "v1.1.4-rc.1", + Commit: "6aaae618", + Codename: "Erbium Earthworm", + Runtime: "go1.12.5", + GOOS: "darwin", + GOARCH: "amd64", + Builder: "jb@kvin.kastelo.net", + Extra: []string{"foo", "bar"}, + }, + }, + { + longVersion: `syncthing v1.23.7-dev.26.gdf7b56ae-stversionextra "Fermium Flea" (go1.20.5 darwin-arm64) jb@ok.kastelo.net 2023-07-12 06:55:26 UTC [Some Wrapper, purego, stnoupgrade]`, + parsed: VersionParts{ + Version: "v1.23.7-dev.26.gdf7b56ae-stversionextra", + Tag: "v1.23.7-dev", + Commit: "df7b56ae", + Codename: "Fermium Flea", + Runtime: "go1.20.5", + GOOS: "darwin", + GOARCH: "arm64", + Builder: "jb@ok.kastelo.net", + Extra: []string{"Some Wrapper", "purego", "stnoupgrade"}, + }, + }, + } + + for _, tc := range cases { + v, err := ParseVersion(tc.longVersion) + if err != nil { + t.Errorf("%s\nerror: %v\n", tc.longVersion, err) + continue + } + if fmt.Sprint(v) != fmt.Sprint(tc.parsed) { + t.Errorf("%s\nA: %v\nE: %v\n", tc.longVersion, v, tc.parsed) + } + } +}