From ba6ac2f604eb1cd27764460b687537c5e40aaaf8 Mon Sep 17 00:00:00 2001 From: Jakob Borg Date: Sat, 18 May 2024 19:31:49 +0200 Subject: [PATCH] lib/geoip, cmd/relaypoolsrv, cmd/ursrv: Automatically manage GeoIP updates (#9342) This adds a small package `geoip` which knows how to download and manage the Maxmind GeoLite2 database we use. This removes the need for various scripts to download and manage the geoip database, something that today happens on Docker startup for the relay pool server and using various hand written hacks for the usage reporting server. The database is downloaded when needed and then refreshed on a best-effort basis weekly. --- Dockerfile.strelaypoolsrv | 10 +-- cmd/strelaypoolsrv/main.go | 40 ++++----- cmd/ursrv/serve/serve.go | 43 +++++----- go.mod | 4 +- go.sum | 7 +- lib/geoip/geoip.go | 124 ++++++++++++++++++++++++++++ lib/geoip/geoip_test.go | 36 ++++++++ script/strelaypoolsrv-entrypoint.sh | 10 --- 8 files changed, 212 insertions(+), 62 deletions(-) create mode 100644 lib/geoip/geoip.go create mode 100644 lib/geoip/geoip_test.go delete mode 100755 script/strelaypoolsrv-entrypoint.sh diff --git a/Dockerfile.strelaypoolsrv b/Dockerfile.strelaypoolsrv index f7e2760e7..a0ad1fd6d 100644 --- a/Dockerfile.strelaypoolsrv +++ b/Dockerfile.strelaypoolsrv @@ -11,14 +11,6 @@ LABEL org.opencontainers.image.authors="The Syncthing Project" \ EXPOSE 8080 -RUN apk add --no-cache ca-certificates su-exec curl -ENV PUID=1000 PGID=1000 MAXMIND_KEY= - -RUN mkdir /var/strelaypoolsrv && chown 1000 /var/strelaypoolsrv -USER 1000 - COPY strelaypoolsrv-linux-${TARGETARCH} /bin/strelaypoolsrv -COPY script/strelaypoolsrv-entrypoint.sh /bin/entrypoint.sh -WORKDIR /var/strelaypoolsrv -ENTRYPOINT ["/bin/entrypoint.sh", "/bin/strelaypoolsrv", "-listen", ":8080"] +ENTRYPOINT ["/bin/strelaypoolsrv", "-listen", ":8080"] diff --git a/cmd/strelaypoolsrv/main.go b/cmd/strelaypoolsrv/main.go index 6d574e3e8..6f1e559ef 100644 --- a/cmd/strelaypoolsrv/main.go +++ b/cmd/strelaypoolsrv/main.go @@ -21,12 +21,12 @@ import ( "time" lru "github.com/hashicorp/golang-lru/v2" - "github.com/oschwald/geoip2-golang" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/syncthing/syncthing/cmd/strelaypoolsrv/auto" "github.com/syncthing/syncthing/lib/assets" _ "github.com/syncthing/syncthing/lib/automaxprocs" + "github.com/syncthing/syncthing/lib/geoip" "github.com/syncthing/syncthing/lib/httpcache" "github.com/syncthing/syncthing/lib/protocol" "github.com/syncthing/syncthing/lib/rand" @@ -100,11 +100,12 @@ var ( debug bool permRelaysFile string ipHeader string - geoipPath string proto string statsRefresh = time.Minute requestQueueLen = 64 requestProcessors = 8 + geoipLicenseKey = os.Getenv("GEOIP_LICENSE_KEY") + geoipAccountID, _ = strconv.Atoi(os.Getenv("GEOIP_ACCOUNT_ID")) requests chan request @@ -130,34 +131,38 @@ func main() { flag.StringVar(&permRelaysFile, "perm-relays", "", "Path to list of permanent relays") flag.StringVar(&knownRelaysFile, "known-relays", knownRelaysFile, "Path to list of current relays") flag.StringVar(&ipHeader, "ip-header", "", "Name of header which holds clients ip:port. Only meaningful when running behind a reverse proxy.") - flag.StringVar(&geoipPath, "geoip", "GeoLite2-City.mmdb", "Path to GeoLite2-City database") flag.StringVar(&proto, "protocol", "tcp", "Protocol used for listening. 'tcp' for IPv4 and IPv6, 'tcp4' for IPv4, 'tcp6' for IPv6") flag.DurationVar(&statsRefresh, "stats-refresh", statsRefresh, "Interval at which to refresh relay stats") flag.IntVar(&requestQueueLen, "request-queue", requestQueueLen, "Queue length for incoming test requests") flag.IntVar(&requestProcessors, "request-processors", requestProcessors, "Number of request processor routines") + flag.StringVar(&geoipLicenseKey, "geoip-license-key", geoipLicenseKey, "License key for GeoIP database") flag.Parse() requests = make(chan request, requestQueueLen) + geoip, err := geoip.NewGeoLite2CityProvider(context.Background(), geoipAccountID, geoipLicenseKey, os.TempDir()) + if err != nil { + log.Fatalln("Failed to create GeoIP provider:", err) + } + go geoip.Serve(context.TODO()) var listener net.Listener - var err error if permRelaysFile != "" { - permanentRelays = loadRelays(permRelaysFile) + permanentRelays = loadRelays(permRelaysFile, geoip) } testCert = createTestCertificate() for i := 0; i < requestProcessors; i++ { - go requestProcessor() + go requestProcessor(geoip) } // Load relays from cache in the background. // Load them in a serial fashion to make sure any genuine requests // are not dropped. go func() { - for _, relay := range loadRelays(knownRelaysFile) { + for _, relay := range loadRelays(knownRelaysFile, geoip) { resultChan := make(chan result) requests <- request{relay, resultChan, nil} result := <-resultChan @@ -425,19 +430,19 @@ func handlePostRequest(w http.ResponseWriter, r *http.Request) { } } -func requestProcessor() { +func requestProcessor(geoip *geoip.Provider) { for request := range requests { if request.queueTimer != nil { request.queueTimer.ObserveDuration() } timer := prometheus.NewTimer(relayTestActionsSeconds.WithLabelValues("test")) - handleRelayTest(request) + handleRelayTest(request, geoip) timer.ObserveDuration() } } -func handleRelayTest(request request) { +func handleRelayTest(request request, geoip *geoip.Provider) { if debug { log.Println("Request for", request.relay) } @@ -450,7 +455,7 @@ func handleRelayTest(request request) { } stats := fetchStats(request.relay) - location := getLocation(request.relay.uri.Host) + location := getLocation(request.relay.uri.Host, geoip) mut.Lock() if stats != nil { @@ -523,7 +528,7 @@ func evict(relay *relay) func() { } } -func loadRelays(file string) []*relay { +func loadRelays(file string, geoip *geoip.Provider) []*relay { content, err := os.ReadFile(file) if err != nil { log.Println("Failed to load relays: " + err.Error()) @@ -547,7 +552,7 @@ func loadRelays(file string) []*relay { relays = append(relays, &relay{ URL: line, - Location: getLocation(uri.Host), + Location: getLocation(uri.Host, geoip), uri: uri, }) if debug { @@ -580,21 +585,16 @@ func createTestCertificate() tls.Certificate { return cert } -func getLocation(host string) location { +func getLocation(host string, geoip *geoip.Provider) location { timer := prometheus.NewTimer(locationLookupSeconds) defer timer.ObserveDuration() - db, err := geoip2.Open(geoipPath) - if err != nil { - return location{} - } - defer db.Close() addr, err := net.ResolveTCPAddr("tcp", host) if err != nil { return location{} } - city, err := db.City(addr.IP) + city, err := geoip.City(addr.IP) if err != nil { return location{} } diff --git a/cmd/ursrv/serve/serve.go b/cmd/ursrv/serve/serve.go index b7d51db3b..14d3123b1 100644 --- a/cmd/ursrv/serve/serve.go +++ b/cmd/ursrv/serve/serve.go @@ -8,6 +8,7 @@ package serve import ( "bytes" + "context" "database/sql" "embed" "encoding/json" @@ -17,6 +18,7 @@ import ( "log" "net" "net/http" + "os" "regexp" "sort" "strconv" @@ -26,20 +28,21 @@ import ( "unicode" _ "github.com/lib/pq" // PostgreSQL driver - "github.com/oschwald/geoip2-golang" "github.com/prometheus/client_golang/prometheus/promhttp" "golang.org/x/text/cases" "golang.org/x/text/language" + "github.com/syncthing/syncthing/lib/geoip" "github.com/syncthing/syncthing/lib/upgrade" "github.com/syncthing/syncthing/lib/ur/contract" ) type CLI struct { - Debug bool `env:"UR_DEBUG"` - DBConn string `env:"UR_DB_URL" default:"postgres://user:password@localhost/ur?sslmode=disable"` - Listen string `env:"UR_LISTEN" default:"0.0.0.0:8080"` - GeoIPPath string `env:"UR_GEOIP" default:"GeoLite2-City.mmdb"` + Debug bool `env:"UR_DEBUG"` + DBConn string `env:"UR_DB_URL" default:"postgres://user:password@localhost/ur?sslmode=disable"` + Listen string `env:"UR_LISTEN" default:"0.0.0.0:8080"` + GeoIPLicenseKey string `env:"UR_GEOIP_LICENSE_KEY"` + GeoIPAccountID int `env:"UR_GEOIP_ACCOUNT_ID"` } //go:embed static @@ -189,10 +192,16 @@ func (cli *CLI) Run() error { log.Fatalln("listen:", err) } + geoip, err := geoip.NewGeoLite2CityProvider(context.Background(), cli.GeoIPAccountID, cli.GeoIPLicenseKey, os.TempDir()) + if err != nil { + log.Fatalln("geoip:", err) + } + go geoip.Serve(context.TODO()) + srv := &server{ - db: db, - debug: cli.Debug, - geoIPPath: cli.GeoIPPath, + db: db, + debug: cli.Debug, + geoip: geoip, } http.HandleFunc("/", srv.rootHandler) http.HandleFunc("/newdata", srv.newDataHandler) @@ -213,9 +222,9 @@ func (cli *CLI) Run() error { } type server struct { - debug bool - db *sql.DB - geoIPPath string + debug bool + db *sql.DB + geoip *geoip.Provider cacheMut sync.Mutex cachedIndex []byte @@ -238,7 +247,7 @@ func (s *server) cacheRefresher() { } func (s *server) refreshCacheLocked() error { - rep := getReport(s.db, s.geoIPPath) + rep := getReport(s.db, s.geoip) buf := new(bytes.Buffer) err := tpl.Execute(buf, rep) if err != nil { @@ -492,15 +501,7 @@ type weightedLocation struct { Weight int `json:"weight"` } -func getReport(db *sql.DB, geoIPPath string) map[string]interface{} { - geoip, err := geoip2.Open(geoIPPath) - if err != nil { - log.Println("opening geoip db", err) - geoip = nil - } else { - defer geoip.Close() - } - +func getReport(db *sql.DB, geoip *geoip.Provider) map[string]interface{} { nodes := 0 countriesTotal := 0 var versions []string diff --git a/go.mod b/go.mod index e0a89690f..3d5ec4426 100644 --- a/go.mod +++ b/go.mod @@ -24,6 +24,7 @@ require ( github.com/lib/pq v1.10.9 github.com/maruel/panicparse/v2 v2.3.1 github.com/maxbrunsfeld/counterfeiter/v6 v6.8.1 + github.com/maxmind/geoipupdate/v6 v6.1.0 github.com/minio/sha256-simd v1.0.1 github.com/miscreant/miscreant.go v0.0.0-20200214223636-26d376326b75 github.com/oschwald/geoip2-golang v1.9.0 @@ -52,6 +53,7 @@ require ( github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/certifi/gocertifi v0.0.0-20210507211836-431795d63e8d // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect @@ -60,13 +62,13 @@ require ( github.com/go-asn1-ber/asn1-ber v1.5.5 // indirect github.com/go-ole/go-ole v1.3.0 // indirect github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect + github.com/gofrs/flock v0.8.1 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/pprof v0.0.0-20240402174815-29b9bb013b0f // indirect github.com/google/uuid v1.6.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/klauspost/cpuid/v2 v2.2.7 // indirect - github.com/kr/text v0.2.0 // indirect github.com/nxadm/tail v1.4.11 // indirect github.com/onsi/ginkgo/v2 v2.17.1 // indirect github.com/onsi/gomega v1.31.1 // indirect diff --git a/go.sum b/go.sum index 7da5eb8c7..7b162b310 100644 --- a/go.sum +++ b/go.sum @@ -22,6 +22,8 @@ github.com/calmh/xdr v1.1.0 h1:U/Dd4CXNLoo8EiQ4ulJUXkgO1/EyQLgDKLgpY1SOoJE= github.com/calmh/xdr v1.1.0/go.mod h1:E8sz2ByAdXC8MbANf1LCRYzedSnnc+/sXXJs/PVqoeg= github.com/ccding/go-stun v0.1.4 h1:lC0co3Q3vjAuu2Jz098WivVPBPbemYFqbwE1syoka4M= github.com/ccding/go-stun v0.1.4/go.mod h1:cCZjJ1J3WFSJV6Wj8Y9Di8JMTsEXh6uv2eNmLzKaUeM= +github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= +github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/certifi/gocertifi v0.0.0-20210507211836-431795d63e8d h1:S2NE3iHSwP0XV47EEXL8mWmRdEfGscSJ+7EgePNgt0s= github.com/certifi/gocertifi v0.0.0-20210507211836-431795d63e8d/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= @@ -34,7 +36,6 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/d4l3k/messagediff v1.2.1 h1:ZcAIMYsUg0EAp9X+tt8/enBE/Q8Yd5kzPynLyKptt9U= github.com/d4l3k/messagediff v1.2.1/go.mod h1:Oozbb1TVXFac9FtSIxHBMnBCq2qeH/2KkEQxENCrlLo= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -62,6 +63,8 @@ github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= +github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= +github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -131,6 +134,8 @@ github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/maxbrunsfeld/counterfeiter/v6 v6.8.1 h1:NicmruxkeqHjDv03SfSxqmaLuisddudfP3h5wdXFbhM= github.com/maxbrunsfeld/counterfeiter/v6 v6.8.1/go.mod h1:eyp4DdUJAKkr9tvxR3jWhw2mDK7CWABMG5r9uyaKC7I= +github.com/maxmind/geoipupdate/v6 v6.1.0 h1:sdtTHzzQNJlXF5+fd/EoPTucRHyMonYt/Cok8xzzfqA= +github.com/maxmind/geoipupdate/v6 v6.1.0/go.mod h1:cZYCDzfMzTY4v6dKRdV7KTB6SStxtn3yFkiJ1btTGGc= github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= diff --git a/lib/geoip/geoip.go b/lib/geoip/geoip.go new file mode 100644 index 000000000..3400587e0 --- /dev/null +++ b/lib/geoip/geoip.go @@ -0,0 +1,124 @@ +// Copyright (C) 2024 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at https://mozilla.org/MPL/2.0/. + +// Package geoip provides an automatically updating MaxMind GeoIP2 database +// provider. +package geoip + +import ( + "context" + "errors" + "fmt" + "net" + "os" + "path/filepath" + "sync" + "time" + + "github.com/maxmind/geoipupdate/v6/pkg/geoipupdate" + "github.com/oschwald/geoip2-golang" +) + +type Provider struct { + edition string + accountID int + licenseKey string + refreshInterval time.Duration + directory string + + mut sync.Mutex + currentDBDir string + db *geoip2.Reader +} + +// NewGeoLite2CityProvider returns a new GeoIP2 database provider for the +// GeoLite2-City database. The database will be stored in the given +// directory (which should exist) and refreshed every 7 days. +func NewGeoLite2CityProvider(ctx context.Context, accountID int, licenseKey string, directory string) (*Provider, error) { + p := &Provider{ + edition: "GeoLite2-City", + accountID: accountID, + licenseKey: licenseKey, + refreshInterval: 7 * 24 * time.Hour, + directory: directory, + } + + if err := p.download(ctx); err != nil { + return nil, err + } + + return p, nil +} + +func (p *Provider) City(ip net.IP) (*geoip2.City, error) { + p.mut.Lock() + defer p.mut.Unlock() + + if p.db == nil { + return nil, errors.New("database not open") + } + + return p.db.City(ip) +} + +// Serve downloads the GeoIP2 database and keeps it up to date. It will return +// when the context is canceled. +func (p *Provider) Serve(ctx context.Context) error { + for { + select { + case <-ctx.Done(): + return ctx.Err() + + case <-time.After(p.refreshInterval): + if err := p.download(ctx); err != nil { + return err + } + } + } +} + +func (p *Provider) download(ctx context.Context) error { + newSubdir, err := os.MkdirTemp(p.directory, "geoipupdate") + if err != nil { + return fmt.Errorf("download: %w", err) + } + + cfg := &geoipupdate.Config{ + URL: "https://updates.maxmind.com", + DatabaseDirectory: newSubdir, + LockFile: filepath.Join(newSubdir, "geoipupdate.lock"), + RetryFor: 5 * time.Minute, + Parallelism: 1, + AccountID: p.accountID, + LicenseKey: p.licenseKey, + EditionIDs: []string{p.edition}, + } + + if err := geoipupdate.NewClient(cfg).Run(ctx); err != nil { + return fmt.Errorf("download: %w", err) + } + + dbPath := filepath.Join(newSubdir, p.edition+".mmdb") + db, err := geoip2.Open(dbPath) + if err != nil { + return fmt.Errorf("open downloaded db: %w", err) + } + + p.mut.Lock() + prevDBDir := p.currentDBDir + if p.db != nil { + p.db.Close() + } + p.currentDBDir = newSubdir + p.db = db + p.mut.Unlock() + + if prevDBDir != "" { + _ = os.RemoveAll(p.currentDBDir) + } + + return nil +} diff --git a/lib/geoip/geoip_test.go b/lib/geoip/geoip_test.go new file mode 100644 index 000000000..bf22c1c36 --- /dev/null +++ b/lib/geoip/geoip_test.go @@ -0,0 +1,36 @@ +// Copyright (C) 2024 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at https://mozilla.org/MPL/2.0/. + +package geoip + +import ( + "context" + "net" + "os" + "strconv" + "testing" +) + +func TestDownloadAndOpen(t *testing.T) { + acctID, _ := strconv.Atoi(os.Getenv("GEOIP_ACCOUNT_ID")) + if acctID == 0 { + t.Skip("No account ID set") + } + license := os.Getenv("GEOIP_LICENSE_KEY") + if license == "" { + t.Skip("No license key set") + } + + p, err := NewGeoLite2CityProvider(context.Background(), acctID, license, t.TempDir()) + if err != nil { + t.Fatal(err) + } + + _, err = p.City(net.ParseIP("8.8.8.8")) + if err != nil { + t.Fatal(err) + } +} diff --git a/script/strelaypoolsrv-entrypoint.sh b/script/strelaypoolsrv-entrypoint.sh deleted file mode 100755 index 945c77d5b..000000000 --- a/script/strelaypoolsrv-entrypoint.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/sh - -set -eu - -if [ "$MAXMIND_KEY" != "" ] ; then - curl "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-City&license_key=${MAXMIND_KEY}&suffix=tar.gz" \ - | tar --strip-components 1 -zxv -fi - -exec "$@"