mirror of
https://github.com/octoleo/syncthing.git
synced 2024-12-21 10:29:06 +00:00
lib/geoip, cmd/relaypoolsrv, cmd/ursrv: Automatically manage GeoIP updates (#9342)
This adds a small package `geoip` which knows how to download and manage the Maxmind GeoLite2 database we use. This removes the need for various scripts to download and manage the geoip database, something that today happens on Docker startup for the relay pool server and using various hand written hacks for the usage reporting server. The database is downloaded when needed and then refreshed on a best-effort basis weekly.
This commit is contained in:
Jakob Borg
GitHub
parent
57d399317e
commit
ba6ac2f604
@ -11,14 +11,6 @@ LABEL org.opencontainers.image.authors="The Syncthing Project" \
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
RUN apk add --no-cache ca-certificates su-exec curl
|
||||
ENV PUID=1000 PGID=1000 MAXMIND_KEY=
|
||||
|
||||
RUN mkdir /var/strelaypoolsrv && chown 1000 /var/strelaypoolsrv
|
||||
USER 1000
|
||||
|
||||
COPY strelaypoolsrv-linux-${TARGETARCH} /bin/strelaypoolsrv
|
||||
COPY script/strelaypoolsrv-entrypoint.sh /bin/entrypoint.sh
|
||||
|
||||
WORKDIR /var/strelaypoolsrv
|
||||
ENTRYPOINT ["/bin/entrypoint.sh", "/bin/strelaypoolsrv", "-listen", ":8080"]
|
||||
ENTRYPOINT ["/bin/strelaypoolsrv", "-listen", ":8080"]
|
||||
|
||||
@ -21,12 +21,12 @@ import (
|
||||
"time"
|
||||
|
||||
lru "github.com/hashicorp/golang-lru/v2"
|
||||
"github.com/oschwald/geoip2-golang"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
"github.com/syncthing/syncthing/cmd/strelaypoolsrv/auto"
|
||||
"github.com/syncthing/syncthing/lib/assets"
|
||||
_ "github.com/syncthing/syncthing/lib/automaxprocs"
|
||||
"github.com/syncthing/syncthing/lib/geoip"
|
||||
"github.com/syncthing/syncthing/lib/httpcache"
|
||||
"github.com/syncthing/syncthing/lib/protocol"
|
||||
"github.com/syncthing/syncthing/lib/rand"
|
||||
@ -100,11 +100,12 @@ var (
|
||||
debug bool
|
||||
permRelaysFile string
|
||||
ipHeader string
|
||||
geoipPath string
|
||||
proto string
|
||||
statsRefresh = time.Minute
|
||||
requestQueueLen = 64
|
||||
requestProcessors = 8
|
||||
geoipLicenseKey = os.Getenv("GEOIP_LICENSE_KEY")
|
||||
geoipAccountID, _ = strconv.Atoi(os.Getenv("GEOIP_ACCOUNT_ID"))
|
||||
|
||||
requests chan request
|
||||
|
||||
@ -130,34 +131,38 @@ func main() {
|
||||
flag.StringVar(&permRelaysFile, "perm-relays", "", "Path to list of permanent relays")
|
||||
flag.StringVar(&knownRelaysFile, "known-relays", knownRelaysFile, "Path to list of current relays")
|
||||
flag.StringVar(&ipHeader, "ip-header", "", "Name of header which holds clients ip:port. Only meaningful when running behind a reverse proxy.")
|
||||
flag.StringVar(&geoipPath, "geoip", "GeoLite2-City.mmdb", "Path to GeoLite2-City database")
|
||||
flag.StringVar(&proto, "protocol", "tcp", "Protocol used for listening. 'tcp' for IPv4 and IPv6, 'tcp4' for IPv4, 'tcp6' for IPv6")
|
||||
flag.DurationVar(&statsRefresh, "stats-refresh", statsRefresh, "Interval at which to refresh relay stats")
|
||||
flag.IntVar(&requestQueueLen, "request-queue", requestQueueLen, "Queue length for incoming test requests")
|
||||
flag.IntVar(&requestProcessors, "request-processors", requestProcessors, "Number of request processor routines")
|
||||
flag.StringVar(&geoipLicenseKey, "geoip-license-key", geoipLicenseKey, "License key for GeoIP database")
|
||||
|
||||
flag.Parse()
|
||||
|
||||
requests = make(chan request, requestQueueLen)
|
||||
geoip, err := geoip.NewGeoLite2CityProvider(context.Background(), geoipAccountID, geoipLicenseKey, os.TempDir())
|
||||
if err != nil {
|
||||
log.Fatalln("Failed to create GeoIP provider:", err)
|
||||
}
|
||||
go geoip.Serve(context.TODO())
|
||||
|
||||
var listener net.Listener
|
||||
var err error
|
||||
|
||||
if permRelaysFile != "" {
|
||||
permanentRelays = loadRelays(permRelaysFile)
|
||||
permanentRelays = loadRelays(permRelaysFile, geoip)
|
||||
}
|
||||
|
||||
testCert = createTestCertificate()
|
||||
|
||||
for i := 0; i < requestProcessors; i++ {
|
||||
go requestProcessor()
|
||||
go requestProcessor(geoip)
|
||||
}
|
||||
|
||||
// Load relays from cache in the background.
|
||||
// Load them in a serial fashion to make sure any genuine requests
|
||||
// are not dropped.
|
||||
go func() {
|
||||
for _, relay := range loadRelays(knownRelaysFile) {
|
||||
for _, relay := range loadRelays(knownRelaysFile, geoip) {
|
||||
resultChan := make(chan result)
|
||||
requests <- request{relay, resultChan, nil}
|
||||
result := <-resultChan
|
||||
@ -425,19 +430,19 @@ func handlePostRequest(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
}
|
||||
|
||||
func requestProcessor() {
|
||||
func requestProcessor(geoip *geoip.Provider) {
|
||||
for request := range requests {
|
||||
if request.queueTimer != nil {
|
||||
request.queueTimer.ObserveDuration()
|
||||
}
|
||||
|
||||
timer := prometheus.NewTimer(relayTestActionsSeconds.WithLabelValues("test"))
|
||||
handleRelayTest(request)
|
||||
handleRelayTest(request, geoip)
|
||||
timer.ObserveDuration()
|
||||
}
|
||||
}
|
||||
|
||||
func handleRelayTest(request request) {
|
||||
func handleRelayTest(request request, geoip *geoip.Provider) {
|
||||
if debug {
|
||||
log.Println("Request for", request.relay)
|
||||
}
|
||||
@ -450,7 +455,7 @@ func handleRelayTest(request request) {
|
||||
}
|
||||
|
||||
stats := fetchStats(request.relay)
|
||||
location := getLocation(request.relay.uri.Host)
|
||||
location := getLocation(request.relay.uri.Host, geoip)
|
||||
|
||||
mut.Lock()
|
||||
if stats != nil {
|
||||
@ -523,7 +528,7 @@ func evict(relay *relay) func() {
|
||||
}
|
||||
}
|
||||
|
||||
func loadRelays(file string) []*relay {
|
||||
func loadRelays(file string, geoip *geoip.Provider) []*relay {
|
||||
content, err := os.ReadFile(file)
|
||||
if err != nil {
|
||||
log.Println("Failed to load relays: " + err.Error())
|
||||
@ -547,7 +552,7 @@ func loadRelays(file string) []*relay {
|
||||
|
||||
relays = append(relays, &relay{
|
||||
URL: line,
|
||||
Location: getLocation(uri.Host),
|
||||
Location: getLocation(uri.Host, geoip),
|
||||
uri: uri,
|
||||
})
|
||||
if debug {
|
||||
@ -580,21 +585,16 @@ func createTestCertificate() tls.Certificate {
|
||||
return cert
|
||||
}
|
||||
|
||||
func getLocation(host string) location {
|
||||
func getLocation(host string, geoip *geoip.Provider) location {
|
||||
timer := prometheus.NewTimer(locationLookupSeconds)
|
||||
defer timer.ObserveDuration()
|
||||
db, err := geoip2.Open(geoipPath)
|
||||
if err != nil {
|
||||
return location{}
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
addr, err := net.ResolveTCPAddr("tcp", host)
|
||||
if err != nil {
|
||||
return location{}
|
||||
}
|
||||
|
||||
city, err := db.City(addr.IP)
|
||||
city, err := geoip.City(addr.IP)
|
||||
if err != nil {
|
||||
return location{}
|
||||
}
|
||||
|
||||
@ -8,6 +8,7 @@ package serve
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"database/sql"
|
||||
"embed"
|
||||
"encoding/json"
|
||||
@ -17,6 +18,7 @@ import (
|
||||
"log"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
@ -26,20 +28,21 @@ import (
|
||||
"unicode"
|
||||
|
||||
_ "github.com/lib/pq" // PostgreSQL driver
|
||||
"github.com/oschwald/geoip2-golang"
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
"golang.org/x/text/cases"
|
||||
"golang.org/x/text/language"
|
||||
|
||||
"github.com/syncthing/syncthing/lib/geoip"
|
||||
"github.com/syncthing/syncthing/lib/upgrade"
|
||||
"github.com/syncthing/syncthing/lib/ur/contract"
|
||||
)
|
||||
|
||||
type CLI struct {
|
||||
Debug bool `env:"UR_DEBUG"`
|
||||
DBConn string `env:"UR_DB_URL" default:"postgres://user:password@localhost/ur?sslmode=disable"`
|
||||
Listen string `env:"UR_LISTEN" default:"0.0.0.0:8080"`
|
||||
GeoIPPath string `env:"UR_GEOIP" default:"GeoLite2-City.mmdb"`
|
||||
Debug bool `env:"UR_DEBUG"`
|
||||
DBConn string `env:"UR_DB_URL" default:"postgres://user:password@localhost/ur?sslmode=disable"`
|
||||
Listen string `env:"UR_LISTEN" default:"0.0.0.0:8080"`
|
||||
GeoIPLicenseKey string `env:"UR_GEOIP_LICENSE_KEY"`
|
||||
GeoIPAccountID int `env:"UR_GEOIP_ACCOUNT_ID"`
|
||||
}
|
||||
|
||||
//go:embed static
|
||||
@ -189,10 +192,16 @@ func (cli *CLI) Run() error {
|
||||
log.Fatalln("listen:", err)
|
||||
}
|
||||
|
||||
geoip, err := geoip.NewGeoLite2CityProvider(context.Background(), cli.GeoIPAccountID, cli.GeoIPLicenseKey, os.TempDir())
|
||||
if err != nil {
|
||||
log.Fatalln("geoip:", err)
|
||||
}
|
||||
go geoip.Serve(context.TODO())
|
||||
|
||||
srv := &server{
|
||||
db: db,
|
||||
debug: cli.Debug,
|
||||
geoIPPath: cli.GeoIPPath,
|
||||
db: db,
|
||||
debug: cli.Debug,
|
||||
geoip: geoip,
|
||||
}
|
||||
http.HandleFunc("/", srv.rootHandler)
|
||||
http.HandleFunc("/newdata", srv.newDataHandler)
|
||||
@ -213,9 +222,9 @@ func (cli *CLI) Run() error {
|
||||
}
|
||||
|
||||
type server struct {
|
||||
debug bool
|
||||
db *sql.DB
|
||||
geoIPPath string
|
||||
debug bool
|
||||
db *sql.DB
|
||||
geoip *geoip.Provider
|
||||
|
||||
cacheMut sync.Mutex
|
||||
cachedIndex []byte
|
||||
@ -238,7 +247,7 @@ func (s *server) cacheRefresher() {
|
||||
}
|
||||
|
||||
func (s *server) refreshCacheLocked() error {
|
||||
rep := getReport(s.db, s.geoIPPath)
|
||||
rep := getReport(s.db, s.geoip)
|
||||
buf := new(bytes.Buffer)
|
||||
err := tpl.Execute(buf, rep)
|
||||
if err != nil {
|
||||
@ -492,15 +501,7 @@ type weightedLocation struct {
|
||||
Weight int `json:"weight"`
|
||||
}
|
||||
|
||||
func getReport(db *sql.DB, geoIPPath string) map[string]interface{} {
|
||||
geoip, err := geoip2.Open(geoIPPath)
|
||||
if err != nil {
|
||||
log.Println("opening geoip db", err)
|
||||
geoip = nil
|
||||
} else {
|
||||
defer geoip.Close()
|
||||
}
|
||||
|
||||
func getReport(db *sql.DB, geoip *geoip.Provider) map[string]interface{} {
|
||||
nodes := 0
|
||||
countriesTotal := 0
|
||||
var versions []string
|
||||
|
||||
4
go.mod
4
go.mod
@ -24,6 +24,7 @@ require (
|
||||
github.com/lib/pq v1.10.9
|
||||
github.com/maruel/panicparse/v2 v2.3.1
|
||||
github.com/maxbrunsfeld/counterfeiter/v6 v6.8.1
|
||||
github.com/maxmind/geoipupdate/v6 v6.1.0
|
||||
github.com/minio/sha256-simd v1.0.1
|
||||
github.com/miscreant/miscreant.go v0.0.0-20200214223636-26d376326b75
|
||||
github.com/oschwald/geoip2-golang v1.9.0
|
||||
@ -52,6 +53,7 @@ require (
|
||||
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
|
||||
github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa // indirect
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/cenkalti/backoff/v4 v4.2.1 // indirect
|
||||
github.com/certifi/gocertifi v0.0.0-20210507211836-431795d63e8d // indirect
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect
|
||||
@ -60,13 +62,13 @@ require (
|
||||
github.com/go-asn1-ber/asn1-ber v1.5.5 // indirect
|
||||
github.com/go-ole/go-ole v1.3.0 // indirect
|
||||
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
|
||||
github.com/gofrs/flock v0.8.1 // indirect
|
||||
github.com/golang/snappy v0.0.4 // indirect
|
||||
github.com/google/pprof v0.0.0-20240402174815-29b9bb013b0f // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/hashicorp/errwrap v1.1.0 // indirect
|
||||
github.com/hashicorp/go-multierror v1.1.1 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.7 // indirect
|
||||
github.com/kr/text v0.2.0 // indirect
|
||||
github.com/nxadm/tail v1.4.11 // indirect
|
||||
github.com/onsi/ginkgo/v2 v2.17.1 // indirect
|
||||
github.com/onsi/gomega v1.31.1 // indirect
|
||||
|
||||
7
go.sum
7
go.sum
@ -22,6 +22,8 @@ github.com/calmh/xdr v1.1.0 h1:U/Dd4CXNLoo8EiQ4ulJUXkgO1/EyQLgDKLgpY1SOoJE=
|
||||
github.com/calmh/xdr v1.1.0/go.mod h1:E8sz2ByAdXC8MbANf1LCRYzedSnnc+/sXXJs/PVqoeg=
|
||||
github.com/ccding/go-stun v0.1.4 h1:lC0co3Q3vjAuu2Jz098WivVPBPbemYFqbwE1syoka4M=
|
||||
github.com/ccding/go-stun v0.1.4/go.mod h1:cCZjJ1J3WFSJV6Wj8Y9Di8JMTsEXh6uv2eNmLzKaUeM=
|
||||
github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM=
|
||||
github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
|
||||
github.com/certifi/gocertifi v0.0.0-20210507211836-431795d63e8d h1:S2NE3iHSwP0XV47EEXL8mWmRdEfGscSJ+7EgePNgt0s=
|
||||
github.com/certifi/gocertifi v0.0.0-20210507211836-431795d63e8d/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA=
|
||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||
@ -34,7 +36,6 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
|
||||
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
||||
github.com/d4l3k/messagediff v1.2.1 h1:ZcAIMYsUg0EAp9X+tt8/enBE/Q8Yd5kzPynLyKptt9U=
|
||||
github.com/d4l3k/messagediff v1.2.1/go.mod h1:Oozbb1TVXFac9FtSIxHBMnBCq2qeH/2KkEQxENCrlLo=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
@ -62,6 +63,8 @@ github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW
|
||||
github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
|
||||
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
|
||||
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls=
|
||||
github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw=
|
||||
github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
|
||||
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
|
||||
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
|
||||
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
@ -131,6 +134,8 @@ github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb
|
||||
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
|
||||
github.com/maxbrunsfeld/counterfeiter/v6 v6.8.1 h1:NicmruxkeqHjDv03SfSxqmaLuisddudfP3h5wdXFbhM=
|
||||
github.com/maxbrunsfeld/counterfeiter/v6 v6.8.1/go.mod h1:eyp4DdUJAKkr9tvxR3jWhw2mDK7CWABMG5r9uyaKC7I=
|
||||
github.com/maxmind/geoipupdate/v6 v6.1.0 h1:sdtTHzzQNJlXF5+fd/EoPTucRHyMonYt/Cok8xzzfqA=
|
||||
github.com/maxmind/geoipupdate/v6 v6.1.0/go.mod h1:cZYCDzfMzTY4v6dKRdV7KTB6SStxtn3yFkiJ1btTGGc=
|
||||
github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
|
||||
github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM=
|
||||
github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8=
|
||||
|
||||
124
lib/geoip/geoip.go
Normal file
124
lib/geoip/geoip.go
Normal file
@ -0,0 +1,124 @@
|
||||
// Copyright (C) 2024 The Syncthing Authors.
|
||||
//
|
||||
// This Source Code Form is subject to the terms of the Mozilla Public
|
||||
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
// You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
// Package geoip provides an automatically updating MaxMind GeoIP2 database
|
||||
// provider.
|
||||
package geoip
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/maxmind/geoipupdate/v6/pkg/geoipupdate"
|
||||
"github.com/oschwald/geoip2-golang"
|
||||
)
|
||||
|
||||
type Provider struct {
|
||||
edition string
|
||||
accountID int
|
||||
licenseKey string
|
||||
refreshInterval time.Duration
|
||||
directory string
|
||||
|
||||
mut sync.Mutex
|
||||
currentDBDir string
|
||||
db *geoip2.Reader
|
||||
}
|
||||
|
||||
// NewGeoLite2CityProvider returns a new GeoIP2 database provider for the
|
||||
// GeoLite2-City database. The database will be stored in the given
|
||||
// directory (which should exist) and refreshed every 7 days.
|
||||
func NewGeoLite2CityProvider(ctx context.Context, accountID int, licenseKey string, directory string) (*Provider, error) {
|
||||
p := &Provider{
|
||||
edition: "GeoLite2-City",
|
||||
accountID: accountID,
|
||||
licenseKey: licenseKey,
|
||||
refreshInterval: 7 * 24 * time.Hour,
|
||||
directory: directory,
|
||||
}
|
||||
|
||||
if err := p.download(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return p, nil
|
||||
}
|
||||
|
||||
func (p *Provider) City(ip net.IP) (*geoip2.City, error) {
|
||||
p.mut.Lock()
|
||||
defer p.mut.Unlock()
|
||||
|
||||
if p.db == nil {
|
||||
return nil, errors.New("database not open")
|
||||
}
|
||||
|
||||
return p.db.City(ip)
|
||||
}
|
||||
|
||||
// Serve downloads the GeoIP2 database and keeps it up to date. It will return
|
||||
// when the context is canceled.
|
||||
func (p *Provider) Serve(ctx context.Context) error {
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
|
||||
case <-time.After(p.refreshInterval):
|
||||
if err := p.download(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Provider) download(ctx context.Context) error {
|
||||
newSubdir, err := os.MkdirTemp(p.directory, "geoipupdate")
|
||||
if err != nil {
|
||||
return fmt.Errorf("download: %w", err)
|
||||
}
|
||||
|
||||
cfg := &geoipupdate.Config{
|
||||
URL: "https://updates.maxmind.com",
|
||||
DatabaseDirectory: newSubdir,
|
||||
LockFile: filepath.Join(newSubdir, "geoipupdate.lock"),
|
||||
RetryFor: 5 * time.Minute,
|
||||
Parallelism: 1,
|
||||
AccountID: p.accountID,
|
||||
LicenseKey: p.licenseKey,
|
||||
EditionIDs: []string{p.edition},
|
||||
}
|
||||
|
||||
if err := geoipupdate.NewClient(cfg).Run(ctx); err != nil {
|
||||
return fmt.Errorf("download: %w", err)
|
||||
}
|
||||
|
||||
dbPath := filepath.Join(newSubdir, p.edition+".mmdb")
|
||||
db, err := geoip2.Open(dbPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("open downloaded db: %w", err)
|
||||
}
|
||||
|
||||
p.mut.Lock()
|
||||
prevDBDir := p.currentDBDir
|
||||
if p.db != nil {
|
||||
p.db.Close()
|
||||
}
|
||||
p.currentDBDir = newSubdir
|
||||
p.db = db
|
||||
p.mut.Unlock()
|
||||
|
||||
if prevDBDir != "" {
|
||||
_ = os.RemoveAll(p.currentDBDir)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
36
lib/geoip/geoip_test.go
Normal file
36
lib/geoip/geoip_test.go
Normal file
@ -0,0 +1,36 @@
|
||||
// Copyright (C) 2024 The Syncthing Authors.
|
||||
//
|
||||
// This Source Code Form is subject to the terms of the Mozilla Public
|
||||
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
|
||||
// You can obtain one at https://mozilla.org/MPL/2.0/.
|
||||
|
||||
package geoip
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"os"
|
||||
"strconv"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestDownloadAndOpen(t *testing.T) {
|
||||
acctID, _ := strconv.Atoi(os.Getenv("GEOIP_ACCOUNT_ID"))
|
||||
if acctID == 0 {
|
||||
t.Skip("No account ID set")
|
||||
}
|
||||
license := os.Getenv("GEOIP_LICENSE_KEY")
|
||||
if license == "" {
|
||||
t.Skip("No license key set")
|
||||
}
|
||||
|
||||
p, err := NewGeoLite2CityProvider(context.Background(), acctID, license, t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
_, err = p.City(net.ParseIP("8.8.8.8"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
@ -1,10 +0,0 @@
|
||||
#!/bin/sh
|
||||
|
||||
set -eu
|
||||
|
||||
if [ "$MAXMIND_KEY" != "" ] ; then
|
||||
curl "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-City&license_key=${MAXMIND_KEY}&suffix=tar.gz" \
|
||||
| tar --strip-components 1 -zxv
|
||||
fi
|
||||
|
||||
exec "$@"
|
||||
Loading…
Reference in New Issue
Block a user