octoleo/syncthing
1
0
Fork 0
mirror of https://github.com/octoleo/syncthing.git synced 2024-12-22 02:48:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

lib/geoip, cmd/relaypoolsrv, cmd/ursrv: Automatically manage GeoIP updates (#9342)

Browse Source 
This adds a small package `geoip` which knows how to download and manage
the Maxmind GeoLite2 database we use. This removes the need for various
scripts to download and manage the geoip database, something that today
happens on Docker startup for the relay pool server and using various
hand written hacks for the usage reporting server.

The database is downloaded when needed and then refreshed on a
best-effort basis weekly.
This commit is contained in:
Jakob Borg 2024-05-18 19:31:49 +02:00 committed by GitHub
parent 57d399317e
commit ba6ac2f604
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 212 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
Dockerfile.strelaypoolsrv
View File

@ -11,14 +11,6 @@ LABEL org.opencontainers.image.authors="The Syncthing Project" \
EXPOSE 8080 EXPOSE 8080
RUN apk add --no-cache ca-certificates su-exec curl
ENV PUID=1000 PGID=1000 MAXMIND_KEY=
RUN mkdir /var/strelaypoolsrv && chown 1000 /var/strelaypoolsrv
USER 1000
COPY strelaypoolsrv-linux-${TARGETARCH} /bin/strelaypoolsrv COPY strelaypoolsrv-linux-${TARGETARCH} /bin/strelaypoolsrv
COPY script/strelaypoolsrv-entrypoint.sh /bin/entrypoint.sh
WORKDIR /var/strelaypoolsrv ENTRYPOINT ["/bin/strelaypoolsrv", "-listen", ":8080"]
ENTRYPOINT ["/bin/entrypoint.sh", "/bin/strelaypoolsrv", "-listen", ":8080"]

40
cmd/strelaypoolsrv/main.go
View File

@ -21,12 +21,12 @@ import (
"time" "time"
lru "github.com/hashicorp/golang-lru/v2" lru "github.com/hashicorp/golang-lru/v2"
"github.com/oschwald/geoip2-golang"
"github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp" "github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/syncthing/syncthing/cmd/strelaypoolsrv/auto" "github.com/syncthing/syncthing/cmd/strelaypoolsrv/auto"
"github.com/syncthing/syncthing/lib/assets" "github.com/syncthing/syncthing/lib/assets"
_ "github.com/syncthing/syncthing/lib/automaxprocs" _ "github.com/syncthing/syncthing/lib/automaxprocs"
"github.com/syncthing/syncthing/lib/geoip"
"github.com/syncthing/syncthing/lib/httpcache" "github.com/syncthing/syncthing/lib/httpcache"
"github.com/syncthing/syncthing/lib/protocol" "github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/rand" "github.com/syncthing/syncthing/lib/rand"
@ -100,11 +100,12 @@ var (
debug bool debug bool
permRelaysFile string permRelaysFile string
ipHeader string ipHeader string
geoipPath string
proto string proto string
statsRefresh = time.Minute statsRefresh = time.Minute
requestQueueLen = 64 requestQueueLen = 64
requestProcessors = 8 requestProcessors = 8
geoipLicenseKey = os.Getenv("GEOIP_LICENSE_KEY")
geoipAccountID, _ = strconv.Atoi(os.Getenv("GEOIP_ACCOUNT_ID"))
requests chan request requests chan request
@ -130,34 +131,38 @@ func main() {
flag.StringVar(&permRelaysFile, "perm-relays", "", "Path to list of permanent relays") flag.StringVar(&permRelaysFile, "perm-relays", "", "Path to list of permanent relays")
flag.StringVar(&knownRelaysFile, "known-relays", knownRelaysFile, "Path to list of current relays") flag.StringVar(&knownRelaysFile, "known-relays", knownRelaysFile, "Path to list of current relays")
flag.StringVar(&ipHeader, "ip-header", "", "Name of header which holds clients ip:port. Only meaningful when running behind a reverse proxy.") flag.StringVar(&ipHeader, "ip-header", "", "Name of header which holds clients ip:port. Only meaningful when running behind a reverse proxy.")
flag.StringVar(&geoipPath, "geoip", "GeoLite2-City.mmdb", "Path to GeoLite2-City database")
flag.StringVar(&proto, "protocol", "tcp", "Protocol used for listening. 'tcp' for IPv4 and IPv6, 'tcp4' for IPv4, 'tcp6' for IPv6") flag.StringVar(&proto, "protocol", "tcp", "Protocol used for listening. 'tcp' for IPv4 and IPv6, 'tcp4' for IPv4, 'tcp6' for IPv6")
flag.DurationVar(&statsRefresh, "stats-refresh", statsRefresh, "Interval at which to refresh relay stats") flag.DurationVar(&statsRefresh, "stats-refresh", statsRefresh, "Interval at which to refresh relay stats")
flag.IntVar(&requestQueueLen, "request-queue", requestQueueLen, "Queue length for incoming test requests") flag.IntVar(&requestQueueLen, "request-queue", requestQueueLen, "Queue length for incoming test requests")
flag.IntVar(&requestProcessors, "request-processors", requestProcessors, "Number of request processor routines") flag.IntVar(&requestProcessors, "request-processors", requestProcessors, "Number of request processor routines")
flag.StringVar(&geoipLicenseKey, "geoip-license-key", geoipLicenseKey, "License key for GeoIP database")
flag.Parse() flag.Parse()
requests = make(chan request, requestQueueLen) requests = make(chan request, requestQueueLen)
geoip, err := geoip.NewGeoLite2CityProvider(context.Background(), geoipAccountID, geoipLicenseKey, os.TempDir())
if err != nil {
log.Fatalln("Failed to create GeoIP provider:", err)
}
go geoip.Serve(context.TODO())
var listener net.Listener var listener net.Listener
var err error
if permRelaysFile != "" { if permRelaysFile != "" {
permanentRelays = loadRelays(permRelaysFile) permanentRelays = loadRelays(permRelaysFile, geoip)
} }
testCert = createTestCertificate() testCert = createTestCertificate()
for i := 0; i < requestProcessors; i++ { for i := 0; i < requestProcessors; i++ {
go requestProcessor() go requestProcessor(geoip)
} }
// Load relays from cache in the background. // Load relays from cache in the background.
// Load them in a serial fashion to make sure any genuine requests // Load them in a serial fashion to make sure any genuine requests
// are not dropped. // are not dropped.
go func() { go func() {
for _, relay := range loadRelays(knownRelaysFile) { for _, relay := range loadRelays(knownRelaysFile, geoip) {
resultChan := make(chan result) resultChan := make(chan result)
requests <- request{relay, resultChan, nil} requests <- request{relay, resultChan, nil}
result := <-resultChan result := <-resultChan
@ -425,19 +430,19 @@ func handlePostRequest(w http.ResponseWriter, r *http.Request) {
} }
} }
func requestProcessor() { func requestProcessor(geoip *geoip.Provider) {
for request := range requests { for request := range requests {
if request.queueTimer != nil { if request.queueTimer != nil {
request.queueTimer.ObserveDuration() request.queueTimer.ObserveDuration()
} }
timer := prometheus.NewTimer(relayTestActionsSeconds.WithLabelValues("test")) timer := prometheus.NewTimer(relayTestActionsSeconds.WithLabelValues("test"))
handleRelayTest(request) handleRelayTest(request, geoip)
timer.ObserveDuration() timer.ObserveDuration()
} }
} }
func handleRelayTest(request request) { func handleRelayTest(request request, geoip *geoip.Provider) {
if debug { if debug {
log.Println("Request for", request.relay) log.Println("Request for", request.relay)
} }
@ -450,7 +455,7 @@ func handleRelayTest(request request) {
} }
stats := fetchStats(request.relay) stats := fetchStats(request.relay)
location := getLocation(request.relay.uri.Host) location := getLocation(request.relay.uri.Host, geoip)
mut.Lock() mut.Lock()
if stats != nil { if stats != nil {
@ -523,7 +528,7 @@ func evict(relay *relay) func() {
} }
} }
func loadRelays(file string) []*relay { func loadRelays(file string, geoip *geoip.Provider) []*relay {
content, err := os.ReadFile(file) content, err := os.ReadFile(file)
if err != nil { if err != nil {
log.Println("Failed to load relays: " + err.Error()) log.Println("Failed to load relays: " + err.Error())
@ -547,7 +552,7 @@ func loadRelays(file string) []*relay {
relays = append(relays, &relay{ relays = append(relays, &relay{
URL: line, URL: line,
Location: getLocation(uri.Host), Location: getLocation(uri.Host, geoip),
uri: uri, uri: uri,
}) })
if debug { if debug {
@ -580,21 +585,16 @@ func createTestCertificate() tls.Certificate {
return cert return cert
} }
func getLocation(host string) location { func getLocation(host string, geoip *geoip.Provider) location {
timer := prometheus.NewTimer(locationLookupSeconds) timer := prometheus.NewTimer(locationLookupSeconds)
defer timer.ObserveDuration() defer timer.ObserveDuration()
db, err := geoip2.Open(geoipPath)
if err != nil {
return location{}
}
defer db.Close()
addr, err := net.ResolveTCPAddr("tcp", host) addr, err := net.ResolveTCPAddr("tcp", host)
if err != nil { if err != nil {
return location{} return location{}
} }
city, err := db.City(addr.IP) city, err := geoip.City(addr.IP)
if err != nil { if err != nil {
return location{} return location{}
} }

43
cmd/ursrv/serve/serve.go
View File

@ -8,6 +8,7 @@ package serve
import ( import (
"bytes" "bytes"
"context"
"database/sql" "database/sql"
"embed" "embed"
"encoding/json" "encoding/json"
@ -17,6 +18,7 @@ import (
"log" "log"
"net" "net"
"net/http" "net/http"
"os"
"regexp" "regexp"
"sort" "sort"
"strconv" "strconv"
@ -26,20 +28,21 @@ import (
"unicode" "unicode"
_ "github.com/lib/pq" // PostgreSQL driver _ "github.com/lib/pq" // PostgreSQL driver
"github.com/oschwald/geoip2-golang"
"github.com/prometheus/client_golang/prometheus/promhttp" "github.com/prometheus/client_golang/prometheus/promhttp"
"golang.org/x/text/cases" "golang.org/x/text/cases"
"golang.org/x/text/language" "golang.org/x/text/language"
"github.com/syncthing/syncthing/lib/geoip"
"github.com/syncthing/syncthing/lib/upgrade" "github.com/syncthing/syncthing/lib/upgrade"
"github.com/syncthing/syncthing/lib/ur/contract" "github.com/syncthing/syncthing/lib/ur/contract"
) )
type CLI struct { type CLI struct {
Debug bool `env:"UR_DEBUG"` Debug bool `env:"UR_DEBUG"`
DBConn string `env:"UR_DB_URL" default:"postgres://user:password@localhost/ur?sslmode=disable"` DBConn string `env:"UR_DB_URL" default:"postgres://user:password@localhost/ur?sslmode=disable"`
Listen string `env:"UR_LISTEN" default:"0.0.0.0:8080"` Listen string `env:"UR_LISTEN" default:"0.0.0.0:8080"`
GeoIPPath string `env:"UR_GEOIP" default:"GeoLite2-City.mmdb"` GeoIPLicenseKey string `env:"UR_GEOIP_LICENSE_KEY"`
GeoIPAccountID int `env:"UR_GEOIP_ACCOUNT_ID"`
} }
//go:embed static //go:embed static
@ -189,10 +192,16 @@ func (cli *CLI) Run() error {
log.Fatalln("listen:", err) log.Fatalln("listen:", err)
} }
geoip, err := geoip.NewGeoLite2CityProvider(context.Background(), cli.GeoIPAccountID, cli.GeoIPLicenseKey, os.TempDir())
if err != nil {
log.Fatalln("geoip:", err)
}
go geoip.Serve(context.TODO())
srv := &server{ srv := &server{
db: db, db: db,
debug: cli.Debug, debug: cli.Debug,
geoIPPath: cli.GeoIPPath, geoip: geoip,
} }
http.HandleFunc("/", srv.rootHandler) http.HandleFunc("/", srv.rootHandler)
http.HandleFunc("/newdata", srv.newDataHandler) http.HandleFunc("/newdata", srv.newDataHandler)
@ -213,9 +222,9 @@ func (cli *CLI) Run() error {
} }
type server struct { type server struct {
debug bool debug bool
db *sql.DB db *sql.DB
geoIPPath string geoip *geoip.Provider
cacheMut sync.Mutex cacheMut sync.Mutex
cachedIndex []byte cachedIndex []byte
@ -238,7 +247,7 @@ func (s *server) cacheRefresher() {
} }
func (s *server) refreshCacheLocked() error { func (s *server) refreshCacheLocked() error {
rep := getReport(s.db, s.geoIPPath) rep := getReport(s.db, s.geoip)
buf := new(bytes.Buffer) buf := new(bytes.Buffer)
err := tpl.Execute(buf, rep) err := tpl.Execute(buf, rep)
if err != nil { if err != nil {
@ -492,15 +501,7 @@ type weightedLocation struct {
Weight int `json:"weight"` Weight int `json:"weight"`
} }
func getReport(db *sql.DB, geoIPPath string) map[string]interface{} { func getReport(db *sql.DB, geoip *geoip.Provider) map[string]interface{} {
geoip, err := geoip2.Open(geoIPPath)
if err != nil {
log.Println("opening geoip db", err)
geoip = nil
} else {
defer geoip.Close()
}
nodes := 0 nodes := 0
countriesTotal := 0 countriesTotal := 0
var versions []string var versions []string

4
go.mod
View File

@ -24,6 +24,7 @@ require (
github.com/lib/pq v1.10.9 github.com/lib/pq v1.10.9
github.com/maruel/panicparse/v2 v2.3.1 github.com/maruel/panicparse/v2 v2.3.1
github.com/maxbrunsfeld/counterfeiter/v6 v6.8.1 github.com/maxbrunsfeld/counterfeiter/v6 v6.8.1
github.com/maxmind/geoipupdate/v6 v6.1.0
github.com/minio/sha256-simd v1.0.1 github.com/minio/sha256-simd v1.0.1
github.com/miscreant/miscreant.go v0.0.0-20200214223636-26d376326b75 github.com/miscreant/miscreant.go v0.0.0-20200214223636-26d376326b75
github.com/oschwald/geoip2-golang v1.9.0 github.com/oschwald/geoip2-golang v1.9.0
@ -52,6 +53,7 @@ require (
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa // indirect github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa // indirect
github.com/beorn7/perks v1.0.1 // indirect github.com/beorn7/perks v1.0.1 // indirect
github.com/cenkalti/backoff/v4 v4.2.1 // indirect
github.com/certifi/gocertifi v0.0.0-20210507211836-431795d63e8d // indirect github.com/certifi/gocertifi v0.0.0-20210507211836-431795d63e8d // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect
@ -60,13 +62,13 @@ require (
github.com/go-asn1-ber/asn1-ber v1.5.5 // indirect github.com/go-asn1-ber/asn1-ber v1.5.5 // indirect
github.com/go-ole/go-ole v1.3.0 // indirect github.com/go-ole/go-ole v1.3.0 // indirect
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
github.com/gofrs/flock v0.8.1 // indirect
github.com/golang/snappy v0.0.4 // indirect github.com/golang/snappy v0.0.4 // indirect
github.com/google/pprof v0.0.0-20240402174815-29b9bb013b0f // indirect github.com/google/pprof v0.0.0-20240402174815-29b9bb013b0f // indirect
github.com/google/uuid v1.6.0 // indirect github.com/google/uuid v1.6.0 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/klauspost/cpuid/v2 v2.2.7 // indirect github.com/klauspost/cpuid/v2 v2.2.7 // indirect
github.com/kr/text v0.2.0 // indirect
github.com/nxadm/tail v1.4.11 // indirect github.com/nxadm/tail v1.4.11 // indirect
github.com/onsi/ginkgo/v2 v2.17.1 // indirect github.com/onsi/ginkgo/v2 v2.17.1 // indirect
github.com/onsi/gomega v1.31.1 // indirect github.com/onsi/gomega v1.31.1 // indirect

7
go.sum
View File

@ -22,6 +22,8 @@ github.com/calmh/xdr v1.1.0 h1:U/Dd4CXNLoo8EiQ4ulJUXkgO1/EyQLgDKLgpY1SOoJE=
github.com/calmh/xdr v1.1.0/go.mod h1:E8sz2ByAdXC8MbANf1LCRYzedSnnc+/sXXJs/PVqoeg= github.com/calmh/xdr v1.1.0/go.mod h1:E8sz2ByAdXC8MbANf1LCRYzedSnnc+/sXXJs/PVqoeg=
github.com/ccding/go-stun v0.1.4 h1:lC0co3Q3vjAuu2Jz098WivVPBPbemYFqbwE1syoka4M= github.com/ccding/go-stun v0.1.4 h1:lC0co3Q3vjAuu2Jz098WivVPBPbemYFqbwE1syoka4M=
github.com/ccding/go-stun v0.1.4/go.mod h1:cCZjJ1J3WFSJV6Wj8Y9Di8JMTsEXh6uv2eNmLzKaUeM= github.com/ccding/go-stun v0.1.4/go.mod h1:cCZjJ1J3WFSJV6Wj8Y9Di8JMTsEXh6uv2eNmLzKaUeM=
github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM=
github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
github.com/certifi/gocertifi v0.0.0-20210507211836-431795d63e8d h1:S2NE3iHSwP0XV47EEXL8mWmRdEfGscSJ+7EgePNgt0s= github.com/certifi/gocertifi v0.0.0-20210507211836-431795d63e8d h1:S2NE3iHSwP0XV47EEXL8mWmRdEfGscSJ+7EgePNgt0s=
github.com/certifi/gocertifi v0.0.0-20210507211836-431795d63e8d/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= github.com/certifi/gocertifi v0.0.0-20210507211836-431795d63e8d/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
@ -34,7 +36,6 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn
github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4= github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4=
github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/d4l3k/messagediff v1.2.1 h1:ZcAIMYsUg0EAp9X+tt8/enBE/Q8Yd5kzPynLyKptt9U= github.com/d4l3k/messagediff v1.2.1 h1:ZcAIMYsUg0EAp9X+tt8/enBE/Q8Yd5kzPynLyKptt9U=
github.com/d4l3k/messagediff v1.2.1/go.mod h1:Oozbb1TVXFac9FtSIxHBMnBCq2qeH/2KkEQxENCrlLo= github.com/d4l3k/messagediff v1.2.1/go.mod h1:Oozbb1TVXFac9FtSIxHBMnBCq2qeH/2KkEQxENCrlLo=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@ -62,6 +63,8 @@ github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW
github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls=
github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw=
github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@ -131,6 +134,8 @@ github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
github.com/maxbrunsfeld/counterfeiter/v6 v6.8.1 h1:NicmruxkeqHjDv03SfSxqmaLuisddudfP3h5wdXFbhM= github.com/maxbrunsfeld/counterfeiter/v6 v6.8.1 h1:NicmruxkeqHjDv03SfSxqmaLuisddudfP3h5wdXFbhM=
github.com/maxbrunsfeld/counterfeiter/v6 v6.8.1/go.mod h1:eyp4DdUJAKkr9tvxR3jWhw2mDK7CWABMG5r9uyaKC7I= github.com/maxbrunsfeld/counterfeiter/v6 v6.8.1/go.mod h1:eyp4DdUJAKkr9tvxR3jWhw2mDK7CWABMG5r9uyaKC7I=
github.com/maxmind/geoipupdate/v6 v6.1.0 h1:sdtTHzzQNJlXF5+fd/EoPTucRHyMonYt/Cok8xzzfqA=
github.com/maxmind/geoipupdate/v6 v6.1.0/go.mod h1:cZYCDzfMzTY4v6dKRdV7KTB6SStxtn3yFkiJ1btTGGc=
github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM=
github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8=

124
lib/geoip/geoip.go Normal file
View File

@ -0,0 +1,124 @@
// Copyright (C) 2024 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
// Package geoip provides an automatically updating MaxMind GeoIP2 database
// provider.
package geoip
import (
"context"
"errors"
"fmt"
"net"
"os"
"path/filepath"
"sync"
"time"
"github.com/maxmind/geoipupdate/v6/pkg/geoipupdate"
"github.com/oschwald/geoip2-golang"
)
type Provider struct {
edition string
accountID int
licenseKey string
refreshInterval time.Duration
directory string
mut sync.Mutex
currentDBDir string
db *geoip2.Reader
}
// NewGeoLite2CityProvider returns a new GeoIP2 database provider for the
// GeoLite2-City database. The database will be stored in the given
// directory (which should exist) and refreshed every 7 days.
func NewGeoLite2CityProvider(ctx context.Context, accountID int, licenseKey string, directory string) (*Provider, error) {
p := &Provider{
edition: "GeoLite2-City",
accountID: accountID,
licenseKey: licenseKey,
refreshInterval: 7 * 24 * time.Hour,
directory: directory,
}
if err := p.download(ctx); err != nil {
return nil, err
}
return p, nil
}
func (p *Provider) City(ip net.IP) (*geoip2.City, error) {
p.mut.Lock()
defer p.mut.Unlock()
if p.db == nil {
return nil, errors.New("database not open")
}
return p.db.City(ip)
}
// Serve downloads the GeoIP2 database and keeps it up to date. It will return
// when the context is canceled.
func (p *Provider) Serve(ctx context.Context) error {
for {
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(p.refreshInterval):
if err := p.download(ctx); err != nil {
return err
}
}
}
}
func (p *Provider) download(ctx context.Context) error {
newSubdir, err := os.MkdirTemp(p.directory, "geoipupdate")
if err != nil {
return fmt.Errorf("download: %w", err)
}
cfg := &geoipupdate.Config{
URL: "https://updates.maxmind.com",
DatabaseDirectory: newSubdir,
LockFile: filepath.Join(newSubdir, "geoipupdate.lock"),
RetryFor: 5 * time.Minute,
Parallelism: 1,
AccountID: p.accountID,
LicenseKey: p.licenseKey,
EditionIDs: []string{p.edition},
}
if err := geoipupdate.NewClient(cfg).Run(ctx); err != nil {
return fmt.Errorf("download: %w", err)
}
dbPath := filepath.Join(newSubdir, p.edition+".mmdb")
db, err := geoip2.Open(dbPath)
if err != nil {
return fmt.Errorf("open downloaded db: %w", err)
}
p.mut.Lock()
prevDBDir := p.currentDBDir
if p.db != nil {
p.db.Close()
}
p.currentDBDir = newSubdir
p.db = db
p.mut.Unlock()
if prevDBDir != "" {
_ = os.RemoveAll(p.currentDBDir)
}
return nil
}

36
lib/geoip/geoip_test.go Normal file
View File

@ -0,0 +1,36 @@
// Copyright (C) 2024 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package geoip
import (
"context"
"net"
"os"
"strconv"
"testing"
)
func TestDownloadAndOpen(t *testing.T) {
acctID, _ := strconv.Atoi(os.Getenv("GEOIP_ACCOUNT_ID"))
if acctID == 0 {
t.Skip("No account ID set")
}
license := os.Getenv("GEOIP_LICENSE_KEY")
if license == "" {
t.Skip("No license key set")
}
p, err := NewGeoLite2CityProvider(context.Background(), acctID, license, t.TempDir())
if err != nil {
t.Fatal(err)
}
_, err = p.City(net.ParseIP("8.8.8.8"))
if err != nil {
t.Fatal(err)
}
}

10
script/strelaypoolsrv-entrypoint.sh
View File

@ -1,10 +0,0 @@
#!/bin/sh
set -eu
if [ "$MAXMIND_KEY" != "" ] ; then
curl "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-City&license_key=${MAXMIND_KEY}&suffix=tar.gz" \
| tar --strip-components 1 -zxv
fi
exec "$@"