From 8f42dedef8dfbdcd989bd90c206e200bdfa97de8 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Tue, 25 May 2021 13:22:28 +0200 Subject: [PATCH 1/3] Add GO111MODULE=off to build.sh, use Golang 1.16 (#966) * Add GO111MODULE=off to build.sh * Use golang 1.16 * Update go version in README.md * Add missing GO111MODULE=off * Add missing GO111MODULE=off again * Use go1.16.3 explicitly * Use 1.16 for CI test * Update min go version * Use go 1.16.4 --- .github/workflows/ci.yml | 4 ++-- .github/workflows/replica-tests.yml | 4 ++-- Dockerfile.packaging | 4 +--- Dockerfile.test | 2 +- README.md | 2 +- build.sh | 7 ++++--- localtests/test.sh | 5 ++++- script/build | 3 ++- script/ensure-go-installed | 4 ++-- script/test | 3 ++- test.sh | 5 ++++- 11 files changed, 25 insertions(+), 18 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cf518ee..84e8c04 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,10 +10,10 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Set up Go 1.15 + - name: Set up Go 1.16 uses: actions/setup-go@v1 with: - go-version: 1.15 + go-version: 1.16 - name: Build run: script/cibuild diff --git a/.github/workflows/replica-tests.yml b/.github/workflows/replica-tests.yml index bbc53a5..f1c887c 100644 --- a/.github/workflows/replica-tests.yml +++ b/.github/workflows/replica-tests.yml @@ -13,10 +13,10 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Set up Go 1.14 + - name: Set up Go 1.16 uses: actions/setup-go@v1 with: - go-version: 1.14 + go-version: 1.16 - name: migration tests env: diff --git a/Dockerfile.packaging b/Dockerfile.packaging index 092fade..64036b1 100644 --- a/Dockerfile.packaging +++ b/Dockerfile.packaging @@ -1,6 +1,4 @@ -# - -FROM golang:1.15.6 +FROM golang:1.16.4 RUN apt-get update RUN apt-get install -y ruby ruby-dev rubygems build-essential diff --git a/Dockerfile.test b/Dockerfile.test index ceb46bf..3ba0126 100644 --- a/Dockerfile.test +++ b/Dockerfile.test @@ -1,4 +1,4 @@ -FROM golang:1.15.6 +FROM golang:1.16.4 LABEL maintainer="github@github.com" RUN apt-get update diff --git a/README.md b/README.md index 8842d5b..e21344d 100644 --- a/README.md +++ b/README.md @@ -95,7 +95,7 @@ Please see [Coding gh-ost](doc/coding-ghost.md) for a guide to getting started d [Download latest release here](https://github.com/github/gh-ost/releases/latest) -`gh-ost` is a Go project; it is built with Go `1.14` and above. To build on your own, use either: +`gh-ost` is a Go project; it is built with Go `1.15` and above. To build on your own, use either: - [script/build](https://github.com/github/gh-ost/blob/master/script/build) - this is the same build script used by CI hence the authoritative; artifact is `./bin/gh-ost` binary. - [build.sh](https://github.com/github/gh-ost/blob/master/build.sh) for building `tar.gz` artifacts in `/tmp/gh-ost` diff --git a/build.sh b/build.sh index 88ecf1e..ef24e81 100755 --- a/build.sh +++ b/build.sh @@ -18,15 +18,16 @@ function build { GOOS=$3 GOARCH=$4 - if ! go version | egrep -q 'go(1\.1[456])' ; then - echo "go version must be 1.14 or above" + if ! go version | egrep -q 'go(1\.1[56])' ; then + echo "go version must be 1.15 or above" exit 1 fi + # TODO: remove GO111MODULE once gh-ost uses Go modules echo "Building ${osname} binary" export GOOS export GOARCH - go build -ldflags "$ldflags" -o $buildpath/$target go/cmd/gh-ost/main.go + GO111MODULE=off go build -ldflags "$ldflags" -o $buildpath/$target go/cmd/gh-ost/main.go if [ $? -ne 0 ]; then echo "Build failed for ${osname}" diff --git a/localtests/test.sh b/localtests/test.sh index eb3d640..5e96e28 100755 --- a/localtests/test.sh +++ b/localtests/test.sh @@ -242,7 +242,10 @@ build_binary() { echo "Using binary: $ghost_binary" return 0 fi - go build -o $ghost_binary go/cmd/gh-ost/main.go + + # TODO: remove GO111MODULE once gh-ost uses Go modules + GO111MODULE=off go build -o $ghost_binary go/cmd/gh-ost/main.go + if [ $? -ne 0 ] ; then echo "Build failure" exit 1 diff --git a/script/build b/script/build index 24be5d1..7107483 100755 --- a/script/build +++ b/script/build @@ -17,4 +17,5 @@ export GOPATH="$PWD/.gopath" cd .gopath/src/github.com/github/gh-ost # We put the binaries directly into the bindir, because we have no need for shim wrappers -go build -o "$bindir/gh-ost" -ldflags "-X main.AppVersion=${version} -X main.BuildDescribe=${describe}" ./go/cmd/gh-ost/main.go +# TODO: remove GO111MODULE once gh-ost uses Go modules +GO111MODULE=off go build -o "$bindir/gh-ost" -ldflags "-X main.AppVersion=${version} -X main.BuildDescribe=${describe}" ./go/cmd/gh-ost/main.go diff --git a/script/ensure-go-installed b/script/ensure-go-installed index baa5bd7..473ba38 100755 --- a/script/ensure-go-installed +++ b/script/ensure-go-installed @@ -1,7 +1,7 @@ #!/bin/bash -PREFERRED_GO_VERSION=go1.14.7 -SUPPORTED_GO_VERSIONS='go1.1[456]' +PREFERRED_GO_VERSION=go1.16.4 +SUPPORTED_GO_VERSIONS='go1.1[56]' GO_PKG_DARWIN=${PREFERRED_GO_VERSION}.darwin-amd64.pkg GO_PKG_DARWIN_SHA=0f215de06019a054a3da46a0722989986c956d719c7a0a8fc38a5f3c216d6f6b diff --git a/script/test b/script/test index 7e757b5..38e7505 100755 --- a/script/test +++ b/script/test @@ -13,5 +13,6 @@ script/build cd .gopath/src/github.com/github/gh-ost +# TODO: remove GO111MODULE once gh-ost uses Go modules echo "Running unit tests" -go test ./go/... +GO111MODULE=off go test ./go/... diff --git a/test.sh b/test.sh index ff514fa..16095bf 100755 --- a/test.sh +++ b/test.sh @@ -5,7 +5,10 @@ retval=0 for testsuite in base mysql sql do pushd go/${testsuite} > /dev/null; - go test $*; + + # TODO: remove GO111MODULE once gh-ost uses Go modules + GO111MODULE=off go test $*; + [ $? -ne 0 ] && retval=1 popd > /dev/null; done From c41823ecc929303e9f6b77fc5ac6f8c73b3e760d Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Thu, 27 May 2021 21:00:58 +0300 Subject: [PATCH 2/3] All MySQL DBs limited to max 3 concurrent/idle connections #15 (#931) * v1.1.0 * WIP: copying AUTO_INCREMENT value to ghost table Initial commit: towards setting up a test suite Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * greping for 'expect_table_structure' content * Adding simple test for 'expect_table_structure' scenario * adding tests for AUTO_INCREMENT value after row deletes. Should initially fail * clear event beforehand * parsing AUTO_INCREMENT from alter query, reading AUTO_INCREMENT from original table, applying AUTO_INCREMENT value onto ghost table if applicable and user has not specified AUTO_INCREMENT in alter statement * support GetUint64 Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * minor update to test Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * adding test for user defined AUTO_INCREMENT statement * Generated column as part of UNIQUE (or PRIMARY) KEY Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * skip analysis of generated column data type in unique key Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * All MySQL DBs limited to max 3 concurrent/idle connections Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/logic/throttler.go | 7 +++++-- go/mysql/utils.go | 28 +++++++++++++++------------- 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/go/logic/throttler.go b/go/logic/throttler.go index d234ea6..abe8669 100644 --- a/go/logic/throttler.go +++ b/go/logic/throttler.go @@ -188,9 +188,12 @@ func (this *Throttler) collectControlReplicasLag() { dbUri := connectionConfig.GetDBUri("information_schema") var heartbeatValue string - if db, _, err := mysql.GetDB(this.migrationContext.Uuid, dbUri); err != nil { + db, _, err := mysql.GetDB(this.migrationContext.Uuid, dbUri) + if err != nil { return lag, err - } else if err = db.QueryRow(replicationLagQuery).Scan(&heartbeatValue); err != nil { + } + + if err := db.QueryRow(replicationLagQuery).Scan(&heartbeatValue); err != nil { return lag, err } diff --git a/go/mysql/utils.go b/go/mysql/utils.go index 17bb5fc..43a228e 100644 --- a/go/mysql/utils.go +++ b/go/mysql/utils.go @@ -18,8 +18,11 @@ import ( "github.com/outbrain/golib/sqlutils" ) -const MaxTableNameLength = 64 -const MaxReplicationPasswordLength = 32 +const ( + MaxTableNameLength = 64 + MaxReplicationPasswordLength = 32 + MaxDBPoolConnections = 3 +) type ReplicationLagResult struct { Key InstanceKey @@ -39,23 +42,22 @@ func (this *ReplicationLagResult) HasLag() bool { var knownDBs map[string]*gosql.DB = make(map[string]*gosql.DB) var knownDBsMutex = &sync.Mutex{} -func GetDB(migrationUuid string, mysql_uri string) (*gosql.DB, bool, error) { +func GetDB(migrationUuid string, mysql_uri string) (db *gosql.DB, exists bool, err error) { cacheKey := migrationUuid + ":" + mysql_uri knownDBsMutex.Lock() - defer func() { - knownDBsMutex.Unlock() - }() + defer knownDBsMutex.Unlock() - var exists bool - if _, exists = knownDBs[cacheKey]; !exists { - if db, err := gosql.Open("mysql", mysql_uri); err == nil { - knownDBs[cacheKey] = db - } else { - return db, exists, err + if db, exists = knownDBs[cacheKey]; !exists { + db, err = gosql.Open("mysql", mysql_uri) + if err != nil { + return nil, false, err } + db.SetMaxOpenConns(MaxDBPoolConnections) + db.SetMaxIdleConns(MaxDBPoolConnections) + knownDBs[cacheKey] = db } - return knownDBs[cacheKey], exists, nil + return db, exists, nil } // GetReplicationLagFromSlaveStatus returns replication lag for a given db; via SHOW SLAVE STATUS From f19f101610cbaaf67113aa4a1c223b702152dafb Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Mon, 31 May 2021 15:15:51 +0300 Subject: [PATCH 3/3] hooks: reporting GH_OST_ETA_SECONDS. ETA as part of migration context (#936) * v1.1.0 * WIP: copying AUTO_INCREMENT value to ghost table Initial commit: towards setting up a test suite Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * greping for 'expect_table_structure' content * Adding simple test for 'expect_table_structure' scenario * adding tests for AUTO_INCREMENT value after row deletes. Should initially fail * clear event beforehand * parsing AUTO_INCREMENT from alter query, reading AUTO_INCREMENT from original table, applying AUTO_INCREMENT value onto ghost table if applicable and user has not specified AUTO_INCREMENT in alter statement * support GetUint64 Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * minor update to test Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * adding test for user defined AUTO_INCREMENT statement * Generated column as part of UNIQUE (or PRIMARY) KEY Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * skip analysis of generated column data type in unique key Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * All MySQL DBs limited to max 3 concurrent/idle connections Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * hooks: reporting GH_OST_ETA_SECONDS. ETA stored as part of migration context Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * GH_OST_ETA_NANOSECONDS Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * N/A denoted by negative value Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * ETAUnknown constant Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/base/context.go | 19 +++++++++++++++++++ go/logic/hooks.go | 1 + go/logic/migrator.go | 19 ++++++++++++++----- 3 files changed, 34 insertions(+), 5 deletions(-) diff --git a/go/base/context.go b/go/base/context.go index 51b43df..b9badc4 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -52,6 +52,7 @@ const ( const ( HTTPStatusOK = 200 MaxEventsBatchSize = 1000 + ETAUnknown = math.MinInt64 ) var ( @@ -182,6 +183,7 @@ type MigrationContext struct { lastHeartbeatOnChangelogMutex *sync.Mutex CurrentLag int64 currentProgress uint64 + etaNanoseonds int64 ThrottleHTTPStatusCode int64 controlReplicasLagResult mysql.ReplicationLagResult TotalRowsCopied int64 @@ -267,6 +269,7 @@ func NewMigrationContext() *MigrationContext { MaxLagMillisecondsThrottleThreshold: 1500, CutOverLockTimeoutSeconds: 3, DMLBatchSize: 10, + etaNanoseonds: ETAUnknown, maxLoad: NewLoadMap(), criticalLoad: NewLoadMap(), throttleMutex: &sync.Mutex{}, @@ -474,6 +477,22 @@ func (this *MigrationContext) SetProgressPct(progressPct float64) { atomic.StoreUint64(&this.currentProgress, math.Float64bits(progressPct)) } +func (this *MigrationContext) GetETADuration() time.Duration { + return time.Duration(atomic.LoadInt64(&this.etaNanoseonds)) +} + +func (this *MigrationContext) SetETADuration(etaDuration time.Duration) { + atomic.StoreInt64(&this.etaNanoseonds, etaDuration.Nanoseconds()) +} + +func (this *MigrationContext) GetETASeconds() int64 { + nano := atomic.LoadInt64(&this.etaNanoseonds) + if nano < 0 { + return ETAUnknown + } + return nano / int64(time.Second) +} + // math.Float64bits([f=0..100]) // GetTotalRowsCopied returns the accurate number of rows being copied (affected) diff --git a/go/logic/hooks.go b/go/logic/hooks.go index 71f070c..2275ede 100644 --- a/go/logic/hooks.go +++ b/go/logic/hooks.go @@ -66,6 +66,7 @@ func (this *HooksExecutor) applyEnvironmentVariables(extraVariables ...string) [ env = append(env, fmt.Sprintf("GH_OST_INSPECTED_LAG=%f", this.migrationContext.GetCurrentLagDuration().Seconds())) env = append(env, fmt.Sprintf("GH_OST_HEARTBEAT_LAG=%f", this.migrationContext.TimeSinceLastHeartbeatOnChangelog().Seconds())) env = append(env, fmt.Sprintf("GH_OST_PROGRESS=%f", this.migrationContext.GetProgressPct())) + env = append(env, fmt.Sprintf("GH_OST_ETA_SECONDS=%d", this.migrationContext.GetETASeconds())) env = append(env, fmt.Sprintf("GH_OST_HOOKS_HINT=%s", this.migrationContext.HooksHintMessage)) env = append(env, fmt.Sprintf("GH_OST_HOOKS_HINT_OWNER=%s", this.migrationContext.HooksHintOwner)) env = append(env, fmt.Sprintf("GH_OST_HOOKS_HINT_TOKEN=%s", this.migrationContext.HooksHintToken)) diff --git a/go/logic/migrator.go b/go/logic/migrator.go index dfddccf..c12c21f 100644 --- a/go/logic/migrator.go +++ b/go/logic/migrator.go @@ -939,20 +939,29 @@ func (this *Migrator) printStatus(rule PrintStatusRule, writers ...io.Writer) { } var etaSeconds float64 = math.MaxFloat64 - eta := "N/A" + var etaDuration = time.Duration(base.ETAUnknown) if progressPct >= 100.0 { - eta = "due" + etaDuration = 0 } else if progressPct >= 0.1 { elapsedRowCopySeconds := this.migrationContext.ElapsedRowCopyTime().Seconds() totalExpectedSeconds := elapsedRowCopySeconds * float64(rowsEstimate) / float64(totalRowsCopied) etaSeconds = totalExpectedSeconds - elapsedRowCopySeconds if etaSeconds >= 0 { - etaDuration := time.Duration(etaSeconds) * time.Second - eta = base.PrettifyDurationOutput(etaDuration) + etaDuration = time.Duration(etaSeconds) * time.Second } else { - eta = "due" + etaDuration = 0 } } + this.migrationContext.SetETADuration(etaDuration) + var eta string + switch etaDuration { + case 0: + eta = "due" + case time.Duration(base.ETAUnknown): + eta = "N/A" + default: + eta = base.PrettifyDurationOutput(etaDuration) + } state := "migrating" if atomic.LoadInt64(&this.migrationContext.CountingRowsFlag) > 0 && !this.migrationContext.ConcurrentCountTableRows {