From 54000ab5169921f2cfd5541e06e89f7f2b65797c Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Sun, 7 Mar 2021 10:47:40 +0200 Subject: [PATCH 1/4] hooks: reporting GH_OST_ETA_SECONDS. ETA stored as part of migration context Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/base/context.go | 9 +++++++++ go/logic/hooks.go | 1 + go/logic/migrator.go | 19 ++++++++++++++----- 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/go/base/context.go b/go/base/context.go index 51b43df..97d0d60 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -182,6 +182,7 @@ type MigrationContext struct { lastHeartbeatOnChangelogMutex *sync.Mutex CurrentLag int64 currentProgress uint64 + etaNanoseonds int64 ThrottleHTTPStatusCode int64 controlReplicasLagResult mysql.ReplicationLagResult TotalRowsCopied int64 @@ -474,6 +475,14 @@ func (this *MigrationContext) SetProgressPct(progressPct float64) { atomic.StoreUint64(&this.currentProgress, math.Float64bits(progressPct)) } +func (this *MigrationContext) GetETADuration() time.Duration { + return time.Duration(atomic.LoadInt64(&this.etaNanoseonds)) +} + +func (this *MigrationContext) SetETADuration(etaDuration time.Duration) { + atomic.StoreInt64(&this.etaNanoseonds, etaDuration.Nanoseconds()) +} + // math.Float64bits([f=0..100]) // GetTotalRowsCopied returns the accurate number of rows being copied (affected) diff --git a/go/logic/hooks.go b/go/logic/hooks.go index 71f070c..ea7822e 100644 --- a/go/logic/hooks.go +++ b/go/logic/hooks.go @@ -66,6 +66,7 @@ func (this *HooksExecutor) applyEnvironmentVariables(extraVariables ...string) [ env = append(env, fmt.Sprintf("GH_OST_INSPECTED_LAG=%f", this.migrationContext.GetCurrentLagDuration().Seconds())) env = append(env, fmt.Sprintf("GH_OST_HEARTBEAT_LAG=%f", this.migrationContext.TimeSinceLastHeartbeatOnChangelog().Seconds())) env = append(env, fmt.Sprintf("GH_OST_PROGRESS=%f", this.migrationContext.GetProgressPct())) + env = append(env, fmt.Sprintf("GH_OST_ETA_SECONDS=%f", this.migrationContext.GetETADuration().Seconds())) env = append(env, fmt.Sprintf("GH_OST_HOOKS_HINT=%s", this.migrationContext.HooksHintMessage)) env = append(env, fmt.Sprintf("GH_OST_HOOKS_HINT_OWNER=%s", this.migrationContext.HooksHintOwner)) env = append(env, fmt.Sprintf("GH_OST_HOOKS_HINT_TOKEN=%s", this.migrationContext.HooksHintToken)) diff --git a/go/logic/migrator.go b/go/logic/migrator.go index dfddccf..e34b2a7 100644 --- a/go/logic/migrator.go +++ b/go/logic/migrator.go @@ -939,20 +939,29 @@ func (this *Migrator) printStatus(rule PrintStatusRule, writers ...io.Writer) { } var etaSeconds float64 = math.MaxFloat64 - eta := "N/A" + var etaDuration = time.Duration(math.MaxInt64) if progressPct >= 100.0 { - eta = "due" + etaDuration = 0 } else if progressPct >= 0.1 { elapsedRowCopySeconds := this.migrationContext.ElapsedRowCopyTime().Seconds() totalExpectedSeconds := elapsedRowCopySeconds * float64(rowsEstimate) / float64(totalRowsCopied) etaSeconds = totalExpectedSeconds - elapsedRowCopySeconds if etaSeconds >= 0 { - etaDuration := time.Duration(etaSeconds) * time.Second - eta = base.PrettifyDurationOutput(etaDuration) + etaDuration = time.Duration(etaSeconds) * time.Second } else { - eta = "due" + etaDuration = 0 } } + this.migrationContext.SetETADuration(etaDuration) + var eta string + switch etaDuration { + case 0: + eta = "due" + case time.Duration(math.MaxInt64): + eta = "N/A" + default: + eta = base.PrettifyDurationOutput(etaDuration) + } state := "migrating" if atomic.LoadInt64(&this.migrationContext.CountingRowsFlag) > 0 && !this.migrationContext.ConcurrentCountTableRows { From 51719a2b769f72f5eee3c869fa1b7eb602d376fd Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Sun, 7 Mar 2021 11:11:50 +0200 Subject: [PATCH 2/4] GH_OST_ETA_NANOSECONDS Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/logic/hooks.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/logic/hooks.go b/go/logic/hooks.go index ea7822e..481c3df 100644 --- a/go/logic/hooks.go +++ b/go/logic/hooks.go @@ -66,7 +66,7 @@ func (this *HooksExecutor) applyEnvironmentVariables(extraVariables ...string) [ env = append(env, fmt.Sprintf("GH_OST_INSPECTED_LAG=%f", this.migrationContext.GetCurrentLagDuration().Seconds())) env = append(env, fmt.Sprintf("GH_OST_HEARTBEAT_LAG=%f", this.migrationContext.TimeSinceLastHeartbeatOnChangelog().Seconds())) env = append(env, fmt.Sprintf("GH_OST_PROGRESS=%f", this.migrationContext.GetProgressPct())) - env = append(env, fmt.Sprintf("GH_OST_ETA_SECONDS=%f", this.migrationContext.GetETADuration().Seconds())) + env = append(env, fmt.Sprintf("GH_OST_ETA_NANOSECONDS=%d", this.migrationContext.GetETADuration().Nanoseconds())) env = append(env, fmt.Sprintf("GH_OST_HOOKS_HINT=%s", this.migrationContext.HooksHintMessage)) env = append(env, fmt.Sprintf("GH_OST_HOOKS_HINT_OWNER=%s", this.migrationContext.HooksHintOwner)) env = append(env, fmt.Sprintf("GH_OST_HOOKS_HINT_TOKEN=%s", this.migrationContext.HooksHintToken)) From 76b9c16a68b43f24bfd658cfee8a71cf7dd74806 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Sun, 7 Mar 2021 11:27:50 +0200 Subject: [PATCH 3/4] N/A denoted by negative value Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/base/context.go | 5 +++++ go/logic/hooks.go | 2 +- go/logic/migrator.go | 4 ++-- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/go/base/context.go b/go/base/context.go index 97d0d60..414ead4 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -483,6 +483,11 @@ func (this *MigrationContext) SetETADuration(etaDuration time.Duration) { atomic.StoreInt64(&this.etaNanoseonds, etaDuration.Nanoseconds()) } +func (this *MigrationContext) GetETASeconds() int64 { + nano := atomic.LoadInt64(&this.etaNanoseonds) + return nano / int64(time.Second) +} + // math.Float64bits([f=0..100]) // GetTotalRowsCopied returns the accurate number of rows being copied (affected) diff --git a/go/logic/hooks.go b/go/logic/hooks.go index 481c3df..2275ede 100644 --- a/go/logic/hooks.go +++ b/go/logic/hooks.go @@ -66,7 +66,7 @@ func (this *HooksExecutor) applyEnvironmentVariables(extraVariables ...string) [ env = append(env, fmt.Sprintf("GH_OST_INSPECTED_LAG=%f", this.migrationContext.GetCurrentLagDuration().Seconds())) env = append(env, fmt.Sprintf("GH_OST_HEARTBEAT_LAG=%f", this.migrationContext.TimeSinceLastHeartbeatOnChangelog().Seconds())) env = append(env, fmt.Sprintf("GH_OST_PROGRESS=%f", this.migrationContext.GetProgressPct())) - env = append(env, fmt.Sprintf("GH_OST_ETA_NANOSECONDS=%d", this.migrationContext.GetETADuration().Nanoseconds())) + env = append(env, fmt.Sprintf("GH_OST_ETA_SECONDS=%d", this.migrationContext.GetETASeconds())) env = append(env, fmt.Sprintf("GH_OST_HOOKS_HINT=%s", this.migrationContext.HooksHintMessage)) env = append(env, fmt.Sprintf("GH_OST_HOOKS_HINT_OWNER=%s", this.migrationContext.HooksHintOwner)) env = append(env, fmt.Sprintf("GH_OST_HOOKS_HINT_TOKEN=%s", this.migrationContext.HooksHintToken)) diff --git a/go/logic/migrator.go b/go/logic/migrator.go index e34b2a7..9ef90e1 100644 --- a/go/logic/migrator.go +++ b/go/logic/migrator.go @@ -939,7 +939,7 @@ func (this *Migrator) printStatus(rule PrintStatusRule, writers ...io.Writer) { } var etaSeconds float64 = math.MaxFloat64 - var etaDuration = time.Duration(math.MaxInt64) + var etaDuration = time.Duration(math.MinInt64) if progressPct >= 100.0 { etaDuration = 0 } else if progressPct >= 0.1 { @@ -957,7 +957,7 @@ func (this *Migrator) printStatus(rule PrintStatusRule, writers ...io.Writer) { switch etaDuration { case 0: eta = "due" - case time.Duration(math.MaxInt64): + case time.Duration(math.MinInt64): eta = "N/A" default: eta = base.PrettifyDurationOutput(etaDuration) From b688c58a45dfc34fe4bc59eb84d1c3504bc8c2ac Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Sun, 7 Mar 2021 14:16:04 +0200 Subject: [PATCH 4/4] ETAUnknown constant Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/base/context.go | 5 +++++ go/logic/migrator.go | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/go/base/context.go b/go/base/context.go index 414ead4..b9badc4 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -52,6 +52,7 @@ const ( const ( HTTPStatusOK = 200 MaxEventsBatchSize = 1000 + ETAUnknown = math.MinInt64 ) var ( @@ -268,6 +269,7 @@ func NewMigrationContext() *MigrationContext { MaxLagMillisecondsThrottleThreshold: 1500, CutOverLockTimeoutSeconds: 3, DMLBatchSize: 10, + etaNanoseonds: ETAUnknown, maxLoad: NewLoadMap(), criticalLoad: NewLoadMap(), throttleMutex: &sync.Mutex{}, @@ -485,6 +487,9 @@ func (this *MigrationContext) SetETADuration(etaDuration time.Duration) { func (this *MigrationContext) GetETASeconds() int64 { nano := atomic.LoadInt64(&this.etaNanoseonds) + if nano < 0 { + return ETAUnknown + } return nano / int64(time.Second) } diff --git a/go/logic/migrator.go b/go/logic/migrator.go index 9ef90e1..c12c21f 100644 --- a/go/logic/migrator.go +++ b/go/logic/migrator.go @@ -939,7 +939,7 @@ func (this *Migrator) printStatus(rule PrintStatusRule, writers ...io.Writer) { } var etaSeconds float64 = math.MaxFloat64 - var etaDuration = time.Duration(math.MinInt64) + var etaDuration = time.Duration(base.ETAUnknown) if progressPct >= 100.0 { etaDuration = 0 } else if progressPct >= 0.1 { @@ -957,7 +957,7 @@ func (this *Migrator) printStatus(rule PrintStatusRule, writers ...io.Writer) { switch etaDuration { case 0: eta = "due" - case time.Duration(math.MinInt64): + case time.Duration(base.ETAUnknown): eta = "N/A" default: eta = base.PrettifyDurationOutput(etaDuration)