From cdf393a30e4f019fa69766eaa075ad32ce7050fc Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Fri, 19 Aug 2016 14:52:49 +0200 Subject: [PATCH 01/20] initial support for hooks --- go/logic/hooks.go | 87 ++++++++++++++++++++++++++++++++++++++++++++ go/logic/migrator.go | 45 ++++++++++++++++++++++- 2 files changed, 130 insertions(+), 2 deletions(-) create mode 100644 go/logic/hooks.go diff --git a/go/logic/hooks.go b/go/logic/hooks.go new file mode 100644 index 0000000..2d71dc7 --- /dev/null +++ b/go/logic/hooks.go @@ -0,0 +1,87 @@ +/* + Copyright 2016 GitHub Inc. + See https://github.com/github/gh-ost/blob/master/LICENSE +*/ + +package logic + +import ( + "fmt" + "os" + "os/exec" + + "github.com/github/gh-ost/go/base" + "github.com/openark/golib/log" +) + +type HooksExecutor struct { + migrationContext *base.MigrationContext +} + +func NewHooksExecutor() *HooksExecutor { + return &HooksExecutor{ + migrationContext: base.GetMigrationContext(), + } +} + +func (this *HooksExecutor) detectHooks() error { + return nil +} + +func (this *HooksExecutor) applyEnvironmentVairables() []string { + env := os.Environ() + env = append(env, fmt.Sprintf("GH_OST_DATABASE_NAME=%s", this.migrationContext.DatabaseName)) + env = append(env, fmt.Sprintf("GH_OST_TABLE_NAME=%s", this.migrationContext.OriginalTableName)) + env = append(env, fmt.Sprintf("GH_OST_GHOST_TABLE_NAME=%s", this.migrationContext.GetGhostTableName())) + env = append(env, fmt.Sprintf("GH_OST_OLD_TABLE_NAME=%s", this.migrationContext.GetOldTableName())) + env = append(env, fmt.Sprintf("GH_OST_DDL=%s", this.migrationContext.AlterStatement)) + env = append(env, fmt.Sprintf("GH_OST_ELAPSED_SECONDS=%f", this.migrationContext.ElapsedTime().Seconds())) + return env +} + +// commandRun executes a command with arguments, and set relevant environment variables +func (this *HooksExecutor) commandRun(commandText string, arguments ...string) error { + cmd := exec.Command(commandText, arguments...) + cmd.Env = this.applyEnvironmentVairables() + + if err := cmd.Run(); err != nil { + return log.Errore(err) + } + return nil +} + +func (this *HooksExecutor) onStartup() error { + return nil +} + +func (this *HooksExecutor) onValidated() error { + return nil +} + +func (this *HooksExecutor) onAboutToRowCopy() error { + return nil +} + +func (this *HooksExecutor) onRowCopyComplete() error { + return nil +} + +func (this *HooksExecutor) onBeginPostponed() error { + return nil +} + +func (this *HooksExecutor) onAboutToCutOver() error { + return nil +} + +func (this *HooksExecutor) onInteractiveCommand(command string) error { + return nil +} + +func (this *HooksExecutor) onSuccess() error { + return nil +} + +func (this *HooksExecutor) onFailure() error { + return nil +} diff --git a/go/logic/migrator.go b/go/logic/migrator.go index 600381e..2850e91 100644 --- a/go/logic/migrator.go +++ b/go/logic/migrator.go @@ -56,6 +56,7 @@ type Migrator struct { applier *Applier eventsStreamer *EventsStreamer server *Server + hooksExecutor *HooksExecutor migrationContext *base.MigrationContext hostname string @@ -110,6 +111,15 @@ func (this *Migrator) acceptSignals() { }() } +// initiateHooksExecutor +func (this *Migrator) initiateHooksExecutor() (err error) { + this.hooksExecutor = NewHooksExecutor() + if err := this.hooksExecutor.detectHooks(); err != nil { + return err + } + return nil +} + // shouldThrottle performs checks to see whether we should currently be throttling. // It also checks for critical-load and panic aborts. func (this *Migrator) shouldThrottle() (result bool, reason string) { @@ -277,7 +287,7 @@ func (this *Migrator) executeAndThrottleOnError(operation func() error) (err err } // consumeRowCopyComplete blocks on the rowCopyComplete channel once, and then -// consumers and drops any further incoming events that may be left hanging. +// consumes and drops any further incoming events that may be left hanging. func (this *Migrator) consumeRowCopyComplete() { <-this.rowCopyComplete atomic.StoreInt64(&this.rowCopyCompleteFlag, 1) @@ -371,6 +381,12 @@ func (this *Migrator) Migrate() (err error) { go this.listenOnPanicAbort() + if err := this.initiateHooksExecutor(); err != nil { + return err + } + if err := this.hooksExecutor.onStartup(); err != nil { + return err + } if err := this.parser.ParseAlterStatement(this.migrationContext.AlterStatement); err != nil { return err } @@ -397,6 +413,11 @@ func (this *Migrator) Migrate() (err error) { if err := this.inspector.InspectOriginalAndGhostTables(); err != nil { return err } + // Validation complete! We're good to execute this migration + if err := this.hooksExecutor.onValidated(); err != nil { + return err + } + if err := this.initiateServer(); err != nil { return err } @@ -419,6 +440,9 @@ func (this *Migrator) Migrate() (err error) { return err } go this.initiateThrottler() + if err := this.hooksExecutor.onAboutToRowCopy(); err != nil { + return err + } go this.executeWriteFuncs() go this.iterateChunks() this.migrationContext.MarkRowCopyStartTime() @@ -427,8 +451,14 @@ func (this *Migrator) Migrate() (err error) { log.Debugf("Operating until row copy is complete") this.consumeRowCopyComplete() log.Infof("Row copy complete") + if err := this.hooksExecutor.onRowCopyComplete(); err != nil { + return err + } this.printStatus(ForcePrintStatusRule) + if err := this.hooksExecutor.onAboutToCutOver(); err != nil { + return err + } if err := this.cutOver(); err != nil { return err } @@ -436,6 +466,9 @@ func (this *Migrator) Migrate() (err error) { if err := this.finalCleanup(); err != nil { return nil } + if err := this.hooksExecutor.onSuccess(); err != nil { + return err + } log.Infof("Done migrating %s.%s", sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName)) return nil } @@ -463,8 +496,12 @@ func (this *Migrator) cutOver() (err error) { } if base.FileExists(this.migrationContext.PostponeCutOverFlagFile) { // Throttle file defined and exists! + if atomic.LoadInt64(&this.migrationContext.IsPostponingCutOver) == 0 { + if err := this.hooksExecutor.onBeginPostponed(); err != nil { + return true, err + } + } atomic.StoreInt64(&this.migrationContext.IsPostponingCutOver, 1) - //log.Debugf("Postponing final table swap as flag file exists: %+v", this.migrationContext.PostponeCutOverFlagFile) return true, nil } return false, nil @@ -659,6 +696,10 @@ func (this *Migrator) onServerCommand(command string, writer *bufio.Writer) (err throttleHint := "# Note: you may only throttle for as long as your binary logs are not purged\n" + if err := this.hooksExecutor.onInteractiveCommand(command); err != nil { + return err + } + switch command { case "help": { From 6acbe7e3ae34b8510299b4e55da5705d4fbf95e2 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Sat, 20 Aug 2016 08:24:20 +0200 Subject: [PATCH 02/20] detecting and executing hooks --- go/base/context.go | 1 + go/logic/hooks.go | 31 +++++++++++++++++++++++++++---- go/logic/migrator.go | 2 +- 3 files changed, 29 insertions(+), 5 deletions(-) diff --git a/go/base/context.go b/go/base/context.go index ea5ab68..78a7a41 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -78,6 +78,7 @@ type MigrationContext struct { PostponeCutOverFlagFile string CutOverLockTimeoutSeconds int64 PanicFlagFile string + HooksPath string DropServeSocket bool ServeSocketFile string diff --git a/go/logic/hooks.go b/go/logic/hooks.go index 2d71dc7..c6c5981 100644 --- a/go/logic/hooks.go +++ b/go/logic/hooks.go @@ -9,6 +9,7 @@ import ( "fmt" "os" "os/exec" + "path/filepath" "github.com/github/gh-ost/go/base" "github.com/openark/golib/log" @@ -24,7 +25,7 @@ func NewHooksExecutor() *HooksExecutor { } } -func (this *HooksExecutor) detectHooks() error { +func (this *HooksExecutor) initHooks() error { return nil } @@ -39,9 +40,9 @@ func (this *HooksExecutor) applyEnvironmentVairables() []string { return env } -// commandRun executes a command with arguments, and set relevant environment variables -func (this *HooksExecutor) commandRun(commandText string, arguments ...string) error { - cmd := exec.Command(commandText, arguments...) +// executeHook executes a command with arguments, and set relevant environment variables +func (this *HooksExecutor) executeHook(hook string, arguments ...string) error { + cmd := exec.Command(hook, arguments...) cmd.Env = this.applyEnvironmentVairables() if err := cmd.Run(); err != nil { @@ -50,6 +51,28 @@ func (this *HooksExecutor) commandRun(commandText string, arguments ...string) e return nil } +func (this *HooksExecutor) detectHooks(baseName string) (hooks []string, err error) { + if this.migrationContext.HooksPath == "" { + return hooks, err + } + pattern := fmt.Sprintf("%s/%s*", this.migrationContext.HooksPath, baseName) + hooks, err = filepath.Glob(pattern) + return hooks, err +} + +func (this *HooksExecutor) executeHooks(baseName string) error { + hooks, err := this.detectHooks(baseName) + if err != nil { + return err + } + for _, hook := range hooks { + if err := this.executeHook(hook); err != nil { + return err + } + } + return nil +} + func (this *HooksExecutor) onStartup() error { return nil } diff --git a/go/logic/migrator.go b/go/logic/migrator.go index 2850e91..efad40a 100644 --- a/go/logic/migrator.go +++ b/go/logic/migrator.go @@ -114,7 +114,7 @@ func (this *Migrator) acceptSignals() { // initiateHooksExecutor func (this *Migrator) initiateHooksExecutor() (err error) { this.hooksExecutor = NewHooksExecutor() - if err := this.hooksExecutor.detectHooks(); err != nil { + if err := this.hooksExecutor.initHooks(); err != nil { return err } return nil From 972728cf40f468f9dbb646cbf0a3ba21fc4a720c Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Mon, 22 Aug 2016 16:24:41 +0200 Subject: [PATCH 03/20] added onStatus hook --- go/logic/hooks.go | 4 ++++ go/logic/migrator.go | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/go/logic/hooks.go b/go/logic/hooks.go index c6c5981..58c1403 100644 --- a/go/logic/hooks.go +++ b/go/logic/hooks.go @@ -108,3 +108,7 @@ func (this *HooksExecutor) onSuccess() error { func (this *HooksExecutor) onFailure() error { return nil } + +func (this *HooksExecutor) onStatus(statusMessage string) error { + return nil +} diff --git a/go/logic/migrator.go b/go/logic/migrator.go index efad40a..9d1ba6a 100644 --- a/go/logic/migrator.go +++ b/go/logic/migrator.go @@ -1065,6 +1065,10 @@ func (this *Migrator) printStatus(rule PrintStatusRule, writers ...io.Writer) { ) w := io.MultiWriter(writers...) fmt.Fprintln(w, status) + + if elapsedSeconds%60 == 0 { + this.hooksExecutor.onStatus(status) + } } // initiateHeartbeatListener listens for heartbeat events. gh-ost implements its own From 1c2a77ef95f2b2874afb474ed14b03230bccb050 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Tue, 23 Aug 2016 11:35:48 +0200 Subject: [PATCH 04/20] hook names; added on-stop-replication hook --- go/base/context.go | 1 + go/cmd/gh-ost/main.go | 1 + go/logic/hooks.go | 61 +++++++++++++++++++++++++++++++------------ go/logic/migrator.go | 12 ++++++--- 4 files changed, 55 insertions(+), 20 deletions(-) diff --git a/go/base/context.go b/go/base/context.go index 78a7a41..438490b 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -92,6 +92,7 @@ type MigrationContext struct { InitiallyDropGhostTable bool CutOverType CutOver + Hostname string TableEngine string RowsEstimate int64 UsedRowsEstimateMethod RowsEstimateMethod diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index b614831..501d04b 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -189,6 +189,7 @@ func main() { migrator := logic.NewMigrator() err := migrator.Migrate() if err != nil { + migrator.ExecOnFailureHook() log.Fatale(err) } log.Info("Done") diff --git a/go/logic/hooks.go b/go/logic/hooks.go index 58c1403..1c562ef 100644 --- a/go/logic/hooks.go +++ b/go/logic/hooks.go @@ -15,6 +15,20 @@ import ( "github.com/openark/golib/log" ) +const ( + onStartup = "gh-ost-on-startup" + onValidated = "gh-ost-on-validated" + onAboutToRowCopy = "gh-ost-on-about-row-copy" + onRowCopyComplete = "gh-ost-on-row-copy-complete" + onBeginPostponed = "gh-ost-on-begin-postponed" + onAboutToCutOver = "gh-ost-on-about-cut-over" + onInteractiveCommand = "gh-ost-on-interactive-command" + onSuccess = "gh-ost-on-success" + onFailure = "gh-ost-on-failure" + onStatus = "gh-ost-on-status" + onStopReplication = "gh-ost-on-stop-replication" +) + type HooksExecutor struct { migrationContext *base.MigrationContext } @@ -29,7 +43,7 @@ func (this *HooksExecutor) initHooks() error { return nil } -func (this *HooksExecutor) applyEnvironmentVairables() []string { +func (this *HooksExecutor) applyEnvironmentVairables(extraVariables ...string) []string { env := os.Environ() env = append(env, fmt.Sprintf("GH_OST_DATABASE_NAME=%s", this.migrationContext.DatabaseName)) env = append(env, fmt.Sprintf("GH_OST_TABLE_NAME=%s", this.migrationContext.OriginalTableName)) @@ -37,13 +51,20 @@ func (this *HooksExecutor) applyEnvironmentVairables() []string { env = append(env, fmt.Sprintf("GH_OST_OLD_TABLE_NAME=%s", this.migrationContext.GetOldTableName())) env = append(env, fmt.Sprintf("GH_OST_DDL=%s", this.migrationContext.AlterStatement)) env = append(env, fmt.Sprintf("GH_OST_ELAPSED_SECONDS=%f", this.migrationContext.ElapsedTime().Seconds())) + env = append(env, fmt.Sprintf("GH_OST_MIGRATED_HOST=%s", this.migrationContext.ApplierConnectionConfig.ImpliedKey.Hostname)) + env = append(env, fmt.Sprintf("GH_OST_INSPECTED_HOST=%s", this.migrationContext.InspectorConnectionConfig.ImpliedKey.Hostname)) + env = append(env, fmt.Sprintf("GH_OST_EXECUTING_HOST=%s", this.migrationContext.Hostname)) + + for _, variable := range extraVariables { + env = append(env, variable) + } return env } -// executeHook executes a command with arguments, and set relevant environment variables -func (this *HooksExecutor) executeHook(hook string, arguments ...string) error { - cmd := exec.Command(hook, arguments...) - cmd.Env = this.applyEnvironmentVairables() +// executeHook executes a command, and sets relevant environment variables +func (this *HooksExecutor) executeHook(hook string, extraVariables ...string) error { + cmd := exec.Command(hook) + cmd.Env = this.applyEnvironmentVairables(extraVariables...) if err := cmd.Run(); err != nil { return log.Errore(err) @@ -60,13 +81,13 @@ func (this *HooksExecutor) detectHooks(baseName string) (hooks []string, err err return hooks, err } -func (this *HooksExecutor) executeHooks(baseName string) error { +func (this *HooksExecutor) executeHooks(baseName string, extraVariables ...string) error { hooks, err := this.detectHooks(baseName) if err != nil { return err } for _, hook := range hooks { - if err := this.executeHook(hook); err != nil { + if err := this.executeHook(hook, extraVariables...); err != nil { return err } } @@ -74,41 +95,47 @@ func (this *HooksExecutor) executeHooks(baseName string) error { } func (this *HooksExecutor) onStartup() error { - return nil + return this.executeHooks(onStartup) } func (this *HooksExecutor) onValidated() error { - return nil + return this.executeHooks(onValidated) } func (this *HooksExecutor) onAboutToRowCopy() error { - return nil + return this.executeHooks(onAboutToRowCopy) } func (this *HooksExecutor) onRowCopyComplete() error { - return nil + return this.executeHooks(onRowCopyComplete) } func (this *HooksExecutor) onBeginPostponed() error { - return nil + return this.executeHooks(onBeginPostponed) } func (this *HooksExecutor) onAboutToCutOver() error { - return nil + return this.executeHooks(onAboutToCutOver) } func (this *HooksExecutor) onInteractiveCommand(command string) error { - return nil + v := fmt.Sprintf("GH_OST_COMMAND='%s'", command) + return this.executeHooks(onInteractiveCommand, v) } func (this *HooksExecutor) onSuccess() error { - return nil + return this.executeHooks(onSuccess) } func (this *HooksExecutor) onFailure() error { - return nil + return this.executeHooks(onFailure) } func (this *HooksExecutor) onStatus(statusMessage string) error { - return nil + v := fmt.Sprintf("GH_OST_STATUS='%s'", statusMessage) + return this.executeHooks(onStatus, v) +} + +func (this *HooksExecutor) onStopReplication() error { + return this.executeHooks(onStopReplication) } diff --git a/go/logic/migrator.go b/go/logic/migrator.go index 9d1ba6a..17c37c7 100644 --- a/go/logic/migrator.go +++ b/go/logic/migrator.go @@ -58,7 +58,6 @@ type Migrator struct { server *Server hooksExecutor *HooksExecutor migrationContext *base.MigrationContext - hostname string tablesInPlace chan bool rowCopyComplete chan bool @@ -375,7 +374,7 @@ func (this *Migrator) validateStatement() (err error) { func (this *Migrator) Migrate() (err error) { log.Infof("Migrating %s.%s", sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName)) this.migrationContext.StartTime = time.Now() - if this.hostname, err = os.Hostname(); err != nil { + if this.migrationContext.Hostname, err = os.Hostname(); err != nil { return err } @@ -473,6 +472,12 @@ func (this *Migrator) Migrate() (err error) { return nil } +// ExecOnFailureHook executes the onFailure hook, and this method is provided as the only external +// hook access point +func (this *Migrator) ExecOnFailureHook() (err error) { + return this.hooksExecutor.onFailure() +} + // cutOver performs the final step of migration, based on migration // type (on replica? bumpy? safe?) func (this *Migrator) cutOver() (err error) { @@ -516,6 +521,7 @@ func (this *Migrator) cutOver() (err error) { // and swap the tables. // The difference is that we will later swap the tables back. log.Debugf("testing on replica. Stopping replication IO thread") + this.hooksExecutor.onStopReplication() if err := this.retryOperation(this.applier.StopReplication); err != nil { return err } @@ -909,7 +915,7 @@ func (this *Migrator) printMigrationStatusHint(writers ...io.Writer) { fmt.Fprintln(w, fmt.Sprintf("# Migrating %+v; inspecting %+v; executing on %+v", *this.applier.connectionConfig.ImpliedKey, *this.inspector.connectionConfig.ImpliedKey, - this.hostname, + this.migrationContext.Hostname, )) fmt.Fprintln(w, fmt.Sprintf("# Migration started at %+v", this.migrationContext.StartTime.Format(time.RubyDate), From a1e191078ac65c2423298ff966e87c7f96952403 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Tue, 23 Aug 2016 11:40:32 +0200 Subject: [PATCH 05/20] rename 'about'->'before' --- go/logic/hooks.go | 12 ++++++------ go/logic/migrator.go | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/go/logic/hooks.go b/go/logic/hooks.go index 1c562ef..dd74973 100644 --- a/go/logic/hooks.go +++ b/go/logic/hooks.go @@ -18,10 +18,10 @@ import ( const ( onStartup = "gh-ost-on-startup" onValidated = "gh-ost-on-validated" - onAboutToRowCopy = "gh-ost-on-about-row-copy" + onBeforeRowCopy = "gh-ost-on-before-row-copy" onRowCopyComplete = "gh-ost-on-row-copy-complete" onBeginPostponed = "gh-ost-on-begin-postponed" - onAboutToCutOver = "gh-ost-on-about-cut-over" + onBeforeCutOver = "gh-ost-on-before-cut-over" onInteractiveCommand = "gh-ost-on-interactive-command" onSuccess = "gh-ost-on-success" onFailure = "gh-ost-on-failure" @@ -102,8 +102,8 @@ func (this *HooksExecutor) onValidated() error { return this.executeHooks(onValidated) } -func (this *HooksExecutor) onAboutToRowCopy() error { - return this.executeHooks(onAboutToRowCopy) +func (this *HooksExecutor) onBeforeRowCopy() error { + return this.executeHooks(onBeforeRowCopy) } func (this *HooksExecutor) onRowCopyComplete() error { @@ -114,8 +114,8 @@ func (this *HooksExecutor) onBeginPostponed() error { return this.executeHooks(onBeginPostponed) } -func (this *HooksExecutor) onAboutToCutOver() error { - return this.executeHooks(onAboutToCutOver) +func (this *HooksExecutor) onBeforeCutOver() error { + return this.executeHooks(onBeforeCutOver) } func (this *HooksExecutor) onInteractiveCommand(command string) error { diff --git a/go/logic/migrator.go b/go/logic/migrator.go index 17c37c7..5956c95 100644 --- a/go/logic/migrator.go +++ b/go/logic/migrator.go @@ -439,7 +439,7 @@ func (this *Migrator) Migrate() (err error) { return err } go this.initiateThrottler() - if err := this.hooksExecutor.onAboutToRowCopy(); err != nil { + if err := this.hooksExecutor.onBeforeRowCopy(); err != nil { return err } go this.executeWriteFuncs() @@ -455,7 +455,7 @@ func (this *Migrator) Migrate() (err error) { } this.printStatus(ForcePrintStatusRule) - if err := this.hooksExecutor.onAboutToCutOver(); err != nil { + if err := this.hooksExecutor.onBeforeCutOver(); err != nil { return err } if err := this.cutOver(); err != nil { From b064174ab4aae5f4caa27ee5e14e44b16aeca839 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Thu, 25 Aug 2016 13:54:21 +0200 Subject: [PATCH 06/20] added elapsedSeconds to status hook --- go/logic/migrator.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/logic/migrator.go b/go/logic/migrator.go index 5aaf9f1..83dffdd 100644 --- a/go/logic/migrator.go +++ b/go/logic/migrator.go @@ -1077,7 +1077,7 @@ func (this *Migrator) printStatus(rule PrintStatusRule, writers ...io.Writer) { fmt.Fprintln(w, status) if elapsedSeconds%60 == 0 { - this.hooksExecutor.onStatus(status) + this.hooksExecutor.onStatus(status, elapsedSeconds) } } From fad6743150ff33ed44e7a64f1cdf632b78ad0781 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Thu, 25 Aug 2016 13:54:42 +0200 Subject: [PATCH 07/20] added hooks hint message --- go/base/context.go | 1 + 1 file changed, 1 insertion(+) diff --git a/go/base/context.go b/go/base/context.go index f1c3111..7a7a0a2 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -79,6 +79,7 @@ type MigrationContext struct { CutOverLockTimeoutSeconds int64 PanicFlagFile string HooksPath string + HooksHintMessage string DropServeSocket bool ServeSocketFile string From ecd33969bf40db6ef67e162b5428dde07702d4eb Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Thu, 25 Aug 2016 13:55:22 +0200 Subject: [PATCH 08/20] added onRowCountComplete(); supporting elapsedSeconds in on-status --- go/logic/hooks.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/go/logic/hooks.go b/go/logic/hooks.go index dd74973..7b831d8 100644 --- a/go/logic/hooks.go +++ b/go/logic/hooks.go @@ -18,6 +18,7 @@ import ( const ( onStartup = "gh-ost-on-startup" onValidated = "gh-ost-on-validated" + onRowCountComplete = "gh-ost-on-rowcount-complete" onBeforeRowCopy = "gh-ost-on-before-row-copy" onRowCopyComplete = "gh-ost-on-row-copy-complete" onBeginPostponed = "gh-ost-on-begin-postponed" @@ -54,6 +55,7 @@ func (this *HooksExecutor) applyEnvironmentVairables(extraVariables ...string) [ env = append(env, fmt.Sprintf("GH_OST_MIGRATED_HOST=%s", this.migrationContext.ApplierConnectionConfig.ImpliedKey.Hostname)) env = append(env, fmt.Sprintf("GH_OST_INSPECTED_HOST=%s", this.migrationContext.InspectorConnectionConfig.ImpliedKey.Hostname)) env = append(env, fmt.Sprintf("GH_OST_EXECUTING_HOST=%s", this.migrationContext.Hostname)) + env = append(env, fmt.Sprintf("GH_OST_HOOKS_HINT=%s", this.migrationContext.HooksHintMessage)) for _, variable := range extraVariables { env = append(env, variable) @@ -102,6 +104,9 @@ func (this *HooksExecutor) onValidated() error { return this.executeHooks(onValidated) } +func (this *HooksExecutor) onRowCountComplete() error { + return this.executeHooks(onRowCountComplete) +} func (this *HooksExecutor) onBeforeRowCopy() error { return this.executeHooks(onBeforeRowCopy) } @@ -131,9 +136,10 @@ func (this *HooksExecutor) onFailure() error { return this.executeHooks(onFailure) } -func (this *HooksExecutor) onStatus(statusMessage string) error { - v := fmt.Sprintf("GH_OST_STATUS='%s'", statusMessage) - return this.executeHooks(onStatus, v) +func (this *HooksExecutor) onStatus(statusMessage string, elapsedSeconds int64) error { + v0 := fmt.Sprintf("GH_OST_STATUS='%s'", statusMessage) + v1 := fmt.Sprintf("GH_OST_ELAPSED_SECONDS='%d'", elapsedSeconds) + return this.executeHooks(onStatus, v0, v1) } func (this *HooksExecutor) onStopReplication() error { From b60875f32ffdaf1151bcae46634406f5b6255ef3 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Thu, 25 Aug 2016 13:55:34 +0200 Subject: [PATCH 09/20] added and improved sample hooks --- resources/hooks-sample/gh-ost-on-before-cut-over-hook | 5 +++++ resources/hooks-sample/gh-ost-on-before-row-copy-hook | 5 +++++ resources/hooks-sample/gh-ost-on-begin-postponed-hook | 5 +++++ resources/hooks-sample/gh-ost-on-failure-hook | 5 +++++ resources/hooks-sample/gh-ost-on-interactive-command-hook | 5 +++++ resources/hooks-sample/gh-ost-on-row-copy-complete-hook | 5 +++++ resources/hooks-sample/gh-ost-on-rowcount-complete-hook | 5 +++++ resources/hooks-sample/gh-ost-on-startup-hook | 5 +++++ resources/hooks-sample/gh-ost-on-status-hook | 5 +++++ resources/hooks-sample/gh-ost-on-stop-replication-hook | 5 +++++ resources/hooks-sample/gh-ost-on-success-hook | 5 +++++ resources/hooks-sample/gh-ost-on-success-hook-2 | 5 +++++ resources/hooks-sample/gh-ost-on-validated-hook | 5 +++++ 13 files changed, 65 insertions(+) create mode 100755 resources/hooks-sample/gh-ost-on-before-cut-over-hook create mode 100755 resources/hooks-sample/gh-ost-on-before-row-copy-hook create mode 100755 resources/hooks-sample/gh-ost-on-begin-postponed-hook create mode 100755 resources/hooks-sample/gh-ost-on-failure-hook create mode 100755 resources/hooks-sample/gh-ost-on-interactive-command-hook create mode 100755 resources/hooks-sample/gh-ost-on-row-copy-complete-hook create mode 100755 resources/hooks-sample/gh-ost-on-rowcount-complete-hook create mode 100755 resources/hooks-sample/gh-ost-on-startup-hook create mode 100755 resources/hooks-sample/gh-ost-on-status-hook create mode 100755 resources/hooks-sample/gh-ost-on-stop-replication-hook create mode 100755 resources/hooks-sample/gh-ost-on-success-hook create mode 100755 resources/hooks-sample/gh-ost-on-success-hook-2 create mode 100755 resources/hooks-sample/gh-ost-on-validated-hook diff --git a/resources/hooks-sample/gh-ost-on-before-cut-over-hook b/resources/hooks-sample/gh-ost-on-before-cut-over-hook new file mode 100755 index 0000000..41bf2d0 --- /dev/null +++ b/resources/hooks-sample/gh-ost-on-before-cut-over-hook @@ -0,0 +1,5 @@ +#!/bin/bash + +# Sample hook file for gh-ost-on-before-cut-over + +echo "$(date) gh-ost-on-before-cut-over $GH_OST_DATABASE_NAME.$GH_OST_TABLE_NAME" >> /tmp/gh-ost.log diff --git a/resources/hooks-sample/gh-ost-on-before-row-copy-hook b/resources/hooks-sample/gh-ost-on-before-row-copy-hook new file mode 100755 index 0000000..4282bcd --- /dev/null +++ b/resources/hooks-sample/gh-ost-on-before-row-copy-hook @@ -0,0 +1,5 @@ +#!/bin/bash + +# Sample hook file for gh-ost-on-before-row-copy + +echo "$(date) gh-ost-on-before-row-copy $GH_OST_DATABASE_NAME.$GH_OST_TABLE_NAME" >> /tmp/gh-ost.log diff --git a/resources/hooks-sample/gh-ost-on-begin-postponed-hook b/resources/hooks-sample/gh-ost-on-begin-postponed-hook new file mode 100755 index 0000000..85cc7a1 --- /dev/null +++ b/resources/hooks-sample/gh-ost-on-begin-postponed-hook @@ -0,0 +1,5 @@ +#!/bin/bash + +# Sample hook file for gh-ost-on-begin-postponed + +echo "$(date) gh-ost-on-begin-postponed $GH_OST_DATABASE_NAME.$GH_OST_TABLE_NAME" >> /tmp/gh-ost.log diff --git a/resources/hooks-sample/gh-ost-on-failure-hook b/resources/hooks-sample/gh-ost-on-failure-hook new file mode 100755 index 0000000..42ccaea --- /dev/null +++ b/resources/hooks-sample/gh-ost-on-failure-hook @@ -0,0 +1,5 @@ +#!/bin/bash + +# Sample hook file for gh-ost-on-failure + +echo "$(date) gh-ost-on-failure $GH_OST_DATABASE_NAME.$GH_OST_TABLE_NAME; ghost: $GH_OST_OLD_TABLE_NAME" >> /tmp/gh-ost.log diff --git a/resources/hooks-sample/gh-ost-on-interactive-command-hook b/resources/hooks-sample/gh-ost-on-interactive-command-hook new file mode 100755 index 0000000..09b35ea --- /dev/null +++ b/resources/hooks-sample/gh-ost-on-interactive-command-hook @@ -0,0 +1,5 @@ +#!/bin/bash + +# Sample hook file for gh-ost-on-interactive-command + +echo "$(date) gh-ost-on-interactive-command $GH_OST_COMMAND" >> /tmp/gh-ost.log diff --git a/resources/hooks-sample/gh-ost-on-row-copy-complete-hook b/resources/hooks-sample/gh-ost-on-row-copy-complete-hook new file mode 100755 index 0000000..1718ee4 --- /dev/null +++ b/resources/hooks-sample/gh-ost-on-row-copy-complete-hook @@ -0,0 +1,5 @@ +#!/bin/bash + +# Sample hook file for gh-ost-on-row-copy-complete + +echo "$(date) gh-ost-on-row-copy-complete $GH_OST_DATABASE_NAME.$GH_OST_TABLE_NAME" >> /tmp/gh-ost.log diff --git a/resources/hooks-sample/gh-ost-on-rowcount-complete-hook b/resources/hooks-sample/gh-ost-on-rowcount-complete-hook new file mode 100755 index 0000000..970e127 --- /dev/null +++ b/resources/hooks-sample/gh-ost-on-rowcount-complete-hook @@ -0,0 +1,5 @@ +#!/bin/bash + +# Sample hook file for gh-ost-on-rowcount-complete + +echo "$(date) gh-ost-on-rowcount-complete $GH_OST_DATABASE_NAME.$GH_OST_TABLE_NAME" >> /tmp/gh-ost.log diff --git a/resources/hooks-sample/gh-ost-on-startup-hook b/resources/hooks-sample/gh-ost-on-startup-hook new file mode 100755 index 0000000..f439af5 --- /dev/null +++ b/resources/hooks-sample/gh-ost-on-startup-hook @@ -0,0 +1,5 @@ +#!/bin/bash + +# Sample hook file for gh-ost-on-startup + +echo "$(date) gh-ost-on-startup $GH_OST_DATABASE_NAME.$GH_OST_TABLE_NAME" >> /tmp/gh-ost.log diff --git a/resources/hooks-sample/gh-ost-on-status-hook b/resources/hooks-sample/gh-ost-on-status-hook new file mode 100755 index 0000000..50bd934 --- /dev/null +++ b/resources/hooks-sample/gh-ost-on-status-hook @@ -0,0 +1,5 @@ +#!/bin/bash + +# Sample hook file for gh-ost-on-status + +echo "$(date) gh-ost-on-status; elapsed: ${GH_OST_ELAPSED_SECONDS}; msg: ${GH_OST_STATUS}" >> /tmp/gh-ost.log diff --git a/resources/hooks-sample/gh-ost-on-stop-replication-hook b/resources/hooks-sample/gh-ost-on-stop-replication-hook new file mode 100755 index 0000000..232ad95 --- /dev/null +++ b/resources/hooks-sample/gh-ost-on-stop-replication-hook @@ -0,0 +1,5 @@ +#!/bin/bash + +# Sample hook file for gh-ost-on-stop-replication + +echo "$(date) gh-ost-on-stop-replication $GH_OST_DATABASE_NAME.$GH_OST_TABLE_NAME $GH_OST_MIGRATED_HOST" >> /tmp/gh-ost.log diff --git a/resources/hooks-sample/gh-ost-on-success-hook b/resources/hooks-sample/gh-ost-on-success-hook new file mode 100755 index 0000000..aa4aedc --- /dev/null +++ b/resources/hooks-sample/gh-ost-on-success-hook @@ -0,0 +1,5 @@ +#!/bin/bash + +# Sample hook file for gh-ost-on-success + +echo "$(date) gh-ost-on-success $GH_OST_DATABASE_NAME.$GH_OST_TABLE_NAME" >> /tmp/gh-ost.log diff --git a/resources/hooks-sample/gh-ost-on-success-hook-2 b/resources/hooks-sample/gh-ost-on-success-hook-2 new file mode 100755 index 0000000..aa4aedc --- /dev/null +++ b/resources/hooks-sample/gh-ost-on-success-hook-2 @@ -0,0 +1,5 @@ +#!/bin/bash + +# Sample hook file for gh-ost-on-success + +echo "$(date) gh-ost-on-success $GH_OST_DATABASE_NAME.$GH_OST_TABLE_NAME" >> /tmp/gh-ost.log diff --git a/resources/hooks-sample/gh-ost-on-validated-hook b/resources/hooks-sample/gh-ost-on-validated-hook new file mode 100755 index 0000000..efef4e5 --- /dev/null +++ b/resources/hooks-sample/gh-ost-on-validated-hook @@ -0,0 +1,5 @@ +#!/bin/bash + +# Sample hook file for gh-ost-on-validated + +echo "$(date) gh-ost-on-validated $GH_OST_DATABASE_NAME.$GH_OST_TABLE_NAME" >> /tmp/gh-ost.log From cbdec6bd3921ca64227f9eef16c7ce27e6d47c8b Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Thu, 25 Aug 2016 13:58:36 +0200 Subject: [PATCH 10/20] supporting --hooks-path and --hooks-hint --- go/cmd/gh-ost/main.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index 480db68..6dafa45 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -89,6 +89,9 @@ func main() { flag.StringVar(&migrationContext.ServeSocketFile, "serve-socket-file", "", "Unix socket file to serve on. Default: auto-determined and advertised upon startup") flag.Int64Var(&migrationContext.ServeTCPPort, "serve-tcp-port", 0, "TCP port to serve on. Default: disabled") + flag.StringVar(&migrationContext.HooksPath, "hooks-path", "", "directory where hook files are found (default: empty, ie. hooks disabled). Hook files found on this path, and conforming to hook naming conventions will be executed") + flag.StringVar(&migrationContext.HooksHintMessage, "hooks-hint", "", "arbitrary message to be injected to hooks via GH_OST_HOOKS_HINT, for your convenience") + maxLoad := flag.String("max-load", "", "Comma delimited status-name=threshold. e.g: 'Threads_running=100,Threads_connected=500'. When status exceeds threshold, app throttles writes") criticalLoad := flag.String("critical-load", "", "Comma delimited status-name=threshold, same format as `--max-load`. When status exceeds threshold, app panics and quits") quiet := flag.Bool("quiet", false, "quiet") From 2c9d71570f3cd9496db965ae9685d82faa6276a3 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Thu, 25 Aug 2016 14:42:49 +0200 Subject: [PATCH 11/20] logging intented hook invocation --- go/logic/hooks.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/go/logic/hooks.go b/go/logic/hooks.go index 7b831d8..ea1fa11 100644 --- a/go/logic/hooks.go +++ b/go/logic/hooks.go @@ -1,3 +1,4 @@ +/* /* Copyright 2016 GitHub Inc. See https://github.com/github/gh-ost/blob/master/LICENSE @@ -89,6 +90,7 @@ func (this *HooksExecutor) executeHooks(baseName string, extraVariables ...strin return err } for _, hook := range hooks { + log.Infof("executing %+v hook: %+v", baseName, hook) if err := this.executeHook(hook, extraVariables...); err != nil { return err } @@ -137,9 +139,11 @@ func (this *HooksExecutor) onFailure() error { } func (this *HooksExecutor) onStatus(statusMessage string, elapsedSeconds int64) error { - v0 := fmt.Sprintf("GH_OST_STATUS='%s'", statusMessage) - v1 := fmt.Sprintf("GH_OST_ELAPSED_SECONDS='%d'", elapsedSeconds) - return this.executeHooks(onStatus, v0, v1) + v := []string{ + fmt.Sprintf("GH_OST_STATUS='%s'", statusMessage), + fmt.Sprintf("GH_OST_ELAPSED_SECONDS='%d'", elapsedSeconds), + } + return this.executeHooks(onStatus, v...) } func (this *HooksExecutor) onStopReplication() error { From ebd53371d64a08388e1a663b54a7b18c65a25edb Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Thu, 25 Aug 2016 14:43:06 +0200 Subject: [PATCH 12/20] adding hooks documentation --- doc/hooks.md | 74 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 doc/hooks.md diff --git a/doc/hooks.md b/doc/hooks.md new file mode 100644 index 0000000..48d3887 --- /dev/null +++ b/doc/hooks.md @@ -0,0 +1,74 @@ +# Hooks + +`gh-ost` supports _hooks_: external processes which `gh-ost` executes at particular points of interest. + +Use cases include: + +- You wish to be notified ny mail when a migration completes/fails +- You wish to be notified when `gh-ost` postpones cut-over (at your demand), thus ready to complete (at your leisure) +- RDS users who wish to `--test-on-replica`, but who cannot have `gh-ost` issue a `STOP SLAVE`, would use a hook to command RDS to stop replication +- Send a status message to your chatops every hour +- Perform cleanup on the _ghost_ table (drop/rename/nibble) once migration completes +- etc. + +`gh-ost` defines certain points of interest (event types), and executes hooks at those points. + +Notes: + +- You may have more than one hook per event type. +- `gh-ost` will invoke relevant hooks _sequentially_ and _synchronously_ + - thus, you would generally like the hooks to execute as fast as possible, or otherwise issue tasks in the background +- A hook returning with error code will propagate the error in `gh-ost`. Thus, you are able to force `gh-ost` to fail migration on your conditions. + - Make sure to only return an error code when you do indeed wish to fail the rest of the migration + +### Creating hooks + +All hooks are expected to reside in a single directory. This directory is indicated by `--hooks-path`. When not provided, no hooks are executed. + +`gh-ost` will dynamically search for hooks in said directory. You may add and remove hooks to/from this directory as `gh-ost` makes progress (though likely you don't want to). Hook files are expected to be executable processes. + +In an effort to simplify code and to standardize usage, `gh-ost` expects hooks in explicit naming conventions. As an example, the `onStartup` hook expects processes named `gh-ost-on-startup*`. It will match and accept files named: + +- `gh-ost-on-startup` +- `gh-ost-on-startup--send-notification-mail` +- `gh-ost-on-startup12345` +- etc. + +The full list of supported hooks is best found in code: [hooks.go](https://github.com/github/gh-ost/blob/master/go/logic/hooks.go). Documentation will always be a bit behind. At this time, though, the following are recognized: + +- `gh-ost-on-startup` +- `gh-ost-on-validated` +- `gh-ost-on-rowcount-complete` +- `gh-ost-on-before-row-copy` +- `gh-ost-on-status` +- `gh-ost-on-interactive-command` +- `gh-ost-on-row-copy-complete` +- `gh-ost-on-stop-replication` +- `gh-ost-on-begin-postponed` +- `gh-ost-on-before-cut-over` +- `gh-ost-on-success` +- `gh-ost-on-failure` + +### Context + +`gh-ost` will set environment variables per hook invocation. Hooks are then able to read those variables, indicating schema name, table name, `alter` statement, migrated host name etc. Some variables are available on all hooks, and some are available on relevant hooks. + +The following variables are available on all hooks: + +- `GH_OST_DATABASE_NAME` +- `GH_OST_TABLE_NAME` +- `GH_OST_GHOST_TABLE_NAME` +- `GH_OST_OLD_TABLE_NAME` +- `GH_OST_DDL` +- `GH_OST_ELAPSED_SECONDS` +- `GH_OST_MIGRATED_HOST` +- `GH_OST_INSPECTED_HOST` +- `GH_OST_EXECUTING_HOST` +- `GH_OST_HOOKS_HINT` + +- `GH_OST_COMMAND` is only available in `gh-ost-on-interactive-command` +- `GH_OST_STATUS` and `GH_OST_ELAPSED_SECONDS` are only available in `gh-ost-on-status` + +### Examples + +See [sample hooks](https://github.com/github/gh-ost/tree/master/resources/hooks-sample), as `bash` implementation samples. From 398e1c75fff6af492a6a78109d1ce7ba35134e35 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Thu, 25 Aug 2016 14:44:15 +0200 Subject: [PATCH 13/20] doc clarification --- doc/hooks.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/hooks.md b/doc/hooks.md index 48d3887..293435e 100644 --- a/doc/hooks.md +++ b/doc/hooks.md @@ -66,6 +66,8 @@ The following variables are available on all hooks: - `GH_OST_EXECUTING_HOST` - `GH_OST_HOOKS_HINT` +The following variable are available on particular hooks: + - `GH_OST_COMMAND` is only available in `gh-ost-on-interactive-command` - `GH_OST_STATUS` and `GH_OST_ELAPSED_SECONDS` are only available in `gh-ost-on-status` From b967e6e96e5eb9cda3c13dcd30bfe1d5bb70626f Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Thu, 25 Aug 2016 14:46:43 +0200 Subject: [PATCH 14/20] doc clarification --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index deb29b9..53d017a 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,7 @@ In addition, it offers many [operational perks](doc/perks.md) that make it safer - Dynamic control: you can [interactively](doc/interactive-commands.md) reconfigure `gh-ost`, even as migration still runs. You may forcibly initiate throttling. - Auditing: you may query `gh-ost` for status. `gh-ost` listens on unix socket or TCP. - Control over cut-over phase: `gh-ost` can be instructed to postpone what is probably the most critical step: the swap of tables, until such time that you're comfortably available. No need to worry about ETA being outside office hours. +- External [hooks](doc/hooks.md) can couple `gh-ost` with your particular environment. Please refer to the [docs](doc) for more information. No, really, read the [docs](doc). From 676020000a210e977bae052394429a47aef2542b Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Thu, 25 Aug 2016 15:11:58 +0200 Subject: [PATCH 15/20] typo --- doc/hooks.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/hooks.md b/doc/hooks.md index 293435e..169dfb2 100644 --- a/doc/hooks.md +++ b/doc/hooks.md @@ -4,7 +4,7 @@ Use cases include: -- You wish to be notified ny mail when a migration completes/fails +- You wish to be notified by mail when a migration completes/fails - You wish to be notified when `gh-ost` postpones cut-over (at your demand), thus ready to complete (at your leisure) - RDS users who wish to `--test-on-replica`, but who cannot have `gh-ost` issue a `STOP SLAVE`, would use a hook to command RDS to stop replication - Send a status message to your chatops every hour From ef386add5cd3171bb508b5ddc6003fedf4675b7f Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Thu, 25 Aug 2016 16:08:49 +0200 Subject: [PATCH 16/20] hook output and err are written to gh-ost's stderr --- go/logic/hooks.go | 8 ++++---- resources/hooks-sample/gh-ost-on-success-hook-2 | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/go/logic/hooks.go b/go/logic/hooks.go index ea1fa11..8906f62 100644 --- a/go/logic/hooks.go +++ b/go/logic/hooks.go @@ -65,14 +65,14 @@ func (this *HooksExecutor) applyEnvironmentVairables(extraVariables ...string) [ } // executeHook executes a command, and sets relevant environment variables +// combined output & error are printed to gh-ost's standard error. func (this *HooksExecutor) executeHook(hook string, extraVariables ...string) error { cmd := exec.Command(hook) cmd.Env = this.applyEnvironmentVairables(extraVariables...) - if err := cmd.Run(); err != nil { - return log.Errore(err) - } - return nil + combinedOutput, err := cmd.CombinedOutput() + fmt.Fprintln(os.Stderr, string(combinedOutput)) + return log.Errore(err) } func (this *HooksExecutor) detectHooks(baseName string) (hooks []string, err error) { diff --git a/resources/hooks-sample/gh-ost-on-success-hook-2 b/resources/hooks-sample/gh-ost-on-success-hook-2 index aa4aedc..29cba49 100755 --- a/resources/hooks-sample/gh-ost-on-success-hook-2 +++ b/resources/hooks-sample/gh-ost-on-success-hook-2 @@ -2,4 +2,4 @@ # Sample hook file for gh-ost-on-success -echo "$(date) gh-ost-on-success $GH_OST_DATABASE_NAME.$GH_OST_TABLE_NAME" >> /tmp/gh-ost.log +echo "$(date) gh-ost-on-success $GH_OST_DATABASE_NAME.$GH_OST_TABLE_NAME -- this message should show on the gh-ost log" From 89a3a261a85f04ca63dd8227255e0d90b700a506 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Fri, 26 Aug 2016 08:39:32 +0200 Subject: [PATCH 17/20] updated version --- build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sh b/build.sh index 20f127a..e00cb54 100755 --- a/build.sh +++ b/build.sh @@ -2,7 +2,7 @@ # # -RELEASE_VERSION="1.0.14" +RELEASE_VERSION="1.0.15" function build { osname=$1 From 6e5db089c8740fec0be8895c4b85a16b34883992 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Mon, 29 Aug 2016 09:58:31 +0200 Subject: [PATCH 18/20] supporting onRowCountComplete hook --- go/logic/migrator.go | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/go/logic/migrator.go b/go/logic/migrator.go index e767455..c44d196 100644 --- a/go/logic/migrator.go +++ b/go/logic/migrator.go @@ -379,13 +379,24 @@ func (this *Migrator) countTableRows() (err error) { log.Debugf("Noop operation; not really counting table rows") return nil } + + countRowsFunc := func() error { + if err := this.inspector.CountTableRows(); err != nil { + return err + } + if err := this.hooksExecutor.onRowCountComplete(); err != nil { + return err + } + return nil + } + if this.migrationContext.ConcurrentCountTableRows { - go this.inspector.CountTableRows() log.Infof("As instructed, counting rows in the background; meanwhile I will use an estimated count, and will update it later on") + go countRowsFunc() // and we ignore errors, because this turns to be a background job return nil } - return this.inspector.CountTableRows() + return countRowsFunc() } // Migrate executes the complete migration logic. This is *the* major gh-ost function. From 6dfa4873c296a7b0472e5b64aafd7902e74a973f Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Mon, 29 Aug 2016 10:44:43 +0200 Subject: [PATCH 19/20] removed excessive argument --- go/logic/hooks.go | 9 +++------ go/logic/migrator.go | 2 +- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/go/logic/hooks.go b/go/logic/hooks.go index 8906f62..ad55b99 100644 --- a/go/logic/hooks.go +++ b/go/logic/hooks.go @@ -138,12 +138,9 @@ func (this *HooksExecutor) onFailure() error { return this.executeHooks(onFailure) } -func (this *HooksExecutor) onStatus(statusMessage string, elapsedSeconds int64) error { - v := []string{ - fmt.Sprintf("GH_OST_STATUS='%s'", statusMessage), - fmt.Sprintf("GH_OST_ELAPSED_SECONDS='%d'", elapsedSeconds), - } - return this.executeHooks(onStatus, v...) +func (this *HooksExecutor) onStatus(statusMessage string) error { + v := fmt.Sprintf("GH_OST_STATUS='%s'", statusMessage) + return this.executeHooks(onStatus, v) } func (this *HooksExecutor) onStopReplication() error { diff --git a/go/logic/migrator.go b/go/logic/migrator.go index c44d196..799e70b 100644 --- a/go/logic/migrator.go +++ b/go/logic/migrator.go @@ -1102,7 +1102,7 @@ func (this *Migrator) printStatus(rule PrintStatusRule, writers ...io.Writer) { fmt.Fprintln(w, status) if elapsedSeconds%60 == 0 { - this.hooksExecutor.onStatus(status, elapsedSeconds) + this.hooksExecutor.onStatus(status) } } From 7450910e707c4aaffa2397ae464d190ddfdeb965 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Mon, 29 Aug 2016 12:47:15 +0200 Subject: [PATCH 20/20] fixed docs --- doc/hooks.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/hooks.md b/doc/hooks.md index 169dfb2..44e59a7 100644 --- a/doc/hooks.md +++ b/doc/hooks.md @@ -69,7 +69,7 @@ The following variables are available on all hooks: The following variable are available on particular hooks: - `GH_OST_COMMAND` is only available in `gh-ost-on-interactive-command` -- `GH_OST_STATUS` and `GH_OST_ELAPSED_SECONDS` are only available in `gh-ost-on-status` +- `GH_OST_STATUS` is only available in `gh-ost-on-status` ### Examples