Merge branch 'master' into tar-xz-dbdeployer-upgrade

.github/workflows/ci.yml

@@ -8,13 +8,18 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@master
+    - uses: actions/checkout@v2
 
-    - name: Set up Go 1.12
+    - name: Set up Go 1.15
       uses: actions/setup-go@v1
       with:
-        version: 1.12
+        go-version: 1.15
-      id: go
 
     - name: Build
       run: script/cibuild
+
+    - name: Upload gh-ost binary artifact
+      uses: actions/upload-artifact@v1
+      with:
+        name: gh-ost
+        path: bin/gh-ost

.github/workflows/replica-tests.yml

@@ -8,13 +8,12 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@master
+    - uses: actions/checkout@v2
 
-    - name: Set up Go 1.12
+    - name: Set up Go 1.14
       uses: actions/setup-go@v1
       with:
-        version: 1.12
+        go-version: 1.14
-      id: go
 
     - name: migration tests
       run:  script/cibuild-gh-ost-replica-tests

.gitignore

@@ -2,3 +2,4 @@
 /bin/
 /libexec/
 /.vendor/
+.idea/

Dockerfile.packaging

@@ -0,0 +1,22 @@
+#
+
+FROM golang:1.15.6
+
+RUN apt-get update
+RUN apt-get install -y ruby ruby-dev rubygems build-essential
+RUN gem install --no-ri --no-rdoc fpm
+ENV GOPATH=/tmp/go
+
+RUN apt-get install -y curl
+RUN apt-get install -y rsync
+RUN apt-get install -y gcc
+RUN apt-get install -y g++
+RUN apt-get install -y bash
+RUN apt-get install -y git
+RUN apt-get install -y tar
+RUN apt-get install -y rpm
+
+RUN mkdir -p $GOPATH/src/github.com/github/gh-ost
+WORKDIR $GOPATH/src/github.com/github/gh-ost
+COPY . .
+RUN bash build.sh

Dockerfile.test

@@ -1,4 +1,4 @@
-FROM golang:1.12.1
+FROM golang:1.15.6
 LABEL maintainer="github@github.com"
 
 RUN apt-get update

README.md

@@ -65,6 +65,7 @@ Also see:
 - [the fine print](doc/the-fine-print.md)
 - [Community questions](https://github.com/github/gh-ost/issues?q=label%3Aquestion)
 - [Using `gh-ost` on AWS RDS](doc/rds.md)
+- [Using `gh-ost` on Azure Database for MySQL](doc/azure.md)
 
 ## What's in a name?
 
@@ -94,7 +95,7 @@ Please see [Coding gh-ost](doc/coding-ghost.md) for a guide to getting started d
 
 [Download latest release here](https://github.com/github/gh-ost/releases/latest)
 
-`gh-ost` is a Go project; it is built with Go `1.12` and above. To build on your own, use either:
+`gh-ost` is a Go project; it is built with Go `1.14` and above. To build on your own, use either:
 - [script/build](https://github.com/github/gh-ost/blob/master/script/build) - this is the same build script used by CI hence the authoritative; artifact is `./bin/gh-ost` binary.
 - [build.sh](https://github.com/github/gh-ost/blob/master/build.sh) for building `tar.gz` artifacts in `/tmp/gh-ost`
 
@@ -109,3 +110,4 @@ Generally speaking, `master` branch is stable, but only [releases](https://githu
 - [@shlomi-noach](https://github.com/shlomi-noach)
 - [@jessbreckenridge](https://github.com/jessbreckenridge)
 - [@gtowey](https://github.com/gtowey)
+- [@timvaillancourt](https://github.com/timvaillancourt)

RELEASE_VERSION

@@ -1 +1 @@
-1.0.48
+1.1.1

build.sh

@@ -18,8 +18,8 @@ function build {
   GOOS=$3
   GOARCH=$4
 
-  if ! go version | egrep -q 'go(1\.1[234])' ; then
+  if ! go version | egrep -q 'go(1\.1[456])' ; then
-    echo "go version must be 1.12 or above"
+    echo "go version must be 1.14 or above"
     exit 1
   fi
 
@@ -40,7 +40,7 @@ function build {
     builddir=$(setuptree)
     cp $buildpath/$target $builddir/gh-ost/usr/bin
     cd $buildpath
-    fpm -v "${RELEASE_VERSION}" --epoch 1 -f -s dir -n gh-ost -m 'shlomi-noach <shlomi-noach+gh-ost-deb@github.com>' --description "GitHub's Online Schema Migrations for MySQL " --url "https://github.com/github/gh-ost" --vendor "GitHub" --license "Apache 2.0" -C $builddir/gh-ost --prefix=/ -t rpm .
+    fpm -v "${RELEASE_VERSION}" --epoch 1 -f -s dir -n gh-ost -m 'shlomi-noach <shlomi-noach+gh-ost-deb@github.com>' --description "GitHub's Online Schema Migrations for MySQL " --url "https://github.com/github/gh-ost" --vendor "GitHub" --license "Apache 2.0" -C $builddir/gh-ost --prefix=/ -t rpm --rpm-rpmbuild-define "_build_id_links none" .
     fpm -v "${RELEASE_VERSION}" --epoch 1 -f -s dir -n gh-ost -m 'shlomi-noach <shlomi-noach+gh-ost-deb@github.com>' --description "GitHub's Online Schema Migrations for MySQL " --url "https://github.com/github/gh-ost" --vendor "GitHub" --license "Apache 2.0" -C $builddir/gh-ost --prefix=/ -t deb --deb-no-default-config-files .
   fi
 }
@@ -61,11 +61,11 @@ main() {
 
   mkdir -p ${buildpath}
   rm -rf ${buildpath:?}/*
-  build macOS osx darwin amd64
   build GNU/Linux linux linux amd64
+  # build macOS osx darwin amd64
 
   echo "Binaries found in:"
-  ls -1 $buildpath/gh-ost-binary*${timestamp}.tar.gz
+  find $buildpath/gh-ost* -type f -maxdepth 1
 }
 
 main "$@"

doc/azure.md

@@ -0,0 +1,26 @@
+`gh-ost` has been updated to work with Azure Database for MySQL however due to GitHub does not use it, this documentation is community driven so if you find a bug please [open an issue][new_issue]!
+
+# Azure Database for MySQL
+
+## Limitations
+
+- `gh-ost` runs should be setup use [`--assume-rbr`][assume_rbr_docs] and use `binlog_row_image=FULL`.
+- Azure Database for MySQL does not use same user name suffix for master and replica, so master host, user and password need to be pointed out. 
+
+## Step
+1. Change the replica server's `binlog_row_image` from `MINIMAL` to `FULL`. See [guide](https://docs.microsoft.com/en-us/azure/mysql/howto-server-parameters) on Azure document.
+2. Use your `gh-ost` always with additional 5 parameter
+```{bash}
+gh-ost \
+--azure \
+--assume-master-host=master-server-dns-name \
+--master-user="master-user-name" \
+--master-password="master-password" \
+--assume-rbr \
+[-- other paramters you need]
+```
+
+
+[new_issue]: https://github.com/github/gh-ost/issues/new
+[assume_rbr_docs]: https://github.com/github/gh-ost/blob/master/doc/command-line-flags.md#assume-rbr
+[migrate_test_on_replica_docs]: https://github.com/github/gh-ost/blob/master/doc/cheatsheet.md#c-migratetest-on-replica

doc/command-line-flags.md

@@ -6,6 +6,10 @@ A more in-depth discussion of various `gh-ost` command line flags: implementatio
 
 Add this flag when executing on Aliyun RDS.
 
+###  azure
+
+Add this flag when executing on Azure Database for MySQL.
+
 ### allow-master-master
 
 See [`--assume-master-host`](#assume-master-host).
@@ -177,6 +181,9 @@ Optionally involve the process ID, for example: `--replica-server-id=$((10000000
 It's on you to choose a number that does not collide with another `gh-ost` or another running replica.
 See also: [`concurrent-migrations`](cheatsheet.md#concurrent-migrations) on the cheatsheet.
 
+### serve-socket-file
+
+Defaults to an auto-determined and advertised upon startup file. Defines Unix socket file to serve on.
 ### skip-foreign-key-checks
 
 By default `gh-ost` verifies no foreign keys exist on the migrated table. On servers with large number of tables this check can take a long time. If you're absolutely certain no foreign keys exist (table does not reference other table nor is referenced by other tables) and wish to save the check time, provide with `--skip-foreign-key-checks`.

doc/hooks.md

@@ -66,6 +66,7 @@ The following variables are available on all hooks:
 - `GH_OST_ESTIMATED_ROWS` - estimated total rows in table
 - `GH_OST_COPIED_ROWS` - number of rows copied by `gh-ost`
 - `GH_OST_INSPECTED_LAG` - lag in seconds (floating point) of inspected server
+- `GH_OST_HEARTBEAT_LAG` - lag in seconds (floating point) of heartbeat
 - `GH_OST_PROGRESS` - progress pct ([0..100], floating point) of migration
 - `GH_OST_MIGRATED_HOST`
 - `GH_OST_INSPECTED_HOST`

doc/interactive-commands.md

@@ -18,6 +18,8 @@ Both interfaces may serve at the same time. Both respond to simple text command,
 - `status`: returns a detailed status summary of migration progress and configuration
 - `sup`: returns a brief status summary of migration progress
 - `coordinates`: returns recent (though not exactly up to date) binary log coordinates of the inspected server
+- `applier`: returns the hostname of the applier
+- `inspector`: returns the hostname of the inspector
 - `chunk-size=<newsize>`: modify the `chunk-size`; applies on next running copy-iteration
 - `dml-batch-size=<newsize>`: modify the `dml-batch-size`; applies on next applying of binary log events
 - `max-lag-millis=<max-lag>`: modify the maximum replication lag threshold (milliseconds, minimum value is `100`, i.e. `0.1` second)

doc/requirements-and-limitations.md

@@ -41,6 +41,7 @@ The `SUPER` privilege is required for `STOP SLAVE`, `START SLAVE` operations. Th
 - Amazon RDS works, but has its own [limitations](rds.md).
 - Google Cloud SQL works, `--gcp` flag required.
 - Aliyun RDS works, `--aliyun-rds` flag required.
+- Azure Database for MySQL works, `--azure` flag required, and have detailed document about it. (azure.md)
 
 - Multisource is not supported when migrating via replica. It _should_ work (but never tested) when connecting directly to master (`--allow-on-master`)
 
@@ -50,4 +51,5 @@ The `SUPER` privilege is required for `STOP SLAVE`, `START SLAVE` operations. Th
 
 - Migrating a `FEDERATED` table is unsupported and is irrelevant to the problem `gh-ost` tackles.
 
+- [Encrypted binary logs](https://www.percona.com/blog/2018/03/08/binlog-encryption-percona-server-mysql/) are not supported.
 - `ALTER TABLE ... RENAME TO some_other_name` is not supported (and you shouldn't use `gh-ost` for such a trivial operation).

go/base/context.go

@@ -19,6 +19,7 @@ import (
 
 	"github.com/github/gh-ost/go/mysql"
 	"github.com/github/gh-ost/go/sql"
+	"github.com/outbrain/golib/log"
 
 	"gopkg.in/gcfg.v1"
 	gcfgscanner "gopkg.in/gcfg.v1/scanner"
@@ -76,9 +77,10 @@ func NewThrottleCheckResult(throttle bool, reason string, reasonHint ThrottleRea
 type MigrationContext struct {
 	Uuid string
 
-	DatabaseName      string
+	DatabaseName          string
-	OriginalTableName string
+	OriginalTableName     string
-	AlterStatement    string
+	AlterStatement        string
+	AlterStatementOptions string // anything following the 'ALTER TABLE [schema.]table' from AlterStatement
 
 	CountTableRows           bool
 	ConcurrentCountTableRows bool
@@ -95,6 +97,7 @@ type MigrationContext struct {
 	DiscardForeignKeys       bool
 	AliyunRDS                bool
 	GoogleCloudPlatform      bool
+	AzureMySQL               bool
 
 	config            ContextConfig
 	configMutex       *sync.Mutex
@@ -119,6 +122,7 @@ type MigrationContext struct {
 	ThrottleAdditionalFlagFile          string
 	throttleQuery                       string
 	throttleHTTP                        string
+	IgnoreHTTPErrors                    bool
 	ThrottleCommandedByUser             int64
 	HibernateUntil                      int64
 	maxLoad                             LoadMap
@@ -174,6 +178,8 @@ type MigrationContext struct {
 	RenameTablesEndTime                    time.Time
 	pointOfInterestTime                    time.Time
 	pointOfInterestTimeMutex               *sync.Mutex
+	lastHeartbeatOnChangelogTime           time.Time
+	lastHeartbeatOnChangelogMutex          *sync.Mutex
 	CurrentLag                             int64
 	currentProgress                        uint64
 	ThrottleHTTPStatusCode                 int64
@@ -216,6 +222,25 @@ type MigrationContext struct {
 	ForceTmpTableName                string
 
 	recentBinlogCoordinates mysql.BinlogCoordinates
+
+	Log Logger
+}
+
+type Logger interface {
+	Debug(args ...interface{})
+	Debugf(format string, args ...interface{})
+	Info(args ...interface{})
+	Infof(format string, args ...interface{})
+	Warning(args ...interface{}) error
+	Warningf(format string, args ...interface{}) error
+	Error(args ...interface{}) error
+	Errorf(format string, args ...interface{}) error
+	Errore(err error) error
+	Fatal(args ...interface{}) error
+	Fatalf(format string, args ...interface{}) error
+	Fatale(err error) error
+	SetLevel(level log.LogLevel)
+	SetPrintStackTrace(printStackTraceFlag bool)
 }
 
 type ContextConfig struct {
@@ -248,8 +273,10 @@ func NewMigrationContext() *MigrationContext {
 		throttleControlReplicaKeys:          mysql.NewInstanceKeyMap(),
 		configMutex:                         &sync.Mutex{},
 		pointOfInterestTimeMutex:            &sync.Mutex{},
+		lastHeartbeatOnChangelogMutex:       &sync.Mutex{},
 		ColumnRenameMap:                     make(map[string]string),
 		PanicAbort:                          make(chan error),
+		Log:                                 NewDefaultLogger(),
 	}
 }
 
@@ -430,6 +457,10 @@ func (this *MigrationContext) MarkRowCopyEndTime() {
 	this.RowCopyEndTime = time.Now()
 }
 
+func (this *MigrationContext) TimeSinceLastHeartbeatOnChangelog() time.Duration {
+	return time.Since(this.GetLastHeartbeatOnChangelogTime())
+}
+
 func (this *MigrationContext) GetCurrentLagDuration() time.Duration {
 	return time.Duration(atomic.LoadInt64(&this.CurrentLag))
 }
@@ -469,6 +500,20 @@ func (this *MigrationContext) TimeSincePointOfInterest() time.Duration {
 	return time.Since(this.pointOfInterestTime)
 }
 
+func (this *MigrationContext) SetLastHeartbeatOnChangelogTime(t time.Time) {
+	this.lastHeartbeatOnChangelogMutex.Lock()
+	defer this.lastHeartbeatOnChangelogMutex.Unlock()
+
+	this.lastHeartbeatOnChangelogTime = t
+}
+
+func (this *MigrationContext) GetLastHeartbeatOnChangelogTime() time.Time {
+	this.lastHeartbeatOnChangelogMutex.Lock()
+	defer this.lastHeartbeatOnChangelogMutex.Unlock()
+
+	return this.lastHeartbeatOnChangelogTime
+}
+
 func (this *MigrationContext) SetHeartbeatIntervalMilliseconds(heartbeatIntervalMilliseconds int64) {
 	if heartbeatIntervalMilliseconds < 100 {
 		heartbeatIntervalMilliseconds = 100
@@ -574,6 +619,13 @@ func (this *MigrationContext) SetThrottleHTTP(throttleHTTP string) {
 	this.throttleHTTP = throttleHTTP
 }
 
+func (this *MigrationContext) SetIgnoreHTTPErrors(ignoreHTTPErrors bool) {
+	this.throttleHTTPMutex.Lock()
+	defer this.throttleHTTPMutex.Unlock()
+
+	this.IgnoreHTTPErrors = ignoreHTTPErrors
+}
+
 func (this *MigrationContext) GetMaxLoad() LoadMap {
 	this.throttleMutex.Lock()
 	defer this.throttleMutex.Unlock()

go/base/default_logger.go

@@ -0,0 +1,73 @@
+package base
+
+import (
+	"github.com/outbrain/golib/log"
+)
+
+type simpleLogger struct{}
+
+func NewDefaultLogger() *simpleLogger {
+	return &simpleLogger{}
+}
+
+func (*simpleLogger) Debug(args ...interface{}) {
+	log.Debug(args[0].(string), args[1:])
+	return
+}
+
+func (*simpleLogger) Debugf(format string, args ...interface{}) {
+	log.Debugf(format, args...)
+	return
+}
+
+func (*simpleLogger) Info(args ...interface{}) {
+	log.Info(args[0].(string), args[1:])
+	return
+}
+
+func (*simpleLogger) Infof(format string, args ...interface{}) {
+	log.Infof(format, args...)
+	return
+}
+
+func (*simpleLogger) Warning(args ...interface{}) error {
+	return log.Warning(args[0].(string), args[1:])
+}
+
+func (*simpleLogger) Warningf(format string, args ...interface{}) error {
+	return log.Warningf(format, args...)
+}
+
+func (*simpleLogger) Error(args ...interface{}) error {
+	return log.Error(args[0].(string), args[1:])
+}
+
+func (*simpleLogger) Errorf(format string, args ...interface{}) error {
+	return log.Errorf(format, args...)
+}
+
+func (*simpleLogger) Errore(err error) error {
+	return log.Errore(err)
+}
+
+func (*simpleLogger) Fatal(args ...interface{}) error {
+	return log.Fatal(args[0].(string), args[1:])
+}
+
+func (*simpleLogger) Fatalf(format string, args ...interface{}) error {
+	return log.Fatalf(format, args...)
+}
+
+func (*simpleLogger) Fatale(err error) error {
+	return log.Fatale(err)
+}
+
+func (*simpleLogger) SetLevel(level log.LogLevel) {
+	log.SetLevel(level)
+	return
+}
+
+func (*simpleLogger) SetPrintStackTrace(printStackTraceFlag bool) {
+	log.SetPrintStackTrace(printStackTraceFlag)
+	return
+}

go/base/utils.go

@@ -13,8 +13,8 @@ import (
 	"time"
 
 	gosql "database/sql"
+
 	"github.com/github/gh-ost/go/mysql"
-	"github.com/outbrain/golib/log"
 )
 
 var (
@@ -63,7 +63,7 @@ func StringContainsAll(s string, substrings ...string) bool {
 	return nonEmptyStringsFound
 }
 
-func ValidateConnection(db *gosql.DB, connectionConfig *mysql.ConnectionConfig, migrationContext *MigrationContext) (string, error) {
+func ValidateConnection(db *gosql.DB, connectionConfig *mysql.ConnectionConfig, migrationContext *MigrationContext, name string) (string, error) {
 	versionQuery := `select @@global.version`
 	var port, extraPort int
 	var version string
@@ -76,7 +76,8 @@ func ValidateConnection(db *gosql.DB, connectionConfig *mysql.ConnectionConfig,
 	}
 	// AliyunRDS set users port to "NULL", replace it by gh-ost param
 	// GCP set users port to "NULL", replace it by gh-ost param
-	if migrationContext.AliyunRDS || migrationContext.GoogleCloudPlatform {
+	// Azure MySQL set users port to a different value by design, replace it by gh-ost para
+	if migrationContext.AliyunRDS || migrationContext.GoogleCloudPlatform || migrationContext.AzureMySQL {
 		port = connectionConfig.Key.Port
 	} else {
 		portQuery := `select @@global.port`
@@ -86,7 +87,7 @@ func ValidateConnection(db *gosql.DB, connectionConfig *mysql.ConnectionConfig,
 	}
 
 	if connectionConfig.Key.Port == port || (extraPort > 0 && connectionConfig.Key.Port == extraPort) {
-		log.Infof("connection validated on %+v", connectionConfig.Key)
+		migrationContext.Log.Infof("%s connection validated on %+v", name, connectionConfig.Key)
 		return version, nil
 	} else if extraPort == 0 {
 		return "", fmt.Errorf("Unexpected database port reported: %+v", port)

go/binlog/gomysql_reader.go

@@ -13,13 +13,13 @@ import (
 	"github.com/github/gh-ost/go/mysql"
 	"github.com/github/gh-ost/go/sql"
 
-	"github.com/outbrain/golib/log"
 	gomysql "github.com/siddontang/go-mysql/mysql"
 	"github.com/siddontang/go-mysql/replication"
 	"golang.org/x/net/context"
 )
 
 type GoMySQLReader struct {
+	migrationContext         *base.MigrationContext
 	connectionConfig         *mysql.ConnectionConfig
 	binlogSyncer             *replication.BinlogSyncer
 	binlogStreamer           *replication.BinlogStreamer
@@ -30,6 +30,7 @@ type GoMySQLReader struct {
 
 func NewGoMySQLReader(migrationContext *base.MigrationContext) (binlogReader *GoMySQLReader, err error) {
 	binlogReader = &GoMySQLReader{
+		migrationContext:        migrationContext,
 		connectionConfig:        migrationContext.InspectorConnectionConfig,
 		currentCoordinates:      mysql.BinlogCoordinates{},
 		currentCoordinatesMutex: &sync.Mutex{},
@@ -57,11 +58,11 @@ func NewGoMySQLReader(migrationContext *base.MigrationContext) (binlogReader *Go
 // ConnectBinlogStreamer
 func (this *GoMySQLReader) ConnectBinlogStreamer(coordinates mysql.BinlogCoordinates) (err error) {
 	if coordinates.IsEmpty() {
-		return log.Errorf("Empty coordinates at ConnectBinlogStreamer()")
+		return this.migrationContext.Log.Errorf("Empty coordinates at ConnectBinlogStreamer()")
 	}
 
 	this.currentCoordinates = coordinates
-	log.Infof("Connecting binlog streamer at %+v", this.currentCoordinates)
+	this.migrationContext.Log.Infof("Connecting binlog streamer at %+v", this.currentCoordinates)
 	// Start sync with specified binlog file and position
 	this.binlogStreamer, err = this.binlogSyncer.StartSync(gomysql.Position{this.currentCoordinates.LogFile, uint32(this.currentCoordinates.LogPos)})
 
@@ -78,7 +79,7 @@ func (this *GoMySQLReader) GetCurrentBinlogCoordinates() *mysql.BinlogCoordinate
 // StreamEvents
 func (this *GoMySQLReader) handleRowsEvent(ev *replication.BinlogEvent, rowsEvent *replication.RowsEvent, entriesChannel chan<- *BinlogEntry) error {
 	if this.currentCoordinates.SmallerThanOrEquals(&this.LastAppliedRowsEventHint) {
-		log.Debugf("Skipping handled query at %+v", this.currentCoordinates)
+		this.migrationContext.Log.Debugf("Skipping handled query at %+v", this.currentCoordinates)
 		return nil
 	}
 
@@ -147,14 +148,14 @@ func (this *GoMySQLReader) StreamEvents(canStopStreaming func() bool, entriesCha
 				defer this.currentCoordinatesMutex.Unlock()
 				this.currentCoordinates.LogFile = string(rotateEvent.NextLogName)
 			}()
-			log.Infof("rotate to next log from %s:%d to %s", this.currentCoordinates.LogFile, int64(ev.Header.LogPos), rotateEvent.NextLogName)
+			this.migrationContext.Log.Infof("rotate to next log from %s:%d to %s", this.currentCoordinates.LogFile, int64(ev.Header.LogPos), rotateEvent.NextLogName)
 		} else if rowsEvent, ok := ev.Event.(*replication.RowsEvent); ok {
 			if err := this.handleRowsEvent(ev, rowsEvent, entriesChannel); err != nil {
 				return err
 			}
 		}
 	}
-	log.Debugf("done streaming events")
+	this.migrationContext.Log.Debugf("done streaming events")
 
 	return nil
 }

go/cmd/gh-ost/main.go

@@ -14,6 +14,7 @@ import (
 
 	"github.com/github/gh-ost/go/base"
 	"github.com/github/gh-ost/go/logic"
+	"github.com/github/gh-ost/go/sql"
 	_ "github.com/go-sql-driver/mysql"
 	"github.com/outbrain/golib/log"
 
@@ -31,7 +32,7 @@ func acceptSignals(migrationContext *base.MigrationContext) {
 		for sig := range c {
 			switch sig {
 			case syscall.SIGHUP:
-				log.Infof("Received SIGHUP. Reloading configuration")
+				migrationContext.Log.Infof("Received SIGHUP. Reloading configuration")
 				if err := migrationContext.ReadConfigFile(); err != nil {
 					log.Errore(err)
 				} else {
@@ -48,6 +49,7 @@ func main() {
 	flag.StringVar(&migrationContext.InspectorConnectionConfig.Key.Hostname, "host", "127.0.0.1", "MySQL hostname (preferably a replica, not the master)")
 	flag.StringVar(&migrationContext.AssumeMasterHostname, "assume-master-host", "", "(optional) explicitly tell gh-ost the identity of the master. Format: some.host.com[:port] This is useful in master-master setups where you wish to pick an explicit master, or in a tungsten-replicator where gh-ost is unable to determine the master")
 	flag.IntVar(&migrationContext.InspectorConnectionConfig.Key.Port, "port", 3306, "MySQL port (preferably a replica, not the master)")
+	flag.Float64Var(&migrationContext.InspectorConnectionConfig.Timeout, "mysql-timeout", 0.0, "Connect, read and write timeout for MySQL")
 	flag.StringVar(&migrationContext.CliUser, "user", "", "MySQL user")
 	flag.StringVar(&migrationContext.CliPassword, "password", "", "MySQL password")
 	flag.StringVar(&migrationContext.CliMasterUser, "master-user", "", "MySQL user on master, if different from that on replica. Requires --assume-master-host")
@@ -77,6 +79,7 @@ func main() {
 	flag.BoolVar(&migrationContext.SkipStrictMode, "skip-strict-mode", false, "explicitly tell gh-ost binlog applier not to enforce strict sql mode")
 	flag.BoolVar(&migrationContext.AliyunRDS, "aliyun-rds", false, "set to 'true' when you execute on Aliyun RDS.")
 	flag.BoolVar(&migrationContext.GoogleCloudPlatform, "gcp", false, "set to 'true' when you execute on a 1st generation Google Cloud Platform (GCP).")
+	flag.BoolVar(&migrationContext.AzureMySQL, "azure", false, "set to 'true' when you execute on Azure Database on MySQL.")
 
 	executeFlag := flag.Bool("execute", false, "actually execute the alter & migrate the table. Default is noop: do some tests and exit")
 	flag.BoolVar(&migrationContext.TestOnReplica, "test-on-replica", false, "Have the migration run on a replica, not on the master. At the end of migration replication is stopped, and tables are swapped and immediately swap-revert. Replication remains stopped and you can compare the two tables for building trust")
@@ -106,6 +109,7 @@ func main() {
 	throttleControlReplicas := flag.String("throttle-control-replicas", "", "List of replicas on which to check for lag; comma delimited. Example: myhost1.com:3306,myhost2.com,myhost3.com:3307")
 	throttleQuery := flag.String("throttle-query", "", "when given, issued (every second) to check if operation should throttle. Expecting to return zero for no-throttle, >0 for throttle. Query is issued on the migrated server. Make sure this query is lightweight")
 	throttleHTTP := flag.String("throttle-http", "", "when given, gh-ost checks given URL via HEAD request; any response code other than 200 (OK) causes throttling; make sure it has low latency response")
+	ignoreHTTPErrors := flag.Bool("ignore-http-errors", false, "ignore HTTP connection errors during throttle check")
 	heartbeatIntervalMillis := flag.Int64("heartbeat-interval-millis", 100, "how frequently would gh-ost inject a heartbeat value")
 	flag.StringVar(&migrationContext.ThrottleFlagFile, "throttle-flag-file", "", "operation pauses when this file exists; hint: use a file that is specific to the table being altered")
 	flag.StringVar(&migrationContext.ThrottleAdditionalFlagFile, "throttle-additional-flag-file", "/tmp/gh-ost.throttle", "operation pauses when this file exists; hint: keep default, use for throttling multiple gh-ost operations")
@@ -156,69 +160,80 @@ func main() {
 		return
 	}
 
-	log.SetLevel(log.ERROR)
+	migrationContext.Log.SetLevel(log.ERROR)
 	if *verbose {
-		log.SetLevel(log.INFO)
+		migrationContext.Log.SetLevel(log.INFO)
 	}
 	if *debug {
-		log.SetLevel(log.DEBUG)
+		migrationContext.Log.SetLevel(log.DEBUG)
 	}
 	if *stack {
-		log.SetPrintStackTrace(*stack)
+		migrationContext.Log.SetPrintStackTrace(*stack)
 	}
 	if *quiet {
 		// Override!!
-		log.SetLevel(log.ERROR)
+		migrationContext.Log.SetLevel(log.ERROR)
 	}
 
-	if migrationContext.DatabaseName == "" {
-		log.Fatalf("--database must be provided and database name must not be empty")
-	}
-	if migrationContext.OriginalTableName == "" {
-		log.Fatalf("--table must be provided and table name must not be empty")
-	}
 	if migrationContext.AlterStatement == "" {
 		log.Fatalf("--alter must be provided and statement must not be empty")
 	}
+	parser := sql.NewParserFromAlterStatement(migrationContext.AlterStatement)
+	migrationContext.AlterStatementOptions = parser.GetAlterStatementOptions()
+
+	if migrationContext.DatabaseName == "" {
+		if parser.HasExplicitSchema() {
+			migrationContext.DatabaseName = parser.GetExplicitSchema()
+		} else {
+			log.Fatalf("--database must be provided and database name must not be empty, or --alter must specify database name")
+		}
+	}
+	if migrationContext.OriginalTableName == "" {
+		if parser.HasExplicitTable() {
+			migrationContext.OriginalTableName = parser.GetExplicitTable()
+		} else {
+			log.Fatalf("--table must be provided and table name must not be empty, or --alter must specify table name")
+		}
+	}
 	migrationContext.Noop = !(*executeFlag)
 	if migrationContext.AllowedRunningOnMaster && migrationContext.TestOnReplica {
-		log.Fatalf("--allow-on-master and --test-on-replica are mutually exclusive")
+		migrationContext.Log.Fatalf("--allow-on-master and --test-on-replica are mutually exclusive")
 	}
 	if migrationContext.AllowedRunningOnMaster && migrationContext.MigrateOnReplica {
-		log.Fatalf("--allow-on-master and --migrate-on-replica are mutually exclusive")
+		migrationContext.Log.Fatalf("--allow-on-master and --migrate-on-replica are mutually exclusive")
 	}
 	if migrationContext.MigrateOnReplica && migrationContext.TestOnReplica {
-		log.Fatalf("--migrate-on-replica and --test-on-replica are mutually exclusive")
+		migrationContext.Log.Fatalf("--migrate-on-replica and --test-on-replica are mutually exclusive")
 	}
 	if migrationContext.SwitchToRowBinlogFormat && migrationContext.AssumeRBR {
-		log.Fatalf("--switch-to-rbr and --assume-rbr are mutually exclusive")
+		migrationContext.Log.Fatalf("--switch-to-rbr and --assume-rbr are mutually exclusive")
 	}
 	if migrationContext.TestOnReplicaSkipReplicaStop {
 		if !migrationContext.TestOnReplica {
-			log.Fatalf("--test-on-replica-skip-replica-stop requires --test-on-replica to be enabled")
+			migrationContext.Log.Fatalf("--test-on-replica-skip-replica-stop requires --test-on-replica to be enabled")
 		}
-		log.Warning("--test-on-replica-skip-replica-stop enabled. We will not stop replication before cut-over. Ensure you have a plugin that does this.")
+		migrationContext.Log.Warning("--test-on-replica-skip-replica-stop enabled. We will not stop replication before cut-over. Ensure you have a plugin that does this.")
 	}
 	if migrationContext.CliMasterUser != "" && migrationContext.AssumeMasterHostname == "" {
-		log.Fatalf("--master-user requires --assume-master-host")
+		migrationContext.Log.Fatalf("--master-user requires --assume-master-host")
 	}
 	if migrationContext.CliMasterPassword != "" && migrationContext.AssumeMasterHostname == "" {
-		log.Fatalf("--master-password requires --assume-master-host")
+		migrationContext.Log.Fatalf("--master-password requires --assume-master-host")
 	}
 	if migrationContext.TLSCACertificate != "" && !migrationContext.UseTLS {
-		log.Fatalf("--ssl-ca requires --ssl")
+		migrationContext.Log.Fatalf("--ssl-ca requires --ssl")
 	}
 	if migrationContext.TLSCertificate != "" && !migrationContext.UseTLS {
-		log.Fatalf("--ssl-cert requires --ssl")
+		migrationContext.Log.Fatalf("--ssl-cert requires --ssl")
 	}
 	if migrationContext.TLSKey != "" && !migrationContext.UseTLS {
-		log.Fatalf("--ssl-key requires --ssl")
+		migrationContext.Log.Fatalf("--ssl-key requires --ssl")
 	}
 	if migrationContext.TLSAllowInsecure && !migrationContext.UseTLS {
-		log.Fatalf("--ssl-allow-insecure requires --ssl")
+		migrationContext.Log.Fatalf("--ssl-allow-insecure requires --ssl")
 	}
 	if *replicationLagQuery != "" {
-		log.Warningf("--replication-lag-query is deprecated")
+		migrationContext.Log.Warningf("--replication-lag-query is deprecated")
 	}
 
 	switch *cutOver {
@@ -227,19 +242,19 @@ func main() {
 	case "two-step":
 		migrationContext.CutOverType = base.CutOverTwoStep
 	default:
-		log.Fatalf("Unknown cut-over: %s", *cutOver)
+		migrationContext.Log.Fatalf("Unknown cut-over: %s", *cutOver)
 	}
 	if err := migrationContext.ReadConfigFile(); err != nil {
-		log.Fatale(err)
+		migrationContext.Log.Fatale(err)
 	}
 	if err := migrationContext.ReadThrottleControlReplicaKeys(*throttleControlReplicas); err != nil {
-		log.Fatale(err)
+		migrationContext.Log.Fatale(err)
 	}
 	if err := migrationContext.ReadMaxLoad(*maxLoad); err != nil {
-		log.Fatale(err)
+		migrationContext.Log.Fatale(err)
 	}
 	if err := migrationContext.ReadCriticalLoad(*criticalLoad); err != nil {
-		log.Fatale(err)
+		migrationContext.Log.Fatale(err)
 	}
 	if migrationContext.ServeSocketFile == "" {
 		migrationContext.ServeSocketFile = fmt.Sprintf("/tmp/gh-ost.%s.%s.sock", migrationContext.DatabaseName, migrationContext.OriginalTableName)
@@ -248,7 +263,7 @@ func main() {
 		fmt.Println("Password:")
 		bytePassword, err := terminal.ReadPassword(int(syscall.Stdin))
 		if err != nil {
-			log.Fatale(err)
+			migrationContext.Log.Fatale(err)
 		}
 		migrationContext.CliPassword = string(bytePassword)
 	}
@@ -259,16 +274,17 @@ func main() {
 	migrationContext.SetMaxLagMillisecondsThrottleThreshold(*maxLagMillis)
 	migrationContext.SetThrottleQuery(*throttleQuery)
 	migrationContext.SetThrottleHTTP(*throttleHTTP)
+	migrationContext.SetIgnoreHTTPErrors(*ignoreHTTPErrors)
 	migrationContext.SetDefaultNumRetries(*defaultRetries)
 	migrationContext.ApplyCredentials()
 	if err := migrationContext.SetupTLS(); err != nil {
-		log.Fatale(err)
+		migrationContext.Log.Fatale(err)
 	}
 	if err := migrationContext.SetCutOverLockTimeoutSeconds(*cutOverLockTimeoutSeconds); err != nil {
-		log.Errore(err)
+		migrationContext.Log.Errore(err)
 	}
 	if err := migrationContext.SetExponentialBackoffMaxInterval(*exponentialBackoffMaxInterval); err != nil {
-		log.Errore(err)
+		migrationContext.Log.Errore(err)
 	}
 
 	log.Infof("starting gh-ost %+v", AppVersion)
@@ -278,7 +294,7 @@ func main() {
 	err := migrator.Migrate()
 	if err != nil {
 		migrator.ExecOnFailureHook()
-		log.Fatale(err)
+		migrationContext.Log.Fatale(err)
 	}
 	fmt.Fprintf(os.Stdout, "# Done\n")
 }

go/logic/applier.go

@@ -16,8 +16,8 @@ import (
 	"github.com/github/gh-ost/go/mysql"
 	"github.com/github/gh-ost/go/sql"
 
-	"github.com/outbrain/golib/log"
 	"github.com/outbrain/golib/sqlutils"
+	"sync"
 )
 
 const (
@@ -57,6 +57,7 @@ type Applier struct {
 	singletonDB       *gosql.DB
 	migrationContext  *base.MigrationContext
 	finishedMigrating int64
+	name              string
 }
 
 func NewApplier(migrationContext *base.MigrationContext) *Applier {
@@ -64,6 +65,7 @@ func NewApplier(migrationContext *base.MigrationContext) *Applier {
 		connectionConfig:  migrationContext.ApplierConnectionConfig,
 		migrationContext:  migrationContext,
 		finishedMigrating: 0,
+		name:              "applier",
 	}
 }
 
@@ -78,18 +80,18 @@ func (this *Applier) InitDBConnections() (err error) {
 		return err
 	}
 	this.singletonDB.SetMaxOpenConns(1)
-	version, err := base.ValidateConnection(this.db, this.connectionConfig, this.migrationContext)
+	version, err := base.ValidateConnection(this.db, this.connectionConfig, this.migrationContext, this.name)
 	if err != nil {
 		return err
 	}
-	if _, err := base.ValidateConnection(this.singletonDB, this.connectionConfig, this.migrationContext); err != nil {
+	if _, err := base.ValidateConnection(this.singletonDB, this.connectionConfig, this.migrationContext, this.name); err != nil {
 		return err
 	}
 	this.migrationContext.ApplierMySQLVersion = version
 	if err := this.validateAndReadTimeZone(); err != nil {
 		return err
 	}
-	if !this.migrationContext.AliyunRDS && !this.migrationContext.GoogleCloudPlatform {
+	if !this.migrationContext.AliyunRDS && !this.migrationContext.GoogleCloudPlatform && !this.migrationContext.AzureMySQL {
 		if impliedKey, err := mysql.GetInstanceKey(this.db); err != nil {
 			return err
 		} else {
@@ -99,7 +101,7 @@ func (this *Applier) InitDBConnections() (err error) {
 	if err := this.readTableColumns(); err != nil {
 		return err
 	}
-	log.Infof("Applier initiated on %+v, version %+v", this.connectionConfig.ImpliedKey, this.migrationContext.ApplierMySQLVersion)
+	this.migrationContext.Log.Infof("Applier initiated on %+v, version %+v", this.connectionConfig.ImpliedKey, this.migrationContext.ApplierMySQLVersion)
 	return nil
 }
 
@@ -110,13 +112,13 @@ func (this *Applier) validateAndReadTimeZone() error {
 		return err
 	}
 
-	log.Infof("will use time_zone='%s' on applier", this.migrationContext.ApplierTimeZone)
+	this.migrationContext.Log.Infof("will use time_zone='%s' on applier", this.migrationContext.ApplierTimeZone)
 	return nil
 }
 
 // readTableColumns reads table columns on applier
 func (this *Applier) readTableColumns() (err error) {
-	log.Infof("Examining table structure on applier")
+	this.migrationContext.Log.Infof("Examining table structure on applier")
 	this.migrationContext.OriginalTableColumnsOnApplier, _, err = mysql.GetTableColumns(this.db, this.migrationContext.DatabaseName, this.migrationContext.OriginalTableName)
 	if err != nil {
 		return err
@@ -157,7 +159,7 @@ func (this *Applier) ValidateOrDropExistingTables() error {
 		}
 	}
 	if len(this.migrationContext.GetOldTableName()) > mysql.MaxTableNameLength {
-		log.Fatalf("--timestamp-old-table defined, but resulting table name (%s) is too long (only %d characters allowed)", this.migrationContext.GetOldTableName(), mysql.MaxTableNameLength)
+		this.migrationContext.Log.Fatalf("--timestamp-old-table defined, but resulting table name (%s) is too long (only %d characters allowed)", this.migrationContext.GetOldTableName(), mysql.MaxTableNameLength)
 	}
 
 	if this.tableExists(this.migrationContext.GetOldTableName()) {
@@ -175,14 +177,14 @@ func (this *Applier) CreateGhostTable() error {
 		sql.EscapeName(this.migrationContext.DatabaseName),
 		sql.EscapeName(this.migrationContext.OriginalTableName),
 	)
-	log.Infof("Creating ghost table %s.%s",
+	this.migrationContext.Log.Infof("Creating ghost table %s.%s",
 		sql.EscapeName(this.migrationContext.DatabaseName),
 		sql.EscapeName(this.migrationContext.GetGhostTableName()),
 	)
 	if _, err := sqlutils.ExecNoPrepare(this.db, query); err != nil {
 		return err
 	}
-	log.Infof("Ghost table created")
+	this.migrationContext.Log.Infof("Ghost table created")
 	return nil
 }
 
@@ -191,17 +193,17 @@ func (this *Applier) AlterGhost() error {
 	query := fmt.Sprintf(`alter /* gh-ost */ table %s.%s %s`,
 		sql.EscapeName(this.migrationContext.DatabaseName),
 		sql.EscapeName(this.migrationContext.GetGhostTableName()),
-		this.migrationContext.AlterStatement,
+		this.migrationContext.AlterStatementOptions,
 	)
-	log.Infof("Altering ghost table %s.%s",
+	this.migrationContext.Log.Infof("Altering ghost table %s.%s",
 		sql.EscapeName(this.migrationContext.DatabaseName),
 		sql.EscapeName(this.migrationContext.GetGhostTableName()),
 	)
-	log.Debugf("ALTER statement: %s", query)
+	this.migrationContext.Log.Debugf("ALTER statement: %s", query)
 	if _, err := sqlutils.ExecNoPrepare(this.db, query); err != nil {
 		return err
 	}
-	log.Infof("Ghost table altered")
+	this.migrationContext.Log.Infof("Ghost table altered")
 	return nil
 }
 
@@ -222,14 +224,14 @@ func (this *Applier) CreateChangelogTable() error {
 		sql.EscapeName(this.migrationContext.DatabaseName),
 		sql.EscapeName(this.migrationContext.GetChangelogTableName()),
 	)
-	log.Infof("Creating changelog table %s.%s",
+	this.migrationContext.Log.Infof("Creating changelog table %s.%s",
 		sql.EscapeName(this.migrationContext.DatabaseName),
 		sql.EscapeName(this.migrationContext.GetChangelogTableName()),
 	)
 	if _, err := sqlutils.ExecNoPrepare(this.db, query); err != nil {
 		return err
 	}
-	log.Infof("Changelog table created")
+	this.migrationContext.Log.Infof("Changelog table created")
 	return nil
 }
 
@@ -239,14 +241,14 @@ func (this *Applier) dropTable(tableName string) error {
 		sql.EscapeName(this.migrationContext.DatabaseName),
 		sql.EscapeName(tableName),
 	)
-	log.Infof("Dropping table %s.%s",
+	this.migrationContext.Log.Infof("Dropping table %s.%s",
 		sql.EscapeName(this.migrationContext.DatabaseName),
 		sql.EscapeName(tableName),
 	)
 	if _, err := sqlutils.ExecNoPrepare(this.db, query); err != nil {
 		return err
 	}
-	log.Infof("Table dropped")
+	this.migrationContext.Log.Infof("Table dropped")
 	return nil
 }
 
@@ -313,7 +315,7 @@ func (this *Applier) InitiateHeartbeat() {
 		if _, err := this.WriteChangelog("heartbeat", time.Now().Format(time.RFC3339Nano)); err != nil {
 			numSuccessiveFailures++
 			if numSuccessiveFailures > this.migrationContext.MaxRetries() {
-				return log.Errore(err)
+				return this.migrationContext.Log.Errore(err)
 			}
 		} else {
 			numSuccessiveFailures = 0
@@ -348,14 +350,14 @@ func (this *Applier) ExecuteThrottleQuery() (int64, error) {
 	}
 	var result int64
 	if err := this.db.QueryRow(throttleQuery).Scan(&result); err != nil {
-		return 0, log.Errore(err)
+		return 0, this.migrationContext.Log.Errore(err)
 	}
 	return result, nil
 }
 
 // ReadMigrationMinValues returns the minimum values to be iterated on rowcopy
 func (this *Applier) ReadMigrationMinValues(uniqueKey *sql.UniqueKey) error {
-	log.Debugf("Reading migration range according to key: %s", uniqueKey.Name)
+	this.migrationContext.Log.Debugf("Reading migration range according to key: %s", uniqueKey.Name)
 	query, err := sql.BuildUniqueKeyMinValuesPreparedQuery(this.migrationContext.DatabaseName, this.migrationContext.OriginalTableName, &uniqueKey.Columns)
 	if err != nil {
 		return err
@@ -370,13 +372,15 @@ func (this *Applier) ReadMigrationMinValues(uniqueKey *sql.UniqueKey) error {
 			return err
 		}
 	}
-	log.Infof("Migration min values: [%s]", this.migrationContext.MigrationRangeMinValues)
+	this.migrationContext.Log.Infof("Migration min values: [%s]", this.migrationContext.MigrationRangeMinValues)
+
+	err = rows.Err()
 	return err
 }
 
 // ReadMigrationMaxValues returns the maximum values to be iterated on rowcopy
 func (this *Applier) ReadMigrationMaxValues(uniqueKey *sql.UniqueKey) error {
-	log.Debugf("Reading migration range according to key: %s", uniqueKey.Name)
+	this.migrationContext.Log.Debugf("Reading migration range according to key: %s", uniqueKey.Name)
 	query, err := sql.BuildUniqueKeyMaxValuesPreparedQuery(this.migrationContext.DatabaseName, this.migrationContext.OriginalTableName, &uniqueKey.Columns)
 	if err != nil {
 		return err
@@ -391,7 +395,9 @@ func (this *Applier) ReadMigrationMaxValues(uniqueKey *sql.UniqueKey) error {
 			return err
 		}
 	}
-	log.Infof("Migration max values: [%s]", this.migrationContext.MigrationRangeMaxValues)
+	this.migrationContext.Log.Infof("Migration max values: [%s]", this.migrationContext.MigrationRangeMaxValues)
+
+	err = rows.Err()
 	return err
 }
 
@@ -444,12 +450,15 @@ func (this *Applier) CalculateNextIterationRangeEndValues() (hasFurtherRange boo
 			}
 			hasFurtherRange = true
 		}
+		if err = rows.Err(); err != nil {
+			return hasFurtherRange, err
+		}
 		if hasFurtherRange {
 			this.migrationContext.MigrationIterationRangeMaxValues = iterationRangeMaxValues
 			return hasFurtherRange, nil
 		}
 	}
-	log.Debugf("Iteration complete: no further range to iterate")
+	this.migrationContext.Log.Debugf("Iteration complete: no further range to iterate")
 	return hasFurtherRange, nil
 }
 
@@ -507,7 +516,7 @@ func (this *Applier) ApplyIterationInsertQuery() (chunkSize int64, rowsAffected
 	}
 	rowsAffected, _ = sqlResult.RowsAffected()
 	duration = time.Since(startTime)
-	log.Debugf(
+	this.migrationContext.Log.Debugf(
 		"Issued INSERT on range: [%s]..[%s]; iteration: %d; chunk-size: %d",
 		this.migrationContext.MigrationIterationRangeMinValues,
 		this.migrationContext.MigrationIterationRangeMaxValues,
@@ -522,7 +531,7 @@ func (this *Applier) LockOriginalTable() error {
 		sql.EscapeName(this.migrationContext.DatabaseName),
 		sql.EscapeName(this.migrationContext.OriginalTableName),
 	)
-	log.Infof("Locking %s.%s",
+	this.migrationContext.Log.Infof("Locking %s.%s",
 		sql.EscapeName(this.migrationContext.DatabaseName),
 		sql.EscapeName(this.migrationContext.OriginalTableName),
 	)
@@ -530,18 +539,18 @@ func (this *Applier) LockOriginalTable() error {
 	if _, err := sqlutils.ExecNoPrepare(this.singletonDB, query); err != nil {
 		return err
 	}
-	log.Infof("Table locked")
+	this.migrationContext.Log.Infof("Table locked")
 	return nil
 }
 
 // UnlockTables makes tea. No wait, it unlocks tables.
 func (this *Applier) UnlockTables() error {
 	query := `unlock /* gh-ost */ tables`
-	log.Infof("Unlocking tables")
+	this.migrationContext.Log.Infof("Unlocking tables")
 	if _, err := sqlutils.ExecNoPrepare(this.singletonDB, query); err != nil {
 		return err
 	}
-	log.Infof("Tables unlocked")
+	this.migrationContext.Log.Infof("Tables unlocked")
 	return nil
 }
 
@@ -555,7 +564,7 @@ func (this *Applier) SwapTablesQuickAndBumpy() error {
 		sql.EscapeName(this.migrationContext.OriginalTableName),
 		sql.EscapeName(this.migrationContext.GetOldTableName()),
 	)
-	log.Infof("Renaming original table")
+	this.migrationContext.Log.Infof("Renaming original table")
 	this.migrationContext.RenameTablesStartTime = time.Now()
 	if _, err := sqlutils.ExecNoPrepare(this.singletonDB, query); err != nil {
 		return err
@@ -565,13 +574,13 @@ func (this *Applier) SwapTablesQuickAndBumpy() error {
 		sql.EscapeName(this.migrationContext.GetGhostTableName()),
 		sql.EscapeName(this.migrationContext.OriginalTableName),
 	)
-	log.Infof("Renaming ghost table")
+	this.migrationContext.Log.Infof("Renaming ghost table")
 	if _, err := sqlutils.ExecNoPrepare(this.db, query); err != nil {
 		return err
 	}
 	this.migrationContext.RenameTablesEndTime = time.Now()
 
-	log.Infof("Tables renamed")
+	this.migrationContext.Log.Infof("Tables renamed")
 	return nil
 }
 
@@ -590,7 +599,7 @@ func (this *Applier) RenameTablesRollback() (renameError error) {
 		sql.EscapeName(this.migrationContext.DatabaseName),
 		sql.EscapeName(this.migrationContext.OriginalTableName),
 	)
-	log.Infof("Renaming back both tables")
+	this.migrationContext.Log.Infof("Renaming back both tables")
 	if _, err := sqlutils.ExecNoPrepare(this.db, query); err == nil {
 		return nil
 	}
@@ -601,7 +610,7 @@ func (this *Applier) RenameTablesRollback() (renameError error) {
 		sql.EscapeName(this.migrationContext.DatabaseName),
 		sql.EscapeName(this.migrationContext.GetGhostTableName()),
 	)
-	log.Infof("Renaming back to ghost table")
+	this.migrationContext.Log.Infof("Renaming back to ghost table")
 	if _, err := sqlutils.ExecNoPrepare(this.db, query); err != nil {
 		renameError = err
 	}
@@ -611,11 +620,11 @@ func (this *Applier) RenameTablesRollback() (renameError error) {
 		sql.EscapeName(this.migrationContext.DatabaseName),
 		sql.EscapeName(this.migrationContext.OriginalTableName),
 	)
-	log.Infof("Renaming back to original table")
+	this.migrationContext.Log.Infof("Renaming back to original table")
 	if _, err := sqlutils.ExecNoPrepare(this.db, query); err != nil {
 		renameError = err
 	}
-	return log.Errore(renameError)
+	return this.migrationContext.Log.Errore(renameError)
 }
 
 // StopSlaveIOThread is applicable with --test-on-replica; it stops the IO thread, duh.
@@ -623,44 +632,44 @@ func (this *Applier) RenameTablesRollback() (renameError error) {
 // and have them written to the binary log, so that we can then read them via streamer.
 func (this *Applier) StopSlaveIOThread() error {
 	query := `stop /* gh-ost */ slave io_thread`
-	log.Infof("Stopping replication IO thread")
+	this.migrationContext.Log.Infof("Stopping replication IO thread")
 	if _, err := sqlutils.ExecNoPrepare(this.db, query); err != nil {
 		return err
 	}
-	log.Infof("Replication IO thread stopped")
+	this.migrationContext.Log.Infof("Replication IO thread stopped")
 	return nil
 }
 
 // StartSlaveIOThread is applicable with --test-on-replica
 func (this *Applier) StartSlaveIOThread() error {
 	query := `start /* gh-ost */ slave io_thread`
-	log.Infof("Starting replication IO thread")
+	this.migrationContext.Log.Infof("Starting replication IO thread")
 	if _, err := sqlutils.ExecNoPrepare(this.db, query); err != nil {
 		return err
 	}
-	log.Infof("Replication IO thread started")
+	this.migrationContext.Log.Infof("Replication IO thread started")
 	return nil
 }
 
 // StartSlaveSQLThread is applicable with --test-on-replica
 func (this *Applier) StopSlaveSQLThread() error {
 	query := `stop /* gh-ost */ slave sql_thread`
-	log.Infof("Verifying SQL thread is stopped")
+	this.migrationContext.Log.Infof("Verifying SQL thread is stopped")
 	if _, err := sqlutils.ExecNoPrepare(this.db, query); err != nil {
 		return err
 	}
-	log.Infof("SQL thread stopped")
+	this.migrationContext.Log.Infof("SQL thread stopped")
 	return nil
 }
 
 // StartSlaveSQLThread is applicable with --test-on-replica
 func (this *Applier) StartSlaveSQLThread() error {
 	query := `start /* gh-ost */ slave sql_thread`
-	log.Infof("Verifying SQL thread is running")
+	this.migrationContext.Log.Infof("Verifying SQL thread is running")
 	if _, err := sqlutils.ExecNoPrepare(this.db, query); err != nil {
 		return err
 	}
-	log.Infof("SQL thread started")
+	this.migrationContext.Log.Infof("SQL thread started")
 	return nil
 }
 
@@ -677,7 +686,7 @@ func (this *Applier) StopReplication() error {
 	if err != nil {
 		return err
 	}
-	log.Infof("Replication IO thread at %+v. SQL thread is at %+v", *readBinlogCoordinates, *executeBinlogCoordinates)
+	this.migrationContext.Log.Infof("Replication IO thread at %+v. SQL thread is at %+v", *readBinlogCoordinates, *executeBinlogCoordinates)
 	return nil
 }
 
@@ -689,7 +698,7 @@ func (this *Applier) StartReplication() error {
 	if err := this.StartSlaveSQLThread(); err != nil {
 		return err
 	}
-	log.Infof("Replication started")
+	this.migrationContext.Log.Infof("Replication started")
 	return nil
 }
 
@@ -703,7 +712,7 @@ func (this *Applier) ExpectUsedLock(sessionId int64) error {
 	var result int64
 	query := `select is_used_lock(?)`
 	lockName := this.GetSessionLockName(sessionId)
-	log.Infof("Checking session lock: %s", lockName)
+	this.migrationContext.Log.Infof("Checking session lock: %s", lockName)
 	if err := this.db.QueryRow(query, lockName).Scan(&result); err != nil || result != sessionId {
 		return fmt.Errorf("Session lock %s expected to be found but wasn't", lockName)
 	}
@@ -738,7 +747,7 @@ func (this *Applier) ExpectProcess(sessionId int64, stateHint, infoHint string)
 // DropAtomicCutOverSentryTableIfExists checks if the "old" table name
 // happens to be a cut-over magic table; if so, it drops it.
 func (this *Applier) DropAtomicCutOverSentryTableIfExists() error {
-	log.Infof("Looking for magic cut-over table")
+	this.migrationContext.Log.Infof("Looking for magic cut-over table")
 	tableName := this.migrationContext.GetOldTableName()
 	rowMap := this.showTableStatus(tableName)
 	if rowMap == nil {
@@ -748,7 +757,7 @@ func (this *Applier) DropAtomicCutOverSentryTableIfExists() error {
 	if rowMap["Comment"].String != atomicCutOverMagicHint {
 		return fmt.Errorf("Expected magic comment on %s, did not find it", tableName)
 	}
-	log.Infof("Dropping magic cut-over table")
+	this.migrationContext.Log.Infof("Dropping magic cut-over table")
 	return this.dropTable(tableName)
 }
 
@@ -768,20 +777,20 @@ func (this *Applier) CreateAtomicCutOverSentryTable() error {
 		this.migrationContext.TableEngine,
 		atomicCutOverMagicHint,
 	)
-	log.Infof("Creating magic cut-over table %s.%s",
+	this.migrationContext.Log.Infof("Creating magic cut-over table %s.%s",
 		sql.EscapeName(this.migrationContext.DatabaseName),
 		sql.EscapeName(tableName),
 	)
 	if _, err := sqlutils.ExecNoPrepare(this.db, query); err != nil {
 		return err
 	}
-	log.Infof("Magic cut-over table created")
+	this.migrationContext.Log.Infof("Magic cut-over table created")
 
 	return nil
 }
 
 // AtomicCutOverMagicLock
-func (this *Applier) AtomicCutOverMagicLock(sessionIdChan chan int64, tableLocked chan<- error, okToUnlockTable <-chan bool, tableUnlocked chan<- error) error {
+func (this *Applier) AtomicCutOverMagicLock(sessionIdChan chan int64, tableLocked chan<- error, okToUnlockTable <-chan bool, tableUnlocked chan<- error, dropCutOverSentryTableOnce *sync.Once) error {
 	tx, err := this.db.Begin()
 	if err != nil {
 		tableLocked <- err
@@ -804,7 +813,7 @@ func (this *Applier) AtomicCutOverMagicLock(sessionIdChan chan int64, tableLocke
 	lockResult := 0
 	query := `select get_lock(?, 0)`
 	lockName := this.GetSessionLockName(sessionId)
-	log.Infof("Grabbing voluntary lock: %s", lockName)
+	this.migrationContext.Log.Infof("Grabbing voluntary lock: %s", lockName)
 	if err := tx.QueryRow(query, lockName).Scan(&lockResult); err != nil || lockResult != 1 {
 		err := fmt.Errorf("Unable to acquire lock %s", lockName)
 		tableLocked <- err
@@ -812,7 +821,7 @@ func (this *Applier) AtomicCutOverMagicLock(sessionIdChan chan int64, tableLocke
 	}
 
 	tableLockTimeoutSeconds := this.migrationContext.CutOverLockTimeoutSeconds * 2
-	log.Infof("Setting LOCK timeout as %d seconds", tableLockTimeoutSeconds)
+	this.migrationContext.Log.Infof("Setting LOCK timeout as %d seconds", tableLockTimeoutSeconds)
 	query = fmt.Sprintf(`set session lock_wait_timeout:=%d`, tableLockTimeoutSeconds)
 	if _, err := tx.Exec(query); err != nil {
 		tableLocked <- err
@@ -830,7 +839,7 @@ func (this *Applier) AtomicCutOverMagicLock(sessionIdChan chan int64, tableLocke
 		sql.EscapeName(this.migrationContext.DatabaseName),
 		sql.EscapeName(this.migrationContext.GetOldTableName()),
 	)
-	log.Infof("Locking %s.%s, %s.%s",
+	this.migrationContext.Log.Infof("Locking %s.%s, %s.%s",
 		sql.EscapeName(this.migrationContext.DatabaseName),
 		sql.EscapeName(this.migrationContext.OriginalTableName),
 		sql.EscapeName(this.migrationContext.DatabaseName),
@@ -841,7 +850,7 @@ func (this *Applier) AtomicCutOverMagicLock(sessionIdChan chan int64, tableLocke
 		tableLocked <- err
 		return err
 	}
-	log.Infof("Tables locked")
+	this.migrationContext.Log.Infof("Tables locked")
 	tableLocked <- nil // No error.
 
 	// From this point on, we are committed to UNLOCK TABLES. No matter what happens,
@@ -850,22 +859,25 @@ func (this *Applier) AtomicCutOverMagicLock(sessionIdChan chan int64, tableLocke
 	// The cut-over phase will proceed to apply remaining backlog onto ghost table,
 	// and issue RENAME. We wait here until told to proceed.
 	<-okToUnlockTable
-	log.Infof("Will now proceed to drop magic table and unlock tables")
+	this.migrationContext.Log.Infof("Will now proceed to drop magic table and unlock tables")
 
 	// The magic table is here because we locked it. And we are the only ones allowed to drop it.
 	// And in fact, we will:
-	log.Infof("Dropping magic cut-over table")
+	this.migrationContext.Log.Infof("Dropping magic cut-over table")
 	query = fmt.Sprintf(`drop /* gh-ost */ table if exists %s.%s`,
 		sql.EscapeName(this.migrationContext.DatabaseName),
 		sql.EscapeName(this.migrationContext.GetOldTableName()),
 	)
-	if _, err := tx.Exec(query); err != nil {
+
-		log.Errore(err)
+	dropCutOverSentryTableOnce.Do(func() {
-		// We DO NOT return here because we must `UNLOCK TABLES`!
+		if _, err := tx.Exec(query); err != nil {
-	}
+			this.migrationContext.Log.Errore(err)
+			// We DO NOT return here because we must `UNLOCK TABLES`!
+		}
+	})
 
 	// Tables still locked
-	log.Infof("Releasing lock from %s.%s, %s.%s",
+	this.migrationContext.Log.Infof("Releasing lock from %s.%s, %s.%s",
 		sql.EscapeName(this.migrationContext.DatabaseName),
 		sql.EscapeName(this.migrationContext.OriginalTableName),
 		sql.EscapeName(this.migrationContext.DatabaseName),
@@ -874,9 +886,9 @@ func (this *Applier) AtomicCutOverMagicLock(sessionIdChan chan int64, tableLocke
 	query = `unlock tables`
 	if _, err := tx.Exec(query); err != nil {
 		tableUnlocked <- err
-		return log.Errore(err)
+		return this.migrationContext.Log.Errore(err)
 	}
-	log.Infof("Tables unlocked")
+	this.migrationContext.Log.Infof("Tables unlocked")
 	tableUnlocked <- nil
 	return nil
 }
@@ -898,7 +910,7 @@ func (this *Applier) AtomicCutoverRename(sessionIdChan chan int64, tablesRenamed
 	}
 	sessionIdChan <- sessionId
 
-	log.Infof("Setting RENAME timeout as %d seconds", this.migrationContext.CutOverLockTimeoutSeconds)
+	this.migrationContext.Log.Infof("Setting RENAME timeout as %d seconds", this.migrationContext.CutOverLockTimeoutSeconds)
 	query := fmt.Sprintf(`set session lock_wait_timeout:=%d`, this.migrationContext.CutOverLockTimeoutSeconds)
 	if _, err := tx.Exec(query); err != nil {
 		return err
@@ -914,13 +926,13 @@ func (this *Applier) AtomicCutoverRename(sessionIdChan chan int64, tablesRenamed
 		sql.EscapeName(this.migrationContext.DatabaseName),
 		sql.EscapeName(this.migrationContext.OriginalTableName),
 	)
-	log.Infof("Issuing and expecting this to block: %s", query)
+	this.migrationContext.Log.Infof("Issuing and expecting this to block: %s", query)
 	if _, err := tx.Exec(query); err != nil {
 		tablesRenamed <- err
-		return log.Errore(err)
+		return this.migrationContext.Log.Errore(err)
 	}
 	tablesRenamed <- nil
-	log.Infof("Tables renamed")
+	this.migrationContext.Log.Infof("Tables renamed")
 	return nil
 }
 
@@ -1026,19 +1038,19 @@ func (this *Applier) ApplyDMLEventQueries(dmlEvents [](*binlog.BinlogDMLEvent))
 	}()
 
 	if err != nil {
-		return log.Errore(err)
+		return this.migrationContext.Log.Errore(err)
 	}
 	// no error
 	atomic.AddInt64(&this.migrationContext.TotalDMLEventsApplied, int64(len(dmlEvents)))
 	if this.migrationContext.CountTableRows {
 		atomic.AddInt64(&this.migrationContext.RowsDeltaEstimate, totalDelta)
 	}
-	log.Debugf("ApplyDMLEventQueries() applied %d events in one transaction", len(dmlEvents))
+	this.migrationContext.Log.Debugf("ApplyDMLEventQueries() applied %d events in one transaction", len(dmlEvents))
 	return nil
 }
 
 func (this *Applier) Teardown() {
-	log.Debugf("Tearing down...")
+	this.migrationContext.Log.Debugf("Tearing down...")
 	this.db.Close()
 	this.singletonDB.Close()
 	atomic.StoreInt64(&this.finishedMigrating, 1)

go/logic/hooks.go

@@ -64,6 +64,7 @@ func (this *HooksExecutor) applyEnvironmentVariables(extraVariables ...string) [
 	env = append(env, fmt.Sprintf("GH_OST_INSPECTED_HOST=%s", this.migrationContext.GetInspectorHostname()))
 	env = append(env, fmt.Sprintf("GH_OST_EXECUTING_HOST=%s", this.migrationContext.Hostname))
 	env = append(env, fmt.Sprintf("GH_OST_INSPECTED_LAG=%f", this.migrationContext.GetCurrentLagDuration().Seconds()))
+	env = append(env, fmt.Sprintf("GH_OST_HEARTBEAT_LAG=%f", this.migrationContext.TimeSinceLastHeartbeatOnChangelog().Seconds()))
 	env = append(env, fmt.Sprintf("GH_OST_PROGRESS=%f", this.migrationContext.GetProgressPct()))
 	env = append(env, fmt.Sprintf("GH_OST_HOOKS_HINT=%s", this.migrationContext.HooksHintMessage))
 	env = append(env, fmt.Sprintf("GH_OST_HOOKS_HINT_OWNER=%s", this.migrationContext.HooksHintOwner))

go/logic/inspect.go

@@ -17,7 +17,6 @@ import (
 	"github.com/github/gh-ost/go/mysql"
 	"github.com/github/gh-ost/go/sql"
 
-	"github.com/outbrain/golib/log"
 	"github.com/outbrain/golib/sqlutils"
 )
 
@@ -30,12 +29,14 @@ type Inspector struct {
 	db                  *gosql.DB
 	informationSchemaDb *gosql.DB
 	migrationContext    *base.MigrationContext
+	name                string
 }
 
 func NewInspector(migrationContext *base.MigrationContext) *Inspector {
 	return &Inspector{
 		connectionConfig: migrationContext.InspectorConnectionConfig,
 		migrationContext: migrationContext,
+		name:             "inspector",
 	}
 }
 
@@ -53,7 +54,7 @@ func (this *Inspector) InitDBConnections() (err error) {
 	if err := this.validateConnection(); err != nil {
 		return err
 	}
-	if !this.migrationContext.AliyunRDS && !this.migrationContext.GoogleCloudPlatform {
+	if !this.migrationContext.AliyunRDS && !this.migrationContext.GoogleCloudPlatform && !this.migrationContext.AzureMySQL {
 		if impliedKey, err := mysql.GetInstanceKey(this.db); err != nil {
 			return err
 		} else {
@@ -69,7 +70,7 @@ func (this *Inspector) InitDBConnections() (err error) {
 	if err := this.applyBinlogFormat(); err != nil {
 		return err
 	}
-	log.Infof("Inspector initiated on %+v, version %+v", this.connectionConfig.ImpliedKey, this.migrationContext.InspectorMySQLVersion)
+	this.migrationContext.Log.Infof("Inspector initiated on %+v, version %+v", this.connectionConfig.ImpliedKey, this.migrationContext.InspectorMySQLVersion)
 	return nil
 }
 
@@ -137,14 +138,14 @@ func (this *Inspector) inspectOriginalAndGhostTables() (err error) {
 			switch column.Type {
 			case sql.FloatColumnType:
 				{
-					log.Warning("Will not use %+v as shared key due to FLOAT data type", sharedUniqueKey.Name)
+					this.migrationContext.Log.Warning("Will not use %+v as shared key due to FLOAT data type", sharedUniqueKey.Name)
 					uniqueKeyIsValid = false
 				}
 			case sql.JSONColumnType:
 				{
 					// Noteworthy that at this time MySQL does not allow JSON indexing anyhow, but this code
 					// will remain in place to potentially handle the future case where JSON is supported in indexes.
-					log.Warning("Will not use %+v as shared key due to JSON data type", sharedUniqueKey.Name)
+					this.migrationContext.Log.Warning("Will not use %+v as shared key due to JSON data type", sharedUniqueKey.Name)
 					uniqueKeyIsValid = false
 				}
 			}
@@ -157,17 +158,17 @@ func (this *Inspector) inspectOriginalAndGhostTables() (err error) {
 	if this.migrationContext.UniqueKey == nil {
 		return fmt.Errorf("No shared unique key can be found after ALTER! Bailing out")
 	}
-	log.Infof("Chosen shared unique key is %s", this.migrationContext.UniqueKey.Name)
+	this.migrationContext.Log.Infof("Chosen shared unique key is %s", this.migrationContext.UniqueKey.Name)
 	if this.migrationContext.UniqueKey.HasNullable {
 		if this.migrationContext.NullableUniqueKeyAllowed {
-			log.Warningf("Chosen key (%s) has nullable columns. You have supplied with --allow-nullable-unique-key and so this migration proceeds. As long as there aren't NULL values in this key's column, migration should be fine. NULL values will corrupt migration's data", this.migrationContext.UniqueKey)
+			this.migrationContext.Log.Warningf("Chosen key (%s) has nullable columns. You have supplied with --allow-nullable-unique-key and so this migration proceeds. As long as there aren't NULL values in this key's column, migration should be fine. NULL values will corrupt migration's data", this.migrationContext.UniqueKey)
 		} else {
 			return fmt.Errorf("Chosen key (%s) has nullable columns. Bailing out. To force this operation to continue, supply --allow-nullable-unique-key flag. Only do so if you are certain there are no actual NULL values in this key. As long as there aren't, migration should be fine. NULL values in columns of this key will corrupt migration's data", this.migrationContext.UniqueKey)
 		}
 	}
 
 	this.migrationContext.SharedColumns, this.migrationContext.MappedSharedColumns = this.getSharedColumns(this.migrationContext.OriginalTableColumns, this.migrationContext.GhostTableColumns, this.migrationContext.OriginalTableVirtualColumns, this.migrationContext.GhostTableVirtualColumns, this.migrationContext.ColumnRenameMap)
-	log.Infof("Shared columns are %s", this.migrationContext.SharedColumns)
+	this.migrationContext.Log.Infof("Shared columns are %s", this.migrationContext.SharedColumns)
 	// By fact that a non-empty unique key exists we also know the shared columns are non-empty
 
 	// This additional step looks at which columns are unsigned. We could have merged this within
@@ -199,7 +200,7 @@ func (this *Inspector) validateConnection() error {
 		return fmt.Errorf("MySQL replication length limited to 32 characters. See https://dev.mysql.com/doc/refman/5.7/en/assigning-passwords.html")
 	}
 
-	version, err := base.ValidateConnection(this.db, this.connectionConfig, this.migrationContext)
+	version, err := base.ValidateConnection(this.db, this.connectionConfig, this.migrationContext, this.name)
 	this.migrationContext.InspectorMySQLVersion = version
 	return err
 }
@@ -250,19 +251,19 @@ func (this *Inspector) validateGrants() error {
 	this.migrationContext.HasSuperPrivilege = foundSuper
 
 	if foundAll {
-		log.Infof("User has ALL privileges")
+		this.migrationContext.Log.Infof("User has ALL privileges")
 		return nil
 	}
 	if foundSuper && foundReplicationSlave && foundDBAll {
-		log.Infof("User has SUPER, REPLICATION SLAVE privileges, and has ALL privileges on %s.*", sql.EscapeName(this.migrationContext.DatabaseName))
+		this.migrationContext.Log.Infof("User has SUPER, REPLICATION SLAVE privileges, and has ALL privileges on %s.*", sql.EscapeName(this.migrationContext.DatabaseName))
 		return nil
 	}
 	if foundReplicationClient && foundReplicationSlave && foundDBAll {
-		log.Infof("User has REPLICATION CLIENT, REPLICATION SLAVE privileges, and has ALL privileges on %s.*", sql.EscapeName(this.migrationContext.DatabaseName))
+		this.migrationContext.Log.Infof("User has REPLICATION CLIENT, REPLICATION SLAVE privileges, and has ALL privileges on %s.*", sql.EscapeName(this.migrationContext.DatabaseName))
 		return nil
 	}
-	log.Debugf("Privileges: Super: %t, REPLICATION CLIENT: %t, REPLICATION SLAVE: %t, ALL on *.*: %t, ALL on %s.*: %t", foundSuper, foundReplicationClient, foundReplicationSlave, foundAll, sql.EscapeName(this.migrationContext.DatabaseName), foundDBAll)
+	this.migrationContext.Log.Debugf("Privileges: Super: %t, REPLICATION CLIENT: %t, REPLICATION SLAVE: %t, ALL on *.*: %t, ALL on %s.*: %t", foundSuper, foundReplicationClient, foundReplicationSlave, foundAll, sql.EscapeName(this.migrationContext.DatabaseName), foundDBAll)
-	return log.Errorf("User has insufficient privileges for migration. Needed: SUPER|REPLICATION CLIENT, REPLICATION SLAVE and ALL on %s.*", sql.EscapeName(this.migrationContext.DatabaseName))
+	return this.migrationContext.Log.Errorf("User has insufficient privileges for migration. Needed: SUPER|REPLICATION CLIENT, REPLICATION SLAVE and ALL on %s.*", sql.EscapeName(this.migrationContext.DatabaseName))
 }
 
 // restartReplication is required so that we are _certain_ the binlog format and
@@ -270,7 +271,7 @@ func (this *Inspector) validateGrants() error {
 // It is entirely possible, for example, that the replication is using 'STATEMENT'
 // binlog format even as the variable says 'ROW'
 func (this *Inspector) restartReplication() error {
-	log.Infof("Restarting replication on %s:%d to make sure binlog settings apply to replication thread", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port)
+	this.migrationContext.Log.Infof("Restarting replication on %s:%d to make sure binlog settings apply to replication thread", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port)
 
 	masterKey, _ := mysql.GetMasterKeyFromSlaveStatus(this.connectionConfig)
 	if masterKey == nil {
@@ -289,7 +290,7 @@ func (this *Inspector) restartReplication() error {
 	}
 	time.Sleep(startSlavePostWaitMilliseconds)
 
-	log.Debugf("Replication restarted")
+	this.migrationContext.Log.Debugf("Replication restarted")
 	return nil
 }
 
@@ -309,7 +310,7 @@ func (this *Inspector) applyBinlogFormat() error {
 		if err := this.restartReplication(); err != nil {
 			return err
 		}
-		log.Debugf("'ROW' binlog format applied")
+		this.migrationContext.Log.Debugf("'ROW' binlog format applied")
 		return nil
 	}
 	// We already have RBR, no explicit switch
@@ -347,7 +348,7 @@ func (this *Inspector) validateBinlogs() error {
 		if countReplicas > 0 {
 			return fmt.Errorf("%s:%d has %s binlog_format, but I'm too scared to change it to ROW because it has replicas. Bailing out", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port, this.migrationContext.OriginalBinlogFormat)
 		}
-		log.Infof("%s:%d has %s binlog_format. I will change it to ROW, and will NOT change it back, even in the event of failure.", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port, this.migrationContext.OriginalBinlogFormat)
+		this.migrationContext.Log.Infof("%s:%d has %s binlog_format. I will change it to ROW, and will NOT change it back, even in the event of failure.", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port, this.migrationContext.OriginalBinlogFormat)
 	}
 	query = `select @@global.binlog_row_image`
 	if err := this.db.QueryRow(query).Scan(&this.migrationContext.OriginalBinlogRowImage); err != nil {
@@ -359,7 +360,7 @@ func (this *Inspector) validateBinlogs() error {
 		return fmt.Errorf("%s:%d has '%s' binlog_row_image, and only 'FULL' is supported. This operation cannot proceed. You may `set global binlog_row_image='full'` and try again", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port, this.migrationContext.OriginalBinlogRowImage)
 	}
 
-	log.Infof("binary logs validated on %s:%d", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port)
+	this.migrationContext.Log.Infof("binary logs validated on %s:%d", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port)
 	return nil
 }
 
@@ -372,12 +373,12 @@ func (this *Inspector) validateLogSlaveUpdates() error {
 	}
 
 	if logSlaveUpdates {
-		log.Infof("log_slave_updates validated on %s:%d", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port)
+		this.migrationContext.Log.Infof("log_slave_updates validated on %s:%d", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port)
 		return nil
 	}
 
 	if this.migrationContext.IsTungsten {
-		log.Warningf("log_slave_updates not found on %s:%d, but --tungsten provided, so I'm proceeding", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port)
+		this.migrationContext.Log.Warningf("log_slave_updates not found on %s:%d, but --tungsten provided, so I'm proceeding", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port)
 		return nil
 	}
 
@@ -386,7 +387,7 @@ func (this *Inspector) validateLogSlaveUpdates() error {
 	}
 
 	if this.migrationContext.InspectorIsAlsoApplier() {
-		log.Warningf("log_slave_updates not found on %s:%d, but executing directly on master, so I'm proceeding", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port)
+		this.migrationContext.Log.Warningf("log_slave_updates not found on %s:%d, but executing directly on master, so I'm proceeding", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port)
 		return nil
 	}
 
@@ -413,17 +414,17 @@ func (this *Inspector) validateTable() error {
 		return err
 	}
 	if !tableFound {
-		return log.Errorf("Cannot find table %s.%s!", sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName))
+		return this.migrationContext.Log.Errorf("Cannot find table %s.%s!", sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName))
 	}
-	log.Infof("Table found. Engine=%s", this.migrationContext.TableEngine)
+	this.migrationContext.Log.Infof("Table found. Engine=%s", this.migrationContext.TableEngine)
-	log.Debugf("Estimated number of rows via STATUS: %d", this.migrationContext.RowsEstimate)
+	this.migrationContext.Log.Debugf("Estimated number of rows via STATUS: %d", this.migrationContext.RowsEstimate)
 	return nil
 }
 
 // validateTableForeignKeys makes sure no foreign keys exist on the migrated table
 func (this *Inspector) validateTableForeignKeys(allowChildForeignKeys bool) error {
 	if this.migrationContext.SkipForeignKeyChecks {
-		log.Warning("--skip-foreign-key-checks provided: will not check for foreign keys")
+		this.migrationContext.Log.Warning("--skip-foreign-key-checks provided: will not check for foreign keys")
 		return nil
 	}
 	query := `
@@ -457,16 +458,16 @@ func (this *Inspector) validateTableForeignKeys(allowChildForeignKeys bool) erro
 		return err
 	}
 	if numParentForeignKeys > 0 {
-		return log.Errorf("Found %d parent-side foreign keys on %s.%s. Parent-side foreign keys are not supported. Bailing out", numParentForeignKeys, sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName))
+		return this.migrationContext.Log.Errorf("Found %d parent-side foreign keys on %s.%s. Parent-side foreign keys are not supported. Bailing out", numParentForeignKeys, sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName))
 	}
 	if numChildForeignKeys > 0 {
 		if allowChildForeignKeys {
-			log.Debugf("Foreign keys found and will be dropped, as per given --discard-foreign-keys flag")
+			this.migrationContext.Log.Debugf("Foreign keys found and will be dropped, as per given --discard-foreign-keys flag")
 			return nil
 		}
-		return log.Errorf("Found %d child-side foreign keys on %s.%s. Child-side foreign keys are not supported. Bailing out", numChildForeignKeys, sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName))
+		return this.migrationContext.Log.Errorf("Found %d child-side foreign keys on %s.%s. Child-side foreign keys are not supported. Bailing out", numChildForeignKeys, sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName))
 	}
-	log.Debugf("Validated no foreign keys exist on table")
+	this.migrationContext.Log.Debugf("Validated no foreign keys exist on table")
 	return nil
 }
 
@@ -492,9 +493,9 @@ func (this *Inspector) validateTableTriggers() error {
 		return err
 	}
 	if numTriggers > 0 {
-		return log.Errorf("Found triggers on %s.%s. Triggers are not supported at this time. Bailing out", sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName))
+		return this.migrationContext.Log.Errorf("Found triggers on %s.%s. Triggers are not supported at this time. Bailing out", sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName))
 	}
-	log.Debugf("Validated no triggers exist on table")
+	this.migrationContext.Log.Debugf("Validated no triggers exist on table")
 	return nil
 }
 
@@ -514,9 +515,9 @@ func (this *Inspector) estimateTableRowsViaExplain() error {
 		return err
 	}
 	if !outputFound {
-		return log.Errorf("Cannot run EXPLAIN on %s.%s!", sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName))
+		return this.migrationContext.Log.Errorf("Cannot run EXPLAIN on %s.%s!", sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName))
 	}
-	log.Infof("Estimated number of rows via EXPLAIN: %d", this.migrationContext.RowsEstimate)
+	this.migrationContext.Log.Infof("Estimated number of rows via EXPLAIN: %d", this.migrationContext.RowsEstimate)
 	return nil
 }
 
@@ -525,9 +526,9 @@ func (this *Inspector) CountTableRows() error {
 	atomic.StoreInt64(&this.migrationContext.CountingRowsFlag, 1)
 	defer atomic.StoreInt64(&this.migrationContext.CountingRowsFlag, 0)
 
-	log.Infof("As instructed, I'm issuing a SELECT COUNT(*) on the table. This may take a while")
+	this.migrationContext.Log.Infof("As instructed, I'm issuing a SELECT COUNT(*) on the table. This may take a while")
 
-	query := fmt.Sprintf(`select /* gh-ost */ count(*) as rows from %s.%s`, sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName))
+	query := fmt.Sprintf(`select /* gh-ost */ count(*) as count_rows from %s.%s`, sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName))
 	var rowsEstimate int64
 	if err := this.db.QueryRow(query).Scan(&rowsEstimate); err != nil {
 		return err
@@ -535,7 +536,7 @@ func (this *Inspector) CountTableRows() error {
 	atomic.StoreInt64(&this.migrationContext.RowsEstimate, rowsEstimate)
 	this.migrationContext.UsedRowsEstimateMethod = base.CountRowsEstimate
 
-	log.Infof("Exact number of rows via COUNT: %d", rowsEstimate)
+	this.migrationContext.Log.Infof("Exact number of rows via COUNT: %d", rowsEstimate)
 
 	return nil
 }
@@ -554,6 +555,7 @@ func (this *Inspector) applyColumnTypes(databaseName, tableName string, columnsL
 	err := sqlutils.QueryRowsMap(this.db, query, func(m sqlutils.RowMap) error {
 		columnName := m.GetString("COLUMN_NAME")
 		columnType := m.GetString("COLUMN_TYPE")
+		columnOctetLength := m.GetUint("CHARACTER_OCTET_LENGTH")
 		for _, columnsList := range columnsLists {
 			column := columnsList.GetColumn(columnName)
 			if column == nil {
@@ -581,6 +583,10 @@ func (this *Inspector) applyColumnTypes(databaseName, tableName string, columnsL
 			if strings.HasPrefix(columnType, "enum") {
 				column.Type = sql.EnumColumnType
 			}
+			if strings.HasPrefix(columnType, "binary") {
+				column.Type = sql.BinaryColumnType
+				column.BinaryOctetLength = columnOctetLength
+			}
 			if charset := m.GetString("CHARACTER_SET_NAME"); charset != "" {
 				column.Charset = charset
 			}
@@ -663,7 +669,7 @@ func (this *Inspector) getCandidateUniqueKeys(tableName string) (uniqueKeys [](*
 	if err != nil {
 		return uniqueKeys, err
 	}
-	log.Debugf("Potential unique keys in %+v: %+v", tableName, uniqueKeys)
+	this.migrationContext.Log.Debugf("Potential unique keys in %+v: %+v", tableName, uniqueKeys)
 	return uniqueKeys, nil
 }
 
@@ -753,7 +759,7 @@ func (this *Inspector) readChangelogState(hint string) (string, error) {
 }
 
 func (this *Inspector) getMasterConnectionConfig() (applierConfig *mysql.ConnectionConfig, err error) {
-	log.Infof("Recursively searching for replication master")
+	this.migrationContext.Log.Infof("Recursively searching for replication master")
 	visitedKeys := mysql.NewInstanceKeyMap()
 	return mysql.GetMasterConnectionConfigSafe(this.connectionConfig, visitedKeys, this.migrationContext.AllowedMasterMaster)
 }

go/logic/migrator.go

@@ -11,6 +11,7 @@ import (
 	"math"
 	"os"
 	"strings"
+	"sync"
 	"sync/atomic"
 	"time"
 
@@ -18,8 +19,6 @@ import (
 	"github.com/github/gh-ost/go/binlog"
 	"github.com/github/gh-ost/go/mysql"
 	"github.com/github/gh-ost/go/sql"
-
-	"github.com/outbrain/golib/log"
 )
 
 type ChangelogState string
@@ -62,7 +61,7 @@ const (
 
 // Migrator is the main schema migration flow manager.
 type Migrator struct {
-	parser           *sql.Parser
+	parser           *sql.AlterTableParser
 	inspector        *Inspector
 	applier          *Applier
 	eventsStreamer   *EventsStreamer
@@ -90,7 +89,7 @@ type Migrator struct {
 func NewMigrator(context *base.MigrationContext) *Migrator {
 	migrator := &Migrator{
 		migrationContext:           context,
-		parser:                     sql.NewParser(),
+		parser:                     sql.NewAlterTableParser(),
 		ghostTableMigrated:         make(chan bool),
 		firstThrottlingCollected:   make(chan bool, 3),
 		rowCopyComplete:            make(chan error),
@@ -208,15 +207,23 @@ func (this *Migrator) canStopStreaming() bool {
 	return atomic.LoadInt64(&this.migrationContext.CutOverCompleteFlag) != 0
 }
 
-// onChangelogStateEvent is called when a binlog event operation on the changelog table is intercepted.
+// onChangelogEvent is called when a binlog event operation on the changelog table is intercepted.
-func (this *Migrator) onChangelogStateEvent(dmlEvent *binlog.BinlogDMLEvent) (err error) {
+func (this *Migrator) onChangelogEvent(dmlEvent *binlog.BinlogDMLEvent) (err error) {
 	// Hey, I created the changelog table, I know the type of columns it has!
-	if hint := dmlEvent.NewColumnValues.StringColumn(2); hint != "state" {
+	switch hint := dmlEvent.NewColumnValues.StringColumn(2); hint {
+	case "state":
+		return this.onChangelogStateEvent(dmlEvent)
+	case "heartbeat":
+		return this.onChangelogHeartbeatEvent(dmlEvent)
+	default:
 		return nil
 	}
+}
+
+func (this *Migrator) onChangelogStateEvent(dmlEvent *binlog.BinlogDMLEvent) (err error) {
 	changelogStateString := dmlEvent.NewColumnValues.StringColumn(3)
 	changelogState := ReadChangelogState(changelogStateString)
-	log.Infof("Intercepted changelog state %s", changelogState)
+	this.migrationContext.Log.Infof("Intercepted changelog state %s", changelogState)
 	switch changelogState {
 	case GhostTableMigrated:
 		{
@@ -242,14 +249,26 @@ func (this *Migrator) onChangelogStateEvent(dmlEvent *binlog.BinlogDMLEvent) (er
 			return fmt.Errorf("Unknown changelog state: %+v", changelogState)
 		}
 	}
-	log.Infof("Handled changelog state %s", changelogState)
+	this.migrationContext.Log.Infof("Handled changelog state %s", changelogState)
 	return nil
 }
 
+func (this *Migrator) onChangelogHeartbeatEvent(dmlEvent *binlog.BinlogDMLEvent) (err error) {
+	changelogHeartbeatString := dmlEvent.NewColumnValues.StringColumn(3)
+
+	heartbeatTime, err := time.Parse(time.RFC3339Nano, changelogHeartbeatString)
+	if err != nil {
+		return this.migrationContext.Log.Errore(err)
+	} else {
+		this.migrationContext.SetLastHeartbeatOnChangelogTime(heartbeatTime)
+		return nil
+	}
+}
+
 // listenOnPanicAbort aborts on abort request
 func (this *Migrator) listenOnPanicAbort() {
 	err := <-this.migrationContext.PanicAbort
-	log.Fatale(err)
+	this.migrationContext.Log.Fatale(err)
 }
 
 // validateStatement validates the `alter` statement meets criteria.
@@ -265,7 +284,7 @@ func (this *Migrator) validateStatement() (err error) {
 		if !this.migrationContext.ApproveRenamedColumns {
 			return fmt.Errorf("gh-ost believes the ALTER statement renames columns, as follows: %v; as precaution, you are asked to confirm gh-ost is correct, and provide with `--approve-renamed-columns`, and we're all happy. Or you can skip renamed columns via `--skip-renamed-columns`, in which case column data may be lost", this.parser.GetNonTrivialRenames())
 		}
-		log.Infof("Alter statement has column(s) renamed. gh-ost finds the following renames: %v; --approve-renamed-columns is given and so migration proceeds.", this.parser.GetNonTrivialRenames())
+		this.migrationContext.Log.Infof("Alter statement has column(s) renamed. gh-ost finds the following renames: %v; --approve-renamed-columns is given and so migration proceeds.", this.parser.GetNonTrivialRenames())
 	}
 	this.migrationContext.DroppedColumnsMap = this.parser.DroppedColumnsMap()
 	return nil
@@ -277,7 +296,7 @@ func (this *Migrator) countTableRows() (err error) {
 		return nil
 	}
 	if this.migrationContext.Noop {
-		log.Debugf("Noop operation; not really counting table rows")
+		this.migrationContext.Log.Debugf("Noop operation; not really counting table rows")
 		return nil
 	}
 
@@ -292,7 +311,7 @@ func (this *Migrator) countTableRows() (err error) {
 	}
 
 	if this.migrationContext.ConcurrentCountTableRows {
-		log.Infof("As instructed, counting rows in the background; meanwhile I will use an estimated count, and will update it later on")
+		this.migrationContext.Log.Infof("As instructed, counting rows in the background; meanwhile I will use an estimated count, and will update it later on")
 		go countRowsFunc()
 		// and we ignore errors, because this turns to be a background job
 		return nil
@@ -304,9 +323,9 @@ func (this *Migrator) createFlagFiles() (err error) {
 	if this.migrationContext.PostponeCutOverFlagFile != "" {
 		if !base.FileExists(this.migrationContext.PostponeCutOverFlagFile) {
 			if err := base.TouchFile(this.migrationContext.PostponeCutOverFlagFile); err != nil {
-				return log.Errorf("--postpone-cut-over-flag-file indicated by gh-ost is unable to create said file: %s", err.Error())
+				return this.migrationContext.Log.Errorf("--postpone-cut-over-flag-file indicated by gh-ost is unable to create said file: %s", err.Error())
 			}
-			log.Infof("Created postpone-cut-over-flag-file: %s", this.migrationContext.PostponeCutOverFlagFile)
+			this.migrationContext.Log.Infof("Created postpone-cut-over-flag-file: %s", this.migrationContext.PostponeCutOverFlagFile)
 		}
 	}
 	return nil
@@ -314,7 +333,7 @@ func (this *Migrator) createFlagFiles() (err error) {
 
 // Migrate executes the complete migration logic. This is *the* major gh-ost function.
 func (this *Migrator) Migrate() (err error) {
-	log.Infof("Migrating %s.%s", sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName))
+	this.migrationContext.Log.Infof("Migrating %s.%s", sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName))
 	this.migrationContext.StartTime = time.Now()
 	if this.migrationContext.Hostname, err = os.Hostname(); err != nil {
 		return err
@@ -353,9 +372,9 @@ func (this *Migrator) Migrate() (err error) {
 	}
 
 	initialLag, _ := this.inspector.getReplicationLag()
-	log.Infof("Waiting for ghost table to be migrated. Current lag is %+v", initialLag)
+	this.migrationContext.Log.Infof("Waiting for ghost table to be migrated. Current lag is %+v", initialLag)
 	<-this.ghostTableMigrated
-	log.Debugf("ghost table migrated")
+	this.migrationContext.Log.Debugf("ghost table migrated")
 	// Yay! We now know the Ghost and Changelog tables are good to examine!
 	// When running on replica, this means the replica has those tables. When running
 	// on master this is always true, of course, and yet it also implies this knowledge
@@ -393,9 +412,9 @@ func (this *Migrator) Migrate() (err error) {
 	this.migrationContext.MarkRowCopyStartTime()
 	go this.initiateStatus()
 
-	log.Debugf("Operating until row copy is complete")
+	this.migrationContext.Log.Debugf("Operating until row copy is complete")
 	this.consumeRowCopyComplete()
-	log.Infof("Row copy complete")
+	this.migrationContext.Log.Infof("Row copy complete")
 	if err := this.hooksExecutor.onRowCopyComplete(); err != nil {
 		return err
 	}
@@ -421,7 +440,7 @@ func (this *Migrator) Migrate() (err error) {
 	if err := this.hooksExecutor.onSuccess(); err != nil {
 		return err
 	}
-	log.Infof("Done migrating %s.%s", sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName))
+	this.migrationContext.Log.Infof("Done migrating %s.%s", sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName))
 	return nil
 }
 
@@ -447,14 +466,14 @@ func (this *Migrator) handleCutOverResult(cutOverError error) (err error) {
 		// and swap the tables.
 		// The difference is that we will later swap the tables back.
 		if err := this.hooksExecutor.onStartReplication(); err != nil {
-			return log.Errore(err)
+			return this.migrationContext.Log.Errore(err)
 		}
 		if this.migrationContext.TestOnReplicaSkipReplicaStop {
-			log.Warningf("--test-on-replica-skip-replica-stop enabled, we are not starting replication.")
+			this.migrationContext.Log.Warningf("--test-on-replica-skip-replica-stop enabled, we are not starting replication.")
 		} else {
-			log.Debugf("testing on replica. Starting replication IO thread after cut-over failure")
+			this.migrationContext.Log.Debugf("testing on replica. Starting replication IO thread after cut-over failure")
 			if err := this.retryOperation(this.applier.StartReplication); err != nil {
-				return log.Errore(err)
+				return this.migrationContext.Log.Errore(err)
 			}
 		}
 	}
@@ -465,18 +484,25 @@ func (this *Migrator) handleCutOverResult(cutOverError error) (err error) {
 // type (on replica? atomic? safe?)
 func (this *Migrator) cutOver() (err error) {
 	if this.migrationContext.Noop {
-		log.Debugf("Noop operation; not really swapping tables")
+		this.migrationContext.Log.Debugf("Noop operation; not really swapping tables")
 		return nil
 	}
 	this.migrationContext.MarkPointOfInterest()
 	this.throttler.throttle(func() {
-		log.Debugf("throttling before swapping tables")
+		this.migrationContext.Log.Debugf("throttling before swapping tables")
 	})
 
 	this.migrationContext.MarkPointOfInterest()
-	log.Debugf("checking for cut-over postpone")
+	this.migrationContext.Log.Debugf("checking for cut-over postpone")
 	this.sleepWhileTrue(
 		func() (bool, error) {
+			heartbeatLag := this.migrationContext.TimeSinceLastHeartbeatOnChangelog()
+			maxLagMillisecondsThrottle := time.Duration(atomic.LoadInt64(&this.migrationContext.MaxLagMillisecondsThrottleThreshold)) * time.Millisecond
+			cutOverLockTimeout := time.Duration(this.migrationContext.CutOverLockTimeoutSeconds) * time.Second
+			if heartbeatLag > maxLagMillisecondsThrottle || heartbeatLag > cutOverLockTimeout {
+				this.migrationContext.Log.Debugf("current HeartbeatLag (%.2fs) is too high, it needs to be less than both --max-lag-millis (%.2fs) and --cut-over-lock-timeout-seconds (%.2fs) to continue", heartbeatLag.Seconds(), maxLagMillisecondsThrottle.Seconds(), cutOverLockTimeout.Seconds())
+				return true, nil
+			}
 			if this.migrationContext.PostponeCutOverFlagFile == "" {
 				return false, nil
 			}
@@ -499,7 +525,7 @@ func (this *Migrator) cutOver() (err error) {
 	)
 	atomic.StoreInt64(&this.migrationContext.IsPostponingCutOver, 0)
 	this.migrationContext.MarkPointOfInterest()
-	log.Debugf("checking for cut-over postpone: complete")
+	this.migrationContext.Log.Debugf("checking for cut-over postpone: complete")
 
 	if this.migrationContext.TestOnReplica {
 		// With `--test-on-replica` we stop replication thread, and then proceed to use
@@ -510,9 +536,9 @@ func (this *Migrator) cutOver() (err error) {
 			return err
 		}
 		if this.migrationContext.TestOnReplicaSkipReplicaStop {
-			log.Warningf("--test-on-replica-skip-replica-stop enabled, we are not stopping replication.")
+			this.migrationContext.Log.Warningf("--test-on-replica-skip-replica-stop enabled, we are not stopping replication.")
 		} else {
-			log.Debugf("testing on replica. Stopping replication IO thread")
+			this.migrationContext.Log.Debugf("testing on replica. Stopping replication IO thread")
 			if err := this.retryOperation(this.applier.StopReplication); err != nil {
 				return err
 			}
@@ -530,7 +556,7 @@ func (this *Migrator) cutOver() (err error) {
 		this.handleCutOverResult(err)
 		return err
 	}
-	return log.Fatalf("Unknown cut-over type: %d; should never get here!", this.migrationContext.CutOverType)
+	return this.migrationContext.Log.Fatalf("Unknown cut-over type: %d; should never get here!", this.migrationContext.CutOverType)
 }
 
 // Inject the "AllEventsUpToLockProcessed" state hint, wait for it to appear in the binary logs,
@@ -542,32 +568,32 @@ func (this *Migrator) waitForEventsUpToLock() (err error) {
 	waitForEventsUpToLockStartTime := time.Now()
 
 	allEventsUpToLockProcessedChallenge := fmt.Sprintf("%s:%d", string(AllEventsUpToLockProcessed), waitForEventsUpToLockStartTime.UnixNano())
-	log.Infof("Writing changelog state: %+v", allEventsUpToLockProcessedChallenge)
+	this.migrationContext.Log.Infof("Writing changelog state: %+v", allEventsUpToLockProcessedChallenge)
 	if _, err := this.applier.WriteChangelogState(allEventsUpToLockProcessedChallenge); err != nil {
 		return err
 	}
-	log.Infof("Waiting for events up to lock")
+	this.migrationContext.Log.Infof("Waiting for events up to lock")
 	atomic.StoreInt64(&this.migrationContext.AllEventsUpToLockProcessedInjectedFlag, 1)
 	for found := false; !found; {
 		select {
 		case <-timeout.C:
 			{
-				return log.Errorf("Timeout while waiting for events up to lock")
+				return this.migrationContext.Log.Errorf("Timeout while waiting for events up to lock")
 			}
 		case state := <-this.allEventsUpToLockProcessed:
 			{
 				if state == allEventsUpToLockProcessedChallenge {
-					log.Infof("Waiting for events up to lock: got %s", state)
+					this.migrationContext.Log.Infof("Waiting for events up to lock: got %s", state)
 					found = true
 				} else {
-					log.Infof("Waiting for events up to lock: skipping %s", state)
+					this.migrationContext.Log.Infof("Waiting for events up to lock: skipping %s", state)
 				}
 			}
 		}
 	}
 	waitForEventsUpToLockDuration := time.Since(waitForEventsUpToLockStartTime)
 
-	log.Infof("Done waiting for events up to lock; duration=%+v", waitForEventsUpToLockDuration)
+	this.migrationContext.Log.Infof("Done waiting for events up to lock; duration=%+v", waitForEventsUpToLockDuration)
 	this.printStatus(ForcePrintStatusAndHintRule)
 
 	return nil
@@ -598,7 +624,7 @@ func (this *Migrator) cutOverTwoStep() (err error) {
 
 	lockAndRenameDuration := this.migrationContext.RenameTablesEndTime.Sub(this.migrationContext.LockTablesStartTime)
 	renameDuration := this.migrationContext.RenameTablesEndTime.Sub(this.migrationContext.RenameTablesStartTime)
-	log.Debugf("Lock & rename duration: %s (rename only: %s). During this time, queries on %s were locked or failing", lockAndRenameDuration, renameDuration, sql.EscapeName(this.migrationContext.OriginalTableName))
+	this.migrationContext.Log.Debugf("Lock & rename duration: %s (rename only: %s). During this time, queries on %s were locked or failing", lockAndRenameDuration, renameDuration, sql.EscapeName(this.migrationContext.OriginalTableName))
 	return nil
 }
 
@@ -608,9 +634,12 @@ func (this *Migrator) atomicCutOver() (err error) {
 	defer atomic.StoreInt64(&this.migrationContext.InCutOverCriticalSectionFlag, 0)
 
 	okToUnlockTable := make(chan bool, 4)
+	var dropCutOverSentryTableOnce sync.Once
 	defer func() {
 		okToUnlockTable <- true
-		this.applier.DropAtomicCutOverSentryTableIfExists()
+		dropCutOverSentryTableOnce.Do(func() {
+			this.applier.DropAtomicCutOverSentryTableIfExists()
+		})
 	}()
 
 	atomic.StoreInt64(&this.migrationContext.AllEventsUpToLockProcessedInjectedFlag, 0)
@@ -619,19 +648,19 @@ func (this *Migrator) atomicCutOver() (err error) {
 	tableLocked := make(chan error, 2)
 	tableUnlocked := make(chan error, 2)
 	go func() {
-		if err := this.applier.AtomicCutOverMagicLock(lockOriginalSessionIdChan, tableLocked, okToUnlockTable, tableUnlocked); err != nil {
+		if err := this.applier.AtomicCutOverMagicLock(lockOriginalSessionIdChan, tableLocked, okToUnlockTable, tableUnlocked, &dropCutOverSentryTableOnce); err != nil {
-			log.Errore(err)
+			this.migrationContext.Log.Errore(err)
 		}
 	}()
 	if err := <-tableLocked; err != nil {
-		return log.Errore(err)
+		return this.migrationContext.Log.Errore(err)
 	}
 	lockOriginalSessionId := <-lockOriginalSessionIdChan
-	log.Infof("Session locking original & magic tables is %+v", lockOriginalSessionId)
+	this.migrationContext.Log.Infof("Session locking original & magic tables is %+v", lockOriginalSessionId)
 	// At this point we know the original table is locked.
 	// We know any newly incoming DML on original table is blocked.
 	if err := this.waitForEventsUpToLock(); err != nil {
-		return log.Errore(err)
+		return this.migrationContext.Log.Errore(err)
 	}
 
 	// Step 2
@@ -649,7 +678,7 @@ func (this *Migrator) atomicCutOver() (err error) {
 		}
 	}()
 	renameSessionId := <-renameSessionIdChan
-	log.Infof("Session renaming tables is %+v", renameSessionId)
+	this.migrationContext.Log.Infof("Session renaming tables is %+v", renameSessionId)
 
 	waitForRename := func() error {
 		if atomic.LoadInt64(&tableRenameKnownToHaveFailed) == 1 {
@@ -666,13 +695,13 @@ func (this *Migrator) atomicCutOver() (err error) {
 		return err
 	}
 	if atomic.LoadInt64(&tableRenameKnownToHaveFailed) == 0 {
-		log.Infof("Found atomic RENAME to be blocking, as expected. Double checking the lock is still in place (though I don't strictly have to)")
+		this.migrationContext.Log.Infof("Found atomic RENAME to be blocking, as expected. Double checking the lock is still in place (though I don't strictly have to)")
 	}
 	if err := this.applier.ExpectUsedLock(lockOriginalSessionId); err != nil {
 		// Abort operation. Just make sure to drop the magic table.
-		return log.Errore(err)
+		return this.migrationContext.Log.Errore(err)
 	}
-	log.Infof("Connection holding lock on original table still exists")
+	this.migrationContext.Log.Infof("Connection holding lock on original table still exists")
 
 	// Now that we've found the RENAME blocking, AND the locking connection still alive,
 	// we know it is safe to proceed to release the lock
@@ -681,16 +710,16 @@ func (this *Migrator) atomicCutOver() (err error) {
 	// BAM! magic table dropped, original table lock is released
 	// -> RENAME released -> queries on original are unblocked.
 	if err := <-tableUnlocked; err != nil {
-		return log.Errore(err)
+		return this.migrationContext.Log.Errore(err)
 	}
 	if err := <-tablesRenamed; err != nil {
-		return log.Errore(err)
+		return this.migrationContext.Log.Errore(err)
 	}
 	this.migrationContext.RenameTablesEndTime = time.Now()
 
 	// ooh nice! We're actually truly and thankfully done
 	lockAndRenameDuration := this.migrationContext.RenameTablesEndTime.Sub(this.migrationContext.LockTablesStartTime)
-	log.Infof("Lock & rename duration: %s. During this time, queries on %s were blocked", lockAndRenameDuration, sql.EscapeName(this.migrationContext.OriginalTableName))
+	this.migrationContext.Log.Infof("Lock & rename duration: %s. During this time, queries on %s were blocked", lockAndRenameDuration, sql.EscapeName(this.migrationContext.OriginalTableName))
 	return nil
 }
 
@@ -736,10 +765,10 @@ func (this *Migrator) initiateInspector() (err error) {
 		if this.migrationContext.ApplierConnectionConfig, err = this.inspector.getMasterConnectionConfig(); err != nil {
 			return err
 		}
-		log.Infof("Master found to be %+v", *this.migrationContext.ApplierConnectionConfig.ImpliedKey)
+		this.migrationContext.Log.Infof("Master found to be %+v", *this.migrationContext.ApplierConnectionConfig.ImpliedKey)
 	} else {
 		// Forced master host.
-		key, err := mysql.ParseRawInstanceKeyLoose(this.migrationContext.AssumeMasterHostname)
+		key, err := mysql.ParseInstanceKey(this.migrationContext.AssumeMasterHostname)
 		if err != nil {
 			return err
 		}
@@ -750,14 +779,14 @@ func (this *Migrator) initiateInspector() (err error) {
 		if this.migrationContext.CliMasterPassword != "" {
 			this.migrationContext.ApplierConnectionConfig.Password = this.migrationContext.CliMasterPassword
 		}
-		log.Infof("Master forced to be %+v", *this.migrationContext.ApplierConnectionConfig.ImpliedKey)
+		this.migrationContext.Log.Infof("Master forced to be %+v", *this.migrationContext.ApplierConnectionConfig.ImpliedKey)
 	}
 	// validate configs
 	if this.migrationContext.TestOnReplica || this.migrationContext.MigrateOnReplica {
 		if this.migrationContext.InspectorIsAlsoApplier() {
 			return fmt.Errorf("Instructed to --test-on-replica or --migrate-on-replica, but the server we connect to doesn't seem to be a replica")
 		}
-		log.Infof("--test-on-replica or --migrate-on-replica given. Will not execute on master %+v but rather on replica %+v itself",
+		this.migrationContext.Log.Infof("--test-on-replica or --migrate-on-replica given. Will not execute on master %+v but rather on replica %+v itself",
 			*this.migrationContext.ApplierConnectionConfig.ImpliedKey, *this.migrationContext.InspectorConnectionConfig.ImpliedKey,
 		)
 		this.migrationContext.ApplierConnectionConfig = this.migrationContext.InspectorConnectionConfig.Duplicate()
@@ -960,13 +989,14 @@ func (this *Migrator) printStatus(rule PrintStatusRule, writers ...io.Writer) {
 
 	currentBinlogCoordinates := *this.eventsStreamer.GetCurrentBinlogCoordinates()
 
-	status := fmt.Sprintf("Copy: %d/%d %.1f%%; Applied: %d; Backlog: %d/%d; Time: %+v(total), %+v(copy); streamer: %+v; Lag: %.2fs, State: %s; ETA: %s",
+	status := fmt.Sprintf("Copy: %d/%d %.1f%%; Applied: %d; Backlog: %d/%d; Time: %+v(total), %+v(copy); streamer: %+v; Lag: %.2fs, HeartbeatLag: %.2fs, State: %s; ETA: %s",
 		totalRowsCopied, rowsEstimate, progressPct,
 		atomic.LoadInt64(&this.migrationContext.TotalDMLEventsApplied),
 		len(this.applyEventsQueue), cap(this.applyEventsQueue),
 		base.PrettifyDurationOutput(elapsedTime), base.PrettifyDurationOutput(this.migrationContext.ElapsedRowCopyTime()),
 		currentBinlogCoordinates,
 		this.migrationContext.GetCurrentLagDuration().Seconds(),
+		this.migrationContext.TimeSinceLastHeartbeatOnChangelog().Seconds(),
 		state,
 		eta,
 	)
@@ -993,17 +1023,17 @@ func (this *Migrator) initiateStreaming() error {
 		this.migrationContext.DatabaseName,
 		this.migrationContext.GetChangelogTableName(),
 		func(dmlEvent *binlog.BinlogDMLEvent) error {
-			return this.onChangelogStateEvent(dmlEvent)
+			return this.onChangelogEvent(dmlEvent)
 		},
 	)
 
 	go func() {
-		log.Debugf("Beginning streaming")
+		this.migrationContext.Log.Debugf("Beginning streaming")
 		err := this.eventsStreamer.StreamEvents(this.canStopStreaming)
 		if err != nil {
 			this.migrationContext.PanicAbort <- err
 		}
-		log.Debugf("Done streaming")
+		this.migrationContext.Log.Debugf("Done streaming")
 	}()
 
 	go func() {
@@ -1038,11 +1068,11 @@ func (this *Migrator) initiateThrottler() error {
 	this.throttler = NewThrottler(this.migrationContext, this.applier, this.inspector)
 
 	go this.throttler.initiateThrottlerCollection(this.firstThrottlingCollected)
-	log.Infof("Waiting for first throttle metrics to be collected")
+	this.migrationContext.Log.Infof("Waiting for first throttle metrics to be collected")
 	<-this.firstThrottlingCollected // replication lag
 	<-this.firstThrottlingCollected // HTTP status
 	<-this.firstThrottlingCollected // other, general metrics
-	log.Infof("First throttle metrics collected")
+	this.migrationContext.Log.Infof("First throttle metrics collected")
 	go this.throttler.initiateThrottlerChecks()
 
 	return nil
@@ -1057,16 +1087,16 @@ func (this *Migrator) initiateApplier() error {
 		return err
 	}
 	if err := this.applier.CreateChangelogTable(); err != nil {
-		log.Errorf("Unable to create changelog table, see further error details. Perhaps a previous migration failed without dropping the table? OR is there a running migration? Bailing out")
+		this.migrationContext.Log.Errorf("Unable to create changelog table, see further error details. Perhaps a previous migration failed without dropping the table? OR is there a running migration? Bailing out")
 		return err
 	}
 	if err := this.applier.CreateGhostTable(); err != nil {
-		log.Errorf("Unable to create ghost table, see further error details. Perhaps a previous migration failed without dropping the table? Bailing out")
+		this.migrationContext.Log.Errorf("Unable to create ghost table, see further error details. Perhaps a previous migration failed without dropping the table? Bailing out")
 		return err
 	}
 
 	if err := this.applier.AlterGhost(); err != nil {
-		log.Errorf("Unable to ALTER ghost table, see further error details. Bailing out")
+		this.migrationContext.Log.Errorf("Unable to ALTER ghost table, see further error details. Bailing out")
 		return err
 	}
 
@@ -1080,14 +1110,14 @@ func (this *Migrator) initiateApplier() error {
 func (this *Migrator) iterateChunks() error {
 	terminateRowIteration := func(err error) error {
 		this.rowCopyComplete <- err
-		return log.Errore(err)
+		return this.migrationContext.Log.Errore(err)
 	}
 	if this.migrationContext.Noop {
-		log.Debugf("Noop operation; not really copying data")
+		this.migrationContext.Log.Debugf("Noop operation; not really copying data")
 		return terminateRowIteration(nil)
 	}
 	if this.migrationContext.MigrationRangeMinValues == nil {
-		log.Debugf("No rows found in table. Rowcopy will be implicitly empty")
+		this.migrationContext.Log.Debugf("No rows found in table. Rowcopy will be implicitly empty")
 		return terminateRowIteration(nil)
 	}
 
@@ -1155,7 +1185,7 @@ func (this *Migrator) onApplyEventStruct(eventStruct *applyEventStruct) error {
 	handleNonDMLEventStruct := func(eventStruct *applyEventStruct) error {
 		if eventStruct.writeFunc != nil {
 			if err := this.retryOperation(*eventStruct.writeFunc); err != nil {
-				return log.Errore(err)
+				return this.migrationContext.Log.Errore(err)
 			}
 		}
 		return nil
@@ -1189,13 +1219,13 @@ func (this *Migrator) onApplyEventStruct(eventStruct *applyEventStruct) error {
 			return this.applier.ApplyDMLEventQueries(dmlEvents)
 		}
 		if err := this.retryOperation(applyEventFunc); err != nil {
-			return log.Errore(err)
+			return this.migrationContext.Log.Errore(err)
 		}
 		if nonDmlStructToApply != nil {
 			// We pulled DML events from the queue, and then we hit a non-DML event. Wait!
 			// We need to handle it!
 			if err := handleNonDMLEventStruct(nonDmlStructToApply); err != nil {
-				return log.Errore(err)
+				return this.migrationContext.Log.Errore(err)
 			}
 		}
 	}
@@ -1207,7 +1237,7 @@ func (this *Migrator) onApplyEventStruct(eventStruct *applyEventStruct) error {
 // Both event backlog and rowcopy events are polled; the backlog events have precedence.
 func (this *Migrator) executeWriteFuncs() error {
 	if this.migrationContext.Noop {
-		log.Debugf("Noop operation; not really executing write funcs")
+		this.migrationContext.Log.Debugf("Noop operation; not really executing write funcs")
 		return nil
 	}
 	for {
@@ -1234,7 +1264,7 @@ func (this *Migrator) executeWriteFuncs() error {
 						copyRowsStartTime := time.Now()
 						// Retries are handled within the copyRowsFunc
 						if err := copyRowsFunc(); err != nil {
-							return log.Errore(err)
+							return this.migrationContext.Log.Errore(err)
 						}
 						if niceRatio := this.migrationContext.GetNiceRatio(); niceRatio > 0 {
 							copyRowsDuration := time.Since(copyRowsStartTime)
@@ -1247,7 +1277,7 @@ func (this *Migrator) executeWriteFuncs() error {
 					{
 						// Hmmmmm... nothing in the queue; no events, but also no row copy.
 						// This is possible upon load. Let's just sleep it over.
-						log.Debugf("Getting nothing in the write queue. Sleeping...")
+						this.migrationContext.Log.Debugf("Getting nothing in the write queue. Sleeping...")
 						time.Sleep(time.Second)
 					}
 				}
@@ -1263,14 +1293,14 @@ func (this *Migrator) finalCleanup() error {
 
 	if this.migrationContext.Noop {
 		if createTableStatement, err := this.inspector.showCreateTable(this.migrationContext.GetGhostTableName()); err == nil {
-			log.Infof("New table structure follows")
+			this.migrationContext.Log.Infof("New table structure follows")
 			fmt.Println(createTableStatement)
 		} else {
-			log.Errore(err)
+			this.migrationContext.Log.Errore(err)
 		}
 	}
 	if err := this.eventsStreamer.Close(); err != nil {
-		log.Errore(err)
+		this.migrationContext.Log.Errore(err)
 	}
 
 	if err := this.retryOperation(this.applier.DropChangelogTable); err != nil {
@@ -1282,8 +1312,8 @@ func (this *Migrator) finalCleanup() error {
 		}
 	} else {
 		if !this.migrationContext.Noop {
-			log.Infof("Am not dropping old table because I want this operation to be as live as possible. If you insist I should do it, please add `--ok-to-drop-table` next time. But I prefer you do not. To drop the old table, issue:")
+			this.migrationContext.Log.Infof("Am not dropping old table because I want this operation to be as live as possible. If you insist I should do it, please add `--ok-to-drop-table` next time. But I prefer you do not. To drop the old table, issue:")
-			log.Infof("-- drop table %s.%s", sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.GetOldTableName()))
+			this.migrationContext.Log.Infof("-- drop table %s.%s", sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.GetOldTableName()))
 		}
 	}
 	if this.migrationContext.Noop {
@@ -1299,22 +1329,22 @@ func (this *Migrator) teardown() {
 	atomic.StoreInt64(&this.finishedMigrating, 1)
 
 	if this.inspector != nil {
-		log.Infof("Tearing down inspector")
+		this.migrationContext.Log.Infof("Tearing down inspector")
 		this.inspector.Teardown()
 	}
 
 	if this.applier != nil {
-		log.Infof("Tearing down applier")
+		this.migrationContext.Log.Infof("Tearing down applier")
 		this.applier.Teardown()
 	}
 
 	if this.eventsStreamer != nil {
-		log.Infof("Tearing down streamer")
+		this.migrationContext.Log.Infof("Tearing down streamer")
 		this.eventsStreamer.Teardown()
 	}
 
 	if this.throttler != nil {
-		log.Infof("Tearing down throttler")
+		this.migrationContext.Log.Infof("Tearing down throttler")
 		this.throttler.Teardown()
 	}
 }

go/logic/server.go

@@ -1,5 +1,5 @@
 /*
-   Copyright 2016 GitHub Inc.
+   Copyright 2021 GitHub Inc.
 	 See https://github.com/github/gh-ost/blob/master/LICENSE
 */
 
@@ -16,7 +16,6 @@ import (
 	"sync/atomic"
 
 	"github.com/github/gh-ost/go/base"
-	"github.com/outbrain/golib/log"
 )
 
 type printStatusFunc func(PrintStatusRule, io.Writer)
@@ -49,12 +48,12 @@ func (this *Server) BindSocketFile() (err error) {
 	if err != nil {
 		return err
 	}
-	log.Infof("Listening on unix socket file: %s", this.migrationContext.ServeSocketFile)
+	this.migrationContext.Log.Infof("Listening on unix socket file: %s", this.migrationContext.ServeSocketFile)
 	return nil
 }
 
 func (this *Server) RemoveSocketFile() (err error) {
-	log.Infof("Removing socket file: %s", this.migrationContext.ServeSocketFile)
+	this.migrationContext.Log.Infof("Removing socket file: %s", this.migrationContext.ServeSocketFile)
 	return os.Remove(this.migrationContext.ServeSocketFile)
 }
 
@@ -66,7 +65,7 @@ func (this *Server) BindTCPPort() (err error) {
 	if err != nil {
 		return err
 	}
-	log.Infof("Listening on tcp port: %d", this.migrationContext.ServeTCPPort)
+	this.migrationContext.Log.Infof("Listening on tcp port: %d", this.migrationContext.ServeTCPPort)
 	return nil
 }
 
@@ -76,7 +75,7 @@ func (this *Server) Serve() (err error) {
 		for {
 			conn, err := this.unixListener.Accept()
 			if err != nil {
-				log.Errore(err)
+				this.migrationContext.Log.Errore(err)
 			}
 			go this.handleConnection(conn)
 		}
@@ -88,7 +87,7 @@ func (this *Server) Serve() (err error) {
 		for {
 			conn, err := this.tcpListener.Accept()
 			if err != nil {
-				log.Errore(err)
+				this.migrationContext.Log.Errore(err)
 			}
 			go this.handleConnection(conn)
 		}
@@ -118,7 +117,7 @@ func (this *Server) onServerCommand(command string, writer *bufio.Writer) (err e
 	} else {
 		fmt.Fprintf(writer, "%s\n", err.Error())
 	}
-	return log.Errore(err)
+	return this.migrationContext.Log.Errore(err)
 }
 
 // applyServerCommand parses and executes commands by user
@@ -147,7 +146,9 @@ func (this *Server) applyServerCommand(command string, writer *bufio.Writer) (pr
 			fmt.Fprint(writer, `available commands:
 status                               # Print a detailed status message
 sup                                  # Print a short status message
-coordinates													 # Print the currently inspected coordinates
+coordinates                          # Print the currently inspected coordinates
+applier                              # Print the hostname of the applier
+inspector                            # Print the hostname of the inspector
 chunk-size=<newsize>                 # Set a new chunk-size
 dml-batch-size=<newsize>             # Set a new dml-batch-size
 nice-ratio=<ratio>                   # Set a new nice-ratio, immediate sleep after each row-copy operation, float (examples: 0 is aggressive, 0.7 adds 70% runtime, 1.0 doubles runtime, 2.0 triples runtime, ...)
@@ -178,6 +179,22 @@ help                                 # This message
 			}
 			return NoPrintStatusRule, fmt.Errorf("coordinates are read-only")
 		}
+	case "applier":
+		if this.migrationContext.ApplierConnectionConfig != nil && this.migrationContext.ApplierConnectionConfig.ImpliedKey != nil {
+			fmt.Fprintf(writer, "Host: %s, Version: %s\n",
+				this.migrationContext.ApplierConnectionConfig.ImpliedKey.String(),
+				this.migrationContext.ApplierMySQLVersion,
+			)
+		}
+		return NoPrintStatusRule, nil
+	case "inspector":
+		if this.migrationContext.InspectorConnectionConfig != nil && this.migrationContext.InspectorConnectionConfig.ImpliedKey != nil {
+			fmt.Fprintf(writer, "Host: %s, Version: %s\n",
+				this.migrationContext.InspectorConnectionConfig.ImpliedKey.String(),
+				this.migrationContext.InspectorMySQLVersion,
+			)
+		}
+		return NoPrintStatusRule, nil
 	case "chunk-size":
 		{
 			if argIsQuestion {

go/logic/streamer.go

@@ -16,7 +16,6 @@ import (
 	"github.com/github/gh-ost/go/binlog"
 	"github.com/github/gh-ost/go/mysql"
 
-	"github.com/outbrain/golib/log"
 	"github.com/outbrain/golib/sqlutils"
 )
 
@@ -43,6 +42,7 @@ type EventsStreamer struct {
 	listenersMutex           *sync.Mutex
 	eventsChannel            chan *binlog.BinlogEntry
 	binlogReader             *binlog.GoMySQLReader
+	name                     string
 }
 
 func NewEventsStreamer(migrationContext *base.MigrationContext) *EventsStreamer {
@@ -52,6 +52,7 @@ func NewEventsStreamer(migrationContext *base.MigrationContext) *EventsStreamer
 		listeners:        [](*BinlogEventListener){},
 		listenersMutex:   &sync.Mutex{},
 		eventsChannel:    make(chan *binlog.BinlogEntry, EventsChannelBufferSize),
+		name:             "streamer",
 	}
 }
 
@@ -107,7 +108,7 @@ func (this *EventsStreamer) InitDBConnections() (err error) {
 	if this.db, _, err = mysql.GetDB(this.migrationContext.Uuid, EventsStreamerUri); err != nil {
 		return err
 	}
-	if _, err := base.ValidateConnection(this.db, this.connectionConfig, this.migrationContext); err != nil {
+	if _, err := base.ValidateConnection(this.db, this.connectionConfig, this.migrationContext, this.name); err != nil {
 		return err
 	}
 	if err := this.readCurrentBinlogCoordinates(); err != nil {
@@ -160,7 +161,7 @@ func (this *EventsStreamer) readCurrentBinlogCoordinates() error {
 	if !foundMasterStatus {
 		return fmt.Errorf("Got no results from SHOW MASTER STATUS. Bailing out")
 	}
-	log.Debugf("Streamer binlog coordinates: %+v", *this.initialBinlogCoordinates)
+	this.migrationContext.Log.Debugf("Streamer binlog coordinates: %+v", *this.initialBinlogCoordinates)
 	return nil
 }
 
@@ -186,7 +187,7 @@ func (this *EventsStreamer) StreamEvents(canStopStreaming func() bool) error {
 				return nil
 			}
 
-			log.Infof("StreamEvents encountered unexpected error: %+v", err)
+			this.migrationContext.Log.Infof("StreamEvents encountered unexpected error: %+v", err)
 			this.migrationContext.MarkPointOfInterest()
 			time.Sleep(ReconnectStreamerSleepSeconds * time.Second)
 
@@ -202,7 +203,7 @@ func (this *EventsStreamer) StreamEvents(canStopStreaming func() bool) error {
 
 			// Reposition at same binlog file.
 			lastAppliedRowsEventHint = this.binlogReader.LastAppliedRowsEventHint
-			log.Infof("Reconnecting... Will resume at %+v", lastAppliedRowsEventHint)
+			this.migrationContext.Log.Infof("Reconnecting... Will resume at %+v", lastAppliedRowsEventHint)
 			if err := this.initBinlogReader(this.GetReconnectBinlogCoordinates()); err != nil {
 				return err
 			}
@@ -213,7 +214,7 @@ func (this *EventsStreamer) StreamEvents(canStopStreaming func() bool) error {
 
 func (this *EventsStreamer) Close() (err error) {
 	err = this.binlogReader.Close()
-	log.Infof("Closed streamer connection. err=%+v", err)
+	this.migrationContext.Log.Infof("Closed streamer connection. err=%+v", err)
 	return err
 }
 

go/logic/throttler.go

@@ -15,24 +15,25 @@ import (
 	"github.com/github/gh-ost/go/base"
 	"github.com/github/gh-ost/go/mysql"
 	"github.com/github/gh-ost/go/sql"
-	"github.com/outbrain/golib/log"
 )
 
 var (
-	httpStatusMessages map[int]string = map[int]string{
+	httpStatusMessages = map[int]string{
 		200: "OK",
 		404: "Not found",
 		417: "Expectation failed",
 		429: "Too many requests",
 		500: "Internal server error",
+		-1:  "Connection error",
 	}
 	// See https://github.com/github/freno/blob/master/doc/http.md
-	httpStatusFrenoMessages map[int]string = map[int]string{
+	httpStatusFrenoMessages = map[int]string{
 		200: "OK",
 		404: "freno: unknown metric",
 		417: "freno: access forbidden",
 		429: "freno: threshold exceeded",
 		500: "freno: internal error",
+		-1:  "freno: connection error",
 	}
 )
 
@@ -84,6 +85,7 @@ func (this *Throttler) shouldThrottle() (result bool, reason string, reasonHint
 	if statusCode != 0 && statusCode != http.StatusOK {
 		return true, this.throttleHttpMessage(int(statusCode)), base.NoThrottleReasonHint
 	}
+
 	// Replication lag throttle
 	maxLagMillisecondsThrottleThreshold := atomic.LoadInt64(&this.migrationContext.MaxLagMillisecondsThrottleThreshold)
 	lag := atomic.LoadInt64(&this.migrationContext.CurrentLag)
@@ -120,7 +122,7 @@ func parseChangelogHeartbeat(heartbeatValue string) (lag time.Duration, err erro
 // parseChangelogHeartbeat parses a string timestamp and deduces replication lag
 func (this *Throttler) parseChangelogHeartbeat(heartbeatValue string) (err error) {
 	if lag, err := parseChangelogHeartbeat(heartbeatValue); err != nil {
-		return log.Errore(err)
+		return this.migrationContext.Log.Errore(err)
 	} else {
 		atomic.StoreInt64(&this.migrationContext.CurrentLag, int64(lag))
 		return nil
@@ -142,13 +144,13 @@ func (this *Throttler) collectReplicationLag(firstThrottlingCollected chan<- boo
 			// This means we will always get a good heartbeat value.
 			// When running on replica, we should instead check the `SHOW SLAVE STATUS` output.
 			if lag, err := mysql.GetReplicationLagFromSlaveStatus(this.inspector.informationSchemaDb); err != nil {
-				return log.Errore(err)
+				return this.migrationContext.Log.Errore(err)
 			} else {
 				atomic.StoreInt64(&this.migrationContext.CurrentLag, int64(lag))
 			}
 		} else {
 			if heartbeatValue, err := this.inspector.readChangelogState("heartbeat"); err != nil {
-				return log.Errore(err)
+				return this.migrationContext.Log.Errore(err)
 			} else {
 				this.parseChangelogHeartbeat(heartbeatValue)
 			}
@@ -288,7 +290,14 @@ func (this *Throttler) collectThrottleHTTPStatus(firstThrottlingCollected chan<-
 		return false, nil
 	}
 
-	collectFunc()
+	_, err := collectFunc()
+	if err != nil {
+		// If not told to ignore errors, we'll throttle on HTTP connection issues
+		if !this.migrationContext.IgnoreHTTPErrors {
+			atomic.StoreInt64(&this.migrationContext.ThrottleHTTPStatusCode, int64(-1))
+		}
+	}
+
 	firstThrottlingCollected <- true
 
 	ticker := time.Tick(100 * time.Millisecond)
@@ -297,7 +306,15 @@ func (this *Throttler) collectThrottleHTTPStatus(firstThrottlingCollected chan<-
 			return
 		}
 
-		if sleep, _ := collectFunc(); sleep {
+		sleep, err := collectFunc()
+		if err != nil {
+			// If not told to ignore errors, we'll throttle on HTTP connection issues
+			if !this.migrationContext.IgnoreHTTPErrors {
+				atomic.StoreInt64(&this.migrationContext.ThrottleHTTPStatusCode, int64(-1))
+			}
+		}
+
+		if sleep {
 			time.Sleep(1 * time.Second)
 		}
 	}
@@ -330,7 +347,7 @@ func (this *Throttler) collectGeneralThrottleMetrics() error {
 		hibernateDuration := time.Duration(this.migrationContext.CriticalLoadHibernateSeconds) * time.Second
 		hibernateUntilTime := time.Now().Add(hibernateDuration)
 		atomic.StoreInt64(&this.migrationContext.HibernateUntil, hibernateUntilTime.UnixNano())
-		log.Errorf("critical-load met: %s=%d, >=%d. Will hibernate for the duration of %+v, until %+v", variableName, value, threshold, hibernateDuration, hibernateUntilTime)
+		this.migrationContext.Log.Errorf("critical-load met: %s=%d, >=%d. Will hibernate for the duration of %+v, until %+v", variableName, value, threshold, hibernateDuration, hibernateUntilTime)
 		go func() {
 			time.Sleep(hibernateDuration)
 			this.migrationContext.SetThrottleGeneralCheckResult(base.NewThrottleCheckResult(true, "leaving hibernation", base.LeavingHibernationThrottleReasonHint))
@@ -343,7 +360,7 @@ func (this *Throttler) collectGeneralThrottleMetrics() error {
 		this.migrationContext.PanicAbort <- fmt.Errorf("critical-load met: %s=%d, >=%d", variableName, value, threshold)
 	}
 	if criticalLoadMet && this.migrationContext.CriticalLoadIntervalMilliseconds > 0 {
-		log.Errorf("critical-load met once: %s=%d, >=%d. Will check again in %d millis", variableName, value, threshold, this.migrationContext.CriticalLoadIntervalMilliseconds)
+		this.migrationContext.Log.Errorf("critical-load met once: %s=%d, >=%d. Will check again in %d millis", variableName, value, threshold, this.migrationContext.CriticalLoadIntervalMilliseconds)
 		go func() {
 			timer := time.NewTimer(time.Millisecond * time.Duration(this.migrationContext.CriticalLoadIntervalMilliseconds))
 			<-timer.C
@@ -461,6 +478,6 @@ func (this *Throttler) throttle(onThrottled func()) {
 }
 
 func (this *Throttler) Teardown() {
-	log.Debugf("Tearing down...")
+	this.migrationContext.Log.Debugf("Tearing down...")
 	atomic.StoreInt64(&this.finishedMigrating, 1)
 }

go/mysql/connection.go

@@ -27,6 +27,7 @@ type ConnectionConfig struct {
 	Password   string
 	ImpliedKey *InstanceKey
 	tlsConfig  *tls.Config
+	Timeout    float64
 }
 
 func NewConnectionConfig() *ConnectionConfig {
@@ -44,6 +45,7 @@ func (this *ConnectionConfig) DuplicateCredentials(key InstanceKey) *ConnectionC
 		User:      this.User,
 		Password:  this.Password,
 		tlsConfig: this.tlsConfig,
+		Timeout:   this.Timeout,
 	}
 	config.ImpliedKey = &config.Key
 	return config
@@ -116,5 +118,5 @@ func (this *ConnectionConfig) GetDBUri(databaseName string) string {
 	if this.tlsConfig != nil {
 		tlsOption = TLS_CONFIG_KEY
 	}
-	return fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?interpolateParams=%t&autocommit=true&charset=utf8mb4,utf8,latin1&tls=%s", this.User, this.Password, hostname, this.Key.Port, databaseName, interpolateParams, tlsOption)
+	return fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?timeout=%fs&readTimeout=%fs&writeTimeout=%fs&interpolateParams=%t&autocommit=true&charset=utf8mb4,utf8,latin1&tls=%s", this.User, this.Password, hostname, this.Key.Port, databaseName, this.Timeout, this.Timeout, this.Timeout, interpolateParams, tlsOption)
 }

go/mysql/connection_test.go

@@ -69,7 +69,7 @@ func TestGetDBUri(t *testing.T) {
 	c.Password = "penguin"
 
 	uri := c.GetDBUri("test")
-	test.S(t).ExpectEquals(uri, "gromit:penguin@tcp(myhost:3306)/test?interpolateParams=true&autocommit=true&charset=utf8mb4,utf8,latin1&tls=false")
+	test.S(t).ExpectEquals(uri, "gromit:penguin@tcp(myhost:3306)/test?timeout=0.000000s&readTimeout=0.000000s&writeTimeout=0.000000s&interpolateParams=true&autocommit=true&charset=utf8mb4,utf8,latin1&tls=false")
 }
 
 func TestGetDBUriWithTLSSetup(t *testing.T) {
@@ -80,5 +80,5 @@ func TestGetDBUriWithTLSSetup(t *testing.T) {
 	c.tlsConfig = &tls.Config{}
 
 	uri := c.GetDBUri("test")
-	test.S(t).ExpectEquals(uri, "gromit:penguin@tcp(myhost:3306)/test?interpolateParams=true&autocommit=true&charset=utf8mb4,utf8,latin1&tls=ghost")
+	test.S(t).ExpectEquals(uri, "gromit:penguin@tcp(myhost:3306)/test?timeout=0.000000s&readTimeout=0.000000s&writeTimeout=0.000000s&interpolateParams=true&autocommit=true&charset=utf8mb4,utf8,latin1&tls=ghost")
 }

go/mysql/instance_key.go

@@ -7,6 +7,7 @@ package mysql
 
 import (
 	"fmt"
+	"regexp"
 	"strconv"
 	"strings"
 )
@@ -15,6 +16,13 @@ const (
 	DefaultInstancePort = 3306
 )
 
+var (
+	ipv4HostPortRegexp = regexp.MustCompile("^([^:]+):([0-9]+)$")
+	ipv4HostRegexp     = regexp.MustCompile("^([^:]+)$")
+	ipv6HostPortRegexp = regexp.MustCompile("^\\[([:0-9a-fA-F]+)\\]:([0-9]+)$") // e.g. [2001:db8:1f70::999:de8:7648:6e8]:3308
+	ipv6HostRegexp     = regexp.MustCompile("^([:0-9a-fA-F]+)$")                // e.g. 2001:db8:1f70::999:de8:7648:6e8
+)
+
 // InstanceKey is an instance indicator, identified by hostname and port
 type InstanceKey struct {
 	Hostname string
@@ -25,25 +33,35 @@ const detachHint = "//"
 
 // ParseInstanceKey will parse an InstanceKey from a string representation such as 127.0.0.1:3306
 func NewRawInstanceKey(hostPort string) (*InstanceKey, error) {
-	tokens := strings.SplitN(hostPort, ":", 2)
+	hostname := ""
-	if len(tokens) != 2 {
+	port := ""
-		return nil, fmt.Errorf("Cannot parse InstanceKey from %s. Expected format is host:port", hostPort)
+	if submatch := ipv4HostPortRegexp.FindStringSubmatch(hostPort); len(submatch) > 0 {
+		hostname = submatch[1]
+		port = submatch[2]
+	} else if submatch := ipv4HostRegexp.FindStringSubmatch(hostPort); len(submatch) > 0 {
+		hostname = submatch[1]
+	} else if submatch := ipv6HostPortRegexp.FindStringSubmatch(hostPort); len(submatch) > 0 {
+		hostname = submatch[1]
+		port = submatch[2]
+	} else if submatch := ipv6HostRegexp.FindStringSubmatch(hostPort); len(submatch) > 0 {
+		hostname = submatch[1]
+	} else {
+		return nil, fmt.Errorf("Cannot parse address: %s", hostPort)
 	}
-	instanceKey := &InstanceKey{Hostname: tokens[0]}
+	instanceKey := &InstanceKey{Hostname: hostname, Port: DefaultInstancePort}
-	var err error
+	if port != "" {
-	if instanceKey.Port, err = strconv.Atoi(tokens[1]); err != nil {
+		var err error
-		return instanceKey, fmt.Errorf("Invalid port: %s", tokens[1])
+		if instanceKey.Port, err = strconv.Atoi(port); err != nil {
+			return instanceKey, fmt.Errorf("Invalid port: %s", port)
+		}
 	}
 
 	return instanceKey, nil
 }
 
-// ParseRawInstanceKeyLoose will parse an InstanceKey from a string representation such as 127.0.0.1:3306.
+// ParseInstanceKey will parse an InstanceKey from a string representation such as 127.0.0.1:3306.
 // The port part is optional; there will be no name resolve
-func ParseRawInstanceKeyLoose(hostPort string) (*InstanceKey, error) {
+func ParseInstanceKey(hostPort string) (*InstanceKey, error) {
-	if !strings.Contains(hostPort, ":") {
-		return &InstanceKey{Hostname: hostPort, Port: DefaultInstancePort}, nil
-	}
 	return NewRawInstanceKey(hostPort)
 }
 

go/mysql/instance_key_map.go

@@ -92,7 +92,7 @@ func (this *InstanceKeyMap) ReadCommaDelimitedList(list string) error {
 	}
 	tokens := strings.Split(list, ",")
 	for _, token := range tokens {
-		key, err := ParseRawInstanceKeyLoose(token)
+		key, err := ParseInstanceKey(token)
 		if err != nil {
 			return err
 		}

go/mysql/instance_key_test.go

@@ -0,0 +1,74 @@
+/*
+   Copyright 2016 GitHub Inc.
+	 See https://github.com/github/gh-ost/blob/master/LICENSE
+*/
+
+package mysql
+
+import (
+	"testing"
+
+	"github.com/outbrain/golib/log"
+	test "github.com/outbrain/golib/tests"
+)
+
+func init() {
+	log.SetLevel(log.ERROR)
+}
+
+func TestParseInstanceKey(t *testing.T) {
+	{
+		key, err := ParseInstanceKey("myhost:1234")
+		test.S(t).ExpectNil(err)
+		test.S(t).ExpectEquals(key.Hostname, "myhost")
+		test.S(t).ExpectEquals(key.Port, 1234)
+	}
+	{
+		key, err := ParseInstanceKey("myhost")
+		test.S(t).ExpectNil(err)
+		test.S(t).ExpectEquals(key.Hostname, "myhost")
+		test.S(t).ExpectEquals(key.Port, 3306)
+	}
+	{
+		key, err := ParseInstanceKey("10.0.0.3:3307")
+		test.S(t).ExpectNil(err)
+		test.S(t).ExpectEquals(key.Hostname, "10.0.0.3")
+		test.S(t).ExpectEquals(key.Port, 3307)
+	}
+	{
+		key, err := ParseInstanceKey("10.0.0.3")
+		test.S(t).ExpectNil(err)
+		test.S(t).ExpectEquals(key.Hostname, "10.0.0.3")
+		test.S(t).ExpectEquals(key.Port, 3306)
+	}
+	{
+		key, err := ParseInstanceKey("[2001:db8:1f70::999:de8:7648:6e8]:3308")
+		test.S(t).ExpectNil(err)
+		test.S(t).ExpectEquals(key.Hostname, "2001:db8:1f70::999:de8:7648:6e8")
+		test.S(t).ExpectEquals(key.Port, 3308)
+	}
+	{
+		key, err := ParseInstanceKey("::1")
+		test.S(t).ExpectNil(err)
+		test.S(t).ExpectEquals(key.Hostname, "::1")
+		test.S(t).ExpectEquals(key.Port, 3306)
+	}
+	{
+		key, err := ParseInstanceKey("0:0:0:0:0:0:0:0")
+		test.S(t).ExpectNil(err)
+		test.S(t).ExpectEquals(key.Hostname, "0:0:0:0:0:0:0:0")
+		test.S(t).ExpectEquals(key.Port, 3306)
+	}
+	{
+		_, err := ParseInstanceKey("[2001:xxxx:1f70::999:de8:7648:6e8]:3308")
+		test.S(t).ExpectNotNil(err)
+	}
+	{
+		_, err := ParseInstanceKey("10.0.0.4:")
+		test.S(t).ExpectNotNil(err)
+	}
+	{
+		_, err := ParseInstanceKey("10.0.0.4:5.6.7")
+		test.S(t).ExpectNotNil(err)
+	}
+}

go/sql/builder.go

@@ -396,7 +396,7 @@ func BuildDMLDeleteQuery(databaseName, tableName string, tableColumns, uniqueKey
 	}
 	for _, column := range uniqueKeyColumns.Columns() {
 		tableOrdinal := tableColumns.Ordinals[column.Name]
-		arg := column.convertArg(args[tableOrdinal])
+		arg := column.convertArg(args[tableOrdinal], true)
 		uniqueKeyArgs = append(uniqueKeyArgs, arg)
 	}
 	databaseName = EscapeName(databaseName)
@@ -433,7 +433,7 @@ func BuildDMLInsertQuery(databaseName, tableName string, tableColumns, sharedCol
 
 	for _, column := range sharedColumns.Columns() {
 		tableOrdinal := tableColumns.Ordinals[column.Name]
-		arg := column.convertArg(args[tableOrdinal])
+		arg := column.convertArg(args[tableOrdinal], false)
 		sharedArgs = append(sharedArgs, arg)
 	}
 
@@ -481,17 +481,20 @@ func BuildDMLUpdateQuery(databaseName, tableName string, tableColumns, sharedCol
 
 	for _, column := range sharedColumns.Columns() {
 		tableOrdinal := tableColumns.Ordinals[column.Name]
-		arg := column.convertArg(valueArgs[tableOrdinal])
+		arg := column.convertArg(valueArgs[tableOrdinal], false)
 		sharedArgs = append(sharedArgs, arg)
 	}
 
 	for _, column := range uniqueKeyColumns.Columns() {
 		tableOrdinal := tableColumns.Ordinals[column.Name]
-		arg := column.convertArg(whereArgs[tableOrdinal])
+		arg := column.convertArg(whereArgs[tableOrdinal], true)
 		uniqueKeyArgs = append(uniqueKeyArgs, arg)
 	}
 
 	setClause, err := BuildSetPreparedClause(mappedSharedColumns)
+	if err != nil {
+		return "", sharedArgs, uniqueKeyArgs, err
+	}
 
 	equalsComparison, err := BuildEqualsPreparedComparison(uniqueKeyColumns.Names())
 	result = fmt.Sprintf(`

go/sql/parser.go

@@ -12,26 +12,54 @@ import (
 )
 
 var (
-	sanitizeQuotesRegexp = regexp.MustCompile("('[^']*')")
+	sanitizeQuotesRegexp                 = regexp.MustCompile("('[^']*')")
-	renameColumnRegexp   = regexp.MustCompile(`(?i)\bchange\s+(column\s+|)([\S]+)\s+([\S]+)\s+`)
+	renameColumnRegexp                   = regexp.MustCompile(`(?i)\bchange\s+(column\s+|)([\S]+)\s+([\S]+)\s+`)
-	dropColumnRegexp     = regexp.MustCompile(`(?i)\bdrop\s+(column\s+|)([\S]+)$`)
+	dropColumnRegexp                     = regexp.MustCompile(`(?i)\bdrop\s+(column\s+|)([\S]+)$`)
-	renameTableRegexp    = regexp.MustCompile(`(?i)\brename\s+(to|as)\s+`)
+	renameTableRegexp                    = regexp.MustCompile(`(?i)\brename\s+(to|as)\s+`)
+	alterTableExplicitSchemaTableRegexps = []*regexp.Regexp{
+		// ALTER TABLE `scm`.`tbl` something
+		regexp.MustCompile(`(?i)\balter\s+table\s+` + "`" + `([^` + "`" + `]+)` + "`" + `[.]` + "`" + `([^` + "`" + `]+)` + "`" + `\s+(.*$)`),
+		// ALTER TABLE `scm`.tbl something
+		regexp.MustCompile(`(?i)\balter\s+table\s+` + "`" + `([^` + "`" + `]+)` + "`" + `[.]([\S]+)\s+(.*$)`),
+		// ALTER TABLE scm.`tbl` something
+		regexp.MustCompile(`(?i)\balter\s+table\s+([\S]+)[.]` + "`" + `([^` + "`" + `]+)` + "`" + `\s+(.*$)`),
+		// ALTER TABLE scm.tbl something
+		regexp.MustCompile(`(?i)\balter\s+table\s+([\S]+)[.]([\S]+)\s+(.*$)`),
+	}
+	alterTableExplicitTableRegexps = []*regexp.Regexp{
+		// ALTER TABLE `tbl` something
+		regexp.MustCompile(`(?i)\balter\s+table\s+` + "`" + `([^` + "`" + `]+)` + "`" + `\s+(.*$)`),
+		// ALTER TABLE tbl something
+		regexp.MustCompile(`(?i)\balter\s+table\s+([\S]+)\s+(.*$)`),
+	}
 )
 
-type Parser struct {
+type AlterTableParser struct {
 	columnRenameMap map[string]string
 	droppedColumns  map[string]bool
 	isRenameTable   bool
+
+	alterStatementOptions string
+	alterTokens           []string
+
+	explicitSchema string
+	explicitTable  string
 }
 
-func NewParser() *Parser {
+func NewAlterTableParser() *AlterTableParser {
-	return &Parser{
+	return &AlterTableParser{
 		columnRenameMap: make(map[string]string),
 		droppedColumns:  make(map[string]bool),
 	}
 }
 
-func (this *Parser) tokenizeAlterStatement(alterStatement string) (tokens []string, err error) {
+func NewParserFromAlterStatement(alterStatement string) *AlterTableParser {
+	parser := NewAlterTableParser()
+	parser.ParseAlterStatement(alterStatement)
+	return parser
+}
+
+func (this *AlterTableParser) tokenizeAlterStatement(alterStatement string) (tokens []string, err error) {
 	terminatingQuote := rune(0)
 	f := func(c rune) bool {
 		switch {
@@ -58,13 +86,13 @@ func (this *Parser) tokenizeAlterStatement(alterStatement string) (tokens []stri
 	return tokens, nil
 }
 
-func (this *Parser) sanitizeQuotesFromAlterStatement(alterStatement string) (strippedStatement string) {
+func (this *AlterTableParser) sanitizeQuotesFromAlterStatement(alterStatement string) (strippedStatement string) {
 	strippedStatement = alterStatement
 	strippedStatement = sanitizeQuotesRegexp.ReplaceAllString(strippedStatement, "''")
 	return strippedStatement
 }
 
-func (this *Parser) parseAlterToken(alterToken string) (err error) {
+func (this *AlterTableParser) parseAlterToken(alterToken string) (err error) {
 	{
 		// rename
 		allStringSubmatch := renameColumnRegexp.FindAllStringSubmatch(alterToken, -1)
@@ -97,16 +125,34 @@ func (this *Parser) parseAlterToken(alterToken string) (err error) {
 	return nil
 }
 
-func (this *Parser) ParseAlterStatement(alterStatement string) (err error) {
+func (this *AlterTableParser) ParseAlterStatement(alterStatement string) (err error) {
-	alterTokens, _ := this.tokenizeAlterStatement(alterStatement)
+
+	this.alterStatementOptions = alterStatement
+	for _, alterTableRegexp := range alterTableExplicitSchemaTableRegexps {
+		if submatch := alterTableRegexp.FindStringSubmatch(this.alterStatementOptions); len(submatch) > 0 {
+			this.explicitSchema = submatch[1]
+			this.explicitTable = submatch[2]
+			this.alterStatementOptions = submatch[3]
+			break
+		}
+	}
+	for _, alterTableRegexp := range alterTableExplicitTableRegexps {
+		if submatch := alterTableRegexp.FindStringSubmatch(this.alterStatementOptions); len(submatch) > 0 {
+			this.explicitTable = submatch[1]
+			this.alterStatementOptions = submatch[2]
+			break
+		}
+	}
+	alterTokens, _ := this.tokenizeAlterStatement(this.alterStatementOptions)
 	for _, alterToken := range alterTokens {
 		alterToken = this.sanitizeQuotesFromAlterStatement(alterToken)
 		this.parseAlterToken(alterToken)
+		this.alterTokens = append(this.alterTokens, alterToken)
 	}
 	return nil
 }
 
-func (this *Parser) GetNonTrivialRenames() map[string]string {
+func (this *AlterTableParser) GetNonTrivialRenames() map[string]string {
 	result := make(map[string]string)
 	for column, renamed := range this.columnRenameMap {
 		if column != renamed {
@@ -116,14 +162,33 @@ func (this *Parser) GetNonTrivialRenames() map[string]string {
 	return result
 }
 
-func (this *Parser) HasNonTrivialRenames() bool {
+func (this *AlterTableParser) HasNonTrivialRenames() bool {
 	return len(this.GetNonTrivialRenames()) > 0
 }
 
-func (this *Parser) DroppedColumnsMap() map[string]bool {
+func (this *AlterTableParser) DroppedColumnsMap() map[string]bool {
 	return this.droppedColumns
 }
 
-func (this *Parser) IsRenameTable() bool {
+func (this *AlterTableParser) IsRenameTable() bool {
 	return this.isRenameTable
 }
+func (this *AlterTableParser) GetExplicitSchema() string {
+	return this.explicitSchema
+}
+
+func (this *AlterTableParser) HasExplicitSchema() bool {
+	return this.GetExplicitSchema() != ""
+}
+
+func (this *AlterTableParser) GetExplicitTable() string {
+	return this.explicitTable
+}
+
+func (this *AlterTableParser) HasExplicitTable() bool {
+	return this.GetExplicitTable() != ""
+}
+
+func (this *AlterTableParser) GetAlterStatementOptions() string {
+	return this.alterStatementOptions
+}

go/sql/parser_test.go

@@ -19,17 +19,19 @@ func init() {
 
 func TestParseAlterStatement(t *testing.T) {
 	statement := "add column t int, engine=innodb"
-	parser := NewParser()
+	parser := NewAlterTableParser()
 	err := parser.ParseAlterStatement(statement)
 	test.S(t).ExpectNil(err)
+	test.S(t).ExpectEquals(parser.alterStatementOptions, statement)
 	test.S(t).ExpectFalse(parser.HasNonTrivialRenames())
 }
 
 func TestParseAlterStatementTrivialRename(t *testing.T) {
 	statement := "add column t int, change ts ts timestamp, engine=innodb"
-	parser := NewParser()
+	parser := NewAlterTableParser()
 	err := parser.ParseAlterStatement(statement)
 	test.S(t).ExpectNil(err)
+	test.S(t).ExpectEquals(parser.alterStatementOptions, statement)
 	test.S(t).ExpectFalse(parser.HasNonTrivialRenames())
 	test.S(t).ExpectEquals(len(parser.columnRenameMap), 1)
 	test.S(t).ExpectEquals(parser.columnRenameMap["ts"], "ts")
@@ -37,9 +39,10 @@ func TestParseAlterStatementTrivialRename(t *testing.T) {
 
 func TestParseAlterStatementTrivialRenames(t *testing.T) {
 	statement := "add column t int, change ts ts timestamp, CHANGE f `f` float, engine=innodb"
-	parser := NewParser()
+	parser := NewAlterTableParser()
 	err := parser.ParseAlterStatement(statement)
 	test.S(t).ExpectNil(err)
+	test.S(t).ExpectEquals(parser.alterStatementOptions, statement)
 	test.S(t).ExpectFalse(parser.HasNonTrivialRenames())
 	test.S(t).ExpectEquals(len(parser.columnRenameMap), 2)
 	test.S(t).ExpectEquals(parser.columnRenameMap["ts"], "ts")
@@ -58,9 +61,10 @@ func TestParseAlterStatementNonTrivial(t *testing.T) {
 	}
 
 	for _, statement := range statements {
-		parser := NewParser()
+		parser := NewAlterTableParser()
 		err := parser.ParseAlterStatement(statement)
 		test.S(t).ExpectNil(err)
+		test.S(t).ExpectEquals(parser.alterStatementOptions, statement)
 		renames := parser.GetNonTrivialRenames()
 		test.S(t).ExpectEquals(len(renames), 2)
 		test.S(t).ExpectEquals(renames["i"], "count")
@@ -69,7 +73,7 @@ func TestParseAlterStatementNonTrivial(t *testing.T) {
 }
 
 func TestTokenizeAlterStatement(t *testing.T) {
-	parser := NewParser()
+	parser := NewAlterTableParser()
 	{
 		alterStatement := "add column t int"
 		tokens, _ := parser.tokenizeAlterStatement(alterStatement)
@@ -108,7 +112,7 @@ func TestTokenizeAlterStatement(t *testing.T) {
 }
 
 func TestSanitizeQuotesFromAlterStatement(t *testing.T) {
-	parser := NewParser()
+	parser := NewAlterTableParser()
 	{
 		alterStatement := "add column e enum('a','b','c')"
 		strippedStatement := parser.sanitizeQuotesFromAlterStatement(alterStatement)
@@ -124,7 +128,7 @@ func TestSanitizeQuotesFromAlterStatement(t *testing.T) {
 func TestParseAlterStatementDroppedColumns(t *testing.T) {
 
 	{
-		parser := NewParser()
+		parser := NewAlterTableParser()
 		statement := "drop column b"
 		err := parser.ParseAlterStatement(statement)
 		test.S(t).ExpectNil(err)
@@ -132,16 +136,17 @@ func TestParseAlterStatementDroppedColumns(t *testing.T) {
 		test.S(t).ExpectTrue(parser.droppedColumns["b"])
 	}
 	{
-		parser := NewParser()
+		parser := NewAlterTableParser()
 		statement := "drop column b, drop key c_idx, drop column `d`"
 		err := parser.ParseAlterStatement(statement)
 		test.S(t).ExpectNil(err)
+		test.S(t).ExpectEquals(parser.alterStatementOptions, statement)
 		test.S(t).ExpectEquals(len(parser.droppedColumns), 2)
 		test.S(t).ExpectTrue(parser.droppedColumns["b"])
 		test.S(t).ExpectTrue(parser.droppedColumns["d"])
 	}
 	{
-		parser := NewParser()
+		parser := NewAlterTableParser()
 		statement := "drop column b, drop key c_idx, drop column `d`, drop `e`, drop primary key, drop foreign key fk_1"
 		err := parser.ParseAlterStatement(statement)
 		test.S(t).ExpectNil(err)
@@ -151,7 +156,7 @@ func TestParseAlterStatementDroppedColumns(t *testing.T) {
 		test.S(t).ExpectTrue(parser.droppedColumns["e"])
 	}
 	{
-		parser := NewParser()
+		parser := NewAlterTableParser()
 		statement := "drop column b, drop bad statement, add column i int"
 		err := parser.ParseAlterStatement(statement)
 		test.S(t).ExpectNil(err)
@@ -163,38 +168,133 @@ func TestParseAlterStatementDroppedColumns(t *testing.T) {
 func TestParseAlterStatementRenameTable(t *testing.T) {
 
 	{
-		parser := NewParser()
+		parser := NewAlterTableParser()
 		statement := "drop column b"
 		err := parser.ParseAlterStatement(statement)
 		test.S(t).ExpectNil(err)
 		test.S(t).ExpectFalse(parser.isRenameTable)
 	}
 	{
-		parser := NewParser()
+		parser := NewAlterTableParser()
 		statement := "rename as something_else"
 		err := parser.ParseAlterStatement(statement)
 		test.S(t).ExpectNil(err)
 		test.S(t).ExpectTrue(parser.isRenameTable)
 	}
 	{
-		parser := NewParser()
+		parser := NewAlterTableParser()
 		statement := "drop column b, rename as something_else"
 		err := parser.ParseAlterStatement(statement)
 		test.S(t).ExpectNil(err)
+		test.S(t).ExpectEquals(parser.alterStatementOptions, statement)
 		test.S(t).ExpectTrue(parser.isRenameTable)
 	}
 	{
-		parser := NewParser()
+		parser := NewAlterTableParser()
 		statement := "engine=innodb rename as something_else"
 		err := parser.ParseAlterStatement(statement)
 		test.S(t).ExpectNil(err)
 		test.S(t).ExpectTrue(parser.isRenameTable)
 	}
 	{
-		parser := NewParser()
+		parser := NewAlterTableParser()
 		statement := "rename as something_else, engine=innodb"
 		err := parser.ParseAlterStatement(statement)
 		test.S(t).ExpectNil(err)
 		test.S(t).ExpectTrue(parser.isRenameTable)
 	}
 }
+
+func TestParseAlterStatementExplicitTable(t *testing.T) {
+
+	{
+		parser := NewAlterTableParser()
+		statement := "drop column b"
+		err := parser.ParseAlterStatement(statement)
+		test.S(t).ExpectNil(err)
+		test.S(t).ExpectEquals(parser.explicitSchema, "")
+		test.S(t).ExpectEquals(parser.explicitTable, "")
+		test.S(t).ExpectEquals(parser.alterStatementOptions, "drop column b")
+		test.S(t).ExpectTrue(reflect.DeepEqual(parser.alterTokens, []string{"drop column b"}))
+	}
+	{
+		parser := NewAlterTableParser()
+		statement := "alter table tbl drop column b"
+		err := parser.ParseAlterStatement(statement)
+		test.S(t).ExpectNil(err)
+		test.S(t).ExpectEquals(parser.explicitSchema, "")
+		test.S(t).ExpectEquals(parser.explicitTable, "tbl")
+		test.S(t).ExpectEquals(parser.alterStatementOptions, "drop column b")
+		test.S(t).ExpectTrue(reflect.DeepEqual(parser.alterTokens, []string{"drop column b"}))
+	}
+	{
+		parser := NewAlterTableParser()
+		statement := "alter table `tbl` drop column b"
+		err := parser.ParseAlterStatement(statement)
+		test.S(t).ExpectNil(err)
+		test.S(t).ExpectEquals(parser.explicitSchema, "")
+		test.S(t).ExpectEquals(parser.explicitTable, "tbl")
+		test.S(t).ExpectEquals(parser.alterStatementOptions, "drop column b")
+		test.S(t).ExpectTrue(reflect.DeepEqual(parser.alterTokens, []string{"drop column b"}))
+	}
+	{
+		parser := NewAlterTableParser()
+		statement := "alter table `scm with spaces`.`tbl` drop column b"
+		err := parser.ParseAlterStatement(statement)
+		test.S(t).ExpectNil(err)
+		test.S(t).ExpectEquals(parser.explicitSchema, "scm with spaces")
+		test.S(t).ExpectEquals(parser.explicitTable, "tbl")
+		test.S(t).ExpectEquals(parser.alterStatementOptions, "drop column b")
+		test.S(t).ExpectTrue(reflect.DeepEqual(parser.alterTokens, []string{"drop column b"}))
+	}
+	{
+		parser := NewAlterTableParser()
+		statement := "alter table `scm`.`tbl with spaces` drop column b"
+		err := parser.ParseAlterStatement(statement)
+		test.S(t).ExpectNil(err)
+		test.S(t).ExpectEquals(parser.explicitSchema, "scm")
+		test.S(t).ExpectEquals(parser.explicitTable, "tbl with spaces")
+		test.S(t).ExpectEquals(parser.alterStatementOptions, "drop column b")
+		test.S(t).ExpectTrue(reflect.DeepEqual(parser.alterTokens, []string{"drop column b"}))
+	}
+	{
+		parser := NewAlterTableParser()
+		statement := "alter table `scm`.tbl drop column b"
+		err := parser.ParseAlterStatement(statement)
+		test.S(t).ExpectNil(err)
+		test.S(t).ExpectEquals(parser.explicitSchema, "scm")
+		test.S(t).ExpectEquals(parser.explicitTable, "tbl")
+		test.S(t).ExpectEquals(parser.alterStatementOptions, "drop column b")
+		test.S(t).ExpectTrue(reflect.DeepEqual(parser.alterTokens, []string{"drop column b"}))
+	}
+	{
+		parser := NewAlterTableParser()
+		statement := "alter table scm.`tbl` drop column b"
+		err := parser.ParseAlterStatement(statement)
+		test.S(t).ExpectNil(err)
+		test.S(t).ExpectEquals(parser.explicitSchema, "scm")
+		test.S(t).ExpectEquals(parser.explicitTable, "tbl")
+		test.S(t).ExpectEquals(parser.alterStatementOptions, "drop column b")
+		test.S(t).ExpectTrue(reflect.DeepEqual(parser.alterTokens, []string{"drop column b"}))
+	}
+	{
+		parser := NewAlterTableParser()
+		statement := "alter table scm.tbl drop column b"
+		err := parser.ParseAlterStatement(statement)
+		test.S(t).ExpectNil(err)
+		test.S(t).ExpectEquals(parser.explicitSchema, "scm")
+		test.S(t).ExpectEquals(parser.explicitTable, "tbl")
+		test.S(t).ExpectEquals(parser.alterStatementOptions, "drop column b")
+		test.S(t).ExpectTrue(reflect.DeepEqual(parser.alterTokens, []string{"drop column b"}))
+	}
+	{
+		parser := NewAlterTableParser()
+		statement := "alter table scm.tbl drop column b, add index idx(i)"
+		err := parser.ParseAlterStatement(statement)
+		test.S(t).ExpectNil(err)
+		test.S(t).ExpectEquals(parser.explicitSchema, "scm")
+		test.S(t).ExpectEquals(parser.explicitTable, "tbl")
+		test.S(t).ExpectEquals(parser.alterStatementOptions, "drop column b, add index idx(i)")
+		test.S(t).ExpectTrue(reflect.DeepEqual(parser.alterTokens, []string{"drop column b", "add index idx(i)"}))
+	}
+}

go/sql/types.go

@@ -6,6 +6,7 @@
 package sql
 
 import (
+	"bytes"
 	"fmt"
 	"reflect"
 	"strconv"
@@ -22,6 +23,7 @@ const (
 	MediumIntColumnType
 	JSONColumnType
 	FloatColumnType
+	BinaryColumnType
 )
 
 const maxMediumintUnsigned int32 = 16777215
@@ -31,19 +33,36 @@ type TimezoneConversion struct {
 }
 
 type Column struct {
-	Name               string
+	Name       string
-	IsUnsigned         bool
+	IsUnsigned bool
-	Charset            string
+	Charset    string
-	Type               ColumnType
+	Type       ColumnType
+
+	// add Octet length for binary type, fix bytes with suffix "00" get clipped in mysql binlog.
+	// https://github.com/github/gh-ost/issues/909
+	BinaryOctetLength  uint
 	timezoneConversion *TimezoneConversion
 }
 
-func (this *Column) convertArg(arg interface{}) interface{} {
+func (this *Column) convertArg(arg interface{}, isUniqueKeyColumn bool) interface{} {
 	if s, ok := arg.(string); ok {
 		// string, charset conversion
 		if encoding, ok := charsetEncodingMap[this.Charset]; ok {
 			arg, _ = encoding.NewDecoder().String(s)
 		}
+
+		if this.Type == BinaryColumnType && isUniqueKeyColumn {
+			arg2Bytes := []byte(arg.(string))
+			size := len(arg2Bytes)
+			if uint(size) < this.BinaryOctetLength {
+				buf := bytes.NewBuffer(arg2Bytes)
+				for i := uint(0); i < (this.BinaryOctetLength - uint(size)); i++ {
+					buf.Write([]byte{0})
+				}
+				arg = buf.String()
+			}
+		}
+
 		return arg
 	}
 

localtests/bigint-change-nullable/create.sql

@@ -0,0 +1,21 @@
+drop table if exists gh_ost_test;
+create table gh_ost_test (
+  id bigint auto_increment,
+  val bigint not null,
+  primary key(id)
+) auto_increment=1;
+
+drop event if exists gh_ost_test;
+delimiter ;;
+create event gh_ost_test
+  on schedule every 1 second
+  starts current_timestamp
+  ends current_timestamp + interval 60 second
+  on completion not preserve
+  enable
+  do
+begin
+  insert into gh_ost_test values (null, 18446744073709551615);
+  insert into gh_ost_test values (null, 18446744073709551614);
+  insert into gh_ost_test values (null, 18446744073709551613);
+end ;;

localtests/bigint-change-nullable/extra_args

@@ -0,0 +1 @@
+--alter="change val val bigint" 

localtests/latin1text/create.sql

@@ -0,0 +1,25 @@
+drop table if exists gh_ost_test;
+create table gh_ost_test (
+  id int auto_increment,
+  t text charset latin1 collate latin1_swedish_ci,
+  primary key(id)
+) auto_increment=1 charset latin1 collate latin1_swedish_ci;
+
+drop event if exists gh_ost_test;
+delimiter ;;
+create event gh_ost_test
+  on schedule every 1 second
+  starts current_timestamp
+  ends current_timestamp + interval 60 second
+  on completion not preserve
+  enable
+  do
+begin
+  insert into gh_ost_test values (null, md5(rand()));
+  insert into gh_ost_test values (null, 'átesting');
+  insert into gh_ost_test values (null, 'ádelete');
+  insert into gh_ost_test values (null, 'testátest');
+  update gh_ost_test set t='áupdated' order by id desc limit 1;
+  update gh_ost_test set t='áupdated1' where t='áupdated' order by id desc limit 1;
+  delete from gh_ost_test where t='ádelete';
+end ;;

localtests/varbinary/create.sql

@@ -0,0 +1,40 @@
+drop table if exists gh_ost_test;
+create table gh_ost_test (
+  id binary(16) NOT NULL,
+  info varchar(255) COLLATE utf8_unicode_ci NOT NULL,
+  data binary(8) NOT NULL,
+  primary key (id),
+  unique key info_uidx (info)
+) auto_increment=1;
+
+drop event if exists gh_ost_test;
+delimiter ;;
+create event gh_ost_test
+  on schedule every 1 second
+  starts current_timestamp
+  ends current_timestamp + interval 60 second
+  on completion not preserve
+  enable
+  do
+begin
+  replace into gh_ost_test (id, info, data) values (X'12ffffffffffffffffffffffffffff00', 'item 1a', X'12ffffffffffffff');
+  replace into gh_ost_test (id, info, data) values (X'34ffffffffffffffffffffffffffffff', 'item 3a', X'34ffffffffffffff');
+  replace into gh_ost_test (id, info, data) values (X'90ffffffffffffffffffffffffffffff', 'item 9a', X'90ffffffffffff00');
+
+  DELETE FROM gh_ost_test WHERE id = X'11ffffffffffffffffffffffffffff00';
+  UPDATE gh_ost_test SET info = 'item 2++' WHERE id = X'22ffffffffffffffffffffffffffff00';
+  UPDATE gh_ost_test SET info = 'item 3++', data = X'33ffffffffffff00' WHERE id = X'33ffffffffffffffffffffffffffffff';
+  DELETE FROM gh_ost_test WHERE id = X'44ffffffffffffffffffffffffffffff';
+  UPDATE gh_ost_test SET info = 'item 5++', data = X'55ffffffffffffee'  WHERE id = X'55ffffffffffffffffffffffffffffff';
+  INSERT INTO gh_ost_test (id, info, data) VALUES (X'66ffffffffffffffffffffffffffff00', 'item 6', X'66ffffffffffffff');
+  INSERT INTO gh_ost_test (id, info, data) VALUES (X'77ffffffffffffffffffffffffffffff', 'item 7', X'77ffffffffffff00');
+  INSERT INTO gh_ost_test (id, info, data) VALUES (X'88ffffffffffffffffffffffffffffff', 'item 8', X'88ffffffffffffff');
+end ;;
+
+INSERT INTO gh_ost_test (id, info, data) VALUES
+ (X'11ffffffffffffffffffffffffffff00', 'item 1', X'11ffffffffffffff'), -- id ends in 00
+ (X'22ffffffffffffffffffffffffffff00', 'item 2', X'22ffffffffffffff'), -- id ends in 00
+ (X'33ffffffffffffffffffffffffffffff', 'item 3', X'33ffffffffffffff'),
+ (X'44ffffffffffffffffffffffffffffff', 'item 4', X'44ffffffffffffff'),
+ (X'55ffffffffffffffffffffffffffffff', 'item 5', X'55ffffffffffffff'),
+ (X'99ffffffffffffffffffffffffffffff', 'item 9', X'99ffffffffffff00'); -- data ends in 00

script/build-deploy-tarball

@@ -28,8 +28,8 @@ gzip $tarball
 mkdir -p "$BUILD_ARTIFACT_DIR"/gh-ost
 cp ${tarball}.gz "$BUILD_ARTIFACT_DIR"/gh-ost/
 
-### HACK HACK HACK ###
+### HACK HACK HACK HACK ###
-# blame @carlosmn and @mattr-
+# blame @carlosmn, @mattr and @timvaillancourt-
-# Allow builds on stretch to also be used for jessie
+# Allow builds on buster to also be used for stretch
-jessie_tarball_name=$(echo $(basename "${tarball}") | sed s/-stretch-/-jessie-/)
+stretch_tarball_name=$(echo $(basename "${tarball}") | sed s/-buster-/-stretch-/)
-cp ${tarball}.gz "$BUILD_ARTIFACT_DIR/gh-ost/${jessie_tarball_name}.gz"
+cp ${tarball}.gz "$BUILD_ARTIFACT_DIR/gh-ost/${stretch_tarball_name}.gz"

script/dock

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+# Usage:
+# dock <test|packages> [arg]
+#   dock test:                build gh-ost & run unit and integration tests
+#   docker pkg [target-path]: build gh-ost release packages and copy to target path (default path: /tmp/gh-ost-release)
+
+command="$1"
+
+case "$command" in
+  "test")
+    docker_target="gh-ost-test"
+    docker build . -f Dockerfile.test -t "${docker_target}" && docker run --rm -it "${docker_target}:latest"
+    ;;
+  "pkg")
+    packages_path="${2:-/tmp/gh-ost-release}"
+    docker_target="gh-ost-packaging"
+    docker build . -f Dockerfile.packaging -t "${docker_target}" && docker run --rm -it -v "${packages_path}:/tmp/pkg" "${docker_target}:latest" bash -c 'find /tmp/gh-ost-release/ -maxdepth 1 -type f | xargs cp -t /tmp/pkg'
+    echo "packages generated on ${packages_path}:"
+    ls -l "${packages_path}"
+    ;;
+  *)
+    >&2 echo "Usage: dock dock <test|alpine|packages> [arg]"
+    exit 1
+esac

script/ensure-go-installed

@@ -1,13 +1,13 @@
 #!/bin/bash
 
-PREFERRED_GO_VERSION=go1.12.6
+PREFERRED_GO_VERSION=go1.14.7
-SUPPORTED_GO_VERSIONS='go1.1[234]'
+SUPPORTED_GO_VERSIONS='go1.1[456]'
 
 GO_PKG_DARWIN=${PREFERRED_GO_VERSION}.darwin-amd64.pkg
-GO_PKG_DARWIN_SHA=ea78245e43de2996fa0973033064b33f48820cfe39f4f3c6e953040925cc5815
+GO_PKG_DARWIN_SHA=0f215de06019a054a3da46a0722989986c956d719c7a0a8fc38a5f3c216d6f6b
 
 GO_PKG_LINUX=${PREFERRED_GO_VERSION}.linux-amd64.tar.gz
-GO_PKG_LINUX_SHA=dbcf71a3c1ea53b8d54ef1b48c85a39a6c9a935d01fc8291ff2b92028e59913c
+GO_PKG_LINUX_SHA=4a7fa60f323ee1416a4b1425aefc37ea359e9d64df19c326a58953a97ad41ea5
 
 export ROOTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )"
 cd $ROOTDIR

vendor/github.com/go-sql-driver/mysql/.travis.yml

@@ -1,10 +1,10 @@
 sudo: false
 language: go
 go:
-  - 1.9.x
   - 1.10.x
   - 1.11.x
   - 1.12.x
+  - 1.13.x
   - master
 
 before_install:
@@ -105,6 +105,7 @@ matrix:
         homebrew:
           packages:
             - mysql
+          update: true
       go: 1.12.x
       before_install:
         - go get golang.org/x/tools/cmd/cover

vendor/github.com/go-sql-driver/mysql/AUTHORS

@@ -13,6 +13,7 @@
 
 Aaron Hopkins <go-sql-driver at die.net>
 Achille Roussel <achille.roussel at gmail.com>
+Alex Snast <alexsn at fb.com>
 Alexey Palazhchenko <alexey.palazhchenko at gmail.com>
 Andrew Reid <andrew.reid at tixtrack.com>
 Arne Hormann <arnehormann at gmail.com>
@@ -44,6 +45,7 @@ James Harr <james.harr at gmail.com>
 Jeff Hodges <jeff at somethingsimilar.com>
 Jeffrey Charles <jeffreycharles at gmail.com>
 Jerome Meyer <jxmeyer at gmail.com>
+Jiajia Zhong <zhong2plus at gmail.com>
 Jian Zhen <zhenjl at gmail.com>
 Joshua Prunier <joshua.prunier at gmail.com>
 Julien Lefevre <julien.lefevr at gmail.com>
@@ -62,6 +64,7 @@ Lucas Liu <extrafliu at gmail.com>
 Luke Scott <luke at webconnex.com>
 Maciej Zimnoch <maciej.zimnoch at codilime.com>
 Michael Woolnough <michael.woolnough at gmail.com>
+Nathanial Murphy <nathanial.murphy at gmail.com>
 Nicola Peduzzi <thenikso at gmail.com>
 Olivier Mengué <dolmen at cpan.org>
 oscarzhao <oscarzhaosl at gmail.com>
@@ -81,6 +84,7 @@ Steven Hartland <steven.hartland at multiplay.co.uk>
 Thomas Wodarek <wodarekwebpage at gmail.com>
 Tim Ruffles <timruffles at gmail.com>
 Tom Jenkinson <tom at tjenkinson.me>
+Vladimir Kovpak <cn007b at gmail.com>
 Xiangyu Hu <xiangyu.hu at outlook.com>
 Xiaobing Jiang <s7v7nislands at gmail.com>
 Xiuming Chen <cc at cxm.cc>
@@ -90,6 +94,7 @@ Zhenye Xie <xiezhenye at gmail.com>
 
 Barracuda Networks, Inc.
 Counting Ltd.
+DigitalOcean Inc.
 Facebook Inc.
 GitHub Inc.
 Google Inc.

vendor/github.com/go-sql-driver/mysql/CHANGELOG.md

@@ -1,3 +1,42 @@
+## Version 1.5 (2020-01-07)
+
+Changes:
+
+  - Dropped support Go 1.9 and lower (#823, #829, #886, #1016, #1017)
+  - Improve buffer handling (#890)
+  - Document potentially insecure TLS configs (#901)
+  - Use a double-buffering scheme to prevent data races (#943)
+  - Pass uint64 values without converting them to string (#838, #955)
+  - Update collations and make utf8mb4 default (#877, #1054)
+  - Make NullTime compatible with sql.NullTime in Go 1.13+ (#995)
+  - Removed CloudSQL support (#993, #1007)
+  - Add Go Module support (#1003)
+
+New Features:
+
+  - Implement support of optional TLS (#900)
+  - Check connection liveness (#934, #964, #997, #1048, #1051, #1052)
+  - Implement Connector Interface (#941, #958, #1020, #1035)
+
+Bugfixes:
+
+  - Mark connections as bad on error during ping (#875)
+  - Mark connections as bad on error during dial (#867)
+  - Fix connection leak caused by rapid context cancellation (#1024)
+  - Mark connections as bad on error during Conn.Prepare (#1030)
+
+
+## Version 1.4.1 (2018-11-14)
+
+Bugfixes:
+
+ - Fix TIME format for binary columns (#818)
+ - Fix handling of empty auth plugin names (#835)
+ - Fix caching_sha2_password with empty password (#826)
+ - Fix canceled context broke mysqlConn (#862)
+ - Fix OldAuthSwitchRequest support (#870)
+ - Fix Auth Response packet for cleartext password (#887)
+
 ## Version 1.4 (2018-06-03)
 
 Changes:

vendor/github.com/go-sql-driver/mysql/README.md

@@ -40,7 +40,7 @@ A MySQL-Driver for Go's [database/sql](https://golang.org/pkg/database/sql/) pac
   * Optional placeholder interpolation
 
 ## Requirements
-  * Go 1.9 or higher. We aim to support the 3 latest versions of Go.
+  * Go 1.10 or higher. We aim to support the 3 latest versions of Go.
   * MySQL (4.1+), MariaDB, Percona Server, Google CloudSQL or Sphinx (2.2.3+)
 
 ---------------------------------------
@@ -166,6 +166,17 @@ Sets the charset used for client-server interaction (`"SET NAMES <value>"`). If
 Usage of the `charset` parameter is discouraged because it issues additional queries to the server.
 Unless you need the fallback behavior, please use `collation` instead.
 
+##### `checkConnLiveness`
+
+```
+Type:           bool
+Valid Values:   true, false
+Default:        true
+```
+
+On supported platforms connections retrieved from the connection pool are checked for liveness before using them. If the check fails, the respective connection is marked as bad and the query retried with another connection.
+`checkConnLiveness=false` disables this liveness check of connections.
+
 ##### `collation`
 
 ```
@@ -396,14 +407,9 @@ TCP on a remote host, e.g. Amazon RDS:
 id:password@tcp(your-amazonaws-uri.com:3306)/dbname
 ```
 
-Google Cloud SQL on App Engine (First Generation MySQL Server):
+Google Cloud SQL on App Engine:
 ```
-user@cloudsql(project-id:instance-name)/dbname
+user:password@unix(/cloudsql/project-id:region-name:instance-name)/dbname
-```
-
-Google Cloud SQL on App Engine (Second Generation MySQL Server):
-```
-user@cloudsql(project-id:regionname:instance-name)/dbname
 ```
 
 TCP using default port (3306) on localhost:
@@ -457,13 +463,13 @@ Alternatively you can use the [`NullTime`](https://godoc.org/github.com/go-sql-d
 
 
 ### Unicode support
-Since version 1.1 Go-MySQL-Driver automatically uses the collation `utf8_general_ci` by default.
+Since version 1.5 Go-MySQL-Driver automatically uses the collation ` utf8mb4_general_ci` by default.
 
 Other collations / charsets can be set using the [`collation`](#collation) DSN parameter.
 
 Version 1.0 of the driver recommended adding `&charset=utf8` (alias for `SET NAMES utf8`) to the DSN to enable proper UTF-8 support. This is not necessary anymore. The [`collation`](#collation) parameter should be preferred to set another collation / charset than the default.
 
-See http://dev.mysql.com/doc/refman/5.7/en/charset-unicode.html for more details on MySQL's Unicode support.
+See http://dev.mysql.com/doc/refman/8.0/en/charset-unicode.html for more details on MySQL's Unicode support.
 
 ## Testing / Development
 To run the driver tests you may need to adjust the configuration. See the [Testing Wiki-Page](https://github.com/go-sql-driver/mysql/wiki/Testing "Testing") for details.

vendor/github.com/go-sql-driver/mysql/appengine.go

@@ -1,25 +0,0 @@
-// Go MySQL Driver - A MySQL-Driver for Go's database/sql package
-//
-// Copyright 2013 The Go-MySQL-Driver Authors. All rights reserved.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
-
-// +build appengine
-
-package mysql
-
-import (
-	"context"
-	"net"
-
-	"google.golang.org/appengine/cloudsql"
-)
-
-func init() {
-	RegisterDialContext("cloudsql", func(_ context.Context, instance string) (net.Conn, error) {
-		// XXX: the cloudsql driver still does not export a Context-aware dialer.
-		return cloudsql.Dial(instance)
-	})
-}

vendor/github.com/go-sql-driver/mysql/conncheck.go

@@ -6,7 +6,7 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
 // You can obtain one at http://mozilla.org/MPL/2.0/.
 
-// +build !windows,!appengine
+// +build linux darwin dragonfly freebsd netbsd openbsd solaris illumos
 
 package mysql
 
@@ -19,35 +19,36 @@ import (
 
 var errUnexpectedRead = errors.New("unexpected read from socket")
 
-func connCheck(c net.Conn) error {
+func connCheck(conn net.Conn) error {
-	var (
+	var sysErr error
-		n    int
-		err  error
-		buff [1]byte
-	)
 
-	sconn, ok := c.(syscall.Conn)
+	sysConn, ok := conn.(syscall.Conn)
 	if !ok {
 		return nil
 	}
-	rc, err := sconn.SyscallConn()
+	rawConn, err := sysConn.SyscallConn()
 	if err != nil {
 		return err
 	}
-	rerr := rc.Read(func(fd uintptr) bool {
+
-		n, err = syscall.Read(int(fd), buff[:])
+	err = rawConn.Read(func(fd uintptr) bool {
+		var buf [1]byte
+		n, err := syscall.Read(int(fd), buf[:])
+		switch {
+		case n == 0 && err == nil:
+			sysErr = io.EOF
+		case n > 0:
+			sysErr = errUnexpectedRead
+		case err == syscall.EAGAIN || err == syscall.EWOULDBLOCK:
+			sysErr = nil
+		default:
+			sysErr = err
+		}
 		return true
 	})
-	switch {
+	if err != nil {
-	case rerr != nil:
-		return rerr
-	case n == 0 && err == nil:
-		return io.EOF
-	case n > 0:
-		return errUnexpectedRead
-	case err == syscall.EAGAIN || err == syscall.EWOULDBLOCK:
-		return nil
-	default:
 		return err
 	}
+
+	return sysErr
 }

vendor/github.com/go-sql-driver/mysql/conncheck_dummy.go

@@ -6,12 +6,12 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
 // You can obtain one at http://mozilla.org/MPL/2.0/.
 
-// +build windows appengine
+// +build !linux,!darwin,!dragonfly,!freebsd,!netbsd,!openbsd,!solaris,!illumos
 
 package mysql
 
 import "net"
 
-func connCheck(c net.Conn) error {
+func connCheck(conn net.Conn) error {
 	return nil
 }

vendor/github.com/go-sql-driver/mysql/conncheck_test.go

@@ -6,7 +6,7 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
 // You can obtain one at http://mozilla.org/MPL/2.0/.
 
-// +build go1.10,!windows
+// +build linux darwin dragonfly freebsd netbsd openbsd solaris illumos
 
 package mysql
 

vendor/github.com/go-sql-driver/mysql/connection.go

@@ -12,6 +12,7 @@ import (
 	"context"
 	"database/sql"
 	"database/sql/driver"
+	"encoding/json"
 	"io"
 	"net"
 	"strconv"
@@ -154,7 +155,9 @@ func (mc *mysqlConn) Prepare(query string) (driver.Stmt, error) {
 	// Send command
 	err := mc.writeCommandPacketStr(comStmtPrepare, query)
 	if err != nil {
-		return nil, mc.markBadConn(err)
+		// STMT_PREPARE is safe to retry.  So we can return ErrBadConn here.
+		errLog.Print(err)
+		return nil, driver.ErrBadConn
 	}
 
 	stmt := &mysqlStmt{
@@ -269,6 +272,14 @@ func (mc *mysqlConn) interpolateParams(query string, args []driver.Value) (strin
 				}
 				buf = append(buf, '\'')
 			}
+		case json.RawMessage:
+			buf = append(buf, '\'')
+			if mc.status&statusNoBackslashEscapes == 0 {
+				buf = escapeBytesBackslash(buf, v)
+			} else {
+				buf = escapeBytesQuotes(buf, v)
+			}
+			buf = append(buf, '\'')
 		case []byte:
 			if v == nil {
 				buf = append(buf, "NULL"...)

vendor/github.com/go-sql-driver/mysql/connection_test.go

@@ -11,6 +11,7 @@ package mysql
 import (
 	"context"
 	"database/sql/driver"
+	"encoding/json"
 	"errors"
 	"net"
 	"testing"
@@ -36,6 +37,33 @@ func TestInterpolateParams(t *testing.T) {
 	}
 }
 
+func TestInterpolateParamsJSONRawMessage(t *testing.T) {
+	mc := &mysqlConn{
+		buf:              newBuffer(nil),
+		maxAllowedPacket: maxPacketSize,
+		cfg: &Config{
+			InterpolateParams: true,
+		},
+	}
+
+	buf, err := json.Marshal(struct {
+		Value int `json:"value"`
+	}{Value: 42})
+	if err != nil {
+		t.Errorf("Expected err=nil, got %#v", err)
+		return
+	}
+	q, err := mc.interpolateParams("SELECT ?", []driver.Value{json.RawMessage(buf)})
+	if err != nil {
+		t.Errorf("Expected err=nil, got %#v", err)
+		return
+	}
+	expected := `SELECT '{\"value\":42}'`
+	if q != expected {
+		t.Errorf("Expected: %q\nGot: %q", expected, q)
+	}
+}
+
 func TestInterpolateParamsTooManyPlaceholders(t *testing.T) {
 	mc := &mysqlConn{
 		buf:              newBuffer(nil),

vendor/github.com/go-sql-driver/mysql/connector.go

@@ -37,17 +37,19 @@ func (c *connector) Connect(ctx context.Context) (driver.Conn, error) {
 	dial, ok := dials[mc.cfg.Net]
 	dialsLock.RUnlock()
 	if ok {
-		mc.netConn, err = dial(ctx, mc.cfg.Addr)
+		dctx := ctx
+		if mc.cfg.Timeout > 0 {
+			var cancel context.CancelFunc
+			dctx, cancel = context.WithTimeout(ctx, c.cfg.Timeout)
+			defer cancel()
+		}
+		mc.netConn, err = dial(dctx, mc.cfg.Addr)
 	} else {
 		nd := net.Dialer{Timeout: mc.cfg.Timeout}
 		mc.netConn, err = nd.DialContext(ctx, mc.cfg.Net, mc.cfg.Addr)
 	}
 
 	if err != nil {
-		if nerr, ok := err.(net.Error); ok && nerr.Temporary() {
-			errLog.Print("net.Error from Dial()': ", nerr.Error())
-			return nil, driver.ErrBadConn
-		}
 		return nil, err
 	}
 
@@ -64,6 +66,7 @@ func (c *connector) Connect(ctx context.Context) (driver.Conn, error) {
 	// Call startWatcher for context support (From Go 1.8)
 	mc.startWatcher()
 	if err := mc.watchCancel(ctx); err != nil {
+		mc.cleanup()
 		return nil, err
 	}
 	defer mc.finish()

vendor/github.com/go-sql-driver/mysql/connector_test.go

@@ -0,0 +1,30 @@
+package mysql
+
+import (
+	"context"
+	"net"
+	"testing"
+	"time"
+)
+
+func TestConnectorReturnsTimeout(t *testing.T) {
+	connector := &connector{&Config{
+		Net:     "tcp",
+		Addr:    "1.1.1.1:1234",
+		Timeout: 10 * time.Millisecond,
+	}}
+
+	_, err := connector.Connect(context.Background())
+	if err == nil {
+		t.Fatal("error expected")
+	}
+
+	if nerr, ok := err.(*net.OpError); ok {
+		expected := "dial tcp 1.1.1.1:1234: i/o timeout"
+		if nerr.Error() != expected {
+			t.Fatalf("expected %q, got %q", expected, nerr.Error())
+		}
+	} else {
+		t.Fatalf("expected %T, got %T", nerr, err)
+	}
+}

vendor/github.com/go-sql-driver/mysql/driver.go

@@ -83,3 +83,25 @@ func (d MySQLDriver) Open(dsn string) (driver.Conn, error) {
 func init() {
 	sql.Register("mysql", &MySQLDriver{})
 }
+
+// NewConnector returns new driver.Connector.
+func NewConnector(cfg *Config) (driver.Connector, error) {
+	cfg = cfg.Clone()
+	// normalize the contents of cfg so calls to NewConnector have the same
+	// behavior as MySQLDriver.OpenConnector
+	if err := cfg.normalize(); err != nil {
+		return nil, err
+	}
+	return &connector{cfg: cfg}, nil
+}
+
+// OpenConnector implements driver.DriverContext.
+func (d MySQLDriver) OpenConnector(dsn string) (driver.Connector, error) {
+	cfg, err := ParseDSN(dsn)
+	if err != nil {
+		return nil, err
+	}
+	return &connector{
+		cfg: cfg,
+	}, nil
+}

vendor/github.com/go-sql-driver/mysql/driver_go110.go

@@ -1,37 +0,0 @@
-// Go MySQL Driver - A MySQL-Driver for Go's database/sql package
-//
-// Copyright 2018 The Go-MySQL-Driver Authors. All rights reserved.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
-
-// +build go1.10
-
-package mysql
-
-import (
-	"database/sql/driver"
-)
-
-// NewConnector returns new driver.Connector.
-func NewConnector(cfg *Config) (driver.Connector, error) {
-	cfg = cfg.Clone()
-	// normalize the contents of cfg so calls to NewConnector have the same
-	// behavior as MySQLDriver.OpenConnector
-	if err := cfg.normalize(); err != nil {
-		return nil, err
-	}
-	return &connector{cfg: cfg}, nil
-}
-
-// OpenConnector implements driver.DriverContext.
-func (d MySQLDriver) OpenConnector(dsn string) (driver.Connector, error) {
-	cfg, err := ParseDSN(dsn)
-	if err != nil {
-		return nil, err
-	}
-	return &connector{
-		cfg: cfg,
-	}, nil
-}

vendor/github.com/go-sql-driver/mysql/driver_go110_test.go

@@ -1,137 +0,0 @@
-// Go MySQL Driver - A MySQL-Driver for Go's database/sql package
-//
-// Copyright 2018 The Go-MySQL-Driver Authors. All rights reserved.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
-
-// +build go1.10
-
-package mysql
-
-import (
-	"context"
-	"database/sql"
-	"database/sql/driver"
-	"fmt"
-	"net"
-	"testing"
-	"time"
-)
-
-var _ driver.DriverContext = &MySQLDriver{}
-
-type dialCtxKey struct{}
-
-func TestConnectorObeysDialTimeouts(t *testing.T) {
-	if !available {
-		t.Skipf("MySQL server not running on %s", netAddr)
-	}
-
-	RegisterDialContext("dialctxtest", func(ctx context.Context, addr string) (net.Conn, error) {
-		var d net.Dialer
-		if !ctx.Value(dialCtxKey{}).(bool) {
-			return nil, fmt.Errorf("test error: query context is not propagated to our dialer")
-		}
-		return d.DialContext(ctx, prot, addr)
-	})
-
-	db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@dialctxtest(%s)/%s?timeout=30s", user, pass, addr, dbname))
-	if err != nil {
-		t.Fatalf("error connecting: %s", err.Error())
-	}
-	defer db.Close()
-
-	ctx := context.WithValue(context.Background(), dialCtxKey{}, true)
-
-	_, err = db.ExecContext(ctx, "DO 1")
-	if err != nil {
-		t.Fatal(err)
-	}
-}
-
-func configForTests(t *testing.T) *Config {
-	if !available {
-		t.Skipf("MySQL server not running on %s", netAddr)
-	}
-
-	mycnf := NewConfig()
-	mycnf.User = user
-	mycnf.Passwd = pass
-	mycnf.Addr = addr
-	mycnf.Net = prot
-	mycnf.DBName = dbname
-	return mycnf
-}
-
-func TestNewConnector(t *testing.T) {
-	mycnf := configForTests(t)
-	conn, err := NewConnector(mycnf)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	db := sql.OpenDB(conn)
-	defer db.Close()
-
-	if err := db.Ping(); err != nil {
-		t.Fatal(err)
-	}
-}
-
-type slowConnection struct {
-	net.Conn
-	slowdown time.Duration
-}
-
-func (sc *slowConnection) Read(b []byte) (int, error) {
-	time.Sleep(sc.slowdown)
-	return sc.Conn.Read(b)
-}
-
-type connectorHijack struct {
-	driver.Connector
-	connErr error
-}
-
-func (cw *connectorHijack) Connect(ctx context.Context) (driver.Conn, error) {
-	var conn driver.Conn
-	conn, cw.connErr = cw.Connector.Connect(ctx)
-	return conn, cw.connErr
-}
-
-func TestConnectorTimeoutsDuringOpen(t *testing.T) {
-	RegisterDialContext("slowconn", func(ctx context.Context, addr string) (net.Conn, error) {
-		var d net.Dialer
-		conn, err := d.DialContext(ctx, prot, addr)
-		if err != nil {
-			return nil, err
-		}
-		return &slowConnection{Conn: conn, slowdown: 100 * time.Millisecond}, nil
-	})
-
-	mycnf := configForTests(t)
-	mycnf.Net = "slowconn"
-
-	conn, err := NewConnector(mycnf)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	hijack := &connectorHijack{Connector: conn}
-
-	db := sql.OpenDB(hijack)
-	defer db.Close()
-
-	ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
-	defer cancel()
-
-	_, err = db.ExecContext(ctx, "DO 1")
-	if err != context.DeadlineExceeded {
-		t.Fatalf("ExecContext should have timed out")
-	}
-	if hijack.connErr != context.DeadlineExceeded {
-		t.Fatalf("(*Connector).Connect should have timed out")
-	}
-}

vendor/github.com/go-sql-driver/mysql/driver_test.go

@@ -1874,7 +1874,7 @@ func TestDialNonRetryableNetErr(t *testing.T) {
 
 func TestDialTemporaryNetErr(t *testing.T) {
 	testErr := netErrorMock{temporary: true}
-	testDialError(t, testErr, driver.ErrBadConn)
+	testDialError(t, testErr, testErr)
 }
 
 // Tests custom dial functions
@@ -2994,3 +2994,172 @@ func TestRawBytesAreNotModified(t *testing.T) {
 		}
 	})
 }
+
+var _ driver.DriverContext = &MySQLDriver{}
+
+type dialCtxKey struct{}
+
+func TestConnectorObeysDialTimeouts(t *testing.T) {
+	if !available {
+		t.Skipf("MySQL server not running on %s", netAddr)
+	}
+
+	RegisterDialContext("dialctxtest", func(ctx context.Context, addr string) (net.Conn, error) {
+		var d net.Dialer
+		if !ctx.Value(dialCtxKey{}).(bool) {
+			return nil, fmt.Errorf("test error: query context is not propagated to our dialer")
+		}
+		return d.DialContext(ctx, prot, addr)
+	})
+
+	db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@dialctxtest(%s)/%s?timeout=30s", user, pass, addr, dbname))
+	if err != nil {
+		t.Fatalf("error connecting: %s", err.Error())
+	}
+	defer db.Close()
+
+	ctx := context.WithValue(context.Background(), dialCtxKey{}, true)
+
+	_, err = db.ExecContext(ctx, "DO 1")
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func configForTests(t *testing.T) *Config {
+	if !available {
+		t.Skipf("MySQL server not running on %s", netAddr)
+	}
+
+	mycnf := NewConfig()
+	mycnf.User = user
+	mycnf.Passwd = pass
+	mycnf.Addr = addr
+	mycnf.Net = prot
+	mycnf.DBName = dbname
+	return mycnf
+}
+
+func TestNewConnector(t *testing.T) {
+	mycnf := configForTests(t)
+	conn, err := NewConnector(mycnf)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	db := sql.OpenDB(conn)
+	defer db.Close()
+
+	if err := db.Ping(); err != nil {
+		t.Fatal(err)
+	}
+}
+
+type slowConnection struct {
+	net.Conn
+	slowdown time.Duration
+}
+
+func (sc *slowConnection) Read(b []byte) (int, error) {
+	time.Sleep(sc.slowdown)
+	return sc.Conn.Read(b)
+}
+
+type connectorHijack struct {
+	driver.Connector
+	connErr error
+}
+
+func (cw *connectorHijack) Connect(ctx context.Context) (driver.Conn, error) {
+	var conn driver.Conn
+	conn, cw.connErr = cw.Connector.Connect(ctx)
+	return conn, cw.connErr
+}
+
+func TestConnectorTimeoutsDuringOpen(t *testing.T) {
+	RegisterDialContext("slowconn", func(ctx context.Context, addr string) (net.Conn, error) {
+		var d net.Dialer
+		conn, err := d.DialContext(ctx, prot, addr)
+		if err != nil {
+			return nil, err
+		}
+		return &slowConnection{Conn: conn, slowdown: 100 * time.Millisecond}, nil
+	})
+
+	mycnf := configForTests(t)
+	mycnf.Net = "slowconn"
+
+	conn, err := NewConnector(mycnf)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	hijack := &connectorHijack{Connector: conn}
+
+	db := sql.OpenDB(hijack)
+	defer db.Close()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
+	defer cancel()
+
+	_, err = db.ExecContext(ctx, "DO 1")
+	if err != context.DeadlineExceeded {
+		t.Fatalf("ExecContext should have timed out")
+	}
+	if hijack.connErr != context.DeadlineExceeded {
+		t.Fatalf("(*Connector).Connect should have timed out")
+	}
+}
+
+// A connection which can only be closed.
+type dummyConnection struct {
+	net.Conn
+	closed bool
+}
+
+func (d *dummyConnection) Close() error {
+	d.closed = true
+	return nil
+}
+
+func TestConnectorTimeoutsWatchCancel(t *testing.T) {
+	var (
+		cancel  func()           // Used to cancel the context just after connecting.
+		created *dummyConnection // The created connection.
+	)
+
+	RegisterDialContext("TestConnectorTimeoutsWatchCancel", func(ctx context.Context, addr string) (net.Conn, error) {
+		// Canceling at this time triggers the watchCancel error branch in Connect().
+		cancel()
+		created = &dummyConnection{}
+		return created, nil
+	})
+
+	mycnf := NewConfig()
+	mycnf.User = "root"
+	mycnf.Addr = "foo"
+	mycnf.Net = "TestConnectorTimeoutsWatchCancel"
+
+	conn, err := NewConnector(mycnf)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	db := sql.OpenDB(conn)
+	defer db.Close()
+
+	var ctx context.Context
+	ctx, cancel = context.WithCancel(context.Background())
+	defer cancel()
+
+	if _, err := db.Conn(ctx); err != context.Canceled {
+		t.Errorf("got %v, want context.Canceled", err)
+	}
+
+	if created == nil {
+		t.Fatal("no connection created")
+	}
+	if !created.closed {
+		t.Errorf("connection not closed")
+	}
+}

vendor/github.com/go-sql-driver/mysql/dsn.go

@@ -55,6 +55,7 @@ type Config struct {
 	AllowCleartextPasswords bool // Allows the cleartext client side plugin
 	AllowNativePasswords    bool // Allows the native password authentication method
 	AllowOldPasswords       bool // Allows the old insecure password method
+	CheckConnLiveness       bool // Check connections for liveness before using them
 	ClientFoundRows         bool // Return number of matching rows instead of rows changed
 	ColumnsWithAlias        bool // Prepend table alias to column names
 	InterpolateParams       bool // Interpolate placeholders into query string
@@ -70,6 +71,7 @@ func NewConfig() *Config {
 		Loc:                  time.UTC,
 		MaxAllowedPacket:     defaultMaxAllowedPacket,
 		AllowNativePasswords: true,
+		CheckConnLiveness:    true,
 	}
 }
 
@@ -148,6 +150,19 @@ func (cfg *Config) normalize() error {
 	return nil
 }
 
+func writeDSNParam(buf *bytes.Buffer, hasParam *bool, name, value string) {
+	buf.Grow(1 + len(name) + 1 + len(value))
+	if !*hasParam {
+		*hasParam = true
+		buf.WriteByte('?')
+	} else {
+		buf.WriteByte('&')
+	}
+	buf.WriteString(name)
+	buf.WriteByte('=')
+	buf.WriteString(value)
+}
+
 // FormatDSN formats the given Config into a DSN string which can be passed to
 // the driver.
 func (cfg *Config) FormatDSN() string {
@@ -186,165 +201,75 @@ func (cfg *Config) FormatDSN() string {
 	}
 
 	if cfg.AllowCleartextPasswords {
-		if hasParam {
+		writeDSNParam(&buf, &hasParam, "allowCleartextPasswords", "true")
-			buf.WriteString("&allowCleartextPasswords=true")
-		} else {
-			hasParam = true
-			buf.WriteString("?allowCleartextPasswords=true")
-		}
 	}
 
 	if !cfg.AllowNativePasswords {
-		if hasParam {
+		writeDSNParam(&buf, &hasParam, "allowNativePasswords", "false")
-			buf.WriteString("&allowNativePasswords=false")
-		} else {
-			hasParam = true
-			buf.WriteString("?allowNativePasswords=false")
-		}
 	}
 
 	if cfg.AllowOldPasswords {
-		if hasParam {
+		writeDSNParam(&buf, &hasParam, "allowOldPasswords", "true")
-			buf.WriteString("&allowOldPasswords=true")
+	}
-		} else {
+
-			hasParam = true
+	if !cfg.CheckConnLiveness {
-			buf.WriteString("?allowOldPasswords=true")
+		writeDSNParam(&buf, &hasParam, "checkConnLiveness", "false")
-		}
 	}
 
 	if cfg.ClientFoundRows {
-		if hasParam {
+		writeDSNParam(&buf, &hasParam, "clientFoundRows", "true")
-			buf.WriteString("&clientFoundRows=true")
-		} else {
-			hasParam = true
-			buf.WriteString("?clientFoundRows=true")
-		}
 	}
 
 	if col := cfg.Collation; col != defaultCollation && len(col) > 0 {
-		if hasParam {
+		writeDSNParam(&buf, &hasParam, "collation", col)
-			buf.WriteString("&collation=")
-		} else {
-			hasParam = true
-			buf.WriteString("?collation=")
-		}
-		buf.WriteString(col)
 	}
 
 	if cfg.ColumnsWithAlias {
-		if hasParam {
+		writeDSNParam(&buf, &hasParam, "columnsWithAlias", "true")
-			buf.WriteString("&columnsWithAlias=true")
-		} else {
-			hasParam = true
-			buf.WriteString("?columnsWithAlias=true")
-		}
 	}
 
 	if cfg.InterpolateParams {
-		if hasParam {
+		writeDSNParam(&buf, &hasParam, "interpolateParams", "true")
-			buf.WriteString("&interpolateParams=true")
-		} else {
-			hasParam = true
-			buf.WriteString("?interpolateParams=true")
-		}
 	}
 
 	if cfg.Loc != time.UTC && cfg.Loc != nil {
-		if hasParam {
+		writeDSNParam(&buf, &hasParam, "loc", url.QueryEscape(cfg.Loc.String()))
-			buf.WriteString("&loc=")
-		} else {
-			hasParam = true
-			buf.WriteString("?loc=")
-		}
-		buf.WriteString(url.QueryEscape(cfg.Loc.String()))
 	}
 
 	if cfg.MultiStatements {
-		if hasParam {
+		writeDSNParam(&buf, &hasParam, "multiStatements", "true")
-			buf.WriteString("&multiStatements=true")
-		} else {
-			hasParam = true
-			buf.WriteString("?multiStatements=true")
-		}
 	}
 
 	if cfg.ParseTime {
-		if hasParam {
+		writeDSNParam(&buf, &hasParam, "parseTime", "true")
-			buf.WriteString("&parseTime=true")
-		} else {
-			hasParam = true
-			buf.WriteString("?parseTime=true")
-		}
 	}
 
 	if cfg.ReadTimeout > 0 {
-		if hasParam {
+		writeDSNParam(&buf, &hasParam, "readTimeout", cfg.ReadTimeout.String())
-			buf.WriteString("&readTimeout=")
-		} else {
-			hasParam = true
-			buf.WriteString("?readTimeout=")
-		}
-		buf.WriteString(cfg.ReadTimeout.String())
 	}
 
 	if cfg.RejectReadOnly {
-		if hasParam {
+		writeDSNParam(&buf, &hasParam, "rejectReadOnly", "true")
-			buf.WriteString("&rejectReadOnly=true")
-		} else {
-			hasParam = true
-			buf.WriteString("?rejectReadOnly=true")
-		}
 	}
 
 	if len(cfg.ServerPubKey) > 0 {
-		if hasParam {
+		writeDSNParam(&buf, &hasParam, "serverPubKey", url.QueryEscape(cfg.ServerPubKey))
-			buf.WriteString("&serverPubKey=")
-		} else {
-			hasParam = true
-			buf.WriteString("?serverPubKey=")
-		}
-		buf.WriteString(url.QueryEscape(cfg.ServerPubKey))
 	}
 
 	if cfg.Timeout > 0 {
-		if hasParam {
+		writeDSNParam(&buf, &hasParam, "timeout", cfg.Timeout.String())
-			buf.WriteString("&timeout=")
-		} else {
-			hasParam = true
-			buf.WriteString("?timeout=")
-		}
-		buf.WriteString(cfg.Timeout.String())
 	}
 
 	if len(cfg.TLSConfig) > 0 {
-		if hasParam {
+		writeDSNParam(&buf, &hasParam, "tls", url.QueryEscape(cfg.TLSConfig))
-			buf.WriteString("&tls=")
-		} else {
-			hasParam = true
-			buf.WriteString("?tls=")
-		}
-		buf.WriteString(url.QueryEscape(cfg.TLSConfig))
 	}
 
 	if cfg.WriteTimeout > 0 {
-		if hasParam {
+		writeDSNParam(&buf, &hasParam, "writeTimeout", cfg.WriteTimeout.String())
-			buf.WriteString("&writeTimeout=")
-		} else {
-			hasParam = true
-			buf.WriteString("?writeTimeout=")
-		}
-		buf.WriteString(cfg.WriteTimeout.String())
 	}
 
 	if cfg.MaxAllowedPacket != defaultMaxAllowedPacket {
-		if hasParam {
+		writeDSNParam(&buf, &hasParam, "maxAllowedPacket", strconv.Itoa(cfg.MaxAllowedPacket))
-			buf.WriteString("&maxAllowedPacket=")
-		} else {
-			hasParam = true
-			buf.WriteString("?maxAllowedPacket=")
-		}
-		buf.WriteString(strconv.Itoa(cfg.MaxAllowedPacket))
-
 	}
 
 	// other params
@@ -355,16 +280,7 @@ func (cfg *Config) FormatDSN() string {
 		}
 		sort.Strings(params)
 		for _, param := range params {
-			if hasParam {
+			writeDSNParam(&buf, &hasParam, param, url.QueryEscape(cfg.Params[param]))
-				buf.WriteByte('&')
-			} else {
-				hasParam = true
-				buf.WriteByte('?')
-			}
-
-			buf.WriteString(param)
-			buf.WriteByte('=')
-			buf.WriteString(url.QueryEscape(cfg.Params[param]))
 		}
 	}
 
@@ -491,6 +407,14 @@ func parseDSNParams(cfg *Config, params string) (err error) {
 				return errors.New("invalid bool value: " + value)
 			}
 
+		// Check connections for Liveness before using them
+		case "checkConnLiveness":
+			var isBool bool
+			cfg.CheckConnLiveness, isBool = readBool(value)
+			if !isBool {
+				return errors.New("invalid bool value: " + value)
+			}
+
 		// Switch "rowsAffected" mode
 		case "clientFoundRows":
 			var isBool bool

vendor/github.com/go-sql-driver/mysql/dsn_test.go

@@ -22,55 +22,55 @@ var testDSNs = []struct {
 	out *Config
 }{{
 	"username:password@protocol(address)/dbname?param=value",
-	&Config{User: "username", Passwd: "password", Net: "protocol", Addr: "address", DBName: "dbname", Params: map[string]string{"param": "value"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true},
+	&Config{User: "username", Passwd: "password", Net: "protocol", Addr: "address", DBName: "dbname", Params: map[string]string{"param": "value"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true},
 }, {
 	"username:password@protocol(address)/dbname?param=value&columnsWithAlias=true",
-	&Config{User: "username", Passwd: "password", Net: "protocol", Addr: "address", DBName: "dbname", Params: map[string]string{"param": "value"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, ColumnsWithAlias: true},
+	&Config{User: "username", Passwd: "password", Net: "protocol", Addr: "address", DBName: "dbname", Params: map[string]string{"param": "value"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true, ColumnsWithAlias: true},
 }, {
 	"username:password@protocol(address)/dbname?param=value&columnsWithAlias=true&multiStatements=true",
-	&Config{User: "username", Passwd: "password", Net: "protocol", Addr: "address", DBName: "dbname", Params: map[string]string{"param": "value"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, ColumnsWithAlias: true, MultiStatements: true},
+	&Config{User: "username", Passwd: "password", Net: "protocol", Addr: "address", DBName: "dbname", Params: map[string]string{"param": "value"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true, ColumnsWithAlias: true, MultiStatements: true},
 }, {
 	"user@unix(/path/to/socket)/dbname?charset=utf8",
-	&Config{User: "user", Net: "unix", Addr: "/path/to/socket", DBName: "dbname", Params: map[string]string{"charset": "utf8"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true},
+	&Config{User: "user", Net: "unix", Addr: "/path/to/socket", DBName: "dbname", Params: map[string]string{"charset": "utf8"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true},
 }, {
 	"user:password@tcp(localhost:5555)/dbname?charset=utf8&tls=true",
-	&Config{User: "user", Passwd: "password", Net: "tcp", Addr: "localhost:5555", DBName: "dbname", Params: map[string]string{"charset": "utf8"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, TLSConfig: "true"},
+	&Config{User: "user", Passwd: "password", Net: "tcp", Addr: "localhost:5555", DBName: "dbname", Params: map[string]string{"charset": "utf8"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true, TLSConfig: "true"},
 }, {
 	"user:password@tcp(localhost:5555)/dbname?charset=utf8mb4,utf8&tls=skip-verify",
-	&Config{User: "user", Passwd: "password", Net: "tcp", Addr: "localhost:5555", DBName: "dbname", Params: map[string]string{"charset": "utf8mb4,utf8"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, TLSConfig: "skip-verify"},
+	&Config{User: "user", Passwd: "password", Net: "tcp", Addr: "localhost:5555", DBName: "dbname", Params: map[string]string{"charset": "utf8mb4,utf8"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true, TLSConfig: "skip-verify"},
 }, {
 	"user:password@/dbname?loc=UTC&timeout=30s&readTimeout=1s&writeTimeout=1s&allowAllFiles=1&clientFoundRows=true&allowOldPasswords=TRUE&collation=utf8mb4_unicode_ci&maxAllowedPacket=16777216&tls=false&allowCleartextPasswords=true&parseTime=true&rejectReadOnly=true",
-	&Config{User: "user", Passwd: "password", Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_unicode_ci", Loc: time.UTC, TLSConfig: "false", AllowCleartextPasswords: true, AllowNativePasswords: true, Timeout: 30 * time.Second, ReadTimeout: time.Second, WriteTimeout: time.Second, AllowAllFiles: true, AllowOldPasswords: true, ClientFoundRows: true, MaxAllowedPacket: 16777216, ParseTime: true, RejectReadOnly: true},
+	&Config{User: "user", Passwd: "password", Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_unicode_ci", Loc: time.UTC, TLSConfig: "false", AllowCleartextPasswords: true, AllowNativePasswords: true, Timeout: 30 * time.Second, ReadTimeout: time.Second, WriteTimeout: time.Second, AllowAllFiles: true, AllowOldPasswords: true, CheckConnLiveness: true, ClientFoundRows: true, MaxAllowedPacket: 16777216, ParseTime: true, RejectReadOnly: true},
 }, {
-	"user:password@/dbname?allowNativePasswords=false&maxAllowedPacket=0",
+	"user:password@/dbname?allowNativePasswords=false&checkConnLiveness=false&maxAllowedPacket=0",
-	&Config{User: "user", Passwd: "password", Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: 0, AllowNativePasswords: false},
+	&Config{User: "user", Passwd: "password", Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: 0, AllowNativePasswords: false, CheckConnLiveness: false},
 }, {
 	"user:p@ss(word)@tcp([de:ad:be:ef::ca:fe]:80)/dbname?loc=Local",
-	&Config{User: "user", Passwd: "p@ss(word)", Net: "tcp", Addr: "[de:ad:be:ef::ca:fe]:80", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.Local, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true},
+	&Config{User: "user", Passwd: "p@ss(word)", Net: "tcp", Addr: "[de:ad:be:ef::ca:fe]:80", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.Local, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true},
 }, {
 	"/dbname",
-	&Config{Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true},
+	&Config{Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true},
 }, {
 	"@/",
-	&Config{Net: "tcp", Addr: "127.0.0.1:3306", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true},
+	&Config{Net: "tcp", Addr: "127.0.0.1:3306", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true},
 }, {
 	"/",
-	&Config{Net: "tcp", Addr: "127.0.0.1:3306", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true},
+	&Config{Net: "tcp", Addr: "127.0.0.1:3306", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true},
 }, {
 	"",
-	&Config{Net: "tcp", Addr: "127.0.0.1:3306", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true},
+	&Config{Net: "tcp", Addr: "127.0.0.1:3306", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true},
 }, {
 	"user:p@/ssword@/",
-	&Config{User: "user", Passwd: "p@/ssword", Net: "tcp", Addr: "127.0.0.1:3306", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true},
+	&Config{User: "user", Passwd: "p@/ssword", Net: "tcp", Addr: "127.0.0.1:3306", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true},
 }, {
 	"unix/?arg=%2Fsome%2Fpath.ext",
-	&Config{Net: "unix", Addr: "/tmp/mysql.sock", Params: map[string]string{"arg": "/some/path.ext"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true},
+	&Config{Net: "unix", Addr: "/tmp/mysql.sock", Params: map[string]string{"arg": "/some/path.ext"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true},
 }, {
 	"tcp(127.0.0.1)/dbname",
-	&Config{Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true},
+	&Config{Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true},
 }, {
 	"tcp(de:ad:be:ef::ca:fe)/dbname",
-	&Config{Net: "tcp", Addr: "[de:ad:be:ef::ca:fe]:3306", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true},
+	&Config{Net: "tcp", Addr: "[de:ad:be:ef::ca:fe]:3306", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true},
 },
 }
 

vendor/github.com/go-sql-driver/mysql/go.mod

@@ -0,0 +1,3 @@
+module github.com/go-sql-driver/mysql
+
+go 1.10

vendor/github.com/go-sql-driver/mysql/nulltime.go

@@ -0,0 +1,50 @@
+// Go MySQL Driver - A MySQL-Driver for Go's database/sql package
+//
+// Copyright 2013 The Go-MySQL-Driver Authors. All rights reserved.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package mysql
+
+import (
+	"database/sql/driver"
+	"fmt"
+	"time"
+)
+
+// Scan implements the Scanner interface.
+// The value type must be time.Time or string / []byte (formatted time-string),
+// otherwise Scan fails.
+func (nt *NullTime) Scan(value interface{}) (err error) {
+	if value == nil {
+		nt.Time, nt.Valid = time.Time{}, false
+		return
+	}
+
+	switch v := value.(type) {
+	case time.Time:
+		nt.Time, nt.Valid = v, true
+		return
+	case []byte:
+		nt.Time, err = parseDateTime(string(v), time.UTC)
+		nt.Valid = (err == nil)
+		return
+	case string:
+		nt.Time, err = parseDateTime(v, time.UTC)
+		nt.Valid = (err == nil)
+		return
+	}
+
+	nt.Valid = false
+	return fmt.Errorf("Can't convert %T to time.Time", value)
+}
+
+// Value implements the driver Valuer interface.
+func (nt NullTime) Value() (driver.Value, error) {
+	if !nt.Valid {
+		return nil, nil
+	}
+	return nt.Time, nil
+}

vendor/github.com/go-sql-driver/mysql/nulltime_go113.go

@@ -0,0 +1,31 @@
+// Go MySQL Driver - A MySQL-Driver for Go's database/sql package
+//
+// Copyright 2013 The Go-MySQL-Driver Authors. All rights reserved.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+// +build go1.13
+
+package mysql
+
+import (
+	"database/sql"
+)
+
+// NullTime represents a time.Time that may be NULL.
+// NullTime implements the Scanner interface so
+// it can be used as a scan destination:
+//
+//  var nt NullTime
+//  err := db.QueryRow("SELECT time FROM foo WHERE id=?", id).Scan(&nt)
+//  ...
+//  if nt.Valid {
+//     // use nt.Time
+//  } else {
+//     // NULL value
+//  }
+//
+// This NullTime implementation is not driver-specific
+type NullTime sql.NullTime

vendor/github.com/go-sql-driver/mysql/nulltime_legacy.go

@@ -0,0 +1,34 @@
+// Go MySQL Driver - A MySQL-Driver for Go's database/sql package
+//
+// Copyright 2013 The Go-MySQL-Driver Authors. All rights reserved.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+// +build !go1.13
+
+package mysql
+
+import (
+	"time"
+)
+
+// NullTime represents a time.Time that may be NULL.
+// NullTime implements the Scanner interface so
+// it can be used as a scan destination:
+//
+//  var nt NullTime
+//  err := db.QueryRow("SELECT time FROM foo WHERE id=?", id).Scan(&nt)
+//  ...
+//  if nt.Valid {
+//     // use nt.Time
+//  } else {
+//     // NULL value
+//  }
+//
+// This NullTime implementation is not driver-specific
+type NullTime struct {
+	Time  time.Time
+	Valid bool // Valid is true if Time is not NULL
+}

vendor/github.com/go-sql-driver/mysql/nulltime_test.go

@@ -0,0 +1,62 @@
+// Go MySQL Driver - A MySQL-Driver for Go's database/sql package
+//
+// Copyright 2013 The Go-MySQL-Driver Authors. All rights reserved.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package mysql
+
+import (
+	"database/sql"
+	"database/sql/driver"
+	"testing"
+	"time"
+)
+
+var (
+	// Check implementation of interfaces
+	_ driver.Valuer = NullTime{}
+	_ sql.Scanner   = (*NullTime)(nil)
+)
+
+func TestScanNullTime(t *testing.T) {
+	var scanTests = []struct {
+		in    interface{}
+		error bool
+		valid bool
+		time  time.Time
+	}{
+		{tDate, false, true, tDate},
+		{sDate, false, true, tDate},
+		{[]byte(sDate), false, true, tDate},
+		{tDateTime, false, true, tDateTime},
+		{sDateTime, false, true, tDateTime},
+		{[]byte(sDateTime), false, true, tDateTime},
+		{tDate0, false, true, tDate0},
+		{sDate0, false, true, tDate0},
+		{[]byte(sDate0), false, true, tDate0},
+		{sDateTime0, false, true, tDate0},
+		{[]byte(sDateTime0), false, true, tDate0},
+		{"", true, false, tDate0},
+		{"1234", true, false, tDate0},
+		{0, true, false, tDate0},
+	}
+
+	var nt = NullTime{}
+	var err error
+
+	for _, tst := range scanTests {
+		err = nt.Scan(tst.in)
+		if (err != nil) != tst.error {
+			t.Errorf("%v: expected error status %t, got %t", tst.in, tst.error, (err != nil))
+		}
+		if nt.Valid != tst.valid {
+			t.Errorf("%v: expected valid status %t, got %t", tst.in, tst.valid, nt.Valid)
+		}
+		if nt.Time != tst.time {
+			t.Errorf("%v: expected time %v, got %v", tst.in, tst.time, nt.Time)
+		}
+	}
+}

vendor/github.com/go-sql-driver/mysql/packets.go

@@ -115,7 +115,7 @@ func (mc *mysqlConn) writePacket(data []byte) error {
 		if mc.cfg.ReadTimeout != 0 {
 			err = conn.SetReadDeadline(time.Time{})
 		}
-		if err == nil {
+		if err == nil && mc.cfg.CheckConnLiveness {
 			err = connCheck(conn)
 		}
 		if err != nil {

vendor/github.com/go-sql-driver/mysql/utils.go

@@ -106,60 +106,6 @@ func readBool(input string) (value bool, valid bool) {
 *                           Time related utils                                *
 ******************************************************************************/
 
-// NullTime represents a time.Time that may be NULL.
-// NullTime implements the Scanner interface so
-// it can be used as a scan destination:
-//
-//  var nt NullTime
-//  err := db.QueryRow("SELECT time FROM foo WHERE id=?", id).Scan(&nt)
-//  ...
-//  if nt.Valid {
-//     // use nt.Time
-//  } else {
-//     // NULL value
-//  }
-//
-// This NullTime implementation is not driver-specific
-type NullTime struct {
-	Time  time.Time
-	Valid bool // Valid is true if Time is not NULL
-}
-
-// Scan implements the Scanner interface.
-// The value type must be time.Time or string / []byte (formatted time-string),
-// otherwise Scan fails.
-func (nt *NullTime) Scan(value interface{}) (err error) {
-	if value == nil {
-		nt.Time, nt.Valid = time.Time{}, false
-		return
-	}
-
-	switch v := value.(type) {
-	case time.Time:
-		nt.Time, nt.Valid = v, true
-		return
-	case []byte:
-		nt.Time, err = parseDateTime(string(v), time.UTC)
-		nt.Valid = (err == nil)
-		return
-	case string:
-		nt.Time, err = parseDateTime(v, time.UTC)
-		nt.Valid = (err == nil)
-		return
-	}
-
-	nt.Valid = false
-	return fmt.Errorf("Can't convert %T to time.Time", value)
-}
-
-// Value implements the driver Valuer interface.
-func (nt NullTime) Value() (driver.Value, error) {
-	if !nt.Valid {
-		return nil, nil
-	}
-	return nt.Time, nil
-}
-
 func parseDateTime(str string, loc *time.Location) (t time.Time, err error) {
 	base := "0000-00-00 00:00:00.0000000"
 	switch len(str) {

vendor/github.com/go-sql-driver/mysql/utils_test.go

@@ -14,49 +14,8 @@ import (
 	"database/sql/driver"
 	"encoding/binary"
 	"testing"
-	"time"
 )
 
-func TestScanNullTime(t *testing.T) {
-	var scanTests = []struct {
-		in    interface{}
-		error bool
-		valid bool
-		time  time.Time
-	}{
-		{tDate, false, true, tDate},
-		{sDate, false, true, tDate},
-		{[]byte(sDate), false, true, tDate},
-		{tDateTime, false, true, tDateTime},
-		{sDateTime, false, true, tDateTime},
-		{[]byte(sDateTime), false, true, tDateTime},
-		{tDate0, false, true, tDate0},
-		{sDate0, false, true, tDate0},
-		{[]byte(sDate0), false, true, tDate0},
-		{sDateTime0, false, true, tDate0},
-		{[]byte(sDateTime0), false, true, tDate0},
-		{"", true, false, tDate0},
-		{"1234", true, false, tDate0},
-		{0, true, false, tDate0},
-	}
-
-	var nt = NullTime{}
-	var err error
-
-	for _, tst := range scanTests {
-		err = nt.Scan(tst.in)
-		if (err != nil) != tst.error {
-			t.Errorf("%v: expected error status %t, got %t", tst.in, tst.error, (err != nil))
-		}
-		if nt.Valid != tst.valid {
-			t.Errorf("%v: expected valid status %t, got %t", tst.in, tst.valid, nt.Valid)
-		}
-		if nt.Time != tst.time {
-			t.Errorf("%v: expected time %v, got %v", tst.in, tst.time, nt.Time)
-		}
-	}
-}
-
 func TestLengthEncodedInteger(t *testing.T) {
 	var integerTests = []struct {
 		num     uint64