merged master

build.sh

@@ -2,7 +2,7 @@
 #
 #
 
-RELEASE_VERSION="1.0.10"
+RELEASE_VERSION="1.0.13"
 
 function build {
     osname=$1

doc/interactive-commands.md

@@ -24,7 +24,7 @@ Both interfaces may serve at the same time. Both respond to simple text command,
 - `critical-load=<load>`: change critical load setting (exceeding given thresholds causes panic and abort)
 - `nice-ratio=<ratio>`: change _nice_ ratio: 0 for aggressive (not nice, not sleeping), positive integer `n`: for any `1ms` spent copying rows, spend `n*1ms` units of time sleeping. Examples: assume a single rows chunk copy takes `100ms` to complete. `nice-ratio=0.5` will cause `gh-ost` to sleep for `50ms` immediately following. `nice-ratio=1` will cause `gh-ost` to sleep for `100ms`, effectively doubling runtime; value of `2` will effectively triple the runtime; etc.
 - `throttle-query`: change throttle query
-- `throttle-control-replicas`: change list of throttle-control replicas, these are replicas `gh-ost` will check
+- `throttle-control-replicas='replica1,replica2'`: change list of throttle-control replicas, these are replicas `gh-ost` will check. This takes a comma separated list of replica's to check and replaces the previous list.
 - `throttle`: force migration suspend
 - `no-throttle`: cancel forced suspension (though other throttling reasons may still apply)
 - `unpostpone`: at a time where `gh-ost` is postponing the [cut-over](cut-over.md) phase, instruct `gh-ost` to stop postponing and proceed immediately to cut-over.

doc/local-tests.md

@@ -0,0 +1,22 @@
+# Local tests
+
+`gh-ost` is continuously tested in production via `--test-on-replica alter='engine=innodb'`. These tests check the GitHub workload and usage, but not necessarily the general case.
+
+Local tests are an additional layer of tests. They will eventually be part of continuous integration tests.
+
+Local tests test explicit use cases, such as column renames, mix of time zones, special types and alters. Traits of a single test:
+
+- Composed of a single table.
+- A single alter.
+- By default the alter is `engine=innodb`, but this can be overridden per-test
+- Scheduled DML operations, executed via `event_scheduler`.
+- `gh-ost` is set to execute and throttle for `5` seconds, at which time all tested DMLs are expected to operate.
+- The test requires a replication topology and utilizes `--test-on-replica`
+- The test checksums the two tables (original and _ghost_) and expects identical checksum
+- By default the test selects all (`*`) columns, but this can be overridden per-test
+
+Tests are found under [localtests](https://github.com/github/gh-ost/tree/master/localtests). A single test is a subdirectory and tests are iterated alphabetically.
+
+New data-integrity, synchronization issues or otherwise concerns are expected to be tested by new test cases.
+
+While this is merged work is still ongoing.

go/base/context.go

@@ -87,6 +87,7 @@ type MigrationContext struct {
 	Noop                         bool
 	TestOnReplica                bool
 	MigrateOnReplica             bool
+	TestOnReplicaSkipReplicaStop bool
 	OkToDropTable                bool
 	InitiallyDropOldTable        bool
 	InitiallyDropGhostTable      bool

go/cmd/gh-ost/main.go

@@ -61,6 +61,7 @@ func main() {
 
 	executeFlag := flag.Bool("execute", false, "actually execute the alter & migrate the table. Default is noop: do some tests and exit")
 	flag.BoolVar(&migrationContext.TestOnReplica, "test-on-replica", false, "Have the migration run on a replica, not on the master. At the end of migration replication is stopped, and tables are swapped and immediately swap-revert. Replication remains stopped and you can compare the two tables for building trust")
+	flag.BoolVar(&migrationContext.TestOnReplicaSkipReplicaStop, "test-on-replica-skip-replica-stop", false, "When --test-on-replica is enabled, do not issue commands stop replication (requires --test-on-replica)")
 	flag.BoolVar(&migrationContext.MigrateOnReplica, "migrate-on-replica", false, "Have the migration run on a replica, not on the master. This will do the full migration on the replica including cut-over (as opposed to --test-on-replica)")
 
 	flag.BoolVar(&migrationContext.OkToDropTable, "ok-to-drop-table", false, "Shall the tool drop the old table at end of operation. DROPping tables can be a long locking operation, which is why I'm not doing it by default. I'm an online tool, yes?")
@@ -149,6 +150,13 @@ func main() {
 	if migrationContext.SwitchToRowBinlogFormat && migrationContext.AssumeRBR {
 		log.Fatalf("--switch-to-rbr and --assume-rbr are mutually exclusive")
 	}
+	if migrationContext.TestOnReplicaSkipReplicaStop {
+		if !migrationContext.TestOnReplica {
+			log.Fatalf("--test-on-replica-skip-replica-stop requires --test-on-replica to be enabled")
+		}
+		log.Warning("--test-on-replica-skip-replica-stop enabled. We will not stop replication before cut-over. Ensure you have a plugin that does this.")
+	}
+
 	switch *cutOver {
 	case "atomic", "default", "":
 		migrationContext.CutOverType = base.CutOverAtomic

go/logic/applier.go

@@ -107,7 +107,7 @@ func (this *Applier) ValidateOrDropExistingTables() error {
 		}
 	}
 	if this.tableExists(this.migrationContext.GetGhostTableName()) {
-		return fmt.Errorf("Table %s already exists. Panicking. Use --initially-drop-ghost-table to force dropping it", sql.EscapeName(this.migrationContext.GetGhostTableName()))
+		return fmt.Errorf("Table %s already exists. Panicking. Use --initially-drop-ghost-table to force dropping it, though I really prefer that you drop it or rename it away", sql.EscapeName(this.migrationContext.GetGhostTableName()))
 	}
 	if this.migrationContext.InitiallyDropOldTable {
 		if err := this.DropOldTable(); err != nil {
@@ -115,7 +115,7 @@ func (this *Applier) ValidateOrDropExistingTables() error {
 		}
 	}
 	if this.tableExists(this.migrationContext.GetOldTableName()) {
-		return fmt.Errorf("Table %s already exists. Panicking. Use --initially-drop-old-table to force dropping it", sql.EscapeName(this.migrationContext.GetOldTableName()))
+		return fmt.Errorf("Table %s already exists. Panicking. Use --initially-drop-old-table to force dropping it, though I really prefer that you drop it or rename it away", sql.EscapeName(this.migrationContext.GetOldTableName()))
 	}
 
 	return nil
@@ -574,6 +574,7 @@ func (this *Applier) StopReplication() error {
 	if err := this.StopSlaveSQLThread(); err != nil {
 		return err
 	}
+
 	readBinlogCoordinates, executeBinlogCoordinates, err := mysql.GetReplicationBinlogCoordinates(this.db)
 	if err != nil {
 		return err
@@ -832,12 +833,12 @@ func (this *Applier) buildDMLEventQuery(dmlEvent *binlog.BinlogDMLEvent) (query
 		}
 	case binlog.InsertDML:
 		{
-			query, sharedArgs, err := sql.BuildDMLInsertQuery(dmlEvent.DatabaseName, this.migrationContext.GetGhostTableName(), this.migrationContext.OriginalTableColumns, this.migrationContext.MappedSharedColumns, dmlEvent.NewColumnValues.AbstractValues())
+			query, sharedArgs, err := sql.BuildDMLInsertQuery(dmlEvent.DatabaseName, this.migrationContext.GetGhostTableName(), this.migrationContext.OriginalTableColumns, this.migrationContext.SharedColumns, this.migrationContext.MappedSharedColumns, dmlEvent.NewColumnValues.AbstractValues())
 			return query, sharedArgs, 1, err
 		}
 	case binlog.UpdateDML:
 		{
-			query, sharedArgs, uniqueKeyArgs, err := sql.BuildDMLUpdateQuery(dmlEvent.DatabaseName, this.migrationContext.GetGhostTableName(), this.migrationContext.OriginalTableColumns, this.migrationContext.MappedSharedColumns, &this.migrationContext.UniqueKey.Columns, dmlEvent.NewColumnValues.AbstractValues(), dmlEvent.WhereColumnValues.AbstractValues())
+			query, sharedArgs, uniqueKeyArgs, err := sql.BuildDMLUpdateQuery(dmlEvent.DatabaseName, this.migrationContext.GetGhostTableName(), this.migrationContext.OriginalTableColumns, this.migrationContext.SharedColumns, this.migrationContext.MappedSharedColumns, &this.migrationContext.UniqueKey.Columns, dmlEvent.NewColumnValues.AbstractValues(), dmlEvent.WhereColumnValues.AbstractValues())
 			args = append(args, sharedArgs...)
 			args = append(args, uniqueKeyArgs...)
 			return query, args, 0, err
@@ -853,7 +854,6 @@ func (this *Applier) ApplyDMLEventQuery(dmlEvent *binlog.BinlogDMLEvent) error {
 	if err != nil {
 		return err
 	}
-
 	// TODO The below is in preparation for transactional writes on the ghost tables.
 	// Such writes would be, for example:
 	// - prepended with sql_mode setup
@@ -871,6 +871,12 @@ func (this *Applier) ApplyDMLEventQuery(dmlEvent *binlog.BinlogDMLEvent) error {
 		if err != nil {
 			return err
 		}
+		if _, err := tx.Exec(`SET
+			SESSION time_zone = '+00:00',
+			sql_mode = CONCAT(@@session.sql_mode, ',STRICT_ALL_TABLES')
+			`); err != nil {
+			return err
+		}
 		if _, err := tx.Exec(query, args...); err != nil {
 			return err
 		}

go/logic/migrator.go

@@ -363,7 +363,7 @@ func (this *Migrator) validateStatement() (err error) {
 	if this.parser.HasNonTrivialRenames() && !this.migrationContext.SkipRenamedColumns {
 		this.migrationContext.ColumnRenameMap = this.parser.GetNonTrivialRenames()
 		if !this.migrationContext.ApproveRenamedColumns {
-			return fmt.Errorf("Alter statement has column(s) renamed. gh-ost suspects the following renames: %v; but to proceed you must approve via `--approve-renamed-columns` (or you can skip renamed columns via `--skip-renamed-columns`)", this.parser.GetNonTrivialRenames())
+			return fmt.Errorf("gh-ost believes the ALTER statement renames columns, as follows: %v; as precation, you are asked to confirm gh-ost is correct, and provide with `--approve-renamed-columns`, and we're all happy. Or you can skip renamed columns via `--skip-renamed-columns`, in which case column data may be lost", this.parser.GetNonTrivialRenames())
 		}
 		log.Infof("Alter statement has column(s) renamed. gh-ost finds the following renames: %v; --approve-renamed-columns is given and so migration proceeds.", this.parser.GetNonTrivialRenames())
 	}
@@ -402,7 +402,7 @@ func (this *Migrator) Migrate() (err error) {
 		return err
 	}
 
-	log.Debugf("Waiting for tables to be in place")
+	log.Infof("Waiting for tables to be in place")
 	<-this.tablesInPlace
 	log.Debugf("Tables are in place")
 	// Yay! We now know the Ghost and Changelog tables are good to examine!
@@ -520,11 +520,15 @@ func (this *Migrator) cutOver() (err error) {
 		// the same cut-over phase as the master would use. That means we take locks
 		// and swap the tables.
 		// The difference is that we will later swap the tables back.
-		log.Debugf("testing on replica. Stopping replication IO thread")
 		this.hooksExecutor.onStopReplication()
+		if this.migrationContext.TestOnReplicaSkipReplicaStop {
+			log.Warningf("--test-on-replica-skip-replica-stop enabled, we are not stopping replication.")
+		} else {
+			log.Debugf("testing on replica. Stopping replication IO thread")
 			if err := this.retryOperation(this.applier.StopReplication); err != nil {
 				return err
 			}
+		}
 		// We're merly testing, we don't want to keep this state. Rollback the renames as possible
 		defer this.applier.RenameTablesRollback()
 		// We further proceed to do the cutover by normal means; the 'defer' above will rollback the swap

go/sql/builder.go

@@ -354,7 +354,7 @@ func BuildDMLDeleteQuery(databaseName, tableName string, tableColumns, uniqueKey
 	return result, uniqueKeyArgs, nil
 }
 
-func BuildDMLInsertQuery(databaseName, tableName string, tableColumns, sharedColumns *ColumnList, args []interface{}) (result string, sharedArgs []interface{}, err error) {
+func BuildDMLInsertQuery(databaseName, tableName string, tableColumns, sharedColumns, mappedSharedColumns *ColumnList, args []interface{}) (result string, sharedArgs []interface{}, err error) {
 	if len(args) != tableColumns.Len() {
 		return result, args, fmt.Errorf("args count differs from table column count in BuildDMLInsertQuery")
 	}
@@ -367,17 +367,17 @@ func BuildDMLInsertQuery(databaseName, tableName string, tableColumns, sharedCol
 	databaseName = EscapeName(databaseName)
 	tableName = EscapeName(tableName)
 
-	for _, column := range sharedColumns.Names {
+	for _, column := range mappedSharedColumns.Names {
 		tableOrdinal := tableColumns.Ordinals[column]
-		arg := fixArgType(args[tableOrdinal], sharedColumns.IsUnsigned(column))
+		arg := fixArgType(args[tableOrdinal], mappedSharedColumns.IsUnsigned(column))
 		sharedArgs = append(sharedArgs, arg)
 	}
 
-	sharedColumnNames := duplicateNames(sharedColumns.Names)
-	for i := range sharedColumnNames {
-		sharedColumnNames[i] = EscapeName(sharedColumnNames[i])
+	mappedSharedColumnNames := duplicateNames(mappedSharedColumns.Names)
+	for i := range mappedSharedColumnNames {
+		mappedSharedColumnNames[i] = EscapeName(mappedSharedColumnNames[i])
 	}
-	preparedValues := buildPreparedValues(sharedColumns.Len())
+	preparedValues := buildPreparedValues(mappedSharedColumns.Len())
 
 	result = fmt.Sprintf(`
 			replace /* gh-ost %s.%s */ into
@@ -387,13 +387,13 @@ func BuildDMLInsertQuery(databaseName, tableName string, tableColumns, sharedCol
 					(%s)
 		`, databaseName, tableName,
 		databaseName, tableName,
-		strings.Join(sharedColumnNames, ", "),
+		strings.Join(mappedSharedColumnNames, ", "),
 		strings.Join(preparedValues, ", "),
 	)
 	return result, sharedArgs, nil
 }
 
-func BuildDMLUpdateQuery(databaseName, tableName string, tableColumns, sharedColumns, uniqueKeyColumns *ColumnList, valueArgs, whereArgs []interface{}) (result string, sharedArgs, uniqueKeyArgs []interface{}, err error) {
+func BuildDMLUpdateQuery(databaseName, tableName string, tableColumns, sharedColumns, mappedSharedColumns, uniqueKeyColumns *ColumnList, valueArgs, whereArgs []interface{}) (result string, sharedArgs, uniqueKeyArgs []interface{}, err error) {
 	if len(valueArgs) != tableColumns.Len() {
 		return result, sharedArgs, uniqueKeyArgs, fmt.Errorf("value args count differs from table column count in BuildDMLUpdateQuery")
 	}
@@ -415,9 +415,10 @@ func BuildDMLUpdateQuery(databaseName, tableName string, tableColumns, sharedCol
 	databaseName = EscapeName(databaseName)
 	tableName = EscapeName(tableName)
 
-	for _, column := range sharedColumns.Names {
+	for i, column := range sharedColumns.Names {
+		mappedColumn := mappedSharedColumns.Names[i]
 		tableOrdinal := tableColumns.Ordinals[column]
-		arg := fixArgType(valueArgs[tableOrdinal], sharedColumns.IsUnsigned(column))
+		arg := fixArgType(valueArgs[tableOrdinal], mappedSharedColumns.IsUnsigned(mappedColumn))
 		sharedArgs = append(sharedArgs, arg)
 	}
 
@@ -427,11 +428,11 @@ func BuildDMLUpdateQuery(databaseName, tableName string, tableColumns, sharedCol
 		uniqueKeyArgs = append(uniqueKeyArgs, arg)
 	}
 
-	sharedColumnNames := duplicateNames(sharedColumns.Names)
-	for i := range sharedColumnNames {
-		sharedColumnNames[i] = EscapeName(sharedColumnNames[i])
+	mappedSharedColumnNames := duplicateNames(mappedSharedColumns.Names)
+	for i := range mappedSharedColumnNames {
+		mappedSharedColumnNames[i] = EscapeName(mappedSharedColumnNames[i])
 	}
-	setClause, err := BuildSetPreparedClause(sharedColumnNames)
+	setClause, err := BuildSetPreparedClause(mappedSharedColumnNames)
 
 	equalsComparison, err := BuildEqualsPreparedComparison(uniqueKeyColumns.Names)
 	result = fmt.Sprintf(`

go/sql/builder_test.go

@@ -442,7 +442,7 @@ func TestBuildDMLInsertQuery(t *testing.T) {
 	args := []interface{}{3, "testname", "first", 17, 23}
 	{
 		sharedColumns := NewColumnList([]string{"id", "name", "position", "age"})
-		query, sharedArgs, err := BuildDMLInsertQuery(databaseName, tableName, tableColumns, sharedColumns, args)
+		query, sharedArgs, err := BuildDMLInsertQuery(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, args)
 		test.S(t).ExpectNil(err)
 		expected := `
 			replace /* gh-ost mydb.tbl */
@@ -456,7 +456,7 @@ func TestBuildDMLInsertQuery(t *testing.T) {
 	}
 	{
 		sharedColumns := NewColumnList([]string{"position", "name", "age", "id"})
-		query, sharedArgs, err := BuildDMLInsertQuery(databaseName, tableName, tableColumns, sharedColumns, args)
+		query, sharedArgs, err := BuildDMLInsertQuery(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, args)
 		test.S(t).ExpectNil(err)
 		expected := `
 			replace /* gh-ost mydb.tbl */
@@ -470,12 +470,12 @@ func TestBuildDMLInsertQuery(t *testing.T) {
 	}
 	{
 		sharedColumns := NewColumnList([]string{"position", "name", "surprise", "id"})
-		_, _, err := BuildDMLInsertQuery(databaseName, tableName, tableColumns, sharedColumns, args)
+		_, _, err := BuildDMLInsertQuery(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, args)
 		test.S(t).ExpectNotNil(err)
 	}
 	{
 		sharedColumns := NewColumnList([]string{})
-		_, _, err := BuildDMLInsertQuery(databaseName, tableName, tableColumns, sharedColumns, args)
+		_, _, err := BuildDMLInsertQuery(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, args)
 		test.S(t).ExpectNotNil(err)
 	}
 }
@@ -489,7 +489,7 @@ func TestBuildDMLInsertQuerySignedUnsigned(t *testing.T) {
 		// testing signed
 		args := []interface{}{3, "testname", "first", int8(-1), 23}
 		sharedColumns := NewColumnList([]string{"id", "name", "position", "age"})
-		query, sharedArgs, err := BuildDMLInsertQuery(databaseName, tableName, tableColumns, sharedColumns, args)
+		query, sharedArgs, err := BuildDMLInsertQuery(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, args)
 		test.S(t).ExpectNil(err)
 		expected := `
 			replace /* gh-ost mydb.tbl */
@@ -505,7 +505,7 @@ func TestBuildDMLInsertQuerySignedUnsigned(t *testing.T) {
 		// testing unsigned
 		args := []interface{}{3, "testname", "first", int8(-1), 23}
 		sharedColumns.SetUnsigned("position")
-		query, sharedArgs, err := BuildDMLInsertQuery(databaseName, tableName, tableColumns, sharedColumns, args)
+		query, sharedArgs, err := BuildDMLInsertQuery(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, args)
 		test.S(t).ExpectNil(err)
 		expected := `
 			replace /* gh-ost mydb.tbl */
@@ -521,7 +521,7 @@ func TestBuildDMLInsertQuerySignedUnsigned(t *testing.T) {
 		// testing unsigned
 		args := []interface{}{3, "testname", "first", int32(-1), 23}
 		sharedColumns.SetUnsigned("position")
-		query, sharedArgs, err := BuildDMLInsertQuery(databaseName, tableName, tableColumns, sharedColumns, args)
+		query, sharedArgs, err := BuildDMLInsertQuery(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, args)
 		test.S(t).ExpectNil(err)
 		expected := `
 			replace /* gh-ost mydb.tbl */
@@ -544,7 +544,7 @@ func TestBuildDMLUpdateQuery(t *testing.T) {
 	{
 		sharedColumns := NewColumnList([]string{"id", "name", "position", "age"})
 		uniqueKeyColumns := NewColumnList([]string{"position"})
-		query, sharedArgs, uniqueKeyArgs, err := BuildDMLUpdateQuery(databaseName, tableName, tableColumns, sharedColumns, uniqueKeyColumns, valueArgs, whereArgs)
+		query, sharedArgs, uniqueKeyArgs, err := BuildDMLUpdateQuery(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, uniqueKeyColumns, valueArgs, whereArgs)
 		test.S(t).ExpectNil(err)
 		expected := `
 			update /* gh-ost mydb.tbl */
@@ -560,7 +560,7 @@ func TestBuildDMLUpdateQuery(t *testing.T) {
 	{
 		sharedColumns := NewColumnList([]string{"id", "name", "position", "age"})
 		uniqueKeyColumns := NewColumnList([]string{"position", "name"})
-		query, sharedArgs, uniqueKeyArgs, err := BuildDMLUpdateQuery(databaseName, tableName, tableColumns, sharedColumns, uniqueKeyColumns, valueArgs, whereArgs)
+		query, sharedArgs, uniqueKeyArgs, err := BuildDMLUpdateQuery(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, uniqueKeyColumns, valueArgs, whereArgs)
 		test.S(t).ExpectNil(err)
 		expected := `
 			update /* gh-ost mydb.tbl */
@@ -576,7 +576,7 @@ func TestBuildDMLUpdateQuery(t *testing.T) {
 	{
 		sharedColumns := NewColumnList([]string{"id", "name", "position", "age"})
 		uniqueKeyColumns := NewColumnList([]string{"age"})
-		query, sharedArgs, uniqueKeyArgs, err := BuildDMLUpdateQuery(databaseName, tableName, tableColumns, sharedColumns, uniqueKeyColumns, valueArgs, whereArgs)
+		query, sharedArgs, uniqueKeyArgs, err := BuildDMLUpdateQuery(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, uniqueKeyColumns, valueArgs, whereArgs)
 		test.S(t).ExpectNil(err)
 		expected := `
 			update /* gh-ost mydb.tbl */
@@ -592,7 +592,7 @@ func TestBuildDMLUpdateQuery(t *testing.T) {
 	{
 		sharedColumns := NewColumnList([]string{"id", "name", "position", "age"})
 		uniqueKeyColumns := NewColumnList([]string{"age", "position", "id", "name"})
-		query, sharedArgs, uniqueKeyArgs, err := BuildDMLUpdateQuery(databaseName, tableName, tableColumns, sharedColumns, uniqueKeyColumns, valueArgs, whereArgs)
+		query, sharedArgs, uniqueKeyArgs, err := BuildDMLUpdateQuery(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, uniqueKeyColumns, valueArgs, whereArgs)
 		test.S(t).ExpectNil(err)
 		expected := `
 			update /* gh-ost mydb.tbl */
@@ -608,15 +608,32 @@ func TestBuildDMLUpdateQuery(t *testing.T) {
 	{
 		sharedColumns := NewColumnList([]string{"id", "name", "position", "age"})
 		uniqueKeyColumns := NewColumnList([]string{"age", "surprise"})
-		_, _, _, err := BuildDMLUpdateQuery(databaseName, tableName, tableColumns, sharedColumns, uniqueKeyColumns, valueArgs, whereArgs)
+		_, _, _, err := BuildDMLUpdateQuery(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, uniqueKeyColumns, valueArgs, whereArgs)
 		test.S(t).ExpectNotNil(err)
 	}
 	{
 		sharedColumns := NewColumnList([]string{"id", "name", "position", "age"})
 		uniqueKeyColumns := NewColumnList([]string{})
-		_, _, _, err := BuildDMLUpdateQuery(databaseName, tableName, tableColumns, sharedColumns, uniqueKeyColumns, valueArgs, whereArgs)
+		_, _, _, err := BuildDMLUpdateQuery(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, uniqueKeyColumns, valueArgs, whereArgs)
 		test.S(t).ExpectNotNil(err)
 	}
+	{
+		sharedColumns := NewColumnList([]string{"id", "name", "position", "age"})
+		mappedColumns := NewColumnList([]string{"id", "name", "role", "age"})
+		uniqueKeyColumns := NewColumnList([]string{"id"})
+		query, sharedArgs, uniqueKeyArgs, err := BuildDMLUpdateQuery(databaseName, tableName, tableColumns, sharedColumns, mappedColumns, uniqueKeyColumns, valueArgs, whereArgs)
+		test.S(t).ExpectNil(err)
+		expected := `
+			update /* gh-ost mydb.tbl */
+			  mydb.tbl
+					set id=?, name=?, role=?, age=?
+				where
+					((id = ?))
+		`
+		test.S(t).ExpectEquals(normalizeQuery(query), normalizeQuery(expected))
+		test.S(t).ExpectTrue(reflect.DeepEqual(sharedArgs, []interface{}{3, "testname", 17, 23}))
+		test.S(t).ExpectTrue(reflect.DeepEqual(uniqueKeyArgs, []interface{}{3}))
+	}
 }
 
 func TestBuildDMLUpdateQuerySignedUnsigned(t *testing.T) {
@@ -629,7 +646,7 @@ func TestBuildDMLUpdateQuerySignedUnsigned(t *testing.T) {
 	uniqueKeyColumns := NewColumnList([]string{"position"})
 	{
 		// test signed
-		query, sharedArgs, uniqueKeyArgs, err := BuildDMLUpdateQuery(databaseName, tableName, tableColumns, sharedColumns, uniqueKeyColumns, valueArgs, whereArgs)
+		query, sharedArgs, uniqueKeyArgs, err := BuildDMLUpdateQuery(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, uniqueKeyColumns, valueArgs, whereArgs)
 		test.S(t).ExpectNil(err)
 		expected := `
 			update /* gh-ost mydb.tbl */
@@ -646,7 +663,7 @@ func TestBuildDMLUpdateQuerySignedUnsigned(t *testing.T) {
 		// test unsigned
 		sharedColumns.SetUnsigned("age")
 		uniqueKeyColumns.SetUnsigned("position")
-		query, sharedArgs, uniqueKeyArgs, err := BuildDMLUpdateQuery(databaseName, tableName, tableColumns, sharedColumns, uniqueKeyColumns, valueArgs, whereArgs)
+		query, sharedArgs, uniqueKeyArgs, err := BuildDMLUpdateQuery(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, uniqueKeyColumns, valueArgs, whereArgs)
 		test.S(t).ExpectNil(err)
 		expected := `
 			update /* gh-ost mydb.tbl */

localtests/enum/create.sql

@@ -0,0 +1,27 @@
+drop table if exists gh_ost_test;
+create table gh_ost_test (
+  id int auto_increment,
+  i int not null,
+  e enum('red', 'green', 'blue', 'orange') null default null collate 'utf8_bin',
+  primary key(id)
+) auto_increment=1;
+
+drop event if exists gh_ost_test;
+delimiter ;;
+create event gh_ost_test
+  on schedule every 1 second
+  starts current_timestamp
+  ends current_timestamp + interval 60 second
+  on completion not preserve
+  enable
+  do
+begin
+  insert into gh_ost_test values (null, 11, 'red');
+  insert into gh_ost_test values (null, 13, 'green');
+  insert into gh_ost_test values (null, 17, 'blue');
+  set @last_insert_id := last_insert_id();
+  update gh_ost_test set e='orange' where id = @last_insert_id;
+  insert into gh_ost_test values (null, 23, null);
+  set @last_insert_id := last_insert_id();
+  update gh_ost_test set i=i+1, e=null where id = @last_insert_id;
+end ;;

localtests/enum/extra_args

@@ -0,0 +1 @@
+--alter="change e e enum('red', 'green', 'blue', 'orange', 'yellow') null default null collate 'utf8_bin'" 

localtests/rename/create.sql

@@ -0,0 +1,26 @@
+drop table if exists gh_ost_test;
+create table gh_ost_test (
+  id int auto_increment,
+  c1 int not null,
+  c2 int not null,
+  primary key (id)
+) auto_increment=1;
+
+drop event if exists gh_ost_test;
+delimiter ;;
+create event gh_ost_test
+  on schedule every 1 second
+  starts current_timestamp
+  ends current_timestamp + interval 60 second
+  on completion not preserve
+  enable
+  do
+begin
+  insert ignore into gh_ost_test values (1, 11, 23);
+  insert ignore into gh_ost_test values (2, 13, 23);
+  insert into gh_ost_test values (null, 17, 23);
+  set @last_insert_id := last_insert_id();
+  update gh_ost_test set c1=c1+@last_insert_id, c2=c2+@last_insert_id where id=@last_insert_id order by id desc limit 1;
+  delete from gh_ost_test where id=1;
+  delete from gh_ost_test where c1=13; -- id=2
+end ;;

localtests/rename/extra_args

@@ -0,0 +1 @@
+--alter="change column c2 c3 int not null" --approve-renamed-columns

localtests/test.sh

@@ -0,0 +1,112 @@
+#!/bin/bash
+
+# Local integration tests. To be used by CI.
+# See https://github.com/github/gh-ost/tree/doc/local-tests.md
+#
+
+tests_path=$(dirname $0)
+test_logfile=/tmp/gh-ost-test.log
+exec_command_file=/tmp/gh-ost-test.bash
+
+master_host=
+master_port=
+replica_host=
+replica_port=
+
+verify_master_and_replica() {
+  if [ "$(gh-ost-test-mysql-master -e "select 1" -ss)" != "1" ] ; then
+    echo "Cannot verify gh-ost-test-mysql-master"
+    exit 1
+  fi
+  read master_host master_port <<< $(gh-ost-test-mysql-master -e "select @@hostname, @@port" -ss)
+  if [ "$(gh-ost-test-mysql-replica -e "select 1" -ss)" != "1" ] ; then
+    echo "Cannot verify gh-ost-test-mysql-replica"
+    exit 1
+  fi
+  read replica_host replica_port <<< $(gh-ost-test-mysql-replica -e "select @@hostname, @@port" -ss)
+}
+
+exec_cmd() {
+  echo "$@"
+  command "$@" 1> $test_logfile 2>&1
+  return $?
+}
+
+test_single() {
+  local test_name
+  test_name="$1"
+
+  echo "Testing: $test_name"
+
+  gh-ost-test-mysql-replica -e "start slave"
+  gh-ost-test-mysql-master test < $tests_path/$test_name/create.sql
+
+  extra_args=""
+  if [ -f $tests_path/$test_name/extra_args ] ; then
+    extra_args=$(cat $tests_path/$test_name/extra_args)
+  fi
+  columns="*"
+  if [ -f $tests_path/$test_name/test_columns ] ; then
+    columns=$(cat $tests_path/$test_name/test_columns)
+  fi
+  # graceful sleep for replica to catch up
+  sleep 1
+  #
+  cmd="go run go/cmd/gh-ost/main.go \
+    --user=gh-ost \
+    --password=gh-ost \
+    --host=$replica_host \
+    --port=$replica_port \
+    --database=test \
+    --table=gh_ost_test \
+    --alter='engine=innodb' \
+    --exact-rowcount \
+    --switch-to-rbr \
+    --initially-drop-old-table \
+    --initially-drop-ghost-table \
+    --throttle-query='select timestampdiff(second, min(last_update), now()) < 5 from _gh_ost_test_ghc' \
+    --serve-socket-file=/tmp/gh-ost.test.sock \
+    --initially-drop-socket-file \
+    --postpone-cut-over-flag-file=/tmp/gh-ost.postpone.flag \
+    --test-on-replica \
+    --default-retries=1 \
+    --verbose \
+    --debug \
+    --stack \
+    --execute ${extra_args[@]}"
+  echo $cmd > $exec_command_file
+  bash $exec_command_file 1> $test_logfile 2>&1
+
+  if [ $? -ne 0 ] ; then
+    echo "ERROR $test_name execution failure. See $test_logfile"
+    return 1
+  fi
+
+  orig_checksum=$(gh-ost-test-mysql-replica test -e "select ${columns} from gh_ost_test" -ss | md5sum)
+  ghost_checksum=$(gh-ost-test-mysql-replica test -e "select ${columns} from _gh_ost_test_gho" -ss | md5sum)
+
+  if [ "$orig_checksum" != "$ghost_checksum" ] ; then
+    echo "ERROR $test_name: checksum mismatch"
+    echo "---"
+    gh-ost-test-mysql-replica test -e "select ${columns} from gh_ost_test" -ss
+    echo "---"
+    gh-ost-test-mysql-replica test -e "select ${columns} from _gh_ost_test_gho" -ss
+    return 1
+  fi
+}
+
+test_all() {
+  find $tests_path ! -path . -type d -mindepth 1 -maxdepth 1 | cut -d "/" -f 3 | while read test_name ; do
+    test_single "$test_name"
+    if [ $? -ne 0 ] ; then
+      echo "+ FAIL"
+      return 1
+    else
+      echo "+ pass"
+    fi
+    gh-ost-test-mysql-replica -e "start slave"
+  done
+}
+
+verify_master_and_replica
+test_all

localtests/tz/create.sql

@@ -0,0 +1,41 @@
+drop table if exists gh_ost_test;
+create table gh_ost_test (
+  id int auto_increment,
+  i int not null,
+  ts0 timestamp default current_timestamp,
+  ts1 timestamp,
+  ts2 timestamp,
+  updated tinyint unsigned default 0,
+  primary key(id),
+  key i_idx(i)
+) auto_increment=1;
+
+drop event if exists gh_ost_test;
+delimiter ;;
+create event gh_ost_test
+  on schedule every 1 second
+  starts current_timestamp
+  ends current_timestamp + interval 60 second
+  on completion not preserve
+  enable
+  do
+begin
+  insert into gh_ost_test values (null, 11, null, now(), now(), 0);
+  update gh_ost_test set ts2=now() + interval 10 minute, updated = 1 where i = 11 order by id desc limit 1;
+
+  set session time_zone='system';
+  insert into gh_ost_test values (null, 13, null, now(), now(), 0);
+  update gh_ost_test set ts2=now() + interval 10 minute, updated = 1 where i = 13 order by id desc limit 1;
+
+  set session time_zone='+00:00';
+  insert into gh_ost_test values (null, 17, null, now(), now(), 0);
+  update gh_ost_test set ts2=now() + interval 10 minute, updated = 1 where i = 17 order by id desc limit 1;
+
+  set session time_zone='-03:00';
+  insert into gh_ost_test values (null, 19, null, now(), now(), 0);
+  update gh_ost_test set ts2=now() + interval 10 minute, updated = 1 where i = 19 order by id desc limit 1;
+
+  set session time_zone='+05:00';
+  insert into gh_ost_test values (null, 23, null, now(), now(), 0);
+  update gh_ost_test set ts2=now() + interval 10 minute, updated = 1 where i = 23 order by id desc limit 1;
+end ;;

localtests/unsigned/create.sql

@@ -0,0 +1,24 @@
+drop table if exists gh_ost_test;
+create table gh_ost_test (
+  id int auto_increment,
+  i int not null,
+  bi bigint not null,
+  iu int unsigned not null,
+  biu bigint unsigned not null,
+  primary key(id)
+) auto_increment=1;
+
+drop event if exists gh_ost_test;
+delimiter ;;
+create event gh_ost_test
+  on schedule every 1 second
+  starts current_timestamp
+  ends current_timestamp + interval 60 second
+  on completion not preserve
+  enable
+  do
+begin
+  insert into gh_ost_test values (null, -2147483647, -9223372036854775807, 4294967295, 18446744073709551615);
+  set @last_insert_id := cast(last_insert_id() as signed);
+  update gh_ost_test set i=-2147483647+@last_insert_id, bi=-9223372036854775807+@last_insert_id, iu=4294967295-@last_insert_id, biu=18446744073709551615-@last_insert_id where id < @last_insert_id order by id desc limit 1;
+end ;;

vendor/github.com/siddontang/go-mysql/replication/row_event.go

@@ -609,7 +609,7 @@ func decodeTimestamp2(data []byte, dec uint16) (string, int, error) {
 		return "0000-00-00 00:00:00", n, nil
 	}
 
-	t := time.Unix(sec, usec*1000)
+	t := time.Unix(sec, usec*1000).UTC()
 	return t.Format(TimeFormat), n, nil
 }