Merge branch 'master' into row-copy-complete

doc/local-tests.md

@@ -0,0 +1,22 @@
+# Local tests
+
+`gh-ost` is continuously tested in production via `--test-on-replica alter='engine=innodb'`. These tests check the GitHub workload and usage, but not necessarily the general case.
+
+Local tests are an additional layer of tests. They will eventually be part of continuous integration tests.
+
+Local tests test explicit use cases, such as column renames, mix of time zones, special types and alters. Traits of a single test:
+
+- Composed of a single table.
+- A single alter.
+- By default the alter is `engine=innodb`, but this can be overridden per-test
+- Scheduled DML operations, executed via `event_scheduler`.
+- `gh-ost` is set to execute and throttle for `5` seconds, at which time all tested DMLs are expected to operate.
+- The test requires a replication topology and utilizes `--test-on-replica`
+- The test checksums the two tables (original and _ghost_) and expects identical checksum
+- By default the test selects all (`*`) columns, but this can be overridden per-test
+
+Tests are found under [localtests](https://github.com/github/gh-ost/tree/master/localtests). A single test is a subdirectory and tests are iterated alphabetically.
+
+New data-integrity, synchronization issues or otherwise concerns are expected to be tested by new test cases.
+
+While this is merged work is still ongoing.

go/base/context.go

@@ -83,13 +83,14 @@ type MigrationContext struct {
 	ServeSocketFile string
 	ServeTCPPort    int64
 
-	Noop                    bool
-	TestOnReplica           bool
-	MigrateOnReplica        bool
-	OkToDropTable           bool
-	InitiallyDropOldTable   bool
-	InitiallyDropGhostTable bool
-	CutOverType             CutOver
+	Noop                         bool
+	TestOnReplica                bool
+	MigrateOnReplica             bool
+	TestOnReplicaSkipReplicaStop bool
+	OkToDropTable                bool
+	InitiallyDropOldTable        bool
+	InitiallyDropGhostTable      bool
+	CutOverType                  CutOver
 
 	TableEngine               string
 	RowsEstimate              int64

go/cmd/gh-ost/main.go

@@ -61,6 +61,7 @@ func main() {
 
 	executeFlag := flag.Bool("execute", false, "actually execute the alter & migrate the table. Default is noop: do some tests and exit")
 	flag.BoolVar(&migrationContext.TestOnReplica, "test-on-replica", false, "Have the migration run on a replica, not on the master. At the end of migration replication is stopped, and tables are swapped and immediately swap-revert. Replication remains stopped and you can compare the two tables for building trust")
+	flag.BoolVar(&migrationContext.TestOnReplicaSkipReplicaStop, "test-on-replica-skip-replica-stop", false, "When --test-on-replica is enabled, do not issue commands stop replication (requires --test-on-replica)")
 	flag.BoolVar(&migrationContext.MigrateOnReplica, "migrate-on-replica", false, "Have the migration run on a replica, not on the master. This will do the full migration on the replica including cut-over (as opposed to --test-on-replica)")
 
 	flag.BoolVar(&migrationContext.OkToDropTable, "ok-to-drop-table", false, "Shall the tool drop the old table at end of operation. DROPping tables can be a long locking operation, which is why I'm not doing it by default. I'm an online tool, yes?")
@@ -149,6 +150,13 @@ func main() {
 	if migrationContext.SwitchToRowBinlogFormat && migrationContext.AssumeRBR {
 		log.Fatalf("--switch-to-rbr and --assume-rbr are mutually exclusive")
 	}
+	if migrationContext.TestOnReplicaSkipReplicaStop {
+		if !migrationContext.TestOnReplica {
+			log.Fatalf("--test-on-replica-skip-replica-stop requires --test-on-replica to be enabled")
+		}
+		log.Warning("--test-on-replica-skip-replica-stop enabled. We will not stop replication before cut-over. Ensure you have a plugin that does this.")
+	}
+
 	switch *cutOver {
 	case "atomic", "default", "":
 		migrationContext.CutOverType = base.CutOverAtomic

go/logic/applier.go

@@ -574,6 +574,7 @@ func (this *Applier) StopReplication() error {
 	if err := this.StopSlaveSQLThread(); err != nil {
 		return err
 	}
+
 	readBinlogCoordinates, executeBinlogCoordinates, err := mysql.GetReplicationBinlogCoordinates(this.db)
 	if err != nil {
 		return err
@@ -870,7 +871,10 @@ func (this *Applier) ApplyDMLEventQuery(dmlEvent *binlog.BinlogDMLEvent) error {
 		if err != nil {
 			return err
 		}
-		if _, err := tx.Exec("SET SESSION time_zone = '+00:00'"); err != nil {
+		if _, err := tx.Exec(`SET
+			SESSION time_zone = '+00:00',
+			sql_mode = CONCAT(@@session.sql_mode, ',STRICT_ALL_TABLES')
+			`); err != nil {
 			return err
 		}
 		if _, err := tx.Exec(query, args...); err != nil {

go/logic/migrator.go

@@ -478,9 +478,14 @@ func (this *Migrator) cutOver() (err error) {
 		// the same cut-over phase as the master would use. That means we take locks
 		// and swap the tables.
 		// The difference is that we will later swap the tables back.
-		log.Debugf("testing on replica. Stopping replication IO thread")
-		if err := this.retryOperation(this.applier.StopReplication); err != nil {
-			return err
+
+		if this.migrationContext.TestOnReplicaSkipReplicaStop {
+			log.Warningf("--test-on-replica-skip-replica-stop enabled, we are not stopping replication.")
+		} else {
+			log.Debugf("testing on replica. Stopping replication IO thread")
+			if err := this.retryOperation(this.applier.StopReplication); err != nil {
+				return err
+			}
 		}
 		// We're merly testing, we don't want to keep this state. Rollback the renames as possible
 		defer this.applier.RenameTablesRollback()

localtests/enum/create.sql

@@ -0,0 +1,27 @@
+drop table if exists gh_ost_test;
+create table gh_ost_test (
+  id int auto_increment,
+  i int not null,
+  e enum('red', 'green', 'blue', 'orange') null default null collate 'utf8_bin',
+  primary key(id)
+) auto_increment=1;
+
+drop event if exists gh_ost_test;
+delimiter ;;
+create event gh_ost_test
+  on schedule every 1 second
+  starts current_timestamp
+  ends current_timestamp + interval 60 second
+  on completion not preserve
+  enable
+  do
+begin
+  insert into gh_ost_test values (null, 11, 'red');
+  insert into gh_ost_test values (null, 13, 'green');
+  insert into gh_ost_test values (null, 17, 'blue');
+  set @last_insert_id := last_insert_id();
+  update gh_ost_test set e='orange' where id = @last_insert_id;
+  insert into gh_ost_test values (null, 23, null);
+  set @last_insert_id := last_insert_id();
+  update gh_ost_test set i=i+1, e=null where id = @last_insert_id;
+end ;;

localtests/enum/extra_args

@@ -0,0 +1 @@
+--alter="change e e enum('red', 'green', 'blue', 'orange', 'yellow') null default null collate 'utf8_bin'" 

localtests/rename/create.sql

@@ -0,0 +1,26 @@
+drop table if exists gh_ost_test;
+create table gh_ost_test (
+  id int auto_increment,
+  c1 int not null,
+  c2 int not null,
+  primary key (id)
+) auto_increment=1;
+
+drop event if exists gh_ost_test;
+delimiter ;;
+create event gh_ost_test
+  on schedule every 1 second
+  starts current_timestamp
+  ends current_timestamp + interval 60 second
+  on completion not preserve
+  enable
+  do
+begin
+  insert ignore into gh_ost_test values (1, 11, 23);
+  insert ignore into gh_ost_test values (2, 13, 23);
+  insert into gh_ost_test values (null, 17, 23);
+  set @last_insert_id := last_insert_id();
+  update gh_ost_test set c1=c1+@last_insert_id, c2=c2+@last_insert_id where id=@last_insert_id order by id desc limit 1;
+  delete from gh_ost_test where id=1;
+  delete from gh_ost_test where c1=13; -- id=2
+end ;;

localtests/rename/extra_args

@@ -0,0 +1 @@
+--alter="change column c2 c3 int not null" --approve-renamed-columns

localtests/test.sh

@@ -0,0 +1,112 @@
+#!/bin/bash
+
+# Local integration tests. To be used by CI.
+# See https://github.com/github/gh-ost/tree/doc/local-tests.md
+#
+
+tests_path=$(dirname $0)
+test_logfile=/tmp/gh-ost-test.log
+exec_command_file=/tmp/gh-ost-test.bash
+
+master_host=
+master_port=
+replica_host=
+replica_port=
+
+verify_master_and_replica() {
+  if [ "$(gh-ost-test-mysql-master -e "select 1" -ss)" != "1" ] ; then
+    echo "Cannot verify gh-ost-test-mysql-master"
+    exit 1
+  fi
+  read master_host master_port <<< $(gh-ost-test-mysql-master -e "select @@hostname, @@port" -ss)
+  if [ "$(gh-ost-test-mysql-replica -e "select 1" -ss)" != "1" ] ; then
+    echo "Cannot verify gh-ost-test-mysql-replica"
+    exit 1
+  fi
+  read replica_host replica_port <<< $(gh-ost-test-mysql-replica -e "select @@hostname, @@port" -ss)
+}
+
+exec_cmd() {
+  echo "$@"
+  command "$@" 1> $test_logfile 2>&1
+  return $?
+}
+
+test_single() {
+  local test_name
+  test_name="$1"
+
+  echo "Testing: $test_name"
+
+  gh-ost-test-mysql-replica -e "start slave"
+  gh-ost-test-mysql-master test < $tests_path/$test_name/create.sql
+
+  extra_args=""
+  if [ -f $tests_path/$test_name/extra_args ] ; then
+    extra_args=$(cat $tests_path/$test_name/extra_args)
+  fi
+  columns="*"
+  if [ -f $tests_path/$test_name/test_columns ] ; then
+    columns=$(cat $tests_path/$test_name/test_columns)
+  fi
+  # graceful sleep for replica to catch up
+  sleep 1
+  #
+  cmd="go run go/cmd/gh-ost/main.go \
+    --user=gh-ost \
+    --password=gh-ost \
+    --host=$replica_host \
+    --port=$replica_port \
+    --database=test \
+    --table=gh_ost_test \
+    --alter='engine=innodb' \
+    --exact-rowcount \
+    --switch-to-rbr \
+    --initially-drop-old-table \
+    --initially-drop-ghost-table \
+    --throttle-query='select timestampdiff(second, min(last_update), now()) < 5 from _gh_ost_test_ghc' \
+    --serve-socket-file=/tmp/gh-ost.test.sock \
+    --initially-drop-socket-file \
+    --postpone-cut-over-flag-file=/tmp/gh-ost.postpone.flag \
+    --test-on-replica \
+    --default-retries=1 \
+    --verbose \
+    --debug \
+    --stack \
+    --execute ${extra_args[@]}"
+  echo $cmd > $exec_command_file
+  bash $exec_command_file 1> $test_logfile 2>&1
+
+  if [ $? -ne 0 ] ; then
+    echo "ERROR $test_name execution failure. See $test_logfile"
+    return 1
+  fi
+
+  orig_checksum=$(gh-ost-test-mysql-replica test -e "select ${columns} from gh_ost_test" -ss | md5sum)
+  ghost_checksum=$(gh-ost-test-mysql-replica test -e "select ${columns} from _gh_ost_test_gho" -ss | md5sum)
+
+  if [ "$orig_checksum" != "$ghost_checksum" ] ; then
+    echo "ERROR $test_name: checksum mismatch"
+    echo "---"
+    gh-ost-test-mysql-replica test -e "select ${columns} from gh_ost_test" -ss
+    echo "---"
+    gh-ost-test-mysql-replica test -e "select ${columns} from _gh_ost_test_gho" -ss
+    return 1
+  fi
+}
+
+test_all() {
+  find $tests_path ! -path . -type d -mindepth 1 -maxdepth 1 | cut -d "/" -f 3 | while read test_name ; do
+    test_single "$test_name"
+    if [ $? -ne 0 ] ; then
+      echo "+ FAIL"
+      return 1
+    else
+      echo "+ pass"
+    fi
+    gh-ost-test-mysql-replica -e "start slave"
+  done
+}
+
+verify_master_and_replica
+test_all

localtests/tz/create.sql

@@ -0,0 +1,41 @@
+drop table if exists gh_ost_test;
+create table gh_ost_test (
+  id int auto_increment,
+  i int not null,
+  ts0 timestamp default current_timestamp,
+  ts1 timestamp,
+  ts2 timestamp,
+  updated tinyint unsigned default 0,
+  primary key(id),
+  key i_idx(i)
+) auto_increment=1;
+
+drop event if exists gh_ost_test;
+delimiter ;;
+create event gh_ost_test
+  on schedule every 1 second
+  starts current_timestamp
+  ends current_timestamp + interval 60 second
+  on completion not preserve
+  enable
+  do
+begin
+  insert into gh_ost_test values (null, 11, null, now(), now(), 0);
+  update gh_ost_test set ts2=now() + interval 10 minute, updated = 1 where i = 11 order by id desc limit 1;
+
+  set session time_zone='system';
+  insert into gh_ost_test values (null, 13, null, now(), now(), 0);
+  update gh_ost_test set ts2=now() + interval 10 minute, updated = 1 where i = 13 order by id desc limit 1;
+
+  set session time_zone='+00:00';
+  insert into gh_ost_test values (null, 17, null, now(), now(), 0);
+  update gh_ost_test set ts2=now() + interval 10 minute, updated = 1 where i = 17 order by id desc limit 1;
+
+  set session time_zone='-03:00';
+  insert into gh_ost_test values (null, 19, null, now(), now(), 0);
+  update gh_ost_test set ts2=now() + interval 10 minute, updated = 1 where i = 19 order by id desc limit 1;
+
+  set session time_zone='+05:00';
+  insert into gh_ost_test values (null, 23, null, now(), now(), 0);
+  update gh_ost_test set ts2=now() + interval 10 minute, updated = 1 where i = 23 order by id desc limit 1;
+end ;;

localtests/unsigned/create.sql

@@ -0,0 +1,24 @@
+drop table if exists gh_ost_test;
+create table gh_ost_test (
+  id int auto_increment,
+  i int not null,
+  bi bigint not null,
+  iu int unsigned not null,
+  biu bigint unsigned not null,
+  primary key(id)
+) auto_increment=1;
+
+drop event if exists gh_ost_test;
+delimiter ;;
+create event gh_ost_test
+  on schedule every 1 second
+  starts current_timestamp
+  ends current_timestamp + interval 60 second
+  on completion not preserve
+  enable
+  do
+begin
+  insert into gh_ost_test values (null, -2147483647, -9223372036854775807, 4294967295, 18446744073709551615);
+  set @last_insert_id := cast(last_insert_id() as signed);
+  update gh_ost_test set i=-2147483647+@last_insert_id, bi=-9223372036854775807+@last_insert_id, iu=4294967295-@last_insert_id, biu=18446744073709551615-@last_insert_id where id < @last_insert_id order by id desc limit 1;
+end ;;