diff --git a/doc/local-tests.md b/doc/local-tests.md new file mode 100644 index 0000000..1614ee4 --- /dev/null +++ b/doc/local-tests.md @@ -0,0 +1,22 @@ +# Local tests + +`gh-ost` is continuously tested in production via `--test-on-replica alter='engine=innodb'`. These tests check the GitHub workload and usage, but not necessarily the general case. + +Local tests are an additional layer of tests. They will eventually be part of continuous integration tests. + +Local tests test explicit use cases, such as column renames, mix of time zones, special types and alters. Traits of a single test: + +- Composed of a single table. +- A single alter. +- By default the alter is `engine=innodb`, but this can be overridden per-test +- Scheduled DML operations, executed via `event_scheduler`. +- `gh-ost` is set to execute and throttle for `5` seconds, at which time all tested DMLs are expected to operate. +- The test requires a replication topology and utilizes `--test-on-replica` +- The test checksums the two tables (original and _ghost_) and expects identical checksum +- By default the test selects all (`*`) columns, but this can be overridden per-test + +Tests are found under [localtests](https://github.com/github/gh-ost/tree/master/localtests). A single test is a subdirectory and tests are iterated alphabetically. + +New data-integrity, synchronization issues or otherwise concerns are expected to be tested by new test cases. + +While this is merged work is still ongoing. diff --git a/go/base/context.go b/go/base/context.go index ea5ab68..dfc69bb 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -83,13 +83,14 @@ type MigrationContext struct { ServeSocketFile string ServeTCPPort int64 - Noop bool - TestOnReplica bool - MigrateOnReplica bool - OkToDropTable bool - InitiallyDropOldTable bool - InitiallyDropGhostTable bool - CutOverType CutOver + Noop bool + TestOnReplica bool + MigrateOnReplica bool + TestOnReplicaSkipReplicaStop bool + OkToDropTable bool + InitiallyDropOldTable bool + InitiallyDropGhostTable bool + CutOverType CutOver TableEngine string RowsEstimate int64 diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index b614831..512c537 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -61,6 +61,7 @@ func main() { executeFlag := flag.Bool("execute", false, "actually execute the alter & migrate the table. Default is noop: do some tests and exit") flag.BoolVar(&migrationContext.TestOnReplica, "test-on-replica", false, "Have the migration run on a replica, not on the master. At the end of migration replication is stopped, and tables are swapped and immediately swap-revert. Replication remains stopped and you can compare the two tables for building trust") + flag.BoolVar(&migrationContext.TestOnReplicaSkipReplicaStop, "test-on-replica-skip-replica-stop", false, "When --test-on-replica is enabled, do not issue commands stop replication (requires --test-on-replica)") flag.BoolVar(&migrationContext.MigrateOnReplica, "migrate-on-replica", false, "Have the migration run on a replica, not on the master. This will do the full migration on the replica including cut-over (as opposed to --test-on-replica)") flag.BoolVar(&migrationContext.OkToDropTable, "ok-to-drop-table", false, "Shall the tool drop the old table at end of operation. DROPping tables can be a long locking operation, which is why I'm not doing it by default. I'm an online tool, yes?") @@ -149,6 +150,13 @@ func main() { if migrationContext.SwitchToRowBinlogFormat && migrationContext.AssumeRBR { log.Fatalf("--switch-to-rbr and --assume-rbr are mutually exclusive") } + if migrationContext.TestOnReplicaSkipReplicaStop { + if !migrationContext.TestOnReplica { + log.Fatalf("--test-on-replica-skip-replica-stop requires --test-on-replica to be enabled") + } + log.Warning("--test-on-replica-skip-replica-stop enabled. We will not stop replication before cut-over. Ensure you have a plugin that does this.") + } + switch *cutOver { case "atomic", "default", "": migrationContext.CutOverType = base.CutOverAtomic diff --git a/go/logic/applier.go b/go/logic/applier.go index a122b71..14d74e1 100644 --- a/go/logic/applier.go +++ b/go/logic/applier.go @@ -574,6 +574,7 @@ func (this *Applier) StopReplication() error { if err := this.StopSlaveSQLThread(); err != nil { return err } + readBinlogCoordinates, executeBinlogCoordinates, err := mysql.GetReplicationBinlogCoordinates(this.db) if err != nil { return err @@ -870,7 +871,10 @@ func (this *Applier) ApplyDMLEventQuery(dmlEvent *binlog.BinlogDMLEvent) error { if err != nil { return err } - if _, err := tx.Exec("SET SESSION time_zone = '+00:00'"); err != nil { + if _, err := tx.Exec(`SET + SESSION time_zone = '+00:00', + sql_mode = CONCAT(@@session.sql_mode, ',STRICT_ALL_TABLES') + `); err != nil { return err } if _, err := tx.Exec(query, args...); err != nil { diff --git a/go/logic/migrator.go b/go/logic/migrator.go index 63f0a33..e4e278f 100644 --- a/go/logic/migrator.go +++ b/go/logic/migrator.go @@ -478,9 +478,14 @@ func (this *Migrator) cutOver() (err error) { // the same cut-over phase as the master would use. That means we take locks // and swap the tables. // The difference is that we will later swap the tables back. - log.Debugf("testing on replica. Stopping replication IO thread") - if err := this.retryOperation(this.applier.StopReplication); err != nil { - return err + + if this.migrationContext.TestOnReplicaSkipReplicaStop { + log.Warningf("--test-on-replica-skip-replica-stop enabled, we are not stopping replication.") + } else { + log.Debugf("testing on replica. Stopping replication IO thread") + if err := this.retryOperation(this.applier.StopReplication); err != nil { + return err + } } // We're merly testing, we don't want to keep this state. Rollback the renames as possible defer this.applier.RenameTablesRollback() diff --git a/localtests/enum/create.sql b/localtests/enum/create.sql new file mode 100644 index 0000000..6420233 --- /dev/null +++ b/localtests/enum/create.sql @@ -0,0 +1,27 @@ +drop table if exists gh_ost_test; +create table gh_ost_test ( + id int auto_increment, + i int not null, + e enum('red', 'green', 'blue', 'orange') null default null collate 'utf8_bin', + primary key(id) +) auto_increment=1; + +drop event if exists gh_ost_test; +delimiter ;; +create event gh_ost_test + on schedule every 1 second + starts current_timestamp + ends current_timestamp + interval 60 second + on completion not preserve + enable + do +begin + insert into gh_ost_test values (null, 11, 'red'); + insert into gh_ost_test values (null, 13, 'green'); + insert into gh_ost_test values (null, 17, 'blue'); + set @last_insert_id := last_insert_id(); + update gh_ost_test set e='orange' where id = @last_insert_id; + insert into gh_ost_test values (null, 23, null); + set @last_insert_id := last_insert_id(); + update gh_ost_test set i=i+1, e=null where id = @last_insert_id; +end ;; diff --git a/localtests/enum/extra_args b/localtests/enum/extra_args new file mode 100644 index 0000000..f369a56 --- /dev/null +++ b/localtests/enum/extra_args @@ -0,0 +1 @@ +--alter="change e e enum('red', 'green', 'blue', 'orange', 'yellow') null default null collate 'utf8_bin'" diff --git a/localtests/rename/create.sql b/localtests/rename/create.sql new file mode 100644 index 0000000..d28a7c1 --- /dev/null +++ b/localtests/rename/create.sql @@ -0,0 +1,26 @@ +drop table if exists gh_ost_test; +create table gh_ost_test ( + id int auto_increment, + c1 int not null, + c2 int not null, + primary key (id) +) auto_increment=1; + +drop event if exists gh_ost_test; +delimiter ;; +create event gh_ost_test + on schedule every 1 second + starts current_timestamp + ends current_timestamp + interval 60 second + on completion not preserve + enable + do +begin + insert ignore into gh_ost_test values (1, 11, 23); + insert ignore into gh_ost_test values (2, 13, 23); + insert into gh_ost_test values (null, 17, 23); + set @last_insert_id := last_insert_id(); + update gh_ost_test set c1=c1+@last_insert_id, c2=c2+@last_insert_id where id=@last_insert_id order by id desc limit 1; + delete from gh_ost_test where id=1; + delete from gh_ost_test where c1=13; -- id=2 +end ;; diff --git a/localtests/rename/extra_args b/localtests/rename/extra_args new file mode 100644 index 0000000..d36d5ee --- /dev/null +++ b/localtests/rename/extra_args @@ -0,0 +1 @@ +--alter="change column c2 c3 int not null" --approve-renamed-columns diff --git a/localtests/test.sh b/localtests/test.sh new file mode 100755 index 0000000..ee82623 --- /dev/null +++ b/localtests/test.sh @@ -0,0 +1,112 @@ +#!/bin/bash + +# Local integration tests. To be used by CI. +# See https://github.com/github/gh-ost/tree/doc/local-tests.md +# + +tests_path=$(dirname $0) +test_logfile=/tmp/gh-ost-test.log +exec_command_file=/tmp/gh-ost-test.bash + +master_host= +master_port= +replica_host= +replica_port= + +verify_master_and_replica() { + if [ "$(gh-ost-test-mysql-master -e "select 1" -ss)" != "1" ] ; then + echo "Cannot verify gh-ost-test-mysql-master" + exit 1 + fi + read master_host master_port <<< $(gh-ost-test-mysql-master -e "select @@hostname, @@port" -ss) + if [ "$(gh-ost-test-mysql-replica -e "select 1" -ss)" != "1" ] ; then + echo "Cannot verify gh-ost-test-mysql-replica" + exit 1 + fi + read replica_host replica_port <<< $(gh-ost-test-mysql-replica -e "select @@hostname, @@port" -ss) +} + +exec_cmd() { + echo "$@" + command "$@" 1> $test_logfile 2>&1 + return $? +} + +test_single() { + local test_name + test_name="$1" + + echo "Testing: $test_name" + + gh-ost-test-mysql-replica -e "start slave" + gh-ost-test-mysql-master test < $tests_path/$test_name/create.sql + + extra_args="" + if [ -f $tests_path/$test_name/extra_args ] ; then + extra_args=$(cat $tests_path/$test_name/extra_args) + fi + columns="*" + if [ -f $tests_path/$test_name/test_columns ] ; then + columns=$(cat $tests_path/$test_name/test_columns) + fi + # graceful sleep for replica to catch up + sleep 1 + # + cmd="go run go/cmd/gh-ost/main.go \ + --user=gh-ost \ + --password=gh-ost \ + --host=$replica_host \ + --port=$replica_port \ + --database=test \ + --table=gh_ost_test \ + --alter='engine=innodb' \ + --exact-rowcount \ + --switch-to-rbr \ + --initially-drop-old-table \ + --initially-drop-ghost-table \ + --throttle-query='select timestampdiff(second, min(last_update), now()) < 5 from _gh_ost_test_ghc' \ + --serve-socket-file=/tmp/gh-ost.test.sock \ + --initially-drop-socket-file \ + --postpone-cut-over-flag-file=/tmp/gh-ost.postpone.flag \ + --test-on-replica \ + --default-retries=1 \ + --verbose \ + --debug \ + --stack \ + --execute ${extra_args[@]}" + echo $cmd > $exec_command_file + bash $exec_command_file 1> $test_logfile 2>&1 + + if [ $? -ne 0 ] ; then + echo "ERROR $test_name execution failure. See $test_logfile" + return 1 + fi + + orig_checksum=$(gh-ost-test-mysql-replica test -e "select ${columns} from gh_ost_test" -ss | md5sum) + ghost_checksum=$(gh-ost-test-mysql-replica test -e "select ${columns} from _gh_ost_test_gho" -ss | md5sum) + + if [ "$orig_checksum" != "$ghost_checksum" ] ; then + echo "ERROR $test_name: checksum mismatch" + echo "---" + gh-ost-test-mysql-replica test -e "select ${columns} from gh_ost_test" -ss + echo "---" + gh-ost-test-mysql-replica test -e "select ${columns} from _gh_ost_test_gho" -ss + return 1 + fi +} + +test_all() { + find $tests_path ! -path . -type d -mindepth 1 -maxdepth 1 | cut -d "/" -f 3 | while read test_name ; do + test_single "$test_name" + if [ $? -ne 0 ] ; then + echo "+ FAIL" + return 1 + else + echo "+ pass" + fi + gh-ost-test-mysql-replica -e "start slave" + done +} + +verify_master_and_replica +test_all diff --git a/localtests/tz/create.sql b/localtests/tz/create.sql new file mode 100644 index 0000000..f908e6e --- /dev/null +++ b/localtests/tz/create.sql @@ -0,0 +1,41 @@ +drop table if exists gh_ost_test; +create table gh_ost_test ( + id int auto_increment, + i int not null, + ts0 timestamp default current_timestamp, + ts1 timestamp, + ts2 timestamp, + updated tinyint unsigned default 0, + primary key(id), + key i_idx(i) +) auto_increment=1; + +drop event if exists gh_ost_test; +delimiter ;; +create event gh_ost_test + on schedule every 1 second + starts current_timestamp + ends current_timestamp + interval 60 second + on completion not preserve + enable + do +begin + insert into gh_ost_test values (null, 11, null, now(), now(), 0); + update gh_ost_test set ts2=now() + interval 10 minute, updated = 1 where i = 11 order by id desc limit 1; + + set session time_zone='system'; + insert into gh_ost_test values (null, 13, null, now(), now(), 0); + update gh_ost_test set ts2=now() + interval 10 minute, updated = 1 where i = 13 order by id desc limit 1; + + set session time_zone='+00:00'; + insert into gh_ost_test values (null, 17, null, now(), now(), 0); + update gh_ost_test set ts2=now() + interval 10 minute, updated = 1 where i = 17 order by id desc limit 1; + + set session time_zone='-03:00'; + insert into gh_ost_test values (null, 19, null, now(), now(), 0); + update gh_ost_test set ts2=now() + interval 10 minute, updated = 1 where i = 19 order by id desc limit 1; + + set session time_zone='+05:00'; + insert into gh_ost_test values (null, 23, null, now(), now(), 0); + update gh_ost_test set ts2=now() + interval 10 minute, updated = 1 where i = 23 order by id desc limit 1; +end ;; diff --git a/localtests/unsigned/create.sql b/localtests/unsigned/create.sql new file mode 100644 index 0000000..d98bb07 --- /dev/null +++ b/localtests/unsigned/create.sql @@ -0,0 +1,24 @@ +drop table if exists gh_ost_test; +create table gh_ost_test ( + id int auto_increment, + i int not null, + bi bigint not null, + iu int unsigned not null, + biu bigint unsigned not null, + primary key(id) +) auto_increment=1; + +drop event if exists gh_ost_test; +delimiter ;; +create event gh_ost_test + on schedule every 1 second + starts current_timestamp + ends current_timestamp + interval 60 second + on completion not preserve + enable + do +begin + insert into gh_ost_test values (null, -2147483647, -9223372036854775807, 4294967295, 18446744073709551615); + set @last_insert_id := cast(last_insert_id() as signed); + update gh_ost_test set i=-2147483647+@last_insert_id, bi=-9223372036854775807+@last_insert_id, iu=4294967295-@last_insert_id, biu=18446744073709551615-@last_insert_id where id < @last_insert_id order by id desc limit 1; +end ;;