Merge branch 'master' into row-copy-complete
This commit is contained in:
commit
c7d88499af
22
doc/local-tests.md
Normal file
22
doc/local-tests.md
Normal file
@ -0,0 +1,22 @@
|
|||||||
|
# Local tests
|
||||||
|
|
||||||
|
`gh-ost` is continuously tested in production via `--test-on-replica alter='engine=innodb'`. These tests check the GitHub workload and usage, but not necessarily the general case.
|
||||||
|
|
||||||
|
Local tests are an additional layer of tests. They will eventually be part of continuous integration tests.
|
||||||
|
|
||||||
|
Local tests test explicit use cases, such as column renames, mix of time zones, special types and alters. Traits of a single test:
|
||||||
|
|
||||||
|
- Composed of a single table.
|
||||||
|
- A single alter.
|
||||||
|
- By default the alter is `engine=innodb`, but this can be overridden per-test
|
||||||
|
- Scheduled DML operations, executed via `event_scheduler`.
|
||||||
|
- `gh-ost` is set to execute and throttle for `5` seconds, at which time all tested DMLs are expected to operate.
|
||||||
|
- The test requires a replication topology and utilizes `--test-on-replica`
|
||||||
|
- The test checksums the two tables (original and _ghost_) and expects identical checksum
|
||||||
|
- By default the test selects all (`*`) columns, but this can be overridden per-test
|
||||||
|
|
||||||
|
Tests are found under [localtests](https://github.com/github/gh-ost/tree/master/localtests). A single test is a subdirectory and tests are iterated alphabetically.
|
||||||
|
|
||||||
|
New data-integrity, synchronization issues or otherwise concerns are expected to be tested by new test cases.
|
||||||
|
|
||||||
|
While this is merged work is still ongoing.
|
@ -83,13 +83,14 @@ type MigrationContext struct {
|
|||||||
ServeSocketFile string
|
ServeSocketFile string
|
||||||
ServeTCPPort int64
|
ServeTCPPort int64
|
||||||
|
|
||||||
Noop bool
|
Noop bool
|
||||||
TestOnReplica bool
|
TestOnReplica bool
|
||||||
MigrateOnReplica bool
|
MigrateOnReplica bool
|
||||||
OkToDropTable bool
|
TestOnReplicaSkipReplicaStop bool
|
||||||
InitiallyDropOldTable bool
|
OkToDropTable bool
|
||||||
InitiallyDropGhostTable bool
|
InitiallyDropOldTable bool
|
||||||
CutOverType CutOver
|
InitiallyDropGhostTable bool
|
||||||
|
CutOverType CutOver
|
||||||
|
|
||||||
TableEngine string
|
TableEngine string
|
||||||
RowsEstimate int64
|
RowsEstimate int64
|
||||||
|
@ -61,6 +61,7 @@ func main() {
|
|||||||
|
|
||||||
executeFlag := flag.Bool("execute", false, "actually execute the alter & migrate the table. Default is noop: do some tests and exit")
|
executeFlag := flag.Bool("execute", false, "actually execute the alter & migrate the table. Default is noop: do some tests and exit")
|
||||||
flag.BoolVar(&migrationContext.TestOnReplica, "test-on-replica", false, "Have the migration run on a replica, not on the master. At the end of migration replication is stopped, and tables are swapped and immediately swap-revert. Replication remains stopped and you can compare the two tables for building trust")
|
flag.BoolVar(&migrationContext.TestOnReplica, "test-on-replica", false, "Have the migration run on a replica, not on the master. At the end of migration replication is stopped, and tables are swapped and immediately swap-revert. Replication remains stopped and you can compare the two tables for building trust")
|
||||||
|
flag.BoolVar(&migrationContext.TestOnReplicaSkipReplicaStop, "test-on-replica-skip-replica-stop", false, "When --test-on-replica is enabled, do not issue commands stop replication (requires --test-on-replica)")
|
||||||
flag.BoolVar(&migrationContext.MigrateOnReplica, "migrate-on-replica", false, "Have the migration run on a replica, not on the master. This will do the full migration on the replica including cut-over (as opposed to --test-on-replica)")
|
flag.BoolVar(&migrationContext.MigrateOnReplica, "migrate-on-replica", false, "Have the migration run on a replica, not on the master. This will do the full migration on the replica including cut-over (as opposed to --test-on-replica)")
|
||||||
|
|
||||||
flag.BoolVar(&migrationContext.OkToDropTable, "ok-to-drop-table", false, "Shall the tool drop the old table at end of operation. DROPping tables can be a long locking operation, which is why I'm not doing it by default. I'm an online tool, yes?")
|
flag.BoolVar(&migrationContext.OkToDropTable, "ok-to-drop-table", false, "Shall the tool drop the old table at end of operation. DROPping tables can be a long locking operation, which is why I'm not doing it by default. I'm an online tool, yes?")
|
||||||
@ -149,6 +150,13 @@ func main() {
|
|||||||
if migrationContext.SwitchToRowBinlogFormat && migrationContext.AssumeRBR {
|
if migrationContext.SwitchToRowBinlogFormat && migrationContext.AssumeRBR {
|
||||||
log.Fatalf("--switch-to-rbr and --assume-rbr are mutually exclusive")
|
log.Fatalf("--switch-to-rbr and --assume-rbr are mutually exclusive")
|
||||||
}
|
}
|
||||||
|
if migrationContext.TestOnReplicaSkipReplicaStop {
|
||||||
|
if !migrationContext.TestOnReplica {
|
||||||
|
log.Fatalf("--test-on-replica-skip-replica-stop requires --test-on-replica to be enabled")
|
||||||
|
}
|
||||||
|
log.Warning("--test-on-replica-skip-replica-stop enabled. We will not stop replication before cut-over. Ensure you have a plugin that does this.")
|
||||||
|
}
|
||||||
|
|
||||||
switch *cutOver {
|
switch *cutOver {
|
||||||
case "atomic", "default", "":
|
case "atomic", "default", "":
|
||||||
migrationContext.CutOverType = base.CutOverAtomic
|
migrationContext.CutOverType = base.CutOverAtomic
|
||||||
|
@ -574,6 +574,7 @@ func (this *Applier) StopReplication() error {
|
|||||||
if err := this.StopSlaveSQLThread(); err != nil {
|
if err := this.StopSlaveSQLThread(); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
readBinlogCoordinates, executeBinlogCoordinates, err := mysql.GetReplicationBinlogCoordinates(this.db)
|
readBinlogCoordinates, executeBinlogCoordinates, err := mysql.GetReplicationBinlogCoordinates(this.db)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@ -870,7 +871,10 @@ func (this *Applier) ApplyDMLEventQuery(dmlEvent *binlog.BinlogDMLEvent) error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if _, err := tx.Exec("SET SESSION time_zone = '+00:00'"); err != nil {
|
if _, err := tx.Exec(`SET
|
||||||
|
SESSION time_zone = '+00:00',
|
||||||
|
sql_mode = CONCAT(@@session.sql_mode, ',STRICT_ALL_TABLES')
|
||||||
|
`); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if _, err := tx.Exec(query, args...); err != nil {
|
if _, err := tx.Exec(query, args...); err != nil {
|
||||||
|
@ -478,9 +478,14 @@ func (this *Migrator) cutOver() (err error) {
|
|||||||
// the same cut-over phase as the master would use. That means we take locks
|
// the same cut-over phase as the master would use. That means we take locks
|
||||||
// and swap the tables.
|
// and swap the tables.
|
||||||
// The difference is that we will later swap the tables back.
|
// The difference is that we will later swap the tables back.
|
||||||
log.Debugf("testing on replica. Stopping replication IO thread")
|
|
||||||
if err := this.retryOperation(this.applier.StopReplication); err != nil {
|
if this.migrationContext.TestOnReplicaSkipReplicaStop {
|
||||||
return err
|
log.Warningf("--test-on-replica-skip-replica-stop enabled, we are not stopping replication.")
|
||||||
|
} else {
|
||||||
|
log.Debugf("testing on replica. Stopping replication IO thread")
|
||||||
|
if err := this.retryOperation(this.applier.StopReplication); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
}
|
}
|
||||||
// We're merly testing, we don't want to keep this state. Rollback the renames as possible
|
// We're merly testing, we don't want to keep this state. Rollback the renames as possible
|
||||||
defer this.applier.RenameTablesRollback()
|
defer this.applier.RenameTablesRollback()
|
||||||
|
27
localtests/enum/create.sql
Normal file
27
localtests/enum/create.sql
Normal file
@ -0,0 +1,27 @@
|
|||||||
|
drop table if exists gh_ost_test;
|
||||||
|
create table gh_ost_test (
|
||||||
|
id int auto_increment,
|
||||||
|
i int not null,
|
||||||
|
e enum('red', 'green', 'blue', 'orange') null default null collate 'utf8_bin',
|
||||||
|
primary key(id)
|
||||||
|
) auto_increment=1;
|
||||||
|
|
||||||
|
drop event if exists gh_ost_test;
|
||||||
|
delimiter ;;
|
||||||
|
create event gh_ost_test
|
||||||
|
on schedule every 1 second
|
||||||
|
starts current_timestamp
|
||||||
|
ends current_timestamp + interval 60 second
|
||||||
|
on completion not preserve
|
||||||
|
enable
|
||||||
|
do
|
||||||
|
begin
|
||||||
|
insert into gh_ost_test values (null, 11, 'red');
|
||||||
|
insert into gh_ost_test values (null, 13, 'green');
|
||||||
|
insert into gh_ost_test values (null, 17, 'blue');
|
||||||
|
set @last_insert_id := last_insert_id();
|
||||||
|
update gh_ost_test set e='orange' where id = @last_insert_id;
|
||||||
|
insert into gh_ost_test values (null, 23, null);
|
||||||
|
set @last_insert_id := last_insert_id();
|
||||||
|
update gh_ost_test set i=i+1, e=null where id = @last_insert_id;
|
||||||
|
end ;;
|
1
localtests/enum/extra_args
Normal file
1
localtests/enum/extra_args
Normal file
@ -0,0 +1 @@
|
|||||||
|
--alter="change e e enum('red', 'green', 'blue', 'orange', 'yellow') null default null collate 'utf8_bin'"
|
26
localtests/rename/create.sql
Normal file
26
localtests/rename/create.sql
Normal file
@ -0,0 +1,26 @@
|
|||||||
|
drop table if exists gh_ost_test;
|
||||||
|
create table gh_ost_test (
|
||||||
|
id int auto_increment,
|
||||||
|
c1 int not null,
|
||||||
|
c2 int not null,
|
||||||
|
primary key (id)
|
||||||
|
) auto_increment=1;
|
||||||
|
|
||||||
|
drop event if exists gh_ost_test;
|
||||||
|
delimiter ;;
|
||||||
|
create event gh_ost_test
|
||||||
|
on schedule every 1 second
|
||||||
|
starts current_timestamp
|
||||||
|
ends current_timestamp + interval 60 second
|
||||||
|
on completion not preserve
|
||||||
|
enable
|
||||||
|
do
|
||||||
|
begin
|
||||||
|
insert ignore into gh_ost_test values (1, 11, 23);
|
||||||
|
insert ignore into gh_ost_test values (2, 13, 23);
|
||||||
|
insert into gh_ost_test values (null, 17, 23);
|
||||||
|
set @last_insert_id := last_insert_id();
|
||||||
|
update gh_ost_test set c1=c1+@last_insert_id, c2=c2+@last_insert_id where id=@last_insert_id order by id desc limit 1;
|
||||||
|
delete from gh_ost_test where id=1;
|
||||||
|
delete from gh_ost_test where c1=13; -- id=2
|
||||||
|
end ;;
|
1
localtests/rename/extra_args
Normal file
1
localtests/rename/extra_args
Normal file
@ -0,0 +1 @@
|
|||||||
|
--alter="change column c2 c3 int not null" --approve-renamed-columns
|
112
localtests/test.sh
Executable file
112
localtests/test.sh
Executable file
@ -0,0 +1,112 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
# Local integration tests. To be used by CI.
|
||||||
|
# See https://github.com/github/gh-ost/tree/doc/local-tests.md
|
||||||
|
#
|
||||||
|
|
||||||
|
tests_path=$(dirname $0)
|
||||||
|
test_logfile=/tmp/gh-ost-test.log
|
||||||
|
exec_command_file=/tmp/gh-ost-test.bash
|
||||||
|
|
||||||
|
master_host=
|
||||||
|
master_port=
|
||||||
|
replica_host=
|
||||||
|
replica_port=
|
||||||
|
|
||||||
|
verify_master_and_replica() {
|
||||||
|
if [ "$(gh-ost-test-mysql-master -e "select 1" -ss)" != "1" ] ; then
|
||||||
|
echo "Cannot verify gh-ost-test-mysql-master"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
read master_host master_port <<< $(gh-ost-test-mysql-master -e "select @@hostname, @@port" -ss)
|
||||||
|
if [ "$(gh-ost-test-mysql-replica -e "select 1" -ss)" != "1" ] ; then
|
||||||
|
echo "Cannot verify gh-ost-test-mysql-replica"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
read replica_host replica_port <<< $(gh-ost-test-mysql-replica -e "select @@hostname, @@port" -ss)
|
||||||
|
}
|
||||||
|
|
||||||
|
exec_cmd() {
|
||||||
|
echo "$@"
|
||||||
|
command "$@" 1> $test_logfile 2>&1
|
||||||
|
return $?
|
||||||
|
}
|
||||||
|
|
||||||
|
test_single() {
|
||||||
|
local test_name
|
||||||
|
test_name="$1"
|
||||||
|
|
||||||
|
echo "Testing: $test_name"
|
||||||
|
|
||||||
|
gh-ost-test-mysql-replica -e "start slave"
|
||||||
|
gh-ost-test-mysql-master test < $tests_path/$test_name/create.sql
|
||||||
|
|
||||||
|
extra_args=""
|
||||||
|
if [ -f $tests_path/$test_name/extra_args ] ; then
|
||||||
|
extra_args=$(cat $tests_path/$test_name/extra_args)
|
||||||
|
fi
|
||||||
|
columns="*"
|
||||||
|
if [ -f $tests_path/$test_name/test_columns ] ; then
|
||||||
|
columns=$(cat $tests_path/$test_name/test_columns)
|
||||||
|
fi
|
||||||
|
# graceful sleep for replica to catch up
|
||||||
|
sleep 1
|
||||||
|
#
|
||||||
|
cmd="go run go/cmd/gh-ost/main.go \
|
||||||
|
--user=gh-ost \
|
||||||
|
--password=gh-ost \
|
||||||
|
--host=$replica_host \
|
||||||
|
--port=$replica_port \
|
||||||
|
--database=test \
|
||||||
|
--table=gh_ost_test \
|
||||||
|
--alter='engine=innodb' \
|
||||||
|
--exact-rowcount \
|
||||||
|
--switch-to-rbr \
|
||||||
|
--initially-drop-old-table \
|
||||||
|
--initially-drop-ghost-table \
|
||||||
|
--throttle-query='select timestampdiff(second, min(last_update), now()) < 5 from _gh_ost_test_ghc' \
|
||||||
|
--serve-socket-file=/tmp/gh-ost.test.sock \
|
||||||
|
--initially-drop-socket-file \
|
||||||
|
--postpone-cut-over-flag-file=/tmp/gh-ost.postpone.flag \
|
||||||
|
--test-on-replica \
|
||||||
|
--default-retries=1 \
|
||||||
|
--verbose \
|
||||||
|
--debug \
|
||||||
|
--stack \
|
||||||
|
--execute ${extra_args[@]}"
|
||||||
|
echo $cmd > $exec_command_file
|
||||||
|
bash $exec_command_file 1> $test_logfile 2>&1
|
||||||
|
|
||||||
|
if [ $? -ne 0 ] ; then
|
||||||
|
echo "ERROR $test_name execution failure. See $test_logfile"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
orig_checksum=$(gh-ost-test-mysql-replica test -e "select ${columns} from gh_ost_test" -ss | md5sum)
|
||||||
|
ghost_checksum=$(gh-ost-test-mysql-replica test -e "select ${columns} from _gh_ost_test_gho" -ss | md5sum)
|
||||||
|
|
||||||
|
if [ "$orig_checksum" != "$ghost_checksum" ] ; then
|
||||||
|
echo "ERROR $test_name: checksum mismatch"
|
||||||
|
echo "---"
|
||||||
|
gh-ost-test-mysql-replica test -e "select ${columns} from gh_ost_test" -ss
|
||||||
|
echo "---"
|
||||||
|
gh-ost-test-mysql-replica test -e "select ${columns} from _gh_ost_test_gho" -ss
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
test_all() {
|
||||||
|
find $tests_path ! -path . -type d -mindepth 1 -maxdepth 1 | cut -d "/" -f 3 | while read test_name ; do
|
||||||
|
test_single "$test_name"
|
||||||
|
if [ $? -ne 0 ] ; then
|
||||||
|
echo "+ FAIL"
|
||||||
|
return 1
|
||||||
|
else
|
||||||
|
echo "+ pass"
|
||||||
|
fi
|
||||||
|
gh-ost-test-mysql-replica -e "start slave"
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
verify_master_and_replica
|
||||||
|
test_all
|
41
localtests/tz/create.sql
Normal file
41
localtests/tz/create.sql
Normal file
@ -0,0 +1,41 @@
|
|||||||
|
drop table if exists gh_ost_test;
|
||||||
|
create table gh_ost_test (
|
||||||
|
id int auto_increment,
|
||||||
|
i int not null,
|
||||||
|
ts0 timestamp default current_timestamp,
|
||||||
|
ts1 timestamp,
|
||||||
|
ts2 timestamp,
|
||||||
|
updated tinyint unsigned default 0,
|
||||||
|
primary key(id),
|
||||||
|
key i_idx(i)
|
||||||
|
) auto_increment=1;
|
||||||
|
|
||||||
|
drop event if exists gh_ost_test;
|
||||||
|
delimiter ;;
|
||||||
|
create event gh_ost_test
|
||||||
|
on schedule every 1 second
|
||||||
|
starts current_timestamp
|
||||||
|
ends current_timestamp + interval 60 second
|
||||||
|
on completion not preserve
|
||||||
|
enable
|
||||||
|
do
|
||||||
|
begin
|
||||||
|
insert into gh_ost_test values (null, 11, null, now(), now(), 0);
|
||||||
|
update gh_ost_test set ts2=now() + interval 10 minute, updated = 1 where i = 11 order by id desc limit 1;
|
||||||
|
|
||||||
|
set session time_zone='system';
|
||||||
|
insert into gh_ost_test values (null, 13, null, now(), now(), 0);
|
||||||
|
update gh_ost_test set ts2=now() + interval 10 minute, updated = 1 where i = 13 order by id desc limit 1;
|
||||||
|
|
||||||
|
set session time_zone='+00:00';
|
||||||
|
insert into gh_ost_test values (null, 17, null, now(), now(), 0);
|
||||||
|
update gh_ost_test set ts2=now() + interval 10 minute, updated = 1 where i = 17 order by id desc limit 1;
|
||||||
|
|
||||||
|
set session time_zone='-03:00';
|
||||||
|
insert into gh_ost_test values (null, 19, null, now(), now(), 0);
|
||||||
|
update gh_ost_test set ts2=now() + interval 10 minute, updated = 1 where i = 19 order by id desc limit 1;
|
||||||
|
|
||||||
|
set session time_zone='+05:00';
|
||||||
|
insert into gh_ost_test values (null, 23, null, now(), now(), 0);
|
||||||
|
update gh_ost_test set ts2=now() + interval 10 minute, updated = 1 where i = 23 order by id desc limit 1;
|
||||||
|
end ;;
|
24
localtests/unsigned/create.sql
Normal file
24
localtests/unsigned/create.sql
Normal file
@ -0,0 +1,24 @@
|
|||||||
|
drop table if exists gh_ost_test;
|
||||||
|
create table gh_ost_test (
|
||||||
|
id int auto_increment,
|
||||||
|
i int not null,
|
||||||
|
bi bigint not null,
|
||||||
|
iu int unsigned not null,
|
||||||
|
biu bigint unsigned not null,
|
||||||
|
primary key(id)
|
||||||
|
) auto_increment=1;
|
||||||
|
|
||||||
|
drop event if exists gh_ost_test;
|
||||||
|
delimiter ;;
|
||||||
|
create event gh_ost_test
|
||||||
|
on schedule every 1 second
|
||||||
|
starts current_timestamp
|
||||||
|
ends current_timestamp + interval 60 second
|
||||||
|
on completion not preserve
|
||||||
|
enable
|
||||||
|
do
|
||||||
|
begin
|
||||||
|
insert into gh_ost_test values (null, -2147483647, -9223372036854775807, 4294967295, 18446744073709551615);
|
||||||
|
set @last_insert_id := cast(last_insert_id() as signed);
|
||||||
|
update gh_ost_test set i=-2147483647+@last_insert_id, bi=-9223372036854775807+@last_insert_id, iu=4294967295-@last_insert_id, biu=18446744073709551615-@last_insert_id where id < @last_insert_id order by id desc limit 1;
|
||||||
|
end ;;
|
Loading…
Reference in New Issue
Block a user