Merge pull request #185 from github/local-tests

WIP: Local tests

doc/local-tests.md

@@ -0,0 +1,22 @@
+# Local tests
+
+`gh-ost` is continuously tested in production via `--test-on-replica alter='engine=innodb'`. These tests check the GitHub workload and usage, but not necessarily the general case.
+
+Local tests are an additional layer of tests. They will eventually be part of continuous integration tests.
+
+Local tests test explicit use cases, such as column renames, mix of time zones, special types and alters. Traits of a single test:
+
+- Composed of a single table.
+- A single alter.
+- By default the alter is `engine=innodb`, but this can be overridden per-test
+- Scheduled DML operations, executed via `event_scheduler`.
+- `gh-ost` is set to execute and throttle for `5` seconds, at which time all tested DMLs are expected to operate.
+- The test requires a replication topology and utilizes `--test-on-replica`
+- The test checksums the two tables (original and _ghost_) and expects identical checksum
+- By default the test selects all (`*`) columns, but this can be overridden per-test
+
+Tests are found under [localtests](https://github.com/github/gh-ost/tree/master/localtests). A single test is a subdirectory and tests are iterated alphabetically.
+
+New data-integrity, synchronization issues or otherwise concerns are expected to be tested by new test cases.
+
+While this is merged work is still ongoing.

localtests/enum/create.sql

@@ -0,0 +1,27 @@
+drop table if exists gh_ost_test;
+create table gh_ost_test (
+  id int auto_increment,
+  i int not null,
+  e enum('red', 'green', 'blue', 'orange') null default null collate 'utf8_bin',
+  primary key(id)
+) auto_increment=1;
+
+drop event if exists gh_ost_test;
+delimiter ;;
+create event gh_ost_test
+  on schedule every 1 second
+  starts current_timestamp
+  ends current_timestamp + interval 60 second
+  on completion not preserve
+  enable
+  do
+begin
+  insert into gh_ost_test values (null, 11, 'red');
+  insert into gh_ost_test values (null, 13, 'green');
+  insert into gh_ost_test values (null, 17, 'blue');
+  set @last_insert_id := last_insert_id();
+  update gh_ost_test set e='orange' where id = @last_insert_id;
+  insert into gh_ost_test values (null, 23, null);
+  set @last_insert_id := last_insert_id();
+  update gh_ost_test set i=i+1, e=null where id = @last_insert_id;
+end ;;

localtests/enum/extra_args

@@ -0,0 +1 @@
+--alter="change e e enum('red', 'green', 'blue', 'orange', 'yellow') null default null collate 'utf8_bin'" 

localtests/rename/create.sql

@@ -0,0 +1,26 @@
+drop table if exists gh_ost_test;
+create table gh_ost_test (
+  id int auto_increment,
+  c1 int not null,
+  c2 int not null,
+  primary key (id)
+) auto_increment=1;
+
+drop event if exists gh_ost_test;
+delimiter ;;
+create event gh_ost_test
+  on schedule every 1 second
+  starts current_timestamp
+  ends current_timestamp + interval 60 second
+  on completion not preserve
+  enable
+  do
+begin
+  insert ignore into gh_ost_test values (1, 11, 23);
+  insert ignore into gh_ost_test values (2, 13, 23);
+  insert into gh_ost_test values (null, 17, 23);
+  set @last_insert_id := last_insert_id();
+  update gh_ost_test set c1=c1+@last_insert_id, c2=c2+@last_insert_id where id=@last_insert_id order by id desc limit 1;
+  delete from gh_ost_test where id=1;
+  delete from gh_ost_test where c1=13; -- id=2
+end ;;

localtests/rename/extra_args

@@ -0,0 +1 @@
+--alter="change column c2 c3 int not null" --approve-renamed-columns

localtests/test.sh

@@ -0,0 +1,112 @@
+#!/bin/bash
+
+# Local integration tests. To be used by CI.
+# See https://github.com/github/gh-ost/tree/doc/local-tests.md
+#
+
+tests_path=$(dirname $0)
+test_logfile=/tmp/gh-ost-test.log
+exec_command_file=/tmp/gh-ost-test.bash
+
+master_host=
+master_port=
+replica_host=
+replica_port=
+
+verify_master_and_replica() {
+  if [ "$(gh-ost-test-mysql-master -e "select 1" -ss)" != "1" ] ; then
+    echo "Cannot verify gh-ost-test-mysql-master"
+    exit 1
+  fi
+  read master_host master_port <<< $(gh-ost-test-mysql-master -e "select @@hostname, @@port" -ss)
+  if [ "$(gh-ost-test-mysql-replica -e "select 1" -ss)" != "1" ] ; then
+    echo "Cannot verify gh-ost-test-mysql-replica"
+    exit 1
+  fi
+  read replica_host replica_port <<< $(gh-ost-test-mysql-replica -e "select @@hostname, @@port" -ss)
+}
+
+exec_cmd() {
+  echo "$@"
+  command "$@" 1> $test_logfile 2>&1
+  return $?
+}
+
+test_single() {
+  local test_name
+  test_name="$1"
+
+  echo "Testing: $test_name"
+
+  gh-ost-test-mysql-replica -e "start slave"
+  gh-ost-test-mysql-master test < $tests_path/$test_name/create.sql
+
+  extra_args=""
+  if [ -f $tests_path/$test_name/extra_args ] ; then
+    extra_args=$(cat $tests_path/$test_name/extra_args)
+  fi
+  columns="*"
+  if [ -f $tests_path/$test_name/test_columns ] ; then
+    columns=$(cat $tests_path/$test_name/test_columns)
+  fi
+  # graceful sleep for replica to catch up
+  sleep 1
+  #
+  cmd="go run go/cmd/gh-ost/main.go \
+    --user=gh-ost \
+    --password=gh-ost \
+    --host=$replica_host \
+    --port=$replica_port \
+    --database=test \
+    --table=gh_ost_test \
+    --alter='engine=innodb' \
+    --exact-rowcount \
+    --switch-to-rbr \
+    --initially-drop-old-table \
+    --initially-drop-ghost-table \
+    --throttle-query='select timestampdiff(second, min(last_update), now()) < 5 from _gh_ost_test_ghc' \
+    --serve-socket-file=/tmp/gh-ost.test.sock \
+    --initially-drop-socket-file \
+    --postpone-cut-over-flag-file=/tmp/gh-ost.postpone.flag \
+    --test-on-replica \
+    --default-retries=1 \
+    --verbose \
+    --debug \
+    --stack \
+    --execute ${extra_args[@]}"
+  echo $cmd > $exec_command_file
+  bash $exec_command_file 1> $test_logfile 2>&1
+
+  if [ $? -ne 0 ] ; then
+    echo "ERROR $test_name execution failure. See $test_logfile"
+    return 1
+  fi
+
+  orig_checksum=$(gh-ost-test-mysql-replica test -e "select ${columns} from gh_ost_test" -ss | md5sum)
+  ghost_checksum=$(gh-ost-test-mysql-replica test -e "select ${columns} from _gh_ost_test_gho" -ss | md5sum)
+
+  if [ "$orig_checksum" != "$ghost_checksum" ] ; then
+    echo "ERROR $test_name: checksum mismatch"
+    echo "---"
+    gh-ost-test-mysql-replica test -e "select ${columns} from gh_ost_test" -ss
+    echo "---"
+    gh-ost-test-mysql-replica test -e "select ${columns} from _gh_ost_test_gho" -ss
+    return 1
+  fi
+}
+
+test_all() {
+  find $tests_path ! -path . -type d -mindepth 1 -maxdepth 1 | cut -d "/" -f 3 | while read test_name ; do
+    test_single "$test_name"
+    if [ $? -ne 0 ] ; then
+      echo "+ FAIL"
+      return 1
+    else
+      echo "+ pass"
+    fi
+    gh-ost-test-mysql-replica -e "start slave"
+  done
+}
+
+verify_master_and_replica
+test_all

localtests/tz/create.sql

@@ -0,0 +1,41 @@
+drop table if exists gh_ost_test;
+create table gh_ost_test (
+  id int auto_increment,
+  i int not null,
+  ts0 timestamp default current_timestamp,
+  ts1 timestamp,
+  ts2 timestamp,
+  updated tinyint unsigned default 0,
+  primary key(id),
+  key i_idx(i)
+) auto_increment=1;
+
+drop event if exists gh_ost_test;
+delimiter ;;
+create event gh_ost_test
+  on schedule every 1 second
+  starts current_timestamp
+  ends current_timestamp + interval 60 second
+  on completion not preserve
+  enable
+  do
+begin
+  insert into gh_ost_test values (null, 11, null, now(), now(), 0);
+  update gh_ost_test set ts2=now() + interval 10 minute, updated = 1 where i = 11 order by id desc limit 1;
+
+  set session time_zone='system';
+  insert into gh_ost_test values (null, 13, null, now(), now(), 0);
+  update gh_ost_test set ts2=now() + interval 10 minute, updated = 1 where i = 13 order by id desc limit 1;
+
+  set session time_zone='+00:00';
+  insert into gh_ost_test values (null, 17, null, now(), now(), 0);
+  update gh_ost_test set ts2=now() + interval 10 minute, updated = 1 where i = 17 order by id desc limit 1;
+
+  set session time_zone='-03:00';
+  insert into gh_ost_test values (null, 19, null, now(), now(), 0);
+  update gh_ost_test set ts2=now() + interval 10 minute, updated = 1 where i = 19 order by id desc limit 1;
+
+  set session time_zone='+05:00';
+  insert into gh_ost_test values (null, 23, null, now(), now(), 0);
+  update gh_ost_test set ts2=now() + interval 10 minute, updated = 1 where i = 23 order by id desc limit 1;
+end ;;

localtests/unsigned/create.sql

@@ -0,0 +1,24 @@
+drop table if exists gh_ost_test;
+create table gh_ost_test (
+  id int auto_increment,
+  i int not null,
+  bi bigint not null,
+  iu int unsigned not null,
+  biu bigint unsigned not null,
+  primary key(id)
+) auto_increment=1;
+
+drop event if exists gh_ost_test;
+delimiter ;;
+create event gh_ost_test
+  on schedule every 1 second
+  starts current_timestamp
+  ends current_timestamp + interval 60 second
+  on completion not preserve
+  enable
+  do
+begin
+  insert into gh_ost_test values (null, -2147483647, -9223372036854775807, 4294967295, 18446744073709551615);
+  set @last_insert_id := cast(last_insert_id() as signed);
+  update gh_ost_test set i=-2147483647+@last_insert_id, bi=-9223372036854775807+@last_insert_id, iu=4294967295-@last_insert_id, biu=18446744073709551615-@last_insert_id where id < @last_insert_id order by id desc limit 1;
+end ;;