diff --git a/.travis.yml b/.travis.yml index 3006208..92cb79c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,13 +8,22 @@ os: env: - MYSQL_USER=root +- CURRENT_CI_ENV=travis + +addons: + apt: + packages: + - git + - numactl + - libaio1 before_install: - mysql -e 'CREATE DATABASE IF NOT EXISTS test;' install: true -script: script/cibuild +script: + - script/cibuild notifications: email: false diff --git a/README.md b/README.md index 0d896bd..2138109 100644 --- a/README.md +++ b/README.md @@ -107,3 +107,5 @@ Generally speaking, `master` branch is stable, but only [releases](https://githu - [@ggunson](https://github.com/ggunson) - [@tomkrouper](https://github.com/tomkrouper) - [@shlomi-noach](https://github.com/shlomi-noach) +- [@jessbreckenridge](https://github.com/jessbreckenridge) +- [@gtowey](https://github.com/gtowey) diff --git a/RELEASE_VERSION b/RELEASE_VERSION index 4ad595c..6ae122f 100644 --- a/RELEASE_VERSION +++ b/RELEASE_VERSION @@ -1 +1 @@ -1.0.42 +1.0.44 diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index 130cff0..b509a25 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -44,7 +44,6 @@ func acceptSignals(migrationContext *base.MigrationContext) { // main is the application's entry point. It will either spawn a CLI or HTTP interfaces. func main() { migrationContext := base.NewMigrationContext() - flag.StringVar(&migrationContext.InspectorConnectionConfig.Key.Hostname, "host", "127.0.0.1", "MySQL hostname (preferably a replica, not the master)") flag.StringVar(&migrationContext.AssumeMasterHostname, "assume-master-host", "", "(optional) explicitly tell gh-ost the identity of the master. Format: some.host.com[:port] This is useful in master-master setups where you wish to pick an explicit master, or in a tungsten-replicator where gh-ost is unable to determine the master") flag.IntVar(&migrationContext.InspectorConnectionConfig.Key.Port, "port", 3306, "MySQL port (preferably a replica, not the master)") diff --git a/go/logic/server.go b/go/logic/server.go index b9903e2..919f178 100644 --- a/go/logic/server.go +++ b/go/logic/server.go @@ -130,6 +130,9 @@ func (this *Server) applyServerCommand(command string, writer *bufio.Writer) (pr arg := "" if len(tokens) > 1 { arg = strings.TrimSpace(tokens[1]) + if unquoted, err := strconv.Unquote(arg); err == nil { + arg = unquoted + } } argIsQuestion := (arg == "?") throttleHint := "# Note: you may only throttle for as long as your binary logs are not purged\n" diff --git a/localtests/datetime-submillis-zeroleading/ignore_versions b/localtests/datetime-submillis-zeroleading/ignore_versions new file mode 100644 index 0000000..7acd3f0 --- /dev/null +++ b/localtests/datetime-submillis-zeroleading/ignore_versions @@ -0,0 +1 @@ +(5.5) diff --git a/localtests/datetime-submillis/ignore_versions b/localtests/datetime-submillis/ignore_versions new file mode 100644 index 0000000..7acd3f0 --- /dev/null +++ b/localtests/datetime-submillis/ignore_versions @@ -0,0 +1 @@ +(5.5) diff --git a/localtests/datetime-to-timestamp-pk-fail/ignore_versions b/localtests/datetime-to-timestamp-pk-fail/ignore_versions new file mode 100644 index 0000000..7acd3f0 --- /dev/null +++ b/localtests/datetime-to-timestamp-pk-fail/ignore_versions @@ -0,0 +1 @@ +(5.5) diff --git a/localtests/datetime/ignore_versions b/localtests/datetime/ignore_versions new file mode 100644 index 0000000..7acd3f0 --- /dev/null +++ b/localtests/datetime/ignore_versions @@ -0,0 +1 @@ +(5.5) diff --git a/localtests/json57/ignore_versions b/localtests/json57/ignore_versions new file mode 100644 index 0000000..b6de5f8 --- /dev/null +++ b/localtests/json57/ignore_versions @@ -0,0 +1 @@ +(5.5|5.6) diff --git a/localtests/json57dml/ignore_versions b/localtests/json57dml/ignore_versions new file mode 100644 index 0000000..b6de5f8 --- /dev/null +++ b/localtests/json57dml/ignore_versions @@ -0,0 +1 @@ +(5.5|5.6) diff --git a/localtests/swap-pk-uk/ignore_versions b/localtests/swap-pk-uk/ignore_versions new file mode 100644 index 0000000..7acd3f0 --- /dev/null +++ b/localtests/swap-pk-uk/ignore_versions @@ -0,0 +1 @@ +(5.5) diff --git a/localtests/swap-uk-uk/ignore_versions b/localtests/swap-uk-uk/ignore_versions new file mode 100644 index 0000000..7acd3f0 --- /dev/null +++ b/localtests/swap-uk-uk/ignore_versions @@ -0,0 +1 @@ +(5.5) diff --git a/localtests/test.sh b/localtests/test.sh index 477ecbb..8d5ee35 100755 --- a/localtests/test.sh +++ b/localtests/test.sh @@ -9,23 +9,38 @@ tests_path=$(dirname $0) test_logfile=/tmp/gh-ost-test.log -ghost_binary=/tmp/gh-ost-test +default_ghost_binary=/tmp/gh-ost-test +ghost_binary="" exec_command_file=/tmp/gh-ost-test.bash -orig_content_output_file=/gh-ost-test.orig.content.csv -ghost_content_output_file=/gh-ost-test.ghost.content.csv -test_pattern="${1:-.}" +orig_content_output_file=/tmp/gh-ost-test.orig.content.csv +ghost_content_output_file=/tmp/gh-ost-test.ghost.content.csv master_host= master_port= replica_host= replica_port= +OPTIND=1 +while getopts "b:" OPTION +do + case $OPTION in + b) + ghost_binary="$OPTARG" + ;; + esac +done +shift $((OPTIND-1)) + +test_pattern="${1:-.}" + verify_master_and_replica() { if [ "$(gh-ost-test-mysql-master -e "select 1" -ss)" != "1" ] ; then echo "Cannot verify gh-ost-test-mysql-master" exit 1 fi read master_host master_port <<< $(gh-ost-test-mysql-master -e "select @@hostname, @@port" -ss) + [ "$master_host" == "$(hostname)" ] && master_host="127.0.0.1" + echo "# master verified at $master_host:$master_port" if [ "$(gh-ost-test-mysql-replica -e "select 1" -ss)" != "1" ] ; then echo "Cannot verify gh-ost-test-mysql-replica" exit 1 @@ -35,6 +50,8 @@ verify_master_and_replica() { exit 1 fi read replica_host replica_port <<< $(gh-ost-test-mysql-replica -e "select @@hostname, @@port" -ss) + [ "$replica_host" == "$(hostname)" ] && replica_host="127.0.0.1" + echo "# replica verified at $replica_host:$replica_port" } exec_cmd() { @@ -66,6 +83,15 @@ test_single() { local test_name test_name="$1" + if [ -f $tests_path/$test_name/ignore_versions ] ; then + ignore_versions=$(cat $tests_path/$test_name/ignore_versions) + mysql_version=$(gh-ost-test-mysql-master -s -s -e "select @@version") + if echo "$mysql_version" | egrep -q "^${ignore_versions}" ; then + echo -n "Skipping: $test_name" + return 0 + fi + fi + echo -n "Testing: $test_name" echo_dot @@ -98,6 +124,7 @@ test_single() { --password=gh-ost \ --host=$replica_host \ --port=$replica_port \ + --assume-master-host=${master_host}:${master_port} --database=test \ --table=gh_ost_test \ --alter='engine=innodb' \ @@ -148,7 +175,8 @@ test_single() { if [ $execution_result -ne 0 ] ; then echo - echo "ERROR $test_name execution failure. cat $test_logfile" + echo "ERROR $test_name execution failure. cat $test_logfile:" + cat $test_logfile return 1 fi @@ -170,7 +198,12 @@ test_single() { build_binary() { echo "Building" - rm -f $ghost_binary + rm -f $default_ghost_binary + [ "$ghost_binary" == "" ] && ghost_binary="$default_ghost_binary" + if [ -f "$ghost_binary" ] ; then + echo "Using binary: $ghost_binary" + return 0 + fi go build -o $ghost_binary go/cmd/gh-ost/main.go if [ $? -ne 0 ] ; then echo "Build failure" diff --git a/localtests/timestamp-to-datetime/ignore_versions b/localtests/timestamp-to-datetime/ignore_versions new file mode 100644 index 0000000..7acd3f0 --- /dev/null +++ b/localtests/timestamp-to-datetime/ignore_versions @@ -0,0 +1 @@ +(5.5) diff --git a/localtests/timestamp/ignore_versions b/localtests/timestamp/ignore_versions new file mode 100644 index 0000000..7acd3f0 --- /dev/null +++ b/localtests/timestamp/ignore_versions @@ -0,0 +1 @@ +(5.5) diff --git a/localtests/tz-datetime-ts/ignore_versions b/localtests/tz-datetime-ts/ignore_versions new file mode 100644 index 0000000..7acd3f0 --- /dev/null +++ b/localtests/tz-datetime-ts/ignore_versions @@ -0,0 +1 @@ +(5.5) diff --git a/localtests/tz/ignore_versions b/localtests/tz/ignore_versions new file mode 100644 index 0000000..7acd3f0 --- /dev/null +++ b/localtests/tz/ignore_versions @@ -0,0 +1 @@ +(5.5) diff --git a/script/cibuild-gh-ost-replica-tests b/script/cibuild-gh-ost-replica-tests new file mode 100755 index 0000000..ab8c689 --- /dev/null +++ b/script/cibuild-gh-ost-replica-tests @@ -0,0 +1,66 @@ +#!/bin/bash + +set -e + +whoami + +# Clone gh-ost-ci-env +# Only clone if not already running locally at latest commit +remote_commit=$(git ls-remote https://github.com/github/gh-ost-ci-env.git HEAD | cut -f1) +local_commit="unknown" +[ -d "gh-ost-ci-env" ] && local_commit=$(cd gh-ost-ci-env && git log --format="%H" -n 1) + +echo "remote commit is: $remote_commit" +echo "local commit is: $local_commit" + +if [ "$remote_commit" != "$local_commit" ] ; then + rm -rf ./gh-ost-ci-env + git clone https://github.com/github/gh-ost-ci-env.git +fi + +test_mysql_version() { + local mysql_version + mysql_version="$1" + + echo "##### Testing $mysql_version" + + echo "### Setting up sandbox for $mysql_version" + + find sandboxes -name "stop_all" | bash + + mkdir -p sandbox/binary + rm -rf sandbox/binary/* + gh-ost-ci-env/bin/linux/dbdeployer unpack gh-ost-ci-env/mysql-tarballs/"$mysql_version".tar.gz --unpack-version="$mysql_version" --sandbox-binary ${PWD}/sandbox/binary + + mkdir -p sandboxes + rm -rf sandboxes/* + + if echo "$mysql_version" | egrep "5[.]5[.]" ; then + gtid="" + else + gtid="--gtid" + fi + gh-ost-ci-env/bin/linux/dbdeployer deploy replication "$mysql_version" --nodes 2 --sandbox-binary ${PWD}/sandbox/binary --sandbox-home ${PWD}/sandboxes ${gtid} --my-cnf-options log_slave_updates --my-cnf-options log_bin --my-cnf-options binlog_format=ROW --sandbox-directory rsandbox + + sed '/sandboxes/d' -i gh-ost-ci-env/bin/gh-ost-test-mysql-master + echo 'sandboxes/rsandbox/m "$@"' >> gh-ost-ci-env/bin/gh-ost-test-mysql-master + + sed '/sandboxes/d' -i gh-ost-ci-env/bin/gh-ost-test-mysql-replica + echo 'sandboxes/rsandbox/s1 "$@"' >> gh-ost-ci-env/bin/gh-ost-test-mysql-replica + + export PATH="${PWD}/gh-ost-ci-env/bin/:${PATH}" + + gh-ost-test-mysql-master -uroot -e "grant all on *.* to 'gh-ost'@'%' identified by 'gh-ost'" + + echo "### Running gh-ost tests for $mysql_version" + ./localtests/test.sh -b bin/gh-ost + + find sandboxes -name "stop_all" | bash +} + +echo "Building..." +. script/build +# Test all versions: +find gh-ost-ci-env/mysql-tarballs/ -name "*.tar.gz" | while read f ; do basename $f ".tar.gz" ; done | sort -r | while read mysql_version ; do + test_mysql_version "$mysql_version" +done