Merge branch 'master' into logging-interface-contrib

.gitignore

@@ -2,3 +2,4 @@
 /bin/
 /libexec/
 /.vendor/
+.idea/

.travis.yml

@@ -1,33 +0,0 @@
-# http://docs.travis-ci.com/user/languages/go/
-language: go
-
-go:
-  - "1.12.x"
-
-os:
-  - linux
-
-services:
-  - mysql
-
-env:
-- MYSQL_USER=root
-- CURRENT_CI_ENV=travis
-
-addons:
-  apt:
-    packages:
-    - git
-    - numactl
-    - libaio1
-
-before_install:
-  - mysql -e 'CREATE DATABASE IF NOT EXISTS test;'
-
-install: true
-
-script:
-  - script/cibuild
-
-notifications:
-  email: false

Dockerfile.packaging

@@ -0,0 +1,22 @@
+#
+
+FROM golang:1.12.6
+
+RUN apt-get update
+RUN apt-get install -y ruby ruby-dev rubygems build-essential
+RUN gem install --no-ri --no-rdoc fpm
+ENV GOPATH=/tmp/go
+
+RUN apt-get install -y curl
+RUN apt-get install -y rsync
+RUN apt-get install -y gcc
+RUN apt-get install -y g++
+RUN apt-get install -y bash
+RUN apt-get install -y git
+RUN apt-get install -y tar
+RUN apt-get install -y rpm
+
+RUN mkdir -p $GOPATH/src/github.com/github/gh-ost
+WORKDIR $GOPATH/src/github.com/github/gh-ost
+COPY . .
+RUN bash build.sh

RELEASE_VERSION

@@ -1 +1 @@
-1.0.48
+1.0.49

build.sh

@@ -61,11 +61,11 @@ main() {
 
   mkdir -p ${buildpath}
   rm -rf ${buildpath:?}/*
-  build macOS osx darwin amd64
   build GNU/Linux linux linux amd64
+  # build macOS osx darwin amd64
 
   echo "Binaries found in:"
-  ls -1 $buildpath/gh-ost-binary*${timestamp}.tar.gz
+  find $buildpath/gh-ost* -type f -maxdepth 1
 }
 
 main "$@"

doc/rds.md

@@ -26,6 +26,14 @@ If you use `pt-table-checksum` as a part of your data integrity checks, you migh
 This tool requires binlog_format=STATEMENT, but the current binlog_format is set to ROW and an error occurred while attempting to change it.  If running MySQL 5.1.29 or newer, setting binlog_format requires the SUPER privilege.  You will need to manually set binlog_format to 'STATEMENT' before running this tool.
 ```
 
+#### Binlog filtering
+
+In Aurora, the [binlog filtering feature][aws_replication_docs_bin_log_filtering] is enabled by default. This becomes an issue when gh-ost tries to do the cut-over, because gh-ost waits for an entry in the binlog to proceed but this entry will never end up in the binlog because it gets filtered out by the binlog filtering feature.  
+You need to turn this feature off during the migration process.  
+Set the `aurora_enable_repl_bin_log_filtering` parameter to 0 in the Parameter Group for your cluster.  
+When the migration is done, set it back to 1 (default).
+
+
 #### Preflight checklist
 
 Before trying to run any `gh-ost` migrations you will want to confirm the following:
@@ -35,6 +43,7 @@ Before trying to run any `gh-ost` migrations you will want to confirm the follow
 - [ ] Executing `SHOW SLAVE STATUS\G` on your replica cluster displays the correct master host, binlog position, etc.
 - [ ] Database backup retention is greater than 1 day to enable binlogs
 - [ ] You have setup [`hooks`][ghost_hooks] to issue RDS procedures for stopping and starting replication. (see [github/gh-ost#163][ghost_rds_issue_tracking] for examples)
+- [ ] The parameter `aurora_enable_repl_bin_log_filtering` is set to 0
 
 [new_issue]: https://github.com/github/gh-ost/issues/new
 [assume_rbr_docs]: https://github.com/github/gh-ost/blob/master/doc/command-line-flags.md#assume-rbr
@@ -43,3 +52,4 @@ Before trying to run any `gh-ost` migrations you will want to confirm the follow
 [percona_toolkit_patch]: https://github.com/jacobbednarz/percona-toolkit/commit/0271ba6a094da446a5e5bb8d99b5c26f1777f2b9
 [ghost_hooks]: https://github.com/github/gh-ost/blob/master/doc/hooks.md
 [ghost_rds_issue_tracking]: https://github.com/github/gh-ost/issues/163
+[aws_replication_docs_bin_log_filtering]: https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/AuroraMySQL.Replication.html#AuroraMySQL.Replication.Performance

go/base/context.go

@@ -120,6 +120,7 @@ type MigrationContext struct {
 	ThrottleAdditionalFlagFile          string
 	throttleQuery                       string
 	throttleHTTP                        string
+	IgnoreHTTPErrors                    bool
 	ThrottleCommandedByUser             int64
 	HibernateUntil                      int64
 	maxLoad                             LoadMap
@@ -595,6 +596,13 @@ func (this *MigrationContext) SetThrottleHTTP(throttleHTTP string) {
 	this.throttleHTTP = throttleHTTP
 }
 
+func (this *MigrationContext) SetIgnoreHTTPErrors(ignoreHTTPErrors bool) {
+	this.throttleHTTPMutex.Lock()
+	defer this.throttleHTTPMutex.Unlock()
+
+	this.IgnoreHTTPErrors = ignoreHTTPErrors
+}
+
 func (this *MigrationContext) GetMaxLoad() LoadMap {
 	this.throttleMutex.Lock()
 	defer this.throttleMutex.Unlock()

go/cmd/gh-ost/main.go

@@ -106,6 +106,7 @@ func main() {
 	throttleControlReplicas := flag.String("throttle-control-replicas", "", "List of replicas on which to check for lag; comma delimited. Example: myhost1.com:3306,myhost2.com,myhost3.com:3307")
 	throttleQuery := flag.String("throttle-query", "", "when given, issued (every second) to check if operation should throttle. Expecting to return zero for no-throttle, >0 for throttle. Query is issued on the migrated server. Make sure this query is lightweight")
 	throttleHTTP := flag.String("throttle-http", "", "when given, gh-ost checks given URL via HEAD request; any response code other than 200 (OK) causes throttling; make sure it has low latency response")
+	ignoreHTTPErrors := flag.Bool("ignore-http-errors", false, "ignore HTTP connection errors during throttle check")
 	heartbeatIntervalMillis := flag.Int64("heartbeat-interval-millis", 100, "how frequently would gh-ost inject a heartbeat value")
 	flag.StringVar(&migrationContext.ThrottleFlagFile, "throttle-flag-file", "", "operation pauses when this file exists; hint: use a file that is specific to the table being altered")
 	flag.StringVar(&migrationContext.ThrottleAdditionalFlagFile, "throttle-additional-flag-file", "/tmp/gh-ost.throttle", "operation pauses when this file exists; hint: keep default, use for throttling multiple gh-ost operations")
@@ -259,6 +260,7 @@ func main() {
 	migrationContext.SetMaxLagMillisecondsThrottleThreshold(*maxLagMillis)
 	migrationContext.SetThrottleQuery(*throttleQuery)
 	migrationContext.SetThrottleHTTP(*throttleHTTP)
+	migrationContext.SetIgnoreHTTPErrors(*ignoreHTTPErrors)
 	migrationContext.SetDefaultNumRetries(*defaultRetries)
 	migrationContext.ApplyCredentials()
 	if err := migrationContext.SetupTLS(); err != nil {

go/logic/throttler.go

@@ -18,20 +18,22 @@ import (
 )
 
 var (
-	httpStatusMessages map[int]string = map[int]string{
+	httpStatusMessages = map[int]string{
 		200: "OK",
 		404: "Not found",
 		417: "Expectation failed",
 		429: "Too many requests",
 		500: "Internal server error",
+		-1:  "Connection error",
 	}
 	// See https://github.com/github/freno/blob/master/doc/http.md
-	httpStatusFrenoMessages map[int]string = map[int]string{
+	httpStatusFrenoMessages = map[int]string{
 		200: "OK",
 		404: "freno: unknown metric",
 		417: "freno: access forbidden",
 		429: "freno: threshold exceeded",
 		500: "freno: internal error",
+		-1:  "freno: connection error",
 	}
 )
 
@@ -83,6 +85,7 @@ func (this *Throttler) shouldThrottle() (result bool, reason string, reasonHint
 	if statusCode != 0 && statusCode != http.StatusOK {
 		return true, this.throttleHttpMessage(int(statusCode)), base.NoThrottleReasonHint
 	}
+
 	// Replication lag throttle
 	maxLagMillisecondsThrottleThreshold := atomic.LoadInt64(&this.migrationContext.MaxLagMillisecondsThrottleThreshold)
 	lag := atomic.LoadInt64(&this.migrationContext.CurrentLag)
@@ -287,7 +290,14 @@ func (this *Throttler) collectThrottleHTTPStatus(firstThrottlingCollected chan<-
 		return false, nil
 	}
 
-	collectFunc()
+	_, err := collectFunc()
+	if err != nil {
+		// If not told to ignore errors, we'll throttle on HTTP connection issues
+		if !this.migrationContext.IgnoreHTTPErrors {
+			atomic.StoreInt64(&this.migrationContext.ThrottleHTTPStatusCode, int64(-1))
+		}
+	}
+
 	firstThrottlingCollected <- true
 
 	ticker := time.Tick(100 * time.Millisecond)
@@ -296,7 +306,15 @@ func (this *Throttler) collectThrottleHTTPStatus(firstThrottlingCollected chan<-
 			return
 		}
 
-		if sleep, _ := collectFunc(); sleep {
+		sleep, err := collectFunc()
+		if err != nil {
+			// If not told to ignore errors, we'll throttle on HTTP connection issues
+			if !this.migrationContext.IgnoreHTTPErrors {
+				atomic.StoreInt64(&this.migrationContext.ThrottleHTTPStatusCode, int64(-1))
+			}
+		}
+
+		if sleep {
 			time.Sleep(1 * time.Second)
 		}
 	}

localtests/bigint-change-nullable/create.sql

@@ -0,0 +1,21 @@
+drop table if exists gh_ost_test;
+create table gh_ost_test (
+  id bigint auto_increment,
+  val bigint not null,
+  primary key(id)
+) auto_increment=1;
+
+drop event if exists gh_ost_test;
+delimiter ;;
+create event gh_ost_test
+  on schedule every 1 second
+  starts current_timestamp
+  ends current_timestamp + interval 60 second
+  on completion not preserve
+  enable
+  do
+begin
+  insert into gh_ost_test values (null, 18446744073709551615);
+  insert into gh_ost_test values (null, 18446744073709551614);
+  insert into gh_ost_test values (null, 18446744073709551613);
+end ;;

localtests/bigint-change-nullable/extra_args

@@ -0,0 +1 @@
+--alter="change val val bigint" 

localtests/latin1text/create.sql

@@ -0,0 +1,25 @@
+drop table if exists gh_ost_test;
+create table gh_ost_test (
+  id int auto_increment,
+  t text charset latin1 collate latin1_swedish_ci,
+  primary key(id)
+) auto_increment=1 charset latin1 collate latin1_swedish_ci;
+
+drop event if exists gh_ost_test;
+delimiter ;;
+create event gh_ost_test
+  on schedule every 1 second
+  starts current_timestamp
+  ends current_timestamp + interval 60 second
+  on completion not preserve
+  enable
+  do
+begin
+  insert into gh_ost_test values (null, md5(rand()));
+  insert into gh_ost_test values (null, 'átesting');
+  insert into gh_ost_test values (null, 'ádelete');
+  insert into gh_ost_test values (null, 'testátest');
+  update gh_ost_test set t='áupdated' order by id desc limit 1;
+  update gh_ost_test set t='áupdated1' where t='áupdated' order by id desc limit 1;
+  delete from gh_ost_test where t='ádelete';
+end ;;

localtests/swap-uk-uk/create.sql

@@ -1,8 +1,8 @@
 drop table if exists gh_ost_test;
 create table gh_ost_test (
-  id bigint,
+  id bigint not null,
   i int not null,
-  ts timestamp(6),
+  ts timestamp(6) not null,
   unique key id_uidx(id),
   unique key its_uidx(i, ts)
 ) ;

localtests/varbinary/create.sql

@@ -0,0 +1,40 @@
+drop table if exists gh_ost_test;
+create table gh_ost_test (
+  id binary(16) NOT NULL,
+  info varchar(255) COLLATE utf8_unicode_ci NOT NULL,
+  data binary(8) NOT NULL,
+  primary key (id),
+  unique key info_uidx (info)
+) auto_increment=1;
+
+drop event if exists gh_ost_test;
+delimiter ;;
+create event gh_ost_test
+  on schedule every 1 second
+  starts current_timestamp
+  ends current_timestamp + interval 60 second
+  on completion not preserve
+  enable
+  do
+begin
+  replace into gh_ost_test (id, info, data) values (X'12ffffffffffffffffffffffffffff00', 'item 1a', X'12ffffffffffffff');
+  replace into gh_ost_test (id, info, data) values (X'34ffffffffffffffffffffffffffffff', 'item 3a', X'34ffffffffffffff');
+  replace into gh_ost_test (id, info, data) values (X'90ffffffffffffffffffffffffffffff', 'item 9a', X'90ffffffffffff00');
+
+  DELETE FROM gh_ost_test WHERE id = X'11ffffffffffffffffffffffffffff00';
+  UPDATE gh_ost_test SET info = 'item 2++' WHERE id = X'22ffffffffffffffffffffffffffff00';
+  UPDATE gh_ost_test SET info = 'item 3++', data = X'33ffffffffffff00' WHERE id = X'33ffffffffffffffffffffffffffffff';
+  DELETE FROM gh_ost_test WHERE id = X'44ffffffffffffffffffffffffffffff';
+  UPDATE gh_ost_test SET info = 'item 5++', data = X'55ffffffffffffee'  WHERE id = X'55ffffffffffffffffffffffffffffff';
+  INSERT INTO gh_ost_test (id, info, data) VALUES (X'66ffffffffffffffffffffffffffff00', 'item 6', X'66ffffffffffffff');
+  INSERT INTO gh_ost_test (id, info, data) VALUES (X'77ffffffffffffffffffffffffffffff', 'item 7', X'77ffffffffffff00');
+  INSERT INTO gh_ost_test (id, info, data) VALUES (X'88ffffffffffffffffffffffffffffff', 'item 8', X'88ffffffffffffff');
+end ;;
+
+INSERT INTO gh_ost_test (id, info, data) VALUES
+ (X'11ffffffffffffffffffffffffffff00', 'item 1', X'11ffffffffffffff'), -- id ends in 00
+ (X'22ffffffffffffffffffffffffffff00', 'item 2', X'22ffffffffffffff'), -- id ends in 00
+ (X'33ffffffffffffffffffffffffffffff', 'item 3', X'33ffffffffffffff'),
+ (X'44ffffffffffffffffffffffffffffff', 'item 4', X'44ffffffffffffff'),
+ (X'55ffffffffffffffffffffffffffffff', 'item 5', X'55ffffffffffffff'),
+ (X'99ffffffffffffffffffffffffffffff', 'item 9', X'99ffffffffffff00'); -- data ends in 00

script/cibuild-gh-ost-replica-tests

@@ -50,7 +50,8 @@ test_mysql_version() {
 
   export PATH="${PWD}/gh-ost-ci-env/bin/:${PATH}"
 
-  gh-ost-test-mysql-master -uroot -e "grant all on *.* to 'gh-ost'@'%' identified by 'gh-ost'"
+  gh-ost-test-mysql-master -uroot -e "create user 'gh-ost'@'%' identified by 'gh-ost'"
+  gh-ost-test-mysql-master -uroot -e "grant all on *.* to 'gh-ost'@'%'"
 
   echo "### Running gh-ost tests for $mysql_version"
   ./localtests/test.sh -b bin/gh-ost
@@ -61,6 +62,9 @@ test_mysql_version() {
 echo "Building..."
 . script/build
 # Test all versions:
+find gh-ost-ci-env/mysql-tarballs/ -name "*.tar.gz" | while read f ; do basename $f ".tar.gz" ; done | sort -r | while read mysql_version ; do
+  echo "found MySQL version: $mysql_version"
+done
 find gh-ost-ci-env/mysql-tarballs/ -name "*.tar.gz" | while read f ; do basename $f ".tar.gz" ; done | sort -r | while read mysql_version ; do
   test_mysql_version "$mysql_version"
 done

script/dock

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+# Usage:
+# dock <test|packages> [arg]
+#   dock test:                build gh-ost & run unit and integration tests
+#   docker pkg [target-path]: build gh-ost release packages and copy to target path (default path: /tmp/gh-ost-release)
+
+command="$1"
+
+case "$command" in
+  "test")
+    docker_target="gh-ost-test"
+    docker build . -f Dockerfile.test -t "${docker_target}" && docker run --rm -it "${docker_target}:latest"
+    ;;
+  "pkg")
+    packages_path="${2:-/tmp/gh-ost-release}"
+    docker_target="gh-ost-packaging"
+    docker build . -f Dockerfile.packaging -t "${docker_target}" && docker run --rm -it -v "${packages_path}:/tmp/pkg" "${docker_target}:latest" bash -c 'find /tmp/gh-ost-release/ -maxdepth 1 -type f | xargs cp -t /tmp/pkg'
+    echo "packages generated on ${packages_path}:"
+    ls -l "${packages_path}"
+    ;;
+  *)
+    >&2 echo "Usage: dock dock <test|alpine|packages> [arg]"
+    exit 1
+esac