Merge branch 'master' into master

.travis.yml

@@ -1,7 +1,9 @@
 # http://docs.travis-ci.com/user/languages/go/
 language: go
 
-go: 1.9
+go:
+  - "1.9"
+  - "1.10"
 
 os:
   - linux

RELEASE_VERSION

@@ -1 +1 @@
-1.0.46
+1.0.47

doc/command-line-flags.md

@@ -2,6 +2,10 @@
 
 A more in-depth discussion of various `gh-ost` command line flags: implementation, implication, use cases.
 
+### aliyun-rds
+
+Add this flag when executing on Aliyun RDS.
+
 ### allow-master-master
 
 See [`--assume-master-host`](#assume-master-host).
@@ -103,6 +107,10 @@ While the ongoing estimated number of rows is still heuristic, it's almost exact
 
 Without this parameter, migration is a _noop_: testing table creation and validity of migration, but not touching data.
 
+### gcp
+
+Add this flag when executing on a 1st generation Google Cloud Platform (GCP).
+
 ### heartbeat-interval-millis
 
 Default 100. See [`subsecond-lag`](subsecond-lag.md) for details.

doc/hooks.md

@@ -69,6 +69,7 @@ The following variables are available on all hooks:
 - `GH_OST_INSPECTED_HOST`
 - `GH_OST_EXECUTING_HOST`
 - `GH_OST_HOOKS_HINT` - copy of `--hooks-hint` value
+- `GH_OST_DRY_RUN` - whether or not the `gh-ost` run is a dry run
 
 The following variable are available on particular hooks:
 

doc/requirements-and-limitations.md

@@ -39,7 +39,8 @@ The `SUPER` privilege is required for `STOP SLAVE`, `START SLAVE` operations. Th
   - For example, you may not migrate `MyTable` if another table called `MYtable` exists in the same schema.
 
 - Amazon RDS works, but has it's own [limitations](rds.md).
-- Google Cloud SQL is currently not supported
+- Google Cloud SQL works, `--gcp` flag required.
+- Aliyun RDS works, `--aliyun-rds` flag required.
 
 - Multisource is not supported when migrating via replica. It _should_ work (but never tested) when connecting directly to master (`--allow-on-master`)
 

doc/shared-key.md

@@ -6,7 +6,7 @@ A requirement for a migration to run is that the two _before_ and _after_ tables
 
 Consider a classic, simple migration. The table is any normal:
 
-```
+```sql
 CREATE TABLE tbl (
   id bigint unsigned not null auto_increment,
   data varchar(255),
@@ -37,7 +37,7 @@ Upon migration, `gh-ost` inspects both the original and _ghost_ table and attemp
 
 ### Examples: allowed and not allowed
 
-```
+```sql
 create table some_table (
   id int auto_increment,
   ts timestamp,

go/base/context.go

@@ -92,6 +92,7 @@ type MigrationContext struct {
 	IsTungsten               bool
 	DiscardForeignKeys       bool
 	AliyunRDS                bool
+	GoogleCloudPlatform      bool
 
 	config            ContextConfig
 	configMutex       *sync.Mutex

go/base/utils.go

@@ -75,7 +75,8 @@ func ValidateConnection(db *gosql.DB, connectionConfig *mysql.ConnectionConfig,
 		// swallow this error. not all servers support extra_port
 	}
 	// AliyunRDS set users port to "NULL", replace it by gh-ost param
-	if migrationContext.AliyunRDS {
+	// GCP set users port to "NULL", replace it by gh-ost param
+	if migrationContext.AliyunRDS || migrationContext.GoogleCloudPlatform {
 		port = connectionConfig.Key.Port
 	} else {
 		portQuery := `select @@global.port`

go/binlog/gomysql_reader.go

@@ -145,7 +145,7 @@ func (this *GoMySQLReader) StreamEvents(canStopStreaming func() bool, entriesCha
 				defer this.currentCoordinatesMutex.Unlock()
 				this.currentCoordinates.LogFile = string(rotateEvent.NextLogName)
 			}()
-			log.Infof("rotate to next log name: %s", rotateEvent.NextLogName)
+			log.Infof("rotate to next log from %s:%d to %s", this.currentCoordinates.LogFile, int64(ev.Header.LogPos), rotateEvent.NextLogName)
 		} else if rowsEvent, ok := ev.Event.(*replication.RowsEvent); ok {
 			if err := this.handleRowsEvent(ev, rowsEvent, entriesChannel); err != nil {
 				return err

go/cmd/gh-ost/main.go

@@ -68,6 +68,7 @@ func main() {
 	flag.BoolVar(&migrationContext.DiscardForeignKeys, "discard-foreign-keys", false, "DANGER! This flag will migrate a table that has foreign keys and will NOT create foreign keys on the ghost table, thus your altered table will have NO foreign keys. This is useful for intentional dropping of foreign keys")
 	flag.BoolVar(&migrationContext.SkipForeignKeyChecks, "skip-foreign-key-checks", false, "set to 'true' when you know for certain there are no foreign keys on your table, and wish to skip the time it takes for gh-ost to verify that")
 	flag.BoolVar(&migrationContext.AliyunRDS, "aliyun-rds", false, "set to 'true' when you execute on Aliyun RDS.")
+	flag.BoolVar(&migrationContext.GoogleCloudPlatform, "gcp", false, "set to 'true' when you execute on a 1st generation Google Cloud Platform (GCP).")
 
 	executeFlag := flag.Bool("execute", false, "actually execute the alter & migrate the table. Default is noop: do some tests and exit")
 	flag.BoolVar(&migrationContext.TestOnReplica, "test-on-replica", false, "Have the migration run on a replica, not on the master. At the end of migration replication is stopped, and tables are swapped and immediately swap-revert. Replication remains stopped and you can compare the two tables for building trust")

go/logic/applier.go

@@ -89,7 +89,7 @@ func (this *Applier) InitDBConnections() (err error) {
 	if err := this.validateAndReadTimeZone(); err != nil {
 		return err
 	}
-	if !this.migrationContext.AliyunRDS {
+	if !this.migrationContext.AliyunRDS && !this.migrationContext.GoogleCloudPlatform {
 		if impliedKey, err := mysql.GetInstanceKey(this.db); err != nil {
 			return err
 		} else {

go/logic/hooks.go

@@ -64,6 +64,7 @@ func (this *HooksExecutor) applyEnvironmentVariables(extraVariables ...string) [
 	env = append(env, fmt.Sprintf("GH_OST_INSPECTED_HOST=%s", this.migrationContext.GetInspectorHostname()))
 	env = append(env, fmt.Sprintf("GH_OST_EXECUTING_HOST=%s", this.migrationContext.Hostname))
 	env = append(env, fmt.Sprintf("GH_OST_HOOKS_HINT=%s", this.migrationContext.HooksHintMessage))
+	env = append(env, fmt.Sprintf("GH_OST_DRY_RUN=%t", this.migrationContext.Noop))
 
 	for _, variable := range extraVariables {
 		env = append(env, variable)

go/logic/inspect.go

@@ -53,7 +53,7 @@ func (this *Inspector) InitDBConnections() (err error) {
 	if err := this.validateConnection(); err != nil {
 		return err
 	}
-	if !this.migrationContext.AliyunRDS {
+	if !this.migrationContext.AliyunRDS && !this.migrationContext.GoogleCloudPlatform {
 		if impliedKey, err := mysql.GetInstanceKey(this.db); err != nil {
 			return err
 		} else {

go/logic/migrator.go

@@ -1087,24 +1087,33 @@ func (this *Migrator) iterateChunks() error {
 		log.Debugf("No rows found in table. Rowcopy will be implicitly empty")
 		return terminateRowIteration(nil)
 	}
+
+	var hasNoFurtherRangeFlag int64
 	// Iterate per chunk:
 	for {
-		if atomic.LoadInt64(&this.rowCopyCompleteFlag) == 1 {
+		if atomic.LoadInt64(&this.rowCopyCompleteFlag) == 1 || atomic.LoadInt64(&hasNoFurtherRangeFlag) == 1 {
 			// Done
 			// There's another such check down the line
 			return nil
 		}
 		copyRowsFunc := func() error {
-			if atomic.LoadInt64(&this.rowCopyCompleteFlag) == 1 {
+			if atomic.LoadInt64(&this.rowCopyCompleteFlag) == 1 || atomic.LoadInt64(&hasNoFurtherRangeFlag) == 1 {
 				// Done.
 				// There's another such check down the line
 				return nil
 			}
-			hasFurtherRange, err := this.applier.CalculateNextIterationRangeEndValues()
-			if err != nil {
+
+			// When hasFurtherRange is false, original table might be write locked and CalculateNextIterationRangeEndValues would hangs forever
+
+			hasFurtherRange := false
+			if err := this.retryOperation(func() (e error) {
+				hasFurtherRange, e = this.applier.CalculateNextIterationRangeEndValues()
+				return e
+			}); err != nil {
 				return terminateRowIteration(err)
 			}
 			if !hasFurtherRange {
+				atomic.StoreInt64(&hasNoFurtherRangeFlag, 1)
 				return terminateRowIteration(nil)
 			}
 			// Copy task:
@@ -1122,7 +1131,7 @@ func (this *Migrator) iterateChunks() error {
 				}
 				_, rowsAffected, _, err := this.applier.ApplyIterationInsertQuery()
 				if err != nil {
-					return terminateRowIteration(err)
+					return err // wrapping call will retry
 				}
 				atomic.AddInt64(&this.migrationContext.TotalRowsCopied, rowsAffected)
 				atomic.AddInt64(&this.migrationContext.Iteration, 1)

localtests/bit-add/create.sql

@@ -0,0 +1,20 @@
+drop table if exists gh_ost_test;
+create table gh_ost_test (
+  id int auto_increment,
+  i int not null,
+  primary key(id)
+) auto_increment=1;
+
+drop event if exists gh_ost_test;
+delimiter ;;
+create event gh_ost_test
+  on schedule every 1 second
+  starts current_timestamp
+  ends current_timestamp + interval 60 second
+  on completion not preserve
+  enable
+  do
+begin
+  insert into gh_ost_test values (null, 11);
+  insert into gh_ost_test values (null, 13);
+end ;;

localtests/bit-add/extra_args

@@ -0,0 +1 @@
+--alter="add column is_good bit null default 0" 

localtests/bit-add/ghost_columns

@@ -0,0 +1 @@
+id, i

localtests/bit-add/orig_columns

@@ -0,0 +1 @@
+id, i

localtests/bit-dml/create.sql

@@ -0,0 +1,24 @@
+drop table if exists gh_ost_test;
+create table gh_ost_test (
+  id int auto_increment,
+  i int not null,
+  is_good bit null default 0,
+  primary key(id)
+) auto_increment=1;
+
+drop event if exists gh_ost_test;
+delimiter ;;
+create event gh_ost_test
+  on schedule every 1 second
+  starts current_timestamp
+  ends current_timestamp + interval 60 second
+  on completion not preserve
+  enable
+  do
+begin
+  insert into gh_ost_test values (null, 11, 0);
+  insert into gh_ost_test values (null, 13, 1);
+  insert into gh_ost_test values (null, 17, 1);
+
+  update gh_ost_test set is_good=0 where i=13 order by id desc limit 1;
+end ;;

localtests/bit-dml/extra_args

@@ -0,0 +1 @@
+--alter="modify column is_good bit not null default 0" --approve-renamed-columns

localtests/convert-utf8mb4/create.sql

@@ -0,0 +1,31 @@
+drop table if exists gh_ost_test;
+create table gh_ost_test (
+  id int auto_increment,
+  t varchar(128) charset utf8 collate utf8_general_ci,
+  tl varchar(128) charset latin1 not null,
+  ta varchar(128) charset ascii not null,
+  primary key(id)
+) auto_increment=1;
+
+insert into gh_ost_test values (null, 'átesting');
+
+
+insert into gh_ost_test values (null, 'Hello world, Καλημέρα κόσμε, コンニチハ', 'átesting0', 'initial');
+
+drop event if exists gh_ost_test;
+delimiter ;;
+create event gh_ost_test
+  on schedule every 1 second
+  starts current_timestamp
+  ends current_timestamp + interval 60 second
+  on completion not preserve
+  enable
+  do
+begin
+  insert into gh_ost_test values (null, md5(rand()), 'átesting-a', 'a');
+  insert into gh_ost_test values (null, 'novo proprietário', 'átesting-b', 'b');
+  insert into gh_ost_test values (null, '2H₂ + O₂ ⇌ 2H₂O, R = 4.7 kΩ, ⌀ 200 mm', 'átesting-c', 'c');
+  insert into gh_ost_test values (null, 'usuário', 'átesting-x', 'x');
+
+  delete from gh_ost_test where ta='x' order by id desc limit 1;
+end ;;

localtests/convert-utf8mb4/extra_args

@@ -0,0 +1 @@
+--alter='convert to character set utf8mb4'

localtests/datetime-1970/create.sql

@@ -1,3 +1,5 @@
+set session time_zone='+00:00';
+
 drop table if exists gh_ost_test;
 create table gh_ost_test (
   id int auto_increment,
@@ -7,6 +9,7 @@ create table gh_ost_test (
   primary key(id)
 ) auto_increment=1;
 
+set session time_zone='+00:00';
 insert into gh_ost_test values (1, '0000-00-00 00:00:00', now(), 0);
 
 drop event if exists gh_ost_test;
@@ -19,5 +22,6 @@ create event gh_ost_test
   enable
   do
 begin
+  set session time_zone='+00:00';
   update gh_ost_test set counter = counter + 1 where id = 1;
 end ;;

localtests/test.sh

@@ -14,11 +14,13 @@ ghost_binary=""
 exec_command_file=/tmp/gh-ost-test.bash
 orig_content_output_file=/tmp/gh-ost-test.orig.content.csv
 ghost_content_output_file=/tmp/gh-ost-test.ghost.content.csv
+throttle_flag_file=/tmp/gh-ost-test.ghost.throttle.flag
 
 master_host=
 master_port=
 replica_host=
 replica_port=
+original_sql_mode=
 
 OPTIND=1
 while getopts "b:" OPTION
@@ -45,6 +47,8 @@ verify_master_and_replica() {
     echo "Cannot enable event_scheduler on master"
     exit 1
   fi
+  original_sql_mode="$(gh-ost-test-mysql-master -e "select @@global.sql_mode" -s -s)"
+  echo "sql_mode on master is ${original_sql_mode}"
 
   if [ "$(gh-ost-test-mysql-replica -e "select 1" -ss)" != "1" ] ; then
     echo "Cannot verify gh-ost-test-mysql-replica"
@@ -87,7 +91,6 @@ start_replication() {
 test_single() {
   local test_name
   test_name="$1"
-  original_sql_mode="$(gh-ost-test-mysql-master -e "select @@global.sql_mode" -s -s)"
 
   if [ -f $tests_path/$test_name/ignore_versions ] ; then
     ignore_versions=$(cat $tests_path/$test_name/ignore_versions)
@@ -108,7 +111,7 @@ test_single() {
     gh-ost-test-mysql-master --default-character-set=utf8mb4 test -e "set @@global.sql_mode='$(cat $tests_path/$test_name/sql_mode)'"
     gh-ost-test-mysql-replica --default-character-set=utf8mb4 test -e "set @@global.sql_mode='$(cat $tests_path/$test_name/sql_mode)'"
   fi
-  
+
   gh-ost-test-mysql-master --default-character-set=utf8mb4 test < $tests_path/$test_name/create.sql
 
   extra_args=""
@@ -145,6 +148,7 @@ test_single() {
     --initially-drop-old-table \
     --initially-drop-ghost-table \
     --throttle-query='select timestampdiff(second, min(last_update), now()) < 5 from _gh_ost_test_ghc' \
+    --throttle-flag-file=$throttle_flag_file \
     --serve-socket-file=/tmp/gh-ost.test.sock \
     --initially-drop-socket-file \
     --test-on-replica \

script/ensure-go-installed

@@ -4,7 +4,7 @@ PREFERRED_GO_VERSION=go1.9.2
 SUPPORTED_GO_VERSIONS='go1.[89]'
 
 GO_PKG_DARWIN=${PREFERRED_GO_VERSION}.darwin-amd64.pkg
-GO_PKG_DARWIN_SHA=73fd5840d55f5566d8db6c0ffdd187577e8ebe650c783f68bd27cbf95bde6743
+GO_PKG_DARWIN_SHA=8b4f6ae6deae1150d2e341d02c247fd18a99af387516540eeb84702ffd76d3a1
 
 GO_PKG_LINUX=${PREFERRED_GO_VERSION}.linux-amd64.tar.gz
 GO_PKG_LINUX_SHA=de874549d9a8d8d8062be05808509c09a88a248e77ec14eb77453530829ac02b