Merge branch 'master' into minor_changes

RELEASE_VERSION

@@ -1 +1 @@
-1.0.45
+1.0.46

build.sh

@@ -2,7 +2,15 @@
 #
 #
 
-RELEASE_VERSION=$(cat RELEASE_VERSION)
+RELEASE_VERSION=
+buildpath=
+
+function setuptree() {
+  b=$( mktemp -d $buildpath/gh-ostXXXXXX ) || return 1
+  mkdir -p $b/gh-ost
+  mkdir -p $b/gh-ost/usr/bin
+  echo $b
+}
 
 function build {
   osname=$1
@@ -10,6 +18,8 @@ function build {
   GOOS=$3
   GOARCH=$4
 
+
+
   if ! go version | egrep -q 'go(1[.]9|1[.]1[0-9])' ; then
     echo "go version is too low. Must use 1.9 or above"
     exit 1
@@ -26,16 +36,38 @@ function build {
   fi
 
   (cd $buildpath && tar cfz ./gh-ost-binary-${osshort}-${timestamp}.tar.gz $target)
+
+  if [ "$GOOS" == "linux" ] ; then
+    echo "Creating Distro full packages"
+    builddir=$(setuptree)
+    cp $buildpath/$target $builddir/gh-ost/usr/bin
+    cd $buildpath
+    fpm -v "${RELEASE_VERSION}" --epoch 1 -f -s dir -n gh-ost -m shlomi-noach --description "GitHub's Online Schema Migrations for MySQL " --url "https://github.com/github/gh-ost" --vendor "GitHub" --license "Apache 2.0" -C $builddir/gh-ost --prefix=/ -t rpm .
+    fpm -v "${RELEASE_VERSION}" --epoch 1 -f -s dir -n gh-ost -m shlomi-noach --description "GitHub's Online Schema Migrations for MySQL " --url "https://github.com/github/gh-ost" --vendor "GitHub" --license "Apache 2.0" -C $builddir/gh-ost --prefix=/ -t deb --deb-no-default-config-files .
+  fi
 }
 
-buildpath=/tmp/gh-ost
+main() {
+  if [ -z "${RELEASE_VERSION}" ] ; then
+    RELEASE_VERSION=$(git describe --abbrev=0 --tags | tr -d 'v')
+  fi
+  if [ -z "${RELEASE_VERSION}" ] ; then
+    RELEASE_VERSION=$(cat RELEASE_VERSION)
+  fi
+
+
+  buildpath=/tmp/gh-ost-release
   target=gh-ost
   timestamp=$(date "+%Y%m%d%H%M%S")
   ldflags="-X main.AppVersion=${RELEASE_VERSION}"
 
   mkdir -p ${buildpath}
+  rm -rf ${buildpath:?}/*
   build macOS osx darwin amd64
   build GNU/Linux linux linux amd64
 
   echo "Binaries found in:"
   ls -1 $buildpath/gh-ost-binary*${timestamp}.tar.gz
+}
+
+main "$@"

doc/command-line-flags.md

@@ -2,6 +2,10 @@
 
 A more in-depth discussion of various `gh-ost` command line flags: implementation, implication, use cases.
 
+### aliyun-rds
+
+Add this flag when executing on Aliyun RDS.
+
 ### allow-master-master
 
 See [`--assume-master-host`](#assume-master-host).
@@ -103,6 +107,10 @@ While the ongoing estimated number of rows is still heuristic, it's almost exact
 
 Without this parameter, migration is a _noop_: testing table creation and validity of migration, but not touching data.
 
+### gcp
+
+Add this flag when executing on a 1st generation Google Cloud Platform (GCP).
+
 ### heartbeat-interval-millis
 
 Default 100. See [`subsecond-lag`](subsecond-lag.md) for details.

doc/requirements-and-limitations.md

@@ -26,10 +26,6 @@ The `SUPER` privilege is required for `STOP SLAVE`, `START SLAVE` operations. Th
 
 - Triggers are not supported. They may be supported in the future.
 
-- MySQL 5.7 generated columns are not supported. They may be supported in the future.
-
-- MySQL 5.7 `POINT` column type is not supported.
-
 - MySQL 5.7 `JSON` columns are supported but not as part of `PRIMARY KEY`
 
 - The two _before_ & _after_ tables must share a `PRIMARY KEY` or other `UNIQUE KEY`. This key will be used by `gh-ost` to iterate through the table rows when copying. [Read more](shared-key.md)
@@ -43,13 +39,14 @@ The `SUPER` privilege is required for `STOP SLAVE`, `START SLAVE` operations. Th
   - For example, you may not migrate `MyTable` if another table called `MYtable` exists in the same schema.
 
 - Amazon RDS works, but has it's own [limitations](rds.md).
-- Google Cloud SQL is currently not supported
+- Google Cloud SQL works, `--gcp` flag required.
+- Aliyun RDS works, `--aliyun-rds` flag required.
 
 - Multisource is not supported when migrating via replica. It _should_ work (but never tested) when connecting directly to master (`--allow-on-master`)
 
 - Master-master setup is only supported in active-passive setup. Active-active (where table is being written to on both masters concurrently) is unsupported. It may be supported in the future.
 
-- If you have en `enum` field as part of your migration key (typically the `PRIMARY KEY`), migration performance will be degraded and potentially bad. [Read more](https://github.com/github/gh-ost/pull/277#issuecomment-254811520)
+- If you have an `enum` field as part of your migration key (typically the `PRIMARY KEY`), migration performance will be degraded and potentially bad. [Read more](https://github.com/github/gh-ost/pull/277#issuecomment-254811520)
 
 - Migrating a `FEDERATED` table is unsupported and is irrelevant to the problem `gh-ost` tackles.
 

go/base/context.go

@@ -92,6 +92,7 @@ type MigrationContext struct {
 	IsTungsten               bool
 	DiscardForeignKeys       bool
 	AliyunRDS                bool
+	GoogleCloudPlatform      bool
 
 	config            ContextConfig
 	configMutex       *sync.Mutex
@@ -186,8 +187,10 @@ type MigrationContext struct {
 
 	OriginalTableColumnsOnApplier    *sql.ColumnList
 	OriginalTableColumns             *sql.ColumnList
+	OriginalTableVirtualColumns      *sql.ColumnList
 	OriginalTableUniqueKeys          [](*sql.UniqueKey)
 	GhostTableColumns                *sql.ColumnList
+	GhostTableVirtualColumns         *sql.ColumnList
 	GhostTableUniqueKeys             [](*sql.UniqueKey)
 	UniqueKey                        *sql.UniqueKey
 	SharedColumns                    *sql.ColumnList

go/base/utils.go

@@ -75,7 +75,8 @@ func ValidateConnection(db *gosql.DB, connectionConfig *mysql.ConnectionConfig,
 		// swallow this error. not all servers support extra_port
 	}
 	// AliyunRDS set users port to "NULL", replace it by gh-ost param
-	if migrationContext.AliyunRDS {
+	// GCP set users port to "NULL", replace it by gh-ost param
+	if migrationContext.AliyunRDS || migrationContext.GoogleCloudPlatform {
 		port = connectionConfig.Key.Port
 	} else {
 		portQuery := `select @@global.port`

go/binlog/gomysql_reader.go

@@ -145,7 +145,7 @@ func (this *GoMySQLReader) StreamEvents(canStopStreaming func() bool, entriesCha
 				defer this.currentCoordinatesMutex.Unlock()
 				this.currentCoordinates.LogFile = string(rotateEvent.NextLogName)
 			}()
-			log.Infof("rotate to next log name: %s", rotateEvent.NextLogName)
+			log.Infof("rotate to next log from %s:%d to %s", this.currentCoordinates.LogFile, int64(ev.Header.LogPos), rotateEvent.NextLogName)
 		} else if rowsEvent, ok := ev.Event.(*replication.RowsEvent); ok {
 			if err := this.handleRowsEvent(ev, rowsEvent, entriesChannel); err != nil {
 				return err

go/cmd/gh-ost/main.go

@@ -68,6 +68,7 @@ func main() {
 	flag.BoolVar(&migrationContext.DiscardForeignKeys, "discard-foreign-keys", false, "DANGER! This flag will migrate a table that has foreign keys and will NOT create foreign keys on the ghost table, thus your altered table will have NO foreign keys. This is useful for intentional dropping of foreign keys")
 	flag.BoolVar(&migrationContext.SkipForeignKeyChecks, "skip-foreign-key-checks", false, "set to 'true' when you know for certain there are no foreign keys on your table, and wish to skip the time it takes for gh-ost to verify that")
 	flag.BoolVar(&migrationContext.AliyunRDS, "aliyun-rds", false, "set to 'true' when you execute on Aliyun RDS.")
+	flag.BoolVar(&migrationContext.GoogleCloudPlatform, "gcp", false, "set to 'true' when you execute on a 1st generation Google Cloud Platform (GCP).")
 
 	executeFlag := flag.Bool("execute", false, "actually execute the alter & migrate the table. Default is noop: do some tests and exit")
 	flag.BoolVar(&migrationContext.TestOnReplica, "test-on-replica", false, "Have the migration run on a replica, not on the master. At the end of migration replication is stopped, and tables are swapped and immediately swap-revert. Replication remains stopped and you can compare the two tables for building trust")

go/logic/applier.go

@@ -89,7 +89,7 @@ func (this *Applier) InitDBConnections() (err error) {
 	if err := this.validateAndReadTimeZone(); err != nil {
 		return err
 	}
-	if !this.migrationContext.AliyunRDS {
+	if !this.migrationContext.AliyunRDS && !this.migrationContext.GoogleCloudPlatform {
 		if impliedKey, err := mysql.GetInstanceKey(this.db); err != nil {
 			return err
 		} else {
@@ -117,7 +117,7 @@ func (this *Applier) validateAndReadTimeZone() error {
 // readTableColumns reads table columns on applier
 func (this *Applier) readTableColumns() (err error) {
 	log.Infof("Examining table structure on applier")
-	this.migrationContext.OriginalTableColumnsOnApplier, err = mysql.GetTableColumns(this.db, this.migrationContext.DatabaseName, this.migrationContext.OriginalTableName)
+	this.migrationContext.OriginalTableColumnsOnApplier, _, err = mysql.GetTableColumns(this.db, this.migrationContext.DatabaseName, this.migrationContext.OriginalTableName)
 	if err != nil {
 		return err
 	}

go/logic/inspect.go

@@ -53,7 +53,7 @@ func (this *Inspector) InitDBConnections() (err error) {
 	if err := this.validateConnection(); err != nil {
 		return err
 	}
-	if !this.migrationContext.AliyunRDS {
+	if !this.migrationContext.AliyunRDS && !this.migrationContext.GoogleCloudPlatform {
 		if impliedKey, err := mysql.GetInstanceKey(this.db); err != nil {
 			return err
 		} else {
@@ -89,24 +89,24 @@ func (this *Inspector) ValidateOriginalTable() (err error) {
 	return nil
 }
 
-func (this *Inspector) InspectTableColumnsAndUniqueKeys(tableName string) (columns *sql.ColumnList, uniqueKeys [](*sql.UniqueKey), err error) {
+func (this *Inspector) InspectTableColumnsAndUniqueKeys(tableName string) (columns *sql.ColumnList, virtualColumns *sql.ColumnList, uniqueKeys [](*sql.UniqueKey), err error) {
 	uniqueKeys, err = this.getCandidateUniqueKeys(tableName)
 	if err != nil {
-		return columns, uniqueKeys, err
+		return columns, virtualColumns, uniqueKeys, err
 	}
 	if len(uniqueKeys) == 0 {
-		return columns, uniqueKeys, fmt.Errorf("No PRIMARY nor UNIQUE key found in table! Bailing out")
+		return columns, virtualColumns, uniqueKeys, fmt.Errorf("No PRIMARY nor UNIQUE key found in table! Bailing out")
 	}
-	columns, err = mysql.GetTableColumns(this.db, this.migrationContext.DatabaseName, tableName)
+	columns, virtualColumns, err = mysql.GetTableColumns(this.db, this.migrationContext.DatabaseName, tableName)
 	if err != nil {
-		return columns, uniqueKeys, err
+		return columns, virtualColumns, uniqueKeys, err
 	}
 
-	return columns, uniqueKeys, nil
+	return columns, virtualColumns, uniqueKeys, nil
 }
 
 func (this *Inspector) InspectOriginalTable() (err error) {
-	this.migrationContext.OriginalTableColumns, this.migrationContext.OriginalTableUniqueKeys, err = this.InspectTableColumnsAndUniqueKeys(this.migrationContext.OriginalTableName)
+	this.migrationContext.OriginalTableColumns, this.migrationContext.OriginalTableVirtualColumns, this.migrationContext.OriginalTableUniqueKeys, err = this.InspectTableColumnsAndUniqueKeys(this.migrationContext.OriginalTableName)
 	if err != nil {
 		return err
 	}
@@ -122,7 +122,7 @@ func (this *Inspector) inspectOriginalAndGhostTables() (err error) {
 		return fmt.Errorf("It seems like table structure is not identical between master and replica. This scenario is not supported.")
 	}
 
-	this.migrationContext.GhostTableColumns, this.migrationContext.GhostTableUniqueKeys, err = this.InspectTableColumnsAndUniqueKeys(this.migrationContext.GetGhostTableName())
+	this.migrationContext.GhostTableColumns, this.migrationContext.GhostTableVirtualColumns, this.migrationContext.GhostTableUniqueKeys, err = this.InspectTableColumnsAndUniqueKeys(this.migrationContext.GetGhostTableName())
 	if err != nil {
 		return err
 	}
@@ -166,7 +166,7 @@ func (this *Inspector) inspectOriginalAndGhostTables() (err error) {
 		}
 	}
 
-	this.migrationContext.SharedColumns, this.migrationContext.MappedSharedColumns = this.getSharedColumns(this.migrationContext.OriginalTableColumns, this.migrationContext.GhostTableColumns, this.migrationContext.ColumnRenameMap)
+	this.migrationContext.SharedColumns, this.migrationContext.MappedSharedColumns = this.getSharedColumns(this.migrationContext.OriginalTableColumns, this.migrationContext.GhostTableColumns, this.migrationContext.OriginalTableVirtualColumns, this.migrationContext.GhostTableVirtualColumns, this.migrationContext.ColumnRenameMap)
 	log.Infof("Shared columns are %s", this.migrationContext.SharedColumns)
 	// By fact that a non-empty unique key exists we also know the shared columns are non-empty
 
@@ -692,7 +692,7 @@ func (this *Inspector) getSharedUniqueKeys(originalUniqueKeys, ghostUniqueKeys [
 }
 
 // getSharedColumns returns the intersection of two lists of columns in same order as the first list
-func (this *Inspector) getSharedColumns(originalColumns, ghostColumns *sql.ColumnList, columnRenameMap map[string]string) (*sql.ColumnList, *sql.ColumnList) {
+func (this *Inspector) getSharedColumns(originalColumns, ghostColumns *sql.ColumnList, originalVirtualColumns, ghostVirtualColumns *sql.ColumnList, columnRenameMap map[string]string) (*sql.ColumnList, *sql.ColumnList) {
 	sharedColumnNames := []string{}
 	for _, originalColumn := range originalColumns.Names() {
 		isSharedColumn := false
@@ -712,6 +712,16 @@ func (this *Inspector) getSharedColumns(originalColumns, ghostColumns *sql.Colum
 				break
 			}
 		}
+		for _, virtualColumn := range originalVirtualColumns.Names() {
+			if strings.EqualFold(originalColumn, virtualColumn) {
+				isSharedColumn = false
+			}
+		}
+		for _, virtualColumn := range ghostVirtualColumns.Names() {
+			if strings.EqualFold(originalColumn, virtualColumn) {
+				isSharedColumn = false
+			}
+		}
 		if isSharedColumn {
 			sharedColumnNames = append(sharedColumnNames, originalColumn)
 		}

go/mysql/utils.go

@@ -8,6 +8,7 @@ package mysql
 import (
 	gosql "database/sql"
 	"fmt"
+	"strings"
 	"sync"
 	"time"
 
@@ -173,7 +174,7 @@ func GetInstanceKey(db *gosql.DB) (instanceKey *InstanceKey, err error) {
 }
 
 // GetTableColumns reads column list from given table
-func GetTableColumns(db *gosql.DB, databaseName, tableName string) (*sql.ColumnList, error) {
+func GetTableColumns(db *gosql.DB, databaseName, tableName string) (*sql.ColumnList, *sql.ColumnList, error) {
 	query := fmt.Sprintf(`
 		show columns from %s.%s
 		`,
@@ -181,18 +182,24 @@ func GetTableColumns(db *gosql.DB, databaseName, tableName string) (*sql.ColumnL
 		sql.EscapeName(tableName),
 	)
 	columnNames := []string{}
+	virtualColumnNames := []string{}
 	err := sqlutils.QueryRowsMap(db, query, func(rowMap sqlutils.RowMap) error {
-		columnNames = append(columnNames, rowMap.GetString("Field"))
+		columnName := rowMap.GetString("Field")
+		columnNames = append(columnNames, columnName)
+		if strings.Contains(rowMap.GetString("Extra"), " GENERATED") {
+			log.Debugf("%s is a generated column", columnName)
+			virtualColumnNames = append(virtualColumnNames, columnName)
+		}
 		return nil
 	})
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 	if len(columnNames) == 0 {
-		return nil, log.Errorf("Found 0 columns on %s.%s. Bailing out",
+		return nil, nil, log.Errorf("Found 0 columns on %s.%s. Bailing out",
 			sql.EscapeName(databaseName),
 			sql.EscapeName(tableName),
 		)
 	}
-	return sql.NewColumnList(columnNames), nil
+	return sql.NewColumnList(columnNames), sql.NewColumnList(virtualColumnNames), nil
 }

localtests/generated-columns-add57/create.sql

@@ -0,0 +1,29 @@
+drop table if exists gh_ost_test;
+create table gh_ost_test (
+  id int auto_increment,
+  a int not null,
+  b int not null,
+  primary key(id)
+) auto_increment=1;
+
+drop event if exists gh_ost_test;
+delimiter ;;
+create event gh_ost_test
+  on schedule every 1 second
+  starts current_timestamp
+  ends current_timestamp + interval 60 second
+  on completion not preserve
+  enable
+  do
+begin
+  insert into gh_ost_test (id, a, b) values (null, 2,3);
+  insert into gh_ost_test (id, a, b) values (null, 2,4);
+  insert into gh_ost_test (id, a, b) values (null, 2,5);
+  insert into gh_ost_test (id, a, b) values (null, 2,6);
+  insert into gh_ost_test (id, a, b) values (null, 2,7);
+  insert into gh_ost_test (id, a, b) values (null, 2,8);
+  insert into gh_ost_test (id, a, b) values (null, 2,9);
+  insert into gh_ost_test (id, a, b) values (null, 2,0);
+  insert into gh_ost_test (id, a, b) values (null, 2,1);
+  insert into gh_ost_test (id, a, b) values (null, 2,2);
+end ;;

localtests/generated-columns-add57/extra_args

@@ -0,0 +1 @@
+--alter="add column sum_ab int as (a + b) virtual not null"

localtests/generated-columns-add57/ghost_columns

@@ -0,0 +1 @@
+id, a, b

localtests/generated-columns-add57/ignore_versions

@@ -0,0 +1 @@
+(5.5|5.6)

localtests/generated-columns-add57/order_by

@@ -0,0 +1 @@
+id

localtests/generated-columns-add57/orig_columns

@@ -0,0 +1 @@
+id, a, b

localtests/generated-columns-rename57/create.sql

@@ -0,0 +1,30 @@
+drop table if exists gh_ost_test;
+create table gh_ost_test (
+  id int auto_increment,
+  a int not null,
+  b int not null,
+  sum_ab int as (a + b) virtual not null,
+  primary key(id)
+) auto_increment=1;
+
+drop event if exists gh_ost_test;
+delimiter ;;
+create event gh_ost_test
+  on schedule every 1 second
+  starts current_timestamp
+  ends current_timestamp + interval 60 second
+  on completion not preserve
+  enable
+  do
+begin
+  insert into gh_ost_test (id, a, b) values (null, 2,3);
+  insert into gh_ost_test (id, a, b) values (null, 2,4);
+  insert into gh_ost_test (id, a, b) values (null, 2,5);
+  insert into gh_ost_test (id, a, b) values (null, 2,6);
+  insert into gh_ost_test (id, a, b) values (null, 2,7);
+  insert into gh_ost_test (id, a, b) values (null, 2,8);
+  insert into gh_ost_test (id, a, b) values (null, 2,9);
+  insert into gh_ost_test (id, a, b) values (null, 2,0);
+  insert into gh_ost_test (id, a, b) values (null, 2,1);
+  insert into gh_ost_test (id, a, b) values (null, 2,2);
+end ;;

localtests/generated-columns-rename57/extra_args

@@ -0,0 +1 @@
+--alter="change sum_ab total_ab int as (a + b) virtual not null" --approve-renamed-columns

localtests/generated-columns-rename57/ignore_versions

@@ -0,0 +1 @@
+(5.5|5.6)

localtests/generated-columns57/create.sql

@@ -0,0 +1,30 @@
+drop table if exists gh_ost_test;
+create table gh_ost_test (
+  id int auto_increment,
+  a int not null,
+  b int not null,
+  sum_ab int as (a + b) virtual not null,
+  primary key(id)
+) auto_increment=1;
+
+drop event if exists gh_ost_test;
+delimiter ;;
+create event gh_ost_test
+  on schedule every 1 second
+  starts current_timestamp
+  ends current_timestamp + interval 60 second
+  on completion not preserve
+  enable
+  do
+begin
+  insert into gh_ost_test (id, a, b) values (null, 2,3);
+  insert into gh_ost_test (id, a, b) values (null, 2,4);
+  insert into gh_ost_test (id, a, b) values (null, 2,5);
+  insert into gh_ost_test (id, a, b) values (null, 2,6);
+  insert into gh_ost_test (id, a, b) values (null, 2,7);
+  insert into gh_ost_test (id, a, b) values (null, 2,8);
+  insert into gh_ost_test (id, a, b) values (null, 2,9);
+  insert into gh_ost_test (id, a, b) values (null, 2,0);
+  insert into gh_ost_test (id, a, b) values (null, 2,1);
+  insert into gh_ost_test (id, a, b) values (null, 2,2);
+end ;;

localtests/generated-columns57/ignore_versions

@@ -0,0 +1 @@
+(5.5|5.6)