Merge branch 'master' into tz-a-different-approach

build.sh

@@ -2,7 +2,7 @@
 #
 #
 
-RELEASE_VERSION="1.0.20"
+RELEASE_VERSION="1.0.21"
 
 function build {
     osname=$1

doc/command-line-flags.md

@@ -57,6 +57,12 @@ This is somewhat similar to a Nagios `n`-times test, where `n` in our case is al
 
 Optional. Default is `safe`. See more discussion in [cut-over](cut-over.md)
 
+### discard-foreign-keys
+
+**Danger**: this flag will _silently_ discard any foreign keys existing on your table.
+
+At this time (10-2016) `gh-ost` does not support foreign keys on migrated tables (it bails out when it notices a FK on the migrated table). However, it is able to support _dropping_ of foreign keys via this flag. If you're trying to get rid of foreign keys in your environment, this is a useful flag.
+
 ### exact-rowcount
 
 A `gh-ost` execution need to copy whatever rows you have in your existing table onto the ghost table. This can, and often be, a large number. Exactly what that number is?

go/base/context.go

@@ -70,6 +70,7 @@ type MigrationContext struct {
 	ApproveRenamedColumns    bool
 	SkipRenamedColumns       bool
 	IsTungsten               bool
+	DiscardForeignKeys       bool
 
 	config      ContextConfig
 	configMutex *sync.Mutex
@@ -239,6 +240,28 @@ func (this *MigrationContext) RequiresBinlogFormatChange() bool {
 	return this.OriginalBinlogFormat != "ROW"
 }
 
+// GetApplierHostname is a safe access method to the applier hostname
+func (this *MigrationContext) GetApplierHostname() string {
+	if this.ApplierConnectionConfig == nil {
+		return ""
+	}
+	if this.ApplierConnectionConfig.ImpliedKey == nil {
+		return ""
+	}
+	return this.ApplierConnectionConfig.ImpliedKey.Hostname
+}
+
+// GetInspectorHostname is a safe access method to the inspector hostname
+func (this *MigrationContext) GetInspectorHostname() string {
+	if this.InspectorConnectionConfig == nil {
+		return ""
+	}
+	if this.InspectorConnectionConfig.ImpliedKey == nil {
+		return ""
+	}
+	return this.InspectorConnectionConfig.ImpliedKey.Hostname
+}
+
 // InspectorIsAlsoApplier is `true` when the both inspector and applier are the
 // same database instance. This would be true when running directly on master or when
 // testing on replica.

go/cmd/gh-ost/main.go

@@ -62,6 +62,7 @@ func main() {
 	flag.BoolVar(&migrationContext.ApproveRenamedColumns, "approve-renamed-columns", false, "in case your `ALTER` statement renames columns, gh-ost will note that and offer its interpretation of the rename. By default gh-ost does not proceed to execute. This flag approves that gh-ost's interpretation si correct")
 	flag.BoolVar(&migrationContext.SkipRenamedColumns, "skip-renamed-columns", false, "in case your `ALTER` statement renames columns, gh-ost will note that and offer its interpretation of the rename. By default gh-ost does not proceed to execute. This flag tells gh-ost to skip the renamed columns, i.e. to treat what gh-ost thinks are renamed columns as unrelated columns. NOTE: you may lose column data")
 	flag.BoolVar(&migrationContext.IsTungsten, "tungsten", false, "explicitly let gh-ost know that you are running on a tungsten-replication based topology (you are likely to also provide --assume-master-host)")
+	flag.BoolVar(&migrationContext.DiscardForeignKeys, "discard-foreign-keys", false, "DANGER! This flag will migrate a table that has foreign keys and will NOT create foreign keys on the ghost table, thus your altered table will have NO foreign keys. This is useful for intentional dropping of foreign keys")
 
 	executeFlag := flag.Bool("execute", false, "actually execute the alter & migrate the table. Default is noop: do some tests and exit")
 	flag.BoolVar(&migrationContext.TestOnReplica, "test-on-replica", false, "Have the migration run on a replica, not on the master. At the end of migration replication is stopped, and tables are swapped and immediately swap-revert. Replication remains stopped and you can compare the two tables for building trust")

go/logic/hooks.go

@@ -59,8 +59,8 @@ func (this *HooksExecutor) applyEnvironmentVairables(extraVariables ...string) [
 	env = append(env, fmt.Sprintf("GH_OST_ESTIMATED_ROWS=%d", estimatedRows))
 	totalRowsCopied := this.migrationContext.GetTotalRowsCopied()
 	env = append(env, fmt.Sprintf("GH_OST_COPIED_ROWS=%d", totalRowsCopied))
-	env = append(env, fmt.Sprintf("GH_OST_MIGRATED_HOST=%s", this.migrationContext.ApplierConnectionConfig.ImpliedKey.Hostname))
+	env = append(env, fmt.Sprintf("GH_OST_MIGRATED_HOST=%s", this.migrationContext.GetApplierHostname()))
-	env = append(env, fmt.Sprintf("GH_OST_INSPECTED_HOST=%s", this.migrationContext.InspectorConnectionConfig.ImpliedKey.Hostname))
+	env = append(env, fmt.Sprintf("GH_OST_INSPECTED_HOST=%s", this.migrationContext.GetInspectorHostname()))
 	env = append(env, fmt.Sprintf("GH_OST_EXECUTING_HOST=%s", this.migrationContext.Hostname))
 	env = append(env, fmt.Sprintf("GH_OST_HOOKS_HINT=%s", this.migrationContext.HooksHintMessage))
 

go/logic/inspect.go

@@ -66,7 +66,7 @@ func (this *Inspector) ValidateOriginalTable() (err error) {
 	if err := this.validateTable(); err != nil {
 		return err
 	}
-	if err := this.validateTableForeignKeys(); err != nil {
+	if err := this.validateTableForeignKeys(this.migrationContext.DiscardForeignKeys); err != nil {
 		return err
 	}
 	if err := this.validateTableTriggers(); err != nil {
@@ -364,9 +364,11 @@ func (this *Inspector) validateTable() error {
 }
 
 // validateTableForeignKeys makes sure no foreign keys exist on the migrated table
-func (this *Inspector) validateTableForeignKeys() error {
+func (this *Inspector) validateTableForeignKeys(allowChildForeignKeys bool) error {
 	query := `
-		SELECT TABLE_SCHEMA, TABLE_NAME
+		SELECT
+			SUM(REFERENCED_TABLE_NAME IS NOT NULL AND TABLE_SCHEMA=? AND TABLE_NAME=?) as num_child_side_fk,
+			SUM(REFERENCED_TABLE_NAME IS NOT NULL AND REFERENCED_TABLE_SCHEMA=? AND REFERENCED_TABLE_NAME=?) as num_parent_side_fk
 		FROM INFORMATION_SCHEMA.KEY_COLUMN_USAGE
 		WHERE
 				REFERENCED_TABLE_NAME IS NOT NULL
@@ -374,24 +376,34 @@ func (this *Inspector) validateTableForeignKeys() error {
 					OR (REFERENCED_TABLE_SCHEMA=? AND REFERENCED_TABLE_NAME=?)
 				)
 	`
-	numForeignKeys := 0
+	numParentForeignKeys := 0
-	err := sqlutils.QueryRowsMap(this.db, query, func(rowMap sqlutils.RowMap) error {
+	numChildForeignKeys := 0
-		fkSchema := rowMap.GetString("TABLE_SCHEMA")
+	err := sqlutils.QueryRowsMap(this.db, query, func(m sqlutils.RowMap) error {
-		fkTable := rowMap.GetString("TABLE_NAME")
+		numChildForeignKeys = m.GetInt("num_child_side_fk")
-		log.Infof("Found foreign key on %s.%s related to %s.%s", sql.EscapeName(fkSchema), sql.EscapeName(fkTable), sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName))
+		numParentForeignKeys = m.GetInt("num_parent_side_fk")
-		numForeignKeys++
 		return nil
 	},
 		this.migrationContext.DatabaseName,
 		this.migrationContext.OriginalTableName,
 		this.migrationContext.DatabaseName,
 		this.migrationContext.OriginalTableName,
+		this.migrationContext.DatabaseName,
+		this.migrationContext.OriginalTableName,
+		this.migrationContext.DatabaseName,
+		this.migrationContext.OriginalTableName,
 	)
 	if err != nil {
 		return err
 	}
-	if numForeignKeys > 0 {
+	if numParentForeignKeys > 0 {
-		return log.Errorf("Found %d foreign keys related to %s.%s. Foreign keys are not supported. Bailing out", numForeignKeys, sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName))
+		return log.Errorf("Found %d parent-side foreign keys on %s.%s. Parent-side foreign keys are not supported. Bailing out", numParentForeignKeys, sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName))
+	}
+	if numChildForeignKeys > 0 {
+		if allowChildForeignKeys {
+			log.Debugf("Foreign keys found and will be dropped, as per given --discard-foreign-keys flag")
+			return nil
+		}
+		return log.Errorf("Found %d child-side foreign keys on %s.%s. Child-side foreign keys are not supported. Bailing out", numChildForeignKeys, sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName))
 	}
 	log.Debugf("Validated no foreign keys exist on table")
 	return nil

go/logic/migrator.go

@@ -616,16 +616,22 @@ func (this *Migrator) initiateInspector() (err error) {
 	}
 	// So far so good, table is accessible and valid.
 	// Let's get master connection config
-	if this.migrationContext.ApplierConnectionConfig, err = this.inspector.getMasterConnectionConfig(); err != nil {
+	if this.migrationContext.AssumeMasterHostname == "" {
-		return err
+		// No forced master host; detect master
-	}
+		if this.migrationContext.ApplierConnectionConfig, err = this.inspector.getMasterConnectionConfig(); err != nil {
-	if this.migrationContext.AssumeMasterHostname != "" {
-		if key, err := mysql.ParseRawInstanceKeyLoose(this.migrationContext.AssumeMasterHostname); err != nil {
 			return err
-		} else {
-			this.migrationContext.ApplierConnectionConfig.Key = *key
 		}
+		log.Infof("Master found to be %+v", *this.migrationContext.ApplierConnectionConfig.ImpliedKey)
+	} else {
+		// Forced master host.
+		key, err := mysql.ParseRawInstanceKeyLoose(this.migrationContext.AssumeMasterHostname)
+		if err != nil {
+			return err
+		}
+		this.migrationContext.ApplierConnectionConfig = this.migrationContext.InspectorConnectionConfig.DuplicateCredentials(*key)
+		log.Infof("Master forced to be %+v", *this.migrationContext.ApplierConnectionConfig.ImpliedKey)
 	}
+	// validate configs
 	if this.migrationContext.TestOnReplica || this.migrationContext.MigrateOnReplica {
 		if this.migrationContext.InspectorIsAlsoApplier() {
 			return fmt.Errorf("Instructed to --test-on-replica or --migrate-on-replica, but the server we connect to doesn't seem to be a replica")
@@ -638,13 +644,12 @@ func (this *Migrator) initiateInspector() (err error) {
 			this.migrationContext.AddThrottleControlReplicaKey(this.migrationContext.InspectorConnectionConfig.Key)
 		}
 	} else if this.migrationContext.InspectorIsAlsoApplier() && !this.migrationContext.AllowedRunningOnMaster {
-		return fmt.Errorf("It seems like this migration attempt to run directly on master. Preferably it would be executed on a replica (and this reduces load from the master). To proceed please provide --allow-on-master")
+		return fmt.Errorf("It seems like this migration attempt to run directly on master. Preferably it would be executed on a replica (and this reduces load from the master). To proceed please provide --allow-on-master. Inspector config=%+v, applier config=%+v", this.migrationContext.InspectorConnectionConfig, this.migrationContext.ApplierConnectionConfig)
 	}
 	if err := this.inspector.validateLogSlaveUpdates(); err != nil {
 		return err
 	}
 
-	log.Infof("Master found to be %+v", *this.migrationContext.ApplierConnectionConfig.ImpliedKey)
 	return nil
 }
 

go/mysql/connection.go

@@ -26,12 +26,10 @@ func NewConnectionConfig() *ConnectionConfig {
 	return config
 }
 
-func (this *ConnectionConfig) Duplicate() *ConnectionConfig {
+// DuplicateCredentials creates a new connection config with given key and with same credentials as this config
+func (this *ConnectionConfig) DuplicateCredentials(key InstanceKey) *ConnectionConfig {
 	config := &ConnectionConfig{
-		Key: InstanceKey{
+		Key:      key,
-			Hostname: this.Key.Hostname,
-			Port:     this.Key.Port,
-		},
 		User:     this.User,
 		Password: this.Password,
 	}
@@ -39,6 +37,10 @@ func (this *ConnectionConfig) Duplicate() *ConnectionConfig {
 	return config
 }
 
+func (this *ConnectionConfig) Duplicate() *ConnectionConfig {
+	return this.DuplicateCredentials(this.Key)
+}
+
 func (this *ConnectionConfig) String() string {
 	return fmt.Sprintf("%s, user=%s", this.Key.DisplayString(), this.User)
 }

go/mysql/connection_test.go

@@ -0,0 +1,57 @@
+/*
+   Copyright 2016 GitHub Inc.
+	 See https://github.com/github/gh-ost/blob/master/LICENSE
+*/
+
+package mysql
+
+import (
+	"testing"
+
+	"github.com/outbrain/golib/log"
+	test "github.com/outbrain/golib/tests"
+)
+
+func init() {
+	log.SetLevel(log.ERROR)
+}
+
+func TestNewConnectionConfig(t *testing.T) {
+	c := NewConnectionConfig()
+	test.S(t).ExpectEquals(c.Key.Hostname, "")
+	test.S(t).ExpectEquals(c.Key.Port, 0)
+	test.S(t).ExpectEquals(c.ImpliedKey.Hostname, "")
+	test.S(t).ExpectEquals(c.ImpliedKey.Port, 0)
+	test.S(t).ExpectEquals(c.User, "")
+	test.S(t).ExpectEquals(c.Password, "")
+}
+
+func TestDuplicateCredentials(t *testing.T) {
+	c := NewConnectionConfig()
+	c.Key = InstanceKey{Hostname: "myhost", Port: 3306}
+	c.User = "gromit"
+	c.Password = "penguin"
+
+	dup := c.DuplicateCredentials(InstanceKey{Hostname: "otherhost", Port: 3310})
+	test.S(t).ExpectEquals(dup.Key.Hostname, "otherhost")
+	test.S(t).ExpectEquals(dup.Key.Port, 3310)
+	test.S(t).ExpectEquals(dup.ImpliedKey.Hostname, "otherhost")
+	test.S(t).ExpectEquals(dup.ImpliedKey.Port, 3310)
+	test.S(t).ExpectEquals(dup.User, "gromit")
+	test.S(t).ExpectEquals(dup.Password, "penguin")
+}
+
+func TestDuplicate(t *testing.T) {
+	c := NewConnectionConfig()
+	c.Key = InstanceKey{Hostname: "myhost", Port: 3306}
+	c.User = "gromit"
+	c.Password = "penguin"
+
+	dup := c.Duplicate()
+	test.S(t).ExpectEquals(dup.Key.Hostname, "myhost")
+	test.S(t).ExpectEquals(dup.Key.Port, 3306)
+	test.S(t).ExpectEquals(dup.ImpliedKey.Hostname, "myhost")
+	test.S(t).ExpectEquals(dup.ImpliedKey.Port, 3306)
+	test.S(t).ExpectEquals(dup.User, "gromit")
+	test.S(t).ExpectEquals(dup.Password, "penguin")
+}

localtests/discard-fk/create.sql

@@ -0,0 +1,32 @@
+drop table if exists gh_ost_test_child;
+drop table if exists gh_ost_test;
+drop table if exists gh_ost_test_fk_parent;
+create table gh_ost_test_fk_parent (
+  id int auto_increment,
+  ts timestamp,
+  primary key(id)
+);
+create table gh_ost_test (
+  id int auto_increment,
+  i int not null,
+  parent_id int not null,
+  primary key(id),
+  constraint test_fk foreign key (parent_id) references gh_ost_test_fk_parent (id) on delete no action
+) auto_increment=1;
+
+insert into gh_ost_test_fk_parent (id) values (1),(2),(3);
+
+drop event if exists gh_ost_test;
+delimiter ;;
+create event gh_ost_test
+  on schedule every 1 second
+  starts current_timestamp
+  ends current_timestamp + interval 60 second
+  on completion not preserve
+  enable
+  do
+begin
+  insert into gh_ost_test values (null, 11, 1);
+  insert into gh_ost_test values (null, 13, 2);
+  insert into gh_ost_test values (null, 17, 3);
+end ;;

localtests/discard-fk/extra_args

@@ -0,0 +1 @@
+--discard-foreign-keys

localtests/fail-fk-parent/create.sql

@@ -0,0 +1,41 @@
+drop table if exists gh_ost_test_child;
+drop table if exists gh_ost_test;
+create table gh_ost_test (
+  id int auto_increment,
+  primary key(id)
+) engine=innodb auto_increment=1;
+
+create table gh_ost_test_child (
+  id int auto_increment,
+  i int not null,
+  parent_id int not null,
+  constraint test_fk foreign key (parent_id) references gh_ost_test (id) on delete no action,
+  primary key(id)
+) engine=innodb;
+insert into gh_ost_test (id) values (1),(2),(3);
+
+drop event if exists gh_ost_test;
+drop event if exists gh_ost_test_cleanup;
+
+delimiter ;;
+create event gh_ost_test
+  on schedule every 1 second
+  starts current_timestamp
+  ends current_timestamp + interval 60 second
+  on completion not preserve
+  enable
+  do
+begin
+  insert into gh_ost_test_child values (null, 11, 1);
+  insert into gh_ost_test_child values (null, 13, 2);
+  insert into gh_ost_test_child values (null, 17, 3);
+end ;;
+
+create event gh_ost_test_cleanup
+  on schedule at current_timestamp + interval 60 second
+  on completion not preserve
+  enable
+  do
+begin
+  drop table if exists gh_ost_test_child;
+end ;;

localtests/fail-fk-parent/extra_args

@@ -0,0 +1 @@
+--discard-foreign-keys

localtests/fail-fk/create.sql

@@ -0,0 +1,32 @@
+drop table if exists gh_ost_test_child;
+drop table if exists gh_ost_test;
+drop table if exists gh_ost_test_fk_parent;
+create table gh_ost_test_fk_parent (
+  id int auto_increment,
+  ts timestamp,
+  primary key(id)
+);
+create table gh_ost_test (
+  id int auto_increment,
+  i int not null,
+  parent_id int not null,
+  primary key(id),
+  constraint test_fk foreign key (parent_id) references gh_ost_test_fk_parent (id) on delete no action
+) auto_increment=1;
+
+insert into gh_ost_test_fk_parent (id) values (1),(2),(3);
+
+drop event if exists gh_ost_test;
+delimiter ;;
+create event gh_ost_test
+  on schedule every 1 second
+  starts current_timestamp
+  ends current_timestamp + interval 60 second
+  on completion not preserve
+  enable
+  do
+begin
+  insert into gh_ost_test values (null, 11, 1);
+  insert into gh_ost_test values (null, 13, 2);
+  insert into gh_ost_test values (null, 17, 3);
+end ;;

localtests/test.sh

@@ -95,7 +95,18 @@ test_single() {
   echo_dot
   bash $exec_command_file 1> $test_logfile 2>&1
 
-  if [ $? -ne 0 ] ; then
+  execution_result=$?
+
+  if [ -f $tests_path/$test_name/expect_failure ] ; then
+    if [ $execution_result -eq 0 ] ; then
+      echo
+      echo "ERROR $test_name execution was expected to exit on error but did not. cat $test_logfile"
+      return 1
+    fi
+    return 0
+  fi
+
+  if [ $execution_result -ne 0 ] ; then
     echo
     echo "ERROR $test_name execution failure. cat $test_logfile"
     return 1