diff --git a/build.sh b/build.sh index 8b09426..3b8dc4b 100755 --- a/build.sh +++ b/build.sh @@ -2,7 +2,7 @@ # # -RELEASE_VERSION="1.0.20" +RELEASE_VERSION="1.0.21" function build { osname=$1 diff --git a/doc/command-line-flags.md b/doc/command-line-flags.md index 52c88cf..404703b 100644 --- a/doc/command-line-flags.md +++ b/doc/command-line-flags.md @@ -57,6 +57,12 @@ This is somewhat similar to a Nagios `n`-times test, where `n` in our case is al Optional. Default is `safe`. See more discussion in [cut-over](cut-over.md) +### discard-foreign-keys + +**Danger**: this flag will _silently_ discard any foreign keys existing on your table. + +At this time (10-2016) `gh-ost` does not support foreign keys on migrated tables (it bails out when it notices a FK on the migrated table). However, it is able to support _dropping_ of foreign keys via this flag. If you're trying to get rid of foreign keys in your environment, this is a useful flag. + ### exact-rowcount A `gh-ost` execution need to copy whatever rows you have in your existing table onto the ghost table. This can, and often be, a large number. Exactly what that number is? diff --git a/go/base/context.go b/go/base/context.go index f605735..3019954 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -70,6 +70,7 @@ type MigrationContext struct { ApproveRenamedColumns bool SkipRenamedColumns bool IsTungsten bool + DiscardForeignKeys bool config ContextConfig configMutex *sync.Mutex @@ -239,6 +240,28 @@ func (this *MigrationContext) RequiresBinlogFormatChange() bool { return this.OriginalBinlogFormat != "ROW" } +// GetApplierHostname is a safe access method to the applier hostname +func (this *MigrationContext) GetApplierHostname() string { + if this.ApplierConnectionConfig == nil { + return "" + } + if this.ApplierConnectionConfig.ImpliedKey == nil { + return "" + } + return this.ApplierConnectionConfig.ImpliedKey.Hostname +} + +// GetInspectorHostname is a safe access method to the inspector hostname +func (this *MigrationContext) GetInspectorHostname() string { + if this.InspectorConnectionConfig == nil { + return "" + } + if this.InspectorConnectionConfig.ImpliedKey == nil { + return "" + } + return this.InspectorConnectionConfig.ImpliedKey.Hostname +} + // InspectorIsAlsoApplier is `true` when the both inspector and applier are the // same database instance. This would be true when running directly on master or when // testing on replica. diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index 948d599..b2198e4 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -62,6 +62,7 @@ func main() { flag.BoolVar(&migrationContext.ApproveRenamedColumns, "approve-renamed-columns", false, "in case your `ALTER` statement renames columns, gh-ost will note that and offer its interpretation of the rename. By default gh-ost does not proceed to execute. This flag approves that gh-ost's interpretation si correct") flag.BoolVar(&migrationContext.SkipRenamedColumns, "skip-renamed-columns", false, "in case your `ALTER` statement renames columns, gh-ost will note that and offer its interpretation of the rename. By default gh-ost does not proceed to execute. This flag tells gh-ost to skip the renamed columns, i.e. to treat what gh-ost thinks are renamed columns as unrelated columns. NOTE: you may lose column data") flag.BoolVar(&migrationContext.IsTungsten, "tungsten", false, "explicitly let gh-ost know that you are running on a tungsten-replication based topology (you are likely to also provide --assume-master-host)") + flag.BoolVar(&migrationContext.DiscardForeignKeys, "discard-foreign-keys", false, "DANGER! This flag will migrate a table that has foreign keys and will NOT create foreign keys on the ghost table, thus your altered table will have NO foreign keys. This is useful for intentional dropping of foreign keys") executeFlag := flag.Bool("execute", false, "actually execute the alter & migrate the table. Default is noop: do some tests and exit") flag.BoolVar(&migrationContext.TestOnReplica, "test-on-replica", false, "Have the migration run on a replica, not on the master. At the end of migration replication is stopped, and tables are swapped and immediately swap-revert. Replication remains stopped and you can compare the two tables for building trust") diff --git a/go/logic/hooks.go b/go/logic/hooks.go index d288218..25d745a 100644 --- a/go/logic/hooks.go +++ b/go/logic/hooks.go @@ -59,8 +59,8 @@ func (this *HooksExecutor) applyEnvironmentVairables(extraVariables ...string) [ env = append(env, fmt.Sprintf("GH_OST_ESTIMATED_ROWS=%d", estimatedRows)) totalRowsCopied := this.migrationContext.GetTotalRowsCopied() env = append(env, fmt.Sprintf("GH_OST_COPIED_ROWS=%d", totalRowsCopied)) - env = append(env, fmt.Sprintf("GH_OST_MIGRATED_HOST=%s", this.migrationContext.ApplierConnectionConfig.ImpliedKey.Hostname)) - env = append(env, fmt.Sprintf("GH_OST_INSPECTED_HOST=%s", this.migrationContext.InspectorConnectionConfig.ImpliedKey.Hostname)) + env = append(env, fmt.Sprintf("GH_OST_MIGRATED_HOST=%s", this.migrationContext.GetApplierHostname())) + env = append(env, fmt.Sprintf("GH_OST_INSPECTED_HOST=%s", this.migrationContext.GetInspectorHostname())) env = append(env, fmt.Sprintf("GH_OST_EXECUTING_HOST=%s", this.migrationContext.Hostname)) env = append(env, fmt.Sprintf("GH_OST_HOOKS_HINT=%s", this.migrationContext.HooksHintMessage)) diff --git a/go/logic/inspect.go b/go/logic/inspect.go index 2c52780..af84b4c 100644 --- a/go/logic/inspect.go +++ b/go/logic/inspect.go @@ -66,7 +66,7 @@ func (this *Inspector) ValidateOriginalTable() (err error) { if err := this.validateTable(); err != nil { return err } - if err := this.validateTableForeignKeys(); err != nil { + if err := this.validateTableForeignKeys(this.migrationContext.DiscardForeignKeys); err != nil { return err } if err := this.validateTableTriggers(); err != nil { @@ -364,9 +364,11 @@ func (this *Inspector) validateTable() error { } // validateTableForeignKeys makes sure no foreign keys exist on the migrated table -func (this *Inspector) validateTableForeignKeys() error { +func (this *Inspector) validateTableForeignKeys(allowChildForeignKeys bool) error { query := ` - SELECT TABLE_SCHEMA, TABLE_NAME + SELECT + SUM(REFERENCED_TABLE_NAME IS NOT NULL AND TABLE_SCHEMA=? AND TABLE_NAME=?) as num_child_side_fk, + SUM(REFERENCED_TABLE_NAME IS NOT NULL AND REFERENCED_TABLE_SCHEMA=? AND REFERENCED_TABLE_NAME=?) as num_parent_side_fk FROM INFORMATION_SCHEMA.KEY_COLUMN_USAGE WHERE REFERENCED_TABLE_NAME IS NOT NULL @@ -374,24 +376,34 @@ func (this *Inspector) validateTableForeignKeys() error { OR (REFERENCED_TABLE_SCHEMA=? AND REFERENCED_TABLE_NAME=?) ) ` - numForeignKeys := 0 - err := sqlutils.QueryRowsMap(this.db, query, func(rowMap sqlutils.RowMap) error { - fkSchema := rowMap.GetString("TABLE_SCHEMA") - fkTable := rowMap.GetString("TABLE_NAME") - log.Infof("Found foreign key on %s.%s related to %s.%s", sql.EscapeName(fkSchema), sql.EscapeName(fkTable), sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName)) - numForeignKeys++ + numParentForeignKeys := 0 + numChildForeignKeys := 0 + err := sqlutils.QueryRowsMap(this.db, query, func(m sqlutils.RowMap) error { + numChildForeignKeys = m.GetInt("num_child_side_fk") + numParentForeignKeys = m.GetInt("num_parent_side_fk") return nil }, this.migrationContext.DatabaseName, this.migrationContext.OriginalTableName, this.migrationContext.DatabaseName, this.migrationContext.OriginalTableName, + this.migrationContext.DatabaseName, + this.migrationContext.OriginalTableName, + this.migrationContext.DatabaseName, + this.migrationContext.OriginalTableName, ) if err != nil { return err } - if numForeignKeys > 0 { - return log.Errorf("Found %d foreign keys related to %s.%s. Foreign keys are not supported. Bailing out", numForeignKeys, sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName)) + if numParentForeignKeys > 0 { + return log.Errorf("Found %d parent-side foreign keys on %s.%s. Parent-side foreign keys are not supported. Bailing out", numParentForeignKeys, sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName)) + } + if numChildForeignKeys > 0 { + if allowChildForeignKeys { + log.Debugf("Foreign keys found and will be dropped, as per given --discard-foreign-keys flag") + return nil + } + return log.Errorf("Found %d child-side foreign keys on %s.%s. Child-side foreign keys are not supported. Bailing out", numChildForeignKeys, sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName)) } log.Debugf("Validated no foreign keys exist on table") return nil diff --git a/go/logic/migrator.go b/go/logic/migrator.go index 2ac4c37..7c96871 100644 --- a/go/logic/migrator.go +++ b/go/logic/migrator.go @@ -616,16 +616,22 @@ func (this *Migrator) initiateInspector() (err error) { } // So far so good, table is accessible and valid. // Let's get master connection config - if this.migrationContext.ApplierConnectionConfig, err = this.inspector.getMasterConnectionConfig(); err != nil { - return err - } - if this.migrationContext.AssumeMasterHostname != "" { - if key, err := mysql.ParseRawInstanceKeyLoose(this.migrationContext.AssumeMasterHostname); err != nil { + if this.migrationContext.AssumeMasterHostname == "" { + // No forced master host; detect master + if this.migrationContext.ApplierConnectionConfig, err = this.inspector.getMasterConnectionConfig(); err != nil { return err - } else { - this.migrationContext.ApplierConnectionConfig.Key = *key } + log.Infof("Master found to be %+v", *this.migrationContext.ApplierConnectionConfig.ImpliedKey) + } else { + // Forced master host. + key, err := mysql.ParseRawInstanceKeyLoose(this.migrationContext.AssumeMasterHostname) + if err != nil { + return err + } + this.migrationContext.ApplierConnectionConfig = this.migrationContext.InspectorConnectionConfig.DuplicateCredentials(*key) + log.Infof("Master forced to be %+v", *this.migrationContext.ApplierConnectionConfig.ImpliedKey) } + // validate configs if this.migrationContext.TestOnReplica || this.migrationContext.MigrateOnReplica { if this.migrationContext.InspectorIsAlsoApplier() { return fmt.Errorf("Instructed to --test-on-replica or --migrate-on-replica, but the server we connect to doesn't seem to be a replica") @@ -638,13 +644,12 @@ func (this *Migrator) initiateInspector() (err error) { this.migrationContext.AddThrottleControlReplicaKey(this.migrationContext.InspectorConnectionConfig.Key) } } else if this.migrationContext.InspectorIsAlsoApplier() && !this.migrationContext.AllowedRunningOnMaster { - return fmt.Errorf("It seems like this migration attempt to run directly on master. Preferably it would be executed on a replica (and this reduces load from the master). To proceed please provide --allow-on-master") + return fmt.Errorf("It seems like this migration attempt to run directly on master. Preferably it would be executed on a replica (and this reduces load from the master). To proceed please provide --allow-on-master. Inspector config=%+v, applier config=%+v", this.migrationContext.InspectorConnectionConfig, this.migrationContext.ApplierConnectionConfig) } if err := this.inspector.validateLogSlaveUpdates(); err != nil { return err } - log.Infof("Master found to be %+v", *this.migrationContext.ApplierConnectionConfig.ImpliedKey) return nil } diff --git a/go/mysql/connection.go b/go/mysql/connection.go index 649da93..daae75e 100644 --- a/go/mysql/connection.go +++ b/go/mysql/connection.go @@ -26,12 +26,10 @@ func NewConnectionConfig() *ConnectionConfig { return config } -func (this *ConnectionConfig) Duplicate() *ConnectionConfig { +// DuplicateCredentials creates a new connection config with given key and with same credentials as this config +func (this *ConnectionConfig) DuplicateCredentials(key InstanceKey) *ConnectionConfig { config := &ConnectionConfig{ - Key: InstanceKey{ - Hostname: this.Key.Hostname, - Port: this.Key.Port, - }, + Key: key, User: this.User, Password: this.Password, } @@ -39,6 +37,10 @@ func (this *ConnectionConfig) Duplicate() *ConnectionConfig { return config } +func (this *ConnectionConfig) Duplicate() *ConnectionConfig { + return this.DuplicateCredentials(this.Key) +} + func (this *ConnectionConfig) String() string { return fmt.Sprintf("%s, user=%s", this.Key.DisplayString(), this.User) } diff --git a/go/mysql/connection_test.go b/go/mysql/connection_test.go new file mode 100644 index 0000000..657feb0 --- /dev/null +++ b/go/mysql/connection_test.go @@ -0,0 +1,57 @@ +/* + Copyright 2016 GitHub Inc. + See https://github.com/github/gh-ost/blob/master/LICENSE +*/ + +package mysql + +import ( + "testing" + + "github.com/outbrain/golib/log" + test "github.com/outbrain/golib/tests" +) + +func init() { + log.SetLevel(log.ERROR) +} + +func TestNewConnectionConfig(t *testing.T) { + c := NewConnectionConfig() + test.S(t).ExpectEquals(c.Key.Hostname, "") + test.S(t).ExpectEquals(c.Key.Port, 0) + test.S(t).ExpectEquals(c.ImpliedKey.Hostname, "") + test.S(t).ExpectEquals(c.ImpliedKey.Port, 0) + test.S(t).ExpectEquals(c.User, "") + test.S(t).ExpectEquals(c.Password, "") +} + +func TestDuplicateCredentials(t *testing.T) { + c := NewConnectionConfig() + c.Key = InstanceKey{Hostname: "myhost", Port: 3306} + c.User = "gromit" + c.Password = "penguin" + + dup := c.DuplicateCredentials(InstanceKey{Hostname: "otherhost", Port: 3310}) + test.S(t).ExpectEquals(dup.Key.Hostname, "otherhost") + test.S(t).ExpectEquals(dup.Key.Port, 3310) + test.S(t).ExpectEquals(dup.ImpliedKey.Hostname, "otherhost") + test.S(t).ExpectEquals(dup.ImpliedKey.Port, 3310) + test.S(t).ExpectEquals(dup.User, "gromit") + test.S(t).ExpectEquals(dup.Password, "penguin") +} + +func TestDuplicate(t *testing.T) { + c := NewConnectionConfig() + c.Key = InstanceKey{Hostname: "myhost", Port: 3306} + c.User = "gromit" + c.Password = "penguin" + + dup := c.Duplicate() + test.S(t).ExpectEquals(dup.Key.Hostname, "myhost") + test.S(t).ExpectEquals(dup.Key.Port, 3306) + test.S(t).ExpectEquals(dup.ImpliedKey.Hostname, "myhost") + test.S(t).ExpectEquals(dup.ImpliedKey.Port, 3306) + test.S(t).ExpectEquals(dup.User, "gromit") + test.S(t).ExpectEquals(dup.Password, "penguin") +} diff --git a/localtests/discard-fk/create.sql b/localtests/discard-fk/create.sql new file mode 100644 index 0000000..6ef2d5c --- /dev/null +++ b/localtests/discard-fk/create.sql @@ -0,0 +1,32 @@ +drop table if exists gh_ost_test_child; +drop table if exists gh_ost_test; +drop table if exists gh_ost_test_fk_parent; +create table gh_ost_test_fk_parent ( + id int auto_increment, + ts timestamp, + primary key(id) +); +create table gh_ost_test ( + id int auto_increment, + i int not null, + parent_id int not null, + primary key(id), + constraint test_fk foreign key (parent_id) references gh_ost_test_fk_parent (id) on delete no action +) auto_increment=1; + +insert into gh_ost_test_fk_parent (id) values (1),(2),(3); + +drop event if exists gh_ost_test; +delimiter ;; +create event gh_ost_test + on schedule every 1 second + starts current_timestamp + ends current_timestamp + interval 60 second + on completion not preserve + enable + do +begin + insert into gh_ost_test values (null, 11, 1); + insert into gh_ost_test values (null, 13, 2); + insert into gh_ost_test values (null, 17, 3); +end ;; diff --git a/localtests/discard-fk/extra_args b/localtests/discard-fk/extra_args new file mode 100644 index 0000000..2d00c49 --- /dev/null +++ b/localtests/discard-fk/extra_args @@ -0,0 +1 @@ +--discard-foreign-keys diff --git a/localtests/fail-fk-parent/create.sql b/localtests/fail-fk-parent/create.sql new file mode 100644 index 0000000..ec713d6 --- /dev/null +++ b/localtests/fail-fk-parent/create.sql @@ -0,0 +1,41 @@ +drop table if exists gh_ost_test_child; +drop table if exists gh_ost_test; +create table gh_ost_test ( + id int auto_increment, + primary key(id) +) engine=innodb auto_increment=1; + +create table gh_ost_test_child ( + id int auto_increment, + i int not null, + parent_id int not null, + constraint test_fk foreign key (parent_id) references gh_ost_test (id) on delete no action, + primary key(id) +) engine=innodb; +insert into gh_ost_test (id) values (1),(2),(3); + +drop event if exists gh_ost_test; +drop event if exists gh_ost_test_cleanup; + +delimiter ;; +create event gh_ost_test + on schedule every 1 second + starts current_timestamp + ends current_timestamp + interval 60 second + on completion not preserve + enable + do +begin + insert into gh_ost_test_child values (null, 11, 1); + insert into gh_ost_test_child values (null, 13, 2); + insert into gh_ost_test_child values (null, 17, 3); +end ;; + +create event gh_ost_test_cleanup + on schedule at current_timestamp + interval 60 second + on completion not preserve + enable + do +begin + drop table if exists gh_ost_test_child; +end ;; diff --git a/localtests/fail-fk-parent/expect_failure b/localtests/fail-fk-parent/expect_failure new file mode 100644 index 0000000..e69de29 diff --git a/localtests/fail-fk-parent/extra_args b/localtests/fail-fk-parent/extra_args new file mode 100644 index 0000000..2d00c49 --- /dev/null +++ b/localtests/fail-fk-parent/extra_args @@ -0,0 +1 @@ +--discard-foreign-keys diff --git a/localtests/fail-fk/create.sql b/localtests/fail-fk/create.sql new file mode 100644 index 0000000..6ef2d5c --- /dev/null +++ b/localtests/fail-fk/create.sql @@ -0,0 +1,32 @@ +drop table if exists gh_ost_test_child; +drop table if exists gh_ost_test; +drop table if exists gh_ost_test_fk_parent; +create table gh_ost_test_fk_parent ( + id int auto_increment, + ts timestamp, + primary key(id) +); +create table gh_ost_test ( + id int auto_increment, + i int not null, + parent_id int not null, + primary key(id), + constraint test_fk foreign key (parent_id) references gh_ost_test_fk_parent (id) on delete no action +) auto_increment=1; + +insert into gh_ost_test_fk_parent (id) values (1),(2),(3); + +drop event if exists gh_ost_test; +delimiter ;; +create event gh_ost_test + on schedule every 1 second + starts current_timestamp + ends current_timestamp + interval 60 second + on completion not preserve + enable + do +begin + insert into gh_ost_test values (null, 11, 1); + insert into gh_ost_test values (null, 13, 2); + insert into gh_ost_test values (null, 17, 3); +end ;; diff --git a/localtests/fail-fk/expect_failure b/localtests/fail-fk/expect_failure new file mode 100644 index 0000000..e69de29 diff --git a/localtests/test.sh b/localtests/test.sh index f12b66f..fc0c9b6 100755 --- a/localtests/test.sh +++ b/localtests/test.sh @@ -95,7 +95,18 @@ test_single() { echo_dot bash $exec_command_file 1> $test_logfile 2>&1 - if [ $? -ne 0 ] ; then + execution_result=$? + + if [ -f $tests_path/$test_name/expect_failure ] ; then + if [ $execution_result -eq 0 ] ; then + echo + echo "ERROR $test_name execution was expected to exit on error but did not. cat $test_logfile" + return 1 + fi + return 0 + fi + + if [ $execution_result -ne 0 ] ; then echo echo "ERROR $test_name execution failure. cat $test_logfile" return 1