From 6f3d54a13698b5079d285f8c2abadfd133021844 Mon Sep 17 00:00:00 2001 From: zj118228 Date: Tue, 23 Jan 2018 13:51:41 +0800 Subject: [PATCH 1/2] Aliyun RDS hide some mysql variables for secrity so gh-ost will get ilegal values. Add --aliyun-rds flag to avoid getting them. --- go/base/context.go | 3 +++ go/base/utils.go | 13 +++++++++++-- go/cmd/gh-ost/main.go | 2 ++ go/logic/applier.go | 6 ++++-- go/logic/inspect.go | 6 ++++-- go/mysql/utils.go | 6 ++++-- 6 files changed, 28 insertions(+), 8 deletions(-) diff --git a/go/base/context.go b/go/base/context.go index 23fe6f6..5f63c88 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -91,6 +91,7 @@ type MigrationContext struct { SkipRenamedColumns bool IsTungsten bool DiscardForeignKeys bool + AliyunRDS bool config ContextConfig configMutex *sync.Mutex @@ -214,6 +215,8 @@ type ContextConfig struct { } } +var Context *MigrationContext + func NewMigrationContext() *MigrationContext { return &MigrationContext{ Uuid: uuid.NewV4().String(), diff --git a/go/base/utils.go b/go/base/utils.go index 727bc57..8d9e4f0 100644 --- a/go/base/utils.go +++ b/go/base/utils.go @@ -65,16 +65,25 @@ func StringContainsAll(s string, substrings ...string) bool { } func ValidateConnection(db *gosql.DB, connectionConfig *mysql.ConnectionConfig) (string, error) { - query := `select @@global.port, @@global.version` + versionQuery := `select @@global.version` var port, extraPort int var version string - if err := db.QueryRow(query).Scan(&port, &version); err != nil { + if err := db.QueryRow(versionQuery).Scan(&version); err != nil { return "", err } extraPortQuery := `select @@global.extra_port` if err := db.QueryRow(extraPortQuery).Scan(&extraPort); err != nil { // swallow this error. not all servers support extra_port } + // AliyunRDS set users port to "NULL", replace it by gh-ost param + if Context.AliyunRDS { + port = connectionConfig.Key.Port + } else { + portQuery := `select @@global.port` + if err := db.QueryRow(portQuery).Scan(&port); err != nil { + return "", err + } + } if connectionConfig.Key.Port == port || (extraPort > 0 && connectionConfig.Key.Port == extraPort) { log.Infof("connection validated on %+v", connectionConfig.Key) diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index 74d0f05..335d6cc 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -44,6 +44,7 @@ func acceptSignals(migrationContext *base.MigrationContext) { // main is the application's entry point. It will either spawn a CLI or HTTP interfaces. func main() { migrationContext := base.NewMigrationContext() + base.Context = migrationContext flag.StringVar(&migrationContext.InspectorConnectionConfig.Key.Hostname, "host", "127.0.0.1", "MySQL hostname (preferably a replica, not the master)") flag.StringVar(&migrationContext.AssumeMasterHostname, "assume-master-host", "", "(optional) explicitly tell gh-ost the identity of the master. Format: some.host.com[:port] This is useful in master-master setups where you wish to pick an explicit master, or in a tungsten-replicator where gh-ost is unable to determine the master") @@ -68,6 +69,7 @@ func main() { flag.BoolVar(&migrationContext.IsTungsten, "tungsten", false, "explicitly let gh-ost know that you are running on a tungsten-replication based topology (you are likely to also provide --assume-master-host)") flag.BoolVar(&migrationContext.DiscardForeignKeys, "discard-foreign-keys", false, "DANGER! This flag will migrate a table that has foreign keys and will NOT create foreign keys on the ghost table, thus your altered table will have NO foreign keys. This is useful for intentional dropping of foreign keys") flag.BoolVar(&migrationContext.SkipForeignKeyChecks, "skip-foreign-key-checks", false, "set to 'true' when you know for certain there are no foreign keys on your table, and wish to skip the time it takes for gh-ost to verify that") + flag.BoolVar(&migrationContext.AliyunRDS, "aliyun-rds", false, "set to 'true' when you execute on Aliyun RDS.") executeFlag := flag.Bool("execute", false, "actually execute the alter & migrate the table. Default is noop: do some tests and exit") flag.BoolVar(&migrationContext.TestOnReplica, "test-on-replica", false, "Have the migration run on a replica, not on the master. At the end of migration replication is stopped, and tables are swapped and immediately swap-revert. Replication remains stopped and you can compare the two tables for building trust") diff --git a/go/logic/applier.go b/go/logic/applier.go index 227b59e..1a3e618 100644 --- a/go/logic/applier.go +++ b/go/logic/applier.go @@ -89,10 +89,12 @@ func (this *Applier) InitDBConnections() (err error) { if err := this.validateAndReadTimeZone(); err != nil { return err } - if impliedKey, err := mysql.GetInstanceKey(this.db); err != nil { + if impliedKey, err := mysql.GetInstanceKey(this.db, this.migrationContext.AliyunRDS); err != nil { return err } else { - this.connectionConfig.ImpliedKey = impliedKey + if this.migrationContext.AliyunRDS != true { + this.connectionConfig.ImpliedKey = impliedKey + } } if err := this.readTableColumns(); err != nil { return err diff --git a/go/logic/inspect.go b/go/logic/inspect.go index 31c81dc..1417ffa 100644 --- a/go/logic/inspect.go +++ b/go/logic/inspect.go @@ -53,10 +53,12 @@ func (this *Inspector) InitDBConnections() (err error) { if err := this.validateConnection(); err != nil { return err } - if impliedKey, err := mysql.GetInstanceKey(this.db); err != nil { + if impliedKey, err := mysql.GetInstanceKey(this.db, this.migrationContext.AliyunRDS); err != nil { return err } else { - this.connectionConfig.ImpliedKey = impliedKey + if this.migrationContext.AliyunRDS != true { + this.connectionConfig.ImpliedKey = impliedKey + } } if err := this.validateGrants(); err != nil { return err diff --git a/go/mysql/utils.go b/go/mysql/utils.go index 532cbb4..c513ae5 100644 --- a/go/mysql/utils.go +++ b/go/mysql/utils.go @@ -171,9 +171,11 @@ func GetSelfBinlogCoordinates(db *gosql.DB) (selfBinlogCoordinates *BinlogCoordi } // GetInstanceKey reads hostname and port on given DB -func GetInstanceKey(db *gosql.DB) (instanceKey *InstanceKey, err error) { +func GetInstanceKey(db *gosql.DB, isAliyunRDS bool) (instanceKey *InstanceKey, err error) { instanceKey = &InstanceKey{} - err = db.QueryRow(`select @@global.hostname, @@global.port`).Scan(&instanceKey.Hostname, &instanceKey.Port) + if isAliyunRDS != true { + err = db.QueryRow(`select @@global.hostname, @@global.port`).Scan(&instanceKey.Hostname, &instanceKey.Port) + } return instanceKey, err } From d3b98d9e746115834e5b9dd64e29a157eddcd854 Mon Sep 17 00:00:00 2001 From: zj118228 Date: Thu, 29 Mar 2018 11:19:20 +0800 Subject: [PATCH 2/2] Refactor the code and remove the global variable of support-aliyun-rds branch. --- go/base/context.go | 2 -- go/base/utils.go | 4 ++-- go/cmd/gh-ost/main.go | 2 -- go/logic/applier.go | 12 ++++++------ go/logic/inspect.go | 10 +++++----- go/logic/streamer.go | 2 +- go/mysql/utils.go | 6 ++---- 7 files changed, 16 insertions(+), 22 deletions(-) diff --git a/go/base/context.go b/go/base/context.go index 5f63c88..981c386 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -215,8 +215,6 @@ type ContextConfig struct { } } -var Context *MigrationContext - func NewMigrationContext() *MigrationContext { return &MigrationContext{ Uuid: uuid.NewV4().String(), diff --git a/go/base/utils.go b/go/base/utils.go index 8d9e4f0..d6bbf47 100644 --- a/go/base/utils.go +++ b/go/base/utils.go @@ -64,7 +64,7 @@ func StringContainsAll(s string, substrings ...string) bool { return nonEmptyStringsFound } -func ValidateConnection(db *gosql.DB, connectionConfig *mysql.ConnectionConfig) (string, error) { +func ValidateConnection(db *gosql.DB, connectionConfig *mysql.ConnectionConfig, migrationContext *MigrationContext) (string, error) { versionQuery := `select @@global.version` var port, extraPort int var version string @@ -76,7 +76,7 @@ func ValidateConnection(db *gosql.DB, connectionConfig *mysql.ConnectionConfig) // swallow this error. not all servers support extra_port } // AliyunRDS set users port to "NULL", replace it by gh-ost param - if Context.AliyunRDS { + if migrationContext.AliyunRDS { port = connectionConfig.Key.Port } else { portQuery := `select @@global.port` diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index 335d6cc..70a4e90 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -44,8 +44,6 @@ func acceptSignals(migrationContext *base.MigrationContext) { // main is the application's entry point. It will either spawn a CLI or HTTP interfaces. func main() { migrationContext := base.NewMigrationContext() - base.Context = migrationContext - flag.StringVar(&migrationContext.InspectorConnectionConfig.Key.Hostname, "host", "127.0.0.1", "MySQL hostname (preferably a replica, not the master)") flag.StringVar(&migrationContext.AssumeMasterHostname, "assume-master-host", "", "(optional) explicitly tell gh-ost the identity of the master. Format: some.host.com[:port] This is useful in master-master setups where you wish to pick an explicit master, or in a tungsten-replicator where gh-ost is unable to determine the master") flag.IntVar(&migrationContext.InspectorConnectionConfig.Key.Port, "port", 3306, "MySQL port (preferably a replica, not the master)") diff --git a/go/logic/applier.go b/go/logic/applier.go index 1a3e618..0a2951e 100644 --- a/go/logic/applier.go +++ b/go/logic/applier.go @@ -78,21 +78,21 @@ func (this *Applier) InitDBConnections() (err error) { return err } this.singletonDB.SetMaxOpenConns(1) - version, err := base.ValidateConnection(this.db, this.connectionConfig) + version, err := base.ValidateConnection(this.db, this.connectionConfig, this.migrationContext) if err != nil { return err } - if _, err := base.ValidateConnection(this.singletonDB, this.connectionConfig); err != nil { + if _, err := base.ValidateConnection(this.singletonDB, this.connectionConfig, this.migrationContext); err != nil { return err } this.migrationContext.ApplierMySQLVersion = version if err := this.validateAndReadTimeZone(); err != nil { return err } - if impliedKey, err := mysql.GetInstanceKey(this.db, this.migrationContext.AliyunRDS); err != nil { - return err - } else { - if this.migrationContext.AliyunRDS != true { + if !this.migrationContext.AliyunRDS { + if impliedKey, err := mysql.GetInstanceKey(this.db); err != nil { + return err + } else { this.connectionConfig.ImpliedKey = impliedKey } } diff --git a/go/logic/inspect.go b/go/logic/inspect.go index 1417ffa..bb57447 100644 --- a/go/logic/inspect.go +++ b/go/logic/inspect.go @@ -53,10 +53,10 @@ func (this *Inspector) InitDBConnections() (err error) { if err := this.validateConnection(); err != nil { return err } - if impliedKey, err := mysql.GetInstanceKey(this.db, this.migrationContext.AliyunRDS); err != nil { - return err - } else { - if this.migrationContext.AliyunRDS != true { + if !this.migrationContext.AliyunRDS { + if impliedKey, err := mysql.GetInstanceKey(this.db); err != nil { + return err + } else { this.connectionConfig.ImpliedKey = impliedKey } } @@ -205,7 +205,7 @@ func (this *Inspector) validateConnection() error { return fmt.Errorf("MySQL replication length limited to 32 characters. See https://dev.mysql.com/doc/refman/5.7/en/assigning-passwords.html") } - version, err := base.ValidateConnection(this.db, this.connectionConfig) + version, err := base.ValidateConnection(this.db, this.connectionConfig, this.migrationContext) this.migrationContext.InspectorMySQLVersion = version return err } diff --git a/go/logic/streamer.go b/go/logic/streamer.go index 37e7195..25d9f53 100644 --- a/go/logic/streamer.go +++ b/go/logic/streamer.go @@ -107,7 +107,7 @@ func (this *EventsStreamer) InitDBConnections() (err error) { if this.db, _, err = mysql.GetDB(this.migrationContext.Uuid, EventsStreamerUri); err != nil { return err } - if _, err := base.ValidateConnection(this.db, this.connectionConfig); err != nil { + if _, err := base.ValidateConnection(this.db, this.connectionConfig, this.migrationContext); err != nil { return err } if err := this.readCurrentBinlogCoordinates(); err != nil { diff --git a/go/mysql/utils.go b/go/mysql/utils.go index c513ae5..532cbb4 100644 --- a/go/mysql/utils.go +++ b/go/mysql/utils.go @@ -171,11 +171,9 @@ func GetSelfBinlogCoordinates(db *gosql.DB) (selfBinlogCoordinates *BinlogCoordi } // GetInstanceKey reads hostname and port on given DB -func GetInstanceKey(db *gosql.DB, isAliyunRDS bool) (instanceKey *InstanceKey, err error) { +func GetInstanceKey(db *gosql.DB) (instanceKey *InstanceKey, err error) { instanceKey = &InstanceKey{} - if isAliyunRDS != true { - err = db.QueryRow(`select @@global.hostname, @@global.port`).Scan(&instanceKey.Hostname, &instanceKey.Port) - } + err = db.QueryRow(`select @@global.hostname, @@global.port`).Scan(&instanceKey.Hostname, &instanceKey.Port) return instanceKey, err }