From ae762694fbb9a40e5e0ae29b62e8ba0651c38263 Mon Sep 17 00:00:00 2001 From: Shuode Li Date: Tue, 17 Dec 2019 03:47:14 +0000 Subject: [PATCH 1/2] Support Azure Database for MySQL. --- README.md | 1 + doc/azure.md | 26 ++++++++++++++++++++++++++ doc/command-line-flags.md | 4 ++++ doc/requirements-and-limitations.md | 1 + go/base/context.go | 1 + go/base/utils.go | 3 ++- go/cmd/gh-ost/main.go | 1 + go/logic/applier.go | 2 +- go/logic/inspect.go | 2 +- go/mysql/connection.go | 2 +- go/mysql/connection_test.go | 4 ++-- 11 files changed, 41 insertions(+), 6 deletions(-) create mode 100644 doc/azure.md diff --git a/README.md b/README.md index cd6f592..a27b143 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,7 @@ Also see: - [the fine print](doc/the-fine-print.md) - [Community questions](https://github.com/github/gh-ost/issues?q=label%3Aquestion) - [Using `gh-ost` on AWS RDS](doc/rds.md) +- [Using `gh-ost` on Azure Database for MySQL](doc/azure.md) ## What's in a name? diff --git a/doc/azure.md b/doc/azure.md new file mode 100644 index 0000000..f544f37 --- /dev/null +++ b/doc/azure.md @@ -0,0 +1,26 @@ +`gh-ost` has been updated to work with Azure Database for MySQL however due to GitHub does not use it, this documentation is community driven so if you find a bug please [open an issue][new_issue]! + +# Azure Database for MySQL + +## Limitations + +- `gh-ost` runs should be setup use [`--assume-rbr`][assume_rbr_docs] and use `binlog_row_image=FULL`. +- Azure Database for MySQL does not use same user name suffix for master and replica, so master host, user and password need to be pointed out. + +## Step +1. Change the replica server's `binlog_row_image` from `MINIMAL` to `FULL`. See [guide](https://docs.microsoft.com/en-us/azure/mysql/howto-server-parameters) on Azure document. +2. Use your `gh-ost` always with additional 5 parameter +```{bash} +gh-ost \ +--azure \ +--assume-master-host=master-server-dns-name \ +--master-user="master-user-name" \ +--master-password="master-password" \ +--assume-rbr \ +[-- other paramters you need] +``` + + +[new_issue]: https://github.com/github/gh-ost/issues/new +[assume_rbr_docs]: https://github.com/github/gh-ost/blob/master/doc/command-line-flags.md#assume-rbr +[migrate_test_on_replica_docs]: https://github.com/github/gh-ost/blob/master/doc/cheatsheet.md#c-migratetest-on-replica \ No newline at end of file diff --git a/doc/command-line-flags.md b/doc/command-line-flags.md index 629e9f9..22dccbd 100644 --- a/doc/command-line-flags.md +++ b/doc/command-line-flags.md @@ -6,6 +6,10 @@ A more in-depth discussion of various `gh-ost` command line flags: implementatio Add this flag when executing on Aliyun RDS. +### azure + +Add this flag when executing on Azure Database for MySQL. + ### allow-master-master See [`--assume-master-host`](#assume-master-host). diff --git a/doc/requirements-and-limitations.md b/doc/requirements-and-limitations.md index 096e2c9..0cd25ed 100644 --- a/doc/requirements-and-limitations.md +++ b/doc/requirements-and-limitations.md @@ -41,6 +41,7 @@ The `SUPER` privilege is required for `STOP SLAVE`, `START SLAVE` operations. Th - Amazon RDS works, but has its own [limitations](rds.md). - Google Cloud SQL works, `--gcp` flag required. - Aliyun RDS works, `--aliyun-rds` flag required. +- Azure Database for MySQL works, `--azure` flag required, and have detailed document about it. (azure.md) - Multisource is not supported when migrating via replica. It _should_ work (but never tested) when connecting directly to master (`--allow-on-master`) diff --git a/go/base/context.go b/go/base/context.go index 5ebf092..4220422 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -95,6 +95,7 @@ type MigrationContext struct { DiscardForeignKeys bool AliyunRDS bool GoogleCloudPlatform bool + AzureMySQL bool config ContextConfig configMutex *sync.Mutex diff --git a/go/base/utils.go b/go/base/utils.go index 99536f8..852f1ab 100644 --- a/go/base/utils.go +++ b/go/base/utils.go @@ -76,7 +76,8 @@ func ValidateConnection(db *gosql.DB, connectionConfig *mysql.ConnectionConfig, } // AliyunRDS set users port to "NULL", replace it by gh-ost param // GCP set users port to "NULL", replace it by gh-ost param - if migrationContext.AliyunRDS || migrationContext.GoogleCloudPlatform { + // Azure MySQL set users port to a different value by design, replace it by gh-ost para + if migrationContext.AliyunRDS || migrationContext.GoogleCloudPlatform || migrationContext.AzureMySQL { port = connectionConfig.Key.Port } else { portQuery := `select @@global.port` diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index ce63b03..9051465 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -77,6 +77,7 @@ func main() { flag.BoolVar(&migrationContext.SkipStrictMode, "skip-strict-mode", false, "explicitly tell gh-ost binlog applier not to enforce strict sql mode") flag.BoolVar(&migrationContext.AliyunRDS, "aliyun-rds", false, "set to 'true' when you execute on Aliyun RDS.") flag.BoolVar(&migrationContext.GoogleCloudPlatform, "gcp", false, "set to 'true' when you execute on a 1st generation Google Cloud Platform (GCP).") + flag.BoolVar(&migrationContext.AzureMySQL, "azure", false, "set to 'true' when you execute on Azure Database on MySQL.") executeFlag := flag.Bool("execute", false, "actually execute the alter & migrate the table. Default is noop: do some tests and exit") flag.BoolVar(&migrationContext.TestOnReplica, "test-on-replica", false, "Have the migration run on a replica, not on the master. At the end of migration replication is stopped, and tables are swapped and immediately swap-revert. Replication remains stopped and you can compare the two tables for building trust") diff --git a/go/logic/applier.go b/go/logic/applier.go index 5fb795b..85265c8 100644 --- a/go/logic/applier.go +++ b/go/logic/applier.go @@ -89,7 +89,7 @@ func (this *Applier) InitDBConnections() (err error) { if err := this.validateAndReadTimeZone(); err != nil { return err } - if !this.migrationContext.AliyunRDS && !this.migrationContext.GoogleCloudPlatform { + if !this.migrationContext.AliyunRDS && !this.migrationContext.GoogleCloudPlatform && !this.migrationContext.AzureMySQL { if impliedKey, err := mysql.GetInstanceKey(this.db); err != nil { return err } else { diff --git a/go/logic/inspect.go b/go/logic/inspect.go index 31184b0..e4bbfde 100644 --- a/go/logic/inspect.go +++ b/go/logic/inspect.go @@ -53,7 +53,7 @@ func (this *Inspector) InitDBConnections() (err error) { if err := this.validateConnection(); err != nil { return err } - if !this.migrationContext.AliyunRDS && !this.migrationContext.GoogleCloudPlatform { + if !this.migrationContext.AliyunRDS && !this.migrationContext.GoogleCloudPlatform && !this.migrationContext.AzureMySQL { if impliedKey, err := mysql.GetInstanceKey(this.db); err != nil { return err } else { diff --git a/go/mysql/connection.go b/go/mysql/connection.go index 654998c..0407044 100644 --- a/go/mysql/connection.go +++ b/go/mysql/connection.go @@ -116,5 +116,5 @@ func (this *ConnectionConfig) GetDBUri(databaseName string) string { if this.tlsConfig != nil { tlsOption = TLS_CONFIG_KEY } - return fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?interpolateParams=%t&autocommit=true&charset=utf8mb4,utf8,latin1&tls=%s", this.User, this.Password, hostname, this.Key.Port, databaseName, interpolateParams, tlsOption) + return fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?interpolateParams=%t&autocommit=true&charset=utf8mb4,utf8,latin1&tls=%s&allowNativePasswords=true", this.User, this.Password, hostname, this.Key.Port, databaseName, interpolateParams, tlsOption) } diff --git a/go/mysql/connection_test.go b/go/mysql/connection_test.go index 6ce85a7..27f37f2 100644 --- a/go/mysql/connection_test.go +++ b/go/mysql/connection_test.go @@ -69,7 +69,7 @@ func TestGetDBUri(t *testing.T) { c.Password = "penguin" uri := c.GetDBUri("test") - test.S(t).ExpectEquals(uri, "gromit:penguin@tcp(myhost:3306)/test?interpolateParams=true&autocommit=true&charset=utf8mb4,utf8,latin1&tls=false") + test.S(t).ExpectEquals(uri, "gromit:penguin@tcp(myhost:3306)/test?interpolateParams=true&autocommit=true&charset=utf8mb4,utf8,latin1&tls=false&allowNativePasswords=true") } func TestGetDBUriWithTLSSetup(t *testing.T) { @@ -80,5 +80,5 @@ func TestGetDBUriWithTLSSetup(t *testing.T) { c.tlsConfig = &tls.Config{} uri := c.GetDBUri("test") - test.S(t).ExpectEquals(uri, "gromit:penguin@tcp(myhost:3306)/test?interpolateParams=true&autocommit=true&charset=utf8mb4,utf8,latin1&tls=ghost") + test.S(t).ExpectEquals(uri, "gromit:penguin@tcp(myhost:3306)/test?interpolateParams=true&autocommit=true&charset=utf8mb4,utf8,latin1&tls=ghost&allowNativePasswords=true") } From 13b5f48ca5e7cc83bf7af2455785304c2c4385fb Mon Sep 17 00:00:00 2001 From: Shuode Li Date: Tue, 17 Dec 2019 04:03:51 +0000 Subject: [PATCH 2/2] Remove useless code --- go/mysql/connection.go | 2 +- go/mysql/connection_test.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go/mysql/connection.go b/go/mysql/connection.go index 0407044..654998c 100644 --- a/go/mysql/connection.go +++ b/go/mysql/connection.go @@ -116,5 +116,5 @@ func (this *ConnectionConfig) GetDBUri(databaseName string) string { if this.tlsConfig != nil { tlsOption = TLS_CONFIG_KEY } - return fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?interpolateParams=%t&autocommit=true&charset=utf8mb4,utf8,latin1&tls=%s&allowNativePasswords=true", this.User, this.Password, hostname, this.Key.Port, databaseName, interpolateParams, tlsOption) + return fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?interpolateParams=%t&autocommit=true&charset=utf8mb4,utf8,latin1&tls=%s", this.User, this.Password, hostname, this.Key.Port, databaseName, interpolateParams, tlsOption) } diff --git a/go/mysql/connection_test.go b/go/mysql/connection_test.go index 27f37f2..6ce85a7 100644 --- a/go/mysql/connection_test.go +++ b/go/mysql/connection_test.go @@ -69,7 +69,7 @@ func TestGetDBUri(t *testing.T) { c.Password = "penguin" uri := c.GetDBUri("test") - test.S(t).ExpectEquals(uri, "gromit:penguin@tcp(myhost:3306)/test?interpolateParams=true&autocommit=true&charset=utf8mb4,utf8,latin1&tls=false&allowNativePasswords=true") + test.S(t).ExpectEquals(uri, "gromit:penguin@tcp(myhost:3306)/test?interpolateParams=true&autocommit=true&charset=utf8mb4,utf8,latin1&tls=false") } func TestGetDBUriWithTLSSetup(t *testing.T) { @@ -80,5 +80,5 @@ func TestGetDBUriWithTLSSetup(t *testing.T) { c.tlsConfig = &tls.Config{} uri := c.GetDBUri("test") - test.S(t).ExpectEquals(uri, "gromit:penguin@tcp(myhost:3306)/test?interpolateParams=true&autocommit=true&charset=utf8mb4,utf8,latin1&tls=ghost&allowNativePasswords=true") + test.S(t).ExpectEquals(uri, "gromit:penguin@tcp(myhost:3306)/test?interpolateParams=true&autocommit=true&charset=utf8mb4,utf8,latin1&tls=ghost") }