diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3556e1e..cf518ee 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,10 +10,10 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Set up Go 1.14 + - name: Set up Go 1.15 uses: actions/setup-go@v1 with: - go-version: 1.14 + go-version: 1.15 - name: Build run: script/cibuild diff --git a/Dockerfile.packaging b/Dockerfile.packaging index 9c5cd29..092fade 100644 --- a/Dockerfile.packaging +++ b/Dockerfile.packaging @@ -1,6 +1,6 @@ # -FROM golang:1.14.7 +FROM golang:1.15.6 RUN apt-get update RUN apt-get install -y ruby ruby-dev rubygems build-essential diff --git a/Dockerfile.test b/Dockerfile.test index 8f56be3..ceb46bf 100644 --- a/Dockerfile.test +++ b/Dockerfile.test @@ -1,4 +1,4 @@ -FROM golang:1.14.7 +FROM golang:1.15.6 LABEL maintainer="github@github.com" RUN apt-get update diff --git a/README.md b/README.md index d4a17e9..d496e08 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,7 @@ Also see: - [the fine print](doc/the-fine-print.md) - [Community questions](https://github.com/github/gh-ost/issues?q=label%3Aquestion) - [Using `gh-ost` on AWS RDS](doc/rds.md) +- [Using `gh-ost` on Azure Database for MySQL](doc/azure.md) ## What's in a name? diff --git a/doc/azure.md b/doc/azure.md new file mode 100644 index 0000000..f544f37 --- /dev/null +++ b/doc/azure.md @@ -0,0 +1,26 @@ +`gh-ost` has been updated to work with Azure Database for MySQL however due to GitHub does not use it, this documentation is community driven so if you find a bug please [open an issue][new_issue]! + +# Azure Database for MySQL + +## Limitations + +- `gh-ost` runs should be setup use [`--assume-rbr`][assume_rbr_docs] and use `binlog_row_image=FULL`. +- Azure Database for MySQL does not use same user name suffix for master and replica, so master host, user and password need to be pointed out. + +## Step +1. Change the replica server's `binlog_row_image` from `MINIMAL` to `FULL`. See [guide](https://docs.microsoft.com/en-us/azure/mysql/howto-server-parameters) on Azure document. +2. Use your `gh-ost` always with additional 5 parameter +```{bash} +gh-ost \ +--azure \ +--assume-master-host=master-server-dns-name \ +--master-user="master-user-name" \ +--master-password="master-password" \ +--assume-rbr \ +[-- other paramters you need] +``` + + +[new_issue]: https://github.com/github/gh-ost/issues/new +[assume_rbr_docs]: https://github.com/github/gh-ost/blob/master/doc/command-line-flags.md#assume-rbr +[migrate_test_on_replica_docs]: https://github.com/github/gh-ost/blob/master/doc/cheatsheet.md#c-migratetest-on-replica \ No newline at end of file diff --git a/doc/command-line-flags.md b/doc/command-line-flags.md index 629e9f9..22dccbd 100644 --- a/doc/command-line-flags.md +++ b/doc/command-line-flags.md @@ -6,6 +6,10 @@ A more in-depth discussion of various `gh-ost` command line flags: implementatio Add this flag when executing on Aliyun RDS. +### azure + +Add this flag when executing on Azure Database for MySQL. + ### allow-master-master See [`--assume-master-host`](#assume-master-host). diff --git a/doc/requirements-and-limitations.md b/doc/requirements-and-limitations.md index f618af6..e09ae4f 100644 --- a/doc/requirements-and-limitations.md +++ b/doc/requirements-and-limitations.md @@ -41,6 +41,7 @@ The `SUPER` privilege is required for `STOP SLAVE`, `START SLAVE` operations. Th - Amazon RDS works, but has its own [limitations](rds.md). - Google Cloud SQL works, `--gcp` flag required. - Aliyun RDS works, `--aliyun-rds` flag required. +- Azure Database for MySQL works, `--azure` flag required, and have detailed document about it. (azure.md) - Multisource is not supported when migrating via replica. It _should_ work (but never tested) when connecting directly to master (`--allow-on-master`) diff --git a/go/base/context.go b/go/base/context.go index 5419d57..c29cdfa 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -97,6 +97,7 @@ type MigrationContext struct { DiscardForeignKeys bool AliyunRDS bool GoogleCloudPlatform bool + AzureMySQL bool config ContextConfig configMutex *sync.Mutex diff --git a/go/base/utils.go b/go/base/utils.go index 1476a21..65e47f0 100644 --- a/go/base/utils.go +++ b/go/base/utils.go @@ -75,7 +75,8 @@ func ValidateConnection(db *gosql.DB, connectionConfig *mysql.ConnectionConfig, } // AliyunRDS set users port to "NULL", replace it by gh-ost param // GCP set users port to "NULL", replace it by gh-ost param - if migrationContext.AliyunRDS || migrationContext.GoogleCloudPlatform { + // Azure MySQL set users port to a different value by design, replace it by gh-ost para + if migrationContext.AliyunRDS || migrationContext.GoogleCloudPlatform || migrationContext.AzureMySQL { port = connectionConfig.Key.Port } else { portQuery := `select @@global.port` diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index f194508..b8557f9 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -79,6 +79,7 @@ func main() { flag.BoolVar(&migrationContext.SkipStrictMode, "skip-strict-mode", false, "explicitly tell gh-ost binlog applier not to enforce strict sql mode") flag.BoolVar(&migrationContext.AliyunRDS, "aliyun-rds", false, "set to 'true' when you execute on Aliyun RDS.") flag.BoolVar(&migrationContext.GoogleCloudPlatform, "gcp", false, "set to 'true' when you execute on a 1st generation Google Cloud Platform (GCP).") + flag.BoolVar(&migrationContext.AzureMySQL, "azure", false, "set to 'true' when you execute on Azure Database on MySQL.") executeFlag := flag.Bool("execute", false, "actually execute the alter & migrate the table. Default is noop: do some tests and exit") flag.BoolVar(&migrationContext.TestOnReplica, "test-on-replica", false, "Have the migration run on a replica, not on the master. At the end of migration replication is stopped, and tables are swapped and immediately swap-revert. Replication remains stopped and you can compare the two tables for building trust") diff --git a/go/logic/applier.go b/go/logic/applier.go index cc0aa01..43b8829 100644 --- a/go/logic/applier.go +++ b/go/logic/applier.go @@ -17,6 +17,7 @@ import ( "github.com/github/gh-ost/go/sql" "github.com/outbrain/golib/sqlutils" + "sync" ) const ( @@ -88,7 +89,7 @@ func (this *Applier) InitDBConnections() (err error) { if err := this.validateAndReadTimeZone(); err != nil { return err } - if !this.migrationContext.AliyunRDS && !this.migrationContext.GoogleCloudPlatform { + if !this.migrationContext.AliyunRDS && !this.migrationContext.GoogleCloudPlatform && !this.migrationContext.AzureMySQL { if impliedKey, err := mysql.GetInstanceKey(this.db); err != nil { return err } else { @@ -806,7 +807,7 @@ func (this *Applier) CreateAtomicCutOverSentryTable() error { } // AtomicCutOverMagicLock -func (this *Applier) AtomicCutOverMagicLock(sessionIdChan chan int64, tableLocked chan<- error, okToUnlockTable <-chan bool, tableUnlocked chan<- error) error { +func (this *Applier) AtomicCutOverMagicLock(sessionIdChan chan int64, tableLocked chan<- error, okToUnlockTable <-chan bool, tableUnlocked chan<- error, dropCutOverSentryTableOnce *sync.Once) error { tx, err := this.db.Begin() if err != nil { tableLocked <- err @@ -884,10 +885,13 @@ func (this *Applier) AtomicCutOverMagicLock(sessionIdChan chan int64, tableLocke sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.GetOldTableName()), ) - if _, err := tx.Exec(query); err != nil { - this.migrationContext.Log.Errore(err) - // We DO NOT return here because we must `UNLOCK TABLES`! - } + + dropCutOverSentryTableOnce.Do(func() { + if _, err := tx.Exec(query); err != nil { + this.migrationContext.Log.Errore(err) + // We DO NOT return here because we must `UNLOCK TABLES`! + } + }) // Tables still locked this.migrationContext.Log.Infof("Releasing lock from %s.%s, %s.%s", diff --git a/go/logic/inspect.go b/go/logic/inspect.go index d2633de..fc70017 100644 --- a/go/logic/inspect.go +++ b/go/logic/inspect.go @@ -52,7 +52,7 @@ func (this *Inspector) InitDBConnections() (err error) { if err := this.validateConnection(); err != nil { return err } - if !this.migrationContext.AliyunRDS && !this.migrationContext.GoogleCloudPlatform { + if !this.migrationContext.AliyunRDS && !this.migrationContext.GoogleCloudPlatform && !this.migrationContext.AzureMySQL { if impliedKey, err := mysql.GetInstanceKey(this.db); err != nil { return err } else { diff --git a/go/logic/migrator.go b/go/logic/migrator.go index 10df555..5ce2ce3 100644 --- a/go/logic/migrator.go +++ b/go/logic/migrator.go @@ -11,6 +11,7 @@ import ( "math" "os" "strings" + "sync" "sync/atomic" "time" @@ -606,9 +607,12 @@ func (this *Migrator) atomicCutOver() (err error) { defer atomic.StoreInt64(&this.migrationContext.InCutOverCriticalSectionFlag, 0) okToUnlockTable := make(chan bool, 4) + var dropCutOverSentryTableOnce sync.Once defer func() { okToUnlockTable <- true - this.applier.DropAtomicCutOverSentryTableIfExists() + dropCutOverSentryTableOnce.Do(func() { + this.applier.DropAtomicCutOverSentryTableIfExists() + }) }() atomic.StoreInt64(&this.migrationContext.AllEventsUpToLockProcessedInjectedFlag, 0) @@ -617,7 +621,7 @@ func (this *Migrator) atomicCutOver() (err error) { tableLocked := make(chan error, 2) tableUnlocked := make(chan error, 2) go func() { - if err := this.applier.AtomicCutOverMagicLock(lockOriginalSessionIdChan, tableLocked, okToUnlockTable, tableUnlocked); err != nil { + if err := this.applier.AtomicCutOverMagicLock(lockOriginalSessionIdChan, tableLocked, okToUnlockTable, tableUnlocked, &dropCutOverSentryTableOnce); err != nil { this.migrationContext.Log.Errore(err) } }() @@ -737,7 +741,7 @@ func (this *Migrator) initiateInspector() (err error) { this.migrationContext.Log.Infof("Master found to be %+v", *this.migrationContext.ApplierConnectionConfig.ImpliedKey) } else { // Forced master host. - key, err := mysql.ParseRawInstanceKeyLoose(this.migrationContext.AssumeMasterHostname) + key, err := mysql.ParseInstanceKey(this.migrationContext.AssumeMasterHostname) if err != nil { return err } diff --git a/go/mysql/instance_key.go b/go/mysql/instance_key.go index 67284d9..eb108d8 100644 --- a/go/mysql/instance_key.go +++ b/go/mysql/instance_key.go @@ -7,6 +7,7 @@ package mysql import ( "fmt" + "regexp" "strconv" "strings" ) @@ -15,6 +16,13 @@ const ( DefaultInstancePort = 3306 ) +var ( + ipv4HostPortRegexp = regexp.MustCompile("^([^:]+):([0-9]+)$") + ipv4HostRegexp = regexp.MustCompile("^([^:]+)$") + ipv6HostPortRegexp = regexp.MustCompile("^\\[([:0-9a-fA-F]+)\\]:([0-9]+)$") // e.g. [2001:db8:1f70::999:de8:7648:6e8]:3308 + ipv6HostRegexp = regexp.MustCompile("^([:0-9a-fA-F]+)$") // e.g. 2001:db8:1f70::999:de8:7648:6e8 +) + // InstanceKey is an instance indicator, identified by hostname and port type InstanceKey struct { Hostname string @@ -25,25 +33,35 @@ const detachHint = "//" // ParseInstanceKey will parse an InstanceKey from a string representation such as 127.0.0.1:3306 func NewRawInstanceKey(hostPort string) (*InstanceKey, error) { - tokens := strings.SplitN(hostPort, ":", 2) - if len(tokens) != 2 { - return nil, fmt.Errorf("Cannot parse InstanceKey from %s. Expected format is host:port", hostPort) + hostname := "" + port := "" + if submatch := ipv4HostPortRegexp.FindStringSubmatch(hostPort); len(submatch) > 0 { + hostname = submatch[1] + port = submatch[2] + } else if submatch := ipv4HostRegexp.FindStringSubmatch(hostPort); len(submatch) > 0 { + hostname = submatch[1] + } else if submatch := ipv6HostPortRegexp.FindStringSubmatch(hostPort); len(submatch) > 0 { + hostname = submatch[1] + port = submatch[2] + } else if submatch := ipv6HostRegexp.FindStringSubmatch(hostPort); len(submatch) > 0 { + hostname = submatch[1] + } else { + return nil, fmt.Errorf("Cannot parse address: %s", hostPort) } - instanceKey := &InstanceKey{Hostname: tokens[0]} - var err error - if instanceKey.Port, err = strconv.Atoi(tokens[1]); err != nil { - return instanceKey, fmt.Errorf("Invalid port: %s", tokens[1]) + instanceKey := &InstanceKey{Hostname: hostname, Port: DefaultInstancePort} + if port != "" { + var err error + if instanceKey.Port, err = strconv.Atoi(port); err != nil { + return instanceKey, fmt.Errorf("Invalid port: %s", port) + } } return instanceKey, nil } -// ParseRawInstanceKeyLoose will parse an InstanceKey from a string representation such as 127.0.0.1:3306. +// ParseInstanceKey will parse an InstanceKey from a string representation such as 127.0.0.1:3306. // The port part is optional; there will be no name resolve -func ParseRawInstanceKeyLoose(hostPort string) (*InstanceKey, error) { - if !strings.Contains(hostPort, ":") { - return &InstanceKey{Hostname: hostPort, Port: DefaultInstancePort}, nil - } +func ParseInstanceKey(hostPort string) (*InstanceKey, error) { return NewRawInstanceKey(hostPort) } diff --git a/go/mysql/instance_key_map.go b/go/mysql/instance_key_map.go index d0900ef..1065fb9 100644 --- a/go/mysql/instance_key_map.go +++ b/go/mysql/instance_key_map.go @@ -92,7 +92,7 @@ func (this *InstanceKeyMap) ReadCommaDelimitedList(list string) error { } tokens := strings.Split(list, ",") for _, token := range tokens { - key, err := ParseRawInstanceKeyLoose(token) + key, err := ParseInstanceKey(token) if err != nil { return err } diff --git a/go/mysql/instance_key_test.go b/go/mysql/instance_key_test.go new file mode 100644 index 0000000..778a5b3 --- /dev/null +++ b/go/mysql/instance_key_test.go @@ -0,0 +1,74 @@ +/* + Copyright 2016 GitHub Inc. + See https://github.com/github/gh-ost/blob/master/LICENSE +*/ + +package mysql + +import ( + "testing" + + "github.com/outbrain/golib/log" + test "github.com/outbrain/golib/tests" +) + +func init() { + log.SetLevel(log.ERROR) +} + +func TestParseInstanceKey(t *testing.T) { + { + key, err := ParseInstanceKey("myhost:1234") + test.S(t).ExpectNil(err) + test.S(t).ExpectEquals(key.Hostname, "myhost") + test.S(t).ExpectEquals(key.Port, 1234) + } + { + key, err := ParseInstanceKey("myhost") + test.S(t).ExpectNil(err) + test.S(t).ExpectEquals(key.Hostname, "myhost") + test.S(t).ExpectEquals(key.Port, 3306) + } + { + key, err := ParseInstanceKey("10.0.0.3:3307") + test.S(t).ExpectNil(err) + test.S(t).ExpectEquals(key.Hostname, "10.0.0.3") + test.S(t).ExpectEquals(key.Port, 3307) + } + { + key, err := ParseInstanceKey("10.0.0.3") + test.S(t).ExpectNil(err) + test.S(t).ExpectEquals(key.Hostname, "10.0.0.3") + test.S(t).ExpectEquals(key.Port, 3306) + } + { + key, err := ParseInstanceKey("[2001:db8:1f70::999:de8:7648:6e8]:3308") + test.S(t).ExpectNil(err) + test.S(t).ExpectEquals(key.Hostname, "2001:db8:1f70::999:de8:7648:6e8") + test.S(t).ExpectEquals(key.Port, 3308) + } + { + key, err := ParseInstanceKey("::1") + test.S(t).ExpectNil(err) + test.S(t).ExpectEquals(key.Hostname, "::1") + test.S(t).ExpectEquals(key.Port, 3306) + } + { + key, err := ParseInstanceKey("0:0:0:0:0:0:0:0") + test.S(t).ExpectNil(err) + test.S(t).ExpectEquals(key.Hostname, "0:0:0:0:0:0:0:0") + test.S(t).ExpectEquals(key.Port, 3306) + } + { + _, err := ParseInstanceKey("[2001:xxxx:1f70::999:de8:7648:6e8]:3308") + test.S(t).ExpectNotNil(err) + } + { + _, err := ParseInstanceKey("10.0.0.4:") + test.S(t).ExpectNotNil(err) + } + { + _, err := ParseInstanceKey("10.0.0.4:5.6.7") + test.S(t).ExpectNotNil(err) + } +} diff --git a/script/build-deploy-tarball b/script/build-deploy-tarball index 95da838..dc28b43 100755 --- a/script/build-deploy-tarball +++ b/script/build-deploy-tarball @@ -30,8 +30,6 @@ cp ${tarball}.gz "$BUILD_ARTIFACT_DIR"/gh-ost/ ### HACK HACK HACK HACK ### # blame @carlosmn, @mattr and @timvaillancourt- -# Allow builds on buster to also be used for stretch + jessie +# Allow builds on buster to also be used for stretch stretch_tarball_name=$(echo $(basename "${tarball}") | sed s/-buster-/-stretch-/) -jessie_tarball_name=$(echo $(basename "${stretch_tarball_name}") | sed s/-stretch-/-jessie-/) cp ${tarball}.gz "$BUILD_ARTIFACT_DIR/gh-ost/${stretch_tarball_name}.gz" -cp ${tarball}.gz "$BUILD_ARTIFACT_DIR/gh-ost/${jessie_tarball_name}.gz"