From 49b80df27b668512914799be756fdfdf6957e8cd Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Sun, 16 Sep 2018 11:44:52 +0300 Subject: [PATCH 1/8] Parsing ipv6 addresses --- go/logic/migrator.go | 2 +- go/mysql/instance_key.go | 39 +++++++++++++++++++++++++----------- go/mysql/instance_key_map.go | 2 +- 3 files changed, 29 insertions(+), 14 deletions(-) diff --git a/go/logic/migrator.go b/go/logic/migrator.go index f52e2b8..eaddbbe 100644 --- a/go/logic/migrator.go +++ b/go/logic/migrator.go @@ -739,7 +739,7 @@ func (this *Migrator) initiateInspector() (err error) { log.Infof("Master found to be %+v", *this.migrationContext.ApplierConnectionConfig.ImpliedKey) } else { // Forced master host. - key, err := mysql.ParseRawInstanceKeyLoose(this.migrationContext.AssumeMasterHostname) + key, err := mysql.ParseInstanceKey(this.migrationContext.AssumeMasterHostname) if err != nil { return err } diff --git a/go/mysql/instance_key.go b/go/mysql/instance_key.go index 67284d9..56fe415 100644 --- a/go/mysql/instance_key.go +++ b/go/mysql/instance_key.go @@ -7,6 +7,7 @@ package mysql import ( "fmt" + "regexp" "strconv" "strings" ) @@ -15,6 +16,12 @@ const ( DefaultInstancePort = 3306 ) +var ( + ipv4HostPortRegexp = regexp.MustCompile("^([^:]+):([0-9]+)$") + ipv4HostRegexp = regexp.MustCompile("^([^:]+)$") + ipv6HostPortRegexp = regexp.MustCompile("^\\[(.+)\\]:([0-9]+)$") // e.g. [2001:db8:1f70::999:de8:7648:6e8]:3308 +) + // InstanceKey is an instance indicator, identified by hostname and port type InstanceKey struct { Hostname string @@ -25,25 +32,33 @@ const detachHint = "//" // ParseInstanceKey will parse an InstanceKey from a string representation such as 127.0.0.1:3306 func NewRawInstanceKey(hostPort string) (*InstanceKey, error) { - tokens := strings.SplitN(hostPort, ":", 2) - if len(tokens) != 2 { - return nil, fmt.Errorf("Cannot parse InstanceKey from %s. Expected format is host:port", hostPort) + hostname := "" + port := "" + if submatch := ipv4HostPortRegexp.FindStringSubmatch(hostPort); len(submatch) > 0 { + hostname = submatch[1] + port = submatch[2] + } else if submatch := ipv4HostRegexp.FindStringSubmatch(hostPort); len(submatch) > 0 { + hostname = submatch[1] + } else if submatch := ipv6HostPortRegexp.FindStringSubmatch(hostPort); len(submatch) > 0 { + hostname = submatch[1] + port = submatch[2] + } else { + return nil, fmt.Errorf("Cannot parse address: %s", hostPort) } - instanceKey := &InstanceKey{Hostname: tokens[0]} - var err error - if instanceKey.Port, err = strconv.Atoi(tokens[1]); err != nil { - return instanceKey, fmt.Errorf("Invalid port: %s", tokens[1]) + instanceKey := &InstanceKey{Hostname: hostname, Port: DefaultInstancePort} + if port != "" { + var err error + if instanceKey.Port, err = strconv.Atoi(port); err != nil { + return instanceKey, fmt.Errorf("Invalid port: %s", port) + } } return instanceKey, nil } -// ParseRawInstanceKeyLoose will parse an InstanceKey from a string representation such as 127.0.0.1:3306. +// ParseInstanceKey will parse an InstanceKey from a string representation such as 127.0.0.1:3306. // The port part is optional; there will be no name resolve -func ParseRawInstanceKeyLoose(hostPort string) (*InstanceKey, error) { - if !strings.Contains(hostPort, ":") { - return &InstanceKey{Hostname: hostPort, Port: DefaultInstancePort}, nil - } +func ParseInstanceKey(hostPort string) (*InstanceKey, error) { return NewRawInstanceKey(hostPort) } diff --git a/go/mysql/instance_key_map.go b/go/mysql/instance_key_map.go index d0900ef..1065fb9 100644 --- a/go/mysql/instance_key_map.go +++ b/go/mysql/instance_key_map.go @@ -92,7 +92,7 @@ func (this *InstanceKeyMap) ReadCommaDelimitedList(list string) error { } tokens := strings.Split(list, ",") for _, token := range tokens { - key, err := ParseRawInstanceKeyLoose(token) + key, err := ParseInstanceKey(token) if err != nil { return err } From a7cfaa4d336c5e1d0084c1120537bbdad3ad239c Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Sun, 16 Sep 2018 11:48:15 +0300 Subject: [PATCH 2/8] added testing --- go/mysql/instance_key_test.go | 58 +++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 go/mysql/instance_key_test.go diff --git a/go/mysql/instance_key_test.go b/go/mysql/instance_key_test.go new file mode 100644 index 0000000..73e22f3 --- /dev/null +++ b/go/mysql/instance_key_test.go @@ -0,0 +1,58 @@ +/* + Copyright 2016 GitHub Inc. + See https://github.com/github/gh-ost/blob/master/LICENSE +*/ + +package mysql + +import ( + "testing" + + "github.com/outbrain/golib/log" + test "github.com/outbrain/golib/tests" +) + +func init() { + log.SetLevel(log.ERROR) +} + +func TestParseInstanceKey(t *testing.T) { + { + key, err := ParseInstanceKey("myhost:1234") + test.S(t).ExpectNil(err) + test.S(t).ExpectEquals(key.Hostname, "myhost") + test.S(t).ExpectEquals(key.Port, 1234) + } + { + key, err := ParseInstanceKey("myhost") + test.S(t).ExpectNil(err) + test.S(t).ExpectEquals(key.Hostname, "myhost") + test.S(t).ExpectEquals(key.Port, 3306) + } + { + key, err := ParseInstanceKey("10.0.0.3:3307") + test.S(t).ExpectNil(err) + test.S(t).ExpectEquals(key.Hostname, "10.0.0.3") + test.S(t).ExpectEquals(key.Port, 3307) + } + { + key, err := ParseInstanceKey("10.0.0.3") + test.S(t).ExpectNil(err) + test.S(t).ExpectEquals(key.Hostname, "10.0.0.3") + test.S(t).ExpectEquals(key.Port, 3306) + } + { + key, err := ParseInstanceKey("[2001:db8:1f70::999:de8:7648:6e8]:3308") + test.S(t).ExpectNil(err) + test.S(t).ExpectEquals(key.Hostname, "2001:db8:1f70::999:de8:7648:6e8") + test.S(t).ExpectEquals(key.Port, 3308) + } + { + _, err := ParseInstanceKey("10.0.0.4:") + test.S(t).ExpectNotNil(err) + } + { + _, err := ParseInstanceKey("10.0.0.4:5.6.7") + test.S(t).ExpectNotNil(err) + } +} From 959d1af211547f428a7d324cc91e0b3a8b6b4bf8 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Sun, 16 Sep 2018 11:52:59 +0300 Subject: [PATCH 3/8] support ipv6 without port --- go/mysql/instance_key.go | 5 ++++- go/mysql/instance_key_test.go | 16 ++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/go/mysql/instance_key.go b/go/mysql/instance_key.go index 56fe415..eb108d8 100644 --- a/go/mysql/instance_key.go +++ b/go/mysql/instance_key.go @@ -19,7 +19,8 @@ const ( var ( ipv4HostPortRegexp = regexp.MustCompile("^([^:]+):([0-9]+)$") ipv4HostRegexp = regexp.MustCompile("^([^:]+)$") - ipv6HostPortRegexp = regexp.MustCompile("^\\[(.+)\\]:([0-9]+)$") // e.g. [2001:db8:1f70::999:de8:7648:6e8]:3308 + ipv6HostPortRegexp = regexp.MustCompile("^\\[([:0-9a-fA-F]+)\\]:([0-9]+)$") // e.g. [2001:db8:1f70::999:de8:7648:6e8]:3308 + ipv6HostRegexp = regexp.MustCompile("^([:0-9a-fA-F]+)$") // e.g. 2001:db8:1f70::999:de8:7648:6e8 ) // InstanceKey is an instance indicator, identified by hostname and port @@ -42,6 +43,8 @@ func NewRawInstanceKey(hostPort string) (*InstanceKey, error) { } else if submatch := ipv6HostPortRegexp.FindStringSubmatch(hostPort); len(submatch) > 0 { hostname = submatch[1] port = submatch[2] + } else if submatch := ipv6HostRegexp.FindStringSubmatch(hostPort); len(submatch) > 0 { + hostname = submatch[1] } else { return nil, fmt.Errorf("Cannot parse address: %s", hostPort) } diff --git a/go/mysql/instance_key_test.go b/go/mysql/instance_key_test.go index 73e22f3..778a5b3 100644 --- a/go/mysql/instance_key_test.go +++ b/go/mysql/instance_key_test.go @@ -47,6 +47,22 @@ func TestParseInstanceKey(t *testing.T) { test.S(t).ExpectEquals(key.Hostname, "2001:db8:1f70::999:de8:7648:6e8") test.S(t).ExpectEquals(key.Port, 3308) } + { + key, err := ParseInstanceKey("::1") + test.S(t).ExpectNil(err) + test.S(t).ExpectEquals(key.Hostname, "::1") + test.S(t).ExpectEquals(key.Port, 3306) + } + { + key, err := ParseInstanceKey("0:0:0:0:0:0:0:0") + test.S(t).ExpectNil(err) + test.S(t).ExpectEquals(key.Hostname, "0:0:0:0:0:0:0:0") + test.S(t).ExpectEquals(key.Port, 3306) + } + { + _, err := ParseInstanceKey("[2001:xxxx:1f70::999:de8:7648:6e8]:3308") + test.S(t).ExpectNotNil(err) + } { _, err := ParseInstanceKey("10.0.0.4:") test.S(t).ExpectNotNil(err) From ae762694fbb9a40e5e0ae29b62e8ba0651c38263 Mon Sep 17 00:00:00 2001 From: Shuode Li Date: Tue, 17 Dec 2019 03:47:14 +0000 Subject: [PATCH 4/8] Support Azure Database for MySQL. --- README.md | 1 + doc/azure.md | 26 ++++++++++++++++++++++++++ doc/command-line-flags.md | 4 ++++ doc/requirements-and-limitations.md | 1 + go/base/context.go | 1 + go/base/utils.go | 3 ++- go/cmd/gh-ost/main.go | 1 + go/logic/applier.go | 2 +- go/logic/inspect.go | 2 +- go/mysql/connection.go | 2 +- go/mysql/connection_test.go | 4 ++-- 11 files changed, 41 insertions(+), 6 deletions(-) create mode 100644 doc/azure.md diff --git a/README.md b/README.md index cd6f592..a27b143 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,7 @@ Also see: - [the fine print](doc/the-fine-print.md) - [Community questions](https://github.com/github/gh-ost/issues?q=label%3Aquestion) - [Using `gh-ost` on AWS RDS](doc/rds.md) +- [Using `gh-ost` on Azure Database for MySQL](doc/azure.md) ## What's in a name? diff --git a/doc/azure.md b/doc/azure.md new file mode 100644 index 0000000..f544f37 --- /dev/null +++ b/doc/azure.md @@ -0,0 +1,26 @@ +`gh-ost` has been updated to work with Azure Database for MySQL however due to GitHub does not use it, this documentation is community driven so if you find a bug please [open an issue][new_issue]! + +# Azure Database for MySQL + +## Limitations + +- `gh-ost` runs should be setup use [`--assume-rbr`][assume_rbr_docs] and use `binlog_row_image=FULL`. +- Azure Database for MySQL does not use same user name suffix for master and replica, so master host, user and password need to be pointed out. + +## Step +1. Change the replica server's `binlog_row_image` from `MINIMAL` to `FULL`. See [guide](https://docs.microsoft.com/en-us/azure/mysql/howto-server-parameters) on Azure document. +2. Use your `gh-ost` always with additional 5 parameter +```{bash} +gh-ost \ +--azure \ +--assume-master-host=master-server-dns-name \ +--master-user="master-user-name" \ +--master-password="master-password" \ +--assume-rbr \ +[-- other paramters you need] +``` + + +[new_issue]: https://github.com/github/gh-ost/issues/new +[assume_rbr_docs]: https://github.com/github/gh-ost/blob/master/doc/command-line-flags.md#assume-rbr +[migrate_test_on_replica_docs]: https://github.com/github/gh-ost/blob/master/doc/cheatsheet.md#c-migratetest-on-replica \ No newline at end of file diff --git a/doc/command-line-flags.md b/doc/command-line-flags.md index 629e9f9..22dccbd 100644 --- a/doc/command-line-flags.md +++ b/doc/command-line-flags.md @@ -6,6 +6,10 @@ A more in-depth discussion of various `gh-ost` command line flags: implementatio Add this flag when executing on Aliyun RDS. +### azure + +Add this flag when executing on Azure Database for MySQL. + ### allow-master-master See [`--assume-master-host`](#assume-master-host). diff --git a/doc/requirements-and-limitations.md b/doc/requirements-and-limitations.md index 096e2c9..0cd25ed 100644 --- a/doc/requirements-and-limitations.md +++ b/doc/requirements-and-limitations.md @@ -41,6 +41,7 @@ The `SUPER` privilege is required for `STOP SLAVE`, `START SLAVE` operations. Th - Amazon RDS works, but has its own [limitations](rds.md). - Google Cloud SQL works, `--gcp` flag required. - Aliyun RDS works, `--aliyun-rds` flag required. +- Azure Database for MySQL works, `--azure` flag required, and have detailed document about it. (azure.md) - Multisource is not supported when migrating via replica. It _should_ work (but never tested) when connecting directly to master (`--allow-on-master`) diff --git a/go/base/context.go b/go/base/context.go index 5ebf092..4220422 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -95,6 +95,7 @@ type MigrationContext struct { DiscardForeignKeys bool AliyunRDS bool GoogleCloudPlatform bool + AzureMySQL bool config ContextConfig configMutex *sync.Mutex diff --git a/go/base/utils.go b/go/base/utils.go index 99536f8..852f1ab 100644 --- a/go/base/utils.go +++ b/go/base/utils.go @@ -76,7 +76,8 @@ func ValidateConnection(db *gosql.DB, connectionConfig *mysql.ConnectionConfig, } // AliyunRDS set users port to "NULL", replace it by gh-ost param // GCP set users port to "NULL", replace it by gh-ost param - if migrationContext.AliyunRDS || migrationContext.GoogleCloudPlatform { + // Azure MySQL set users port to a different value by design, replace it by gh-ost para + if migrationContext.AliyunRDS || migrationContext.GoogleCloudPlatform || migrationContext.AzureMySQL { port = connectionConfig.Key.Port } else { portQuery := `select @@global.port` diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index ce63b03..9051465 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -77,6 +77,7 @@ func main() { flag.BoolVar(&migrationContext.SkipStrictMode, "skip-strict-mode", false, "explicitly tell gh-ost binlog applier not to enforce strict sql mode") flag.BoolVar(&migrationContext.AliyunRDS, "aliyun-rds", false, "set to 'true' when you execute on Aliyun RDS.") flag.BoolVar(&migrationContext.GoogleCloudPlatform, "gcp", false, "set to 'true' when you execute on a 1st generation Google Cloud Platform (GCP).") + flag.BoolVar(&migrationContext.AzureMySQL, "azure", false, "set to 'true' when you execute on Azure Database on MySQL.") executeFlag := flag.Bool("execute", false, "actually execute the alter & migrate the table. Default is noop: do some tests and exit") flag.BoolVar(&migrationContext.TestOnReplica, "test-on-replica", false, "Have the migration run on a replica, not on the master. At the end of migration replication is stopped, and tables are swapped and immediately swap-revert. Replication remains stopped and you can compare the two tables for building trust") diff --git a/go/logic/applier.go b/go/logic/applier.go index 5fb795b..85265c8 100644 --- a/go/logic/applier.go +++ b/go/logic/applier.go @@ -89,7 +89,7 @@ func (this *Applier) InitDBConnections() (err error) { if err := this.validateAndReadTimeZone(); err != nil { return err } - if !this.migrationContext.AliyunRDS && !this.migrationContext.GoogleCloudPlatform { + if !this.migrationContext.AliyunRDS && !this.migrationContext.GoogleCloudPlatform && !this.migrationContext.AzureMySQL { if impliedKey, err := mysql.GetInstanceKey(this.db); err != nil { return err } else { diff --git a/go/logic/inspect.go b/go/logic/inspect.go index 31184b0..e4bbfde 100644 --- a/go/logic/inspect.go +++ b/go/logic/inspect.go @@ -53,7 +53,7 @@ func (this *Inspector) InitDBConnections() (err error) { if err := this.validateConnection(); err != nil { return err } - if !this.migrationContext.AliyunRDS && !this.migrationContext.GoogleCloudPlatform { + if !this.migrationContext.AliyunRDS && !this.migrationContext.GoogleCloudPlatform && !this.migrationContext.AzureMySQL { if impliedKey, err := mysql.GetInstanceKey(this.db); err != nil { return err } else { diff --git a/go/mysql/connection.go b/go/mysql/connection.go index 654998c..0407044 100644 --- a/go/mysql/connection.go +++ b/go/mysql/connection.go @@ -116,5 +116,5 @@ func (this *ConnectionConfig) GetDBUri(databaseName string) string { if this.tlsConfig != nil { tlsOption = TLS_CONFIG_KEY } - return fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?interpolateParams=%t&autocommit=true&charset=utf8mb4,utf8,latin1&tls=%s", this.User, this.Password, hostname, this.Key.Port, databaseName, interpolateParams, tlsOption) + return fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?interpolateParams=%t&autocommit=true&charset=utf8mb4,utf8,latin1&tls=%s&allowNativePasswords=true", this.User, this.Password, hostname, this.Key.Port, databaseName, interpolateParams, tlsOption) } diff --git a/go/mysql/connection_test.go b/go/mysql/connection_test.go index 6ce85a7..27f37f2 100644 --- a/go/mysql/connection_test.go +++ b/go/mysql/connection_test.go @@ -69,7 +69,7 @@ func TestGetDBUri(t *testing.T) { c.Password = "penguin" uri := c.GetDBUri("test") - test.S(t).ExpectEquals(uri, "gromit:penguin@tcp(myhost:3306)/test?interpolateParams=true&autocommit=true&charset=utf8mb4,utf8,latin1&tls=false") + test.S(t).ExpectEquals(uri, "gromit:penguin@tcp(myhost:3306)/test?interpolateParams=true&autocommit=true&charset=utf8mb4,utf8,latin1&tls=false&allowNativePasswords=true") } func TestGetDBUriWithTLSSetup(t *testing.T) { @@ -80,5 +80,5 @@ func TestGetDBUriWithTLSSetup(t *testing.T) { c.tlsConfig = &tls.Config{} uri := c.GetDBUri("test") - test.S(t).ExpectEquals(uri, "gromit:penguin@tcp(myhost:3306)/test?interpolateParams=true&autocommit=true&charset=utf8mb4,utf8,latin1&tls=ghost") + test.S(t).ExpectEquals(uri, "gromit:penguin@tcp(myhost:3306)/test?interpolateParams=true&autocommit=true&charset=utf8mb4,utf8,latin1&tls=ghost&allowNativePasswords=true") } From 13b5f48ca5e7cc83bf7af2455785304c2c4385fb Mon Sep 17 00:00:00 2001 From: Shuode Li Date: Tue, 17 Dec 2019 04:03:51 +0000 Subject: [PATCH 5/8] Remove useless code --- go/mysql/connection.go | 2 +- go/mysql/connection_test.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go/mysql/connection.go b/go/mysql/connection.go index 0407044..654998c 100644 --- a/go/mysql/connection.go +++ b/go/mysql/connection.go @@ -116,5 +116,5 @@ func (this *ConnectionConfig) GetDBUri(databaseName string) string { if this.tlsConfig != nil { tlsOption = TLS_CONFIG_KEY } - return fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?interpolateParams=%t&autocommit=true&charset=utf8mb4,utf8,latin1&tls=%s&allowNativePasswords=true", this.User, this.Password, hostname, this.Key.Port, databaseName, interpolateParams, tlsOption) + return fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?interpolateParams=%t&autocommit=true&charset=utf8mb4,utf8,latin1&tls=%s", this.User, this.Password, hostname, this.Key.Port, databaseName, interpolateParams, tlsOption) } diff --git a/go/mysql/connection_test.go b/go/mysql/connection_test.go index 27f37f2..6ce85a7 100644 --- a/go/mysql/connection_test.go +++ b/go/mysql/connection_test.go @@ -69,7 +69,7 @@ func TestGetDBUri(t *testing.T) { c.Password = "penguin" uri := c.GetDBUri("test") - test.S(t).ExpectEquals(uri, "gromit:penguin@tcp(myhost:3306)/test?interpolateParams=true&autocommit=true&charset=utf8mb4,utf8,latin1&tls=false&allowNativePasswords=true") + test.S(t).ExpectEquals(uri, "gromit:penguin@tcp(myhost:3306)/test?interpolateParams=true&autocommit=true&charset=utf8mb4,utf8,latin1&tls=false") } func TestGetDBUriWithTLSSetup(t *testing.T) { @@ -80,5 +80,5 @@ func TestGetDBUriWithTLSSetup(t *testing.T) { c.tlsConfig = &tls.Config{} uri := c.GetDBUri("test") - test.S(t).ExpectEquals(uri, "gromit:penguin@tcp(myhost:3306)/test?interpolateParams=true&autocommit=true&charset=utf8mb4,utf8,latin1&tls=ghost&allowNativePasswords=true") + test.S(t).ExpectEquals(uri, "gromit:penguin@tcp(myhost:3306)/test?interpolateParams=true&autocommit=true&charset=utf8mb4,utf8,latin1&tls=ghost") } From 0e2d33ad86c11a26abda216027e78831646d04c4 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Tue, 20 Oct 2020 16:08:49 +0200 Subject: [PATCH 6/8] Merge in https://github.com/github/gh-ost/pull/755 --- go/logic/applier.go | 14 +++++++++----- go/logic/migrator.go | 8 ++++++-- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/go/logic/applier.go b/go/logic/applier.go index 089978c..8aa1c9c 100644 --- a/go/logic/applier.go +++ b/go/logic/applier.go @@ -17,6 +17,7 @@ import ( "github.com/github/gh-ost/go/sql" "github.com/outbrain/golib/sqlutils" + "sync" ) const ( @@ -787,7 +788,7 @@ func (this *Applier) CreateAtomicCutOverSentryTable() error { } // AtomicCutOverMagicLock -func (this *Applier) AtomicCutOverMagicLock(sessionIdChan chan int64, tableLocked chan<- error, okToUnlockTable <-chan bool, tableUnlocked chan<- error) error { +func (this *Applier) AtomicCutOverMagicLock(sessionIdChan chan int64, tableLocked chan<- error, okToUnlockTable <-chan bool, tableUnlocked chan<- error, dropCutOverSentryTableOnce *sync.Once) error { tx, err := this.db.Begin() if err != nil { tableLocked <- err @@ -865,10 +866,13 @@ func (this *Applier) AtomicCutOverMagicLock(sessionIdChan chan int64, tableLocke sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.GetOldTableName()), ) - if _, err := tx.Exec(query); err != nil { - this.migrationContext.Log.Errore(err) - // We DO NOT return here because we must `UNLOCK TABLES`! - } + + dropCutOverSentryTableOnce.Do(func() { + if _, err := tx.Exec(query); err != nil { + this.migrationContext.Log.Errore(err) + // We DO NOT return here because we must `UNLOCK TABLES`! + } + }) // Tables still locked this.migrationContext.Log.Infof("Releasing lock from %s.%s, %s.%s", diff --git a/go/logic/migrator.go b/go/logic/migrator.go index 70af08d..85dae29 100644 --- a/go/logic/migrator.go +++ b/go/logic/migrator.go @@ -11,6 +11,7 @@ import ( "math" "os" "strings" + "sync" "sync/atomic" "time" @@ -606,9 +607,12 @@ func (this *Migrator) atomicCutOver() (err error) { defer atomic.StoreInt64(&this.migrationContext.InCutOverCriticalSectionFlag, 0) okToUnlockTable := make(chan bool, 4) + var dropCutOverSentryTableOnce sync.Once defer func() { okToUnlockTable <- true - this.applier.DropAtomicCutOverSentryTableIfExists() + dropCutOverSentryTableOnce.Do(func() { + this.applier.DropAtomicCutOverSentryTableIfExists() + }) }() atomic.StoreInt64(&this.migrationContext.AllEventsUpToLockProcessedInjectedFlag, 0) @@ -617,7 +621,7 @@ func (this *Migrator) atomicCutOver() (err error) { tableLocked := make(chan error, 2) tableUnlocked := make(chan error, 2) go func() { - if err := this.applier.AtomicCutOverMagicLock(lockOriginalSessionIdChan, tableLocked, okToUnlockTable, tableUnlocked); err != nil { + if err := this.applier.AtomicCutOverMagicLock(lockOriginalSessionIdChan, tableLocked, okToUnlockTable, tableUnlocked, &dropCutOverSentryTableOnce); err != nil { this.migrationContext.Log.Errore(err) } }() From dd2568631a8e4523981c898a930a0119d338ccd9 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Thu, 17 Dec 2020 17:26:51 +0100 Subject: [PATCH 7/8] Use golang:1.15.x, remove jessie tarball build --- .github/workflows/ci.yml | 4 ++-- Dockerfile.packaging | 2 +- Dockerfile.test | 2 +- script/build-deploy-tarball | 2 -- 4 files changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3556e1e..cf518ee 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,10 +10,10 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Set up Go 1.14 + - name: Set up Go 1.15 uses: actions/setup-go@v1 with: - go-version: 1.14 + go-version: 1.15 - name: Build run: script/cibuild diff --git a/Dockerfile.packaging b/Dockerfile.packaging index 9c5cd29..092fade 100644 --- a/Dockerfile.packaging +++ b/Dockerfile.packaging @@ -1,6 +1,6 @@ # -FROM golang:1.14.7 +FROM golang:1.15.6 RUN apt-get update RUN apt-get install -y ruby ruby-dev rubygems build-essential diff --git a/Dockerfile.test b/Dockerfile.test index 8f56be3..ceb46bf 100644 --- a/Dockerfile.test +++ b/Dockerfile.test @@ -1,4 +1,4 @@ -FROM golang:1.14.7 +FROM golang:1.15.6 LABEL maintainer="github@github.com" RUN apt-get update diff --git a/script/build-deploy-tarball b/script/build-deploy-tarball index 95da838..ebd7c64 100755 --- a/script/build-deploy-tarball +++ b/script/build-deploy-tarball @@ -32,6 +32,4 @@ cp ${tarball}.gz "$BUILD_ARTIFACT_DIR"/gh-ost/ # blame @carlosmn, @mattr and @timvaillancourt- # Allow builds on buster to also be used for stretch + jessie stretch_tarball_name=$(echo $(basename "${tarball}") | sed s/-buster-/-stretch-/) -jessie_tarball_name=$(echo $(basename "${stretch_tarball_name}") | sed s/-stretch-/-jessie-/) cp ${tarball}.gz "$BUILD_ARTIFACT_DIR/gh-ost/${stretch_tarball_name}.gz" -cp ${tarball}.gz "$BUILD_ARTIFACT_DIR/gh-ost/${jessie_tarball_name}.gz" From 60294109ca895f0fcc4b98dec2cd4e32304a5a49 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Thu, 17 Dec 2020 23:17:34 +0100 Subject: [PATCH 8/8] Update comment --- script/build-deploy-tarball | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/build-deploy-tarball b/script/build-deploy-tarball index ebd7c64..dc28b43 100755 --- a/script/build-deploy-tarball +++ b/script/build-deploy-tarball @@ -30,6 +30,6 @@ cp ${tarball}.gz "$BUILD_ARTIFACT_DIR"/gh-ost/ ### HACK HACK HACK HACK ### # blame @carlosmn, @mattr and @timvaillancourt- -# Allow builds on buster to also be used for stretch + jessie +# Allow builds on buster to also be used for stretch stretch_tarball_name=$(echo $(basename "${tarball}") | sed s/-buster-/-stretch-/) cp ${tarball}.gz "$BUILD_ARTIFACT_DIR/gh-ost/${stretch_tarball_name}.gz"