Merge branch 'master' into patch-1

.github/workflows/ci.yml

@@ -5,7 +5,7 @@ on: [pull_request]
 jobs:
   build:
 
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
 
     steps:
     - uses: actions/checkout@v2

.github/workflows/replica-tests.yml

@@ -5,7 +5,7 @@ on: [pull_request]
 jobs:
   build:
 
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     strategy:
       matrix:
         version: [mysql-5.7.25,mysql-8.0.16]

doc/command-line-flags.md

@@ -45,6 +45,22 @@ If you happen to _know_ your servers use RBR (Row Based Replication, i.e. `binlo
 Skipping this step means `gh-ost` would not need the `SUPER` privilege in order to operate.
 You may want to use this on Amazon RDS.
 
+### attempt-instant-ddl
+
+MySQL 8.0 supports "instant DDL" for some operations. If an alter statement can be completed with instant DDL, only a metadata change is required internally. Instant operations include:
+
+- Adding a column
+- Dropping a column
+- Dropping an index
+- Extending a varchar column
+- Adding a virtual generated column
+
+It is not reliable to parse the `ALTER` statement to determine if it is instant or not. This is because the table might be in an older row format, or have some other incompatibility that is difficult to identify.
+
+`--attempt-instant-ddl` is disabled by default, but the risks of enabling it are relatively minor: `gh-ost` may need to acquire a metadata lock at the start of the operation. This is not a problem for most scenarios, but it could be a problem for users that start the DDL during a period with long running transactions.
+
+`gh-ost` will automatically fallback to the normal DDL process if the attempt to use instant DDL is unsuccessful.
+
 ### conf
 
 `--conf=/path/to/my.cnf`: file where credentials are specified. Should be in (or contain) the following format:

go/base/context.go

@@ -101,6 +101,7 @@ type MigrationContext struct {
 	AliyunRDS                bool
 	GoogleCloudPlatform      bool
 	AzureMySQL               bool
+	AttemptInstantDDL        bool
 
 	config            ContextConfig
 	configMutex       *sync.Mutex

go/cmd/gh-ost/main.go

@@ -67,6 +67,8 @@ func main() {
 	flag.StringVar(&migrationContext.DatabaseName, "database", "", "database name (mandatory)")
 	flag.StringVar(&migrationContext.OriginalTableName, "table", "", "table name (mandatory)")
 	flag.StringVar(&migrationContext.AlterStatement, "alter", "", "alter statement (mandatory)")
+	flag.BoolVar(&migrationContext.AttemptInstantDDL, "attempt-instant-ddl", false, "Attempt to use instant DDL for this migration first")
+
 	flag.BoolVar(&migrationContext.CountTableRows, "exact-rowcount", false, "actually count table rows as opposed to estimate them (results in more accurate progress estimation)")
 	flag.BoolVar(&migrationContext.ConcurrentCountTableRows, "concurrent-rowcount", true, "(with --exact-rowcount), when true (default): count rows after row-copy begins, concurrently, and adjust row estimate later on; when false: first count rows, then start row copy")
 	flag.BoolVar(&migrationContext.AllowedRunningOnMaster, "allow-on-master", false, "allow this migration to run directly on master. Preferably it would run on a replica")

go/logic/applier.go

@@ -135,6 +135,16 @@ func (this *Applier) generateSqlModeQuery() string {
 	return fmt.Sprintf("sql_mode = %s", sqlModeQuery)
 }
 
+// generateInstantDDLQuery returns the SQL for this ALTER operation
+// with an INSTANT assertion (requires MySQL 8.0+)
+func (this *Applier) generateInstantDDLQuery() string {
+	return fmt.Sprintf(`ALTER /* gh-ost */ TABLE %s.%s %s, ALGORITHM=INSTANT`,
+		sql.EscapeName(this.migrationContext.DatabaseName),
+		sql.EscapeName(this.migrationContext.OriginalTableName),
+		this.migrationContext.AlterStatementOptions,
+	)
+}
+
 // readTableColumns reads table columns on applier
 func (this *Applier) readTableColumns() (err error) {
 	this.migrationContext.Log.Infof("Examining table structure on applier")
@@ -188,6 +198,27 @@ func (this *Applier) ValidateOrDropExistingTables() error {
 	return nil
 }
 
+// AttemptInstantDDL attempts to use instant DDL (from MySQL 8.0, and earlier in Aurora and some others).
+// If successful, the operation is only a meta-data change so a lot of time is saved!
+// The risk of attempting to instant DDL when not supported is that a metadata lock may be acquired.
+// This is minor, since gh-ost will eventually require a metadata lock anyway, but at the cut-over stage.
+// Instant operations include:
+// - Adding a column
+// - Dropping a column
+// - Dropping an index
+// - Extending a VARCHAR column
+// - Adding a virtual generated column
+// It is not reliable to parse the `alter` statement to determine if it is instant or not.
+// This is because the table might be in an older row format, or have some other incompatibility
+// that is difficult to identify.
+func (this *Applier) AttemptInstantDDL() error {
+	query := this.generateInstantDDLQuery()
+	this.migrationContext.Log.Infof("INSTANT DDL query is: %s", query)
+	// We don't need a trx, because for instant DDL the SQL mode doesn't matter.
+	_, err := this.db.Exec(query)
+	return err
+}
+
 // CreateGhostTable creates the ghost table on the applier host
 func (this *Applier) CreateGhostTable() error {
 	query := fmt.Sprintf(`create /* gh-ost */ table %s.%s like %s.%s`,

go/logic/applier_test.go

@@ -170,3 +170,16 @@ func TestApplierBuildDMLEventQuery(t *testing.T) {
 		test.S(t).ExpectEquals(res[0].args[3], 42)
 	})
 }
+
+func TestApplierInstantDDL(t *testing.T) {
+	migrationContext := base.NewMigrationContext()
+	migrationContext.DatabaseName = "test"
+	migrationContext.OriginalTableName = "mytable"
+	migrationContext.AlterStatementOptions = "ADD INDEX (foo)"
+	applier := NewApplier(migrationContext)
+
+	t.Run("instantDDLstmt", func(t *testing.T) {
+		stmt := applier.generateInstantDDLQuery()
+		test.S(t).ExpectEquals(stmt, "ALTER /* gh-ost */ TABLE `test`.`mytable` ADD INDEX (foo), ALGORITHM=INSTANT")
+	})
+}

go/logic/migrator.go

@@ -360,6 +360,17 @@ func (this *Migrator) Migrate() (err error) {
 	if err := this.createFlagFiles(); err != nil {
 		return err
 	}
+	// In MySQL 8.0 (and possibly earlier) some DDL statements can be applied instantly.
+	// Attempt to do this if AttemptInstantDDL is set.
+	if this.migrationContext.AttemptInstantDDL {
+		this.migrationContext.Log.Infof("Attempting to execute alter with ALGORITHM=INSTANT")
+		if err := this.applier.AttemptInstantDDL(); err == nil {
+			this.migrationContext.Log.Infof("Success! table %s.%s migrated instantly", sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName))
+			return nil
+		} else {
+			this.migrationContext.Log.Infof("ALGORITHM=INSTANT not supported for this operation, proceeding with original algorithm: %s", err)
+		}
+	}
 
 	initialLag, _ := this.inspector.getReplicationLag()
 	this.migrationContext.Log.Infof("Waiting for ghost table to be migrated. Current lag is %+v", initialLag)
@@ -1036,6 +1047,7 @@ func (this *Migrator) printStatus(rule PrintStatusRule, writers ...io.Writer) {
 	)
 	w := io.MultiWriter(writers...)
 	fmt.Fprintln(w, status)
+	this.migrationContext.Log.Infof(status)
 
 	hooksStatusIntervalSec := this.migrationContext.HooksStatusIntervalSec
 	if hooksStatusIntervalSec > 0 && elapsedSeconds%hooksStatusIntervalSec == 0 {

localtests/attempt-instant-ddl/create.sql

@@ -0,0 +1,13 @@
+drop table if exists gh_ost_test;
+create table gh_ost_test (
+  id int auto_increment,
+  i int not null,
+  color varchar(32),
+  primary key(id)
+) auto_increment=1;
+
+drop event if exists gh_ost_test;
+
+insert into gh_ost_test values (null, 11, 'red');
+insert into gh_ost_test values (null, 13, 'green');
+insert into gh_ost_test values (null, 17, 'blue');

localtests/attempt-instant-ddl/extra_args

@@ -0,0 +1 @@
+--attempt-instant-ddl