Add tests, incorporate feedback

This commit is contained in:
Morgan Tocker 2022-11-14 11:29:26 -07:00
parent 05f32ebf29
commit 75a346be93
7 changed files with 70 additions and 18 deletions

View File

@ -45,6 +45,22 @@ If you happen to _know_ your servers use RBR (Row Based Replication, i.e. `binlo
Skipping this step means `gh-ost` would not need the `SUPER` privilege in order to operate. Skipping this step means `gh-ost` would not need the `SUPER` privilege in order to operate.
You may want to use this on Amazon RDS. You may want to use this on Amazon RDS.
### attempt-instant-ddl
MySQL 8.0 support "instant DDL" for some operations. If an alter statement can be completed with instant DDL, only a metadata change is required internally, so MySQL will return _instantly_ (only requiring a metadata lock to complete). Instant operations include:
- Adding a column
- Dropping a column
- Dropping an index
- Extending a varchar column
- Adding a virtual generated column
It is not reliable to parse the `ALTER` statement to determine if it is instant or not. This is because the table might be in an older row format, or have some other incompatibility that is difficult to identify.
The risks of attempting to instant DDL are relatively minor: Gh-ost may need to acquire a metadata lock at the start of the operation. This is not a problem for most scenarios, but it could be a problem for users that start the DDL during a period with long running transactions.
gh-ost will automatically fallback to the normal DDL process if the attempt to use instant DDL is unsuccessful.
### conf ### conf
`--conf=/path/to/my.cnf`: file where credentials are specified. Should be in (or contain) the following format: `--conf=/path/to/my.cnf`: file where credentials are specified. Should be in (or contain) the following format:

View File

@ -67,7 +67,7 @@ func main() {
flag.StringVar(&migrationContext.DatabaseName, "database", "", "database name (mandatory)") flag.StringVar(&migrationContext.DatabaseName, "database", "", "database name (mandatory)")
flag.StringVar(&migrationContext.OriginalTableName, "table", "", "table name (mandatory)") flag.StringVar(&migrationContext.OriginalTableName, "table", "", "table name (mandatory)")
flag.StringVar(&migrationContext.AlterStatement, "alter", "", "alter statement (mandatory)") flag.StringVar(&migrationContext.AlterStatement, "alter", "", "alter statement (mandatory)")
flag.BoolVar(&migrationContext.AttemptInstantDDL, "attempt-instant-ddl", true, "Attempt to use instant DDL for this migration first.") flag.BoolVar(&migrationContext.AttemptInstantDDL, "attempt-instant-ddl", false, "Attempt to use instant DDL for this migration first")
flag.BoolVar(&migrationContext.CountTableRows, "exact-rowcount", false, "actually count table rows as opposed to estimate them (results in more accurate progress estimation)") flag.BoolVar(&migrationContext.CountTableRows, "exact-rowcount", false, "actually count table rows as opposed to estimate them (results in more accurate progress estimation)")
flag.BoolVar(&migrationContext.ConcurrentCountTableRows, "concurrent-rowcount", true, "(with --exact-rowcount), when true (default): count rows after row-copy begins, concurrently, and adjust row estimate later on; when false: first count rows, then start row copy") flag.BoolVar(&migrationContext.ConcurrentCountTableRows, "concurrent-rowcount", true, "(with --exact-rowcount), when true (default): count rows after row-copy begins, concurrently, and adjust row estimate later on; when false: first count rows, then start row copy")

View File

@ -135,6 +135,16 @@ func (this *Applier) generateSqlModeQuery() string {
return fmt.Sprintf("sql_mode = %s", sqlModeQuery) return fmt.Sprintf("sql_mode = %s", sqlModeQuery)
} }
// generateInstantDDLQuery returns the SQL for this ALTER operation
// with an INSTANT assertion (requires MySQL 8.0+)
func (this *Applier) generateInstantDDLQuery() string {
return fmt.Sprintf(`ALTER /* gh-ost */ TABLE %s.%s %s, ALGORITHM=INSTANT`,
sql.EscapeName(this.migrationContext.DatabaseName),
sql.EscapeName(this.migrationContext.OriginalTableName),
this.migrationContext.AlterStatementOptions,
)
}
// readTableColumns reads table columns on applier // readTableColumns reads table columns on applier
func (this *Applier) readTableColumns() (err error) { func (this *Applier) readTableColumns() (err error) {
this.migrationContext.Log.Infof("Examining table structure on applier") this.migrationContext.Log.Infof("Examining table structure on applier")
@ -188,22 +198,21 @@ func (this *Applier) ValidateOrDropExistingTables() error {
return nil return nil
} }
// AttemptInstantDDL attempts to use instant DDL (from MySQL 8.0, and earlier in Aurora and some others.) // AttemptInstantDDL attempts to use instant DDL (from MySQL 8.0, and earlier in Aurora and some others).
// to apply the ALTER statement immediately. If it errors, the original // If successful, the operation is only a meta-data change so a lot of time is saved!
// gh-ost algorithm can be used. However, if it's successful -- a lot // The risk of attempting to instant DDL when not supported is that a metadata lock may be acquired.
// of time can potentially be saved. Instant operations include: // This is minor, since gh-ost will eventually require a metadata lock anyway, but at the cut-over stage.
// Instant operations include:
// - Adding a column // - Adding a column
// - Dropping a column // - Dropping a column
// - Dropping an index // - Dropping an index
// - Extending a varchar column // - Extending a VARCHAR column
// It is safer to attempt the change than try and parse the DDL, since // - Adding a virtual generated column
// there might be specifics about the table which make it not possible to apply instantly. // It is not reliable to parse the `alter` statement to determine if it is instant or not.
// This is because the table might be in an older row format, or have some other incompatibility
// that is difficult to identify.
func (this *Applier) AttemptInstantDDL() error { func (this *Applier) AttemptInstantDDL() error {
query := fmt.Sprintf(`ALTER /* gh-ost */ TABLE %s.%s %s, ALGORITHM=INSTANT`, query := this.generateInstantDDLQuery()
sql.EscapeName(this.migrationContext.DatabaseName),
sql.EscapeName(this.migrationContext.OriginalTableName),
this.migrationContext.AlterStatementOptions,
)
this.migrationContext.Log.Infof("INSTANT DDL query is: %s", query) this.migrationContext.Log.Infof("INSTANT DDL query is: %s", query)
// We don't need a trx, because for instant DDL the SQL mode doesn't matter. // We don't need a trx, because for instant DDL the SQL mode doesn't matter.
_, err := this.db.Exec(query) _, err := this.db.Exec(query)

View File

@ -170,3 +170,17 @@ func TestApplierBuildDMLEventQuery(t *testing.T) {
test.S(t).ExpectEquals(res[0].args[3], 42) test.S(t).ExpectEquals(res[0].args[3], 42)
}) })
} }
func TestApplierInstantDDL(t *testing.T) {
migrationContext := base.NewMigrationContext()
migrationContext.DatabaseName = "test"
migrationContext.OriginalTableName = "mytable"
migrationContext.AlterStatementOptions = "ADD INDEX (foo)"
applier := NewApplier(migrationContext)
t.Run("instantDDLstmt", func(t *testing.T) {
stmt := applier.generateInstantDDLQuery()
test.S(t).ExpectEquals(stmt, "ALTER /* gh-ost */ TABLE `test`.`mytable` ADD INDEX (foo), ALGORITHM=INSTANT")
})
}

View File

@ -352,15 +352,14 @@ func (this *Migrator) Migrate() (err error) {
return err return err
} }
// In MySQL 8.0 (and possibly earlier) some DDL statements can be applied instantly. // In MySQL 8.0 (and possibly earlier) some DDL statements can be applied instantly.
// As just a metadata change. We can't detect this unless we attempt the statement // Attempt to do this if AttemptInstantDDL is set.
// (i.e. there is no explain for DDL).
if this.migrationContext.AttemptInstantDDL { if this.migrationContext.AttemptInstantDDL {
this.migrationContext.Log.Infof("Attempting to execute ALTER TABLE as INSTANT DDL") this.migrationContext.Log.Infof("Attempting to execute alter with ALGORITHM=INSTANT")
if err := this.attemptInstantDDL(); err == nil { if err := this.attemptInstantDDL(); err == nil {
this.migrationContext.Log.Infof("Success! Table %s.%s migrated instantly", sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName)) this.migrationContext.Log.Infof("Success! table %s.%s migrated instantly", sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName))
return nil return nil
} else { } else {
this.migrationContext.Log.Infof("INSTANT DDL failed, will proceed with original algorithm: %s", err) this.migrationContext.Log.Infof("ALGORITHM=INSTANT failed, proceeding with original algorithm: %s", err)
} }
} }

View File

@ -0,0 +1,13 @@
drop table if exists gh_ost_test;
create table gh_ost_test (
id int auto_increment,
i int not null,
color varchar(32),
primary key(id)
) auto_increment=1;
drop event if exists gh_ost_test;
insert into gh_ost_test values (null, 11, 'red');
insert into gh_ost_test values (null, 13, 'green');
insert into gh_ost_test values (null, 17, 'blue');

View File

@ -0,0 +1 @@
--attempt-instant-ddl