Merge pull request #24 from github/initially-dropping-tables

old/ghost drop tables

go/base/context.go

@@ -58,6 +58,8 @@ type MigrationContext struct {
 	TestOnReplica           bool
 	OkToDropTable           bool
 	QuickAndBumpySwapTables bool
+	InitiallyDropOldTable   bool
+	InitiallyDropGhostTable bool
 
 	TableEngine               string
 	RowsEstimate              int64

go/cmd/gh-osc/main.go

@@ -35,6 +35,8 @@ func main() {
 	flag.BoolVar(&migrationContext.TestOnReplica, "test-on-replica", false, "Have the migration run on a replica, not on the master. At the end of migration tables are not swapped; gh-osc issues `STOP SLAVE` and you can compare the two tables for building trust")
 	flag.BoolVar(&migrationContext.OkToDropTable, "ok-to-drop-table", false, "Shall the tool drop the old table at end of operation. DROPping tables can be a long locking operation, which is why I'm not doing it by default. I'm an online tool, yes?")
 	flag.BoolVar(&migrationContext.QuickAndBumpySwapTables, "quick-and-bumpy-swap-tables", false, "Shall the tool issue a faster swapping of tables at end of operation, at the cost of causing a brief period of time when the table does not exist? This will cause queries on table to fail with error (as opposed to being locked for a longer duration of a swap)")
+	flag.BoolVar(&migrationContext.InitiallyDropOldTable, "initially-drop-old-table", false, "Drop a possibly existing OLD table (remains from a previous run?) before beginning operation. Default is to panic and abort if such table exists")
+	flag.BoolVar(&migrationContext.InitiallyDropGhostTable, "initially-drop-ghost-table", false, "Drop a possibly existing Ghost table (remains from a previous run?) before beginning operation. Default is to panic and abort if such table exists")
 
 	flag.BoolVar(&migrationContext.SwitchToRowBinlogFormat, "switch-to-rbr", false, "let this tool automatically switch binary log format to 'ROW' on the replica, if needed. The format will NOT be switched back. I'm too scared to do that, and wish to protect you if you happen to execute another migration while this one is running")
 	flag.Int64Var(&migrationContext.ChunkSize, "chunk-size", 1000, "amount of rows to handle in each iteration (allowed range: 100-100,000)")

go/logic/applier.go

@@ -69,6 +69,37 @@ func (this *Applier) validateConnection(db *gosql.DB) error {
 	return nil
 }
 
+func (this *Applier) tableExists(tableName string) (tableFound bool) {
+	query := fmt.Sprintf(`show /* gh-osc */ table status from %s like '%s'`, sql.EscapeName(this.migrationContext.DatabaseName), tableName)
+
+	sqlutils.QueryRowsMap(this.db, query, func(m sqlutils.RowMap) error {
+		tableFound = true
+		return nil
+	})
+	return tableFound
+}
+
+func (this *Applier) ValidateOrDropExistingTables() error {
+	if this.migrationContext.InitiallyDropGhostTable {
+		if err := this.DropGhostTable(); err != nil {
+			return err
+		}
+	}
+	if this.tableExists(this.migrationContext.GetGhostTableName()) {
+		return fmt.Errorf("Table %s already exists. Panicking. Use --initially-drop-ghost-table to force dropping it", sql.EscapeName(this.migrationContext.GetGhostTableName()))
+	}
+	if this.migrationContext.InitiallyDropOldTable {
+		if err := this.DropOldTable(); err != nil {
+			return err
+		}
+	}
+	if this.tableExists(this.migrationContext.GetOldTableName()) {
+		return fmt.Errorf("Table %s already exists. Panicking. Use --initially-drop-old-table to force dropping it", sql.EscapeName(this.migrationContext.GetOldTableName()))
+	}
+
+	return nil
+}
+
 // CreateGhostTable creates the ghost table on the applier host
 func (this *Applier) CreateGhostTable() error {
 	query := fmt.Sprintf(`create /* gh-osc */ table %s.%s like %s.%s`,
@@ -109,6 +140,9 @@ func (this *Applier) AlterGhost() error {
 
 // CreateChangelogTable creates the changelog table on the applier host
 func (this *Applier) CreateChangelogTable() error {
+	if err := this.DropChangelogTable(); err != nil {
+		return err
+	}
 	query := fmt.Sprintf(`create /* gh-osc */ table %s.%s (
 			id bigint auto_increment,
 			last_update timestamp not null DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
@@ -154,6 +188,11 @@ func (this *Applier) DropChangelogTable() error {
 	return this.dropTable(this.migrationContext.GetChangelogTableName())
 }
 
+// DropOldTable drops the _Old table on the applier host
+func (this *Applier) DropOldTable() error {
+	return this.dropTable(this.migrationContext.GetOldTableName())
+}
+
 // DropGhostTable drops the ghost table on the applier host
 func (this *Applier) DropGhostTable() error {
 	return this.dropTable(this.migrationContext.GetGhostTableName())

go/logic/migrator.go

@@ -344,10 +344,26 @@ func (this *Migrator) stopWritesAndCompleteMigration() (err error) {
 		}); err != nil {
 		return err
 	}
+	if err := this.dropOldTableIfRequired(); err != nil {
+		return err
+	}
 
 	return
 }
 
+func (this *Migrator) dropOldTableIfRequired() (err error) {
+	if !this.migrationContext.OkToDropTable {
+		return nil
+	}
+	dropTableFunc := func() error {
+		return this.applier.dropTable(this.migrationContext.GetOldTableName())
+	}
+	if err := this.retryOperation(dropTableFunc); err != nil {
+		return err
+	}
+	return nil
+}
+
 // stopWritesAndCompleteMigrationOnMasterQuickAndBumpy will lock down the original table, execute
 // what's left of last DML entries, and **non-atomically** swap original->old, then new->original.
 // There is a point in time where the "original" table does not exist and queries are non-blocked
@@ -368,14 +384,9 @@ func (this *Migrator) stopWritesAndCompleteMigrationOnMasterQuickAndBumpy() (err
 	if err := this.retryOperation(this.applier.UnlockTables); err != nil {
 		return err
 	}
-	if this.migrationContext.OkToDropTable {
+	if err := this.dropOldTableIfRequired(); err != nil {
-		dropTableFunc := func() error {
-			return this.applier.dropTable(this.migrationContext.GetOldTableName())
-		}
-		if err := this.retryOperation(dropTableFunc); err != nil {
 		return err
 	}
-	}
 
 	lockAndRenameDuration := this.migrationContext.RenameTablesEndTime.Sub(this.migrationContext.LockTablesStartTime)
 	renameDuration := this.migrationContext.RenameTablesEndTime.Sub(this.migrationContext.RenameTablesStartTime)
@@ -614,6 +625,9 @@ func (this *Migrator) initiateApplier() error {
 	if err := this.applier.InitDBConnections(); err != nil {
 		return err
 	}
+	if err := this.applier.ValidateOrDropExistingTables(); err != nil {
+		return err
+	}
 	if err := this.applier.CreateGhostTable(); err != nil {
 		log.Errorf("Unable to create ghost table, see further error details. Perhaps a previous migration failed without dropping the table? Bailing out")
 		return err