Merge pull request #12 from openark/copy-auto-increment

Copying AUTO_INCREMENT value to ghost table

.github/workflows/ci.yml

@@ -10,10 +10,10 @@ jobs:
     steps:
     - uses: actions/checkout@v2
 
-    - name: Set up Go 1.14
+    - name: Set up Go 1.15
       uses: actions/setup-go@v1
       with:
-        go-version: 1.14
+        go-version: 1.15
 
     - name: Build
       run: script/cibuild

Dockerfile.packaging

@@ -1,6 +1,6 @@
 #
 
-FROM golang:1.14.7
+FROM golang:1.15.6
 
 RUN apt-get update
 RUN apt-get install -y ruby ruby-dev rubygems build-essential

Dockerfile.test

@@ -1,4 +1,4 @@
-FROM golang:1.14.7
+FROM golang:1.15.6
 LABEL maintainer="github@github.com"
 
 RUN apt-get update

README.md

@@ -65,6 +65,7 @@ Also see:
 - [the fine print](doc/the-fine-print.md)
 - [Community questions](https://github.com/github/gh-ost/issues?q=label%3Aquestion)
 - [Using `gh-ost` on AWS RDS](doc/rds.md)
+- [Using `gh-ost` on Azure Database for MySQL](doc/azure.md)
 
 ## What's in a name?
 

doc/azure.md

@@ -0,0 +1,26 @@
+`gh-ost` has been updated to work with Azure Database for MySQL however due to GitHub does not use it, this documentation is community driven so if you find a bug please [open an issue][new_issue]!
+
+# Azure Database for MySQL
+
+## Limitations
+
+- `gh-ost` runs should be setup use [`--assume-rbr`][assume_rbr_docs] and use `binlog_row_image=FULL`.
+- Azure Database for MySQL does not use same user name suffix for master and replica, so master host, user and password need to be pointed out. 
+
+## Step
+1. Change the replica server's `binlog_row_image` from `MINIMAL` to `FULL`. See [guide](https://docs.microsoft.com/en-us/azure/mysql/howto-server-parameters) on Azure document.
+2. Use your `gh-ost` always with additional 5 parameter
+```{bash}
+gh-ost \
+--azure \
+--assume-master-host=master-server-dns-name \
+--master-user="master-user-name" \
+--master-password="master-password" \
+--assume-rbr \
+[-- other paramters you need]
+```
+
+
+[new_issue]: https://github.com/github/gh-ost/issues/new
+[assume_rbr_docs]: https://github.com/github/gh-ost/blob/master/doc/command-line-flags.md#assume-rbr
+[migrate_test_on_replica_docs]: https://github.com/github/gh-ost/blob/master/doc/cheatsheet.md#c-migratetest-on-replica

doc/command-line-flags.md

@@ -6,6 +6,10 @@ A more in-depth discussion of various `gh-ost` command line flags: implementatio
 
 Add this flag when executing on Aliyun RDS.
 
+###  azure
+
+Add this flag when executing on Azure Database for MySQL.
+
 ### allow-master-master
 
 See [`--assume-master-host`](#assume-master-host).

doc/requirements-and-limitations.md

@@ -41,6 +41,7 @@ The `SUPER` privilege is required for `STOP SLAVE`, `START SLAVE` operations. Th
 - Amazon RDS works, but has its own [limitations](rds.md).
 - Google Cloud SQL works, `--gcp` flag required.
 - Aliyun RDS works, `--aliyun-rds` flag required.
+- Azure Database for MySQL works, `--azure` flag required, and have detailed document about it. (azure.md)
 
 - Multisource is not supported when migrating via replica. It _should_ work (but never tested) when connecting directly to master (`--allow-on-master`)
 

go/base/context.go

@@ -97,6 +97,7 @@ type MigrationContext struct {
 	DiscardForeignKeys       bool
 	AliyunRDS                bool
 	GoogleCloudPlatform      bool
+	AzureMySQL               bool
 
 	config            ContextConfig
 	configMutex       *sync.Mutex
@@ -203,6 +204,7 @@ type MigrationContext struct {
 	OriginalTableColumns             *sql.ColumnList
 	OriginalTableVirtualColumns      *sql.ColumnList
 	OriginalTableUniqueKeys          [](*sql.UniqueKey)
+	OriginalTableAutoIncrement       uint64
 	GhostTableColumns                *sql.ColumnList
 	GhostTableVirtualColumns         *sql.ColumnList
 	GhostTableUniqueKeys             [](*sql.UniqueKey)

go/base/utils.go

@@ -75,7 +75,8 @@ func ValidateConnection(db *gosql.DB, connectionConfig *mysql.ConnectionConfig,
 	}
 	// AliyunRDS set users port to "NULL", replace it by gh-ost param
 	// GCP set users port to "NULL", replace it by gh-ost param
-	if migrationContext.AliyunRDS || migrationContext.GoogleCloudPlatform {
+	// Azure MySQL set users port to a different value by design, replace it by gh-ost para
+	if migrationContext.AliyunRDS || migrationContext.GoogleCloudPlatform || migrationContext.AzureMySQL {
 		port = connectionConfig.Key.Port
 	} else {
 		portQuery := `select @@global.port`

go/cmd/gh-ost/main.go

@@ -79,6 +79,7 @@ func main() {
 	flag.BoolVar(&migrationContext.SkipStrictMode, "skip-strict-mode", false, "explicitly tell gh-ost binlog applier not to enforce strict sql mode")
 	flag.BoolVar(&migrationContext.AliyunRDS, "aliyun-rds", false, "set to 'true' when you execute on Aliyun RDS.")
 	flag.BoolVar(&migrationContext.GoogleCloudPlatform, "gcp", false, "set to 'true' when you execute on a 1st generation Google Cloud Platform (GCP).")
+	flag.BoolVar(&migrationContext.AzureMySQL, "azure", false, "set to 'true' when you execute on Azure Database on MySQL.")
 
 	executeFlag := flag.Bool("execute", false, "actually execute the alter & migrate the table. Default is noop: do some tests and exit")
 	flag.BoolVar(&migrationContext.TestOnReplica, "test-on-replica", false, "Have the migration run on a replica, not on the master. At the end of migration replication is stopped, and tables are swapped and immediately swap-revert. Replication remains stopped and you can compare the two tables for building trust")

go/logic/applier.go

@@ -17,6 +17,7 @@ import (
 	"github.com/github/gh-ost/go/sql"
 
 	"github.com/outbrain/golib/sqlutils"
+	"sync"
 )
 
 const (
@@ -88,7 +89,7 @@ func (this *Applier) InitDBConnections() (err error) {
 	if err := this.validateAndReadTimeZone(); err != nil {
 		return err
 	}
-	if !this.migrationContext.AliyunRDS && !this.migrationContext.GoogleCloudPlatform {
+	if !this.migrationContext.AliyunRDS && !this.migrationContext.GoogleCloudPlatform && !this.migrationContext.AzureMySQL {
 		if impliedKey, err := mysql.GetInstanceKey(this.db); err != nil {
 			return err
 		} else {
@@ -204,6 +205,25 @@ func (this *Applier) AlterGhost() error {
 	return nil
 }
 
+// AlterGhost applies `alter` statement on ghost table
+func (this *Applier) AlterGhostAutoIncrement() error {
+	query := fmt.Sprintf(`alter /* gh-ost */ table %s.%s AUTO_INCREMENT=%d`,
+		sql.EscapeName(this.migrationContext.DatabaseName),
+		sql.EscapeName(this.migrationContext.GetGhostTableName()),
+		this.migrationContext.OriginalTableAutoIncrement,
+	)
+	this.migrationContext.Log.Infof("Altering ghost table AUTO_INCREMENT value %s.%s",
+		sql.EscapeName(this.migrationContext.DatabaseName),
+		sql.EscapeName(this.migrationContext.GetGhostTableName()),
+	)
+	this.migrationContext.Log.Debugf("AUTO_INCREMENT ALTER statement: %s", query)
+	if _, err := sqlutils.ExecNoPrepare(this.db, query); err != nil {
+		return err
+	}
+	this.migrationContext.Log.Infof("Ghost table AUTO_INCREMENT altered")
+	return nil
+}
+
 // CreateChangelogTable creates the changelog table on the applier host
 func (this *Applier) CreateChangelogTable() error {
 	if err := this.DropChangelogTable(); err != nil {
@@ -787,7 +807,7 @@ func (this *Applier) CreateAtomicCutOverSentryTable() error {
 }
 
 // AtomicCutOverMagicLock
-func (this *Applier) AtomicCutOverMagicLock(sessionIdChan chan int64, tableLocked chan<- error, okToUnlockTable <-chan bool, tableUnlocked chan<- error) error {
+func (this *Applier) AtomicCutOverMagicLock(sessionIdChan chan int64, tableLocked chan<- error, okToUnlockTable <-chan bool, tableUnlocked chan<- error, dropCutOverSentryTableOnce *sync.Once) error {
 	tx, err := this.db.Begin()
 	if err != nil {
 		tableLocked <- err
@@ -865,10 +885,13 @@ func (this *Applier) AtomicCutOverMagicLock(sessionIdChan chan int64, tableLocke
 		sql.EscapeName(this.migrationContext.DatabaseName),
 		sql.EscapeName(this.migrationContext.GetOldTableName()),
 	)
-	if _, err := tx.Exec(query); err != nil {
-		this.migrationContext.Log.Errore(err)
-		// We DO NOT return here because we must `UNLOCK TABLES`!
-	}
+
+	dropCutOverSentryTableOnce.Do(func() {
+		if _, err := tx.Exec(query); err != nil {
+			this.migrationContext.Log.Errore(err)
+			// We DO NOT return here because we must `UNLOCK TABLES`!
+		}
+	})
 
 	// Tables still locked
 	this.migrationContext.Log.Infof("Releasing lock from %s.%s, %s.%s",

go/logic/inspect.go

@@ -52,7 +52,7 @@ func (this *Inspector) InitDBConnections() (err error) {
 	if err := this.validateConnection(); err != nil {
 		return err
 	}
-	if !this.migrationContext.AliyunRDS && !this.migrationContext.GoogleCloudPlatform {
+	if !this.migrationContext.AliyunRDS && !this.migrationContext.GoogleCloudPlatform && !this.migrationContext.AzureMySQL {
 		if impliedKey, err := mysql.GetInstanceKey(this.db); err != nil {
 			return err
 		} else {
@@ -109,6 +109,10 @@ func (this *Inspector) InspectOriginalTable() (err error) {
 	if err != nil {
 		return err
 	}
+	this.migrationContext.OriginalTableAutoIncrement, err = this.getAutoIncrementValue(this.migrationContext.OriginalTableName)
+	if err != nil {
+		return err
+	}
 	return nil
 }
 
@@ -589,6 +593,24 @@ func (this *Inspector) applyColumnTypes(databaseName, tableName string, columnsL
 	return err
 }
 
+// getAutoIncrementValue get's the original table's AUTO_INCREMENT value, if exists (0 value if not exists)
+func (this *Inspector) getAutoIncrementValue(tableName string) (autoIncrement uint64, err error) {
+	query := `
+		SELECT
+			AUTO_INCREMENT
+		FROM INFORMATION_SCHEMA.TABLES
+		WHERE
+			TABLES.TABLE_SCHEMA = ?
+			AND TABLES.TABLE_NAME = ?
+			AND AUTO_INCREMENT IS NOT NULL
+  `
+	err = sqlutils.QueryRowsMap(this.db, query, func(m sqlutils.RowMap) error {
+		autoIncrement = m.GetUint64("AUTO_INCREMENT")
+		return nil
+	}, this.migrationContext.DatabaseName, tableName)
+	return autoIncrement, err
+}
+
 // getCandidateUniqueKeys investigates a table and returns the list of unique keys
 // candidate for chunking
 func (this *Inspector) getCandidateUniqueKeys(tableName string) (uniqueKeys [](*sql.UniqueKey), err error) {

go/logic/migrator.go

@@ -11,6 +11,7 @@ import (
 	"math"
 	"os"
 	"strings"
+	"sync"
 	"sync/atomic"
 	"time"
 
@@ -606,9 +607,12 @@ func (this *Migrator) atomicCutOver() (err error) {
 	defer atomic.StoreInt64(&this.migrationContext.InCutOverCriticalSectionFlag, 0)
 
 	okToUnlockTable := make(chan bool, 4)
+	var dropCutOverSentryTableOnce sync.Once
 	defer func() {
 		okToUnlockTable <- true
-		this.applier.DropAtomicCutOverSentryTableIfExists()
+		dropCutOverSentryTableOnce.Do(func() {
+			this.applier.DropAtomicCutOverSentryTableIfExists()
+		})
 	}()
 
 	atomic.StoreInt64(&this.migrationContext.AllEventsUpToLockProcessedInjectedFlag, 0)
@@ -617,7 +621,7 @@ func (this *Migrator) atomicCutOver() (err error) {
 	tableLocked := make(chan error, 2)
 	tableUnlocked := make(chan error, 2)
 	go func() {
-		if err := this.applier.AtomicCutOverMagicLock(lockOriginalSessionIdChan, tableLocked, okToUnlockTable, tableUnlocked); err != nil {
+		if err := this.applier.AtomicCutOverMagicLock(lockOriginalSessionIdChan, tableLocked, okToUnlockTable, tableUnlocked, &dropCutOverSentryTableOnce); err != nil {
 			this.migrationContext.Log.Errore(err)
 		}
 	}()
@@ -737,7 +741,7 @@ func (this *Migrator) initiateInspector() (err error) {
 		this.migrationContext.Log.Infof("Master found to be %+v", *this.migrationContext.ApplierConnectionConfig.ImpliedKey)
 	} else {
 		// Forced master host.
-		key, err := mysql.ParseRawInstanceKeyLoose(this.migrationContext.AssumeMasterHostname)
+		key, err := mysql.ParseInstanceKey(this.migrationContext.AssumeMasterHostname)
 		if err != nil {
 			return err
 		}
@@ -1068,6 +1072,14 @@ func (this *Migrator) initiateApplier() error {
 		return err
 	}
 
+	if this.migrationContext.OriginalTableAutoIncrement > 0 && !this.parser.IsAutoIncrementDefined() {
+		// Original table has AUTO_INCREMENT value and the -alter statement does not indicate any override,
+		// so we should copy AUTO_INCREMENT value onto our ghost table.
+		if err := this.applier.AlterGhostAutoIncrement(); err != nil {
+			this.migrationContext.Log.Errorf("Unable to ALTER ghost table AUTO_INCREMENT value, see further error details. Bailing out")
+			return err
+		}
+	}
 	this.applier.WriteChangelogState(string(GhostTableMigrated))
 	go this.applier.InitiateHeartbeat()
 	return nil

go/mysql/instance_key.go

@@ -7,6 +7,7 @@ package mysql
 
 import (
 	"fmt"
+	"regexp"
 	"strconv"
 	"strings"
 )
@@ -15,6 +16,13 @@ const (
 	DefaultInstancePort = 3306
 )
 
+var (
+	ipv4HostPortRegexp = regexp.MustCompile("^([^:]+):([0-9]+)$")
+	ipv4HostRegexp     = regexp.MustCompile("^([^:]+)$")
+	ipv6HostPortRegexp = regexp.MustCompile("^\\[([:0-9a-fA-F]+)\\]:([0-9]+)$") // e.g. [2001:db8:1f70::999:de8:7648:6e8]:3308
+	ipv6HostRegexp     = regexp.MustCompile("^([:0-9a-fA-F]+)$")                // e.g. 2001:db8:1f70::999:de8:7648:6e8
+)
+
 // InstanceKey is an instance indicator, identified by hostname and port
 type InstanceKey struct {
 	Hostname string
@@ -25,25 +33,35 @@ const detachHint = "//"
 
 // ParseInstanceKey will parse an InstanceKey from a string representation such as 127.0.0.1:3306
 func NewRawInstanceKey(hostPort string) (*InstanceKey, error) {
-	tokens := strings.SplitN(hostPort, ":", 2)
-	if len(tokens) != 2 {
-		return nil, fmt.Errorf("Cannot parse InstanceKey from %s. Expected format is host:port", hostPort)
+	hostname := ""
+	port := ""
+	if submatch := ipv4HostPortRegexp.FindStringSubmatch(hostPort); len(submatch) > 0 {
+		hostname = submatch[1]
+		port = submatch[2]
+	} else if submatch := ipv4HostRegexp.FindStringSubmatch(hostPort); len(submatch) > 0 {
+		hostname = submatch[1]
+	} else if submatch := ipv6HostPortRegexp.FindStringSubmatch(hostPort); len(submatch) > 0 {
+		hostname = submatch[1]
+		port = submatch[2]
+	} else if submatch := ipv6HostRegexp.FindStringSubmatch(hostPort); len(submatch) > 0 {
+		hostname = submatch[1]
+	} else {
+		return nil, fmt.Errorf("Cannot parse address: %s", hostPort)
 	}
-	instanceKey := &InstanceKey{Hostname: tokens[0]}
-	var err error
-	if instanceKey.Port, err = strconv.Atoi(tokens[1]); err != nil {
-		return instanceKey, fmt.Errorf("Invalid port: %s", tokens[1])
+	instanceKey := &InstanceKey{Hostname: hostname, Port: DefaultInstancePort}
+	if port != "" {
+		var err error
+		if instanceKey.Port, err = strconv.Atoi(port); err != nil {
+			return instanceKey, fmt.Errorf("Invalid port: %s", port)
+		}
 	}
 
 	return instanceKey, nil
 }
 
-// ParseRawInstanceKeyLoose will parse an InstanceKey from a string representation such as 127.0.0.1:3306.
+// ParseInstanceKey will parse an InstanceKey from a string representation such as 127.0.0.1:3306.
 // The port part is optional; there will be no name resolve
-func ParseRawInstanceKeyLoose(hostPort string) (*InstanceKey, error) {
-	if !strings.Contains(hostPort, ":") {
-		return &InstanceKey{Hostname: hostPort, Port: DefaultInstancePort}, nil
-	}
+func ParseInstanceKey(hostPort string) (*InstanceKey, error) {
 	return NewRawInstanceKey(hostPort)
 }
 

go/mysql/instance_key_map.go

@@ -92,7 +92,7 @@ func (this *InstanceKeyMap) ReadCommaDelimitedList(list string) error {
 	}
 	tokens := strings.Split(list, ",")
 	for _, token := range tokens {
-		key, err := ParseRawInstanceKeyLoose(token)
+		key, err := ParseInstanceKey(token)
 		if err != nil {
 			return err
 		}

go/mysql/instance_key_test.go

@@ -0,0 +1,74 @@
+/*
+   Copyright 2016 GitHub Inc.
+	 See https://github.com/github/gh-ost/blob/master/LICENSE
+*/
+
+package mysql
+
+import (
+	"testing"
+
+	"github.com/outbrain/golib/log"
+	test "github.com/outbrain/golib/tests"
+)
+
+func init() {
+	log.SetLevel(log.ERROR)
+}
+
+func TestParseInstanceKey(t *testing.T) {
+	{
+		key, err := ParseInstanceKey("myhost:1234")
+		test.S(t).ExpectNil(err)
+		test.S(t).ExpectEquals(key.Hostname, "myhost")
+		test.S(t).ExpectEquals(key.Port, 1234)
+	}
+	{
+		key, err := ParseInstanceKey("myhost")
+		test.S(t).ExpectNil(err)
+		test.S(t).ExpectEquals(key.Hostname, "myhost")
+		test.S(t).ExpectEquals(key.Port, 3306)
+	}
+	{
+		key, err := ParseInstanceKey("10.0.0.3:3307")
+		test.S(t).ExpectNil(err)
+		test.S(t).ExpectEquals(key.Hostname, "10.0.0.3")
+		test.S(t).ExpectEquals(key.Port, 3307)
+	}
+	{
+		key, err := ParseInstanceKey("10.0.0.3")
+		test.S(t).ExpectNil(err)
+		test.S(t).ExpectEquals(key.Hostname, "10.0.0.3")
+		test.S(t).ExpectEquals(key.Port, 3306)
+	}
+	{
+		key, err := ParseInstanceKey("[2001:db8:1f70::999:de8:7648:6e8]:3308")
+		test.S(t).ExpectNil(err)
+		test.S(t).ExpectEquals(key.Hostname, "2001:db8:1f70::999:de8:7648:6e8")
+		test.S(t).ExpectEquals(key.Port, 3308)
+	}
+	{
+		key, err := ParseInstanceKey("::1")
+		test.S(t).ExpectNil(err)
+		test.S(t).ExpectEquals(key.Hostname, "::1")
+		test.S(t).ExpectEquals(key.Port, 3306)
+	}
+	{
+		key, err := ParseInstanceKey("0:0:0:0:0:0:0:0")
+		test.S(t).ExpectNil(err)
+		test.S(t).ExpectEquals(key.Hostname, "0:0:0:0:0:0:0:0")
+		test.S(t).ExpectEquals(key.Port, 3306)
+	}
+	{
+		_, err := ParseInstanceKey("[2001:xxxx:1f70::999:de8:7648:6e8]:3308")
+		test.S(t).ExpectNotNil(err)
+	}
+	{
+		_, err := ParseInstanceKey("10.0.0.4:")
+		test.S(t).ExpectNotNil(err)
+	}
+	{
+		_, err := ParseInstanceKey("10.0.0.4:5.6.7")
+		test.S(t).ExpectNotNil(err)
+	}
+}

go/sql/parser.go

@@ -16,6 +16,7 @@ var (
 	renameColumnRegexp                   = regexp.MustCompile(`(?i)\bchange\s+(column\s+|)([\S]+)\s+([\S]+)\s+`)
 	dropColumnRegexp                     = regexp.MustCompile(`(?i)\bdrop\s+(column\s+|)([\S]+)$`)
 	renameTableRegexp                    = regexp.MustCompile(`(?i)\brename\s+(to|as)\s+`)
+	autoIncrementRegexp                  = regexp.MustCompile(`(?i)\bauto_increment[\s]*=[\s]*([0-9]+)`)
 	alterTableExplicitSchemaTableRegexps = []*regexp.Regexp{
 		// ALTER TABLE `scm`.`tbl` something
 		regexp.MustCompile(`(?i)\balter\s+table\s+` + "`" + `([^` + "`" + `]+)` + "`" + `[.]` + "`" + `([^` + "`" + `]+)` + "`" + `\s+(.*$)`),
@@ -35,9 +36,10 @@ var (
 )
 
 type AlterTableParser struct {
-	columnRenameMap map[string]string
-	droppedColumns  map[string]bool
-	isRenameTable   bool
+	columnRenameMap        map[string]string
+	droppedColumns         map[string]bool
+	isRenameTable          bool
+	isAutoIncrementDefined bool
 
 	alterStatementOptions string
 	alterTokens           []string
@@ -122,6 +124,12 @@ func (this *AlterTableParser) parseAlterToken(alterToken string) (err error) {
 			this.isRenameTable = true
 		}
 	}
+	{
+		// auto_increment
+		if autoIncrementRegexp.MatchString(alterToken) {
+			this.isAutoIncrementDefined = true
+		}
+	}
 	return nil
 }
 
@@ -173,6 +181,11 @@ func (this *AlterTableParser) DroppedColumnsMap() map[string]bool {
 func (this *AlterTableParser) IsRenameTable() bool {
 	return this.isRenameTable
 }
+
+func (this *AlterTableParser) IsAutoIncrementDefined() bool {
+	return this.isAutoIncrementDefined
+}
+
 func (this *AlterTableParser) GetExplicitSchema() string {
 	return this.explicitSchema
 }

go/sql/parser_test.go

@@ -24,6 +24,7 @@ func TestParseAlterStatement(t *testing.T) {
 	test.S(t).ExpectNil(err)
 	test.S(t).ExpectEquals(parser.alterStatementOptions, statement)
 	test.S(t).ExpectFalse(parser.HasNonTrivialRenames())
+	test.S(t).ExpectFalse(parser.IsAutoIncrementDefined())
 }
 
 func TestParseAlterStatementTrivialRename(t *testing.T) {
@@ -33,10 +34,31 @@ func TestParseAlterStatementTrivialRename(t *testing.T) {
 	test.S(t).ExpectNil(err)
 	test.S(t).ExpectEquals(parser.alterStatementOptions, statement)
 	test.S(t).ExpectFalse(parser.HasNonTrivialRenames())
+	test.S(t).ExpectFalse(parser.IsAutoIncrementDefined())
 	test.S(t).ExpectEquals(len(parser.columnRenameMap), 1)
 	test.S(t).ExpectEquals(parser.columnRenameMap["ts"], "ts")
 }
 
+func TestParseAlterStatementWithAutoIncrement(t *testing.T) {
+
+	statements := []string{
+		"auto_increment=7",
+		"auto_increment = 7",
+		"AUTO_INCREMENT = 71",
+		"add column t int, change ts ts timestamp, auto_increment=7 engine=innodb",
+		"add column t int, change ts ts timestamp, auto_increment =7 engine=innodb",
+		"add column t int, change ts ts timestamp, AUTO_INCREMENT = 7 engine=innodb",
+		"add column t int, change ts ts timestamp, engine=innodb auto_increment=73425",
+	}
+	for _, statement := range statements {
+		parser := NewAlterTableParser()
+		err := parser.ParseAlterStatement(statement)
+		test.S(t).ExpectNil(err)
+		test.S(t).ExpectEquals(parser.alterStatementOptions, statement)
+		test.S(t).ExpectTrue(parser.IsAutoIncrementDefined())
+	}
+}
+
 func TestParseAlterStatementTrivialRenames(t *testing.T) {
 	statement := "add column t int, change ts ts timestamp, CHANGE f `f` float, engine=innodb"
 	parser := NewAlterTableParser()
@@ -44,6 +66,7 @@ func TestParseAlterStatementTrivialRenames(t *testing.T) {
 	test.S(t).ExpectNil(err)
 	test.S(t).ExpectEquals(parser.alterStatementOptions, statement)
 	test.S(t).ExpectFalse(parser.HasNonTrivialRenames())
+	test.S(t).ExpectFalse(parser.IsAutoIncrementDefined())
 	test.S(t).ExpectEquals(len(parser.columnRenameMap), 2)
 	test.S(t).ExpectEquals(parser.columnRenameMap["ts"], "ts")
 	test.S(t).ExpectEquals(parser.columnRenameMap["f"], "f")
@@ -64,6 +87,7 @@ func TestParseAlterStatementNonTrivial(t *testing.T) {
 		parser := NewAlterTableParser()
 		err := parser.ParseAlterStatement(statement)
 		test.S(t).ExpectNil(err)
+		test.S(t).ExpectFalse(parser.IsAutoIncrementDefined())
 		test.S(t).ExpectEquals(parser.alterStatementOptions, statement)
 		renames := parser.GetNonTrivialRenames()
 		test.S(t).ExpectEquals(len(renames), 2)

localtests/autoinc-copy-deletes-user-defined/create.sql

@@ -0,0 +1,17 @@
+drop event if exists gh_ost_test;
+
+drop table if exists gh_ost_test;
+create table gh_ost_test (
+  id int auto_increment,
+  i int not null,
+  primary key(id)
+) auto_increment=1;
+
+insert into gh_ost_test values (NULL, 11);
+insert into gh_ost_test values (NULL, 13);
+insert into gh_ost_test values (NULL, 17);
+insert into gh_ost_test values (NULL, 23);
+insert into gh_ost_test values (NULL, 29);
+insert into gh_ost_test values (NULL, 31);
+insert into gh_ost_test values (NULL, 37);
+delete from gh_ost_test where id>=5;

localtests/autoinc-copy-deletes-user-defined/expect_table_structure

@@ -0,0 +1 @@
+AUTO_INCREMENT=7

localtests/autoinc-copy-deletes-user-defined/extra_args

@@ -0,0 +1 @@
+--alter='AUTO_INCREMENT=7'

localtests/autoinc-copy-deletes/create.sql

@@ -0,0 +1,17 @@
+drop event if exists gh_ost_test;
+
+drop table if exists gh_ost_test;
+create table gh_ost_test (
+  id int auto_increment,
+  i int not null,
+  primary key(id)
+) auto_increment=1;
+
+insert into gh_ost_test values (NULL, 11);
+insert into gh_ost_test values (NULL, 13);
+insert into gh_ost_test values (NULL, 17);
+insert into gh_ost_test values (NULL, 23);
+insert into gh_ost_test values (NULL, 29);
+insert into gh_ost_test values (NULL, 31);
+insert into gh_ost_test values (NULL, 37);
+delete from gh_ost_test where id>=5;

localtests/autoinc-copy-deletes/expect_table_structure

@@ -0,0 +1 @@
+AUTO_INCREMENT=8

localtests/autoinc-copy-simple/create.sql

@@ -0,0 +1,13 @@
+drop event if exists gh_ost_test;
+
+drop table if exists gh_ost_test;
+create table gh_ost_test (
+  id int auto_increment,
+  i int not null,
+  primary key(id)
+) auto_increment=1;
+
+insert into gh_ost_test values (NULL, 11);
+insert into gh_ost_test values (NULL, 13);
+insert into gh_ost_test values (NULL, 17);
+insert into gh_ost_test values (NULL, 23);

localtests/autoinc-copy-simple/expect_table_structure

@@ -0,0 +1 @@
+AUTO_INCREMENT=5

localtests/test.sh

@@ -12,6 +12,7 @@ test_logfile=/tmp/gh-ost-test.log
 default_ghost_binary=/tmp/gh-ost-test
 ghost_binary=""
 exec_command_file=/tmp/gh-ost-test.bash
+ghost_structure_output_file=/tmp/gh-ost-test.ghost.structure.sql
 orig_content_output_file=/tmp/gh-ost-test.orig.content.csv
 ghost_content_output_file=/tmp/gh-ost-test.ghost.content.csv
 throttle_flag_file=/tmp/gh-ost-test.ghost.throttle.flag
@@ -204,6 +205,18 @@ test_single() {
     return 1
   fi
 
+  gh-ost-test-mysql-replica --default-character-set=utf8mb4 test -e "show create table _gh_ost_test_gho\G" -ss > $ghost_structure_output_file
+
+  if [ -f $tests_path/$test_name/expect_table_structure ] ; then
+    expected_table_structure="$(cat $tests_path/$test_name/expect_table_structure)"
+    if ! grep -q "$expected_table_structure" $ghost_structure_output_file ; then
+      echo
+      echo "ERROR $test_name: table structure was expected to include ${expected_table_structure} but did not. cat $ghost_structure_output_file:"
+      cat $ghost_structure_output_file
+      return 1
+    fi
+  fi
+
   echo_dot
   gh-ost-test-mysql-replica --default-character-set=utf8mb4 test -e "select ${orig_columns} from gh_ost_test ${order_by}" -ss > $orig_content_output_file
   gh-ost-test-mysql-replica --default-character-set=utf8mb4 test -e "select ${ghost_columns} from _gh_ost_test_gho ${order_by}" -ss > $ghost_content_output_file

script/build-deploy-tarball

@@ -30,8 +30,6 @@ cp ${tarball}.gz "$BUILD_ARTIFACT_DIR"/gh-ost/
 
 ### HACK HACK HACK HACK ###
 # blame @carlosmn, @mattr and @timvaillancourt-
-# Allow builds on buster to also be used for stretch + jessie
+# Allow builds on buster to also be used for stretch
 stretch_tarball_name=$(echo $(basename "${tarball}") | sed s/-buster-/-stretch-/)
-jessie_tarball_name=$(echo $(basename "${stretch_tarball_name}") | sed s/-stretch-/-jessie-/)
 cp ${tarball}.gz "$BUILD_ARTIFACT_DIR/gh-ost/${stretch_tarball_name}.gz"
-cp ${tarball}.gz "$BUILD_ARTIFACT_DIR/gh-ost/${jessie_tarball_name}.gz"

vendor/github.com/outbrain/golib/sqlutils/sqlutils.go

@@ -117,6 +117,19 @@ func (this *RowMap) GetUintD(key string, def uint) uint {
 	return uint(res)
 }
 
+func (this *RowMap) GetUint64(key string) uint64 {
+	res, _ := strconv.ParseUint(this.GetString(key), 10, 0)
+	return res
+}
+
+func (this *RowMap) GetUint64D(key string, def uint64) uint64 {
+	res, err := strconv.ParseUint(this.GetString(key), 10, 0)
+	if err != nil {
+		return def
+	}
+	return uint64(res)
+}
+
 func (this *RowMap) GetBool(key string) bool {
 	return this.GetInt(key) != 0
 }