octoleo/gh-ost
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Support a complete ALTER TABLE statement in --alter

Browse Source
This commit is contained in:
Shlomi Noach 2020-07-22 12:33:02 +03:00
parent 4dab06e92b
commit 6c7b4736e1
4 changed files with 178 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
go/cmd/gh-ost/main.go
View File

@ -14,6 +14,7 @@ import (
"github.com/github/gh-ost/go/base"
"github.com/github/gh-ost/go/logic"
"github.com/github/gh-ost/go/sql"
_ "github.com/go-sql-driver/mysql"
"github.com/outbrain/golib/log"
@ -172,15 +173,25 @@ func main() {
log.SetLevel(log.ERROR)
}
if migrationContext.DatabaseName == "" {
log.Fatalf("--database must be provided and database name must not be empty")
}
if migrationContext.OriginalTableName == "" {
log.Fatalf("--table must be provided and table name must not be empty")
}
if migrationContext.AlterStatement == "" {
log.Fatalf("--alter must be provided and statement must not be empty")
}
parser := sql.NewParserFromAlterStatement(migrationContext.AlterStatement)
if migrationContext.DatabaseName == "" {
if parser.HasExplicitSchema() {
migrationContext.DatabaseName = parser.GetExplicitSchema()
} else {
log.Fatalf("--database must be provided and database name must not be empty, or --alter must specify database name")
}
}
if migrationContext.OriginalTableName == "" {
if parser.HasExplicitTable() {
migrationContext.OriginalTableName = parser.GetExplicitTable()
} else {
log.Fatalf("--table must be provided and table name must not be empty, or --alter must specify table name")
}
}
migrationContext.Noop = !(*executeFlag)
if migrationContext.AllowedRunningOnMaster && migrationContext.TestOnReplica {
log.Fatalf("--allow-on-master and --test-on-replica are mutually exclusive")

4
go/logic/migrator.go
View File

@ -62,7 +62,7 @@ const (
// Migrator is the main schema migration flow manager.
type Migrator struct {
parser *sql.Parser
parser *sql.AlterTableParser
inspector *Inspector
applier *Applier
eventsStreamer *EventsStreamer
@ -90,7 +90,7 @@ type Migrator struct {
func NewMigrator(context *base.MigrationContext) *Migrator {
migrator := &Migrator{
migrationContext: context,
parser: sql.NewParser(),
parser: sql.NewAlterTableParser(),
ghostTableMigrated: make(chan bool),
firstThrottlingCollected: make(chan bool, 3),
rowCopyComplete: make(chan error),

83
go/sql/parser.go
View File

@ -12,26 +12,47 @@ import (
)
var (
sanitizeQuotesRegexp = regexp.MustCompile("('[^']*')")
renameColumnRegexp = regexp.MustCompile(`(?i)\bchange\s+(column\s+|)([\S]+)\s+([\S]+)\s+`)
dropColumnRegexp = regexp.MustCompile(`(?i)\bdrop\s+(column\s+|)([\S]+)$`)
renameTableRegexp = regexp.MustCompile(`(?i)\brename\s+(to|as)\s+`)
sanitizeQuotesRegexp = regexp.MustCompile("('[^']*')")
renameColumnRegexp = regexp.MustCompile(`(?i)\bchange\s+(column\s+|)([\S]+)\s+([\S]+)\s+`)
dropColumnRegexp = regexp.MustCompile(`(?i)\bdrop\s+(column\s+|)([\S]+)$`)
renameTableRegexp = regexp.MustCompile(`(?i)\brename\s+(to|as)\s+`)
alterTableExplicitSchemaTableRegexps = []*regexp.Regexp{
regexp.MustCompile(`(?i)\balter\s+table\s+` + "`" + `([^` + "`" + `]+)` + "`" + `[.]` + "`" + `([^` + "`" + `]+)` + "`" + `\s+(.*$)`),
regexp.MustCompile(`(?i)\balter\s+table\s+` + "`" + `([^` + "`" + `]+)` + "`" + `[.]([\S]+)\s+(.*$)`),
regexp.MustCompile(`(?i)\balter\s+table\s+([\S]+)[.]` + "`" + `([^` + "`" + `]+)` + "`" + `\s+(.*$)`),
regexp.MustCompile(`(?i)\balter\s+table\s+([\S]+)[.]([\S]+)\s+(.*$)`),
}
alterTableExplicitTableRegexps = []*regexp.Regexp{
regexp.MustCompile(`(?i)\balter\s+table\s+` + "`" + `([^` + "`" + `]+)` + "`" + `\s+(.*$)`),
regexp.MustCompile(`(?i)\balter\s+table\s+([\S]+)\s+(.*$)`),
}
)
type Parser struct {
type AlterTableParser struct {
columnRenameMap map[string]string
droppedColumns map[string]bool
isRenameTable bool
alterTokens []string
explicitSchema string
explicitTable string
}
func NewParser() *Parser {
return &Parser{
func NewAlterTableParser() *AlterTableParser {
return &AlterTableParser{
columnRenameMap: make(map[string]string),
droppedColumns: make(map[string]bool),
}
}
func (this *Parser) tokenizeAlterStatement(alterStatement string) (tokens []string, err error) {
func NewParserFromAlterStatement(alterStatement string) *AlterTableParser {
parser := NewAlterTableParser()
parser.ParseAlterStatement(alterStatement)
return parser
}
func (this *AlterTableParser) tokenizeAlterStatement(alterStatement string) (tokens []string, err error) {
terminatingQuote := rune(0)
f := func(c rune) bool {
switch {
@ -58,13 +79,13 @@ func (this *Parser) tokenizeAlterStatement(alterStatement string) (tokens []stri
return tokens, nil
}
func (this *Parser) sanitizeQuotesFromAlterStatement(alterStatement string) (strippedStatement string) {
func (this *AlterTableParser) sanitizeQuotesFromAlterStatement(alterStatement string) (strippedStatement string) {
strippedStatement = alterStatement
strippedStatement = sanitizeQuotesRegexp.ReplaceAllString(strippedStatement, "''")
return strippedStatement
}
func (this *Parser) parseAlterToken(alterToken string) (err error) {
func (this *AlterTableParser) parseAlterToken(alterToken string) (err error) {
{
// rename
allStringSubmatch := renameColumnRegexp.FindAllStringSubmatch(alterToken, -1)
@ -97,16 +118,33 @@ func (this *Parser) parseAlterToken(alterToken string) (err error) {
return nil
}
func (this *Parser) ParseAlterStatement(alterStatement string) (err error) {
func (this *AlterTableParser) ParseAlterStatement(alterStatement string) (err error) {
for _, alterTableRegexp := range alterTableExplicitSchemaTableRegexps {
if submatch := alterTableRegexp.FindStringSubmatch(alterStatement); len(submatch) > 0 {
this.explicitSchema = submatch[1]
this.explicitTable = submatch[2]
alterStatement = submatch[3]
break
}
}
for _, alterTableRegexp := range alterTableExplicitTableRegexps {
if submatch := alterTableRegexp.FindStringSubmatch(alterStatement); len(submatch) > 0 {
this.explicitTable = submatch[1]
alterStatement = submatch[2]
break
}
}
alterTokens, _ := this.tokenizeAlterStatement(alterStatement)
for _, alterToken := range alterTokens {
alterToken = this.sanitizeQuotesFromAlterStatement(alterToken)
this.parseAlterToken(alterToken)
this.alterTokens = append(this.alterTokens, alterToken)
}
return nil
}
func (this *Parser) GetNonTrivialRenames() map[string]string {
func (this *AlterTableParser) GetNonTrivialRenames() map[string]string {
result := make(map[string]string)
for column, renamed := range this.columnRenameMap {
if column != renamed {
@ -116,14 +154,29 @@ func (this *Parser) GetNonTrivialRenames() map[string]string {
return result
}
func (this *Parser) HasNonTrivialRenames() bool {
func (this *AlterTableParser) HasNonTrivialRenames() bool {
return len(this.GetNonTrivialRenames()) > 0
}
func (this *Parser) DroppedColumnsMap() map[string]bool {
func (this *AlterTableParser) DroppedColumnsMap() map[string]bool {
return this.droppedColumns
}
func (this *Parser) IsRenameTable() bool {
func (this *AlterTableParser) IsRenameTable() bool {
return this.isRenameTable
}
func (this *AlterTableParser) GetExplicitSchema() string {
return this.explicitSchema
}
func (this *AlterTableParser) HasExplicitSchema() bool {
return this.GetExplicitSchema() != ""
}
func (this *AlterTableParser) GetExplicitTable() string {
return this.explicitTable
}
func (this *AlterTableParser) HasExplicitTable() bool {
return this.GetExplicitTable() != ""
}

106
go/sql/parser_test.go
View File

@ -19,7 +19,7 @@ func init() {
func TestParseAlterStatement(t *testing.T) {
statement := "add column t int, engine=innodb"
parser := NewParser()
parser := NewAlterTableParser()
err := parser.ParseAlterStatement(statement)
test.S(t).ExpectNil(err)
test.S(t).ExpectFalse(parser.HasNonTrivialRenames())
@ -27,7 +27,7 @@ func TestParseAlterStatement(t *testing.T) {
func TestParseAlterStatementTrivialRename(t *testing.T) {
statement := "add column t int, change ts ts timestamp, engine=innodb"
parser := NewParser()
parser := NewAlterTableParser()
err := parser.ParseAlterStatement(statement)
test.S(t).ExpectNil(err)
test.S(t).ExpectFalse(parser.HasNonTrivialRenames())
@ -37,7 +37,7 @@ func TestParseAlterStatementTrivialRename(t *testing.T) {
func TestParseAlterStatementTrivialRenames(t *testing.T) {
statement := "add column t int, change ts ts timestamp, CHANGE f `f` float, engine=innodb"
parser := NewParser()
parser := NewAlterTableParser()
err := parser.ParseAlterStatement(statement)
test.S(t).ExpectNil(err)
test.S(t).ExpectFalse(parser.HasNonTrivialRenames())
@ -58,7 +58,7 @@ func TestParseAlterStatementNonTrivial(t *testing.T) {
}
for _, statement := range statements {
parser := NewParser()
parser := NewAlterTableParser()
err := parser.ParseAlterStatement(statement)
test.S(t).ExpectNil(err)
renames := parser.GetNonTrivialRenames()
@ -69,7 +69,7 @@ func TestParseAlterStatementNonTrivial(t *testing.T) {
}
func TestTokenizeAlterStatement(t *testing.T) {
parser := NewParser()
parser := NewAlterTableParser()
{
alterStatement := "add column t int"
tokens, _ := parser.tokenizeAlterStatement(alterStatement)
@ -108,7 +108,7 @@ func TestTokenizeAlterStatement(t *testing.T) {
}
func TestSanitizeQuotesFromAlterStatement(t *testing.T) {
parser := NewParser()
parser := NewAlterTableParser()
{
alterStatement := "add column e enum('a','b','c')"
strippedStatement := parser.sanitizeQuotesFromAlterStatement(alterStatement)
@ -124,7 +124,7 @@ func TestSanitizeQuotesFromAlterStatement(t *testing.T) {
func TestParseAlterStatementDroppedColumns(t *testing.T) {
{
parser := NewParser()
parser := NewAlterTableParser()
statement := "drop column b"
err := parser.ParseAlterStatement(statement)
test.S(t).ExpectNil(err)
@ -132,7 +132,7 @@ func TestParseAlterStatementDroppedColumns(t *testing.T) {
test.S(t).ExpectTrue(parser.droppedColumns["b"])
}
{
parser := NewParser()
parser := NewAlterTableParser()
statement := "drop column b, drop key c_idx, drop column `d`"
err := parser.ParseAlterStatement(statement)
test.S(t).ExpectNil(err)
@ -141,7 +141,7 @@ func TestParseAlterStatementDroppedColumns(t *testing.T) {
test.S(t).ExpectTrue(parser.droppedColumns["d"])
}
{
parser := NewParser()
parser := NewAlterTableParser()
statement := "drop column b, drop key c_idx, drop column `d`, drop `e`, drop primary key, drop foreign key fk_1"
err := parser.ParseAlterStatement(statement)
test.S(t).ExpectNil(err)
@ -151,7 +151,7 @@ func TestParseAlterStatementDroppedColumns(t *testing.T) {
test.S(t).ExpectTrue(parser.droppedColumns["e"])
}
{
parser := NewParser()
parser := NewAlterTableParser()
statement := "drop column b, drop bad statement, add column i int"
err := parser.ParseAlterStatement(statement)
test.S(t).ExpectNil(err)
@ -163,38 +163,114 @@ func TestParseAlterStatementDroppedColumns(t *testing.T) {
func TestParseAlterStatementRenameTable(t *testing.T) {
{
parser := NewParser()
parser := NewAlterTableParser()
statement := "drop column b"
err := parser.ParseAlterStatement(statement)
test.S(t).ExpectNil(err)
test.S(t).ExpectFalse(parser.isRenameTable)
}
{
parser := NewParser()
parser := NewAlterTableParser()
statement := "rename as something_else"
err := parser.ParseAlterStatement(statement)
test.S(t).ExpectNil(err)
test.S(t).ExpectTrue(parser.isRenameTable)
}
{
parser := NewParser()
parser := NewAlterTableParser()
statement := "drop column b, rename as something_else"
err := parser.ParseAlterStatement(statement)
test.S(t).ExpectNil(err)
test.S(t).ExpectTrue(parser.isRenameTable)
}
{
parser := NewParser()
parser := NewAlterTableParser()
statement := "engine=innodb rename as something_else"
err := parser.ParseAlterStatement(statement)
test.S(t).ExpectNil(err)
test.S(t).ExpectTrue(parser.isRenameTable)
}
{
parser := NewParser()
parser := NewAlterTableParser()
statement := "rename as something_else, engine=innodb"
err := parser.ParseAlterStatement(statement)
test.S(t).ExpectNil(err)
test.S(t).ExpectTrue(parser.isRenameTable)
}
}
func TestParseAlterStatementExplicitTable(t *testing.T) {
{
parser := NewAlterTableParser()
statement := "drop column b"
err := parser.ParseAlterStatement(statement)
test.S(t).ExpectNil(err)
test.S(t).ExpectEquals(parser.explicitSchema, "")
test.S(t).ExpectEquals(parser.explicitTable, "")
test.S(t).ExpectTrue(reflect.DeepEqual(parser.alterTokens, []string{"drop column b"}))
}
{
parser := NewAlterTableParser()
statement := "alter table tbl drop column b"
err := parser.ParseAlterStatement(statement)
test.S(t).ExpectNil(err)
test.S(t).ExpectEquals(parser.explicitSchema, "")
test.S(t).ExpectEquals(parser.explicitTable, "tbl")
test.S(t).ExpectTrue(reflect.DeepEqual(parser.alterTokens, []string{"drop column b"}))
}
{
parser := NewAlterTableParser()
statement := "alter table `tbl` drop column b"
err := parser.ParseAlterStatement(statement)
test.S(t).ExpectNil(err)
test.S(t).ExpectEquals(parser.explicitSchema, "")
test.S(t).ExpectEquals(parser.explicitTable, "tbl")
test.S(t).ExpectTrue(reflect.DeepEqual(parser.alterTokens, []string{"drop column b"}))
}
{
parser := NewAlterTableParser()
statement := "alter table `scm with spaces`.`tbl` drop column b"
err := parser.ParseAlterStatement(statement)
test.S(t).ExpectNil(err)
test.S(t).ExpectEquals(parser.explicitSchema, "scm with spaces")
test.S(t).ExpectEquals(parser.explicitTable, "tbl")
test.S(t).ExpectTrue(reflect.DeepEqual(parser.alterTokens, []string{"drop column b"}))
}
{
parser := NewAlterTableParser()
statement := "alter table `scm`.`tbl with spaces` drop column b"
err := parser.ParseAlterStatement(statement)
test.S(t).ExpectNil(err)
test.S(t).ExpectEquals(parser.explicitSchema, "scm")
test.S(t).ExpectEquals(parser.explicitTable, "tbl with spaces")
test.S(t).ExpectTrue(reflect.DeepEqual(parser.alterTokens, []string{"drop column b"}))
}
{
parser := NewAlterTableParser()
statement := "alter table `scm`.tbl drop column b"
err := parser.ParseAlterStatement(statement)
test.S(t).ExpectNil(err)
test.S(t).ExpectEquals(parser.explicitSchema, "scm")
test.S(t).ExpectEquals(parser.explicitTable, "tbl")
test.S(t).ExpectTrue(reflect.DeepEqual(parser.alterTokens, []string{"drop column b"}))
}
{
parser := NewAlterTableParser()
statement := "alter table scm.`tbl` drop column b"
err := parser.ParseAlterStatement(statement)
test.S(t).ExpectNil(err)
test.S(t).ExpectEquals(parser.explicitSchema, "scm")
test.S(t).ExpectEquals(parser.explicitTable, "tbl")
test.S(t).ExpectTrue(reflect.DeepEqual(parser.alterTokens, []string{"drop column b"}))
}
{
parser := NewAlterTableParser()
statement := "alter table scm.tbl drop column b"
err := parser.ParseAlterStatement(statement)
test.S(t).ExpectNil(err)
test.S(t).ExpectEquals(parser.explicitSchema, "scm")
test.S(t).ExpectEquals(parser.explicitTable, "tbl")
test.S(t).ExpectTrue(reflect.DeepEqual(parser.alterTokens, []string{"drop column b"}))
}
}