gh-ost/go/sql/parser.go

213 lines
6.2 KiB
Go
Raw Normal View History

/*
Copyright 2022 GitHub Inc.
See https://github.com/github/gh-ost/blob/master/LICENSE
*/
package sql
import (
"regexp"
"strconv"
"strings"
)
var (
sanitizeQuotesRegexp = regexp.MustCompile("('[^']*')")
renameColumnRegexp = regexp.MustCompile(`(?i)\bchange\s+(column\s+|)([\S]+)\s+([\S]+)\s+`)
dropColumnRegexp = regexp.MustCompile(`(?i)\bdrop\s+(column\s+|)([\S]+)$`)
renameTableRegexp = regexp.MustCompile(`(?i)\brename\s+(to|as)\s+`)
autoIncrementRegexp = regexp.MustCompile(`(?i)\bauto_increment[\s]*=[\s]*([0-9]+)`)
alterTableExplicitSchemaTableRegexps = []*regexp.Regexp{
2020-07-23 11:04:14 +00:00
// ALTER TABLE `scm`.`tbl` something
regexp.MustCompile(`(?i)\balter\s+table\s+` + "`" + `([^` + "`" + `]+)` + "`" + `[.]` + "`" + `([^` + "`" + `]+)` + "`" + `\s+(.*$)`),
2020-07-23 11:04:14 +00:00
// ALTER TABLE `scm`.tbl something
regexp.MustCompile(`(?i)\balter\s+table\s+` + "`" + `([^` + "`" + `]+)` + "`" + `[.]([\S]+)\s+(.*$)`),
2020-07-23 11:04:14 +00:00
// ALTER TABLE scm.`tbl` something
regexp.MustCompile(`(?i)\balter\s+table\s+([\S]+)[.]` + "`" + `([^` + "`" + `]+)` + "`" + `\s+(.*$)`),
2020-07-23 11:04:14 +00:00
// ALTER TABLE scm.tbl something
regexp.MustCompile(`(?i)\balter\s+table\s+([\S]+)[.]([\S]+)\s+(.*$)`),
}
alterTableExplicitTableRegexps = []*regexp.Regexp{
2020-07-23 11:04:14 +00:00
// ALTER TABLE `tbl` something
regexp.MustCompile(`(?i)\balter\s+table\s+` + "`" + `([^` + "`" + `]+)` + "`" + `\s+(.*$)`),
2020-07-23 11:04:14 +00:00
// ALTER TABLE tbl something
regexp.MustCompile(`(?i)\balter\s+table\s+([\S]+)\s+(.*$)`),
}
Enum to varchar (#963) * v1.1.0 * WIP: copying AUTO_INCREMENT value to ghost table Initial commit: towards setting up a test suite Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * greping for 'expect_table_structure' content * Adding simple test for 'expect_table_structure' scenario * adding tests for AUTO_INCREMENT value after row deletes. Should initially fail * clear event beforehand * parsing AUTO_INCREMENT from alter query, reading AUTO_INCREMENT from original table, applying AUTO_INCREMENT value onto ghost table if applicable and user has not specified AUTO_INCREMENT in alter statement * support GetUint64 Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * minor update to test Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * adding test for user defined AUTO_INCREMENT statement * Generated column as part of UNIQUE (or PRIMARY) KEY Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * skip analysis of generated column data type in unique key Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * All MySQL DBs limited to max 3 concurrent/idle connections Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * hooks: reporting GH_OST_ETA_SECONDS. ETA stored as part of migration context Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * GH_OST_ETA_NANOSECONDS Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * N/A denoted by negative value Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * ETAUnknown constant Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * Convering enum to varchar Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * test: not null Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * first attempt at setting enum-to-string right Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * fix insert query Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * store enum values, use when populating Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * apply EnumValues to mapped column Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * fix compilation error Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * gofmt Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com>
2021-06-10 15:17:49 +00:00
enumValuesRegexp = regexp.MustCompile("^enum[(](.*)[)]$")
)
type AlterTableParser struct {
columnRenameMap map[string]string
droppedColumns map[string]bool
isRenameTable bool
isAutoIncrementDefined bool
2020-07-23 08:38:05 +00:00
alterStatementOptions string
alterTokens []string
explicitSchema string
explicitTable string
}
func NewAlterTableParser() *AlterTableParser {
return &AlterTableParser{
columnRenameMap: make(map[string]string),
2017-04-23 05:23:56 +00:00
droppedColumns: make(map[string]bool),
}
}
func NewParserFromAlterStatement(alterStatement string) *AlterTableParser {
parser := NewAlterTableParser()
parser.ParseAlterStatement(alterStatement)
return parser
}
func (this *AlterTableParser) tokenizeAlterStatement(alterStatement string) (tokens []string) {
terminatingQuote := rune(0)
f := func(c rune) bool {
switch {
case c == terminatingQuote:
terminatingQuote = rune(0)
return false
case terminatingQuote != rune(0):
return false
case c == '\'':
terminatingQuote = c
return false
case c == '(':
terminatingQuote = ')'
return false
default:
return c == ','
}
}
tokens = strings.FieldsFunc(alterStatement, f)
for i := range tokens {
tokens[i] = strings.TrimSpace(tokens[i])
}
return tokens
}
func (this *AlterTableParser) sanitizeQuotesFromAlterStatement(alterStatement string) (strippedStatement string) {
strippedStatement = alterStatement
2016-11-29 14:47:39 +00:00
strippedStatement = sanitizeQuotesRegexp.ReplaceAllString(strippedStatement, "''")
return strippedStatement
}
func (this *AlterTableParser) parseAlterToken(alterToken string) {
2017-04-23 05:23:56 +00:00
{
// rename
allStringSubmatch := renameColumnRegexp.FindAllStringSubmatch(alterToken, -1)
for _, submatch := range allStringSubmatch {
if unquoted, err := strconv.Unquote(submatch[2]); err == nil {
submatch[2] = unquoted
}
if unquoted, err := strconv.Unquote(submatch[3]); err == nil {
submatch[3] = unquoted
}
this.columnRenameMap[submatch[2]] = submatch[3]
}
}
2017-04-23 05:23:56 +00:00
{
// drop
allStringSubmatch := dropColumnRegexp.FindAllStringSubmatch(alterToken, -1)
for _, submatch := range allStringSubmatch {
if unquoted, err := strconv.Unquote(submatch[2]); err == nil {
submatch[2] = unquoted
}
this.droppedColumns[submatch[2]] = true
}
}
2018-05-06 08:19:03 +00:00
{
// rename table
if renameTableRegexp.MatchString(alterToken) {
this.isRenameTable = true
}
}
{
// auto_increment
if autoIncrementRegexp.MatchString(alterToken) {
this.isAutoIncrementDefined = true
}
}
2017-04-23 05:23:56 +00:00
}
func (this *AlterTableParser) ParseAlterStatement(alterStatement string) (err error) {
2020-07-23 08:38:05 +00:00
this.alterStatementOptions = alterStatement
for _, alterTableRegexp := range alterTableExplicitSchemaTableRegexps {
2020-07-23 08:38:05 +00:00
if submatch := alterTableRegexp.FindStringSubmatch(this.alterStatementOptions); len(submatch) > 0 {
this.explicitSchema = submatch[1]
this.explicitTable = submatch[2]
2020-07-23 08:38:05 +00:00
this.alterStatementOptions = submatch[3]
break
}
}
for _, alterTableRegexp := range alterTableExplicitTableRegexps {
2020-07-23 08:38:05 +00:00
if submatch := alterTableRegexp.FindStringSubmatch(this.alterStatementOptions); len(submatch) > 0 {
this.explicitTable = submatch[1]
2020-07-23 08:38:05 +00:00
this.alterStatementOptions = submatch[2]
break
}
}
for _, alterToken := range this.tokenizeAlterStatement(this.alterStatementOptions) {
2017-04-23 05:23:56 +00:00
alterToken = this.sanitizeQuotesFromAlterStatement(alterToken)
this.parseAlterToken(alterToken)
this.alterTokens = append(this.alterTokens, alterToken)
2017-04-23 05:23:56 +00:00
}
return nil
}
func (this *AlterTableParser) GetNonTrivialRenames() map[string]string {
result := make(map[string]string)
for column, renamed := range this.columnRenameMap {
if column != renamed {
result[column] = renamed
}
}
return result
}
func (this *AlterTableParser) HasNonTrivialRenames() bool {
return len(this.GetNonTrivialRenames()) > 0
}
func (this *AlterTableParser) DroppedColumnsMap() map[string]bool {
return this.droppedColumns
}
2018-05-06 08:19:03 +00:00
func (this *AlterTableParser) IsRenameTable() bool {
2018-05-06 08:19:03 +00:00
return this.isRenameTable
}
func (this *AlterTableParser) IsAutoIncrementDefined() bool {
return this.isAutoIncrementDefined
}
func (this *AlterTableParser) GetExplicitSchema() string {
return this.explicitSchema
}
func (this *AlterTableParser) HasExplicitSchema() bool {
return this.GetExplicitSchema() != ""
}
func (this *AlterTableParser) GetExplicitTable() string {
return this.explicitTable
}
func (this *AlterTableParser) HasExplicitTable() bool {
return this.GetExplicitTable() != ""
}
2020-07-23 08:38:05 +00:00
func (this *AlterTableParser) GetAlterStatementOptions() string {
return this.alterStatementOptions
}
Enum to varchar (#963) * v1.1.0 * WIP: copying AUTO_INCREMENT value to ghost table Initial commit: towards setting up a test suite Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * greping for 'expect_table_structure' content * Adding simple test for 'expect_table_structure' scenario * adding tests for AUTO_INCREMENT value after row deletes. Should initially fail * clear event beforehand * parsing AUTO_INCREMENT from alter query, reading AUTO_INCREMENT from original table, applying AUTO_INCREMENT value onto ghost table if applicable and user has not specified AUTO_INCREMENT in alter statement * support GetUint64 Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * minor update to test Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * adding test for user defined AUTO_INCREMENT statement * Generated column as part of UNIQUE (or PRIMARY) KEY Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * skip analysis of generated column data type in unique key Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * All MySQL DBs limited to max 3 concurrent/idle connections Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * hooks: reporting GH_OST_ETA_SECONDS. ETA stored as part of migration context Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * GH_OST_ETA_NANOSECONDS Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * N/A denoted by negative value Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * ETAUnknown constant Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * Convering enum to varchar Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * test: not null Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * first attempt at setting enum-to-string right Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * fix insert query Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * store enum values, use when populating Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * apply EnumValues to mapped column Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * fix compilation error Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * gofmt Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com>
2021-06-10 15:17:49 +00:00
func ParseEnumValues(enumColumnType string) string {
if submatch := enumValuesRegexp.FindStringSubmatch(enumColumnType); len(submatch) > 0 {
return submatch[1]
}
return enumColumnType
}