From 9bc508f068b28bd8f38cfcb558036e1f03dd2fa5 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Thu, 10 Jun 2021 18:17:49 +0300 Subject: [PATCH] Enum to varchar (#963) * v1.1.0 * WIP: copying AUTO_INCREMENT value to ghost table Initial commit: towards setting up a test suite Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * greping for 'expect_table_structure' content * Adding simple test for 'expect_table_structure' scenario * adding tests for AUTO_INCREMENT value after row deletes. Should initially fail * clear event beforehand * parsing AUTO_INCREMENT from alter query, reading AUTO_INCREMENT from original table, applying AUTO_INCREMENT value onto ghost table if applicable and user has not specified AUTO_INCREMENT in alter statement * support GetUint64 Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * minor update to test Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * adding test for user defined AUTO_INCREMENT statement * Generated column as part of UNIQUE (or PRIMARY) KEY Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * skip analysis of generated column data type in unique key Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * All MySQL DBs limited to max 3 concurrent/idle connections Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * hooks: reporting GH_OST_ETA_SECONDS. ETA stored as part of migration context Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * GH_OST_ETA_NANOSECONDS Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * N/A denoted by negative value Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * ETAUnknown constant Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * Convering enum to varchar Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * test: not null Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * first attempt at setting enum-to-string right Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * fix insert query Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * store enum values, use when populating Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * apply EnumValues to mapped column Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * fix compilation error Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * gofmt Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/logic/inspect.go | 5 +++++ go/sql/builder.go | 4 ++++ go/sql/parser.go | 8 ++++++++ go/sql/parser_test.go | 18 ++++++++++++++++++ go/sql/types.go | 27 ++++++++++++++++++++------- localtests/enum-to-varchar/create.sql | 26 ++++++++++++++++++++++++++ localtests/enum-to-varchar/extra_args | 1 + 7 files changed, 82 insertions(+), 7 deletions(-) create mode 100644 localtests/enum-to-varchar/create.sql create mode 100644 localtests/enum-to-varchar/extra_args diff --git a/go/logic/inspect.go b/go/logic/inspect.go index 097b58f..fb473b8 100644 --- a/go/logic/inspect.go +++ b/go/logic/inspect.go @@ -187,6 +187,10 @@ func (this *Inspector) inspectOriginalAndGhostTables() (err error) { if column.Name == mappedColumn.Name && column.Type == sql.DateTimeColumnType && mappedColumn.Type == sql.TimestampColumnType { this.migrationContext.MappedSharedColumns.SetConvertDatetimeToTimestamp(column.Name, this.migrationContext.ApplierTimeZone) } + if column.Name == mappedColumn.Name && column.Type == sql.EnumColumnType && mappedColumn.Charset != "" { + this.migrationContext.MappedSharedColumns.SetEnumToTextConversion(column.Name) + this.migrationContext.MappedSharedColumns.SetEnumValues(column.Name, column.EnumValues) + } } for _, column := range this.migrationContext.UniqueKey.Columns.Columns() { @@ -590,6 +594,7 @@ func (this *Inspector) applyColumnTypes(databaseName, tableName string, columnsL } if strings.HasPrefix(columnType, "enum") { column.Type = sql.EnumColumnType + column.EnumValues = sql.ParseEnumValues(m.GetString("COLUMN_TYPE")) } if strings.HasPrefix(columnType, "binary") { column.Type = sql.BinaryColumnType diff --git a/go/sql/builder.go b/go/sql/builder.go index 776a10d..7fe366c 100644 --- a/go/sql/builder.go +++ b/go/sql/builder.go @@ -38,6 +38,8 @@ func buildColumnsPreparedValues(columns *ColumnList) []string { var token string if column.timezoneConversion != nil { token = fmt.Sprintf("convert_tz(?, '%s', '%s')", column.timezoneConversion.ToTimezone, "+00:00") + } else if column.enumToTextConversion { + token = fmt.Sprintf("ELT(?, %s)", column.EnumValues) } else if column.Type == JSONColumnType { token = "convert(? using utf8mb4)" } else { @@ -108,6 +110,8 @@ func BuildSetPreparedClause(columns *ColumnList) (result string, err error) { var setToken string if column.timezoneConversion != nil { setToken = fmt.Sprintf("%s=convert_tz(?, '%s', '%s')", EscapeName(column.Name), column.timezoneConversion.ToTimezone, "+00:00") + } else if column.enumToTextConversion { + setToken = fmt.Sprintf("%s=ELT(?, %s)", EscapeName(column.Name), column.EnumValues) } else if column.Type == JSONColumnType { setToken = fmt.Sprintf("%s=convert(? using utf8mb4)", EscapeName(column.Name)) } else { diff --git a/go/sql/parser.go b/go/sql/parser.go index d9c0c3f..eac0bdc 100644 --- a/go/sql/parser.go +++ b/go/sql/parser.go @@ -33,6 +33,7 @@ var ( // ALTER TABLE tbl something regexp.MustCompile(`(?i)\balter\s+table\s+([\S]+)\s+(.*$)`), } + enumValuesRegexp = regexp.MustCompile("^enum[(](.*)[)]$") ) type AlterTableParser struct { @@ -205,3 +206,10 @@ func (this *AlterTableParser) HasExplicitTable() bool { func (this *AlterTableParser) GetAlterStatementOptions() string { return this.alterStatementOptions } + +func ParseEnumValues(enumColumnType string) string { + if submatch := enumValuesRegexp.FindStringSubmatch(enumColumnType); len(submatch) > 0 { + return submatch[1] + } + return enumColumnType +} diff --git a/go/sql/parser_test.go b/go/sql/parser_test.go index 6cdbb39..3157d09 100644 --- a/go/sql/parser_test.go +++ b/go/sql/parser_test.go @@ -322,3 +322,21 @@ func TestParseAlterStatementExplicitTable(t *testing.T) { test.S(t).ExpectTrue(reflect.DeepEqual(parser.alterTokens, []string{"drop column b", "add index idx(i)"})) } } + +func TestParseEnumValues(t *testing.T) { + { + s := "enum('red','green','blue','orange')" + values := ParseEnumValues(s) + test.S(t).ExpectEquals(values, "'red','green','blue','orange'") + } + { + s := "('red','green','blue','orange')" + values := ParseEnumValues(s) + test.S(t).ExpectEquals(values, "('red','green','blue','orange')") + } + { + s := "zzz" + values := ParseEnumValues(s) + test.S(t).ExpectEquals(values, "zzz") + } +} diff --git a/go/sql/types.go b/go/sql/types.go index fa6b74e..3c4ce5e 100644 --- a/go/sql/types.go +++ b/go/sql/types.go @@ -33,15 +33,16 @@ type TimezoneConversion struct { } type Column struct { - Name string - IsUnsigned bool - Charset string - Type ColumnType - + Name string + IsUnsigned bool + Charset string + Type ColumnType + EnumValues string + timezoneConversion *TimezoneConversion + enumToTextConversion bool // add Octet length for binary type, fix bytes with suffix "00" get clipped in mysql binlog. // https://github.com/github/gh-ost/issues/909 - BinaryOctetLength uint - timezoneConversion *TimezoneConversion + BinaryOctetLength uint } func (this *Column) convertArg(arg interface{}, isUniqueKeyColumn bool) interface{} { @@ -198,6 +199,18 @@ func (this *ColumnList) HasTimezoneConversion(columnName string) bool { return this.GetColumn(columnName).timezoneConversion != nil } +func (this *ColumnList) SetEnumToTextConversion(columnName string) { + this.GetColumn(columnName).enumToTextConversion = true +} + +func (this *ColumnList) IsEnumToTextConversion(columnName string) bool { + return this.GetColumn(columnName).enumToTextConversion +} + +func (this *ColumnList) SetEnumValues(columnName string, enumValues string) { + this.GetColumn(columnName).EnumValues = enumValues +} + func (this *ColumnList) String() string { return strings.Join(this.Names(), ",") } diff --git a/localtests/enum-to-varchar/create.sql b/localtests/enum-to-varchar/create.sql new file mode 100644 index 0000000..0dbab17 --- /dev/null +++ b/localtests/enum-to-varchar/create.sql @@ -0,0 +1,26 @@ +drop table if exists gh_ost_test; +create table gh_ost_test ( + id int auto_increment, + i int not null, + e enum('red', 'green', 'blue', 'orange') null default null collate 'utf8_bin', + primary key(id) +) auto_increment=1; + +insert into gh_ost_test values (null, 7, 'red'); + +drop event if exists gh_ost_test; +delimiter ;; +create event gh_ost_test + on schedule every 1 second + starts current_timestamp + ends current_timestamp + interval 60 second + on completion not preserve + enable + do +begin + insert into gh_ost_test values (null, 11, 'red'); + insert into gh_ost_test values (null, 13, 'green'); + insert into gh_ost_test values (null, 17, 'blue'); + set @last_insert_id := last_insert_id(); + update gh_ost_test set e='orange' where id = @last_insert_id; +end ;; diff --git a/localtests/enum-to-varchar/extra_args b/localtests/enum-to-varchar/extra_args new file mode 100644 index 0000000..68524e4 --- /dev/null +++ b/localtests/enum-to-varchar/extra_args @@ -0,0 +1 @@ +--alter="change e e varchar(32) not null default ''"