From c1bfe94b0fa744ee96c973f77211fa32ce60c6d7 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Sun, 2 May 2021 18:12:35 +0300 Subject: [PATCH 1/7] Convering enum to varchar Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- localtests/enum-to-varchar/create.sql | 29 +++++++++++++++++++++++++++ localtests/enum-to-varchar/extra_args | 1 + 2 files changed, 30 insertions(+) create mode 100644 localtests/enum-to-varchar/create.sql create mode 100644 localtests/enum-to-varchar/extra_args diff --git a/localtests/enum-to-varchar/create.sql b/localtests/enum-to-varchar/create.sql new file mode 100644 index 0000000..8b25e39 --- /dev/null +++ b/localtests/enum-to-varchar/create.sql @@ -0,0 +1,29 @@ +drop table if exists gh_ost_test; +create table gh_ost_test ( + id int auto_increment, + i int not null, + e enum('red', 'green', 'blue', 'orange') null default null collate 'utf8_bin', + primary key(id) +) auto_increment=1; + +insert into gh_ost_test values (null, 7, 'red'); + +drop event if exists gh_ost_test; +delimiter ;; +create event gh_ost_test + on schedule every 1 second + starts current_timestamp + ends current_timestamp + interval 60 second + on completion not preserve + enable + do +begin + insert into gh_ost_test values (null, 11, 'red'); + insert into gh_ost_test values (null, 13, 'green'); + insert into gh_ost_test values (null, 17, 'blue'); + set @last_insert_id := last_insert_id(); + update gh_ost_test set e='orange' where id = @last_insert_id; + insert into gh_ost_test values (null, 23, null); + set @last_insert_id := last_insert_id(); + update gh_ost_test set i=i+1, e=null where id = @last_insert_id; +end ;; diff --git a/localtests/enum-to-varchar/extra_args b/localtests/enum-to-varchar/extra_args new file mode 100644 index 0000000..68524e4 --- /dev/null +++ b/localtests/enum-to-varchar/extra_args @@ -0,0 +1 @@ +--alter="change e e varchar(32) not null default ''" From 9bb2daaf158a67054addd1e69f609754ad004390 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Sun, 2 May 2021 18:35:52 +0300 Subject: [PATCH 2/7] test: not null Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- localtests/enum-to-varchar/create.sql | 3 --- 1 file changed, 3 deletions(-) diff --git a/localtests/enum-to-varchar/create.sql b/localtests/enum-to-varchar/create.sql index 8b25e39..0dbab17 100644 --- a/localtests/enum-to-varchar/create.sql +++ b/localtests/enum-to-varchar/create.sql @@ -23,7 +23,4 @@ begin insert into gh_ost_test values (null, 17, 'blue'); set @last_insert_id := last_insert_id(); update gh_ost_test set e='orange' where id = @last_insert_id; - insert into gh_ost_test values (null, 23, null); - set @last_insert_id := last_insert_id(); - update gh_ost_test set i=i+1, e=null where id = @last_insert_id; end ;; From 939b898ea9cec4964d0d03a27eeed977982cd05c Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Sun, 2 May 2021 19:28:19 +0300 Subject: [PATCH 3/7] first attempt at setting enum-to-string right Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/logic/inspect.go | 3 +++ go/sql/builder.go | 2 ++ go/sql/types.go | 19 ++++++++++++++----- 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/go/logic/inspect.go b/go/logic/inspect.go index e61f97b..4ad0131 100644 --- a/go/logic/inspect.go +++ b/go/logic/inspect.go @@ -187,6 +187,9 @@ func (this *Inspector) inspectOriginalAndGhostTables() (err error) { if column.Name == mappedColumn.Name && column.Type == sql.DateTimeColumnType && mappedColumn.Type == sql.TimestampColumnType { this.migrationContext.MappedSharedColumns.SetConvertDatetimeToTimestamp(column.Name, this.migrationContext.ApplierTimeZone) } + if column.Name == mappedColumn.Name && column.Type == sql.EnumColumnType && mappedColumn.Charset != "" { + this.migrationContext.MappedSharedColumns.SetEnumToTextConversion(column.Name) + } } for _, column := range this.migrationContext.UniqueKey.Columns.Columns() { diff --git a/go/sql/builder.go b/go/sql/builder.go index 4b019bc..4b970e9 100644 --- a/go/sql/builder.go +++ b/go/sql/builder.go @@ -108,6 +108,8 @@ func BuildSetPreparedClause(columns *ColumnList) (result string, err error) { var setToken string if column.timezoneConversion != nil { setToken = fmt.Sprintf("%s=convert_tz(?, '%s', '%s')", EscapeName(column.Name), column.timezoneConversion.ToTimezone, "+00:00") + } else if column.enumToTextConversion { + setToken = fmt.Sprintf("%s=concat('', ?)", EscapeName(column.Name)) } else if column.Type == JSONColumnType { setToken = fmt.Sprintf("%s=convert(? using utf8mb4)", EscapeName(column.Name)) } else { diff --git a/go/sql/types.go b/go/sql/types.go index ef83819..30db646 100644 --- a/go/sql/types.go +++ b/go/sql/types.go @@ -31,11 +31,12 @@ type TimezoneConversion struct { } type Column struct { - Name string - IsUnsigned bool - Charset string - Type ColumnType - timezoneConversion *TimezoneConversion + Name string + IsUnsigned bool + Charset string + Type ColumnType + timezoneConversion *TimezoneConversion + enumToTextConversion bool } func (this *Column) convertArg(arg interface{}) interface{} { @@ -179,6 +180,14 @@ func (this *ColumnList) HasTimezoneConversion(columnName string) bool { return this.GetColumn(columnName).timezoneConversion != nil } +func (this *ColumnList) SetEnumToTextConversion(columnName string) { + this.GetColumn(columnName).enumToTextConversion = true +} + +func (this *ColumnList) IsEnumToTextConversion(columnName string) bool { + return this.GetColumn(columnName).enumToTextConversion +} + func (this *ColumnList) String() string { return strings.Join(this.Names(), ",") } From 95ee9e2144ff97b75638822590d24497205f0da7 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Sun, 2 May 2021 19:37:25 +0300 Subject: [PATCH 4/7] fix insert query Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/sql/builder.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/go/sql/builder.go b/go/sql/builder.go index 4b970e9..4618ef1 100644 --- a/go/sql/builder.go +++ b/go/sql/builder.go @@ -38,6 +38,8 @@ func buildColumnsPreparedValues(columns *ColumnList) []string { var token string if column.timezoneConversion != nil { token = fmt.Sprintf("convert_tz(?, '%s', '%s')", column.timezoneConversion.ToTimezone, "+00:00") + } else if column.enumToTextConversion { + token = "concat('', ?)" } else if column.Type == JSONColumnType { token = "convert(? using utf8mb4)" } else { From e80ddb42c9c488eac0c950711b0f6e4cb4cd2215 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Sun, 2 May 2021 20:44:39 +0300 Subject: [PATCH 5/7] store enum values, use when populating Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/logic/inspect.go | 1 + go/sql/builder.go | 4 ++-- go/sql/parser.go | 8 ++++++++ go/sql/parser_test.go | 18 ++++++++++++++++++ go/sql/types.go | 1 + 5 files changed, 30 insertions(+), 2 deletions(-) diff --git a/go/logic/inspect.go b/go/logic/inspect.go index 4ad0131..7b8002d 100644 --- a/go/logic/inspect.go +++ b/go/logic/inspect.go @@ -592,6 +592,7 @@ func (this *Inspector) applyColumnTypes(databaseName, tableName string, columnsL } if strings.HasPrefix(columnType, "enum") { column.Type = sql.EnumColumnType + column.EnumValues = sql.ParseEnumValues(m.GetString("COLUMN_TYPE")) } if charset := m.GetString("CHARACTER_SET_NAME"); charset != "" { column.Charset = charset diff --git a/go/sql/builder.go b/go/sql/builder.go index 4618ef1..878850c 100644 --- a/go/sql/builder.go +++ b/go/sql/builder.go @@ -39,7 +39,7 @@ func buildColumnsPreparedValues(columns *ColumnList) []string { if column.timezoneConversion != nil { token = fmt.Sprintf("convert_tz(?, '%s', '%s')", column.timezoneConversion.ToTimezone, "+00:00") } else if column.enumToTextConversion { - token = "concat('', ?)" + token = fmt.Sprintf("ELT(?, %s)", column.EnumValues) } else if column.Type == JSONColumnType { token = "convert(? using utf8mb4)" } else { @@ -111,7 +111,7 @@ func BuildSetPreparedClause(columns *ColumnList) (result string, err error) { if column.timezoneConversion != nil { setToken = fmt.Sprintf("%s=convert_tz(?, '%s', '%s')", EscapeName(column.Name), column.timezoneConversion.ToTimezone, "+00:00") } else if column.enumToTextConversion { - setToken = fmt.Sprintf("%s=concat('', ?)", EscapeName(column.Name)) + setToken = fmt.Sprintf("%s=ELT(?, %s)", EscapeName(column.Name), column.EnumValues) } else if column.Type == JSONColumnType { setToken = fmt.Sprintf("%s=convert(? using utf8mb4)", EscapeName(column.Name)) } else { diff --git a/go/sql/parser.go b/go/sql/parser.go index d9c0c3f..eac0bdc 100644 --- a/go/sql/parser.go +++ b/go/sql/parser.go @@ -33,6 +33,7 @@ var ( // ALTER TABLE tbl something regexp.MustCompile(`(?i)\balter\s+table\s+([\S]+)\s+(.*$)`), } + enumValuesRegexp = regexp.MustCompile("^enum[(](.*)[)]$") ) type AlterTableParser struct { @@ -205,3 +206,10 @@ func (this *AlterTableParser) HasExplicitTable() bool { func (this *AlterTableParser) GetAlterStatementOptions() string { return this.alterStatementOptions } + +func ParseEnumValues(enumColumnType string) string { + if submatch := enumValuesRegexp.FindStringSubmatch(enumColumnType); len(submatch) > 0 { + return submatch[1] + } + return enumColumnType +} diff --git a/go/sql/parser_test.go b/go/sql/parser_test.go index 6cdbb39..3157d09 100644 --- a/go/sql/parser_test.go +++ b/go/sql/parser_test.go @@ -322,3 +322,21 @@ func TestParseAlterStatementExplicitTable(t *testing.T) { test.S(t).ExpectTrue(reflect.DeepEqual(parser.alterTokens, []string{"drop column b", "add index idx(i)"})) } } + +func TestParseEnumValues(t *testing.T) { + { + s := "enum('red','green','blue','orange')" + values := ParseEnumValues(s) + test.S(t).ExpectEquals(values, "'red','green','blue','orange'") + } + { + s := "('red','green','blue','orange')" + values := ParseEnumValues(s) + test.S(t).ExpectEquals(values, "('red','green','blue','orange')") + } + { + s := "zzz" + values := ParseEnumValues(s) + test.S(t).ExpectEquals(values, "zzz") + } +} diff --git a/go/sql/types.go b/go/sql/types.go index 30db646..44e9725 100644 --- a/go/sql/types.go +++ b/go/sql/types.go @@ -35,6 +35,7 @@ type Column struct { IsUnsigned bool Charset string Type ColumnType + EnumValues string timezoneConversion *TimezoneConversion enumToTextConversion bool } From 6e5b665c557e2aa4c1456410a0f231022f950966 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Sun, 2 May 2021 20:56:02 +0300 Subject: [PATCH 6/7] apply EnumValues to mapped column Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/logic/inspect.go | 1 + go/sql/types.go | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/go/logic/inspect.go b/go/logic/inspect.go index 7b8002d..6c873c8 100644 --- a/go/logic/inspect.go +++ b/go/logic/inspect.go @@ -189,6 +189,7 @@ func (this *Inspector) inspectOriginalAndGhostTables() (err error) { } if column.Name == mappedColumn.Name && column.Type == sql.EnumColumnType && mappedColumn.Charset != "" { this.migrationContext.MappedSharedColumns.SetEnumToTextConversion(column.Name) + this.migrationContext.MappedSharedColumns.SetEnumValues(column.Name, column.EnumValues) } } diff --git a/go/sql/types.go b/go/sql/types.go index 44e9725..c4be371 100644 --- a/go/sql/types.go +++ b/go/sql/types.go @@ -189,6 +189,10 @@ func (this *ColumnList) IsEnumToTextConversion(columnName string) bool { return this.GetColumn(columnName).enumToTextConversion } +func (this *ColumnList) SetEnumValues(columnName string, enumValues string) { + this.GetColumn(columnName).EnumValues = enumValues + + func (this *ColumnList) String() string { return strings.Join(this.Names(), ",") } From 82bdf066e9edbeb9e0887ec15121fc4e27e50bc3 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Sun, 2 May 2021 20:59:50 +0300 Subject: [PATCH 7/7] fix compilation error Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/sql/types.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/sql/types.go b/go/sql/types.go index c4be371..c005316 100644 --- a/go/sql/types.go +++ b/go/sql/types.go @@ -191,7 +191,7 @@ func (this *ColumnList) IsEnumToTextConversion(columnName string) bool { func (this *ColumnList) SetEnumValues(columnName string, enumValues string) { this.GetColumn(columnName).EnumValues = enumValues - +} func (this *ColumnList) String() string { return strings.Join(this.Names(), ",")