From e9cb4a233f74c07f1c6f70cf7dece173ddf996cd Mon Sep 17 00:00:00 2001 From: wangzihuacool Date: Sat, 27 Aug 2022 01:55:36 +0000 Subject: [PATCH] convert to bytes if character string without charsetConversion. --- go/logic/inspect.go | 3 +++ go/sql/builder_test.go | 32 ++++++++++++++++---------------- go/sql/types.go | 19 ++++++++++++++++++- 3 files changed, 37 insertions(+), 17 deletions(-) diff --git a/go/logic/inspect.go b/go/logic/inspect.go index a562102..83b778a 100644 --- a/go/logic/inspect.go +++ b/go/logic/inspect.go @@ -192,6 +192,9 @@ func (this *Inspector) inspectOriginalAndGhostTables() (err error) { this.migrationContext.MappedSharedColumns.SetEnumToTextConversion(column.Name) this.migrationContext.MappedSharedColumns.SetEnumValues(column.Name, column.EnumValues) } + if column.Name == mappedColumn.Name && column.Charset != mappedColumn.Charset { + this.migrationContext.SharedColumns.SetCharsetConversion(column.Name, column.Charset, mappedColumn.Charset) + } } for _, column := range this.migrationContext.UniqueKey.Columns.Columns() { diff --git a/go/sql/builder_test.go b/go/sql/builder_test.go index 5ffd599..2998242 100644 --- a/go/sql/builder_test.go +++ b/go/sql/builder_test.go @@ -371,7 +371,7 @@ func TestBuildDMLDeleteQuery(t *testing.T) { ((name = ?) and (position = ?)) ` test.S(t).ExpectEquals(normalizeQuery(query), normalizeQuery(expected)) - test.S(t).ExpectTrue(reflect.DeepEqual(uniqueKeyArgs, []interface{}{[]byte("testname"), 17})) + test.S(t).ExpectTrue(reflect.DeepEqual(uniqueKeyArgs, []interface{}{"testname", 17})) } { uniqueKeyColumns := NewColumnList([]string{"position", "name"}) @@ -386,7 +386,7 @@ func TestBuildDMLDeleteQuery(t *testing.T) { ((position = ?) and (name = ?)) ` test.S(t).ExpectEquals(normalizeQuery(query), normalizeQuery(expected)) - test.S(t).ExpectTrue(reflect.DeepEqual(uniqueKeyArgs, []interface{}{17, []byte("testname")})) + test.S(t).ExpectTrue(reflect.DeepEqual(uniqueKeyArgs, []interface{}{17, "testname"})) } { uniqueKeyColumns := NewColumnList([]string{"position", "name"}) @@ -452,7 +452,7 @@ func TestBuildDMLInsertQuery(t *testing.T) { (?, ?, ?, ?) ` test.S(t).ExpectEquals(normalizeQuery(query), normalizeQuery(expected)) - test.S(t).ExpectTrue(reflect.DeepEqual(sharedArgs, []interface{}{3, []byte("testname"), 17, 23})) + test.S(t).ExpectTrue(reflect.DeepEqual(sharedArgs, []interface{}{3, "testname", 17, 23})) } { sharedColumns := NewColumnList([]string{"position", "name", "age", "id"}) @@ -466,7 +466,7 @@ func TestBuildDMLInsertQuery(t *testing.T) { (?, ?, ?, ?) ` test.S(t).ExpectEquals(normalizeQuery(query), normalizeQuery(expected)) - test.S(t).ExpectTrue(reflect.DeepEqual(sharedArgs, []interface{}{17, []byte("testname"), 23, 3})) + test.S(t).ExpectTrue(reflect.DeepEqual(sharedArgs, []interface{}{17, "testname", 23, 3})) } { sharedColumns := NewColumnList([]string{"position", "name", "surprise", "id"}) @@ -499,7 +499,7 @@ func TestBuildDMLInsertQuerySignedUnsigned(t *testing.T) { (?, ?, ?, ?) ` test.S(t).ExpectEquals(normalizeQuery(query), normalizeQuery(expected)) - test.S(t).ExpectTrue(reflect.DeepEqual(sharedArgs, []interface{}{3, []byte("testname"), int8(-1), 23})) + test.S(t).ExpectTrue(reflect.DeepEqual(sharedArgs, []interface{}{3, "testname", int8(-1), 23})) } { // testing unsigned @@ -515,7 +515,7 @@ func TestBuildDMLInsertQuerySignedUnsigned(t *testing.T) { (?, ?, ?, ?) ` test.S(t).ExpectEquals(normalizeQuery(query), normalizeQuery(expected)) - test.S(t).ExpectTrue(reflect.DeepEqual(sharedArgs, []interface{}{3, []byte("testname"), uint8(255), 23})) + test.S(t).ExpectTrue(reflect.DeepEqual(sharedArgs, []interface{}{3, "testname", uint8(255), 23})) } { // testing unsigned @@ -531,7 +531,7 @@ func TestBuildDMLInsertQuerySignedUnsigned(t *testing.T) { (?, ?, ?, ?) ` test.S(t).ExpectEquals(normalizeQuery(query), normalizeQuery(expected)) - test.S(t).ExpectTrue(reflect.DeepEqual(sharedArgs, []interface{}{3, []byte("testname"), uint32(4294967295), 23})) + test.S(t).ExpectTrue(reflect.DeepEqual(sharedArgs, []interface{}{3, "testname", uint32(4294967295), 23})) } } @@ -554,7 +554,7 @@ func TestBuildDMLUpdateQuery(t *testing.T) { ((position = ?)) ` test.S(t).ExpectEquals(normalizeQuery(query), normalizeQuery(expected)) - test.S(t).ExpectTrue(reflect.DeepEqual(sharedArgs, []interface{}{3, []byte("testname"), 17, 23})) + test.S(t).ExpectTrue(reflect.DeepEqual(sharedArgs, []interface{}{3, "testname", 17, 23})) test.S(t).ExpectTrue(reflect.DeepEqual(uniqueKeyArgs, []interface{}{17})) } { @@ -570,8 +570,8 @@ func TestBuildDMLUpdateQuery(t *testing.T) { ((position = ?) and (name = ?)) ` test.S(t).ExpectEquals(normalizeQuery(query), normalizeQuery(expected)) - test.S(t).ExpectTrue(reflect.DeepEqual(sharedArgs, []interface{}{3, []byte("testname"), 17, 23})) - test.S(t).ExpectTrue(reflect.DeepEqual(uniqueKeyArgs, []interface{}{17, []byte("testname")})) + test.S(t).ExpectTrue(reflect.DeepEqual(sharedArgs, []interface{}{3, "testname", 17, 23})) + test.S(t).ExpectTrue(reflect.DeepEqual(uniqueKeyArgs, []interface{}{17, "testname"})) } { sharedColumns := NewColumnList([]string{"id", "name", "position", "age"}) @@ -586,7 +586,7 @@ func TestBuildDMLUpdateQuery(t *testing.T) { ((age = ?)) ` test.S(t).ExpectEquals(normalizeQuery(query), normalizeQuery(expected)) - test.S(t).ExpectTrue(reflect.DeepEqual(sharedArgs, []interface{}{3, []byte("testname"), 17, 23})) + test.S(t).ExpectTrue(reflect.DeepEqual(sharedArgs, []interface{}{3, "testname", 17, 23})) test.S(t).ExpectTrue(reflect.DeepEqual(uniqueKeyArgs, []interface{}{56})) } { @@ -602,8 +602,8 @@ func TestBuildDMLUpdateQuery(t *testing.T) { ((age = ?) and (position = ?) and (id = ?) and (name = ?)) ` test.S(t).ExpectEquals(normalizeQuery(query), normalizeQuery(expected)) - test.S(t).ExpectTrue(reflect.DeepEqual(sharedArgs, []interface{}{3, []byte("testname"), 17, 23})) - test.S(t).ExpectTrue(reflect.DeepEqual(uniqueKeyArgs, []interface{}{56, 17, 3, []byte("testname")})) + test.S(t).ExpectTrue(reflect.DeepEqual(sharedArgs, []interface{}{3, "testname", 17, 23})) + test.S(t).ExpectTrue(reflect.DeepEqual(uniqueKeyArgs, []interface{}{56, 17, 3, "testname"})) } { sharedColumns := NewColumnList([]string{"id", "name", "position", "age"}) @@ -631,7 +631,7 @@ func TestBuildDMLUpdateQuery(t *testing.T) { ((id = ?)) ` test.S(t).ExpectEquals(normalizeQuery(query), normalizeQuery(expected)) - test.S(t).ExpectTrue(reflect.DeepEqual(sharedArgs, []interface{}{3, []byte("testname"), 17, 23})) + test.S(t).ExpectTrue(reflect.DeepEqual(sharedArgs, []interface{}{3, "testname", 17, 23})) test.S(t).ExpectTrue(reflect.DeepEqual(uniqueKeyArgs, []interface{}{3})) } } @@ -656,7 +656,7 @@ func TestBuildDMLUpdateQuerySignedUnsigned(t *testing.T) { ((position = ?)) ` test.S(t).ExpectEquals(normalizeQuery(query), normalizeQuery(expected)) - test.S(t).ExpectTrue(reflect.DeepEqual(sharedArgs, []interface{}{3, []byte("testname"), int8(-17), int8(-2)})) + test.S(t).ExpectTrue(reflect.DeepEqual(sharedArgs, []interface{}{3, "testname", int8(-17), int8(-2)})) test.S(t).ExpectTrue(reflect.DeepEqual(uniqueKeyArgs, []interface{}{int8(-3)})) } { @@ -673,7 +673,7 @@ func TestBuildDMLUpdateQuerySignedUnsigned(t *testing.T) { ((position = ?)) ` test.S(t).ExpectEquals(normalizeQuery(query), normalizeQuery(expected)) - test.S(t).ExpectTrue(reflect.DeepEqual(sharedArgs, []interface{}{3, []byte("testname"), int8(-17), uint8(254)})) + test.S(t).ExpectTrue(reflect.DeepEqual(sharedArgs, []interface{}{3, "testname", int8(-17), uint8(254)})) test.S(t).ExpectTrue(reflect.DeepEqual(uniqueKeyArgs, []interface{}{uint8(253)})) } } diff --git a/go/sql/types.go b/go/sql/types.go index 5411e47..3be1a44 100644 --- a/go/sql/types.go +++ b/go/sql/types.go @@ -32,6 +32,11 @@ type TimezoneConversion struct { ToTimezone string } +type CharacterSetConversion struct { + ToCharset string + FromCharset string +} + type Column struct { Name string IsUnsigned bool @@ -43,12 +48,20 @@ type Column struct { // add Octet length for binary type, fix bytes with suffix "00" get clipped in mysql binlog. // https://github.com/github/gh-ost/issues/909 BinaryOctetLength uint + charsetConversion *CharacterSetConversion } func (this *Column) convertArg(arg interface{}, isUniqueKeyColumn bool) interface{} { if s, ok := arg.(string); ok { arg2Bytes := []byte(s) - arg = arg2Bytes + // convert to bytes if character string without charsetConversion. + if this.Charset != "" && this.charsetConversion == nil { + arg = arg2Bytes + } else { + if encoding, ok := charsetEncodingMap[this.Charset]; ok { + arg, _ = encoding.NewDecoder().String(s) + } + } if this.Type == BinaryColumnType && isUniqueKeyColumn { size := len(arg2Bytes) @@ -235,6 +248,10 @@ func (this *ColumnList) Len() int { return len(this.columns) } +func (this *ColumnList) SetCharsetConversion(columnName string, fromCharset string, toCharset string) { + this.GetColumn(columnName).charsetConversion = &CharacterSetConversion{FromCharset: fromCharset, ToCharset: toCharset} +} + // UniqueKey is the combination of a key's name and columns type UniqueKey struct { Name string