octoleo/gh-ost
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

convert to bytes if character string without charsetConversion.

Browse Source
This commit is contained in:
wangzihuacool 2022-08-27 01:55:36 +00:00
parent a17389e97a
commit e9cb4a233f
3 changed files with 37 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
go/logic/inspect.go
View File

@ -192,6 +192,9 @@ func (this *Inspector) inspectOriginalAndGhostTables() (err error) {
this.migrationContext.MappedSharedColumns.SetEnumToTextConversion(column.Name) this.migrationContext.MappedSharedColumns.SetEnumToTextConversion(column.Name)
this.migrationContext.MappedSharedColumns.SetEnumValues(column.Name, column.EnumValues) this.migrationContext.MappedSharedColumns.SetEnumValues(column.Name, column.EnumValues)
} }
if column.Name == mappedColumn.Name && column.Charset != mappedColumn.Charset {
this.migrationContext.SharedColumns.SetCharsetConversion(column.Name, column.Charset, mappedColumn.Charset)
}
} }
for _, column := range this.migrationContext.UniqueKey.Columns.Columns() { for _, column := range this.migrationContext.UniqueKey.Columns.Columns() {

32
go/sql/builder_test.go
View File

@ -371,7 +371,7 @@ func TestBuildDMLDeleteQuery(t *testing.T) {
((name = ?) and (position = ?)) ((name = ?) and (position = ?))
` `
test.S(t).ExpectEquals(normalizeQuery(query), normalizeQuery(expected)) test.S(t).ExpectEquals(normalizeQuery(query), normalizeQuery(expected))
test.S(t).ExpectTrue(reflect.DeepEqual(uniqueKeyArgs, []interface{}{[]byte("testname"), 17})) test.S(t).ExpectTrue(reflect.DeepEqual(uniqueKeyArgs, []interface{}{"testname", 17}))
} }
{ {
uniqueKeyColumns := NewColumnList([]string{"position", "name"}) uniqueKeyColumns := NewColumnList([]string{"position", "name"})
@ -386,7 +386,7 @@ func TestBuildDMLDeleteQuery(t *testing.T) {
((position = ?) and (name = ?)) ((position = ?) and (name = ?))
` `
test.S(t).ExpectEquals(normalizeQuery(query), normalizeQuery(expected)) test.S(t).ExpectEquals(normalizeQuery(query), normalizeQuery(expected))
test.S(t).ExpectTrue(reflect.DeepEqual(uniqueKeyArgs, []interface{}{17, []byte("testname")})) test.S(t).ExpectTrue(reflect.DeepEqual(uniqueKeyArgs, []interface{}{17, "testname"}))
} }
{ {
uniqueKeyColumns := NewColumnList([]string{"position", "name"}) uniqueKeyColumns := NewColumnList([]string{"position", "name"})
@ -452,7 +452,7 @@ func TestBuildDMLInsertQuery(t *testing.T) {
(?, ?, ?, ?) (?, ?, ?, ?)
` `
test.S(t).ExpectEquals(normalizeQuery(query), normalizeQuery(expected)) test.S(t).ExpectEquals(normalizeQuery(query), normalizeQuery(expected))
test.S(t).ExpectTrue(reflect.DeepEqual(sharedArgs, []interface{}{3, []byte("testname"), 17, 23})) test.S(t).ExpectTrue(reflect.DeepEqual(sharedArgs, []interface{}{3, "testname", 17, 23}))
} }
{ {
sharedColumns := NewColumnList([]string{"position", "name", "age", "id"}) sharedColumns := NewColumnList([]string{"position", "name", "age", "id"})
@ -466,7 +466,7 @@ func TestBuildDMLInsertQuery(t *testing.T) {
(?, ?, ?, ?) (?, ?, ?, ?)
` `
test.S(t).ExpectEquals(normalizeQuery(query), normalizeQuery(expected)) test.S(t).ExpectEquals(normalizeQuery(query), normalizeQuery(expected))
test.S(t).ExpectTrue(reflect.DeepEqual(sharedArgs, []interface{}{17, []byte("testname"), 23, 3})) test.S(t).ExpectTrue(reflect.DeepEqual(sharedArgs, []interface{}{17, "testname", 23, 3}))
} }
{ {
sharedColumns := NewColumnList([]string{"position", "name", "surprise", "id"}) sharedColumns := NewColumnList([]string{"position", "name", "surprise", "id"})
@ -499,7 +499,7 @@ func TestBuildDMLInsertQuerySignedUnsigned(t *testing.T) {
(?, ?, ?, ?) (?, ?, ?, ?)
` `
test.S(t).ExpectEquals(normalizeQuery(query), normalizeQuery(expected)) test.S(t).ExpectEquals(normalizeQuery(query), normalizeQuery(expected))
test.S(t).ExpectTrue(reflect.DeepEqual(sharedArgs, []interface{}{3, []byte("testname"), int8(-1), 23})) test.S(t).ExpectTrue(reflect.DeepEqual(sharedArgs, []interface{}{3, "testname", int8(-1), 23}))
} }
{ {
// testing unsigned // testing unsigned
@ -515,7 +515,7 @@ func TestBuildDMLInsertQuerySignedUnsigned(t *testing.T) {
(?, ?, ?, ?) (?, ?, ?, ?)
` `
test.S(t).ExpectEquals(normalizeQuery(query), normalizeQuery(expected)) test.S(t).ExpectEquals(normalizeQuery(query), normalizeQuery(expected))
test.S(t).ExpectTrue(reflect.DeepEqual(sharedArgs, []interface{}{3, []byte("testname"), uint8(255), 23})) test.S(t).ExpectTrue(reflect.DeepEqual(sharedArgs, []interface{}{3, "testname", uint8(255), 23}))
} }
{ {
// testing unsigned // testing unsigned
@ -531,7 +531,7 @@ func TestBuildDMLInsertQuerySignedUnsigned(t *testing.T) {
(?, ?, ?, ?) (?, ?, ?, ?)
` `
test.S(t).ExpectEquals(normalizeQuery(query), normalizeQuery(expected)) test.S(t).ExpectEquals(normalizeQuery(query), normalizeQuery(expected))
test.S(t).ExpectTrue(reflect.DeepEqual(sharedArgs, []interface{}{3, []byte("testname"), uint32(4294967295), 23})) test.S(t).ExpectTrue(reflect.DeepEqual(sharedArgs, []interface{}{3, "testname", uint32(4294967295), 23}))
} }
} }
@ -554,7 +554,7 @@ func TestBuildDMLUpdateQuery(t *testing.T) {
((position = ?)) ((position = ?))
` `
test.S(t).ExpectEquals(normalizeQuery(query), normalizeQuery(expected)) test.S(t).ExpectEquals(normalizeQuery(query), normalizeQuery(expected))
test.S(t).ExpectTrue(reflect.DeepEqual(sharedArgs, []interface{}{3, []byte("testname"), 17, 23})) test.S(t).ExpectTrue(reflect.DeepEqual(sharedArgs, []interface{}{3, "testname", 17, 23}))
test.S(t).ExpectTrue(reflect.DeepEqual(uniqueKeyArgs, []interface{}{17})) test.S(t).ExpectTrue(reflect.DeepEqual(uniqueKeyArgs, []interface{}{17}))
} }
{ {
@ -570,8 +570,8 @@ func TestBuildDMLUpdateQuery(t *testing.T) {
((position = ?) and (name = ?)) ((position = ?) and (name = ?))
` `
test.S(t).ExpectEquals(normalizeQuery(query), normalizeQuery(expected)) test.S(t).ExpectEquals(normalizeQuery(query), normalizeQuery(expected))
test.S(t).ExpectTrue(reflect.DeepEqual(sharedArgs, []interface{}{3, []byte("testname"), 17, 23})) test.S(t).ExpectTrue(reflect.DeepEqual(sharedArgs, []interface{}{3, "testname", 17, 23}))
test.S(t).ExpectTrue(reflect.DeepEqual(uniqueKeyArgs, []interface{}{17, []byte("testname")})) test.S(t).ExpectTrue(reflect.DeepEqual(uniqueKeyArgs, []interface{}{17, "testname"}))
} }
{ {
sharedColumns := NewColumnList([]string{"id", "name", "position", "age"}) sharedColumns := NewColumnList([]string{"id", "name", "position", "age"})
@ -586,7 +586,7 @@ func TestBuildDMLUpdateQuery(t *testing.T) {
((age = ?)) ((age = ?))
` `
test.S(t).ExpectEquals(normalizeQuery(query), normalizeQuery(expected)) test.S(t).ExpectEquals(normalizeQuery(query), normalizeQuery(expected))
test.S(t).ExpectTrue(reflect.DeepEqual(sharedArgs, []interface{}{3, []byte("testname"), 17, 23})) test.S(t).ExpectTrue(reflect.DeepEqual(sharedArgs, []interface{}{3, "testname", 17, 23}))
test.S(t).ExpectTrue(reflect.DeepEqual(uniqueKeyArgs, []interface{}{56})) test.S(t).ExpectTrue(reflect.DeepEqual(uniqueKeyArgs, []interface{}{56}))
} }
{ {
@ -602,8 +602,8 @@ func TestBuildDMLUpdateQuery(t *testing.T) {
((age = ?) and (position = ?) and (id = ?) and (name = ?)) ((age = ?) and (position = ?) and (id = ?) and (name = ?))
` `
test.S(t).ExpectEquals(normalizeQuery(query), normalizeQuery(expected)) test.S(t).ExpectEquals(normalizeQuery(query), normalizeQuery(expected))
test.S(t).ExpectTrue(reflect.DeepEqual(sharedArgs, []interface{}{3, []byte("testname"), 17, 23})) test.S(t).ExpectTrue(reflect.DeepEqual(sharedArgs, []interface{}{3, "testname", 17, 23}))
test.S(t).ExpectTrue(reflect.DeepEqual(uniqueKeyArgs, []interface{}{56, 17, 3, []byte("testname")})) test.S(t).ExpectTrue(reflect.DeepEqual(uniqueKeyArgs, []interface{}{56, 17, 3, "testname"}))
} }
{ {
sharedColumns := NewColumnList([]string{"id", "name", "position", "age"}) sharedColumns := NewColumnList([]string{"id", "name", "position", "age"})
@ -631,7 +631,7 @@ func TestBuildDMLUpdateQuery(t *testing.T) {
((id = ?)) ((id = ?))
` `
test.S(t).ExpectEquals(normalizeQuery(query), normalizeQuery(expected)) test.S(t).ExpectEquals(normalizeQuery(query), normalizeQuery(expected))
test.S(t).ExpectTrue(reflect.DeepEqual(sharedArgs, []interface{}{3, []byte("testname"), 17, 23})) test.S(t).ExpectTrue(reflect.DeepEqual(sharedArgs, []interface{}{3, "testname", 17, 23}))
test.S(t).ExpectTrue(reflect.DeepEqual(uniqueKeyArgs, []interface{}{3})) test.S(t).ExpectTrue(reflect.DeepEqual(uniqueKeyArgs, []interface{}{3}))
} }
} }
@ -656,7 +656,7 @@ func TestBuildDMLUpdateQuerySignedUnsigned(t *testing.T) {
((position = ?)) ((position = ?))
` `
test.S(t).ExpectEquals(normalizeQuery(query), normalizeQuery(expected)) test.S(t).ExpectEquals(normalizeQuery(query), normalizeQuery(expected))
test.S(t).ExpectTrue(reflect.DeepEqual(sharedArgs, []interface{}{3, []byte("testname"), int8(-17), int8(-2)})) test.S(t).ExpectTrue(reflect.DeepEqual(sharedArgs, []interface{}{3, "testname", int8(-17), int8(-2)}))
test.S(t).ExpectTrue(reflect.DeepEqual(uniqueKeyArgs, []interface{}{int8(-3)})) test.S(t).ExpectTrue(reflect.DeepEqual(uniqueKeyArgs, []interface{}{int8(-3)}))
} }
{ {
@ -673,7 +673,7 @@ func TestBuildDMLUpdateQuerySignedUnsigned(t *testing.T) {
((position = ?)) ((position = ?))
` `
test.S(t).ExpectEquals(normalizeQuery(query), normalizeQuery(expected)) test.S(t).ExpectEquals(normalizeQuery(query), normalizeQuery(expected))
test.S(t).ExpectTrue(reflect.DeepEqual(sharedArgs, []interface{}{3, []byte("testname"), int8(-17), uint8(254)})) test.S(t).ExpectTrue(reflect.DeepEqual(sharedArgs, []interface{}{3, "testname", int8(-17), uint8(254)}))
test.S(t).ExpectTrue(reflect.DeepEqual(uniqueKeyArgs, []interface{}{uint8(253)})) test.S(t).ExpectTrue(reflect.DeepEqual(uniqueKeyArgs, []interface{}{uint8(253)}))
} }
} }

17
go/sql/types.go
View File

@ -32,6 +32,11 @@ type TimezoneConversion struct {
ToTimezone string ToTimezone string
} }
type CharacterSetConversion struct {
ToCharset string
FromCharset string
}
type Column struct { type Column struct {
Name string Name string
IsUnsigned bool IsUnsigned bool
@ -43,12 +48,20 @@ type Column struct {
// add Octet length for binary type, fix bytes with suffix "00" get clipped in mysql binlog. // add Octet length for binary type, fix bytes with suffix "00" get clipped in mysql binlog.
// https://github.com/github/gh-ost/issues/909 // https://github.com/github/gh-ost/issues/909
BinaryOctetLength uint BinaryOctetLength uint
charsetConversion *CharacterSetConversion
} }
func (this *Column) convertArg(arg interface{}, isUniqueKeyColumn bool) interface{} { func (this *Column) convertArg(arg interface{}, isUniqueKeyColumn bool) interface{} {
if s, ok := arg.(string); ok { if s, ok := arg.(string); ok {
arg2Bytes := []byte(s) arg2Bytes := []byte(s)
// convert to bytes if character string without charsetConversion.
if this.Charset != "" && this.charsetConversion == nil {
arg = arg2Bytes arg = arg2Bytes
} else {
if encoding, ok := charsetEncodingMap[this.Charset]; ok {
arg, _ = encoding.NewDecoder().String(s)
}
}
if this.Type == BinaryColumnType && isUniqueKeyColumn { if this.Type == BinaryColumnType && isUniqueKeyColumn {
size := len(arg2Bytes) size := len(arg2Bytes)
@ -235,6 +248,10 @@ func (this *ColumnList) Len() int {
return len(this.columns) return len(this.columns)
} }
func (this *ColumnList) SetCharsetConversion(columnName string, fromCharset string, toCharset string) {
this.GetColumn(columnName).charsetConversion = &CharacterSetConversion{FromCharset: fromCharset, ToCharset: toCharset}
}
// UniqueKey is the combination of a key's name and columns // UniqueKey is the combination of a key's name and columns
type UniqueKey struct { type UniqueKey struct {
Name string Name string