gh-ost/go/sql/types.go

311 lines
7.5 KiB
Go
Raw Normal View History

/*
Copyright 2016 GitHub Inc.
2016-05-16 09:09:17 +00:00
See https://github.com/github/gh-ost/blob/master/LICENSE
*/
package sql
import (
2021-01-14 06:51:14 +00:00
"bytes"
"fmt"
"reflect"
"strconv"
"strings"
)
type ColumnType int
const (
2018-05-03 16:45:14 +00:00
UnknownColumnType ColumnType = iota
TimestampColumnType
DateTimeColumnType
EnumColumnType
MediumIntColumnType
JSONColumnType
FloatColumnType
2021-01-13 09:21:21 +00:00
BinaryColumnType
)
2017-01-10 07:57:42 +00:00
const maxMediumintUnsigned int32 = 16777215
2017-11-08 00:45:31 +00:00
type TimezoneConversion struct {
ToTimezone string
}
type Column struct {
Enum to varchar (#963) * v1.1.0 * WIP: copying AUTO_INCREMENT value to ghost table Initial commit: towards setting up a test suite Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * greping for 'expect_table_structure' content * Adding simple test for 'expect_table_structure' scenario * adding tests for AUTO_INCREMENT value after row deletes. Should initially fail * clear event beforehand * parsing AUTO_INCREMENT from alter query, reading AUTO_INCREMENT from original table, applying AUTO_INCREMENT value onto ghost table if applicable and user has not specified AUTO_INCREMENT in alter statement * support GetUint64 Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * minor update to test Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * adding test for user defined AUTO_INCREMENT statement * Generated column as part of UNIQUE (or PRIMARY) KEY Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * skip analysis of generated column data type in unique key Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * All MySQL DBs limited to max 3 concurrent/idle connections Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * hooks: reporting GH_OST_ETA_SECONDS. ETA stored as part of migration context Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * GH_OST_ETA_NANOSECONDS Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * N/A denoted by negative value Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * ETAUnknown constant Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * Convering enum to varchar Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * test: not null Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * first attempt at setting enum-to-string right Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * fix insert query Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * store enum values, use when populating Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * apply EnumValues to mapped column Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * fix compilation error Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * gofmt Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com>
2021-06-10 15:17:49 +00:00
Name string
IsUnsigned bool
Charset string
Type ColumnType
EnumValues string
timezoneConversion *TimezoneConversion
enumToTextConversion bool
2021-01-13 09:21:21 +00:00
// add Octet length for binary type, fix bytes with suffix "00" get clipped in mysql binlog.
// https://github.com/github/gh-ost/issues/909
Enum to varchar (#963) * v1.1.0 * WIP: copying AUTO_INCREMENT value to ghost table Initial commit: towards setting up a test suite Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * greping for 'expect_table_structure' content * Adding simple test for 'expect_table_structure' scenario * adding tests for AUTO_INCREMENT value after row deletes. Should initially fail * clear event beforehand * parsing AUTO_INCREMENT from alter query, reading AUTO_INCREMENT from original table, applying AUTO_INCREMENT value onto ghost table if applicable and user has not specified AUTO_INCREMENT in alter statement * support GetUint64 Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * minor update to test Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * adding test for user defined AUTO_INCREMENT statement * Generated column as part of UNIQUE (or PRIMARY) KEY Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * skip analysis of generated column data type in unique key Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * All MySQL DBs limited to max 3 concurrent/idle connections Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * hooks: reporting GH_OST_ETA_SECONDS. ETA stored as part of migration context Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * GH_OST_ETA_NANOSECONDS Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * N/A denoted by negative value Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * ETAUnknown constant Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * Convering enum to varchar Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * test: not null Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * first attempt at setting enum-to-string right Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * fix insert query Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * store enum values, use when populating Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * apply EnumValues to mapped column Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * fix compilation error Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * gofmt Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com>
2021-06-10 15:17:49 +00:00
BinaryOctetLength uint
}
2021-02-26 06:23:24 +00:00
func (this *Column) convertArg(arg interface{}, isUniqueKeyColumn bool) interface{} {
if s, ok := arg.(string); ok {
arg2Bytes := []byte(s)
arg = arg2Bytes
2021-01-13 09:21:21 +00:00
2021-02-26 06:23:24 +00:00
if this.Type == BinaryColumnType && isUniqueKeyColumn {
2021-02-24 09:18:21 +00:00
size := len(arg2Bytes)
2021-01-13 09:21:21 +00:00
if uint(size) < this.BinaryOctetLength {
2021-02-24 09:18:21 +00:00
buf := bytes.NewBuffer(arg2Bytes)
2021-01-13 09:21:21 +00:00
for i := uint(0); i < (this.BinaryOctetLength - uint(size)); i++ {
buf.Write([]byte{0})
}
arg = buf.String()
}
}
return arg
}
if this.IsUnsigned {
if i, ok := arg.(int8); ok {
return uint8(i)
}
if i, ok := arg.(int16); ok {
return uint16(i)
}
if i, ok := arg.(int32); ok {
2017-01-10 07:57:42 +00:00
if this.Type == MediumIntColumnType {
// problem with mediumint is that it's a 3-byte type. There is no compatible golang type to match that.
// So to convert from negative to positive we'd need to convert the value manually
if i >= 0 {
return i
}
return uint32(maxMediumintUnsigned + i + 1)
}
return uint32(i)
}
if i, ok := arg.(int64); ok {
return strconv.FormatUint(uint64(i), 10)
}
if i, ok := arg.(int); ok {
return uint(i)
}
}
return arg
}
func NewColumns(names []string) []Column {
result := make([]Column, len(names))
for i := range names {
result[i].Name = names[i]
}
return result
}
func ParseColumns(names string) []Column {
namesArray := strings.Split(names, ",")
return NewColumns(namesArray)
}
// ColumnsMap maps a column name onto its ordinal position
type ColumnsMap map[string]int
2016-08-17 04:49:20 +00:00
func NewEmptyColumnsMap() ColumnsMap {
columnsMap := make(map[string]int)
2016-08-17 04:49:20 +00:00
return ColumnsMap(columnsMap)
}
func NewColumnsMap(orderedColumns []Column) ColumnsMap {
2016-08-17 04:49:20 +00:00
columnsMap := NewEmptyColumnsMap()
for i, column := range orderedColumns {
columnsMap[column.Name] = i
}
2016-08-17 04:49:20 +00:00
return columnsMap
}
// ColumnList makes for a named list of columns
type ColumnList struct {
columns []Column
Ordinals ColumnsMap
}
// NewColumnList creates an object given ordered list of column names
func NewColumnList(names []string) *ColumnList {
result := &ColumnList{
columns: NewColumns(names),
}
result.Ordinals = NewColumnsMap(result.columns)
return result
}
// ParseColumnList parses a comma delimited list of column names
func ParseColumnList(names string) *ColumnList {
result := &ColumnList{
columns: ParseColumns(names),
}
result.Ordinals = NewColumnsMap(result.columns)
return result
}
func (this *ColumnList) Columns() []Column {
return this.columns
}
func (this *ColumnList) Names() []string {
names := make([]string, len(this.columns))
for i := range this.columns {
names[i] = this.columns[i].Name
}
return names
}
func (this *ColumnList) GetColumn(columnName string) *Column {
if ordinal, ok := this.Ordinals[columnName]; ok {
return &this.columns[ordinal]
}
return nil
}
2016-08-17 04:49:20 +00:00
func (this *ColumnList) SetUnsigned(columnName string) {
this.GetColumn(columnName).IsUnsigned = true
2016-08-17 04:49:20 +00:00
}
func (this *ColumnList) IsUnsigned(columnName string) bool {
return this.GetColumn(columnName).IsUnsigned
}
func (this *ColumnList) SetCharset(columnName string, charset string) {
this.GetColumn(columnName).Charset = charset
}
func (this *ColumnList) GetCharset(columnName string) string {
return this.GetColumn(columnName).Charset
}
func (this *ColumnList) SetColumnType(columnName string, columnType ColumnType) {
this.GetColumn(columnName).Type = columnType
}
func (this *ColumnList) GetColumnType(columnName string) ColumnType {
return this.GetColumn(columnName).Type
}
func (this *ColumnList) SetConvertDatetimeToTimestamp(columnName string, toTimezone string) {
2017-11-08 00:45:31 +00:00
this.GetColumn(columnName).timezoneConversion = &TimezoneConversion{ToTimezone: toTimezone}
2016-08-17 04:49:20 +00:00
}
func (this *ColumnList) HasTimezoneConversion(columnName string) bool {
return this.GetColumn(columnName).timezoneConversion != nil
}
Enum to varchar (#963) * v1.1.0 * WIP: copying AUTO_INCREMENT value to ghost table Initial commit: towards setting up a test suite Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * greping for 'expect_table_structure' content * Adding simple test for 'expect_table_structure' scenario * adding tests for AUTO_INCREMENT value after row deletes. Should initially fail * clear event beforehand * parsing AUTO_INCREMENT from alter query, reading AUTO_INCREMENT from original table, applying AUTO_INCREMENT value onto ghost table if applicable and user has not specified AUTO_INCREMENT in alter statement * support GetUint64 Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * minor update to test Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * adding test for user defined AUTO_INCREMENT statement * Generated column as part of UNIQUE (or PRIMARY) KEY Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * skip analysis of generated column data type in unique key Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * All MySQL DBs limited to max 3 concurrent/idle connections Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * hooks: reporting GH_OST_ETA_SECONDS. ETA stored as part of migration context Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * GH_OST_ETA_NANOSECONDS Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * N/A denoted by negative value Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * ETAUnknown constant Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * Convering enum to varchar Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * test: not null Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * first attempt at setting enum-to-string right Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * fix insert query Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * store enum values, use when populating Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * apply EnumValues to mapped column Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * fix compilation error Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * gofmt Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com>
2021-06-10 15:17:49 +00:00
func (this *ColumnList) SetEnumToTextConversion(columnName string) {
this.GetColumn(columnName).enumToTextConversion = true
}
func (this *ColumnList) IsEnumToTextConversion(columnName string) bool {
return this.GetColumn(columnName).enumToTextConversion
}
func (this *ColumnList) SetEnumValues(columnName string, enumValues string) {
this.GetColumn(columnName).EnumValues = enumValues
}
func (this *ColumnList) String() string {
return strings.Join(this.Names(), ",")
}
func (this *ColumnList) Equals(other *ColumnList) bool {
return reflect.DeepEqual(this.Columns, other.Columns)
}
func (this *ColumnList) EqualsByNames(other *ColumnList) bool {
return reflect.DeepEqual(this.Names(), other.Names())
}
// IsSubsetOf returns 'true' when column names of this list are a subset of
// another list, in arbitrary order (order agnostic)
func (this *ColumnList) IsSubsetOf(other *ColumnList) bool {
for _, column := range this.columns {
if _, exists := other.Ordinals[column.Name]; !exists {
return false
}
}
return true
}
func (this *ColumnList) Len() int {
return len(this.columns)
}
// UniqueKey is the combination of a key's name and columns
type UniqueKey struct {
Name string
Columns ColumnList
HasNullable bool
IsAutoIncrement bool
}
// IsPrimary checks if this unique key is primary
func (this *UniqueKey) IsPrimary() bool {
return this.Name == "PRIMARY"
}
func (this *UniqueKey) Len() int {
return this.Columns.Len()
}
func (this *UniqueKey) String() string {
description := this.Name
if this.IsAutoIncrement {
description = fmt.Sprintf("%s (auto_increment)", description)
}
return fmt.Sprintf("%s: %s; has nullable: %+v", description, this.Columns.Names(), this.HasNullable)
}
type ColumnValues struct {
abstractValues []interface{}
ValuesPointers []interface{}
}
func NewColumnValues(length int) *ColumnValues {
result := &ColumnValues{
abstractValues: make([]interface{}, length),
ValuesPointers: make([]interface{}, length),
}
for i := 0; i < length; i++ {
result.ValuesPointers[i] = &result.abstractValues[i]
}
return result
}
func ToColumnValues(abstractValues []interface{}) *ColumnValues {
result := &ColumnValues{
abstractValues: abstractValues,
ValuesPointers: make([]interface{}, len(abstractValues)),
}
for i := 0; i < len(abstractValues); i++ {
result.ValuesPointers[i] = &result.abstractValues[i]
}
return result
}
func (this *ColumnValues) AbstractValues() []interface{} {
return this.abstractValues
}
func (this *ColumnValues) StringColumn(index int) string {
val := this.AbstractValues()[index]
if ints, ok := val.([]uint8); ok {
return string(ints)
}
return fmt.Sprintf("%+v", val)
}
func (this *ColumnValues) String() string {
stringValues := []string{}
for i := range this.AbstractValues() {
stringValues = append(stringValues, this.StringColumn(i))
}
return strings.Join(stringValues, ",")
}