diff --git a/doc/requirements-and-limitations.md b/doc/requirements-and-limitations.md index 3d174a7..99e3ee3 100644 --- a/doc/requirements-and-limitations.md +++ b/doc/requirements-and-limitations.md @@ -26,10 +26,6 @@ The `SUPER` privilege is required for `STOP SLAVE`, `START SLAVE` operations. Th - Triggers are not supported. They may be supported in the future. -- MySQL 5.7 generated columns are not supported. They may be supported in the future. - -- MySQL 5.7 `POINT` column type is not supported. - - MySQL 5.7 `JSON` columns are supported but not as part of `PRIMARY KEY` - The two _before_ & _after_ tables must share a `PRIMARY KEY` or other `UNIQUE KEY`. This key will be used by `gh-ost` to iterate through the table rows when copying. [Read more](shared-key.md) @@ -53,4 +49,4 @@ The `SUPER` privilege is required for `STOP SLAVE`, `START SLAVE` operations. Th - Migrating a `FEDERATED` table is unsupported and is irrelevant to the problem `gh-ost` tackles. -- `ALTER TABLE ... RENAME TO some_other_name` is not supported (and you shouldn't use `gh-ost` for such a trivial operation). \ No newline at end of file +- `ALTER TABLE ... RENAME TO some_other_name` is not supported (and you shouldn't use `gh-ost` for such a trivial operation). diff --git a/go/base/context.go b/go/base/context.go index 94039e3..2357c54 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -186,8 +186,10 @@ type MigrationContext struct { OriginalTableColumnsOnApplier *sql.ColumnList OriginalTableColumns *sql.ColumnList + OriginalTableVirtualColumns *sql.ColumnList OriginalTableUniqueKeys [](*sql.UniqueKey) GhostTableColumns *sql.ColumnList + GhostTableVirtualColumns *sql.ColumnList GhostTableUniqueKeys [](*sql.UniqueKey) UniqueKey *sql.UniqueKey SharedColumns *sql.ColumnList diff --git a/go/logic/applier.go b/go/logic/applier.go index afdc45f..ae59f62 100644 --- a/go/logic/applier.go +++ b/go/logic/applier.go @@ -117,7 +117,7 @@ func (this *Applier) validateAndReadTimeZone() error { // readTableColumns reads table columns on applier func (this *Applier) readTableColumns() (err error) { log.Infof("Examining table structure on applier") - this.migrationContext.OriginalTableColumnsOnApplier, err = mysql.GetTableColumns(this.db, this.migrationContext.DatabaseName, this.migrationContext.OriginalTableName) + this.migrationContext.OriginalTableColumnsOnApplier, _, err = mysql.GetTableColumns(this.db, this.migrationContext.DatabaseName, this.migrationContext.OriginalTableName) if err != nil { return err } diff --git a/go/logic/inspect.go b/go/logic/inspect.go index 2568afe..dc69b81 100644 --- a/go/logic/inspect.go +++ b/go/logic/inspect.go @@ -89,24 +89,24 @@ func (this *Inspector) ValidateOriginalTable() (err error) { return nil } -func (this *Inspector) InspectTableColumnsAndUniqueKeys(tableName string) (columns *sql.ColumnList, uniqueKeys [](*sql.UniqueKey), err error) { +func (this *Inspector) InspectTableColumnsAndUniqueKeys(tableName string) (columns *sql.ColumnList, virtualColumns *sql.ColumnList, uniqueKeys [](*sql.UniqueKey), err error) { uniqueKeys, err = this.getCandidateUniqueKeys(tableName) if err != nil { - return columns, uniqueKeys, err + return columns, virtualColumns, uniqueKeys, err } if len(uniqueKeys) == 0 { - return columns, uniqueKeys, fmt.Errorf("No PRIMARY nor UNIQUE key found in table! Bailing out") + return columns, virtualColumns, uniqueKeys, fmt.Errorf("No PRIMARY nor UNIQUE key found in table! Bailing out") } - columns, err = mysql.GetTableColumns(this.db, this.migrationContext.DatabaseName, tableName) + columns, virtualColumns, err = mysql.GetTableColumns(this.db, this.migrationContext.DatabaseName, tableName) if err != nil { - return columns, uniqueKeys, err + return columns, virtualColumns, uniqueKeys, err } - return columns, uniqueKeys, nil + return columns, virtualColumns, uniqueKeys, nil } func (this *Inspector) InspectOriginalTable() (err error) { - this.migrationContext.OriginalTableColumns, this.migrationContext.OriginalTableUniqueKeys, err = this.InspectTableColumnsAndUniqueKeys(this.migrationContext.OriginalTableName) + this.migrationContext.OriginalTableColumns, this.migrationContext.OriginalTableVirtualColumns, this.migrationContext.OriginalTableUniqueKeys, err = this.InspectTableColumnsAndUniqueKeys(this.migrationContext.OriginalTableName) if err != nil { return err } @@ -122,7 +122,7 @@ func (this *Inspector) inspectOriginalAndGhostTables() (err error) { return fmt.Errorf("It seems like table structure is not identical between master and replica. This scenario is not supported.") } - this.migrationContext.GhostTableColumns, this.migrationContext.GhostTableUniqueKeys, err = this.InspectTableColumnsAndUniqueKeys(this.migrationContext.GetGhostTableName()) + this.migrationContext.GhostTableColumns, this.migrationContext.GhostTableVirtualColumns, this.migrationContext.GhostTableUniqueKeys, err = this.InspectTableColumnsAndUniqueKeys(this.migrationContext.GetGhostTableName()) if err != nil { return err } @@ -166,7 +166,7 @@ func (this *Inspector) inspectOriginalAndGhostTables() (err error) { } } - this.migrationContext.SharedColumns, this.migrationContext.MappedSharedColumns = this.getSharedColumns(this.migrationContext.OriginalTableColumns, this.migrationContext.GhostTableColumns, this.migrationContext.ColumnRenameMap) + this.migrationContext.SharedColumns, this.migrationContext.MappedSharedColumns = this.getSharedColumns(this.migrationContext.OriginalTableColumns, this.migrationContext.GhostTableColumns, this.migrationContext.OriginalTableVirtualColumns, this.migrationContext.GhostTableVirtualColumns, this.migrationContext.ColumnRenameMap) log.Infof("Shared columns are %s", this.migrationContext.SharedColumns) // By fact that a non-empty unique key exists we also know the shared columns are non-empty @@ -692,7 +692,7 @@ func (this *Inspector) getSharedUniqueKeys(originalUniqueKeys, ghostUniqueKeys [ } // getSharedColumns returns the intersection of two lists of columns in same order as the first list -func (this *Inspector) getSharedColumns(originalColumns, ghostColumns *sql.ColumnList, columnRenameMap map[string]string) (*sql.ColumnList, *sql.ColumnList) { +func (this *Inspector) getSharedColumns(originalColumns, ghostColumns *sql.ColumnList, originalVirtualColumns, ghostVirtualColumns *sql.ColumnList, columnRenameMap map[string]string) (*sql.ColumnList, *sql.ColumnList) { sharedColumnNames := []string{} for _, originalColumn := range originalColumns.Names() { isSharedColumn := false @@ -709,6 +709,16 @@ func (this *Inspector) getSharedColumns(originalColumns, ghostColumns *sql.Colum isSharedColumn = false } } + for _, virtualColumn := range originalVirtualColumns.Names() { + if strings.EqualFold(originalColumn, virtualColumn) { + isSharedColumn = false + } + } + for _, virtualColumn := range ghostVirtualColumns.Names() { + if strings.EqualFold(originalColumn, virtualColumn) { + isSharedColumn = false + } + } if isSharedColumn { sharedColumnNames = append(sharedColumnNames, originalColumn) } diff --git a/go/mysql/utils.go b/go/mysql/utils.go index 532cbb4..67ed7f8 100644 --- a/go/mysql/utils.go +++ b/go/mysql/utils.go @@ -8,6 +8,7 @@ package mysql import ( gosql "database/sql" "fmt" + "strings" "sync" "time" @@ -178,7 +179,7 @@ func GetInstanceKey(db *gosql.DB) (instanceKey *InstanceKey, err error) { } // GetTableColumns reads column list from given table -func GetTableColumns(db *gosql.DB, databaseName, tableName string) (*sql.ColumnList, error) { +func GetTableColumns(db *gosql.DB, databaseName, tableName string) (*sql.ColumnList, *sql.ColumnList, error) { query := fmt.Sprintf(` show columns from %s.%s `, @@ -186,18 +187,24 @@ func GetTableColumns(db *gosql.DB, databaseName, tableName string) (*sql.ColumnL sql.EscapeName(tableName), ) columnNames := []string{} + virtualColumnNames := []string{} err := sqlutils.QueryRowsMap(db, query, func(rowMap sqlutils.RowMap) error { - columnNames = append(columnNames, rowMap.GetString("Field")) + columnName := rowMap.GetString("Field") + columnNames = append(columnNames, columnName) + if strings.Contains(rowMap.GetString("Extra"), " GENERATED") { + log.Debugf("%s is a generated column", columnName) + virtualColumnNames = append(virtualColumnNames, columnName) + } return nil }) if err != nil { - return nil, err + return nil, nil, err } if len(columnNames) == 0 { - return nil, log.Errorf("Found 0 columns on %s.%s. Bailing out", + return nil, nil, log.Errorf("Found 0 columns on %s.%s. Bailing out", sql.EscapeName(databaseName), sql.EscapeName(tableName), ) } - return sql.NewColumnList(columnNames), nil + return sql.NewColumnList(columnNames), sql.NewColumnList(virtualColumnNames), nil } diff --git a/localtests/generated-columns-add57/create.sql b/localtests/generated-columns-add57/create.sql new file mode 100644 index 0000000..ef795ba --- /dev/null +++ b/localtests/generated-columns-add57/create.sql @@ -0,0 +1,29 @@ +drop table if exists gh_ost_test; +create table gh_ost_test ( + id int auto_increment, + a int not null, + b int not null, + primary key(id) +) auto_increment=1; + +drop event if exists gh_ost_test; +delimiter ;; +create event gh_ost_test + on schedule every 1 second + starts current_timestamp + ends current_timestamp + interval 60 second + on completion not preserve + enable + do +begin + insert into gh_ost_test (id, a, b) values (null, 2,3); + insert into gh_ost_test (id, a, b) values (null, 2,4); + insert into gh_ost_test (id, a, b) values (null, 2,5); + insert into gh_ost_test (id, a, b) values (null, 2,6); + insert into gh_ost_test (id, a, b) values (null, 2,7); + insert into gh_ost_test (id, a, b) values (null, 2,8); + insert into gh_ost_test (id, a, b) values (null, 2,9); + insert into gh_ost_test (id, a, b) values (null, 2,0); + insert into gh_ost_test (id, a, b) values (null, 2,1); + insert into gh_ost_test (id, a, b) values (null, 2,2); +end ;; diff --git a/localtests/generated-columns-add57/extra_args b/localtests/generated-columns-add57/extra_args new file mode 100644 index 0000000..b2bf5bc --- /dev/null +++ b/localtests/generated-columns-add57/extra_args @@ -0,0 +1 @@ +--alter="add column sum_ab int as (a + b) virtual not null" diff --git a/localtests/generated-columns-add57/ghost_columns b/localtests/generated-columns-add57/ghost_columns new file mode 100644 index 0000000..bd17155 --- /dev/null +++ b/localtests/generated-columns-add57/ghost_columns @@ -0,0 +1 @@ +id, a, b diff --git a/localtests/generated-columns-add57/ignore_versions b/localtests/generated-columns-add57/ignore_versions new file mode 100644 index 0000000..b6de5f8 --- /dev/null +++ b/localtests/generated-columns-add57/ignore_versions @@ -0,0 +1 @@ +(5.5|5.6) diff --git a/localtests/generated-columns-add57/order_by b/localtests/generated-columns-add57/order_by new file mode 100644 index 0000000..074d1ee --- /dev/null +++ b/localtests/generated-columns-add57/order_by @@ -0,0 +1 @@ +id diff --git a/localtests/generated-columns-add57/orig_columns b/localtests/generated-columns-add57/orig_columns new file mode 100644 index 0000000..bd17155 --- /dev/null +++ b/localtests/generated-columns-add57/orig_columns @@ -0,0 +1 @@ +id, a, b diff --git a/localtests/generated-columns-rename57/create.sql b/localtests/generated-columns-rename57/create.sql new file mode 100644 index 0000000..e244ca3 --- /dev/null +++ b/localtests/generated-columns-rename57/create.sql @@ -0,0 +1,30 @@ +drop table if exists gh_ost_test; +create table gh_ost_test ( + id int auto_increment, + a int not null, + b int not null, + sum_ab int as (a + b) virtual not null, + primary key(id) +) auto_increment=1; + +drop event if exists gh_ost_test; +delimiter ;; +create event gh_ost_test + on schedule every 1 second + starts current_timestamp + ends current_timestamp + interval 60 second + on completion not preserve + enable + do +begin + insert into gh_ost_test (id, a, b) values (null, 2,3); + insert into gh_ost_test (id, a, b) values (null, 2,4); + insert into gh_ost_test (id, a, b) values (null, 2,5); + insert into gh_ost_test (id, a, b) values (null, 2,6); + insert into gh_ost_test (id, a, b) values (null, 2,7); + insert into gh_ost_test (id, a, b) values (null, 2,8); + insert into gh_ost_test (id, a, b) values (null, 2,9); + insert into gh_ost_test (id, a, b) values (null, 2,0); + insert into gh_ost_test (id, a, b) values (null, 2,1); + insert into gh_ost_test (id, a, b) values (null, 2,2); +end ;; diff --git a/localtests/generated-columns-rename57/extra_args b/localtests/generated-columns-rename57/extra_args new file mode 100644 index 0000000..6a19098 --- /dev/null +++ b/localtests/generated-columns-rename57/extra_args @@ -0,0 +1 @@ +--alter="change sum_ab total_ab int as (a + b) virtual not null" --approve-renamed-columns diff --git a/localtests/generated-columns-rename57/ignore_versions b/localtests/generated-columns-rename57/ignore_versions new file mode 100644 index 0000000..b6de5f8 --- /dev/null +++ b/localtests/generated-columns-rename57/ignore_versions @@ -0,0 +1 @@ +(5.5|5.6) diff --git a/localtests/generated-columns57/create.sql b/localtests/generated-columns57/create.sql new file mode 100644 index 0000000..e244ca3 --- /dev/null +++ b/localtests/generated-columns57/create.sql @@ -0,0 +1,30 @@ +drop table if exists gh_ost_test; +create table gh_ost_test ( + id int auto_increment, + a int not null, + b int not null, + sum_ab int as (a + b) virtual not null, + primary key(id) +) auto_increment=1; + +drop event if exists gh_ost_test; +delimiter ;; +create event gh_ost_test + on schedule every 1 second + starts current_timestamp + ends current_timestamp + interval 60 second + on completion not preserve + enable + do +begin + insert into gh_ost_test (id, a, b) values (null, 2,3); + insert into gh_ost_test (id, a, b) values (null, 2,4); + insert into gh_ost_test (id, a, b) values (null, 2,5); + insert into gh_ost_test (id, a, b) values (null, 2,6); + insert into gh_ost_test (id, a, b) values (null, 2,7); + insert into gh_ost_test (id, a, b) values (null, 2,8); + insert into gh_ost_test (id, a, b) values (null, 2,9); + insert into gh_ost_test (id, a, b) values (null, 2,0); + insert into gh_ost_test (id, a, b) values (null, 2,1); + insert into gh_ost_test (id, a, b) values (null, 2,2); +end ;; diff --git a/localtests/generated-columns57/ignore_versions b/localtests/generated-columns57/ignore_versions new file mode 100644 index 0000000..b6de5f8 --- /dev/null +++ b/localtests/generated-columns57/ignore_versions @@ -0,0 +1 @@ +(5.5|5.6) diff --git a/localtests/json57dml/create.sql b/localtests/json57dml/create.sql index ac20941..0e76b2e 100644 --- a/localtests/json57dml/create.sql +++ b/localtests/json57dml/create.sql @@ -19,7 +19,7 @@ create event gh_ost_test begin insert into gh_ost_test (id, i, j) values (null, 11, '"sometext"'); insert into gh_ost_test (id, i, j) values (null, 13, '{"key":"val"}'); - insert into gh_ost_test (id, i, j) values (null, 17, '{"is-it": true, "count": 3, "elements": []}'); + insert into gh_ost_test (id, i, j) values (null, 17, '{"is-it": true, "count": 3, "elements": []}'); insert into gh_ost_test (id, i, j) values (null, 19, '{"text":"Lorem ipsum dolor sit amet, consectetuer adipiscing elit. Aenean commodo ligula eget dolor. Aenean massa. Cum sociis natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Donec quam felis, ultricies nec, pellentesque eu, pretium quis, sem. Nulla consequat massa quis enim. Donec pede justo, fringilla vel, aliquet nec, vulputate eget, arcu. In enim justo, rhoncus ut, imperdiet a, venenatis vitae, justo. Nullam dictum felis eu pede mollis pretium. Integer tincidunt. Cras dapibus. Vivamus elementum semper nisi. Aenean vulputate eleifend tellus. Aenean leo ligula, porttitor eu, consequat vitae, eleifend ac, enim. Aliquam lorem ante, dapibus in, viverra quis, feugiat a, tellus. Phasellus viverra nulla ut metus varius laoreet. Quisque rutrum. Aenean imperdiet. Etiam ultricies nisi vel augue. Curabitur ullamcorper ultricies nisi. Nam eget dui. Etiam rhoncus. Maecenas tempus, tellus eget condimentum rhoncus, sem quam semper libero, sit amet adipiscing sem neque sed ipsum. Nam quam nunc, blandit vel, luctus pulvinar, hendrerit id, lorem. Maecenas nec odio et ante tincidunt tempus. Donec vitae sapien ut libero venenatis faucibus. Nullam quis ante. Etiam sit amet orci eget eros faucibus tincidunt. Duis leo. Sed fringilla mauris sit amet nibh. Donec sodales sagittis magna. Sed consequat, leo eget bibendum sodales, augue velit cursus nunc, quis gravida magna mi a libero. Fusce vulputate eleifend sapien. Vestibulum purus quam, scelerisque ut, mollis sed, nonummy id, metus. Nullam accumsan lorem in dui. Cras ultricies mi eu turpis hendrerit fringilla. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia Curae; In ac dui quis mi consectetuer lacinia. Nam pretium turpis et arcu. Duis arcu tortor, suscipit eget, imperdiet nec, imperdiet iaculis, ipsum. Sed aliquam ultrices mauris. Integer ante arcu, accumsan a, consectetuer eget, posuere ut, mauris. Praesent adipiscing. Phasellus ullamcorper ipsum rutrum nunc. Nunc nonummy metus. Vestibulum volutpat pretium libero. Cras id dui. Aenean ut eros et nisl sagittis vestibulum. Nullam nulla eros, ultricies sit amet, nonummy id, imperdiet feugiat, pede. Sed lectus. Donec mollis hendrerit risus. Phasellus nec sem in justo pellentesque facilisis. Etiam imperdiet imperdiet orci. Nunc nec neque. Phasellus leo dolor, tempus non, auctor et, hendrerit quis, nisi. Curabitur ligula sapien, tincidunt non, euismod vitae, posuere imperdiet, leo. Maecenas malesuada. Praesent congue erat at massa. Sed cursus turpis vitae tortor. Donec posuere vulputate arcu. Phasellus accumsan cursus velit. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia Curae; Sed aliquam, nisi quis porttitor congue, elit erat euismod orci, ac"}'); update gh_ost_test set j = '{"updated": 11}', updated = 1 where i = 11 and updated = 0;