diff --git a/build.sh b/build.sh index 150250b..36c4215 100755 --- a/build.sh +++ b/build.sh @@ -2,7 +2,7 @@ # # -RELEASE_VERSION="1.0.10" +RELEASE_VERSION="1.0.13" function build { osname=$1 diff --git a/doc/interactive-commands.md b/doc/interactive-commands.md index 5449627..fa971b0 100644 --- a/doc/interactive-commands.md +++ b/doc/interactive-commands.md @@ -24,7 +24,7 @@ Both interfaces may serve at the same time. Both respond to simple text command, - `critical-load=`: change critical load setting (exceeding given thresholds causes panic and abort) - `nice-ratio=`: change _nice_ ratio: 0 for aggressive (not nice, not sleeping), positive integer `n`: for any `1ms` spent copying rows, spend `n*1ms` units of time sleeping. Examples: assume a single rows chunk copy takes `100ms` to complete. `nice-ratio=0.5` will cause `gh-ost` to sleep for `50ms` immediately following. `nice-ratio=1` will cause `gh-ost` to sleep for `100ms`, effectively doubling runtime; value of `2` will effectively triple the runtime; etc. - `throttle-query`: change throttle query -- `throttle-control-replicas`: change list of throttle-control replicas, these are replicas `gh-ost` will check +- `throttle-control-replicas='replica1,replica2'`: change list of throttle-control replicas, these are replicas `gh-ost` will check. This takes a comma separated list of replica's to check and replaces the previous list. - `throttle`: force migration suspend - `no-throttle`: cancel forced suspension (though other throttling reasons may still apply) - `unpostpone`: at a time where `gh-ost` is postponing the [cut-over](cut-over.md) phase, instruct `gh-ost` to stop postponing and proceed immediately to cut-over. diff --git a/doc/local-tests.md b/doc/local-tests.md new file mode 100644 index 0000000..1614ee4 --- /dev/null +++ b/doc/local-tests.md @@ -0,0 +1,22 @@ +# Local tests + +`gh-ost` is continuously tested in production via `--test-on-replica alter='engine=innodb'`. These tests check the GitHub workload and usage, but not necessarily the general case. + +Local tests are an additional layer of tests. They will eventually be part of continuous integration tests. + +Local tests test explicit use cases, such as column renames, mix of time zones, special types and alters. Traits of a single test: + +- Composed of a single table. +- A single alter. +- By default the alter is `engine=innodb`, but this can be overridden per-test +- Scheduled DML operations, executed via `event_scheduler`. +- `gh-ost` is set to execute and throttle for `5` seconds, at which time all tested DMLs are expected to operate. +- The test requires a replication topology and utilizes `--test-on-replica` +- The test checksums the two tables (original and _ghost_) and expects identical checksum +- By default the test selects all (`*`) columns, but this can be overridden per-test + +Tests are found under [localtests](https://github.com/github/gh-ost/tree/master/localtests). A single test is a subdirectory and tests are iterated alphabetically. + +New data-integrity, synchronization issues or otherwise concerns are expected to be tested by new test cases. + +While this is merged work is still ongoing. diff --git a/go/base/context.go b/go/base/context.go index 438490b..f1c3111 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -84,13 +84,14 @@ type MigrationContext struct { ServeSocketFile string ServeTCPPort int64 - Noop bool - TestOnReplica bool - MigrateOnReplica bool - OkToDropTable bool - InitiallyDropOldTable bool - InitiallyDropGhostTable bool - CutOverType CutOver + Noop bool + TestOnReplica bool + MigrateOnReplica bool + TestOnReplicaSkipReplicaStop bool + OkToDropTable bool + InitiallyDropOldTable bool + InitiallyDropGhostTable bool + CutOverType CutOver Hostname string TableEngine string diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index 501d04b..480db68 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -61,6 +61,7 @@ func main() { executeFlag := flag.Bool("execute", false, "actually execute the alter & migrate the table. Default is noop: do some tests and exit") flag.BoolVar(&migrationContext.TestOnReplica, "test-on-replica", false, "Have the migration run on a replica, not on the master. At the end of migration replication is stopped, and tables are swapped and immediately swap-revert. Replication remains stopped and you can compare the two tables for building trust") + flag.BoolVar(&migrationContext.TestOnReplicaSkipReplicaStop, "test-on-replica-skip-replica-stop", false, "When --test-on-replica is enabled, do not issue commands stop replication (requires --test-on-replica)") flag.BoolVar(&migrationContext.MigrateOnReplica, "migrate-on-replica", false, "Have the migration run on a replica, not on the master. This will do the full migration on the replica including cut-over (as opposed to --test-on-replica)") flag.BoolVar(&migrationContext.OkToDropTable, "ok-to-drop-table", false, "Shall the tool drop the old table at end of operation. DROPping tables can be a long locking operation, which is why I'm not doing it by default. I'm an online tool, yes?") @@ -149,6 +150,13 @@ func main() { if migrationContext.SwitchToRowBinlogFormat && migrationContext.AssumeRBR { log.Fatalf("--switch-to-rbr and --assume-rbr are mutually exclusive") } + if migrationContext.TestOnReplicaSkipReplicaStop { + if !migrationContext.TestOnReplica { + log.Fatalf("--test-on-replica-skip-replica-stop requires --test-on-replica to be enabled") + } + log.Warning("--test-on-replica-skip-replica-stop enabled. We will not stop replication before cut-over. Ensure you have a plugin that does this.") + } + switch *cutOver { case "atomic", "default", "": migrationContext.CutOverType = base.CutOverAtomic diff --git a/go/logic/applier.go b/go/logic/applier.go index 10da785..14d74e1 100644 --- a/go/logic/applier.go +++ b/go/logic/applier.go @@ -107,7 +107,7 @@ func (this *Applier) ValidateOrDropExistingTables() error { } } if this.tableExists(this.migrationContext.GetGhostTableName()) { - return fmt.Errorf("Table %s already exists. Panicking. Use --initially-drop-ghost-table to force dropping it", sql.EscapeName(this.migrationContext.GetGhostTableName())) + return fmt.Errorf("Table %s already exists. Panicking. Use --initially-drop-ghost-table to force dropping it, though I really prefer that you drop it or rename it away", sql.EscapeName(this.migrationContext.GetGhostTableName())) } if this.migrationContext.InitiallyDropOldTable { if err := this.DropOldTable(); err != nil { @@ -115,7 +115,7 @@ func (this *Applier) ValidateOrDropExistingTables() error { } } if this.tableExists(this.migrationContext.GetOldTableName()) { - return fmt.Errorf("Table %s already exists. Panicking. Use --initially-drop-old-table to force dropping it", sql.EscapeName(this.migrationContext.GetOldTableName())) + return fmt.Errorf("Table %s already exists. Panicking. Use --initially-drop-old-table to force dropping it, though I really prefer that you drop it or rename it away", sql.EscapeName(this.migrationContext.GetOldTableName())) } return nil @@ -574,6 +574,7 @@ func (this *Applier) StopReplication() error { if err := this.StopSlaveSQLThread(); err != nil { return err } + readBinlogCoordinates, executeBinlogCoordinates, err := mysql.GetReplicationBinlogCoordinates(this.db) if err != nil { return err @@ -832,12 +833,12 @@ func (this *Applier) buildDMLEventQuery(dmlEvent *binlog.BinlogDMLEvent) (query } case binlog.InsertDML: { - query, sharedArgs, err := sql.BuildDMLInsertQuery(dmlEvent.DatabaseName, this.migrationContext.GetGhostTableName(), this.migrationContext.OriginalTableColumns, this.migrationContext.MappedSharedColumns, dmlEvent.NewColumnValues.AbstractValues()) + query, sharedArgs, err := sql.BuildDMLInsertQuery(dmlEvent.DatabaseName, this.migrationContext.GetGhostTableName(), this.migrationContext.OriginalTableColumns, this.migrationContext.SharedColumns, this.migrationContext.MappedSharedColumns, dmlEvent.NewColumnValues.AbstractValues()) return query, sharedArgs, 1, err } case binlog.UpdateDML: { - query, sharedArgs, uniqueKeyArgs, err := sql.BuildDMLUpdateQuery(dmlEvent.DatabaseName, this.migrationContext.GetGhostTableName(), this.migrationContext.OriginalTableColumns, this.migrationContext.MappedSharedColumns, &this.migrationContext.UniqueKey.Columns, dmlEvent.NewColumnValues.AbstractValues(), dmlEvent.WhereColumnValues.AbstractValues()) + query, sharedArgs, uniqueKeyArgs, err := sql.BuildDMLUpdateQuery(dmlEvent.DatabaseName, this.migrationContext.GetGhostTableName(), this.migrationContext.OriginalTableColumns, this.migrationContext.SharedColumns, this.migrationContext.MappedSharedColumns, &this.migrationContext.UniqueKey.Columns, dmlEvent.NewColumnValues.AbstractValues(), dmlEvent.WhereColumnValues.AbstractValues()) args = append(args, sharedArgs...) args = append(args, uniqueKeyArgs...) return query, args, 0, err @@ -853,7 +854,6 @@ func (this *Applier) ApplyDMLEventQuery(dmlEvent *binlog.BinlogDMLEvent) error { if err != nil { return err } - // TODO The below is in preparation for transactional writes on the ghost tables. // Such writes would be, for example: // - prepended with sql_mode setup @@ -871,6 +871,12 @@ func (this *Applier) ApplyDMLEventQuery(dmlEvent *binlog.BinlogDMLEvent) error { if err != nil { return err } + if _, err := tx.Exec(`SET + SESSION time_zone = '+00:00', + sql_mode = CONCAT(@@session.sql_mode, ',STRICT_ALL_TABLES') + `); err != nil { + return err + } if _, err := tx.Exec(query, args...); err != nil { return err } diff --git a/go/logic/migrator.go b/go/logic/migrator.go index 5956c95..5aaf9f1 100644 --- a/go/logic/migrator.go +++ b/go/logic/migrator.go @@ -363,7 +363,7 @@ func (this *Migrator) validateStatement() (err error) { if this.parser.HasNonTrivialRenames() && !this.migrationContext.SkipRenamedColumns { this.migrationContext.ColumnRenameMap = this.parser.GetNonTrivialRenames() if !this.migrationContext.ApproveRenamedColumns { - return fmt.Errorf("Alter statement has column(s) renamed. gh-ost suspects the following renames: %v; but to proceed you must approve via `--approve-renamed-columns` (or you can skip renamed columns via `--skip-renamed-columns`)", this.parser.GetNonTrivialRenames()) + return fmt.Errorf("gh-ost believes the ALTER statement renames columns, as follows: %v; as precation, you are asked to confirm gh-ost is correct, and provide with `--approve-renamed-columns`, and we're all happy. Or you can skip renamed columns via `--skip-renamed-columns`, in which case column data may be lost", this.parser.GetNonTrivialRenames()) } log.Infof("Alter statement has column(s) renamed. gh-ost finds the following renames: %v; --approve-renamed-columns is given and so migration proceeds.", this.parser.GetNonTrivialRenames()) } @@ -402,7 +402,7 @@ func (this *Migrator) Migrate() (err error) { return err } - log.Debugf("Waiting for tables to be in place") + log.Infof("Waiting for tables to be in place") <-this.tablesInPlace log.Debugf("Tables are in place") // Yay! We now know the Ghost and Changelog tables are good to examine! @@ -520,10 +520,14 @@ func (this *Migrator) cutOver() (err error) { // the same cut-over phase as the master would use. That means we take locks // and swap the tables. // The difference is that we will later swap the tables back. - log.Debugf("testing on replica. Stopping replication IO thread") this.hooksExecutor.onStopReplication() - if err := this.retryOperation(this.applier.StopReplication); err != nil { - return err + if this.migrationContext.TestOnReplicaSkipReplicaStop { + log.Warningf("--test-on-replica-skip-replica-stop enabled, we are not stopping replication.") + } else { + log.Debugf("testing on replica. Stopping replication IO thread") + if err := this.retryOperation(this.applier.StopReplication); err != nil { + return err + } } // We're merly testing, we don't want to keep this state. Rollback the renames as possible defer this.applier.RenameTablesRollback() diff --git a/go/sql/builder.go b/go/sql/builder.go index 7c670a9..79cea47 100644 --- a/go/sql/builder.go +++ b/go/sql/builder.go @@ -354,7 +354,7 @@ func BuildDMLDeleteQuery(databaseName, tableName string, tableColumns, uniqueKey return result, uniqueKeyArgs, nil } -func BuildDMLInsertQuery(databaseName, tableName string, tableColumns, sharedColumns *ColumnList, args []interface{}) (result string, sharedArgs []interface{}, err error) { +func BuildDMLInsertQuery(databaseName, tableName string, tableColumns, sharedColumns, mappedSharedColumns *ColumnList, args []interface{}) (result string, sharedArgs []interface{}, err error) { if len(args) != tableColumns.Len() { return result, args, fmt.Errorf("args count differs from table column count in BuildDMLInsertQuery") } @@ -367,17 +367,17 @@ func BuildDMLInsertQuery(databaseName, tableName string, tableColumns, sharedCol databaseName = EscapeName(databaseName) tableName = EscapeName(tableName) - for _, column := range sharedColumns.Names { + for _, column := range mappedSharedColumns.Names { tableOrdinal := tableColumns.Ordinals[column] - arg := fixArgType(args[tableOrdinal], sharedColumns.IsUnsigned(column)) + arg := fixArgType(args[tableOrdinal], mappedSharedColumns.IsUnsigned(column)) sharedArgs = append(sharedArgs, arg) } - sharedColumnNames := duplicateNames(sharedColumns.Names) - for i := range sharedColumnNames { - sharedColumnNames[i] = EscapeName(sharedColumnNames[i]) + mappedSharedColumnNames := duplicateNames(mappedSharedColumns.Names) + for i := range mappedSharedColumnNames { + mappedSharedColumnNames[i] = EscapeName(mappedSharedColumnNames[i]) } - preparedValues := buildPreparedValues(sharedColumns.Len()) + preparedValues := buildPreparedValues(mappedSharedColumns.Len()) result = fmt.Sprintf(` replace /* gh-ost %s.%s */ into @@ -387,13 +387,13 @@ func BuildDMLInsertQuery(databaseName, tableName string, tableColumns, sharedCol (%s) `, databaseName, tableName, databaseName, tableName, - strings.Join(sharedColumnNames, ", "), + strings.Join(mappedSharedColumnNames, ", "), strings.Join(preparedValues, ", "), ) return result, sharedArgs, nil } -func BuildDMLUpdateQuery(databaseName, tableName string, tableColumns, sharedColumns, uniqueKeyColumns *ColumnList, valueArgs, whereArgs []interface{}) (result string, sharedArgs, uniqueKeyArgs []interface{}, err error) { +func BuildDMLUpdateQuery(databaseName, tableName string, tableColumns, sharedColumns, mappedSharedColumns, uniqueKeyColumns *ColumnList, valueArgs, whereArgs []interface{}) (result string, sharedArgs, uniqueKeyArgs []interface{}, err error) { if len(valueArgs) != tableColumns.Len() { return result, sharedArgs, uniqueKeyArgs, fmt.Errorf("value args count differs from table column count in BuildDMLUpdateQuery") } @@ -415,9 +415,10 @@ func BuildDMLUpdateQuery(databaseName, tableName string, tableColumns, sharedCol databaseName = EscapeName(databaseName) tableName = EscapeName(tableName) - for _, column := range sharedColumns.Names { + for i, column := range sharedColumns.Names { + mappedColumn := mappedSharedColumns.Names[i] tableOrdinal := tableColumns.Ordinals[column] - arg := fixArgType(valueArgs[tableOrdinal], sharedColumns.IsUnsigned(column)) + arg := fixArgType(valueArgs[tableOrdinal], mappedSharedColumns.IsUnsigned(mappedColumn)) sharedArgs = append(sharedArgs, arg) } @@ -427,11 +428,11 @@ func BuildDMLUpdateQuery(databaseName, tableName string, tableColumns, sharedCol uniqueKeyArgs = append(uniqueKeyArgs, arg) } - sharedColumnNames := duplicateNames(sharedColumns.Names) - for i := range sharedColumnNames { - sharedColumnNames[i] = EscapeName(sharedColumnNames[i]) + mappedSharedColumnNames := duplicateNames(mappedSharedColumns.Names) + for i := range mappedSharedColumnNames { + mappedSharedColumnNames[i] = EscapeName(mappedSharedColumnNames[i]) } - setClause, err := BuildSetPreparedClause(sharedColumnNames) + setClause, err := BuildSetPreparedClause(mappedSharedColumnNames) equalsComparison, err := BuildEqualsPreparedComparison(uniqueKeyColumns.Names) result = fmt.Sprintf(` diff --git a/go/sql/builder_test.go b/go/sql/builder_test.go index db2617e..55f27f9 100644 --- a/go/sql/builder_test.go +++ b/go/sql/builder_test.go @@ -442,7 +442,7 @@ func TestBuildDMLInsertQuery(t *testing.T) { args := []interface{}{3, "testname", "first", 17, 23} { sharedColumns := NewColumnList([]string{"id", "name", "position", "age"}) - query, sharedArgs, err := BuildDMLInsertQuery(databaseName, tableName, tableColumns, sharedColumns, args) + query, sharedArgs, err := BuildDMLInsertQuery(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, args) test.S(t).ExpectNil(err) expected := ` replace /* gh-ost mydb.tbl */ @@ -456,7 +456,7 @@ func TestBuildDMLInsertQuery(t *testing.T) { } { sharedColumns := NewColumnList([]string{"position", "name", "age", "id"}) - query, sharedArgs, err := BuildDMLInsertQuery(databaseName, tableName, tableColumns, sharedColumns, args) + query, sharedArgs, err := BuildDMLInsertQuery(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, args) test.S(t).ExpectNil(err) expected := ` replace /* gh-ost mydb.tbl */ @@ -470,12 +470,12 @@ func TestBuildDMLInsertQuery(t *testing.T) { } { sharedColumns := NewColumnList([]string{"position", "name", "surprise", "id"}) - _, _, err := BuildDMLInsertQuery(databaseName, tableName, tableColumns, sharedColumns, args) + _, _, err := BuildDMLInsertQuery(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, args) test.S(t).ExpectNotNil(err) } { sharedColumns := NewColumnList([]string{}) - _, _, err := BuildDMLInsertQuery(databaseName, tableName, tableColumns, sharedColumns, args) + _, _, err := BuildDMLInsertQuery(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, args) test.S(t).ExpectNotNil(err) } } @@ -489,7 +489,7 @@ func TestBuildDMLInsertQuerySignedUnsigned(t *testing.T) { // testing signed args := []interface{}{3, "testname", "first", int8(-1), 23} sharedColumns := NewColumnList([]string{"id", "name", "position", "age"}) - query, sharedArgs, err := BuildDMLInsertQuery(databaseName, tableName, tableColumns, sharedColumns, args) + query, sharedArgs, err := BuildDMLInsertQuery(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, args) test.S(t).ExpectNil(err) expected := ` replace /* gh-ost mydb.tbl */ @@ -505,7 +505,7 @@ func TestBuildDMLInsertQuerySignedUnsigned(t *testing.T) { // testing unsigned args := []interface{}{3, "testname", "first", int8(-1), 23} sharedColumns.SetUnsigned("position") - query, sharedArgs, err := BuildDMLInsertQuery(databaseName, tableName, tableColumns, sharedColumns, args) + query, sharedArgs, err := BuildDMLInsertQuery(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, args) test.S(t).ExpectNil(err) expected := ` replace /* gh-ost mydb.tbl */ @@ -521,7 +521,7 @@ func TestBuildDMLInsertQuerySignedUnsigned(t *testing.T) { // testing unsigned args := []interface{}{3, "testname", "first", int32(-1), 23} sharedColumns.SetUnsigned("position") - query, sharedArgs, err := BuildDMLInsertQuery(databaseName, tableName, tableColumns, sharedColumns, args) + query, sharedArgs, err := BuildDMLInsertQuery(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, args) test.S(t).ExpectNil(err) expected := ` replace /* gh-ost mydb.tbl */ @@ -544,7 +544,7 @@ func TestBuildDMLUpdateQuery(t *testing.T) { { sharedColumns := NewColumnList([]string{"id", "name", "position", "age"}) uniqueKeyColumns := NewColumnList([]string{"position"}) - query, sharedArgs, uniqueKeyArgs, err := BuildDMLUpdateQuery(databaseName, tableName, tableColumns, sharedColumns, uniqueKeyColumns, valueArgs, whereArgs) + query, sharedArgs, uniqueKeyArgs, err := BuildDMLUpdateQuery(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, uniqueKeyColumns, valueArgs, whereArgs) test.S(t).ExpectNil(err) expected := ` update /* gh-ost mydb.tbl */ @@ -560,7 +560,7 @@ func TestBuildDMLUpdateQuery(t *testing.T) { { sharedColumns := NewColumnList([]string{"id", "name", "position", "age"}) uniqueKeyColumns := NewColumnList([]string{"position", "name"}) - query, sharedArgs, uniqueKeyArgs, err := BuildDMLUpdateQuery(databaseName, tableName, tableColumns, sharedColumns, uniqueKeyColumns, valueArgs, whereArgs) + query, sharedArgs, uniqueKeyArgs, err := BuildDMLUpdateQuery(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, uniqueKeyColumns, valueArgs, whereArgs) test.S(t).ExpectNil(err) expected := ` update /* gh-ost mydb.tbl */ @@ -576,7 +576,7 @@ func TestBuildDMLUpdateQuery(t *testing.T) { { sharedColumns := NewColumnList([]string{"id", "name", "position", "age"}) uniqueKeyColumns := NewColumnList([]string{"age"}) - query, sharedArgs, uniqueKeyArgs, err := BuildDMLUpdateQuery(databaseName, tableName, tableColumns, sharedColumns, uniqueKeyColumns, valueArgs, whereArgs) + query, sharedArgs, uniqueKeyArgs, err := BuildDMLUpdateQuery(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, uniqueKeyColumns, valueArgs, whereArgs) test.S(t).ExpectNil(err) expected := ` update /* gh-ost mydb.tbl */ @@ -592,7 +592,7 @@ func TestBuildDMLUpdateQuery(t *testing.T) { { sharedColumns := NewColumnList([]string{"id", "name", "position", "age"}) uniqueKeyColumns := NewColumnList([]string{"age", "position", "id", "name"}) - query, sharedArgs, uniqueKeyArgs, err := BuildDMLUpdateQuery(databaseName, tableName, tableColumns, sharedColumns, uniqueKeyColumns, valueArgs, whereArgs) + query, sharedArgs, uniqueKeyArgs, err := BuildDMLUpdateQuery(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, uniqueKeyColumns, valueArgs, whereArgs) test.S(t).ExpectNil(err) expected := ` update /* gh-ost mydb.tbl */ @@ -608,15 +608,32 @@ func TestBuildDMLUpdateQuery(t *testing.T) { { sharedColumns := NewColumnList([]string{"id", "name", "position", "age"}) uniqueKeyColumns := NewColumnList([]string{"age", "surprise"}) - _, _, _, err := BuildDMLUpdateQuery(databaseName, tableName, tableColumns, sharedColumns, uniqueKeyColumns, valueArgs, whereArgs) + _, _, _, err := BuildDMLUpdateQuery(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, uniqueKeyColumns, valueArgs, whereArgs) test.S(t).ExpectNotNil(err) } { sharedColumns := NewColumnList([]string{"id", "name", "position", "age"}) uniqueKeyColumns := NewColumnList([]string{}) - _, _, _, err := BuildDMLUpdateQuery(databaseName, tableName, tableColumns, sharedColumns, uniqueKeyColumns, valueArgs, whereArgs) + _, _, _, err := BuildDMLUpdateQuery(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, uniqueKeyColumns, valueArgs, whereArgs) test.S(t).ExpectNotNil(err) } + { + sharedColumns := NewColumnList([]string{"id", "name", "position", "age"}) + mappedColumns := NewColumnList([]string{"id", "name", "role", "age"}) + uniqueKeyColumns := NewColumnList([]string{"id"}) + query, sharedArgs, uniqueKeyArgs, err := BuildDMLUpdateQuery(databaseName, tableName, tableColumns, sharedColumns, mappedColumns, uniqueKeyColumns, valueArgs, whereArgs) + test.S(t).ExpectNil(err) + expected := ` + update /* gh-ost mydb.tbl */ + mydb.tbl + set id=?, name=?, role=?, age=? + where + ((id = ?)) + ` + test.S(t).ExpectEquals(normalizeQuery(query), normalizeQuery(expected)) + test.S(t).ExpectTrue(reflect.DeepEqual(sharedArgs, []interface{}{3, "testname", 17, 23})) + test.S(t).ExpectTrue(reflect.DeepEqual(uniqueKeyArgs, []interface{}{3})) + } } func TestBuildDMLUpdateQuerySignedUnsigned(t *testing.T) { @@ -629,7 +646,7 @@ func TestBuildDMLUpdateQuerySignedUnsigned(t *testing.T) { uniqueKeyColumns := NewColumnList([]string{"position"}) { // test signed - query, sharedArgs, uniqueKeyArgs, err := BuildDMLUpdateQuery(databaseName, tableName, tableColumns, sharedColumns, uniqueKeyColumns, valueArgs, whereArgs) + query, sharedArgs, uniqueKeyArgs, err := BuildDMLUpdateQuery(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, uniqueKeyColumns, valueArgs, whereArgs) test.S(t).ExpectNil(err) expected := ` update /* gh-ost mydb.tbl */ @@ -646,7 +663,7 @@ func TestBuildDMLUpdateQuerySignedUnsigned(t *testing.T) { // test unsigned sharedColumns.SetUnsigned("age") uniqueKeyColumns.SetUnsigned("position") - query, sharedArgs, uniqueKeyArgs, err := BuildDMLUpdateQuery(databaseName, tableName, tableColumns, sharedColumns, uniqueKeyColumns, valueArgs, whereArgs) + query, sharedArgs, uniqueKeyArgs, err := BuildDMLUpdateQuery(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, uniqueKeyColumns, valueArgs, whereArgs) test.S(t).ExpectNil(err) expected := ` update /* gh-ost mydb.tbl */ diff --git a/localtests/enum/create.sql b/localtests/enum/create.sql new file mode 100644 index 0000000..6420233 --- /dev/null +++ b/localtests/enum/create.sql @@ -0,0 +1,27 @@ +drop table if exists gh_ost_test; +create table gh_ost_test ( + id int auto_increment, + i int not null, + e enum('red', 'green', 'blue', 'orange') null default null collate 'utf8_bin', + primary key(id) +) auto_increment=1; + +drop event if exists gh_ost_test; +delimiter ;; +create event gh_ost_test + on schedule every 1 second + starts current_timestamp + ends current_timestamp + interval 60 second + on completion not preserve + enable + do +begin + insert into gh_ost_test values (null, 11, 'red'); + insert into gh_ost_test values (null, 13, 'green'); + insert into gh_ost_test values (null, 17, 'blue'); + set @last_insert_id := last_insert_id(); + update gh_ost_test set e='orange' where id = @last_insert_id; + insert into gh_ost_test values (null, 23, null); + set @last_insert_id := last_insert_id(); + update gh_ost_test set i=i+1, e=null where id = @last_insert_id; +end ;; diff --git a/localtests/enum/extra_args b/localtests/enum/extra_args new file mode 100644 index 0000000..f369a56 --- /dev/null +++ b/localtests/enum/extra_args @@ -0,0 +1 @@ +--alter="change e e enum('red', 'green', 'blue', 'orange', 'yellow') null default null collate 'utf8_bin'" diff --git a/localtests/rename/create.sql b/localtests/rename/create.sql new file mode 100644 index 0000000..d28a7c1 --- /dev/null +++ b/localtests/rename/create.sql @@ -0,0 +1,26 @@ +drop table if exists gh_ost_test; +create table gh_ost_test ( + id int auto_increment, + c1 int not null, + c2 int not null, + primary key (id) +) auto_increment=1; + +drop event if exists gh_ost_test; +delimiter ;; +create event gh_ost_test + on schedule every 1 second + starts current_timestamp + ends current_timestamp + interval 60 second + on completion not preserve + enable + do +begin + insert ignore into gh_ost_test values (1, 11, 23); + insert ignore into gh_ost_test values (2, 13, 23); + insert into gh_ost_test values (null, 17, 23); + set @last_insert_id := last_insert_id(); + update gh_ost_test set c1=c1+@last_insert_id, c2=c2+@last_insert_id where id=@last_insert_id order by id desc limit 1; + delete from gh_ost_test where id=1; + delete from gh_ost_test where c1=13; -- id=2 +end ;; diff --git a/localtests/rename/extra_args b/localtests/rename/extra_args new file mode 100644 index 0000000..d36d5ee --- /dev/null +++ b/localtests/rename/extra_args @@ -0,0 +1 @@ +--alter="change column c2 c3 int not null" --approve-renamed-columns diff --git a/localtests/test.sh b/localtests/test.sh new file mode 100755 index 0000000..ee82623 --- /dev/null +++ b/localtests/test.sh @@ -0,0 +1,112 @@ +#!/bin/bash + +# Local integration tests. To be used by CI. +# See https://github.com/github/gh-ost/tree/doc/local-tests.md +# + +tests_path=$(dirname $0) +test_logfile=/tmp/gh-ost-test.log +exec_command_file=/tmp/gh-ost-test.bash + +master_host= +master_port= +replica_host= +replica_port= + +verify_master_and_replica() { + if [ "$(gh-ost-test-mysql-master -e "select 1" -ss)" != "1" ] ; then + echo "Cannot verify gh-ost-test-mysql-master" + exit 1 + fi + read master_host master_port <<< $(gh-ost-test-mysql-master -e "select @@hostname, @@port" -ss) + if [ "$(gh-ost-test-mysql-replica -e "select 1" -ss)" != "1" ] ; then + echo "Cannot verify gh-ost-test-mysql-replica" + exit 1 + fi + read replica_host replica_port <<< $(gh-ost-test-mysql-replica -e "select @@hostname, @@port" -ss) +} + +exec_cmd() { + echo "$@" + command "$@" 1> $test_logfile 2>&1 + return $? +} + +test_single() { + local test_name + test_name="$1" + + echo "Testing: $test_name" + + gh-ost-test-mysql-replica -e "start slave" + gh-ost-test-mysql-master test < $tests_path/$test_name/create.sql + + extra_args="" + if [ -f $tests_path/$test_name/extra_args ] ; then + extra_args=$(cat $tests_path/$test_name/extra_args) + fi + columns="*" + if [ -f $tests_path/$test_name/test_columns ] ; then + columns=$(cat $tests_path/$test_name/test_columns) + fi + # graceful sleep for replica to catch up + sleep 1 + # + cmd="go run go/cmd/gh-ost/main.go \ + --user=gh-ost \ + --password=gh-ost \ + --host=$replica_host \ + --port=$replica_port \ + --database=test \ + --table=gh_ost_test \ + --alter='engine=innodb' \ + --exact-rowcount \ + --switch-to-rbr \ + --initially-drop-old-table \ + --initially-drop-ghost-table \ + --throttle-query='select timestampdiff(second, min(last_update), now()) < 5 from _gh_ost_test_ghc' \ + --serve-socket-file=/tmp/gh-ost.test.sock \ + --initially-drop-socket-file \ + --postpone-cut-over-flag-file=/tmp/gh-ost.postpone.flag \ + --test-on-replica \ + --default-retries=1 \ + --verbose \ + --debug \ + --stack \ + --execute ${extra_args[@]}" + echo $cmd > $exec_command_file + bash $exec_command_file 1> $test_logfile 2>&1 + + if [ $? -ne 0 ] ; then + echo "ERROR $test_name execution failure. See $test_logfile" + return 1 + fi + + orig_checksum=$(gh-ost-test-mysql-replica test -e "select ${columns} from gh_ost_test" -ss | md5sum) + ghost_checksum=$(gh-ost-test-mysql-replica test -e "select ${columns} from _gh_ost_test_gho" -ss | md5sum) + + if [ "$orig_checksum" != "$ghost_checksum" ] ; then + echo "ERROR $test_name: checksum mismatch" + echo "---" + gh-ost-test-mysql-replica test -e "select ${columns} from gh_ost_test" -ss + echo "---" + gh-ost-test-mysql-replica test -e "select ${columns} from _gh_ost_test_gho" -ss + return 1 + fi +} + +test_all() { + find $tests_path ! -path . -type d -mindepth 1 -maxdepth 1 | cut -d "/" -f 3 | while read test_name ; do + test_single "$test_name" + if [ $? -ne 0 ] ; then + echo "+ FAIL" + return 1 + else + echo "+ pass" + fi + gh-ost-test-mysql-replica -e "start slave" + done +} + +verify_master_and_replica +test_all diff --git a/localtests/tz/create.sql b/localtests/tz/create.sql new file mode 100644 index 0000000..f908e6e --- /dev/null +++ b/localtests/tz/create.sql @@ -0,0 +1,41 @@ +drop table if exists gh_ost_test; +create table gh_ost_test ( + id int auto_increment, + i int not null, + ts0 timestamp default current_timestamp, + ts1 timestamp, + ts2 timestamp, + updated tinyint unsigned default 0, + primary key(id), + key i_idx(i) +) auto_increment=1; + +drop event if exists gh_ost_test; +delimiter ;; +create event gh_ost_test + on schedule every 1 second + starts current_timestamp + ends current_timestamp + interval 60 second + on completion not preserve + enable + do +begin + insert into gh_ost_test values (null, 11, null, now(), now(), 0); + update gh_ost_test set ts2=now() + interval 10 minute, updated = 1 where i = 11 order by id desc limit 1; + + set session time_zone='system'; + insert into gh_ost_test values (null, 13, null, now(), now(), 0); + update gh_ost_test set ts2=now() + interval 10 minute, updated = 1 where i = 13 order by id desc limit 1; + + set session time_zone='+00:00'; + insert into gh_ost_test values (null, 17, null, now(), now(), 0); + update gh_ost_test set ts2=now() + interval 10 minute, updated = 1 where i = 17 order by id desc limit 1; + + set session time_zone='-03:00'; + insert into gh_ost_test values (null, 19, null, now(), now(), 0); + update gh_ost_test set ts2=now() + interval 10 minute, updated = 1 where i = 19 order by id desc limit 1; + + set session time_zone='+05:00'; + insert into gh_ost_test values (null, 23, null, now(), now(), 0); + update gh_ost_test set ts2=now() + interval 10 minute, updated = 1 where i = 23 order by id desc limit 1; +end ;; diff --git a/localtests/unsigned/create.sql b/localtests/unsigned/create.sql new file mode 100644 index 0000000..d98bb07 --- /dev/null +++ b/localtests/unsigned/create.sql @@ -0,0 +1,24 @@ +drop table if exists gh_ost_test; +create table gh_ost_test ( + id int auto_increment, + i int not null, + bi bigint not null, + iu int unsigned not null, + biu bigint unsigned not null, + primary key(id) +) auto_increment=1; + +drop event if exists gh_ost_test; +delimiter ;; +create event gh_ost_test + on schedule every 1 second + starts current_timestamp + ends current_timestamp + interval 60 second + on completion not preserve + enable + do +begin + insert into gh_ost_test values (null, -2147483647, -9223372036854775807, 4294967295, 18446744073709551615); + set @last_insert_id := cast(last_insert_id() as signed); + update gh_ost_test set i=-2147483647+@last_insert_id, bi=-9223372036854775807+@last_insert_id, iu=4294967295-@last_insert_id, biu=18446744073709551615-@last_insert_id where id < @last_insert_id order by id desc limit 1; +end ;; diff --git a/vendor/github.com/siddontang/go-mysql/replication/row_event.go b/vendor/github.com/siddontang/go-mysql/replication/row_event.go index 3fe8598..0e4eb17 100644 --- a/vendor/github.com/siddontang/go-mysql/replication/row_event.go +++ b/vendor/github.com/siddontang/go-mysql/replication/row_event.go @@ -609,7 +609,7 @@ func decodeTimestamp2(data []byte, dec uint16) (string, int, error) { return "0000-00-00 00:00:00", n, nil } - t := time.Unix(sec, usec*1000) + t := time.Unix(sec, usec*1000).UTC() return t.Format(TimeFormat), n, nil }