merged master

This commit is contained in:
Shlomi Noach 2016-08-25 12:32:03 +02:00
commit cb1a7e2805
17 changed files with 341 additions and 50 deletions

View File

@ -2,7 +2,7 @@
#
#
RELEASE_VERSION="1.0.10"
RELEASE_VERSION="1.0.13"
function build {
osname=$1

View File

@ -24,7 +24,7 @@ Both interfaces may serve at the same time. Both respond to simple text command,
- `critical-load=<load>`: change critical load setting (exceeding given thresholds causes panic and abort)
- `nice-ratio=<ratio>`: change _nice_ ratio: 0 for aggressive (not nice, not sleeping), positive integer `n`: for any `1ms` spent copying rows, spend `n*1ms` units of time sleeping. Examples: assume a single rows chunk copy takes `100ms` to complete. `nice-ratio=0.5` will cause `gh-ost` to sleep for `50ms` immediately following. `nice-ratio=1` will cause `gh-ost` to sleep for `100ms`, effectively doubling runtime; value of `2` will effectively triple the runtime; etc.
- `throttle-query`: change throttle query
- `throttle-control-replicas`: change list of throttle-control replicas, these are replicas `gh-ost` will check
- `throttle-control-replicas='replica1,replica2'`: change list of throttle-control replicas, these are replicas `gh-ost` will check. This takes a comma separated list of replica's to check and replaces the previous list.
- `throttle`: force migration suspend
- `no-throttle`: cancel forced suspension (though other throttling reasons may still apply)
- `unpostpone`: at a time where `gh-ost` is postponing the [cut-over](cut-over.md) phase, instruct `gh-ost` to stop postponing and proceed immediately to cut-over.

22
doc/local-tests.md Normal file
View File

@ -0,0 +1,22 @@
# Local tests
`gh-ost` is continuously tested in production via `--test-on-replica alter='engine=innodb'`. These tests check the GitHub workload and usage, but not necessarily the general case.
Local tests are an additional layer of tests. They will eventually be part of continuous integration tests.
Local tests test explicit use cases, such as column renames, mix of time zones, special types and alters. Traits of a single test:
- Composed of a single table.
- A single alter.
- By default the alter is `engine=innodb`, but this can be overridden per-test
- Scheduled DML operations, executed via `event_scheduler`.
- `gh-ost` is set to execute and throttle for `5` seconds, at which time all tested DMLs are expected to operate.
- The test requires a replication topology and utilizes `--test-on-replica`
- The test checksums the two tables (original and _ghost_) and expects identical checksum
- By default the test selects all (`*`) columns, but this can be overridden per-test
Tests are found under [localtests](https://github.com/github/gh-ost/tree/master/localtests). A single test is a subdirectory and tests are iterated alphabetically.
New data-integrity, synchronization issues or otherwise concerns are expected to be tested by new test cases.
While this is merged work is still ongoing.

View File

@ -84,13 +84,14 @@ type MigrationContext struct {
ServeSocketFile string
ServeTCPPort int64
Noop bool
TestOnReplica bool
MigrateOnReplica bool
OkToDropTable bool
InitiallyDropOldTable bool
InitiallyDropGhostTable bool
CutOverType CutOver
Noop bool
TestOnReplica bool
MigrateOnReplica bool
TestOnReplicaSkipReplicaStop bool
OkToDropTable bool
InitiallyDropOldTable bool
InitiallyDropGhostTable bool
CutOverType CutOver
Hostname string
TableEngine string

View File

@ -61,6 +61,7 @@ func main() {
executeFlag := flag.Bool("execute", false, "actually execute the alter & migrate the table. Default is noop: do some tests and exit")
flag.BoolVar(&migrationContext.TestOnReplica, "test-on-replica", false, "Have the migration run on a replica, not on the master. At the end of migration replication is stopped, and tables are swapped and immediately swap-revert. Replication remains stopped and you can compare the two tables for building trust")
flag.BoolVar(&migrationContext.TestOnReplicaSkipReplicaStop, "test-on-replica-skip-replica-stop", false, "When --test-on-replica is enabled, do not issue commands stop replication (requires --test-on-replica)")
flag.BoolVar(&migrationContext.MigrateOnReplica, "migrate-on-replica", false, "Have the migration run on a replica, not on the master. This will do the full migration on the replica including cut-over (as opposed to --test-on-replica)")
flag.BoolVar(&migrationContext.OkToDropTable, "ok-to-drop-table", false, "Shall the tool drop the old table at end of operation. DROPping tables can be a long locking operation, which is why I'm not doing it by default. I'm an online tool, yes?")
@ -149,6 +150,13 @@ func main() {
if migrationContext.SwitchToRowBinlogFormat && migrationContext.AssumeRBR {
log.Fatalf("--switch-to-rbr and --assume-rbr are mutually exclusive")
}
if migrationContext.TestOnReplicaSkipReplicaStop {
if !migrationContext.TestOnReplica {
log.Fatalf("--test-on-replica-skip-replica-stop requires --test-on-replica to be enabled")
}
log.Warning("--test-on-replica-skip-replica-stop enabled. We will not stop replication before cut-over. Ensure you have a plugin that does this.")
}
switch *cutOver {
case "atomic", "default", "":
migrationContext.CutOverType = base.CutOverAtomic

View File

@ -107,7 +107,7 @@ func (this *Applier) ValidateOrDropExistingTables() error {
}
}
if this.tableExists(this.migrationContext.GetGhostTableName()) {
return fmt.Errorf("Table %s already exists. Panicking. Use --initially-drop-ghost-table to force dropping it", sql.EscapeName(this.migrationContext.GetGhostTableName()))
return fmt.Errorf("Table %s already exists. Panicking. Use --initially-drop-ghost-table to force dropping it, though I really prefer that you drop it or rename it away", sql.EscapeName(this.migrationContext.GetGhostTableName()))
}
if this.migrationContext.InitiallyDropOldTable {
if err := this.DropOldTable(); err != nil {
@ -115,7 +115,7 @@ func (this *Applier) ValidateOrDropExistingTables() error {
}
}
if this.tableExists(this.migrationContext.GetOldTableName()) {
return fmt.Errorf("Table %s already exists. Panicking. Use --initially-drop-old-table to force dropping it", sql.EscapeName(this.migrationContext.GetOldTableName()))
return fmt.Errorf("Table %s already exists. Panicking. Use --initially-drop-old-table to force dropping it, though I really prefer that you drop it or rename it away", sql.EscapeName(this.migrationContext.GetOldTableName()))
}
return nil
@ -574,6 +574,7 @@ func (this *Applier) StopReplication() error {
if err := this.StopSlaveSQLThread(); err != nil {
return err
}
readBinlogCoordinates, executeBinlogCoordinates, err := mysql.GetReplicationBinlogCoordinates(this.db)
if err != nil {
return err
@ -832,12 +833,12 @@ func (this *Applier) buildDMLEventQuery(dmlEvent *binlog.BinlogDMLEvent) (query
}
case binlog.InsertDML:
{
query, sharedArgs, err := sql.BuildDMLInsertQuery(dmlEvent.DatabaseName, this.migrationContext.GetGhostTableName(), this.migrationContext.OriginalTableColumns, this.migrationContext.MappedSharedColumns, dmlEvent.NewColumnValues.AbstractValues())
query, sharedArgs, err := sql.BuildDMLInsertQuery(dmlEvent.DatabaseName, this.migrationContext.GetGhostTableName(), this.migrationContext.OriginalTableColumns, this.migrationContext.SharedColumns, this.migrationContext.MappedSharedColumns, dmlEvent.NewColumnValues.AbstractValues())
return query, sharedArgs, 1, err
}
case binlog.UpdateDML:
{
query, sharedArgs, uniqueKeyArgs, err := sql.BuildDMLUpdateQuery(dmlEvent.DatabaseName, this.migrationContext.GetGhostTableName(), this.migrationContext.OriginalTableColumns, this.migrationContext.MappedSharedColumns, &this.migrationContext.UniqueKey.Columns, dmlEvent.NewColumnValues.AbstractValues(), dmlEvent.WhereColumnValues.AbstractValues())
query, sharedArgs, uniqueKeyArgs, err := sql.BuildDMLUpdateQuery(dmlEvent.DatabaseName, this.migrationContext.GetGhostTableName(), this.migrationContext.OriginalTableColumns, this.migrationContext.SharedColumns, this.migrationContext.MappedSharedColumns, &this.migrationContext.UniqueKey.Columns, dmlEvent.NewColumnValues.AbstractValues(), dmlEvent.WhereColumnValues.AbstractValues())
args = append(args, sharedArgs...)
args = append(args, uniqueKeyArgs...)
return query, args, 0, err
@ -853,7 +854,6 @@ func (this *Applier) ApplyDMLEventQuery(dmlEvent *binlog.BinlogDMLEvent) error {
if err != nil {
return err
}
// TODO The below is in preparation for transactional writes on the ghost tables.
// Such writes would be, for example:
// - prepended with sql_mode setup
@ -871,6 +871,12 @@ func (this *Applier) ApplyDMLEventQuery(dmlEvent *binlog.BinlogDMLEvent) error {
if err != nil {
return err
}
if _, err := tx.Exec(`SET
SESSION time_zone = '+00:00',
sql_mode = CONCAT(@@session.sql_mode, ',STRICT_ALL_TABLES')
`); err != nil {
return err
}
if _, err := tx.Exec(query, args...); err != nil {
return err
}

View File

@ -363,7 +363,7 @@ func (this *Migrator) validateStatement() (err error) {
if this.parser.HasNonTrivialRenames() && !this.migrationContext.SkipRenamedColumns {
this.migrationContext.ColumnRenameMap = this.parser.GetNonTrivialRenames()
if !this.migrationContext.ApproveRenamedColumns {
return fmt.Errorf("Alter statement has column(s) renamed. gh-ost suspects the following renames: %v; but to proceed you must approve via `--approve-renamed-columns` (or you can skip renamed columns via `--skip-renamed-columns`)", this.parser.GetNonTrivialRenames())
return fmt.Errorf("gh-ost believes the ALTER statement renames columns, as follows: %v; as precation, you are asked to confirm gh-ost is correct, and provide with `--approve-renamed-columns`, and we're all happy. Or you can skip renamed columns via `--skip-renamed-columns`, in which case column data may be lost", this.parser.GetNonTrivialRenames())
}
log.Infof("Alter statement has column(s) renamed. gh-ost finds the following renames: %v; --approve-renamed-columns is given and so migration proceeds.", this.parser.GetNonTrivialRenames())
}
@ -402,7 +402,7 @@ func (this *Migrator) Migrate() (err error) {
return err
}
log.Debugf("Waiting for tables to be in place")
log.Infof("Waiting for tables to be in place")
<-this.tablesInPlace
log.Debugf("Tables are in place")
// Yay! We now know the Ghost and Changelog tables are good to examine!
@ -520,10 +520,14 @@ func (this *Migrator) cutOver() (err error) {
// the same cut-over phase as the master would use. That means we take locks
// and swap the tables.
// The difference is that we will later swap the tables back.
log.Debugf("testing on replica. Stopping replication IO thread")
this.hooksExecutor.onStopReplication()
if err := this.retryOperation(this.applier.StopReplication); err != nil {
return err
if this.migrationContext.TestOnReplicaSkipReplicaStop {
log.Warningf("--test-on-replica-skip-replica-stop enabled, we are not stopping replication.")
} else {
log.Debugf("testing on replica. Stopping replication IO thread")
if err := this.retryOperation(this.applier.StopReplication); err != nil {
return err
}
}
// We're merly testing, we don't want to keep this state. Rollback the renames as possible
defer this.applier.RenameTablesRollback()

View File

@ -354,7 +354,7 @@ func BuildDMLDeleteQuery(databaseName, tableName string, tableColumns, uniqueKey
return result, uniqueKeyArgs, nil
}
func BuildDMLInsertQuery(databaseName, tableName string, tableColumns, sharedColumns *ColumnList, args []interface{}) (result string, sharedArgs []interface{}, err error) {
func BuildDMLInsertQuery(databaseName, tableName string, tableColumns, sharedColumns, mappedSharedColumns *ColumnList, args []interface{}) (result string, sharedArgs []interface{}, err error) {
if len(args) != tableColumns.Len() {
return result, args, fmt.Errorf("args count differs from table column count in BuildDMLInsertQuery")
}
@ -367,17 +367,17 @@ func BuildDMLInsertQuery(databaseName, tableName string, tableColumns, sharedCol
databaseName = EscapeName(databaseName)
tableName = EscapeName(tableName)
for _, column := range sharedColumns.Names {
for _, column := range mappedSharedColumns.Names {
tableOrdinal := tableColumns.Ordinals[column]
arg := fixArgType(args[tableOrdinal], sharedColumns.IsUnsigned(column))
arg := fixArgType(args[tableOrdinal], mappedSharedColumns.IsUnsigned(column))
sharedArgs = append(sharedArgs, arg)
}
sharedColumnNames := duplicateNames(sharedColumns.Names)
for i := range sharedColumnNames {
sharedColumnNames[i] = EscapeName(sharedColumnNames[i])
mappedSharedColumnNames := duplicateNames(mappedSharedColumns.Names)
for i := range mappedSharedColumnNames {
mappedSharedColumnNames[i] = EscapeName(mappedSharedColumnNames[i])
}
preparedValues := buildPreparedValues(sharedColumns.Len())
preparedValues := buildPreparedValues(mappedSharedColumns.Len())
result = fmt.Sprintf(`
replace /* gh-ost %s.%s */ into
@ -387,13 +387,13 @@ func BuildDMLInsertQuery(databaseName, tableName string, tableColumns, sharedCol
(%s)
`, databaseName, tableName,
databaseName, tableName,
strings.Join(sharedColumnNames, ", "),
strings.Join(mappedSharedColumnNames, ", "),
strings.Join(preparedValues, ", "),
)
return result, sharedArgs, nil
}
func BuildDMLUpdateQuery(databaseName, tableName string, tableColumns, sharedColumns, uniqueKeyColumns *ColumnList, valueArgs, whereArgs []interface{}) (result string, sharedArgs, uniqueKeyArgs []interface{}, err error) {
func BuildDMLUpdateQuery(databaseName, tableName string, tableColumns, sharedColumns, mappedSharedColumns, uniqueKeyColumns *ColumnList, valueArgs, whereArgs []interface{}) (result string, sharedArgs, uniqueKeyArgs []interface{}, err error) {
if len(valueArgs) != tableColumns.Len() {
return result, sharedArgs, uniqueKeyArgs, fmt.Errorf("value args count differs from table column count in BuildDMLUpdateQuery")
}
@ -415,9 +415,10 @@ func BuildDMLUpdateQuery(databaseName, tableName string, tableColumns, sharedCol
databaseName = EscapeName(databaseName)
tableName = EscapeName(tableName)
for _, column := range sharedColumns.Names {
for i, column := range sharedColumns.Names {
mappedColumn := mappedSharedColumns.Names[i]
tableOrdinal := tableColumns.Ordinals[column]
arg := fixArgType(valueArgs[tableOrdinal], sharedColumns.IsUnsigned(column))
arg := fixArgType(valueArgs[tableOrdinal], mappedSharedColumns.IsUnsigned(mappedColumn))
sharedArgs = append(sharedArgs, arg)
}
@ -427,11 +428,11 @@ func BuildDMLUpdateQuery(databaseName, tableName string, tableColumns, sharedCol
uniqueKeyArgs = append(uniqueKeyArgs, arg)
}
sharedColumnNames := duplicateNames(sharedColumns.Names)
for i := range sharedColumnNames {
sharedColumnNames[i] = EscapeName(sharedColumnNames[i])
mappedSharedColumnNames := duplicateNames(mappedSharedColumns.Names)
for i := range mappedSharedColumnNames {
mappedSharedColumnNames[i] = EscapeName(mappedSharedColumnNames[i])
}
setClause, err := BuildSetPreparedClause(sharedColumnNames)
setClause, err := BuildSetPreparedClause(mappedSharedColumnNames)
equalsComparison, err := BuildEqualsPreparedComparison(uniqueKeyColumns.Names)
result = fmt.Sprintf(`

View File

@ -442,7 +442,7 @@ func TestBuildDMLInsertQuery(t *testing.T) {
args := []interface{}{3, "testname", "first", 17, 23}
{
sharedColumns := NewColumnList([]string{"id", "name", "position", "age"})
query, sharedArgs, err := BuildDMLInsertQuery(databaseName, tableName, tableColumns, sharedColumns, args)
query, sharedArgs, err := BuildDMLInsertQuery(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, args)
test.S(t).ExpectNil(err)
expected := `
replace /* gh-ost mydb.tbl */
@ -456,7 +456,7 @@ func TestBuildDMLInsertQuery(t *testing.T) {
}
{
sharedColumns := NewColumnList([]string{"position", "name", "age", "id"})
query, sharedArgs, err := BuildDMLInsertQuery(databaseName, tableName, tableColumns, sharedColumns, args)
query, sharedArgs, err := BuildDMLInsertQuery(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, args)
test.S(t).ExpectNil(err)
expected := `
replace /* gh-ost mydb.tbl */
@ -470,12 +470,12 @@ func TestBuildDMLInsertQuery(t *testing.T) {
}
{
sharedColumns := NewColumnList([]string{"position", "name", "surprise", "id"})
_, _, err := BuildDMLInsertQuery(databaseName, tableName, tableColumns, sharedColumns, args)
_, _, err := BuildDMLInsertQuery(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, args)
test.S(t).ExpectNotNil(err)
}
{
sharedColumns := NewColumnList([]string{})
_, _, err := BuildDMLInsertQuery(databaseName, tableName, tableColumns, sharedColumns, args)
_, _, err := BuildDMLInsertQuery(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, args)
test.S(t).ExpectNotNil(err)
}
}
@ -489,7 +489,7 @@ func TestBuildDMLInsertQuerySignedUnsigned(t *testing.T) {
// testing signed
args := []interface{}{3, "testname", "first", int8(-1), 23}
sharedColumns := NewColumnList([]string{"id", "name", "position", "age"})
query, sharedArgs, err := BuildDMLInsertQuery(databaseName, tableName, tableColumns, sharedColumns, args)
query, sharedArgs, err := BuildDMLInsertQuery(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, args)
test.S(t).ExpectNil(err)
expected := `
replace /* gh-ost mydb.tbl */
@ -505,7 +505,7 @@ func TestBuildDMLInsertQuerySignedUnsigned(t *testing.T) {
// testing unsigned
args := []interface{}{3, "testname", "first", int8(-1), 23}
sharedColumns.SetUnsigned("position")
query, sharedArgs, err := BuildDMLInsertQuery(databaseName, tableName, tableColumns, sharedColumns, args)
query, sharedArgs, err := BuildDMLInsertQuery(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, args)
test.S(t).ExpectNil(err)
expected := `
replace /* gh-ost mydb.tbl */
@ -521,7 +521,7 @@ func TestBuildDMLInsertQuerySignedUnsigned(t *testing.T) {
// testing unsigned
args := []interface{}{3, "testname", "first", int32(-1), 23}
sharedColumns.SetUnsigned("position")
query, sharedArgs, err := BuildDMLInsertQuery(databaseName, tableName, tableColumns, sharedColumns, args)
query, sharedArgs, err := BuildDMLInsertQuery(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, args)
test.S(t).ExpectNil(err)
expected := `
replace /* gh-ost mydb.tbl */
@ -544,7 +544,7 @@ func TestBuildDMLUpdateQuery(t *testing.T) {
{
sharedColumns := NewColumnList([]string{"id", "name", "position", "age"})
uniqueKeyColumns := NewColumnList([]string{"position"})
query, sharedArgs, uniqueKeyArgs, err := BuildDMLUpdateQuery(databaseName, tableName, tableColumns, sharedColumns, uniqueKeyColumns, valueArgs, whereArgs)
query, sharedArgs, uniqueKeyArgs, err := BuildDMLUpdateQuery(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, uniqueKeyColumns, valueArgs, whereArgs)
test.S(t).ExpectNil(err)
expected := `
update /* gh-ost mydb.tbl */
@ -560,7 +560,7 @@ func TestBuildDMLUpdateQuery(t *testing.T) {
{
sharedColumns := NewColumnList([]string{"id", "name", "position", "age"})
uniqueKeyColumns := NewColumnList([]string{"position", "name"})
query, sharedArgs, uniqueKeyArgs, err := BuildDMLUpdateQuery(databaseName, tableName, tableColumns, sharedColumns, uniqueKeyColumns, valueArgs, whereArgs)
query, sharedArgs, uniqueKeyArgs, err := BuildDMLUpdateQuery(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, uniqueKeyColumns, valueArgs, whereArgs)
test.S(t).ExpectNil(err)
expected := `
update /* gh-ost mydb.tbl */
@ -576,7 +576,7 @@ func TestBuildDMLUpdateQuery(t *testing.T) {
{
sharedColumns := NewColumnList([]string{"id", "name", "position", "age"})
uniqueKeyColumns := NewColumnList([]string{"age"})
query, sharedArgs, uniqueKeyArgs, err := BuildDMLUpdateQuery(databaseName, tableName, tableColumns, sharedColumns, uniqueKeyColumns, valueArgs, whereArgs)
query, sharedArgs, uniqueKeyArgs, err := BuildDMLUpdateQuery(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, uniqueKeyColumns, valueArgs, whereArgs)
test.S(t).ExpectNil(err)
expected := `
update /* gh-ost mydb.tbl */
@ -592,7 +592,7 @@ func TestBuildDMLUpdateQuery(t *testing.T) {
{
sharedColumns := NewColumnList([]string{"id", "name", "position", "age"})
uniqueKeyColumns := NewColumnList([]string{"age", "position", "id", "name"})
query, sharedArgs, uniqueKeyArgs, err := BuildDMLUpdateQuery(databaseName, tableName, tableColumns, sharedColumns, uniqueKeyColumns, valueArgs, whereArgs)
query, sharedArgs, uniqueKeyArgs, err := BuildDMLUpdateQuery(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, uniqueKeyColumns, valueArgs, whereArgs)
test.S(t).ExpectNil(err)
expected := `
update /* gh-ost mydb.tbl */
@ -608,15 +608,32 @@ func TestBuildDMLUpdateQuery(t *testing.T) {
{
sharedColumns := NewColumnList([]string{"id", "name", "position", "age"})
uniqueKeyColumns := NewColumnList([]string{"age", "surprise"})
_, _, _, err := BuildDMLUpdateQuery(databaseName, tableName, tableColumns, sharedColumns, uniqueKeyColumns, valueArgs, whereArgs)
_, _, _, err := BuildDMLUpdateQuery(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, uniqueKeyColumns, valueArgs, whereArgs)
test.S(t).ExpectNotNil(err)
}
{
sharedColumns := NewColumnList([]string{"id", "name", "position", "age"})
uniqueKeyColumns := NewColumnList([]string{})
_, _, _, err := BuildDMLUpdateQuery(databaseName, tableName, tableColumns, sharedColumns, uniqueKeyColumns, valueArgs, whereArgs)
_, _, _, err := BuildDMLUpdateQuery(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, uniqueKeyColumns, valueArgs, whereArgs)
test.S(t).ExpectNotNil(err)
}
{
sharedColumns := NewColumnList([]string{"id", "name", "position", "age"})
mappedColumns := NewColumnList([]string{"id", "name", "role", "age"})
uniqueKeyColumns := NewColumnList([]string{"id"})
query, sharedArgs, uniqueKeyArgs, err := BuildDMLUpdateQuery(databaseName, tableName, tableColumns, sharedColumns, mappedColumns, uniqueKeyColumns, valueArgs, whereArgs)
test.S(t).ExpectNil(err)
expected := `
update /* gh-ost mydb.tbl */
mydb.tbl
set id=?, name=?, role=?, age=?
where
((id = ?))
`
test.S(t).ExpectEquals(normalizeQuery(query), normalizeQuery(expected))
test.S(t).ExpectTrue(reflect.DeepEqual(sharedArgs, []interface{}{3, "testname", 17, 23}))
test.S(t).ExpectTrue(reflect.DeepEqual(uniqueKeyArgs, []interface{}{3}))
}
}
func TestBuildDMLUpdateQuerySignedUnsigned(t *testing.T) {
@ -629,7 +646,7 @@ func TestBuildDMLUpdateQuerySignedUnsigned(t *testing.T) {
uniqueKeyColumns := NewColumnList([]string{"position"})
{
// test signed
query, sharedArgs, uniqueKeyArgs, err := BuildDMLUpdateQuery(databaseName, tableName, tableColumns, sharedColumns, uniqueKeyColumns, valueArgs, whereArgs)
query, sharedArgs, uniqueKeyArgs, err := BuildDMLUpdateQuery(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, uniqueKeyColumns, valueArgs, whereArgs)
test.S(t).ExpectNil(err)
expected := `
update /* gh-ost mydb.tbl */
@ -646,7 +663,7 @@ func TestBuildDMLUpdateQuerySignedUnsigned(t *testing.T) {
// test unsigned
sharedColumns.SetUnsigned("age")
uniqueKeyColumns.SetUnsigned("position")
query, sharedArgs, uniqueKeyArgs, err := BuildDMLUpdateQuery(databaseName, tableName, tableColumns, sharedColumns, uniqueKeyColumns, valueArgs, whereArgs)
query, sharedArgs, uniqueKeyArgs, err := BuildDMLUpdateQuery(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, uniqueKeyColumns, valueArgs, whereArgs)
test.S(t).ExpectNil(err)
expected := `
update /* gh-ost mydb.tbl */

View File

@ -0,0 +1,27 @@
drop table if exists gh_ost_test;
create table gh_ost_test (
id int auto_increment,
i int not null,
e enum('red', 'green', 'blue', 'orange') null default null collate 'utf8_bin',
primary key(id)
) auto_increment=1;
drop event if exists gh_ost_test;
delimiter ;;
create event gh_ost_test
on schedule every 1 second
starts current_timestamp
ends current_timestamp + interval 60 second
on completion not preserve
enable
do
begin
insert into gh_ost_test values (null, 11, 'red');
insert into gh_ost_test values (null, 13, 'green');
insert into gh_ost_test values (null, 17, 'blue');
set @last_insert_id := last_insert_id();
update gh_ost_test set e='orange' where id = @last_insert_id;
insert into gh_ost_test values (null, 23, null);
set @last_insert_id := last_insert_id();
update gh_ost_test set i=i+1, e=null where id = @last_insert_id;
end ;;

View File

@ -0,0 +1 @@
--alter="change e e enum('red', 'green', 'blue', 'orange', 'yellow') null default null collate 'utf8_bin'"

View File

@ -0,0 +1,26 @@
drop table if exists gh_ost_test;
create table gh_ost_test (
id int auto_increment,
c1 int not null,
c2 int not null,
primary key (id)
) auto_increment=1;
drop event if exists gh_ost_test;
delimiter ;;
create event gh_ost_test
on schedule every 1 second
starts current_timestamp
ends current_timestamp + interval 60 second
on completion not preserve
enable
do
begin
insert ignore into gh_ost_test values (1, 11, 23);
insert ignore into gh_ost_test values (2, 13, 23);
insert into gh_ost_test values (null, 17, 23);
set @last_insert_id := last_insert_id();
update gh_ost_test set c1=c1+@last_insert_id, c2=c2+@last_insert_id where id=@last_insert_id order by id desc limit 1;
delete from gh_ost_test where id=1;
delete from gh_ost_test where c1=13; -- id=2
end ;;

View File

@ -0,0 +1 @@
--alter="change column c2 c3 int not null" --approve-renamed-columns

112
localtests/test.sh Executable file
View File

@ -0,0 +1,112 @@
#!/bin/bash
# Local integration tests. To be used by CI.
# See https://github.com/github/gh-ost/tree/doc/local-tests.md
#
tests_path=$(dirname $0)
test_logfile=/tmp/gh-ost-test.log
exec_command_file=/tmp/gh-ost-test.bash
master_host=
master_port=
replica_host=
replica_port=
verify_master_and_replica() {
if [ "$(gh-ost-test-mysql-master -e "select 1" -ss)" != "1" ] ; then
echo "Cannot verify gh-ost-test-mysql-master"
exit 1
fi
read master_host master_port <<< $(gh-ost-test-mysql-master -e "select @@hostname, @@port" -ss)
if [ "$(gh-ost-test-mysql-replica -e "select 1" -ss)" != "1" ] ; then
echo "Cannot verify gh-ost-test-mysql-replica"
exit 1
fi
read replica_host replica_port <<< $(gh-ost-test-mysql-replica -e "select @@hostname, @@port" -ss)
}
exec_cmd() {
echo "$@"
command "$@" 1> $test_logfile 2>&1
return $?
}
test_single() {
local test_name
test_name="$1"
echo "Testing: $test_name"
gh-ost-test-mysql-replica -e "start slave"
gh-ost-test-mysql-master test < $tests_path/$test_name/create.sql
extra_args=""
if [ -f $tests_path/$test_name/extra_args ] ; then
extra_args=$(cat $tests_path/$test_name/extra_args)
fi
columns="*"
if [ -f $tests_path/$test_name/test_columns ] ; then
columns=$(cat $tests_path/$test_name/test_columns)
fi
# graceful sleep for replica to catch up
sleep 1
#
cmd="go run go/cmd/gh-ost/main.go \
--user=gh-ost \
--password=gh-ost \
--host=$replica_host \
--port=$replica_port \
--database=test \
--table=gh_ost_test \
--alter='engine=innodb' \
--exact-rowcount \
--switch-to-rbr \
--initially-drop-old-table \
--initially-drop-ghost-table \
--throttle-query='select timestampdiff(second, min(last_update), now()) < 5 from _gh_ost_test_ghc' \
--serve-socket-file=/tmp/gh-ost.test.sock \
--initially-drop-socket-file \
--postpone-cut-over-flag-file=/tmp/gh-ost.postpone.flag \
--test-on-replica \
--default-retries=1 \
--verbose \
--debug \
--stack \
--execute ${extra_args[@]}"
echo $cmd > $exec_command_file
bash $exec_command_file 1> $test_logfile 2>&1
if [ $? -ne 0 ] ; then
echo "ERROR $test_name execution failure. See $test_logfile"
return 1
fi
orig_checksum=$(gh-ost-test-mysql-replica test -e "select ${columns} from gh_ost_test" -ss | md5sum)
ghost_checksum=$(gh-ost-test-mysql-replica test -e "select ${columns} from _gh_ost_test_gho" -ss | md5sum)
if [ "$orig_checksum" != "$ghost_checksum" ] ; then
echo "ERROR $test_name: checksum mismatch"
echo "---"
gh-ost-test-mysql-replica test -e "select ${columns} from gh_ost_test" -ss
echo "---"
gh-ost-test-mysql-replica test -e "select ${columns} from _gh_ost_test_gho" -ss
return 1
fi
}
test_all() {
find $tests_path ! -path . -type d -mindepth 1 -maxdepth 1 | cut -d "/" -f 3 | while read test_name ; do
test_single "$test_name"
if [ $? -ne 0 ] ; then
echo "+ FAIL"
return 1
else
echo "+ pass"
fi
gh-ost-test-mysql-replica -e "start slave"
done
}
verify_master_and_replica
test_all

41
localtests/tz/create.sql Normal file
View File

@ -0,0 +1,41 @@
drop table if exists gh_ost_test;
create table gh_ost_test (
id int auto_increment,
i int not null,
ts0 timestamp default current_timestamp,
ts1 timestamp,
ts2 timestamp,
updated tinyint unsigned default 0,
primary key(id),
key i_idx(i)
) auto_increment=1;
drop event if exists gh_ost_test;
delimiter ;;
create event gh_ost_test
on schedule every 1 second
starts current_timestamp
ends current_timestamp + interval 60 second
on completion not preserve
enable
do
begin
insert into gh_ost_test values (null, 11, null, now(), now(), 0);
update gh_ost_test set ts2=now() + interval 10 minute, updated = 1 where i = 11 order by id desc limit 1;
set session time_zone='system';
insert into gh_ost_test values (null, 13, null, now(), now(), 0);
update gh_ost_test set ts2=now() + interval 10 minute, updated = 1 where i = 13 order by id desc limit 1;
set session time_zone='+00:00';
insert into gh_ost_test values (null, 17, null, now(), now(), 0);
update gh_ost_test set ts2=now() + interval 10 minute, updated = 1 where i = 17 order by id desc limit 1;
set session time_zone='-03:00';
insert into gh_ost_test values (null, 19, null, now(), now(), 0);
update gh_ost_test set ts2=now() + interval 10 minute, updated = 1 where i = 19 order by id desc limit 1;
set session time_zone='+05:00';
insert into gh_ost_test values (null, 23, null, now(), now(), 0);
update gh_ost_test set ts2=now() + interval 10 minute, updated = 1 where i = 23 order by id desc limit 1;
end ;;

View File

@ -0,0 +1,24 @@
drop table if exists gh_ost_test;
create table gh_ost_test (
id int auto_increment,
i int not null,
bi bigint not null,
iu int unsigned not null,
biu bigint unsigned not null,
primary key(id)
) auto_increment=1;
drop event if exists gh_ost_test;
delimiter ;;
create event gh_ost_test
on schedule every 1 second
starts current_timestamp
ends current_timestamp + interval 60 second
on completion not preserve
enable
do
begin
insert into gh_ost_test values (null, -2147483647, -9223372036854775807, 4294967295, 18446744073709551615);
set @last_insert_id := cast(last_insert_id() as signed);
update gh_ost_test set i=-2147483647+@last_insert_id, bi=-9223372036854775807+@last_insert_id, iu=4294967295-@last_insert_id, biu=18446744073709551615-@last_insert_id where id < @last_insert_id order by id desc limit 1;
end ;;

View File

@ -609,7 +609,7 @@ func decodeTimestamp2(data []byte, dec uint16) (string, int, error) {
return "0000-00-00 00:00:00", n, nil
}
t := time.Unix(sec, usec*1000)
t := time.Unix(sec, usec*1000).UTC()
return t.Format(TimeFormat), n, nil
}