octoleo/gh-ost
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

beginning support for ranges and iteration. Still WIP

Browse Source
This commit is contained in:
Shlomi Noach 2016-04-05 09:14:22 +02:00
parent ea0906f4e5
commit 3583ab5dc5
5 changed files with 135 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
go/base/context.go
View File

@ -12,6 +12,7 @@ import (
"github.com/github/gh-osc/go/sql"
)
// RowsEstimateMethod is the type of row number estimation
type RowsEstimateMethod string
const (
@ -20,22 +21,28 @@ const (
CountRowsEstimate = "CountRowsEstimate"
)
// MigrationContext has the general, global state of migration. It is used by
// all components throughout the migration process.
type MigrationContext struct {
DatabaseName string
OriginalTableName string
AlterStatement string
TableEngine string
CountTableRows bool
RowsEstimate int64
UsedRowsEstimateMethod RowsEstimateMethod
ChunkSize int
OriginalBinlogFormat string
OriginalBinlogRowImage string
AllowedRunningOnMaster bool
InspectorConnectionConfig *mysql.ConnectionConfig
MasterConnectionConfig *mysql.ConnectionConfig
MigrationRangeMinValues *sql.ColumnValues
MigrationRangeMaxValues *sql.ColumnValues
DatabaseName string
OriginalTableName string
AlterStatement string
TableEngine string
CountTableRows bool
RowsEstimate int64
UsedRowsEstimateMethod RowsEstimateMethod
ChunkSize int
OriginalBinlogFormat string
OriginalBinlogRowImage string
AllowedRunningOnMaster bool
InspectorConnectionConfig *mysql.ConnectionConfig
MasterConnectionConfig *mysql.ConnectionConfig
MigrationRangeMinValues *sql.ColumnValues
MigrationRangeMaxValues *sql.ColumnValues
Iteration int64
MigrationIterationRangeMinValues *sql.ColumnValues
MigrationIterationRangeMaxValues *sql.ColumnValues
UniqueKey *sql.UniqueKey
}
var context *MigrationContext
@ -57,22 +64,24 @@ func GetMigrationContext() *MigrationContext {
return context
}
// GetGhostTableName
// GetGhostTableName generates the name of ghost table, based on original table name
func (this *MigrationContext) GetGhostTableName() string {
return fmt.Sprintf("_%s_New", this.OriginalTableName)
}
// RequiresBinlogFormatChange
// RequiresBinlogFormatChange is `true` when the original binlog format isn't `ROW`
func (this *MigrationContext) RequiresBinlogFormatChange() bool {
return this.OriginalBinlogFormat != "ROW"
}
// IsRunningOnMaster
// IsRunningOnMaster is `true` when the app connects directly to the master (typically
// it should be executed on replica and infer the master)
func (this *MigrationContext) IsRunningOnMaster() bool {
return this.InspectorConnectionConfig.Equals(this.MasterConnectionConfig)
}
// HasMigrationRange
// HasMigrationRange tells us whether there's a range to iterate for copying rows.
// It will be `false` if the table is initially empty
func (this *MigrationContext) HasMigrationRange() bool {
return this.MigrationRangeMinValues != nil && MigrationRangeMaxValues != nil
return this.MigrationRangeMinValues != nil && this.MigrationRangeMaxValues != nil
}

81
go/logic/applier.go
View File

@ -137,16 +137,91 @@ func (this *Applier) ReadMigrationMaxValues(uniqueKey *sql.UniqueKey) error {
return err
}
func (this *Applier) ReadMigrationRangeValues(uniqueKey *sql.UniqueKey) error {
if err := this.ReadMigrationMinValues(uniqueKey); err != nil {
func (this *Applier) ReadMigrationRangeValues() error {
if err := this.ReadMigrationMinValues(this.migrationContext.UniqueKey); err != nil {
return err
}
if err := this.ReadMigrationMaxValues(uniqueKey); err != nil {
if err := this.ReadMigrationMaxValues(this.migrationContext.UniqueKey); err != nil {
return err
}
return nil
}
// IterationIsComplete lets us know when the copy-iteration phase is complete, i.e.
// we've exhausted all rows
func (this *Applier) IterationIsComplete() (bool, error) {
if !this.migrationContext.HasMigrationRange() {
return false, nil
}
if this.migrationContext.MigrationIterationRangeMinValues == nil {
return false, nil
}
compareWithIterationRangeStart, err := sql.BuildRangePreparedComparison(this.migrationContext.UniqueKey.Columns, sql.GreaterThanOrEqualsComparisonSign)
if err != nil {
return false, err
}
compareWithRangeEnd, err := sql.BuildRangePreparedComparison(this.migrationContext.UniqueKey.Columns, sql.LessThanComparisonSign)
if err != nil {
return false, err
}
args := sqlutils.Args()
args = append(args, this.migrationContext.MigrationIterationRangeMinValues.AbstractValues()...)
args = append(args, this.migrationContext.MigrationRangeMaxValues.AbstractValues()...)
query := fmt.Sprintf(`
select /* gh-osc IterationIsComplete */ 1
from %s.%s
where (%s) and (%s)
limit 1
`,
sql.EscapeName(this.migrationContext.DatabaseName),
sql.EscapeName(this.migrationContext.OriginalTableName),
compareWithIterationRangeStart,
compareWithRangeEnd,
)
log.Debugf(query)
moreRowsFound := false
err = sqlutils.QueryRowsMap(this.db, query, func(rowMap sqlutils.RowMap) error {
moreRowsFound = true
return nil
}, args)
if err != nil {
return false, err
}
return !moreRowsFound, nil
}
func (this *Applier) CalculateNextIterationRangeEndValues() error {
query, err := sql.BuildUniqueKeyRangeEndPreparedQuery(this.migrationContext.DatabaseName, this.migrationContext.OriginalTableName, this.migrationContext.UniqueKey.Columns, this.migrationContext.ChunkSize)
if err != nil {
return err
}
startingFromValues := this.migrationContext.MigrationRangeMinValues
this.migrationContext.MigrationIterationRangeMinValues = this.migrationContext.MigrationIterationRangeMaxValues
if this.migrationContext.MigrationIterationRangeMinValues != nil {
startingFromValues = this.migrationContext.MigrationIterationRangeMinValues
}
rows, err := this.db.Query(query, startingFromValues.AbstractValues()...)
if err != nil {
return err
}
iterationRangeMaxValues := sql.NewColumnValues(len(this.migrationContext.UniqueKey.Columns))
iterationRangeEndFound := false
for rows.Next() {
if err = rows.Scan(iterationRangeMaxValues.ValuesPointers...); err != nil {
return err
}
iterationRangeEndFound = true
}
log.Debugf("5")
if !iterationRangeEndFound {
return fmt.Errorf("Cannot find iteration range end")
}
this.migrationContext.MigrationIterationRangeMaxValues = iterationRangeMaxValues
log.Debugf("column values: %s", this.migrationContext.MigrationIterationRangeMaxValues)
return nil
}
// IterateTable
func (this *Applier) IterateTable(uniqueKey *sql.UniqueKey) error {
query, err := sql.BuildUniqueKeyMinValuesPreparedQuery(this.migrationContext.DatabaseName, this.migrationContext.OriginalTableName, uniqueKey.Columns)

19
go/logic/migrator.go
View File

@ -55,10 +55,25 @@ func (this *Migrator) Migrate() (err error) {
// log.Errorf("Unable to ALTER ghost table, see further error details. Bailing out")
// return err
// }
if err := this.applier.ReadMigrationRangeValues(uniqueKeys[0]); err != nil {
this.migrationContext.UniqueKey = uniqueKeys[0]
this.migrationContext.ChunkSize = 100
if err := this.applier.ReadMigrationRangeValues(); err != nil {
return err
}
for {
log.Debugf("checking if complete")
isComplete, err := this.applier.IterationIsComplete()
if err != nil {
return err
}
if isComplete {
break
}
err = this.applier.CalculateNextIterationRangeEndValues()
if err != nil {
return err
}
}
if err := this.applier.IterateTable(uniqueKeys[0]); err != nil {
return err
}

12
go/sql/builder.go
View File

@ -121,7 +121,7 @@ func BuildRangePreparedComparison(columns []string, comparisonSign ValueComparis
return BuildRangeComparison(columns, values, comparisonSign)
}
func BuildRangeInsertQuery(databaseName, originalTableName, ghostTableName string, sharedColumns []string, uniqueKey string, uniqueKeyColumns, rangeStartValues, rangeEndValues []string) (string, error) {
func BuildRangeInsertQuery(databaseName, originalTableName, ghostTableName string, sharedColumns []string, uniqueKey string, uniqueKeyColumns, rangeStartValues, rangeEndValues []string, includeRangeStartValues bool) (string, error) {
if len(sharedColumns) == 0 {
return "", fmt.Errorf("Got 0 shared columns in BuildRangeInsertQuery")
}
@ -134,7 +134,11 @@ func BuildRangeInsertQuery(databaseName, originalTableName, ghostTableName strin
uniqueKey = EscapeName(uniqueKey)
sharedColumnsListing := strings.Join(sharedColumns, ", ")
rangeStartComparison, err := BuildRangeComparison(uniqueKeyColumns, rangeStartValues, GreaterThanOrEqualsComparisonSign)
var minRangeComparisonSign ValueComparisonSign = GreaterThanComparisonSign
if includeRangeStartValues {
minRangeComparisonSign = GreaterThanOrEqualsComparisonSign
}
rangeStartComparison, err := BuildRangeComparison(uniqueKeyColumns, rangeStartValues, minRangeComparisonSign)
if err != nil {
return "", err
}
@ -153,14 +157,14 @@ func BuildRangeInsertQuery(databaseName, originalTableName, ghostTableName strin
return query, nil
}
func BuildRangeInsertPreparedQuery(databaseName, originalTableName, ghostTableName string, sharedColumns []string, uniqueKey string, uniqueKeyColumns []string) (string, error) {
func BuildRangeInsertPreparedQuery(databaseName, originalTableName, ghostTableName string, sharedColumns []string, uniqueKey string, uniqueKeyColumns []string, includeRangeStartValues bool) (string, error) {
rangeStartValues := make([]string, len(uniqueKeyColumns), len(uniqueKeyColumns))
rangeEndValues := make([]string, len(uniqueKeyColumns), len(uniqueKeyColumns))
for i := range uniqueKeyColumns {
rangeStartValues[i] = "?"
rangeEndValues[i] = "?"
}
return BuildRangeInsertQuery(databaseName, originalTableName, ghostTableName, sharedColumns, uniqueKey, uniqueKeyColumns, rangeStartValues, rangeEndValues)
return BuildRangeInsertQuery(databaseName, originalTableName, ghostTableName, sharedColumns, uniqueKey, uniqueKeyColumns, rangeStartValues, rangeEndValues, includeRangeStartValues)
}
func BuildUniqueKeyRangeEndPreparedQuery(databaseName, tableName string, uniqueKeyColumns []string, chunkSize int) (string, error) {

6
go/sql/builder_test.go
View File

@ -128,7 +128,7 @@ func TestBuildRangeInsertQuery(t *testing.T) {
rangeStartValues := []string{"@v1s"}
rangeEndValues := []string{"@v1e"}
query, err := BuildRangeInsertQuery(databaseName, originalTableName, ghostTableName, sharedColumns, uniqueKey, uniqueKeyColumns, rangeStartValues, rangeEndValues)
query, err := BuildRangeInsertQuery(databaseName, originalTableName, ghostTableName, sharedColumns, uniqueKey, uniqueKeyColumns, rangeStartValues, rangeEndValues, true)
test.S(t).ExpectNil(err)
expected := `
insert /* gh-osc mydb.tbl */ ignore into mydb.ghost (id, name, position)
@ -144,7 +144,7 @@ func TestBuildRangeInsertQuery(t *testing.T) {
rangeStartValues := []string{"@v1s", "@v2s"}
rangeEndValues := []string{"@v1e", "@v2e"}
query, err := BuildRangeInsertQuery(databaseName, originalTableName, ghostTableName, sharedColumns, uniqueKey, uniqueKeyColumns, rangeStartValues, rangeEndValues)
query, err := BuildRangeInsertQuery(databaseName, originalTableName, ghostTableName, sharedColumns, uniqueKey, uniqueKeyColumns, rangeStartValues, rangeEndValues, true)
test.S(t).ExpectNil(err)
expected := `
insert /* gh-osc mydb.tbl */ ignore into mydb.ghost (id, name, position)
@ -165,7 +165,7 @@ func TestBuildRangeInsertPreparedQuery(t *testing.T) {
uniqueKey := "name_position_uidx"
uniqueKeyColumns := []string{"name", "position"}
query, err := BuildRangeInsertPreparedQuery(databaseName, originalTableName, ghostTableName, sharedColumns, uniqueKey, uniqueKeyColumns)
query, err := BuildRangeInsertPreparedQuery(databaseName, originalTableName, ghostTableName, sharedColumns, uniqueKey, uniqueKeyColumns, true)
test.S(t).ExpectNil(err)
expected := `
insert /* gh-osc mydb.tbl */ ignore into mydb.ghost (id, name, position)