Merge branch 'master' into json-57-test

go/logic/applier.go

@@ -398,35 +398,41 @@ func (this *Applier) CalculateNextIterationRangeEndValues() (hasFurtherRange boo
 	if this.migrationContext.MigrationIterationRangeMinValues == nil {
 		this.migrationContext.MigrationIterationRangeMinValues = this.migrationContext.MigrationRangeMinValues
 	}
-	query, explodedArgs, err := sql.BuildUniqueKeyRangeEndPreparedQuery(
+	for i := 0; i < 2; i++ {
-		this.migrationContext.DatabaseName,
+		buildFunc := sql.BuildUniqueKeyRangeEndPreparedQueryViaOffset
-		this.migrationContext.OriginalTableName,
+		if i == 1 {
-		&this.migrationContext.UniqueKey.Columns,
+			buildFunc = sql.BuildUniqueKeyRangeEndPreparedQueryViaTemptable
-		this.migrationContext.MigrationIterationRangeMinValues.AbstractValues(),
+		}
-		this.migrationContext.MigrationRangeMaxValues.AbstractValues(),
+		query, explodedArgs, err := buildFunc(
-		atomic.LoadInt64(&this.migrationContext.ChunkSize),
+			this.migrationContext.DatabaseName,
-		this.migrationContext.GetIteration() == 0,
+			this.migrationContext.OriginalTableName,
-		fmt.Sprintf("iteration:%d", this.migrationContext.GetIteration()),
+			&this.migrationContext.UniqueKey.Columns,
-	)
+			this.migrationContext.MigrationIterationRangeMinValues.AbstractValues(),
-	if err != nil {
+			this.migrationContext.MigrationRangeMaxValues.AbstractValues(),
-		return hasFurtherRange, err
+			atomic.LoadInt64(&this.migrationContext.ChunkSize),
-	}
+			this.migrationContext.GetIteration() == 0,
-	rows, err := this.db.Query(query, explodedArgs...)
+			fmt.Sprintf("iteration:%d", this.migrationContext.GetIteration()),
-	if err != nil {
+		)
-		return hasFurtherRange, err
+		if err != nil {
-	}
-	iterationRangeMaxValues := sql.NewColumnValues(this.migrationContext.UniqueKey.Len())
-	for rows.Next() {
-		if err = rows.Scan(iterationRangeMaxValues.ValuesPointers...); err != nil {
 			return hasFurtherRange, err
 		}
-		hasFurtherRange = true
+		rows, err := this.db.Query(query, explodedArgs...)
+		if err != nil {
+			return hasFurtherRange, err
+		}
+		iterationRangeMaxValues := sql.NewColumnValues(this.migrationContext.UniqueKey.Len())
+		for rows.Next() {
+			if err = rows.Scan(iterationRangeMaxValues.ValuesPointers...); err != nil {
+				return hasFurtherRange, err
+			}
+			hasFurtherRange = true
+		}
+		if hasFurtherRange {
+			this.migrationContext.MigrationIterationRangeMaxValues = iterationRangeMaxValues
+			return hasFurtherRange, nil
+		}
 	}
-	if !hasFurtherRange {
+	log.Debugf("Iteration complete: no further range to iterate")
-		log.Debugf("Iteration complete: no further range to iterate")
-		return hasFurtherRange, nil
-	}
-	this.migrationContext.MigrationIterationRangeMaxValues = iterationRangeMaxValues
 	return hasFurtherRange, nil
 }
 

go/sql/builder.go

@@ -235,7 +235,62 @@ func BuildRangeInsertPreparedQuery(databaseName, originalTableName, ghostTableNa
 	return BuildRangeInsertQuery(databaseName, originalTableName, ghostTableName, sharedColumns, mappedSharedColumns, uniqueKey, uniqueKeyColumns, rangeStartValues, rangeEndValues, rangeStartArgs, rangeEndArgs, includeRangeStartValues, transactionalTable)
 }
 
-func BuildUniqueKeyRangeEndPreparedQuery(databaseName, tableName string, uniqueKeyColumns *ColumnList, rangeStartArgs, rangeEndArgs []interface{}, chunkSize int64, includeRangeStartValues bool, hint string) (result string, explodedArgs []interface{}, err error) {
+func BuildUniqueKeyRangeEndPreparedQueryViaOffset(databaseName, tableName string, uniqueKeyColumns *ColumnList, rangeStartArgs, rangeEndArgs []interface{}, chunkSize int64, includeRangeStartValues bool, hint string) (result string, explodedArgs []interface{}, err error) {
+	if uniqueKeyColumns.Len() == 0 {
+		return "", explodedArgs, fmt.Errorf("Got 0 columns in BuildUniqueKeyRangeEndPreparedQuery")
+	}
+	databaseName = EscapeName(databaseName)
+	tableName = EscapeName(tableName)
+
+	var startRangeComparisonSign ValueComparisonSign = GreaterThanComparisonSign
+	if includeRangeStartValues {
+		startRangeComparisonSign = GreaterThanOrEqualsComparisonSign
+	}
+	rangeStartComparison, rangeExplodedArgs, err := BuildRangePreparedComparison(uniqueKeyColumns, rangeStartArgs, startRangeComparisonSign)
+	if err != nil {
+		return "", explodedArgs, err
+	}
+	explodedArgs = append(explodedArgs, rangeExplodedArgs...)
+	rangeEndComparison, rangeExplodedArgs, err := BuildRangePreparedComparison(uniqueKeyColumns, rangeEndArgs, LessThanOrEqualsComparisonSign)
+	if err != nil {
+		return "", explodedArgs, err
+	}
+	explodedArgs = append(explodedArgs, rangeExplodedArgs...)
+
+	uniqueKeyColumnNames := duplicateNames(uniqueKeyColumns.Names())
+	uniqueKeyColumnAscending := make([]string, len(uniqueKeyColumnNames), len(uniqueKeyColumnNames))
+	uniqueKeyColumnDescending := make([]string, len(uniqueKeyColumnNames), len(uniqueKeyColumnNames))
+	for i, column := range uniqueKeyColumns.Columns() {
+		uniqueKeyColumnNames[i] = EscapeName(uniqueKeyColumnNames[i])
+		if column.Type == EnumColumnType {
+			uniqueKeyColumnAscending[i] = fmt.Sprintf("concat(%s) asc", uniqueKeyColumnNames[i])
+			uniqueKeyColumnDescending[i] = fmt.Sprintf("concat(%s) desc", uniqueKeyColumnNames[i])
+		} else {
+			uniqueKeyColumnAscending[i] = fmt.Sprintf("%s asc", uniqueKeyColumnNames[i])
+			uniqueKeyColumnDescending[i] = fmt.Sprintf("%s desc", uniqueKeyColumnNames[i])
+		}
+	}
+	result = fmt.Sprintf(`
+				select  /* gh-ost %s.%s %s */
+						%s
+					from
+						%s.%s
+					where %s and %s
+					order by
+						%s
+					limit 1
+					offset %d
+    `, databaseName, tableName, hint,
+		strings.Join(uniqueKeyColumnNames, ", "),
+		databaseName, tableName,
+		rangeStartComparison, rangeEndComparison,
+		strings.Join(uniqueKeyColumnAscending, ", "),
+		(chunkSize - 1),
+	)
+	return result, explodedArgs, nil
+}
+
+func BuildUniqueKeyRangeEndPreparedQueryViaTemptable(databaseName, tableName string, uniqueKeyColumns *ColumnList, rangeStartArgs, rangeEndArgs []interface{}, chunkSize int64, includeRangeStartValues bool, hint string) (result string, explodedArgs []interface{}, err error) {
 	if uniqueKeyColumns.Len() == 0 {
 		return "", explodedArgs, fmt.Errorf("Got 0 columns in BuildUniqueKeyRangeEndPreparedQuery")
 	}

go/sql/builder_test.go

@@ -283,7 +283,7 @@ func TestBuildUniqueKeyRangeEndPreparedQuery(t *testing.T) {
 		rangeStartArgs := []interface{}{3, 17}
 		rangeEndArgs := []interface{}{103, 117}
 
-		query, explodedArgs, err := BuildUniqueKeyRangeEndPreparedQuery(databaseName, originalTableName, uniqueKeyColumns, rangeStartArgs, rangeEndArgs, chunkSize, false, "test")
+		query, explodedArgs, err := BuildUniqueKeyRangeEndPreparedQueryViaTemptable(databaseName, originalTableName, uniqueKeyColumns, rangeStartArgs, rangeEndArgs, chunkSize, false, "test")
 		test.S(t).ExpectNil(err)
 		expected := `
 				select /* gh-ost mydb.tbl test */ name, position

localtests/test.sh

@@ -91,6 +91,7 @@ test_single() {
     --postpone-cut-over-flag-file=/tmp/gh-ost.test.postpone.flag \
     --test-on-replica \
     --default-retries=1 \
+    --chunk-size=10 \
     --verbose \
     --debug \
     --stack \