Merge branch 'master' into master

This commit is contained in:
Tim Vaillancourt 2023-01-28 00:19:37 +01:00 committed by GitHub
commit bea5323816
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
25 changed files with 209 additions and 30 deletions

View File

@ -5,7 +5,7 @@ on: [pull_request]
jobs:
build:
runs-on: ubuntu-latest
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v2

View File

@ -5,10 +5,10 @@ on: [pull_request]
jobs:
build:
runs-on: ubuntu-latest
runs-on: ubuntu-20.04
strategy:
matrix:
version: [mysql-5.7.25,mysql-8.0.16]
version: [mysql-5.7.25,mysql-8.0.16,PerconaServer-8.0.21]
steps:
- uses: actions/checkout@v2

View File

@ -45,6 +45,22 @@ If you happen to _know_ your servers use RBR (Row Based Replication, i.e. `binlo
Skipping this step means `gh-ost` would not need the `SUPER` privilege in order to operate.
You may want to use this on Amazon RDS.
### attempt-instant-ddl
MySQL 8.0 supports "instant DDL" for some operations. If an alter statement can be completed with instant DDL, only a metadata change is required internally. Instant operations include:
- Adding a column
- Dropping a column
- Dropping an index
- Extending a varchar column
- Adding a virtual generated column
It is not reliable to parse the `ALTER` statement to determine if it is instant or not. This is because the table might be in an older row format, or have some other incompatibility that is difficult to identify.
`--attempt-instant-ddl` is disabled by default, but the risks of enabling it are relatively minor: `gh-ost` may need to acquire a metadata lock at the start of the operation. This is not a problem for most scenarios, but it could be a problem for users that start the DDL during a period with long running transactions.
`gh-ost` will automatically fallback to the normal DDL process if the attempt to use instant DDL is unsuccessful.
### conf
`--conf=/path/to/my.cnf`: file where credentials are specified. Should be in (or contain) the following format:
@ -230,6 +246,18 @@ Allows `gh-ost` to connect to the MySQL servers using encrypted connections, but
`--ssl-key=/path/to/ssl-key.key`: SSL private key file (in PEM format).
### storage-engine
Default is `innodb`, and `rocksdb` support is currently experimental. InnoDB and RocksDB are both transactional engines, supporting both shared and exclusive row locks.
But RocksDB currently lacks a few features support compared to InnoDB:
- Gap Locks
- Foreign Key
- Generated Columns
- Spatial
- Geometry
When `--storage-engine=rocksdb`, `gh-ost` will make some changes necessary (e.g. sets isolation level to `READ_COMMITTED`) to support RocksDB.
### test-on-replica
Issue the migration on a replica; do not modify data on master. Useful for validating, testing and benchmarking. See [`testing-on-replica`](testing-on-replica.md)

View File

@ -29,7 +29,7 @@ CREATE TABLE tbl (
(This is also the definition of the _ghost_ table, except that that table would be called `_tbl_gho`).
In this migration, the _before_ and _after_ versions contain the same unique not-null key (the PRIMARY KEY). To run this migration, `gh-ost` would iterate through the `tbl` table using the primary key, copy rows from `tbl` to the _ghost_ table `_tbl_gho` in primary key order, while also applying the binlog event writes from `tble` onto `_tbl_gho`.
In this migration, the _before_ and _after_ versions contain the same unique not-null key (the PRIMARY KEY). To run this migration, `gh-ost` would iterate through the `tbl` table using the primary key, copy rows from `tbl` to the _ghost_ table `_tbl_gho` in primary key order, while also applying the binlog event writes from `tbl` onto `_tbl_gho`.
The applying of the binlog events is what requires the shared unique key. For example, an `UPDATE` statement to `tbl` translates to a `REPLACE` statement which `gh-ost` applies to `_tbl_gho`. A `REPLACE` statement expects to insert or replace an existing row based on its row's values and the table's unique key constraints. In particular, if inserting that row would result in a unique key violation (e.g., a row with that primary key already exists), it would _replace_ that existing row with the new values.

View File

@ -101,6 +101,7 @@ type MigrationContext struct {
AliyunRDS bool
GoogleCloudPlatform bool
AzureMySQL bool
AttemptInstantDDL bool
config ContextConfig
configMutex *sync.Mutex
@ -289,6 +290,19 @@ func NewMigrationContext() *MigrationContext {
}
}
func (this *MigrationContext) SetConnectionConfig(storageEngine string) error {
var transactionIsolation string
switch storageEngine {
case "rocksdb":
transactionIsolation = "READ-COMMITTED"
default:
transactionIsolation = "REPEATABLE-READ"
}
this.InspectorConnectionConfig.TransactionIsolation = transactionIsolation
this.ApplierConnectionConfig.TransactionIsolation = transactionIsolation
return nil
}
func getSafeTableName(baseName string, suffix string) string {
name := fmt.Sprintf("_%s_%s", baseName, suffix)
if len(name) <= mysql.MaxTableNameLength {
@ -427,6 +441,10 @@ func (this *MigrationContext) IsTransactionalTable() bool {
{
return true
}
case "rocksdb":
{
return true
}
}
return false
}

View File

@ -67,6 +67,9 @@ func main() {
flag.StringVar(&migrationContext.DatabaseName, "database", "", "database name (mandatory)")
flag.StringVar(&migrationContext.OriginalTableName, "table", "", "table name (mandatory)")
flag.StringVar(&migrationContext.AlterStatement, "alter", "", "alter statement (mandatory)")
flag.BoolVar(&migrationContext.AttemptInstantDDL, "attempt-instant-ddl", false, "Attempt to use instant DDL for this migration first")
storageEngine := flag.String("storage-engine", "innodb", "Specify table storage engine (default: 'innodb'). When 'rocksdb': the session transaction isolation level is changed from REPEATABLE_READ to READ_COMMITTED.")
flag.BoolVar(&migrationContext.CountTableRows, "exact-rowcount", false, "actually count table rows as opposed to estimate them (results in more accurate progress estimation)")
flag.BoolVar(&migrationContext.ConcurrentCountTableRows, "concurrent-rowcount", true, "(with --exact-rowcount), when true (default): count rows after row-copy begins, concurrently, and adjust row estimate later on; when false: first count rows, then start row copy")
flag.BoolVar(&migrationContext.AllowedRunningOnMaster, "allow-on-master", false, "allow this migration to run directly on master. Preferably it would run on a replica")
@ -180,6 +183,10 @@ func main() {
migrationContext.Log.SetLevel(log.ERROR)
}
if err := migrationContext.SetConnectionConfig(*storageEngine); err != nil {
migrationContext.Log.Fatale(err)
}
if migrationContext.AlterStatement == "" {
log.Fatal("--alter must be provided and statement must not be empty")
}
@ -245,6 +252,9 @@ func main() {
if *replicationLagQuery != "" {
migrationContext.Log.Warning("--replication-lag-query is deprecated")
}
if *storageEngine == "rocksdb" {
migrationContext.Log.Warning("RocksDB storage engine support is experimental")
}
switch *cutOver {
case "atomic", "default", "":

View File

@ -134,6 +134,16 @@ func (this *Applier) generateSqlModeQuery() string {
return fmt.Sprintf("sql_mode = %s", sqlModeQuery)
}
// generateInstantDDLQuery returns the SQL for this ALTER operation
// with an INSTANT assertion (requires MySQL 8.0+)
func (this *Applier) generateInstantDDLQuery() string {
return fmt.Sprintf(`ALTER /* gh-ost */ TABLE %s.%s %s, ALGORITHM=INSTANT`,
sql.EscapeName(this.migrationContext.DatabaseName),
sql.EscapeName(this.migrationContext.OriginalTableName),
this.migrationContext.AlterStatementOptions,
)
}
// readTableColumns reads table columns on applier
func (this *Applier) readTableColumns() (err error) {
this.migrationContext.Log.Infof("Examining table structure on applier")
@ -187,6 +197,27 @@ func (this *Applier) ValidateOrDropExistingTables() error {
return nil
}
// AttemptInstantDDL attempts to use instant DDL (from MySQL 8.0, and earlier in Aurora and some others).
// If successful, the operation is only a meta-data change so a lot of time is saved!
// The risk of attempting to instant DDL when not supported is that a metadata lock may be acquired.
// This is minor, since gh-ost will eventually require a metadata lock anyway, but at the cut-over stage.
// Instant operations include:
// - Adding a column
// - Dropping a column
// - Dropping an index
// - Extending a VARCHAR column
// - Adding a virtual generated column
// It is not reliable to parse the `alter` statement to determine if it is instant or not.
// This is because the table might be in an older row format, or have some other incompatibility
// that is difficult to identify.
func (this *Applier) AttemptInstantDDL() error {
query := this.generateInstantDDLQuery()
this.migrationContext.Log.Infof("INSTANT DDL query is: %s", query)
// We don't need a trx, because for instant DDL the SQL mode doesn't matter.
_, err := this.db.Exec(query)
return err
}
// CreateGhostTable creates the ghost table on the applier host
func (this *Applier) CreateGhostTable() error {
query := fmt.Sprintf(`create /* gh-ost */ table %s.%s like %s.%s`,

View File

@ -170,3 +170,16 @@ func TestApplierBuildDMLEventQuery(t *testing.T) {
test.S(t).ExpectEquals(res[0].args[3], 42)
})
}
func TestApplierInstantDDL(t *testing.T) {
migrationContext := base.NewMigrationContext()
migrationContext.DatabaseName = "test"
migrationContext.OriginalTableName = "mytable"
migrationContext.AlterStatementOptions = "ADD INDEX (foo)"
applier := NewApplier(migrationContext)
t.Run("instantDDLstmt", func(t *testing.T) {
stmt := applier.generateInstantDDLQuery()
test.S(t).ExpectEquals(stmt, "ALTER /* gh-ost */ TABLE `test`.`mytable` ADD INDEX (foo), ALGORITHM=INSTANT")
})
}

View File

@ -359,6 +359,17 @@ func (this *Migrator) Migrate() (err error) {
if err := this.createFlagFiles(); err != nil {
return err
}
// In MySQL 8.0 (and possibly earlier) some DDL statements can be applied instantly.
// Attempt to do this if AttemptInstantDDL is set.
if this.migrationContext.AttemptInstantDDL {
this.migrationContext.Log.Infof("Attempting to execute alter with ALGORITHM=INSTANT")
if err := this.applier.AttemptInstantDDL(); err == nil {
this.migrationContext.Log.Infof("Success! table %s.%s migrated instantly", sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName))
return nil
} else {
this.migrationContext.Log.Infof("ALGORITHM=INSTANT not supported for this operation, proceeding with original algorithm: %s", err)
}
}
initialLag, _ := this.inspector.getReplicationLag()
this.migrationContext.Log.Infof("Waiting for ghost table to be migrated. Current lag is %+v", initialLag)
@ -1031,6 +1042,7 @@ func (this *Migrator) printStatus(rule PrintStatusRule, writers ...io.Writer) {
)
w := io.MultiWriter(writers...)
fmt.Fprintln(w, status)
this.migrationContext.Log.Infof(status)
hooksStatusIntervalSec := this.migrationContext.HooksStatusIntervalSec
if hooksStatusIntervalSec > 0 && elapsedSeconds%hooksStatusIntervalSec == 0 {

View File

@ -18,7 +18,6 @@ import (
)
const (
transactionIsolation = "REPEATABLE-READ"
TLS_CONFIG_KEY = "ghost"
)
@ -30,6 +29,7 @@ type ConnectionConfig struct {
ImpliedKey *InstanceKey
tlsConfig *tls.Config
Timeout float64
TransactionIsolation string
}
func NewConnectionConfig() *ConnectionConfig {
@ -48,6 +48,7 @@ func (this *ConnectionConfig) DuplicateCredentials(key InstanceKey) *ConnectionC
Password: this.Password,
tlsConfig: this.tlsConfig,
Timeout: this.Timeout,
TransactionIsolation: this.TransactionIsolation,
}
config.ImpliedKey = &config.Key
return config
@ -126,7 +127,7 @@ func (this *ConnectionConfig) GetDBUri(databaseName string) string {
"charset=utf8mb4,utf8,latin1",
"interpolateParams=true",
fmt.Sprintf("tls=%s", tlsOption),
fmt.Sprintf("transaction_isolation=%q", transactionIsolation),
fmt.Sprintf("transaction_isolation=%q", this.TransactionIsolation),
fmt.Sprintf("timeout=%fs", this.Timeout),
fmt.Sprintf("readTimeout=%fs", this.Timeout),
fmt.Sprintf("writeTimeout=%fs", this.Timeout),

View File

@ -13,6 +13,10 @@ import (
test "github.com/openark/golib/tests"
)
const (
transactionIsolation = "REPEATABLE-READ"
)
func init() {
log.SetLevel(log.ERROR)
}
@ -25,6 +29,7 @@ func TestNewConnectionConfig(t *testing.T) {
test.S(t).ExpectEquals(c.ImpliedKey.Port, 0)
test.S(t).ExpectEquals(c.User, "")
test.S(t).ExpectEquals(c.Password, "")
test.S(t).ExpectEquals(c.TransactionIsolation, "")
}
func TestDuplicateCredentials(t *testing.T) {
@ -36,6 +41,7 @@ func TestDuplicateCredentials(t *testing.T) {
InsecureSkipVerify: true,
ServerName: "feathers",
}
c.TransactionIsolation = transactionIsolation
dup := c.DuplicateCredentials(InstanceKey{Hostname: "otherhost", Port: 3310})
test.S(t).ExpectEquals(dup.Key.Hostname, "otherhost")
@ -45,6 +51,7 @@ func TestDuplicateCredentials(t *testing.T) {
test.S(t).ExpectEquals(dup.User, "gromit")
test.S(t).ExpectEquals(dup.Password, "penguin")
test.S(t).ExpectEquals(dup.tlsConfig, c.tlsConfig)
test.S(t).ExpectEquals(dup.TransactionIsolation, c.TransactionIsolation)
}
func TestDuplicate(t *testing.T) {
@ -52,6 +59,7 @@ func TestDuplicate(t *testing.T) {
c.Key = InstanceKey{Hostname: "myhost", Port: 3306}
c.User = "gromit"
c.Password = "penguin"
c.TransactionIsolation = transactionIsolation
dup := c.Duplicate()
test.S(t).ExpectEquals(dup.Key.Hostname, "myhost")
@ -60,6 +68,7 @@ func TestDuplicate(t *testing.T) {
test.S(t).ExpectEquals(dup.ImpliedKey.Port, 3306)
test.S(t).ExpectEquals(dup.User, "gromit")
test.S(t).ExpectEquals(dup.Password, "penguin")
test.S(t).ExpectEquals(dup.TransactionIsolation, transactionIsolation)
}
func TestGetDBUri(t *testing.T) {
@ -68,6 +77,7 @@ func TestGetDBUri(t *testing.T) {
c.User = "gromit"
c.Password = "penguin"
c.Timeout = 1.2345
c.TransactionIsolation = transactionIsolation
uri := c.GetDBUri("test")
test.S(t).ExpectEquals(uri, `gromit:penguin@tcp(myhost:3306)/test?autocommit=true&charset=utf8mb4,utf8,latin1&interpolateParams=true&tls=false&transaction_isolation="REPEATABLE-READ"&timeout=1.234500s&readTimeout=1.234500s&writeTimeout=1.234500s`)
@ -80,6 +90,7 @@ func TestGetDBUriWithTLSSetup(t *testing.T) {
c.Password = "penguin"
c.Timeout = 1.2345
c.tlsConfig = &tls.Config{}
c.TransactionIsolation = transactionIsolation
uri := c.GetDBUri("test")
test.S(t).ExpectEquals(uri, `gromit:penguin@tcp(myhost:3306)/test?autocommit=true&charset=utf8mb4,utf8,latin1&interpolateParams=true&tls=ghost&transaction_isolation="REPEATABLE-READ"&timeout=1.234500s&readTimeout=1.234500s&writeTimeout=1.234500s`)

View File

@ -0,0 +1,13 @@
drop table if exists gh_ost_test;
create table gh_ost_test (
id int auto_increment,
i int not null,
color varchar(32),
primary key(id)
) auto_increment=1;
drop event if exists gh_ost_test;
insert into gh_ost_test values (null, 11, 'red');
insert into gh_ost_test values (null, 13, 'green');
insert into gh_ost_test values (null, 17, 'blue');

View File

@ -0,0 +1 @@
--attempt-instant-ddl

View File

@ -7,9 +7,6 @@ create table gh_ost_test (
primary key(id)
) auto_increment=1;
insert into gh_ost_test values (null, 'átesting');
insert into gh_ost_test values (null, 'Hello world, Καλημέρα κόσμε, コンニチハ', 'átesting0', 'initial');
drop event if exists gh_ost_test;

View File

@ -0,0 +1 @@
Percona

View File

@ -0,0 +1 @@
Percona

View File

@ -0,0 +1 @@
Percona

View File

@ -0,0 +1 @@
Percona

View File

@ -0,0 +1 @@
Percona

View File

@ -0,0 +1 @@
Percona

View File

@ -0,0 +1 @@
Percona

View File

@ -0,0 +1 @@
Percona

View File

@ -0,0 +1 @@
Percona

View File

@ -11,6 +11,7 @@ tests_path=$(dirname $0)
test_logfile=/tmp/gh-ost-test.log
default_ghost_binary=/tmp/gh-ost-test
ghost_binary=""
storage_engine=innodb
exec_command_file=/tmp/gh-ost-test.bash
ghost_structure_output_file=/tmp/gh-ost-test.ghost.structure.sql
orig_content_output_file=/tmp/gh-ost-test.orig.content.csv
@ -24,12 +25,13 @@ replica_port=
original_sql_mode=
OPTIND=1
while getopts "b:" OPTION
while getopts "b:s:" OPTION
do
case $OPTION in
b)
ghost_binary="$OPTARG"
;;
ghost_binary="$OPTARG";;
s)
storage_engine="$OPTARG";;
esac
done
shift $((OPTIND-1))
@ -99,9 +101,13 @@ test_single() {
if [ -f $tests_path/$test_name/ignore_versions ] ; then
ignore_versions=$(cat $tests_path/$test_name/ignore_versions)
mysql_version=$(gh-ost-test-mysql-master -s -s -e "select @@version")
mysql_version_comment=$(gh-ost-test-mysql-master -s -s -e "select @@version_comment")
if echo "$mysql_version" | egrep -q "^${ignore_versions}" ; then
echo -n "Skipping: $test_name"
return 0
elif echo "$mysql_version_comment" | egrep -i -q "^${ignore_versions}" ; then
echo -n "Skipping: $test_name"
return 0
fi
fi
@ -117,6 +123,14 @@ test_single() {
fi
gh-ost-test-mysql-master --default-character-set=utf8mb4 test < $tests_path/$test_name/create.sql
test_create_result=$?
if [ $test_create_result -ne 0 ] ; then
echo
echo "ERROR $test_name create failure. cat $tests_path/$test_name/create.sql:"
cat $tests_path/$test_name/create.sql
return 1
fi
extra_args=""
if [ -f $tests_path/$test_name/extra_args ] ; then
@ -146,7 +160,8 @@ test_single() {
--assume-master-host=${master_host}:${master_port}
--database=test \
--table=gh_ost_test \
--alter='engine=innodb' \
--storage-engine=${storage_engine} \
--alter='engine=${storage_engine}' \
--exact-rowcount \
--assume-rbr \
--initially-drop-old-table \
@ -255,7 +270,7 @@ build_binary() {
test_all() {
build_binary
find $tests_path ! -path . -type d -mindepth 1 -maxdepth 1 | cut -d "/" -f 3 | egrep "$test_pattern" | while read test_name ; do
find $tests_path ! -path . -type d -mindepth 1 -maxdepth 1 | cut -d "/" -f 3 | egrep "$test_pattern" | sort | while read test_name ; do
test_single "$test_name"
if [ $? -ne 0 ] ; then
create_statement=$(gh-ost-test-mysql-replica test -t -e "show create table _gh_ost_test_gho \G")

View File

@ -36,8 +36,16 @@ test_mysql_version() {
mkdir -p sandbox/binary
rm -rf sandbox/binary/*
local mysql_server=${mysql_version%-*}
if echo "$mysql_server" | egrep -i "percona" ; then
tarball_name=Percona-Server-${mysql_version#*-}-12-Linux.x86_64.glibc2.12-minimal.tar.gz
rm -f gh-ost-ci-env/mysql-tarballs/${tarball_name}
ln -s "$mysql_version".tar.xz gh-ost-ci-env/mysql-tarballs/${tarball_name}
gh-ost-ci-env/bin/linux/dbdeployer unpack gh-ost-ci-env/mysql-tarballs/${tarball_name} --sandbox-binary ${PWD}/sandbox/binary
rm -f gh-ost-ci-env/mysql-tarballs/${tarball_name}
else
gh-ost-ci-env/bin/linux/dbdeployer unpack gh-ost-ci-env/mysql-tarballs/"$mysql_version".tar.xz --sandbox-binary ${PWD}/sandbox/binary
fi
mkdir -p sandboxes
rm -rf sandboxes/*
@ -60,9 +68,21 @@ test_mysql_version() {
gh-ost-test-mysql-master -uroot -e "create user 'gh-ost'@'%' identified by 'gh-ost'"
gh-ost-test-mysql-master -uroot -e "grant all on *.* to 'gh-ost'@'%'"
echo "### Running gh-ost tests for $mysql_version"
./localtests/test.sh -b bin/gh-ost
if echo "$mysql_server" | egrep -i "percona" ; then
echo "### Preparing for rocksdb in PerconaServer"
gh-ost-test-mysql-master -uroot -e 'INSTALL PLUGIN ROCKSDB SONAME "ha_rocksdb.so"'
gh-ost-test-mysql-master -uroot -e 'set global default_storage_engine="ROCKSDB"'
gh-ost-test-mysql-master -uroot -e 'set global transaction_isolation="READ-COMMITTED"'
gh-ost-test-mysql-replica -uroot -e 'INSTALL PLUGIN ROCKSDB SONAME "ha_rocksdb.so"'
gh-ost-test-mysql-replica -uroot -e 'set global default_storage_engine="ROCKSDB"'
gh-ost-test-mysql-replica -uroot -e 'set global transaction_isolation="READ-COMMITTED"'
echo "### Running gh-ost tests for $mysql_version"
./localtests/test.sh -b bin/gh-ost -s rocksdb
else
echo "### Running gh-ost tests for $mysql_version"
./localtests/test.sh -b bin/gh-ost -s innodb
fi
find sandboxes -name "stop_all" | bash
}