diff --git a/.github/workflows/replica-tests.yml b/.github/workflows/replica-tests.yml index 0a82f00..f2a52ec 100644 --- a/.github/workflows/replica-tests.yml +++ b/.github/workflows/replica-tests.yml @@ -8,7 +8,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - version: [mysql-5.7.25,mysql-8.0.16] + version: [mysql-5.7.25,mysql-8.0.16,PerconaServer-8.0.21] steps: - uses: actions/checkout@v2 diff --git a/doc/command-line-flags.md b/doc/command-line-flags.md index 56bc642..021462f 100644 --- a/doc/command-line-flags.md +++ b/doc/command-line-flags.md @@ -246,6 +246,18 @@ Allows `gh-ost` to connect to the MySQL servers using encrypted connections, but `--ssl-key=/path/to/ssl-key.key`: SSL private key file (in PEM format). +### storage-engine +Default is `innodb`, and `rocksdb` support is currently experimental. InnoDB and RocksDB are both transactional engines, supporting both shared and exclusive row locks. + +But RocksDB currently lacks a few features support compared to InnoDB: +- Gap Locks +- Foreign Key +- Generated Columns +- Spatial +- Geometry + +When `--storage-engine=rocksdb`, `gh-ost` will make some changes necessary (e.g. sets isolation level to `READ_COMMITTED`) to support RocksDB. + ### test-on-replica Issue the migration on a replica; do not modify data on master. Useful for validating, testing and benchmarking. See [`testing-on-replica`](testing-on-replica.md) diff --git a/go/base/context.go b/go/base/context.go index 6032a93..e3472f5 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -290,6 +290,19 @@ func NewMigrationContext() *MigrationContext { } } +func (this *MigrationContext) SetConnectionConfig(storageEngine string) error { + var transactionIsolation string + switch storageEngine { + case "rocksdb": + transactionIsolation = "READ-COMMITTED" + default: + transactionIsolation = "REPEATABLE-READ" + } + this.InspectorConnectionConfig.TransactionIsolation = transactionIsolation + this.ApplierConnectionConfig.TransactionIsolation = transactionIsolation + return nil +} + func getSafeTableName(baseName string, suffix string) string { name := fmt.Sprintf("_%s_%s", baseName, suffix) if len(name) <= mysql.MaxTableNameLength { @@ -428,6 +441,10 @@ func (this *MigrationContext) IsTransactionalTable() bool { { return true } + case "rocksdb": + { + return true + } } return false } diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index c00b206..3daf244 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -68,6 +68,7 @@ func main() { flag.StringVar(&migrationContext.OriginalTableName, "table", "", "table name (mandatory)") flag.StringVar(&migrationContext.AlterStatement, "alter", "", "alter statement (mandatory)") flag.BoolVar(&migrationContext.AttemptInstantDDL, "attempt-instant-ddl", false, "Attempt to use instant DDL for this migration first") + storageEngine := flag.String("storage-engine", "innodb", "Specify table storage engine (default: 'innodb'). When 'rocksdb': the session transaction isolation level is changed from REPEATABLE_READ to READ_COMMITTED.") flag.BoolVar(&migrationContext.CountTableRows, "exact-rowcount", false, "actually count table rows as opposed to estimate them (results in more accurate progress estimation)") flag.BoolVar(&migrationContext.ConcurrentCountTableRows, "concurrent-rowcount", true, "(with --exact-rowcount), when true (default): count rows after row-copy begins, concurrently, and adjust row estimate later on; when false: first count rows, then start row copy") @@ -182,6 +183,10 @@ func main() { migrationContext.Log.SetLevel(log.ERROR) } + if err := migrationContext.SetConnectionConfig(*storageEngine); err != nil { + migrationContext.Log.Fatale(err) + } + if migrationContext.AlterStatement == "" { log.Fatal("--alter must be provided and statement must not be empty") } @@ -247,6 +252,9 @@ func main() { if *replicationLagQuery != "" { migrationContext.Log.Warning("--replication-lag-query is deprecated") } + if *storageEngine == "rocksdb" { + migrationContext.Log.Warning("RocksDB storage engine support is experimental") + } switch *cutOver { case "atomic", "default", "": diff --git a/go/mysql/connection.go b/go/mysql/connection.go index 6a5c890..6250925 100644 --- a/go/mysql/connection.go +++ b/go/mysql/connection.go @@ -18,18 +18,18 @@ import ( ) const ( - transactionIsolation = "REPEATABLE-READ" - TLS_CONFIG_KEY = "ghost" + TLS_CONFIG_KEY = "ghost" ) // ConnectionConfig is the minimal configuration required to connect to a MySQL server type ConnectionConfig struct { - Key InstanceKey - User string - Password string - ImpliedKey *InstanceKey - tlsConfig *tls.Config - Timeout float64 + Key InstanceKey + User string + Password string + ImpliedKey *InstanceKey + tlsConfig *tls.Config + Timeout float64 + TransactionIsolation string } func NewConnectionConfig() *ConnectionConfig { @@ -43,11 +43,12 @@ func NewConnectionConfig() *ConnectionConfig { // DuplicateCredentials creates a new connection config with given key and with same credentials as this config func (this *ConnectionConfig) DuplicateCredentials(key InstanceKey) *ConnectionConfig { config := &ConnectionConfig{ - Key: key, - User: this.User, - Password: this.Password, - tlsConfig: this.tlsConfig, - Timeout: this.Timeout, + Key: key, + User: this.User, + Password: this.Password, + tlsConfig: this.tlsConfig, + Timeout: this.Timeout, + TransactionIsolation: this.TransactionIsolation, } config.ImpliedKey = &config.Key return config @@ -126,7 +127,7 @@ func (this *ConnectionConfig) GetDBUri(databaseName string) string { "charset=utf8mb4,utf8,latin1", "interpolateParams=true", fmt.Sprintf("tls=%s", tlsOption), - fmt.Sprintf("transaction_isolation=%q", transactionIsolation), + fmt.Sprintf("transaction_isolation=%q", this.TransactionIsolation), fmt.Sprintf("timeout=%fs", this.Timeout), fmt.Sprintf("readTimeout=%fs", this.Timeout), fmt.Sprintf("writeTimeout=%fs", this.Timeout), diff --git a/go/mysql/connection_test.go b/go/mysql/connection_test.go index 390774c..5667235 100644 --- a/go/mysql/connection_test.go +++ b/go/mysql/connection_test.go @@ -13,6 +13,10 @@ import ( test "github.com/openark/golib/tests" ) +const ( + transactionIsolation = "REPEATABLE-READ" +) + func init() { log.SetLevel(log.ERROR) } @@ -25,6 +29,7 @@ func TestNewConnectionConfig(t *testing.T) { test.S(t).ExpectEquals(c.ImpliedKey.Port, 0) test.S(t).ExpectEquals(c.User, "") test.S(t).ExpectEquals(c.Password, "") + test.S(t).ExpectEquals(c.TransactionIsolation, "") } func TestDuplicateCredentials(t *testing.T) { @@ -36,6 +41,7 @@ func TestDuplicateCredentials(t *testing.T) { InsecureSkipVerify: true, ServerName: "feathers", } + c.TransactionIsolation = transactionIsolation dup := c.DuplicateCredentials(InstanceKey{Hostname: "otherhost", Port: 3310}) test.S(t).ExpectEquals(dup.Key.Hostname, "otherhost") @@ -45,6 +51,7 @@ func TestDuplicateCredentials(t *testing.T) { test.S(t).ExpectEquals(dup.User, "gromit") test.S(t).ExpectEquals(dup.Password, "penguin") test.S(t).ExpectEquals(dup.tlsConfig, c.tlsConfig) + test.S(t).ExpectEquals(dup.TransactionIsolation, c.TransactionIsolation) } func TestDuplicate(t *testing.T) { @@ -52,6 +59,7 @@ func TestDuplicate(t *testing.T) { c.Key = InstanceKey{Hostname: "myhost", Port: 3306} c.User = "gromit" c.Password = "penguin" + c.TransactionIsolation = transactionIsolation dup := c.Duplicate() test.S(t).ExpectEquals(dup.Key.Hostname, "myhost") @@ -60,6 +68,7 @@ func TestDuplicate(t *testing.T) { test.S(t).ExpectEquals(dup.ImpliedKey.Port, 3306) test.S(t).ExpectEquals(dup.User, "gromit") test.S(t).ExpectEquals(dup.Password, "penguin") + test.S(t).ExpectEquals(dup.TransactionIsolation, transactionIsolation) } func TestGetDBUri(t *testing.T) { @@ -68,6 +77,7 @@ func TestGetDBUri(t *testing.T) { c.User = "gromit" c.Password = "penguin" c.Timeout = 1.2345 + c.TransactionIsolation = transactionIsolation uri := c.GetDBUri("test") test.S(t).ExpectEquals(uri, `gromit:penguin@tcp(myhost:3306)/test?autocommit=true&charset=utf8mb4,utf8,latin1&interpolateParams=true&tls=false&transaction_isolation="REPEATABLE-READ"&timeout=1.234500s&readTimeout=1.234500s&writeTimeout=1.234500s`) @@ -80,6 +90,7 @@ func TestGetDBUriWithTLSSetup(t *testing.T) { c.Password = "penguin" c.Timeout = 1.2345 c.tlsConfig = &tls.Config{} + c.TransactionIsolation = transactionIsolation uri := c.GetDBUri("test") test.S(t).ExpectEquals(uri, `gromit:penguin@tcp(myhost:3306)/test?autocommit=true&charset=utf8mb4,utf8,latin1&interpolateParams=true&tls=ghost&transaction_isolation="REPEATABLE-READ"&timeout=1.234500s&readTimeout=1.234500s&writeTimeout=1.234500s`) diff --git a/localtests/discard-fk/ignore_versions b/localtests/discard-fk/ignore_versions new file mode 100644 index 0000000..cf02abe --- /dev/null +++ b/localtests/discard-fk/ignore_versions @@ -0,0 +1 @@ +Percona \ No newline at end of file diff --git a/localtests/fail-fk-parent/ignore_versions b/localtests/fail-fk-parent/ignore_versions new file mode 100644 index 0000000..cf02abe --- /dev/null +++ b/localtests/fail-fk-parent/ignore_versions @@ -0,0 +1 @@ +Percona \ No newline at end of file diff --git a/localtests/fail-fk/ignore_versions b/localtests/fail-fk/ignore_versions new file mode 100644 index 0000000..cf02abe --- /dev/null +++ b/localtests/fail-fk/ignore_versions @@ -0,0 +1 @@ +Percona \ No newline at end of file diff --git a/localtests/generated-columns-add/ignore_versions b/localtests/generated-columns-add/ignore_versions new file mode 100644 index 0000000..cf02abe --- /dev/null +++ b/localtests/generated-columns-add/ignore_versions @@ -0,0 +1 @@ +Percona \ No newline at end of file diff --git a/localtests/generated-columns-rename/ignore_versions b/localtests/generated-columns-rename/ignore_versions new file mode 100644 index 0000000..cf02abe --- /dev/null +++ b/localtests/generated-columns-rename/ignore_versions @@ -0,0 +1 @@ +Percona \ No newline at end of file diff --git a/localtests/generated-columns-unique/ignore_versions b/localtests/generated-columns-unique/ignore_versions new file mode 100644 index 0000000..cf02abe --- /dev/null +++ b/localtests/generated-columns-unique/ignore_versions @@ -0,0 +1 @@ +Percona \ No newline at end of file diff --git a/localtests/generated-columns/ignore_versions b/localtests/generated-columns/ignore_versions new file mode 100644 index 0000000..cf02abe --- /dev/null +++ b/localtests/generated-columns/ignore_versions @@ -0,0 +1 @@ +Percona \ No newline at end of file diff --git a/localtests/geometry/ignore_versions b/localtests/geometry/ignore_versions new file mode 100644 index 0000000..cf02abe --- /dev/null +++ b/localtests/geometry/ignore_versions @@ -0,0 +1 @@ +Percona \ No newline at end of file diff --git a/localtests/spatial/ignore_versions b/localtests/spatial/ignore_versions new file mode 100644 index 0000000..cf02abe --- /dev/null +++ b/localtests/spatial/ignore_versions @@ -0,0 +1 @@ +Percona \ No newline at end of file diff --git a/localtests/test.sh b/localtests/test.sh index 5ca06ad..14ecd83 100755 --- a/localtests/test.sh +++ b/localtests/test.sh @@ -11,6 +11,7 @@ tests_path=$(dirname $0) test_logfile=/tmp/gh-ost-test.log default_ghost_binary=/tmp/gh-ost-test ghost_binary="" +storage_engine=innodb exec_command_file=/tmp/gh-ost-test.bash ghost_structure_output_file=/tmp/gh-ost-test.ghost.structure.sql orig_content_output_file=/tmp/gh-ost-test.orig.content.csv @@ -24,12 +25,13 @@ replica_port= original_sql_mode= OPTIND=1 -while getopts "b:" OPTION +while getopts "b:s:" OPTION do case $OPTION in b) - ghost_binary="$OPTARG" - ;; + ghost_binary="$OPTARG";; + s) + storage_engine="$OPTARG";; esac done shift $((OPTIND-1)) @@ -99,9 +101,13 @@ test_single() { if [ -f $tests_path/$test_name/ignore_versions ] ; then ignore_versions=$(cat $tests_path/$test_name/ignore_versions) mysql_version=$(gh-ost-test-mysql-master -s -s -e "select @@version") + mysql_version_comment=$(gh-ost-test-mysql-master -s -s -e "select @@version_comment") if echo "$mysql_version" | egrep -q "^${ignore_versions}" ; then echo -n "Skipping: $test_name" return 0 + elif echo "$mysql_version_comment" | egrep -i -q "^${ignore_versions}" ; then + echo -n "Skipping: $test_name" + return 0 fi fi @@ -154,7 +160,8 @@ test_single() { --assume-master-host=${master_host}:${master_port} --database=test \ --table=gh_ost_test \ - --alter='engine=innodb' \ + --storage-engine=${storage_engine} \ + --alter='engine=${storage_engine}' \ --exact-rowcount \ --assume-rbr \ --initially-drop-old-table \ diff --git a/script/cibuild-gh-ost-replica-tests b/script/cibuild-gh-ost-replica-tests index c4dbfd2..90eb856 100755 --- a/script/cibuild-gh-ost-replica-tests +++ b/script/cibuild-gh-ost-replica-tests @@ -36,8 +36,16 @@ test_mysql_version() { mkdir -p sandbox/binary rm -rf sandbox/binary/* - gh-ost-ci-env/bin/linux/dbdeployer unpack gh-ost-ci-env/mysql-tarballs/"$mysql_version".tar.xz --sandbox-binary ${PWD}/sandbox/binary - + local mysql_server=${mysql_version%-*} + if echo "$mysql_server" | egrep -i "percona" ; then + tarball_name=Percona-Server-${mysql_version#*-}-12-Linux.x86_64.glibc2.12-minimal.tar.gz + rm -f gh-ost-ci-env/mysql-tarballs/${tarball_name} + ln -s "$mysql_version".tar.xz gh-ost-ci-env/mysql-tarballs/${tarball_name} + gh-ost-ci-env/bin/linux/dbdeployer unpack gh-ost-ci-env/mysql-tarballs/${tarball_name} --sandbox-binary ${PWD}/sandbox/binary + rm -f gh-ost-ci-env/mysql-tarballs/${tarball_name} + else + gh-ost-ci-env/bin/linux/dbdeployer unpack gh-ost-ci-env/mysql-tarballs/"$mysql_version".tar.xz --sandbox-binary ${PWD}/sandbox/binary + fi mkdir -p sandboxes rm -rf sandboxes/* @@ -60,9 +68,21 @@ test_mysql_version() { gh-ost-test-mysql-master -uroot -e "create user 'gh-ost'@'%' identified by 'gh-ost'" gh-ost-test-mysql-master -uroot -e "grant all on *.* to 'gh-ost'@'%'" - echo "### Running gh-ost tests for $mysql_version" - ./localtests/test.sh -b bin/gh-ost + if echo "$mysql_server" | egrep -i "percona" ; then + echo "### Preparing for rocksdb in PerconaServer" + gh-ost-test-mysql-master -uroot -e 'INSTALL PLUGIN ROCKSDB SONAME "ha_rocksdb.so"' + gh-ost-test-mysql-master -uroot -e 'set global default_storage_engine="ROCKSDB"' + gh-ost-test-mysql-master -uroot -e 'set global transaction_isolation="READ-COMMITTED"' + gh-ost-test-mysql-replica -uroot -e 'INSTALL PLUGIN ROCKSDB SONAME "ha_rocksdb.so"' + gh-ost-test-mysql-replica -uroot -e 'set global default_storage_engine="ROCKSDB"' + gh-ost-test-mysql-replica -uroot -e 'set global transaction_isolation="READ-COMMITTED"' + echo "### Running gh-ost tests for $mysql_version" + ./localtests/test.sh -b bin/gh-ost -s rocksdb + else + echo "### Running gh-ost tests for $mysql_version" + ./localtests/test.sh -b bin/gh-ost -s innodb + fi find sandboxes -name "stop_all" | bash }