diff --git a/go/base/context.go b/go/base/context.go index b68bf7b..be28ac2 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -270,6 +270,8 @@ func NewMigrationContext() *MigrationContext { Uuid: uuid.NewV4().String(), defaultNumRetries: 60, ChunkSize: 1000, + InspectorConnectionConfig: mysql.NewConnectionConfig(), + ApplierConnectionConfig: mysql.NewConnectionConfig(), MaxLagMillisecondsThrottleThreshold: 1500, CutOverLockTimeoutSeconds: 3, DMLBatchSize: 10, @@ -288,7 +290,7 @@ func NewMigrationContext() *MigrationContext { } } -func (this *MigrationContext) SetConnectionConfig(storageEngine string, host string, port int, timeout float64) error { +func (this *MigrationContext) SetConnectionConfig(storageEngine string) error { transactionIsolation := "REPEATABLE-READ" switch storageEngine { case "rocksdb": @@ -298,11 +300,8 @@ func (this *MigrationContext) SetConnectionConfig(storageEngine string, host str default: transactionIsolation = "REPEATABLE-READ" } - this.InspectorConnectionConfig = mysql.NewConnectionConfig(transactionIsolation) - this.InspectorConnectionConfig.Key.Hostname = host - this.InspectorConnectionConfig.Key.Port = port - this.InspectorConnectionConfig.Timeout = timeout - this.ApplierConnectionConfig = mysql.NewConnectionConfig(transactionIsolation) + this.InspectorConnectionConfig.TransactionIsolation = transactionIsolation + this.ApplierConnectionConfig.TransactionIsolation = transactionIsolation return nil } diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index 145ef65..98310f3 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -47,10 +47,10 @@ func acceptSignals(migrationContext *base.MigrationContext) { // main is the application's entry point. It will either spawn a CLI or HTTP interfaces. func main() { migrationContext := base.NewMigrationContext() - host := flag.String("host", "127.0.0.1", "MySQL hostname (preferably a replica, not the master)") - port := flag.Int("port", 3306, "MySQL port (preferably a replica, not the master)") - timeout := flag.Float64("mysql-timeout", 0.0, "Connect, read and write timeout for MySQL") + flag.StringVar(&migrationContext.InspectorConnectionConfig.Key.Hostname, "host", "127.0.0.1", "MySQL hostname (preferably a replica, not the master)") flag.StringVar(&migrationContext.AssumeMasterHostname, "assume-master-host", "", "(optional) explicitly tell gh-ost the identity of the master. Format: some.host.com[:port] This is useful in master-master setups where you wish to pick an explicit master, or in a tungsten-replicator where gh-ost is unable to determine the master") + flag.IntVar(&migrationContext.InspectorConnectionConfig.Key.Port, "port", 3306, "MySQL port (preferably a replica, not the master)") + flag.Float64Var(&migrationContext.InspectorConnectionConfig.Timeout, "mysql-timeout", 0.0, "Connect, read and write timeout for MySQL") flag.StringVar(&migrationContext.CliUser, "user", "", "MySQL user") flag.StringVar(&migrationContext.CliPassword, "password", "", "MySQL password") flag.StringVar(&migrationContext.CliMasterUser, "master-user", "", "MySQL user on master, if different from that on replica. Requires --assume-master-host") @@ -183,7 +183,7 @@ func main() { migrationContext.Log.SetLevel(log.ERROR) } - if err := migrationContext.SetConnectionConfig(*storageEngine, *host, *port, *timeout); err != nil { + if err := migrationContext.SetConnectionConfig(*storageEngine); err != nil { migrationContext.Log.Fatale(err) } diff --git a/go/mysql/connection.go b/go/mysql/connection.go index 15d7cb0..6250925 100644 --- a/go/mysql/connection.go +++ b/go/mysql/connection.go @@ -29,13 +29,12 @@ type ConnectionConfig struct { ImpliedKey *InstanceKey tlsConfig *tls.Config Timeout float64 - transactionIsolation string + TransactionIsolation string } -func NewConnectionConfig(transactionIsolation string) *ConnectionConfig { +func NewConnectionConfig() *ConnectionConfig { config := &ConnectionConfig{ - Key: InstanceKey{}, - transactionIsolation: transactionIsolation, + Key: InstanceKey{}, } config.ImpliedKey = &config.Key return config @@ -44,11 +43,12 @@ func NewConnectionConfig(transactionIsolation string) *ConnectionConfig { // DuplicateCredentials creates a new connection config with given key and with same credentials as this config func (this *ConnectionConfig) DuplicateCredentials(key InstanceKey) *ConnectionConfig { config := &ConnectionConfig{ - Key: key, - User: this.User, - Password: this.Password, - tlsConfig: this.tlsConfig, - Timeout: this.Timeout, + Key: key, + User: this.User, + Password: this.Password, + tlsConfig: this.tlsConfig, + Timeout: this.Timeout, + TransactionIsolation: this.TransactionIsolation, } config.ImpliedKey = &config.Key return config @@ -127,7 +127,7 @@ func (this *ConnectionConfig) GetDBUri(databaseName string) string { "charset=utf8mb4,utf8,latin1", "interpolateParams=true", fmt.Sprintf("tls=%s", tlsOption), - fmt.Sprintf("transaction_isolation=%q", this.transactionIsolation), + fmt.Sprintf("transaction_isolation=%q", this.TransactionIsolation), fmt.Sprintf("timeout=%fs", this.Timeout), fmt.Sprintf("readTimeout=%fs", this.Timeout), fmt.Sprintf("writeTimeout=%fs", this.Timeout), diff --git a/go/mysql/connection_test.go b/go/mysql/connection_test.go index e8cdc27..5667235 100644 --- a/go/mysql/connection_test.go +++ b/go/mysql/connection_test.go @@ -22,17 +22,18 @@ func init() { } func TestNewConnectionConfig(t *testing.T) { - c := NewConnectionConfig(transactionIsolation) + c := NewConnectionConfig() test.S(t).ExpectEquals(c.Key.Hostname, "") test.S(t).ExpectEquals(c.Key.Port, 0) test.S(t).ExpectEquals(c.ImpliedKey.Hostname, "") test.S(t).ExpectEquals(c.ImpliedKey.Port, 0) test.S(t).ExpectEquals(c.User, "") test.S(t).ExpectEquals(c.Password, "") + test.S(t).ExpectEquals(c.TransactionIsolation, "") } func TestDuplicateCredentials(t *testing.T) { - c := NewConnectionConfig(transactionIsolation) + c := NewConnectionConfig() c.Key = InstanceKey{Hostname: "myhost", Port: 3306} c.User = "gromit" c.Password = "penguin" @@ -40,6 +41,7 @@ func TestDuplicateCredentials(t *testing.T) { InsecureSkipVerify: true, ServerName: "feathers", } + c.TransactionIsolation = transactionIsolation dup := c.DuplicateCredentials(InstanceKey{Hostname: "otherhost", Port: 3310}) test.S(t).ExpectEquals(dup.Key.Hostname, "otherhost") @@ -49,13 +51,15 @@ func TestDuplicateCredentials(t *testing.T) { test.S(t).ExpectEquals(dup.User, "gromit") test.S(t).ExpectEquals(dup.Password, "penguin") test.S(t).ExpectEquals(dup.tlsConfig, c.tlsConfig) + test.S(t).ExpectEquals(dup.TransactionIsolation, c.TransactionIsolation) } func TestDuplicate(t *testing.T) { - c := NewConnectionConfig(transactionIsolation) + c := NewConnectionConfig() c.Key = InstanceKey{Hostname: "myhost", Port: 3306} c.User = "gromit" c.Password = "penguin" + c.TransactionIsolation = transactionIsolation dup := c.Duplicate() test.S(t).ExpectEquals(dup.Key.Hostname, "myhost") @@ -64,26 +68,29 @@ func TestDuplicate(t *testing.T) { test.S(t).ExpectEquals(dup.ImpliedKey.Port, 3306) test.S(t).ExpectEquals(dup.User, "gromit") test.S(t).ExpectEquals(dup.Password, "penguin") + test.S(t).ExpectEquals(dup.TransactionIsolation, transactionIsolation) } func TestGetDBUri(t *testing.T) { - c := NewConnectionConfig(transactionIsolation) + c := NewConnectionConfig() c.Key = InstanceKey{Hostname: "myhost", Port: 3306} c.User = "gromit" c.Password = "penguin" c.Timeout = 1.2345 + c.TransactionIsolation = transactionIsolation uri := c.GetDBUri("test") test.S(t).ExpectEquals(uri, `gromit:penguin@tcp(myhost:3306)/test?autocommit=true&charset=utf8mb4,utf8,latin1&interpolateParams=true&tls=false&transaction_isolation="REPEATABLE-READ"&timeout=1.234500s&readTimeout=1.234500s&writeTimeout=1.234500s`) } func TestGetDBUriWithTLSSetup(t *testing.T) { - c := NewConnectionConfig(transactionIsolation) + c := NewConnectionConfig() c.Key = InstanceKey{Hostname: "myhost", Port: 3306} c.User = "gromit" c.Password = "penguin" c.Timeout = 1.2345 c.tlsConfig = &tls.Config{} + c.TransactionIsolation = transactionIsolation uri := c.GetDBUri("test") test.S(t).ExpectEquals(uri, `gromit:penguin@tcp(myhost:3306)/test?autocommit=true&charset=utf8mb4,utf8,latin1&interpolateParams=true&tls=ghost&transaction_isolation="REPEATABLE-READ"&timeout=1.234500s&readTimeout=1.234500s&writeTimeout=1.234500s`) diff --git a/localtests/test.sh b/localtests/test.sh index d73dab2..14ecd83 100755 --- a/localtests/test.sh +++ b/localtests/test.sh @@ -11,6 +11,7 @@ tests_path=$(dirname $0) test_logfile=/tmp/gh-ost-test.log default_ghost_binary=/tmp/gh-ost-test ghost_binary="" +storage_engine=innodb exec_command_file=/tmp/gh-ost-test.bash ghost_structure_output_file=/tmp/gh-ost-test.ghost.structure.sql orig_content_output_file=/tmp/gh-ost-test.orig.content.csv @@ -24,12 +25,13 @@ replica_port= original_sql_mode= OPTIND=1 -while getopts "b:" OPTION +while getopts "b:s:" OPTION do case $OPTION in b) - ghost_binary="$OPTARG" - ;; + ghost_binary="$OPTARG";; + s) + storage_engine="$OPTARG";; esac done shift $((OPTIND-1)) @@ -158,7 +160,8 @@ test_single() { --assume-master-host=${master_host}:${master_port} --database=test \ --table=gh_ost_test \ - --alter='engine=innodb' \ + --storage-engine=${storage_engine} \ + --alter='engine=${storage_engine}' \ --exact-rowcount \ --assume-rbr \ --initially-drop-old-table \ diff --git a/script/cibuild-gh-ost-replica-tests b/script/cibuild-gh-ost-replica-tests index bb60d71..31b3139 100755 --- a/script/cibuild-gh-ost-replica-tests +++ b/script/cibuild-gh-ost-replica-tests @@ -36,8 +36,16 @@ test_mysql_version() { mkdir -p sandbox/binary rm -rf sandbox/binary/* - gh-ost-ci-env/bin/linux/dbdeployer unpack gh-ost-ci-env/mysql-tarballs/"$mysql_version".tar.xz --sandbox-binary ${PWD}/sandbox/binary - + local mysql_server=${mysql_version%-*} + if echo "$mysql_server" | egrep -i "percona" ; then + tarball_name=Percona-Server-${mysql_version#*-}-12-Linux.x86_64.glibc2.12-minimal.tar.gz + rm -f gh-ost-ci-env/mysql-tarballs/${tarball_name} + ln -s "$mysql_version".tar.xz gh-ost-ci-env/mysql-tarballs/${tarball_name} + gh-ost-ci-env/bin/linux/dbdeployer unpack gh-ost-ci-env/mysql-tarballs/${tarball_name} --sandbox-binary ${PWD}/sandbox/binary + rm -f gh-ost-ci-env/mysql-tarballs/${tarball_name} + else + gh-ost-ci-env/bin/linux/dbdeployer unpack gh-ost-ci-env/mysql-tarballs/"$mysql_version".tar.xz --sandbox-binary ${PWD}/sandbox/binary + fi mkdir -p sandboxes rm -rf sandboxes/* @@ -60,49 +68,45 @@ test_mysql_version() { gh-ost-test-mysql-master -uroot -e "create user 'gh-ost'@'%' identified by 'gh-ost'" gh-ost-test-mysql-master -uroot -e "grant all on *.* to 'gh-ost'@'%'" - local mysql_server=${mysql_version%-*} if echo "$mysql_server" | egrep -i "percona" ; then echo "### Preparing for rocksdb in PerconaServer" - gh-ost-test-mysql-master -uroot -e "INSTALL PLUGIN ROCKSDB SONAME 'ha_rocksdb.so'" - gh-ost-test-mysql-master -uroot -e "INSTALL PLUGIN ROCKSDB_CFSTATS SONAME 'ha_rocksdb.so'" - gh-ost-test-mysql-master -uroot -e "INSTALL PLUGIN ROCKSDB_DBSTATS SONAME 'ha_rocksdb.so'" - gh-ost-test-mysql-master -uroot -e "INSTALL PLUGIN ROCKSDB_PERF_CONTEXT SONAME 'ha_rocksdb.so'" - gh-ost-test-mysql-master -uroot -e "INSTALL PLUGIN ROCKSDB_PERF_CONTEXT_GLOBAL SONAME 'ha_rocksdb.so'" - gh-ost-test-mysql-master -uroot -e "INSTALL PLUGIN ROCKSDB_CF_OPTIONS SONAME 'ha_rocksdb.so'" - gh-ost-test-mysql-master -uroot -e "INSTALL PLUGIN ROCKSDB_GLOBAL_INFO SONAME 'ha_rocksdb.so'" - gh-ost-test-mysql-master -uroot -e "INSTALL PLUGIN ROCKSDB_COMPACTION_HISTORY SONAME 'ha_rocksdb.so'" - gh-ost-test-mysql-master -uroot -e "INSTALL PLUGIN ROCKSDB_COMPACTION_STATS SONAME 'ha_rocksdb.so'" - gh-ost-test-mysql-master -uroot -e "INSTALL PLUGIN ROCKSDB_ACTIVE_COMPACTION_STATS SONAME 'ha_rocksdb.so'" - gh-ost-test-mysql-master -uroot -e "INSTALL PLUGIN ROCKSDB_DDL SONAME 'ha_rocksdb.so'" - gh-ost-test-mysql-master -uroot -e "INSTALL PLUGIN ROCKSDB_INDEX_FILE_MAP SONAME 'ha_rocksdb.so'" - gh-ost-test-mysql-master -uroot -e "INSTALL PLUGIN ROCKSDB_LOCKS SONAME 'ha_rocksdb.so'" - gh-ost-test-mysql-master -uroot -e "INSTALL PLUGIN ROCKSDB_TRX SONAME 'ha_rocksdb.so'" - gh-ost-test-mysql-master -uroot -e "INSTALL PLUGIN ROCKSDB_DEADLOCK SONAME 'ha_rocksdb.so'" - gh-ost-test-mysql-master -uroot -e "set global default_storage_engine='ROCKSDB'" - gh-ost-test-mysql-master -uroot -e "set global transaction_isolation='READ-COMMITTED'" + gh-ost-test-mysql-master -uroot -e 'INSTALL PLUGIN ROCKSDB SONAME "ha_rocksdb.so"' + gh-ost-test-mysql-master -uroot -e 'INSTALL PLUGIN ROCKSDB_CFSTATS SONAME "ha_rocksdb.so"' + gh-ost-test-mysql-master -uroot -e 'INSTALL PLUGIN ROCKSDB_DBSTATS SONAME "ha_rocksdb.so"' + gh-ost-test-mysql-master -uroot -e 'INSTALL PLUGIN ROCKSDB_PERF_CONTEXT SONAME "ha_rocksdb.so"' + gh-ost-test-mysql-master -uroot -e 'INSTALL PLUGIN ROCKSDB_PERF_CONTEXT_GLOBAL SONAME "ha_rocksdb.so"' + gh-ost-test-mysql-master -uroot -e 'INSTALL PLUGIN ROCKSDB_CF_OPTIONS SONAME "ha_rocksdb.so"' + gh-ost-test-mysql-master -uroot -e 'INSTALL PLUGIN ROCKSDB_GLOBAL_INFO SONAME "ha_rocksdb.so"' + gh-ost-test-mysql-master -uroot -e 'INSTALL PLUGIN ROCKSDB_COMPACTION_STATS SONAME "ha_rocksdb.so"' + gh-ost-test-mysql-master -uroot -e 'INSTALL PLUGIN ROCKSDB_DDL SONAME "ha_rocksdb.so"' + gh-ost-test-mysql-master -uroot -e 'INSTALL PLUGIN ROCKSDB_INDEX_FILE_MAP SONAME "ha_rocksdb.so"' + gh-ost-test-mysql-master -uroot -e 'INSTALL PLUGIN ROCKSDB_LOCKS SONAME "ha_rocksdb.so"' + gh-ost-test-mysql-master -uroot -e 'INSTALL PLUGIN ROCKSDB_TRX SONAME "ha_rocksdb.so"' + gh-ost-test-mysql-master -uroot -e 'INSTALL PLUGIN ROCKSDB_DEADLOCK SONAME "ha_rocksdb.so"' + gh-ost-test-mysql-master -uroot -e 'set global default_storage_engine="ROCKSDB"' + gh-ost-test-mysql-master -uroot -e 'set global transaction_isolation="READ-COMMITTED"' + gh-ost-test-mysql-replica -uroot -e 'INSTALL PLUGIN ROCKSDB SONAME "ha_rocksdb.so"' + gh-ost-test-mysql-replica -uroot -e 'INSTALL PLUGIN ROCKSDB_CFSTATS SONAME "ha_rocksdb.so"' + gh-ost-test-mysql-replica -uroot -e 'INSTALL PLUGIN ROCKSDB_DBSTATS SONAME "ha_rocksdb.so"' + gh-ost-test-mysql-replica -uroot -e 'INSTALL PLUGIN ROCKSDB_PERF_CONTEXT SONAME "ha_rocksdb.so"' + gh-ost-test-mysql-replica -uroot -e 'INSTALL PLUGIN ROCKSDB_PERF_CONTEXT_GLOBAL SONAME "ha_rocksdb.so"' + gh-ost-test-mysql-replica -uroot -e 'INSTALL PLUGIN ROCKSDB_CF_OPTIONS SONAME "ha_rocksdb.so"' + gh-ost-test-mysql-replica -uroot -e 'INSTALL PLUGIN ROCKSDB_GLOBAL_INFO SONAME "ha_rocksdb.so"' + gh-ost-test-mysql-replica -uroot -e 'INSTALL PLUGIN ROCKSDB_COMPACTION_STATS SONAME "ha_rocksdb.so"' + gh-ost-test-mysql-replica -uroot -e 'INSTALL PLUGIN ROCKSDB_DDL SONAME "ha_rocksdb.so"' + gh-ost-test-mysql-replica -uroot -e 'INSTALL PLUGIN ROCKSDB_INDEX_FILE_MAP SONAME "ha_rocksdb.so"' + gh-ost-test-mysql-replica -uroot -e 'INSTALL PLUGIN ROCKSDB_LOCKS SONAME "ha_rocksdb.so"' + gh-ost-test-mysql-replica -uroot -e 'INSTALL PLUGIN ROCKSDB_TRX SONAME "ha_rocksdb.so"' + gh-ost-test-mysql-replica -uroot -e 'INSTALL PLUGIN ROCKSDB_DEADLOCK SONAME "ha_rocksdb.so"' + gh-ost-test-mysql-replica -uroot -e 'set global default_storage_engine="ROCKSDB"' + gh-ost-test-mysql-replica -uroot -e 'set global transaction_isolation="READ-COMMITTED"' - gh-ost-test-mysql-replica -uroot -e "INSTALL PLUGIN ROCKSDB SONAME 'ha_rocksdb.so'" - gh-ost-test-mysql-replica -uroot -e "INSTALL PLUGIN ROCKSDB_CFSTATS SONAME 'ha_rocksdb.so'" - gh-ost-test-mysql-replica -uroot -e "INSTALL PLUGIN ROCKSDB_DBSTATS SONAME 'ha_rocksdb.so'" - gh-ost-test-mysql-replica -uroot -e "INSTALL PLUGIN ROCKSDB_PERF_CONTEXT SONAME 'ha_rocksdb.so'" - gh-ost-test-mysql-replica -uroot -e "INSTALL PLUGIN ROCKSDB_PERF_CONTEXT_GLOBAL SONAME 'ha_rocksdb.so'" - gh-ost-test-mysql-replica -uroot -e "INSTALL PLUGIN ROCKSDB_CF_OPTIONS SONAME 'ha_rocksdb.so'" - gh-ost-test-mysql-replica -uroot -e "INSTALL PLUGIN ROCKSDB_GLOBAL_INFO SONAME 'ha_rocksdb.so'" - gh-ost-test-mysql-replica -uroot -e "INSTALL PLUGIN ROCKSDB_COMPACTION_HISTORY SONAME 'ha_rocksdb.so'" - gh-ost-test-mysql-replica -uroot -e "INSTALL PLUGIN ROCKSDB_COMPACTION_STATS SONAME 'ha_rocksdb.so'" - gh-ost-test-mysql-replica -uroot -e "INSTALL PLUGIN ROCKSDB_ACTIVE_COMPACTION_STATS SONAME 'ha_rocksdb.so'" - gh-ost-test-mysql-replica -uroot -e "INSTALL PLUGIN ROCKSDB_DDL SONAME 'ha_rocksdb.so'" - gh-ost-test-mysql-replica -uroot -e "INSTALL PLUGIN ROCKSDB_INDEX_FILE_MAP SONAME 'ha_rocksdb.so'" - gh-ost-test-mysql-replica -uroot -e "INSTALL PLUGIN ROCKSDB_LOCKS SONAME 'ha_rocksdb.so'" - gh-ost-test-mysql-replica -uroot -e "INSTALL PLUGIN ROCKSDB_TRX SONAME 'ha_rocksdb.so'" - gh-ost-test-mysql-replica -uroot -e "INSTALL PLUGIN ROCKSDB_DEADLOCK SONAME 'ha_rocksdb.so'" - gh-ost-test-mysql-replica -uroot -e "set global default_storage_engine='ROCKSDB'" - gh-ost-test-mysql-replica -uroot -e "set global transaction_isolation='READ-COMMITTED'" + echo "### Running gh-ost tests for $mysql_version" + ./localtests/test.sh -b bin/gh-ost -s rocksdb + else + echo "### Running gh-ost tests for $mysql_version" + ./localtests/test.sh -b bin/gh-ost -s innodb fi - - echo "### Running gh-ost tests for $mysql_version" - ./localtests/test.sh -b bin/gh-ost - find sandboxes -name "stop_all" | bash }