octoleo/gh-ost
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

add support for rocksdb

Browse Source
This commit is contained in:
lukelewang 2022-11-26 01:38:12 +08:00
parent 094909f267
commit da3514253f
6 changed files with 83 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
go/base/context.go
View File

@ -270,6 +270,8 @@ func NewMigrationContext() *MigrationContext {
Uuid: uuid.NewV4().String(),
defaultNumRetries: 60,
ChunkSize: 1000,
InspectorConnectionConfig: mysql.NewConnectionConfig(),
ApplierConnectionConfig: mysql.NewConnectionConfig(),
MaxLagMillisecondsThrottleThreshold: 1500,
CutOverLockTimeoutSeconds: 3,
DMLBatchSize: 10,
@ -288,7 +290,7 @@ func NewMigrationContext() *MigrationContext {
}
}
func (this *MigrationContext) SetConnectionConfig(storageEngine string, host string, port int, timeout float64) error {
func (this *MigrationContext) SetConnectionConfig(storageEngine string) error {
transactionIsolation := "REPEATABLE-READ"
switch storageEngine {
case "rocksdb":
@ -298,11 +300,8 @@ func (this *MigrationContext) SetConnectionConfig(storageEngine string, host str
default:
transactionIsolation = "REPEATABLE-READ"
}
this.InspectorConnectionConfig = mysql.NewConnectionConfig(transactionIsolation)
this.InspectorConnectionConfig.Key.Hostname = host
this.InspectorConnectionConfig.Key.Port = port
this.InspectorConnectionConfig.Timeout = timeout
this.ApplierConnectionConfig = mysql.NewConnectionConfig(transactionIsolation)
this.InspectorConnectionConfig.TransactionIsolation = transactionIsolation
this.ApplierConnectionConfig.TransactionIsolation = transactionIsolation
return nil
}

8
go/cmd/gh-ost/main.go
View File

@ -47,10 +47,10 @@ func acceptSignals(migrationContext *base.MigrationContext) {
// main is the application's entry point. It will either spawn a CLI or HTTP interfaces.
func main() {
migrationContext := base.NewMigrationContext()
host := flag.String("host", "127.0.0.1", "MySQL hostname (preferably a replica, not the master)")
port := flag.Int("port", 3306, "MySQL port (preferably a replica, not the master)")
timeout := flag.Float64("mysql-timeout", 0.0, "Connect, read and write timeout for MySQL")
flag.StringVar(&migrationContext.InspectorConnectionConfig.Key.Hostname, "host", "127.0.0.1", "MySQL hostname (preferably a replica, not the master)")
flag.StringVar(&migrationContext.AssumeMasterHostname, "assume-master-host", "", "(optional) explicitly tell gh-ost the identity of the master. Format: some.host.com[:port] This is useful in master-master setups where you wish to pick an explicit master, or in a tungsten-replicator where gh-ost is unable to determine the master")
flag.IntVar(&migrationContext.InspectorConnectionConfig.Key.Port, "port", 3306, "MySQL port (preferably a replica, not the master)")
flag.Float64Var(&migrationContext.InspectorConnectionConfig.Timeout, "mysql-timeout", 0.0, "Connect, read and write timeout for MySQL")
flag.StringVar(&migrationContext.CliUser, "user", "", "MySQL user")
flag.StringVar(&migrationContext.CliPassword, "password", "", "MySQL password")
flag.StringVar(&migrationContext.CliMasterUser, "master-user", "", "MySQL user on master, if different from that on replica. Requires --assume-master-host")
@ -183,7 +183,7 @@ func main() {
migrationContext.Log.SetLevel(log.ERROR)
}
if err := migrationContext.SetConnectionConfig(*storageEngine, *host, *port, *timeout); err != nil {
if err := migrationContext.SetConnectionConfig(*storageEngine); err != nil {
migrationContext.Log.Fatale(err)
}

20
go/mysql/connection.go
View File

@ -29,13 +29,12 @@ type ConnectionConfig struct {
ImpliedKey *InstanceKey
tlsConfig *tls.Config
Timeout float64
transactionIsolation string
TransactionIsolation string
}
func NewConnectionConfig(transactionIsolation string) *ConnectionConfig {
func NewConnectionConfig() *ConnectionConfig {
config := &ConnectionConfig{
Key: InstanceKey{},
transactionIsolation: transactionIsolation,
Key: InstanceKey{},
}
config.ImpliedKey = &config.Key
return config
@ -44,11 +43,12 @@ func NewConnectionConfig(transactionIsolation string) *ConnectionConfig {
// DuplicateCredentials creates a new connection config with given key and with same credentials as this config
func (this *ConnectionConfig) DuplicateCredentials(key InstanceKey) *ConnectionConfig {
config := &ConnectionConfig{
Key: key,
User: this.User,
Password: this.Password,
tlsConfig: this.tlsConfig,
Timeout: this.Timeout,
Key: key,
User: this.User,
Password: this.Password,
tlsConfig: this.tlsConfig,
Timeout: this.Timeout,
TransactionIsolation: this.TransactionIsolation,
}
config.ImpliedKey = &config.Key
return config
@ -127,7 +127,7 @@ func (this *ConnectionConfig) GetDBUri(databaseName string) string {
"charset=utf8mb4,utf8,latin1",
"interpolateParams=true",
fmt.Sprintf("tls=%s", tlsOption),
fmt.Sprintf("transaction_isolation=%q", this.transactionIsolation),
fmt.Sprintf("transaction_isolation=%q", this.TransactionIsolation),
fmt.Sprintf("timeout=%fs", this.Timeout),
fmt.Sprintf("readTimeout=%fs", this.Timeout),
fmt.Sprintf("writeTimeout=%fs", this.Timeout),

17
go/mysql/connection_test.go
View File

@ -22,17 +22,18 @@ func init() {
}
func TestNewConnectionConfig(t *testing.T) {
c := NewConnectionConfig(transactionIsolation)
c := NewConnectionConfig()
test.S(t).ExpectEquals(c.Key.Hostname, "")
test.S(t).ExpectEquals(c.Key.Port, 0)
test.S(t).ExpectEquals(c.ImpliedKey.Hostname, "")
test.S(t).ExpectEquals(c.ImpliedKey.Port, 0)
test.S(t).ExpectEquals(c.User, "")
test.S(t).ExpectEquals(c.Password, "")
test.S(t).ExpectEquals(c.TransactionIsolation, "")
}
func TestDuplicateCredentials(t *testing.T) {
c := NewConnectionConfig(transactionIsolation)
c := NewConnectionConfig()
c.Key = InstanceKey{Hostname: "myhost", Port: 3306}
c.User = "gromit"
c.Password = "penguin"
@ -40,6 +41,7 @@ func TestDuplicateCredentials(t *testing.T) {
InsecureSkipVerify: true,
ServerName: "feathers",
}
c.TransactionIsolation = transactionIsolation
dup := c.DuplicateCredentials(InstanceKey{Hostname: "otherhost", Port: 3310})
test.S(t).ExpectEquals(dup.Key.Hostname, "otherhost")
@ -49,13 +51,15 @@ func TestDuplicateCredentials(t *testing.T) {
test.S(t).ExpectEquals(dup.User, "gromit")
test.S(t).ExpectEquals(dup.Password, "penguin")
test.S(t).ExpectEquals(dup.tlsConfig, c.tlsConfig)
test.S(t).ExpectEquals(dup.TransactionIsolation, c.TransactionIsolation)
}
func TestDuplicate(t *testing.T) {
c := NewConnectionConfig(transactionIsolation)
c := NewConnectionConfig()
c.Key = InstanceKey{Hostname: "myhost", Port: 3306}
c.User = "gromit"
c.Password = "penguin"
c.TransactionIsolation = transactionIsolation
dup := c.Duplicate()
test.S(t).ExpectEquals(dup.Key.Hostname, "myhost")
@ -64,26 +68,29 @@ func TestDuplicate(t *testing.T) {
test.S(t).ExpectEquals(dup.ImpliedKey.Port, 3306)
test.S(t).ExpectEquals(dup.User, "gromit")
test.S(t).ExpectEquals(dup.Password, "penguin")
test.S(t).ExpectEquals(dup.TransactionIsolation, transactionIsolation)
}
func TestGetDBUri(t *testing.T) {
c := NewConnectionConfig(transactionIsolation)
c := NewConnectionConfig()
c.Key = InstanceKey{Hostname: "myhost", Port: 3306}
c.User = "gromit"
c.Password = "penguin"
c.Timeout = 1.2345
c.TransactionIsolation = transactionIsolation
uri := c.GetDBUri("test")
test.S(t).ExpectEquals(uri, `gromit:penguin@tcp(myhost:3306)/test?autocommit=true&charset=utf8mb4,utf8,latin1&interpolateParams=true&tls=false&transaction_isolation="REPEATABLE-READ"&timeout=1.234500s&readTimeout=1.234500s&writeTimeout=1.234500s`)
}
func TestGetDBUriWithTLSSetup(t *testing.T) {
c := NewConnectionConfig(transactionIsolation)
c := NewConnectionConfig()
c.Key = InstanceKey{Hostname: "myhost", Port: 3306}
c.User = "gromit"
c.Password = "penguin"
c.Timeout = 1.2345
c.tlsConfig = &tls.Config{}
c.TransactionIsolation = transactionIsolation
uri := c.GetDBUri("test")
test.S(t).ExpectEquals(uri, `gromit:penguin@tcp(myhost:3306)/test?autocommit=true&charset=utf8mb4,utf8,latin1&interpolateParams=true&tls=ghost&transaction_isolation="REPEATABLE-READ"&timeout=1.234500s&readTimeout=1.234500s&writeTimeout=1.234500s`)

11
localtests/test.sh
View File

@ -11,6 +11,7 @@ tests_path=$(dirname $0)
test_logfile=/tmp/gh-ost-test.log
default_ghost_binary=/tmp/gh-ost-test
ghost_binary=""
storage_engine=innodb
exec_command_file=/tmp/gh-ost-test.bash
ghost_structure_output_file=/tmp/gh-ost-test.ghost.structure.sql
orig_content_output_file=/tmp/gh-ost-test.orig.content.csv
@ -24,12 +25,13 @@ replica_port=
original_sql_mode=
OPTIND=1
while getopts "b:" OPTION
while getopts "b:s:" OPTION
do
case $OPTION in
b)
ghost_binary="$OPTARG"
;;
ghost_binary="$OPTARG";;
s)
storage_engine="$OPTARG";;
esac
done
shift $((OPTIND-1))
@ -158,7 +160,8 @@ test_single() {
--assume-master-host=${master_host}:${master_port}
--database=test \
--table=gh_ost_test \
--alter='engine=innodb' \
--storage-engine=${storage_engine} \
--alter='engine=${storage_engine}' \
--exact-rowcount \
--assume-rbr \
--initially-drop-old-table \

86
script/cibuild-gh-ost-replica-tests
View File

@ -36,8 +36,16 @@ test_mysql_version() {
mkdir -p sandbox/binary
rm -rf sandbox/binary/*
gh-ost-ci-env/bin/linux/dbdeployer unpack gh-ost-ci-env/mysql-tarballs/"$mysql_version".tar.xz --sandbox-binary ${PWD}/sandbox/binary
local mysql_server=${mysql_version%-*}
if echo "$mysql_server" | egrep -i "percona" ; then
tarball_name=Percona-Server-${mysql_version#*-}-12-Linux.x86_64.glibc2.12-minimal.tar.gz
rm -f gh-ost-ci-env/mysql-tarballs/${tarball_name}
ln -s "$mysql_version".tar.xz gh-ost-ci-env/mysql-tarballs/${tarball_name}
gh-ost-ci-env/bin/linux/dbdeployer unpack gh-ost-ci-env/mysql-tarballs/${tarball_name} --sandbox-binary ${PWD}/sandbox/binary
rm -f gh-ost-ci-env/mysql-tarballs/${tarball_name}
else
gh-ost-ci-env/bin/linux/dbdeployer unpack gh-ost-ci-env/mysql-tarballs/"$mysql_version".tar.xz --sandbox-binary ${PWD}/sandbox/binary
fi
mkdir -p sandboxes
rm -rf sandboxes/*
@ -60,49 +68,45 @@ test_mysql_version() {
gh-ost-test-mysql-master -uroot -e "create user 'gh-ost'@'%' identified by 'gh-ost'"
gh-ost-test-mysql-master -uroot -e "grant all on *.* to 'gh-ost'@'%'"
local mysql_server=${mysql_version%-*}
if echo "$mysql_server" | egrep -i "percona" ; then
echo "### Preparing for rocksdb in PerconaServer"
gh-ost-test-mysql-master -uroot -e "INSTALL PLUGIN ROCKSDB SONAME 'ha_rocksdb.so'"
gh-ost-test-mysql-master -uroot -e "INSTALL PLUGIN ROCKSDB_CFSTATS SONAME 'ha_rocksdb.so'"
gh-ost-test-mysql-master -uroot -e "INSTALL PLUGIN ROCKSDB_DBSTATS SONAME 'ha_rocksdb.so'"
gh-ost-test-mysql-master -uroot -e "INSTALL PLUGIN ROCKSDB_PERF_CONTEXT SONAME 'ha_rocksdb.so'"
gh-ost-test-mysql-master -uroot -e "INSTALL PLUGIN ROCKSDB_PERF_CONTEXT_GLOBAL SONAME 'ha_rocksdb.so'"
gh-ost-test-mysql-master -uroot -e "INSTALL PLUGIN ROCKSDB_CF_OPTIONS SONAME 'ha_rocksdb.so'"
gh-ost-test-mysql-master -uroot -e "INSTALL PLUGIN ROCKSDB_GLOBAL_INFO SONAME 'ha_rocksdb.so'"
gh-ost-test-mysql-master -uroot -e "INSTALL PLUGIN ROCKSDB_COMPACTION_HISTORY SONAME 'ha_rocksdb.so'"
gh-ost-test-mysql-master -uroot -e "INSTALL PLUGIN ROCKSDB_COMPACTION_STATS SONAME 'ha_rocksdb.so'"
gh-ost-test-mysql-master -uroot -e "INSTALL PLUGIN ROCKSDB_ACTIVE_COMPACTION_STATS SONAME 'ha_rocksdb.so'"
gh-ost-test-mysql-master -uroot -e "INSTALL PLUGIN ROCKSDB_DDL SONAME 'ha_rocksdb.so'"
gh-ost-test-mysql-master -uroot -e "INSTALL PLUGIN ROCKSDB_INDEX_FILE_MAP SONAME 'ha_rocksdb.so'"
gh-ost-test-mysql-master -uroot -e "INSTALL PLUGIN ROCKSDB_LOCKS SONAME 'ha_rocksdb.so'"
gh-ost-test-mysql-master -uroot -e "INSTALL PLUGIN ROCKSDB_TRX SONAME 'ha_rocksdb.so'"
gh-ost-test-mysql-master -uroot -e "INSTALL PLUGIN ROCKSDB_DEADLOCK SONAME 'ha_rocksdb.so'"
gh-ost-test-mysql-master -uroot -e "set global default_storage_engine='ROCKSDB'"
gh-ost-test-mysql-master -uroot -e "set global transaction_isolation='READ-COMMITTED'"
gh-ost-test-mysql-master -uroot -e 'INSTALL PLUGIN ROCKSDB SONAME "ha_rocksdb.so"'
gh-ost-test-mysql-master -uroot -e 'INSTALL PLUGIN ROCKSDB_CFSTATS SONAME "ha_rocksdb.so"'
gh-ost-test-mysql-master -uroot -e 'INSTALL PLUGIN ROCKSDB_DBSTATS SONAME "ha_rocksdb.so"'
gh-ost-test-mysql-master -uroot -e 'INSTALL PLUGIN ROCKSDB_PERF_CONTEXT SONAME "ha_rocksdb.so"'
gh-ost-test-mysql-master -uroot -e 'INSTALL PLUGIN ROCKSDB_PERF_CONTEXT_GLOBAL SONAME "ha_rocksdb.so"'
gh-ost-test-mysql-master -uroot -e 'INSTALL PLUGIN ROCKSDB_CF_OPTIONS SONAME "ha_rocksdb.so"'
gh-ost-test-mysql-master -uroot -e 'INSTALL PLUGIN ROCKSDB_GLOBAL_INFO SONAME "ha_rocksdb.so"'
gh-ost-test-mysql-master -uroot -e 'INSTALL PLUGIN ROCKSDB_COMPACTION_STATS SONAME "ha_rocksdb.so"'
gh-ost-test-mysql-master -uroot -e 'INSTALL PLUGIN ROCKSDB_DDL SONAME "ha_rocksdb.so"'
gh-ost-test-mysql-master -uroot -e 'INSTALL PLUGIN ROCKSDB_INDEX_FILE_MAP SONAME "ha_rocksdb.so"'
gh-ost-test-mysql-master -uroot -e 'INSTALL PLUGIN ROCKSDB_LOCKS SONAME "ha_rocksdb.so"'
gh-ost-test-mysql-master -uroot -e 'INSTALL PLUGIN ROCKSDB_TRX SONAME "ha_rocksdb.so"'
gh-ost-test-mysql-master -uroot -e 'INSTALL PLUGIN ROCKSDB_DEADLOCK SONAME "ha_rocksdb.so"'
gh-ost-test-mysql-master -uroot -e 'set global default_storage_engine="ROCKSDB"'
gh-ost-test-mysql-master -uroot -e 'set global transaction_isolation="READ-COMMITTED"'
gh-ost-test-mysql-replica -uroot -e 'INSTALL PLUGIN ROCKSDB SONAME "ha_rocksdb.so"'
gh-ost-test-mysql-replica -uroot -e 'INSTALL PLUGIN ROCKSDB_CFSTATS SONAME "ha_rocksdb.so"'
gh-ost-test-mysql-replica -uroot -e 'INSTALL PLUGIN ROCKSDB_DBSTATS SONAME "ha_rocksdb.so"'
gh-ost-test-mysql-replica -uroot -e 'INSTALL PLUGIN ROCKSDB_PERF_CONTEXT SONAME "ha_rocksdb.so"'
gh-ost-test-mysql-replica -uroot -e 'INSTALL PLUGIN ROCKSDB_PERF_CONTEXT_GLOBAL SONAME "ha_rocksdb.so"'
gh-ost-test-mysql-replica -uroot -e 'INSTALL PLUGIN ROCKSDB_CF_OPTIONS SONAME "ha_rocksdb.so"'
gh-ost-test-mysql-replica -uroot -e 'INSTALL PLUGIN ROCKSDB_GLOBAL_INFO SONAME "ha_rocksdb.so"'
gh-ost-test-mysql-replica -uroot -e 'INSTALL PLUGIN ROCKSDB_COMPACTION_STATS SONAME "ha_rocksdb.so"'
gh-ost-test-mysql-replica -uroot -e 'INSTALL PLUGIN ROCKSDB_DDL SONAME "ha_rocksdb.so"'
gh-ost-test-mysql-replica -uroot -e 'INSTALL PLUGIN ROCKSDB_INDEX_FILE_MAP SONAME "ha_rocksdb.so"'
gh-ost-test-mysql-replica -uroot -e 'INSTALL PLUGIN ROCKSDB_LOCKS SONAME "ha_rocksdb.so"'
gh-ost-test-mysql-replica -uroot -e 'INSTALL PLUGIN ROCKSDB_TRX SONAME "ha_rocksdb.so"'
gh-ost-test-mysql-replica -uroot -e 'INSTALL PLUGIN ROCKSDB_DEADLOCK SONAME "ha_rocksdb.so"'
gh-ost-test-mysql-replica -uroot -e 'set global default_storage_engine="ROCKSDB"'
gh-ost-test-mysql-replica -uroot -e 'set global transaction_isolation="READ-COMMITTED"'
gh-ost-test-mysql-replica -uroot -e "INSTALL PLUGIN ROCKSDB SONAME 'ha_rocksdb.so'"
gh-ost-test-mysql-replica -uroot -e "INSTALL PLUGIN ROCKSDB_CFSTATS SONAME 'ha_rocksdb.so'"
gh-ost-test-mysql-replica -uroot -e "INSTALL PLUGIN ROCKSDB_DBSTATS SONAME 'ha_rocksdb.so'"
gh-ost-test-mysql-replica -uroot -e "INSTALL PLUGIN ROCKSDB_PERF_CONTEXT SONAME 'ha_rocksdb.so'"
gh-ost-test-mysql-replica -uroot -e "INSTALL PLUGIN ROCKSDB_PERF_CONTEXT_GLOBAL SONAME 'ha_rocksdb.so'"
gh-ost-test-mysql-replica -uroot -e "INSTALL PLUGIN ROCKSDB_CF_OPTIONS SONAME 'ha_rocksdb.so'"
gh-ost-test-mysql-replica -uroot -e "INSTALL PLUGIN ROCKSDB_GLOBAL_INFO SONAME 'ha_rocksdb.so'"
gh-ost-test-mysql-replica -uroot -e "INSTALL PLUGIN ROCKSDB_COMPACTION_HISTORY SONAME 'ha_rocksdb.so'"
gh-ost-test-mysql-replica -uroot -e "INSTALL PLUGIN ROCKSDB_COMPACTION_STATS SONAME 'ha_rocksdb.so'"
gh-ost-test-mysql-replica -uroot -e "INSTALL PLUGIN ROCKSDB_ACTIVE_COMPACTION_STATS SONAME 'ha_rocksdb.so'"
gh-ost-test-mysql-replica -uroot -e "INSTALL PLUGIN ROCKSDB_DDL SONAME 'ha_rocksdb.so'"
gh-ost-test-mysql-replica -uroot -e "INSTALL PLUGIN ROCKSDB_INDEX_FILE_MAP SONAME 'ha_rocksdb.so'"
gh-ost-test-mysql-replica -uroot -e "INSTALL PLUGIN ROCKSDB_LOCKS SONAME 'ha_rocksdb.so'"
gh-ost-test-mysql-replica -uroot -e "INSTALL PLUGIN ROCKSDB_TRX SONAME 'ha_rocksdb.so'"
gh-ost-test-mysql-replica -uroot -e "INSTALL PLUGIN ROCKSDB_DEADLOCK SONAME 'ha_rocksdb.so'"
gh-ost-test-mysql-replica -uroot -e "set global default_storage_engine='ROCKSDB'"
gh-ost-test-mysql-replica -uroot -e "set global transaction_isolation='READ-COMMITTED'"
echo "### Running gh-ost tests for $mysql_version"
./localtests/test.sh -b bin/gh-ost -s rocksdb
else
echo "### Running gh-ost tests for $mysql_version"
./localtests/test.sh -b bin/gh-ost -s innodb
fi
echo "### Running gh-ost tests for $mysql_version"
./localtests/test.sh -b bin/gh-ost
find sandboxes -name "stop_all" | bash
}