From b72ebfbbd2ca45c6d8cb2f3728f9c98a4d8c376f Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Wed, 5 Feb 2020 10:28:54 +0200 Subject: [PATCH 01/21] replication tests CI: switch to .tar.xz binaries, upgrade dbdeployer --- script/cibuild-gh-ost-replica-tests | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/script/cibuild-gh-ost-replica-tests b/script/cibuild-gh-ost-replica-tests index c0c925c..af3155f 100755 --- a/script/cibuild-gh-ost-replica-tests +++ b/script/cibuild-gh-ost-replica-tests @@ -6,7 +6,7 @@ whoami # Clone gh-ost-ci-env # Only clone if not already running locally at latest commit -remote_commit=$(git ls-remote https://github.com/github/gh-ost-ci-env.git real-tar-gz | cut -f1) +remote_commit=$(git ls-remote https://github.com/github/gh-ost-ci-env.git tar-xz-binaries | cut -f1) local_commit="unknown" [ -d "gh-ost-ci-env" ] && local_commit=$(cd gh-ost-ci-env && git log --format="%H" -n 1) @@ -18,8 +18,8 @@ if [ "$remote_commit" != "$local_commit" ] ; then git clone https://github.com/github/gh-ost-ci-env.git ( cd gh-ost-ci-env - git fetch origin real-tar-gz - git checkout real-tar-gz + git fetch origin tar-xz-binaries + git checkout tar-xz-binaries ) fi @@ -35,7 +35,7 @@ test_mysql_version() { mkdir -p sandbox/binary rm -rf sandbox/binary/* - gh-ost-ci-env/bin/linux/dbdeployer unpack gh-ost-ci-env/mysql-tarballs/"$mysql_version".tar.gz --unpack-version="$mysql_version" --sandbox-binary ${PWD}/sandbox/binary + gh-ost-ci-env/bin/linux/dbdeployer unpack gh-ost-ci-env/mysql-tarballs/"$mysql_version".tar.xz --unpack-version="$mysql_version" --sandbox-binary ${PWD}/sandbox/binary mkdir -p sandboxes rm -rf sandboxes/* @@ -67,9 +67,9 @@ test_mysql_version() { echo "Building..." . script/build # Test all versions: -find gh-ost-ci-env/mysql-tarballs/ -name "*.tar.gz" | while read f ; do basename $f ".tar.gz" ; done | sort -r | while read mysql_version ; do +find gh-ost-ci-env/mysql-tarballs/ -name "*.tar.xz" | while read f ; do basename $f ".tar.xz" ; done | sort -r | while read mysql_version ; do echo "found MySQL version: $mysql_version" done -find gh-ost-ci-env/mysql-tarballs/ -name "*.tar.gz" | while read f ; do basename $f ".tar.gz" ; done | sort -r | while read mysql_version ; do +find gh-ost-ci-env/mysql-tarballs/ -name "*.tar.xz" | while read f ; do basename $f ".tar.xz" ; done | sort -r | while read mysql_version ; do test_mysql_version "$mysql_version" done From 4e5a708f307b94cfecd71a7c302dc6a2e2d115a2 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Wed, 5 Feb 2020 10:49:52 +0200 Subject: [PATCH 02/21] testing dbdeployer version --- script/cibuild-gh-ost-replica-tests | 63 +++++++++++++++++------------ 1 file changed, 38 insertions(+), 25 deletions(-) diff --git a/script/cibuild-gh-ost-replica-tests b/script/cibuild-gh-ost-replica-tests index af3155f..f6504b4 100755 --- a/script/cibuild-gh-ost-replica-tests +++ b/script/cibuild-gh-ost-replica-tests @@ -4,24 +4,30 @@ set -e whoami -# Clone gh-ost-ci-env -# Only clone if not already running locally at latest commit -remote_commit=$(git ls-remote https://github.com/github/gh-ost-ci-env.git tar-xz-binaries | cut -f1) -local_commit="unknown" -[ -d "gh-ost-ci-env" ] && local_commit=$(cd gh-ost-ci-env && git log --format="%H" -n 1) +fetch_ci_env() { + # Clone gh-ost-ci-env + # Only clone if not already running locally at latest commit + remote_commit=$(git ls-remote https://github.com/github/gh-ost-ci-env.git tar-xz-binaries | cut -f1) + local_commit="unknown" + [ -d "gh-ost-ci-env" ] && local_commit=$(cd gh-ost-ci-env && git log --format="%H" -n 1) -echo "remote commit is: $remote_commit" -echo "local commit is: $local_commit" + echo "remote commit is: $remote_commit" + echo "local commit is: $local_commit" -if [ "$remote_commit" != "$local_commit" ] ; then - rm -rf ./gh-ost-ci-env - git clone https://github.com/github/gh-ost-ci-env.git - ( - cd gh-ost-ci-env - git fetch origin tar-xz-binaries - git checkout tar-xz-binaries - ) -fi + if [ "$remote_commit" != "$local_commit" ] ; then + rm -rf ./gh-ost-ci-env + git clone https://github.com/github/gh-ost-ci-env.git + ( + cd gh-ost-ci-env + git fetch origin tar-xz-binaries + git checkout tar-xz-binaries + ) + fi +} + +test_dbdeployer() { + gh-ost-ci-env/bin/linux/dbdeployer --version +} test_mysql_version() { local mysql_version @@ -64,12 +70,19 @@ test_mysql_version() { find sandboxes -name "stop_all" | bash } -echo "Building..." -. script/build -# Test all versions: -find gh-ost-ci-env/mysql-tarballs/ -name "*.tar.xz" | while read f ; do basename $f ".tar.xz" ; done | sort -r | while read mysql_version ; do - echo "found MySQL version: $mysql_version" -done -find gh-ost-ci-env/mysql-tarballs/ -name "*.tar.xz" | while read f ; do basename $f ".tar.xz" ; done | sort -r | while read mysql_version ; do - test_mysql_version "$mysql_version" -done +main() { + fetch_ci_env + test_dbdeployer + + echo "Building..." + . script/build + # Test all versions: + find gh-ost-ci-env/mysql-tarballs/ -name "*.tar.xz" | while read f ; do basename $f ".tar.xz" ; done | sort -r | while read mysql_version ; do + echo "found MySQL version: $mysql_version" + done + find gh-ost-ci-env/mysql-tarballs/ -name "*.tar.xz" | while read f ; do basename $f ".tar.xz" ; done | sort -r | while read mysql_version ; do + test_mysql_version "$mysql_version" + done +} + +main() From abc30b96c88bc5a87240c90721029210bff0beef Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Wed, 5 Feb 2020 10:52:25 +0200 Subject: [PATCH 03/21] typo --- script/cibuild-gh-ost-replica-tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/cibuild-gh-ost-replica-tests b/script/cibuild-gh-ost-replica-tests index f6504b4..ca435b8 100755 --- a/script/cibuild-gh-ost-replica-tests +++ b/script/cibuild-gh-ost-replica-tests @@ -85,4 +85,4 @@ main() { done } -main() +main From 5a2bcfc09b97224e6f12f7cacf2a90398a8d66ce Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Wed, 5 Feb 2020 10:59:39 +0200 Subject: [PATCH 04/21] try not specify version --- script/cibuild-gh-ost-replica-tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/cibuild-gh-ost-replica-tests b/script/cibuild-gh-ost-replica-tests index ca435b8..237c483 100755 --- a/script/cibuild-gh-ost-replica-tests +++ b/script/cibuild-gh-ost-replica-tests @@ -41,7 +41,7 @@ test_mysql_version() { mkdir -p sandbox/binary rm -rf sandbox/binary/* - gh-ost-ci-env/bin/linux/dbdeployer unpack gh-ost-ci-env/mysql-tarballs/"$mysql_version".tar.xz --unpack-version="$mysql_version" --sandbox-binary ${PWD}/sandbox/binary + gh-ost-ci-env/bin/linux/dbdeployer unpack gh-ost-ci-env/mysql-tarballs/"$mysql_version".tar.xz --sandbox-binary ${PWD}/sandbox/binary mkdir -p sandboxes rm -rf sandboxes/* From c480ff133790d6558fcb6457c1a173ca62bdd906 Mon Sep 17 00:00:00 2001 From: Yakir Gibraltar Date: Tue, 6 Apr 2021 18:15:05 +0300 Subject: [PATCH 05/21] Remove build_id files from rpm --- build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sh b/build.sh index b5d4659..88ecf1e 100755 --- a/build.sh +++ b/build.sh @@ -40,7 +40,7 @@ function build { builddir=$(setuptree) cp $buildpath/$target $builddir/gh-ost/usr/bin cd $buildpath - fpm -v "${RELEASE_VERSION}" --epoch 1 -f -s dir -n gh-ost -m 'shlomi-noach ' --description "GitHub's Online Schema Migrations for MySQL " --url "https://github.com/github/gh-ost" --vendor "GitHub" --license "Apache 2.0" -C $builddir/gh-ost --prefix=/ -t rpm . + fpm -v "${RELEASE_VERSION}" --epoch 1 -f -s dir -n gh-ost -m 'shlomi-noach ' --description "GitHub's Online Schema Migrations for MySQL " --url "https://github.com/github/gh-ost" --vendor "GitHub" --license "Apache 2.0" -C $builddir/gh-ost --prefix=/ -t rpm --rpm-rpmbuild-define "_build_id_links none" . fpm -v "${RELEASE_VERSION}" --epoch 1 -f -s dir -n gh-ost -m 'shlomi-noach ' --description "GitHub's Online Schema Migrations for MySQL " --url "https://github.com/github/gh-ost" --vendor "GitHub" --license "Apache 2.0" -C $builddir/gh-ost --prefix=/ -t deb --deb-no-default-config-files . fi } From f40f14b9eeba6205fdcc303132548a6e29ce3318 Mon Sep 17 00:00:00 2001 From: "Fan()" <18501341937@163.com> Date: Thu, 29 Apr 2021 13:43:24 +0800 Subject: [PATCH 06/21] Update inspect.go fix https://github.com/github/gh-ost/issues/961 --- go/logic/inspect.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/logic/inspect.go b/go/logic/inspect.go index 0128010..3d101c0 100644 --- a/go/logic/inspect.go +++ b/go/logic/inspect.go @@ -528,7 +528,7 @@ func (this *Inspector) CountTableRows() error { this.migrationContext.Log.Infof("As instructed, I'm issuing a SELECT COUNT(*) on the table. This may take a while") - query := fmt.Sprintf(`select /* gh-ost */ count(*) as rows from %s.%s`, sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName)) + query := fmt.Sprintf(`select /* gh-ost */ count(*) as 'rows' from %s.%s`, sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName)) var rowsEstimate int64 if err := this.db.QueryRow(query).Scan(&rowsEstimate); err != nil { return err From 74f807103ee4391693054c892c845ea1b99edffc Mon Sep 17 00:00:00 2001 From: "Fan()" <18501341937@163.com> Date: Thu, 29 Apr 2021 14:33:10 +0800 Subject: [PATCH 07/21] Update inspect.go --- go/logic/inspect.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/logic/inspect.go b/go/logic/inspect.go index 3d101c0..584c56b 100644 --- a/go/logic/inspect.go +++ b/go/logic/inspect.go @@ -528,7 +528,7 @@ func (this *Inspector) CountTableRows() error { this.migrationContext.Log.Infof("As instructed, I'm issuing a SELECT COUNT(*) on the table. This may take a while") - query := fmt.Sprintf(`select /* gh-ost */ count(*) as 'rows' from %s.%s`, sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName)) + query := fmt.Sprintf(`select /* gh-ost */ count(*) as count_rows from %s.%s`, sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName)) var rowsEstimate int64 if err := this.db.QueryRow(query).Scan(&rowsEstimate); err != nil { return err From afa221bb45a5d0c8065c78be63b0d2696267c225 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Fri, 7 May 2021 22:38:25 +0200 Subject: [PATCH 08/21] Use HEAD --- script/cibuild-gh-ost-replica-tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/cibuild-gh-ost-replica-tests b/script/cibuild-gh-ost-replica-tests index 237c483..161fc99 100755 --- a/script/cibuild-gh-ost-replica-tests +++ b/script/cibuild-gh-ost-replica-tests @@ -7,7 +7,7 @@ whoami fetch_ci_env() { # Clone gh-ost-ci-env # Only clone if not already running locally at latest commit - remote_commit=$(git ls-remote https://github.com/github/gh-ost-ci-env.git tar-xz-binaries | cut -f1) + remote_commit=$(git ls-remote https://github.com/github/gh-ost-ci-env.git HEAD | cut -f1) local_commit="unknown" [ -d "gh-ost-ci-env" ] && local_commit=$(cd gh-ost-ci-env && git log --format="%H" -n 1) From 6aaab786573a41a4cae7d8d3126da8dd08494613 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Fri, 7 May 2021 23:22:18 +0200 Subject: [PATCH 09/21] Use fix-8016-tarball branch --- script/cibuild-gh-ost-replica-tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/cibuild-gh-ost-replica-tests b/script/cibuild-gh-ost-replica-tests index 161fc99..c71b18d 100755 --- a/script/cibuild-gh-ost-replica-tests +++ b/script/cibuild-gh-ost-replica-tests @@ -7,7 +7,7 @@ whoami fetch_ci_env() { # Clone gh-ost-ci-env # Only clone if not already running locally at latest commit - remote_commit=$(git ls-remote https://github.com/github/gh-ost-ci-env.git HEAD | cut -f1) + remote_commit=$(git ls-remote https://github.com/github/gh-ost-ci-env.git fix-8016-tarball | cut -f1) local_commit="unknown" [ -d "gh-ost-ci-env" ] && local_commit=$(cd gh-ost-ci-env && git log --format="%H" -n 1) From 8df36d4bfd44d3bf568d986c38bbde6aeefcb168 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Fri, 7 May 2021 23:32:24 +0200 Subject: [PATCH 10/21] Fix git checkout --- script/cibuild-gh-ost-replica-tests | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script/cibuild-gh-ost-replica-tests b/script/cibuild-gh-ost-replica-tests index c71b18d..d95c90b 100755 --- a/script/cibuild-gh-ost-replica-tests +++ b/script/cibuild-gh-ost-replica-tests @@ -19,8 +19,8 @@ fetch_ci_env() { git clone https://github.com/github/gh-ost-ci-env.git ( cd gh-ost-ci-env - git fetch origin tar-xz-binaries - git checkout tar-xz-binaries + git fetch origin fix-8016-tarball + git checkout fix-8016-tarball ) fi } From 3976b7acf2cf3e09400c636438bbf708198ef917 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Fri, 7 May 2021 23:50:24 +0200 Subject: [PATCH 11/21] Remove 'mysql-' prefix from mysql version so dbdeployer can make a port num --- script/cibuild-gh-ost-replica-tests | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/script/cibuild-gh-ost-replica-tests b/script/cibuild-gh-ost-replica-tests index d95c90b..ab9d7e7 100755 --- a/script/cibuild-gh-ost-replica-tests +++ b/script/cibuild-gh-ost-replica-tests @@ -46,12 +46,13 @@ test_mysql_version() { mkdir -p sandboxes rm -rf sandboxes/* - if echo "$mysql_version" | egrep "5[.]5[.]" ; then + local mysql_version_num=${version#*-} + if echo "$mysql_version_num" | egrep "5[.]5[.]" ; then gtid="" else gtid="--gtid" fi - gh-ost-ci-env/bin/linux/dbdeployer deploy replication "$mysql_version" --nodes 2 --sandbox-binary ${PWD}/sandbox/binary --sandbox-home ${PWD}/sandboxes ${gtid} --my-cnf-options log_slave_updates --my-cnf-options log_bin --my-cnf-options binlog_format=ROW --sandbox-directory rsandbox + gh-ost-ci-env/bin/linux/dbdeployer deploy replication "$mysql_version_num" --nodes 2 --sandbox-binary ${PWD}/sandbox/binary --sandbox-home ${PWD}/sandboxes ${gtid} --my-cnf-options log_slave_updates --my-cnf-options log_bin --my-cnf-options binlog_format=ROW --sandbox-directory rsandbox sed '/sandboxes/d' -i gh-ost-ci-env/bin/gh-ost-test-mysql-master echo 'sandboxes/rsandbox/m "$@"' >> gh-ost-ci-env/bin/gh-ost-test-mysql-master From 7ae7cc67f7b773c14b9c896ee27dfe0a8328006f Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Fri, 7 May 2021 23:52:01 +0200 Subject: [PATCH 12/21] Use right var name --- script/cibuild-gh-ost-replica-tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/cibuild-gh-ost-replica-tests b/script/cibuild-gh-ost-replica-tests index ab9d7e7..10d8e67 100755 --- a/script/cibuild-gh-ost-replica-tests +++ b/script/cibuild-gh-ost-replica-tests @@ -46,7 +46,7 @@ test_mysql_version() { mkdir -p sandboxes rm -rf sandboxes/* - local mysql_version_num=${version#*-} + local mysql_version_num=${mysql_version#*-} if echo "$mysql_version_num" | egrep "5[.]5[.]" ; then gtid="" else From 5ebbfaea893c533dc840bea30b1aeb199b1e55fa Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Sat, 8 May 2021 00:23:31 +0200 Subject: [PATCH 13/21] Use matrix build for replica test CI --- .github/workflows/replica-tests.yml | 5 +++++ script/cibuild-gh-ost-replica-tests | 22 +++++++++++++--------- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/.github/workflows/replica-tests.yml b/.github/workflows/replica-tests.yml index 31e2052..bbc53a5 100644 --- a/.github/workflows/replica-tests.yml +++ b/.github/workflows/replica-tests.yml @@ -6,6 +6,9 @@ jobs: build: runs-on: ubuntu-latest + strategy: + matrix: + version: [mysql-5.5.62,mysql-5.6.43,mysql-5.7.25,mysql-8.0.16] steps: - uses: actions/checkout@v2 @@ -16,4 +19,6 @@ jobs: go-version: 1.14 - name: migration tests + env: + TEST_MYSQL_VERSION: ${{ matrix.version }} run: script/cibuild-gh-ost-replica-tests diff --git a/script/cibuild-gh-ost-replica-tests b/script/cibuild-gh-ost-replica-tests index 3de9e05..a7f6035 100755 --- a/script/cibuild-gh-ost-replica-tests +++ b/script/cibuild-gh-ost-replica-tests @@ -59,12 +59,16 @@ test_mysql_version() { find sandboxes -name "stop_all" | bash } -echo "Building..." -. script/build -# Test all versions: -find gh-ost-ci-env/mysql-tarballs/ -name "*.tar.gz" | while read f ; do basename $f ".tar.gz" ; done | sort -r | while read mysql_version ; do - echo "found MySQL version: $mysql_version" -done -find gh-ost-ci-env/mysql-tarballs/ -name "*.tar.gz" | while read f ; do basename $f ".tar.gz" ; done | sort -r | while read mysql_version ; do - test_mysql_version "$mysql_version" -done +# TEST_MYSQL_VERSION is set by the replica-tests CI job +if [ -z "$TEST_MYSQL_VERSION" ]; then + # Test all versions: + find gh-ost-ci-env/mysql-tarballs/ -name "*.tar.gz" | while read f ; do basename $f ".tar.gz" ; done | sort -r | while read mysql_version ; do + echo "found MySQL version: $mysql_version" + done + find gh-ost-ci-env/mysql-tarballs/ -name "*.tar.gz" | while read f ; do basename $f ".tar.gz" ; done | sort -r | while read mysql_version ; do + test_mysql_version "$mysql_version" + done +else + echo "found MySQL version: $MYSQL_VERSION" + test_mysql_version "$MYSQL_VERSION" +fi From 2634534cf505b06bb39bcef3ab0f26e8f7114a2a Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Sat, 8 May 2021 00:26:46 +0200 Subject: [PATCH 14/21] Fix var name --- script/cibuild-gh-ost-replica-tests | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script/cibuild-gh-ost-replica-tests b/script/cibuild-gh-ost-replica-tests index a7f6035..40f7380 100755 --- a/script/cibuild-gh-ost-replica-tests +++ b/script/cibuild-gh-ost-replica-tests @@ -69,6 +69,6 @@ if [ -z "$TEST_MYSQL_VERSION" ]; then test_mysql_version "$mysql_version" done else - echo "found MySQL version: $MYSQL_VERSION" - test_mysql_version "$MYSQL_VERSION" + echo "found MySQL version: $TEST_MYSQL_VERSION" + test_mysql_version "$TEST_MYSQL_VERSION" fi From 535687023eebdf928f167a7239182cb1b3c5f29f Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Sat, 8 May 2021 00:34:38 +0200 Subject: [PATCH 15/21] Fetch master branch of ci env --- script/cibuild-gh-ost-replica-tests | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/script/cibuild-gh-ost-replica-tests b/script/cibuild-gh-ost-replica-tests index 10d8e67..0d664c9 100755 --- a/script/cibuild-gh-ost-replica-tests +++ b/script/cibuild-gh-ost-replica-tests @@ -7,7 +7,7 @@ whoami fetch_ci_env() { # Clone gh-ost-ci-env # Only clone if not already running locally at latest commit - remote_commit=$(git ls-remote https://github.com/github/gh-ost-ci-env.git fix-8016-tarball | cut -f1) + remote_commit=$(git ls-remote https://github.com/github/gh-ost-ci-env.git HEAD | cut -f1) local_commit="unknown" [ -d "gh-ost-ci-env" ] && local_commit=$(cd gh-ost-ci-env && git log --format="%H" -n 1) @@ -19,8 +19,8 @@ fetch_ci_env() { git clone https://github.com/github/gh-ost-ci-env.git ( cd gh-ost-ci-env - git fetch origin fix-8016-tarball - git checkout fix-8016-tarball + git fetch origin master + git checkout master ) fi } From cd5beeb495c19478457c708870100e45284042bf Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Sat, 8 May 2021 00:35:19 +0200 Subject: [PATCH 16/21] Remove git branch steps --- script/cibuild-gh-ost-replica-tests | 5 ----- 1 file changed, 5 deletions(-) diff --git a/script/cibuild-gh-ost-replica-tests b/script/cibuild-gh-ost-replica-tests index 0d664c9..b886fa0 100755 --- a/script/cibuild-gh-ost-replica-tests +++ b/script/cibuild-gh-ost-replica-tests @@ -17,11 +17,6 @@ fetch_ci_env() { if [ "$remote_commit" != "$local_commit" ] ; then rm -rf ./gh-ost-ci-env git clone https://github.com/github/gh-ost-ci-env.git - ( - cd gh-ost-ci-env - git fetch origin master - git checkout master - ) fi } From a25f63aa2abe4d39c2b8136f6f39e3299cb2c9ec Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Sat, 8 May 2021 00:44:28 +0200 Subject: [PATCH 17/21] merge tar-xz-dbdeployer-upgrade --- script/cibuild-gh-ost-replica-tests | 71 +++++++++++++++++++---------- 1 file changed, 46 insertions(+), 25 deletions(-) diff --git a/script/cibuild-gh-ost-replica-tests b/script/cibuild-gh-ost-replica-tests index 40f7380..0edf617 100755 --- a/script/cibuild-gh-ost-replica-tests +++ b/script/cibuild-gh-ost-replica-tests @@ -4,19 +4,30 @@ set -e whoami -# Clone gh-ost-ci-env -# Only clone if not already running locally at latest commit -remote_commit=$(git ls-remote https://github.com/github/gh-ost-ci-env.git HEAD | cut -f1) -local_commit="unknown" -[ -d "gh-ost-ci-env" ] && local_commit=$(cd gh-ost-ci-env && git log --format="%H" -n 1) +fetch_ci_env() { + # Clone gh-ost-ci-env + # Only clone if not already running locally at latest commit + remote_commit=$(git ls-remote https://github.com/github/gh-ost-ci-env.git tar-xz-binaries | cut -f1) + local_commit="unknown" + [ -d "gh-ost-ci-env" ] && local_commit=$(cd gh-ost-ci-env && git log --format="%H" -n 1) -echo "remote commit is: $remote_commit" -echo "local commit is: $local_commit" + echo "remote commit is: $remote_commit" + echo "local commit is: $local_commit" -if [ "$remote_commit" != "$local_commit" ] ; then - rm -rf ./gh-ost-ci-env - git clone https://github.com/github/gh-ost-ci-env.git -fi + if [ "$remote_commit" != "$local_commit" ] ; then + rm -rf ./gh-ost-ci-env + git clone https://github.com/github/gh-ost-ci-env.git + ( + cd gh-ost-ci-env + git fetch origin tar-xz-binaries + git checkout tar-xz-binaries + ) + fi +} + +test_dbdeployer() { + gh-ost-ci-env/bin/linux/dbdeployer --version +} test_mysql_version() { local mysql_version @@ -30,7 +41,7 @@ test_mysql_version() { mkdir -p sandbox/binary rm -rf sandbox/binary/* - gh-ost-ci-env/bin/linux/dbdeployer unpack gh-ost-ci-env/mysql-tarballs/"$mysql_version".tar.gz --unpack-version="$mysql_version" --sandbox-binary ${PWD}/sandbox/binary + gh-ost-ci-env/bin/linux/dbdeployer unpack gh-ost-ci-env/mysql-tarballs/"$mysql_version".tar.xz --sandbox-binary ${PWD}/sandbox/binary mkdir -p sandboxes rm -rf sandboxes/* @@ -59,16 +70,26 @@ test_mysql_version() { find sandboxes -name "stop_all" | bash } -# TEST_MYSQL_VERSION is set by the replica-tests CI job -if [ -z "$TEST_MYSQL_VERSION" ]; then - # Test all versions: - find gh-ost-ci-env/mysql-tarballs/ -name "*.tar.gz" | while read f ; do basename $f ".tar.gz" ; done | sort -r | while read mysql_version ; do - echo "found MySQL version: $mysql_version" - done - find gh-ost-ci-env/mysql-tarballs/ -name "*.tar.gz" | while read f ; do basename $f ".tar.gz" ; done | sort -r | while read mysql_version ; do - test_mysql_version "$mysql_version" - done -else - echo "found MySQL version: $TEST_MYSQL_VERSION" - test_mysql_version "$TEST_MYSQL_VERSION" -fi +main() { + fetch_ci_env + test_dbdeployer + + echo "Building..." + . script/build + + # TEST_MYSQL_VERSION is set by the replica-tests CI job + if [ -z "$TEST_MYSQL_VERSION" ]; then + # Test all versions: + find gh-ost-ci-env/mysql-tarballs/ -name "*.tar.xz" | while read f ; do basename $f ".tar.xz" ; done | sort -r | while read mysql_version ; do + echo "found MySQL version: $mysql_version" + done + find gh-ost-ci-env/mysql-tarballs/ -name "*.tar.xz" | while read f ; do basename $f ".tar.xz" ; done | sort -r | while read mysql_version ; do + test_mysql_version "$mysql_version" + done + else + echo "found MySQL version: $TEST_MYSQL_VERSION" + test_mysql_version "$TEST_MYSQL_VERSION" + fi +} + +main From 5e37110cfd92f61013baa89b164dd5edd6afe592 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Sat, 8 May 2021 00:45:59 +0200 Subject: [PATCH 18/21] merge tar-xz-dbdeployer-upgrade --- build.sh | 2 +- doc/hooks.md | 1 + go/base/context.go | 21 ++++++++++++++++ go/logic/hooks.go | 1 + go/logic/inspect.go | 2 +- go/logic/migrator.go | 38 +++++++++++++++++++++++++---- script/cibuild-gh-ost-replica-tests | 12 +++------ 7 files changed, 62 insertions(+), 15 deletions(-) diff --git a/build.sh b/build.sh index b5d4659..88ecf1e 100755 --- a/build.sh +++ b/build.sh @@ -40,7 +40,7 @@ function build { builddir=$(setuptree) cp $buildpath/$target $builddir/gh-ost/usr/bin cd $buildpath - fpm -v "${RELEASE_VERSION}" --epoch 1 -f -s dir -n gh-ost -m 'shlomi-noach ' --description "GitHub's Online Schema Migrations for MySQL " --url "https://github.com/github/gh-ost" --vendor "GitHub" --license "Apache 2.0" -C $builddir/gh-ost --prefix=/ -t rpm . + fpm -v "${RELEASE_VERSION}" --epoch 1 -f -s dir -n gh-ost -m 'shlomi-noach ' --description "GitHub's Online Schema Migrations for MySQL " --url "https://github.com/github/gh-ost" --vendor "GitHub" --license "Apache 2.0" -C $builddir/gh-ost --prefix=/ -t rpm --rpm-rpmbuild-define "_build_id_links none" . fpm -v "${RELEASE_VERSION}" --epoch 1 -f -s dir -n gh-ost -m 'shlomi-noach ' --description "GitHub's Online Schema Migrations for MySQL " --url "https://github.com/github/gh-ost" --vendor "GitHub" --license "Apache 2.0" -C $builddir/gh-ost --prefix=/ -t deb --deb-no-default-config-files . fi } diff --git a/doc/hooks.md b/doc/hooks.md index 4c49c85..91e1311 100644 --- a/doc/hooks.md +++ b/doc/hooks.md @@ -66,6 +66,7 @@ The following variables are available on all hooks: - `GH_OST_ESTIMATED_ROWS` - estimated total rows in table - `GH_OST_COPIED_ROWS` - number of rows copied by `gh-ost` - `GH_OST_INSPECTED_LAG` - lag in seconds (floating point) of inspected server +- `GH_OST_HEARTBEAT_LAG` - lag in seconds (floating point) of heartbeat - `GH_OST_PROGRESS` - progress pct ([0..100], floating point) of migration - `GH_OST_MIGRATED_HOST` - `GH_OST_INSPECTED_HOST` diff --git a/go/base/context.go b/go/base/context.go index 3211067..3757653 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -178,6 +178,8 @@ type MigrationContext struct { RenameTablesEndTime time.Time pointOfInterestTime time.Time pointOfInterestTimeMutex *sync.Mutex + lastHeartbeatOnChangelogTime time.Time + lastHeartbeatOnChangelogMutex *sync.Mutex CurrentLag int64 currentProgress uint64 ThrottleHTTPStatusCode int64 @@ -271,6 +273,7 @@ func NewMigrationContext() *MigrationContext { throttleControlReplicaKeys: mysql.NewInstanceKeyMap(), configMutex: &sync.Mutex{}, pointOfInterestTimeMutex: &sync.Mutex{}, + lastHeartbeatOnChangelogMutex: &sync.Mutex{}, ColumnRenameMap: make(map[string]string), PanicAbort: make(chan error), Log: NewDefaultLogger(), @@ -454,6 +457,10 @@ func (this *MigrationContext) MarkRowCopyEndTime() { this.RowCopyEndTime = time.Now() } +func (this *MigrationContext) TimeSinceLastHeartbeatOnChangelog() time.Duration { + return time.Since(this.GetLastHeartbeatOnChangelogTime()) +} + func (this *MigrationContext) GetCurrentLagDuration() time.Duration { return time.Duration(atomic.LoadInt64(&this.CurrentLag)) } @@ -493,6 +500,20 @@ func (this *MigrationContext) TimeSincePointOfInterest() time.Duration { return time.Since(this.pointOfInterestTime) } +func (this *MigrationContext) SetLastHeartbeatOnChangelogTime(t time.Time) { + this.lastHeartbeatOnChangelogMutex.Lock() + defer this.lastHeartbeatOnChangelogMutex.Unlock() + + this.lastHeartbeatOnChangelogTime = t +} + +func (this *MigrationContext) GetLastHeartbeatOnChangelogTime() time.Time { + this.lastHeartbeatOnChangelogMutex.Lock() + defer this.lastHeartbeatOnChangelogMutex.Unlock() + + return this.lastHeartbeatOnChangelogTime +} + func (this *MigrationContext) SetHeartbeatIntervalMilliseconds(heartbeatIntervalMilliseconds int64) { if heartbeatIntervalMilliseconds < 100 { heartbeatIntervalMilliseconds = 100 diff --git a/go/logic/hooks.go b/go/logic/hooks.go index fa5011e..71f070c 100644 --- a/go/logic/hooks.go +++ b/go/logic/hooks.go @@ -64,6 +64,7 @@ func (this *HooksExecutor) applyEnvironmentVariables(extraVariables ...string) [ env = append(env, fmt.Sprintf("GH_OST_INSPECTED_HOST=%s", this.migrationContext.GetInspectorHostname())) env = append(env, fmt.Sprintf("GH_OST_EXECUTING_HOST=%s", this.migrationContext.Hostname)) env = append(env, fmt.Sprintf("GH_OST_INSPECTED_LAG=%f", this.migrationContext.GetCurrentLagDuration().Seconds())) + env = append(env, fmt.Sprintf("GH_OST_HEARTBEAT_LAG=%f", this.migrationContext.TimeSinceLastHeartbeatOnChangelog().Seconds())) env = append(env, fmt.Sprintf("GH_OST_PROGRESS=%f", this.migrationContext.GetProgressPct())) env = append(env, fmt.Sprintf("GH_OST_HOOKS_HINT=%s", this.migrationContext.HooksHintMessage)) env = append(env, fmt.Sprintf("GH_OST_HOOKS_HINT_OWNER=%s", this.migrationContext.HooksHintOwner)) diff --git a/go/logic/inspect.go b/go/logic/inspect.go index 0128010..584c56b 100644 --- a/go/logic/inspect.go +++ b/go/logic/inspect.go @@ -528,7 +528,7 @@ func (this *Inspector) CountTableRows() error { this.migrationContext.Log.Infof("As instructed, I'm issuing a SELECT COUNT(*) on the table. This may take a while") - query := fmt.Sprintf(`select /* gh-ost */ count(*) as rows from %s.%s`, sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName)) + query := fmt.Sprintf(`select /* gh-ost */ count(*) as count_rows from %s.%s`, sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName)) var rowsEstimate int64 if err := this.db.QueryRow(query).Scan(&rowsEstimate); err != nil { return err diff --git a/go/logic/migrator.go b/go/logic/migrator.go index 291a490..2a038a8 100644 --- a/go/logic/migrator.go +++ b/go/logic/migrator.go @@ -207,12 +207,20 @@ func (this *Migrator) canStopStreaming() bool { return atomic.LoadInt64(&this.migrationContext.CutOverCompleteFlag) != 0 } -// onChangelogStateEvent is called when a binlog event operation on the changelog table is intercepted. -func (this *Migrator) onChangelogStateEvent(dmlEvent *binlog.BinlogDMLEvent) (err error) { +// onChangelogEvent is called when a binlog event operation on the changelog table is intercepted. +func (this *Migrator) onChangelogEvent(dmlEvent *binlog.BinlogDMLEvent) (err error) { // Hey, I created the changelog table, I know the type of columns it has! - if hint := dmlEvent.NewColumnValues.StringColumn(2); hint != "state" { + switch hint := dmlEvent.NewColumnValues.StringColumn(2); hint { + case "state": + return this.onChangelogStateEvent(dmlEvent) + case "heartbeat": + return this.onChangelogHeartbeatEvent(dmlEvent) + default: return nil } +} + +func (this *Migrator) onChangelogStateEvent(dmlEvent *binlog.BinlogDMLEvent) (err error) { changelogStateString := dmlEvent.NewColumnValues.StringColumn(3) changelogState := ReadChangelogState(changelogStateString) this.migrationContext.Log.Infof("Intercepted changelog state %s", changelogState) @@ -245,6 +253,18 @@ func (this *Migrator) onChangelogStateEvent(dmlEvent *binlog.BinlogDMLEvent) (er return nil } +func (this *Migrator) onChangelogHeartbeatEvent(dmlEvent *binlog.BinlogDMLEvent) (err error) { + changelogHeartbeatString := dmlEvent.NewColumnValues.StringColumn(3) + + heartbeatTime, err := time.Parse(time.RFC3339Nano, changelogHeartbeatString) + if err != nil { + return this.migrationContext.Log.Errore(err) + } else { + this.migrationContext.SetLastHeartbeatOnChangelogTime(heartbeatTime) + return nil + } +} + // listenOnPanicAbort aborts on abort request func (this *Migrator) listenOnPanicAbort() { err := <-this.migrationContext.PanicAbort @@ -476,6 +496,13 @@ func (this *Migrator) cutOver() (err error) { this.migrationContext.Log.Debugf("checking for cut-over postpone") this.sleepWhileTrue( func() (bool, error) { + heartbeatLag := this.migrationContext.TimeSinceLastHeartbeatOnChangelog() + maxLagMillisecondsThrottle := time.Duration(atomic.LoadInt64(&this.migrationContext.MaxLagMillisecondsThrottleThreshold)) * time.Millisecond + cutOverLockTimeout := time.Duration(this.migrationContext.CutOverLockTimeoutSeconds) * time.Second + if heartbeatLag > maxLagMillisecondsThrottle || heartbeatLag > cutOverLockTimeout { + this.migrationContext.Log.Debugf("current HeartbeatLag (%.2fs) is too high, it needs to be less than both --max-lag-millis (%.2fs) and --cut-over-lock-timeout-seconds (%.2fs) to continue", heartbeatLag.Seconds(), maxLagMillisecondsThrottle.Seconds(), cutOverLockTimeout.Seconds()) + return true, nil + } if this.migrationContext.PostponeCutOverFlagFile == "" { return false, nil } @@ -962,13 +989,14 @@ func (this *Migrator) printStatus(rule PrintStatusRule, writers ...io.Writer) { currentBinlogCoordinates := *this.eventsStreamer.GetCurrentBinlogCoordinates() - status := fmt.Sprintf("Copy: %d/%d %.1f%%; Applied: %d; Backlog: %d/%d; Time: %+v(total), %+v(copy); streamer: %+v; Lag: %.2fs, State: %s; ETA: %s", + status := fmt.Sprintf("Copy: %d/%d %.1f%%; Applied: %d; Backlog: %d/%d; Time: %+v(total), %+v(copy); streamer: %+v; Lag: %.2fs, HeartbeatLag: %.2fs, State: %s; ETA: %s", totalRowsCopied, rowsEstimate, progressPct, atomic.LoadInt64(&this.migrationContext.TotalDMLEventsApplied), len(this.applyEventsQueue), cap(this.applyEventsQueue), base.PrettifyDurationOutput(elapsedTime), base.PrettifyDurationOutput(this.migrationContext.ElapsedRowCopyTime()), currentBinlogCoordinates, this.migrationContext.GetCurrentLagDuration().Seconds(), + this.migrationContext.TimeSinceLastHeartbeatOnChangelog().Seconds(), state, eta, ) @@ -995,7 +1023,7 @@ func (this *Migrator) initiateStreaming() error { this.migrationContext.DatabaseName, this.migrationContext.GetChangelogTableName(), func(dmlEvent *binlog.BinlogDMLEvent) error { - return this.onChangelogStateEvent(dmlEvent) + return this.onChangelogEvent(dmlEvent) }, ) diff --git a/script/cibuild-gh-ost-replica-tests b/script/cibuild-gh-ost-replica-tests index 0edf617..c4dbfd2 100755 --- a/script/cibuild-gh-ost-replica-tests +++ b/script/cibuild-gh-ost-replica-tests @@ -7,7 +7,7 @@ whoami fetch_ci_env() { # Clone gh-ost-ci-env # Only clone if not already running locally at latest commit - remote_commit=$(git ls-remote https://github.com/github/gh-ost-ci-env.git tar-xz-binaries | cut -f1) + remote_commit=$(git ls-remote https://github.com/github/gh-ost-ci-env.git HEAD | cut -f1) local_commit="unknown" [ -d "gh-ost-ci-env" ] && local_commit=$(cd gh-ost-ci-env && git log --format="%H" -n 1) @@ -17,11 +17,6 @@ fetch_ci_env() { if [ "$remote_commit" != "$local_commit" ] ; then rm -rf ./gh-ost-ci-env git clone https://github.com/github/gh-ost-ci-env.git - ( - cd gh-ost-ci-env - git fetch origin tar-xz-binaries - git checkout tar-xz-binaries - ) fi } @@ -46,12 +41,13 @@ test_mysql_version() { mkdir -p sandboxes rm -rf sandboxes/* - if echo "$mysql_version" | egrep "5[.]5[.]" ; then + local mysql_version_num=${mysql_version#*-} + if echo "$mysql_version_num" | egrep "5[.]5[.]" ; then gtid="" else gtid="--gtid" fi - gh-ost-ci-env/bin/linux/dbdeployer deploy replication "$mysql_version" --nodes 2 --sandbox-binary ${PWD}/sandbox/binary --sandbox-home ${PWD}/sandboxes ${gtid} --my-cnf-options log_slave_updates --my-cnf-options log_bin --my-cnf-options binlog_format=ROW --sandbox-directory rsandbox + gh-ost-ci-env/bin/linux/dbdeployer deploy replication "$mysql_version_num" --nodes 2 --sandbox-binary ${PWD}/sandbox/binary --sandbox-home ${PWD}/sandboxes ${gtid} --my-cnf-options log_slave_updates --my-cnf-options log_bin --my-cnf-options binlog_format=ROW --sandbox-directory rsandbox sed '/sandboxes/d' -i gh-ost-ci-env/bin/gh-ost-test-mysql-master echo 'sandboxes/rsandbox/m "$@"' >> gh-ost-ci-env/bin/gh-ost-test-mysql-master From c71dbf9ef3e8511b1eefb3ec38ef6d3ce768f0e6 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Fri, 14 May 2021 15:32:56 +0200 Subject: [PATCH 19/21] Copy auto increment (#967) * v1.1.0 * WIP: copying AUTO_INCREMENT value to ghost table Initial commit: towards setting up a test suite Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * greping for 'expect_table_structure' content * Adding simple test for 'expect_table_structure' scenario * adding tests for AUTO_INCREMENT value after row deletes. Should initially fail * clear event beforehand * parsing AUTO_INCREMENT from alter query, reading AUTO_INCREMENT from original table, applying AUTO_INCREMENT value onto ghost table if applicable and user has not specified AUTO_INCREMENT in alter statement * support GetUint64 Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * minor update to test Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * adding test for user defined AUTO_INCREMENT statement Co-authored-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/base/context.go | 1 + go/logic/applier.go | 19 +++++++++++++++ go/logic/inspect.go | 22 +++++++++++++++++ go/logic/migrator.go | 8 +++++++ go/sql/parser.go | 19 ++++++++++++--- go/sql/parser_test.go | 24 +++++++++++++++++++ .../create.sql | 17 +++++++++++++ .../expect_table_structure | 1 + .../extra_args | 1 + localtests/autoinc-copy-deletes/create.sql | 17 +++++++++++++ .../expect_table_structure | 1 + localtests/autoinc-copy-simple/create.sql | 13 ++++++++++ .../expect_table_structure | 1 + localtests/test.sh | 13 ++++++++++ .../outbrain/golib/sqlutils/sqlutils.go | 13 ++++++++++ 15 files changed, 167 insertions(+), 3 deletions(-) create mode 100644 localtests/autoinc-copy-deletes-user-defined/create.sql create mode 100644 localtests/autoinc-copy-deletes-user-defined/expect_table_structure create mode 100644 localtests/autoinc-copy-deletes-user-defined/extra_args create mode 100644 localtests/autoinc-copy-deletes/create.sql create mode 100644 localtests/autoinc-copy-deletes/expect_table_structure create mode 100644 localtests/autoinc-copy-simple/create.sql create mode 100644 localtests/autoinc-copy-simple/expect_table_structure diff --git a/go/base/context.go b/go/base/context.go index 3757653..51b43df 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -206,6 +206,7 @@ type MigrationContext struct { OriginalTableColumns *sql.ColumnList OriginalTableVirtualColumns *sql.ColumnList OriginalTableUniqueKeys [](*sql.UniqueKey) + OriginalTableAutoIncrement uint64 GhostTableColumns *sql.ColumnList GhostTableVirtualColumns *sql.ColumnList GhostTableUniqueKeys [](*sql.UniqueKey) diff --git a/go/logic/applier.go b/go/logic/applier.go index 52628ec..67b519e 100644 --- a/go/logic/applier.go +++ b/go/logic/applier.go @@ -207,6 +207,25 @@ func (this *Applier) AlterGhost() error { return nil } +// AlterGhost applies `alter` statement on ghost table +func (this *Applier) AlterGhostAutoIncrement() error { + query := fmt.Sprintf(`alter /* gh-ost */ table %s.%s AUTO_INCREMENT=%d`, + sql.EscapeName(this.migrationContext.DatabaseName), + sql.EscapeName(this.migrationContext.GetGhostTableName()), + this.migrationContext.OriginalTableAutoIncrement, + ) + this.migrationContext.Log.Infof("Altering ghost table AUTO_INCREMENT value %s.%s", + sql.EscapeName(this.migrationContext.DatabaseName), + sql.EscapeName(this.migrationContext.GetGhostTableName()), + ) + this.migrationContext.Log.Debugf("AUTO_INCREMENT ALTER statement: %s", query) + if _, err := sqlutils.ExecNoPrepare(this.db, query); err != nil { + return err + } + this.migrationContext.Log.Infof("Ghost table AUTO_INCREMENT altered") + return nil +} + // CreateChangelogTable creates the changelog table on the applier host func (this *Applier) CreateChangelogTable() error { if err := this.DropChangelogTable(); err != nil { diff --git a/go/logic/inspect.go b/go/logic/inspect.go index 584c56b..3e3e303 100644 --- a/go/logic/inspect.go +++ b/go/logic/inspect.go @@ -111,6 +111,10 @@ func (this *Inspector) InspectOriginalTable() (err error) { if err != nil { return err } + this.migrationContext.OriginalTableAutoIncrement, err = this.getAutoIncrementValue(this.migrationContext.OriginalTableName) + if err != nil { + return err + } return nil } @@ -596,6 +600,24 @@ func (this *Inspector) applyColumnTypes(databaseName, tableName string, columnsL return err } +// getAutoIncrementValue get's the original table's AUTO_INCREMENT value, if exists (0 value if not exists) +func (this *Inspector) getAutoIncrementValue(tableName string) (autoIncrement uint64, err error) { + query := ` + SELECT + AUTO_INCREMENT + FROM INFORMATION_SCHEMA.TABLES + WHERE + TABLES.TABLE_SCHEMA = ? + AND TABLES.TABLE_NAME = ? + AND AUTO_INCREMENT IS NOT NULL + ` + err = sqlutils.QueryRowsMap(this.db, query, func(m sqlutils.RowMap) error { + autoIncrement = m.GetUint64("AUTO_INCREMENT") + return nil + }, this.migrationContext.DatabaseName, tableName) + return autoIncrement, err +} + // getCandidateUniqueKeys investigates a table and returns the list of unique keys // candidate for chunking func (this *Inspector) getCandidateUniqueKeys(tableName string) (uniqueKeys [](*sql.UniqueKey), err error) { diff --git a/go/logic/migrator.go b/go/logic/migrator.go index 2a038a8..dfddccf 100644 --- a/go/logic/migrator.go +++ b/go/logic/migrator.go @@ -1100,6 +1100,14 @@ func (this *Migrator) initiateApplier() error { return err } + if this.migrationContext.OriginalTableAutoIncrement > 0 && !this.parser.IsAutoIncrementDefined() { + // Original table has AUTO_INCREMENT value and the -alter statement does not indicate any override, + // so we should copy AUTO_INCREMENT value onto our ghost table. + if err := this.applier.AlterGhostAutoIncrement(); err != nil { + this.migrationContext.Log.Errorf("Unable to ALTER ghost table AUTO_INCREMENT value, see further error details. Bailing out") + return err + } + } this.applier.WriteChangelogState(string(GhostTableMigrated)) go this.applier.InitiateHeartbeat() return nil diff --git a/go/sql/parser.go b/go/sql/parser.go index ebb8b38..d9c0c3f 100644 --- a/go/sql/parser.go +++ b/go/sql/parser.go @@ -16,6 +16,7 @@ var ( renameColumnRegexp = regexp.MustCompile(`(?i)\bchange\s+(column\s+|)([\S]+)\s+([\S]+)\s+`) dropColumnRegexp = regexp.MustCompile(`(?i)\bdrop\s+(column\s+|)([\S]+)$`) renameTableRegexp = regexp.MustCompile(`(?i)\brename\s+(to|as)\s+`) + autoIncrementRegexp = regexp.MustCompile(`(?i)\bauto_increment[\s]*=[\s]*([0-9]+)`) alterTableExplicitSchemaTableRegexps = []*regexp.Regexp{ // ALTER TABLE `scm`.`tbl` something regexp.MustCompile(`(?i)\balter\s+table\s+` + "`" + `([^` + "`" + `]+)` + "`" + `[.]` + "`" + `([^` + "`" + `]+)` + "`" + `\s+(.*$)`), @@ -35,9 +36,10 @@ var ( ) type AlterTableParser struct { - columnRenameMap map[string]string - droppedColumns map[string]bool - isRenameTable bool + columnRenameMap map[string]string + droppedColumns map[string]bool + isRenameTable bool + isAutoIncrementDefined bool alterStatementOptions string alterTokens []string @@ -122,6 +124,12 @@ func (this *AlterTableParser) parseAlterToken(alterToken string) (err error) { this.isRenameTable = true } } + { + // auto_increment + if autoIncrementRegexp.MatchString(alterToken) { + this.isAutoIncrementDefined = true + } + } return nil } @@ -173,6 +181,11 @@ func (this *AlterTableParser) DroppedColumnsMap() map[string]bool { func (this *AlterTableParser) IsRenameTable() bool { return this.isRenameTable } + +func (this *AlterTableParser) IsAutoIncrementDefined() bool { + return this.isAutoIncrementDefined +} + func (this *AlterTableParser) GetExplicitSchema() string { return this.explicitSchema } diff --git a/go/sql/parser_test.go b/go/sql/parser_test.go index 79faa63..6cdbb39 100644 --- a/go/sql/parser_test.go +++ b/go/sql/parser_test.go @@ -24,6 +24,7 @@ func TestParseAlterStatement(t *testing.T) { test.S(t).ExpectNil(err) test.S(t).ExpectEquals(parser.alterStatementOptions, statement) test.S(t).ExpectFalse(parser.HasNonTrivialRenames()) + test.S(t).ExpectFalse(parser.IsAutoIncrementDefined()) } func TestParseAlterStatementTrivialRename(t *testing.T) { @@ -33,10 +34,31 @@ func TestParseAlterStatementTrivialRename(t *testing.T) { test.S(t).ExpectNil(err) test.S(t).ExpectEquals(parser.alterStatementOptions, statement) test.S(t).ExpectFalse(parser.HasNonTrivialRenames()) + test.S(t).ExpectFalse(parser.IsAutoIncrementDefined()) test.S(t).ExpectEquals(len(parser.columnRenameMap), 1) test.S(t).ExpectEquals(parser.columnRenameMap["ts"], "ts") } +func TestParseAlterStatementWithAutoIncrement(t *testing.T) { + + statements := []string{ + "auto_increment=7", + "auto_increment = 7", + "AUTO_INCREMENT = 71", + "add column t int, change ts ts timestamp, auto_increment=7 engine=innodb", + "add column t int, change ts ts timestamp, auto_increment =7 engine=innodb", + "add column t int, change ts ts timestamp, AUTO_INCREMENT = 7 engine=innodb", + "add column t int, change ts ts timestamp, engine=innodb auto_increment=73425", + } + for _, statement := range statements { + parser := NewAlterTableParser() + err := parser.ParseAlterStatement(statement) + test.S(t).ExpectNil(err) + test.S(t).ExpectEquals(parser.alterStatementOptions, statement) + test.S(t).ExpectTrue(parser.IsAutoIncrementDefined()) + } +} + func TestParseAlterStatementTrivialRenames(t *testing.T) { statement := "add column t int, change ts ts timestamp, CHANGE f `f` float, engine=innodb" parser := NewAlterTableParser() @@ -44,6 +66,7 @@ func TestParseAlterStatementTrivialRenames(t *testing.T) { test.S(t).ExpectNil(err) test.S(t).ExpectEquals(parser.alterStatementOptions, statement) test.S(t).ExpectFalse(parser.HasNonTrivialRenames()) + test.S(t).ExpectFalse(parser.IsAutoIncrementDefined()) test.S(t).ExpectEquals(len(parser.columnRenameMap), 2) test.S(t).ExpectEquals(parser.columnRenameMap["ts"], "ts") test.S(t).ExpectEquals(parser.columnRenameMap["f"], "f") @@ -64,6 +87,7 @@ func TestParseAlterStatementNonTrivial(t *testing.T) { parser := NewAlterTableParser() err := parser.ParseAlterStatement(statement) test.S(t).ExpectNil(err) + test.S(t).ExpectFalse(parser.IsAutoIncrementDefined()) test.S(t).ExpectEquals(parser.alterStatementOptions, statement) renames := parser.GetNonTrivialRenames() test.S(t).ExpectEquals(len(renames), 2) diff --git a/localtests/autoinc-copy-deletes-user-defined/create.sql b/localtests/autoinc-copy-deletes-user-defined/create.sql new file mode 100644 index 0000000..2058b0b --- /dev/null +++ b/localtests/autoinc-copy-deletes-user-defined/create.sql @@ -0,0 +1,17 @@ +drop event if exists gh_ost_test; + +drop table if exists gh_ost_test; +create table gh_ost_test ( + id int auto_increment, + i int not null, + primary key(id) +) auto_increment=1; + +insert into gh_ost_test values (NULL, 11); +insert into gh_ost_test values (NULL, 13); +insert into gh_ost_test values (NULL, 17); +insert into gh_ost_test values (NULL, 23); +insert into gh_ost_test values (NULL, 29); +insert into gh_ost_test values (NULL, 31); +insert into gh_ost_test values (NULL, 37); +delete from gh_ost_test where id>=5; diff --git a/localtests/autoinc-copy-deletes-user-defined/expect_table_structure b/localtests/autoinc-copy-deletes-user-defined/expect_table_structure new file mode 100644 index 0000000..5e180af --- /dev/null +++ b/localtests/autoinc-copy-deletes-user-defined/expect_table_structure @@ -0,0 +1 @@ +AUTO_INCREMENT=7 diff --git a/localtests/autoinc-copy-deletes-user-defined/extra_args b/localtests/autoinc-copy-deletes-user-defined/extra_args new file mode 100644 index 0000000..cce91e1 --- /dev/null +++ b/localtests/autoinc-copy-deletes-user-defined/extra_args @@ -0,0 +1 @@ +--alter='AUTO_INCREMENT=7' diff --git a/localtests/autoinc-copy-deletes/create.sql b/localtests/autoinc-copy-deletes/create.sql new file mode 100644 index 0000000..2058b0b --- /dev/null +++ b/localtests/autoinc-copy-deletes/create.sql @@ -0,0 +1,17 @@ +drop event if exists gh_ost_test; + +drop table if exists gh_ost_test; +create table gh_ost_test ( + id int auto_increment, + i int not null, + primary key(id) +) auto_increment=1; + +insert into gh_ost_test values (NULL, 11); +insert into gh_ost_test values (NULL, 13); +insert into gh_ost_test values (NULL, 17); +insert into gh_ost_test values (NULL, 23); +insert into gh_ost_test values (NULL, 29); +insert into gh_ost_test values (NULL, 31); +insert into gh_ost_test values (NULL, 37); +delete from gh_ost_test where id>=5; diff --git a/localtests/autoinc-copy-deletes/expect_table_structure b/localtests/autoinc-copy-deletes/expect_table_structure new file mode 100644 index 0000000..5a755ff --- /dev/null +++ b/localtests/autoinc-copy-deletes/expect_table_structure @@ -0,0 +1 @@ +AUTO_INCREMENT=8 diff --git a/localtests/autoinc-copy-simple/create.sql b/localtests/autoinc-copy-simple/create.sql new file mode 100644 index 0000000..677f08e --- /dev/null +++ b/localtests/autoinc-copy-simple/create.sql @@ -0,0 +1,13 @@ +drop event if exists gh_ost_test; + +drop table if exists gh_ost_test; +create table gh_ost_test ( + id int auto_increment, + i int not null, + primary key(id) +) auto_increment=1; + +insert into gh_ost_test values (NULL, 11); +insert into gh_ost_test values (NULL, 13); +insert into gh_ost_test values (NULL, 17); +insert into gh_ost_test values (NULL, 23); diff --git a/localtests/autoinc-copy-simple/expect_table_structure b/localtests/autoinc-copy-simple/expect_table_structure new file mode 100644 index 0000000..3ed5902 --- /dev/null +++ b/localtests/autoinc-copy-simple/expect_table_structure @@ -0,0 +1 @@ +AUTO_INCREMENT=5 diff --git a/localtests/test.sh b/localtests/test.sh index d4b3f17..eb3d640 100755 --- a/localtests/test.sh +++ b/localtests/test.sh @@ -12,6 +12,7 @@ test_logfile=/tmp/gh-ost-test.log default_ghost_binary=/tmp/gh-ost-test ghost_binary="" exec_command_file=/tmp/gh-ost-test.bash +ghost_structure_output_file=/tmp/gh-ost-test.ghost.structure.sql orig_content_output_file=/tmp/gh-ost-test.orig.content.csv ghost_content_output_file=/tmp/gh-ost-test.ghost.content.csv throttle_flag_file=/tmp/gh-ost-test.ghost.throttle.flag @@ -204,6 +205,18 @@ test_single() { return 1 fi + gh-ost-test-mysql-replica --default-character-set=utf8mb4 test -e "show create table _gh_ost_test_gho\G" -ss > $ghost_structure_output_file + + if [ -f $tests_path/$test_name/expect_table_structure ] ; then + expected_table_structure="$(cat $tests_path/$test_name/expect_table_structure)" + if ! grep -q "$expected_table_structure" $ghost_structure_output_file ; then + echo + echo "ERROR $test_name: table structure was expected to include ${expected_table_structure} but did not. cat $ghost_structure_output_file:" + cat $ghost_structure_output_file + return 1 + fi + fi + echo_dot gh-ost-test-mysql-replica --default-character-set=utf8mb4 test -e "select ${orig_columns} from gh_ost_test ${order_by}" -ss > $orig_content_output_file gh-ost-test-mysql-replica --default-character-set=utf8mb4 test -e "select ${ghost_columns} from _gh_ost_test_gho ${order_by}" -ss > $ghost_content_output_file diff --git a/vendor/github.com/outbrain/golib/sqlutils/sqlutils.go b/vendor/github.com/outbrain/golib/sqlutils/sqlutils.go index 8d98690..56761c6 100644 --- a/vendor/github.com/outbrain/golib/sqlutils/sqlutils.go +++ b/vendor/github.com/outbrain/golib/sqlutils/sqlutils.go @@ -117,6 +117,19 @@ func (this *RowMap) GetUintD(key string, def uint) uint { return uint(res) } +func (this *RowMap) GetUint64(key string) uint64 { + res, _ := strconv.ParseUint(this.GetString(key), 10, 0) + return res +} + +func (this *RowMap) GetUint64D(key string, def uint64) uint64 { + res, err := strconv.ParseUint(this.GetString(key), 10, 0) + if err != nil { + return def + } + return uint64(res) +} + func (this *RowMap) GetBool(key string) bool { return this.GetInt(key) != 0 } From 57a81ceff58fe253f29dea4766430a6e836214b2 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Wed, 19 May 2021 14:08:32 +0200 Subject: [PATCH 20/21] Use GitHub Actions CI badges (#970) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d496e08..8842d5b 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # gh-ost -[![build status](https://travis-ci.org/github/gh-ost.svg)](https://travis-ci.org/github/gh-ost) [![downloads](https://img.shields.io/github/downloads/github/gh-ost/total.svg)](https://github.com/github/gh-ost/releases) [![release](https://img.shields.io/github/release/github/gh-ost.svg)](https://github.com/github/gh-ost/releases) +[![ci](https://github.com/github/gh-ost/actions/workflows/ci.yml/badge.svg)](https://github.com/github/gh-ost/actions/workflows/ci.yml) [![replica-tests](https://github.com/github/gh-ost/actions/workflows/replica-tests.yml/badge.svg)](https://github.com/github/gh-ost/actions/workflows/replica-tests.yml) [![downloads](https://img.shields.io/github/downloads/github/gh-ost/total.svg)](https://github.com/github/gh-ost/releases) [![release](https://img.shields.io/github/release/github/gh-ost.svg)](https://github.com/github/gh-ost/releases) #### GitHub's online schema migration for MySQL From 36c669dd759d4cefedbebf8461f68527b9ea22ce Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Mon, 24 May 2021 21:16:49 +0300 Subject: [PATCH 21/21] Generated column as part of UNIQUE (or PRIMARY) KEY (#919) * v1.1.0 * WIP: copying AUTO_INCREMENT value to ghost table Initial commit: towards setting up a test suite Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * greping for 'expect_table_structure' content * Adding simple test for 'expect_table_structure' scenario * adding tests for AUTO_INCREMENT value after row deletes. Should initially fail * clear event beforehand * parsing AUTO_INCREMENT from alter query, reading AUTO_INCREMENT from original table, applying AUTO_INCREMENT value onto ghost table if applicable and user has not specified AUTO_INCREMENT in alter statement * support GetUint64 Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * minor update to test Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * adding test for user defined AUTO_INCREMENT statement * Generated column as part of UNIQUE (or PRIMARY) KEY Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * skip analysis of generated column data type in unique key Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/logic/inspect.go | 4 +++ .../generated-columns57-unique/create.sql | 30 +++++++++++++++++++ .../ignore_versions | 1 + 3 files changed, 35 insertions(+) create mode 100644 localtests/generated-columns57-unique/create.sql create mode 100644 localtests/generated-columns57-unique/ignore_versions diff --git a/go/logic/inspect.go b/go/logic/inspect.go index 3e3e303..097b58f 100644 --- a/go/logic/inspect.go +++ b/go/logic/inspect.go @@ -190,6 +190,10 @@ func (this *Inspector) inspectOriginalAndGhostTables() (err error) { } for _, column := range this.migrationContext.UniqueKey.Columns.Columns() { + if this.migrationContext.GhostTableVirtualColumns.GetColumn(column.Name) != nil { + // this is a virtual column + continue + } if this.migrationContext.MappedSharedColumns.HasTimezoneConversion(column.Name) { return fmt.Errorf("No support at this time for converting a column from DATETIME to TIMESTAMP that is also part of the chosen unique key. Column: %s, key: %s", column.Name, this.migrationContext.UniqueKey.Name) } diff --git a/localtests/generated-columns57-unique/create.sql b/localtests/generated-columns57-unique/create.sql new file mode 100644 index 0000000..7a63dd9 --- /dev/null +++ b/localtests/generated-columns57-unique/create.sql @@ -0,0 +1,30 @@ +drop table if exists gh_ost_test; +create table gh_ost_test ( + id int auto_increment, + `idb` varchar(36) CHARACTER SET utf8mb4 GENERATED ALWAYS AS (json_unquote(json_extract(`jsonobj`,_utf8mb4'$._id'))) STORED NOT NULL, + `jsonobj` json NOT NULL, + PRIMARY KEY (`id`,`idb`) +) auto_increment=1; + +insert into gh_ost_test (id, jsonobj) values (null, '{"_id":2}'); +insert into gh_ost_test (id, jsonobj) values (null, '{"_id":3}'); + +drop event if exists gh_ost_test; +delimiter ;; +create event gh_ost_test + on schedule every 1 second + starts current_timestamp + ends current_timestamp + interval 60 second + on completion not preserve + enable + do +begin + insert into gh_ost_test (id, jsonobj) values (null, '{"_id":5}'); + insert into gh_ost_test (id, jsonobj) values (null, '{"_id":7}'); + insert into gh_ost_test (id, jsonobj) values (null, '{"_id":11}'); + insert into gh_ost_test (id, jsonobj) values (null, '{"_id":13}'); + insert into gh_ost_test (id, jsonobj) values (null, '{"_id":17}'); + insert into gh_ost_test (id, jsonobj) values (null, '{"_id":19}'); + insert into gh_ost_test (id, jsonobj) values (null, '{"_id":23}'); + insert into gh_ost_test (id, jsonobj) values (null, '{"_id":27}'); +end ;; diff --git a/localtests/generated-columns57-unique/ignore_versions b/localtests/generated-columns57-unique/ignore_versions new file mode 100644 index 0000000..b6de5f8 --- /dev/null +++ b/localtests/generated-columns57-unique/ignore_versions @@ -0,0 +1 @@ +(5.5|5.6)