diff --git a/.gitignore b/.gitignore index 63f0df9..605546d 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ /bin/ /libexec/ /.vendor/ +.idea/ diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index c1c1891..0000000 --- a/.travis.yml +++ /dev/null @@ -1,33 +0,0 @@ -# http://docs.travis-ci.com/user/languages/go/ -language: go - -go: - - "1.12.x" - -os: - - linux - -services: - - mysql - -env: -- MYSQL_USER=root -- CURRENT_CI_ENV=travis - -addons: - apt: - packages: - - git - - numactl - - libaio1 - -before_install: - - mysql -e 'CREATE DATABASE IF NOT EXISTS test;' - -install: true - -script: - - script/cibuild - -notifications: - email: false diff --git a/Dockerfile.packaging b/Dockerfile.packaging new file mode 100644 index 0000000..214c70c --- /dev/null +++ b/Dockerfile.packaging @@ -0,0 +1,22 @@ +# + +FROM golang:1.12.6 + +RUN apt-get update +RUN apt-get install -y ruby ruby-dev rubygems build-essential +RUN gem install --no-ri --no-rdoc fpm +ENV GOPATH=/tmp/go + +RUN apt-get install -y curl +RUN apt-get install -y rsync +RUN apt-get install -y gcc +RUN apt-get install -y g++ +RUN apt-get install -y bash +RUN apt-get install -y git +RUN apt-get install -y tar +RUN apt-get install -y rpm + +RUN mkdir -p $GOPATH/src/github.com/github/gh-ost +WORKDIR $GOPATH/src/github.com/github/gh-ost +COPY . . +RUN bash build.sh diff --git a/RELEASE_VERSION b/RELEASE_VERSION index 56d0dad..f133985 100644 --- a/RELEASE_VERSION +++ b/RELEASE_VERSION @@ -1 +1 @@ -1.0.48 +1.0.49 diff --git a/build.sh b/build.sh index b7f3a54..46db9c2 100755 --- a/build.sh +++ b/build.sh @@ -61,11 +61,11 @@ main() { mkdir -p ${buildpath} rm -rf ${buildpath:?}/* - build macOS osx darwin amd64 build GNU/Linux linux linux amd64 + # build macOS osx darwin amd64 echo "Binaries found in:" - ls -1 $buildpath/gh-ost-binary*${timestamp}.tar.gz + find $buildpath/gh-ost* -type f -maxdepth 1 } main "$@" diff --git a/doc/rds.md b/doc/rds.md index 37d1d80..da59abb 100644 --- a/doc/rds.md +++ b/doc/rds.md @@ -26,6 +26,14 @@ If you use `pt-table-checksum` as a part of your data integrity checks, you migh This tool requires binlog_format=STATEMENT, but the current binlog_format is set to ROW and an error occurred while attempting to change it. If running MySQL 5.1.29 or newer, setting binlog_format requires the SUPER privilege. You will need to manually set binlog_format to 'STATEMENT' before running this tool. ``` +#### Binlog filtering + +In Aurora, the [binlog filtering feature][aws_replication_docs_bin_log_filtering] is enabled by default. This becomes an issue when gh-ost tries to do the cut-over, because gh-ost waits for an entry in the binlog to proceed but this entry will never end up in the binlog because it gets filtered out by the binlog filtering feature. +You need to turn this feature off during the migration process. +Set the `aurora_enable_repl_bin_log_filtering` parameter to 0 in the Parameter Group for your cluster. +When the migration is done, set it back to 1 (default). + + #### Preflight checklist Before trying to run any `gh-ost` migrations you will want to confirm the following: @@ -35,6 +43,7 @@ Before trying to run any `gh-ost` migrations you will want to confirm the follow - [ ] Executing `SHOW SLAVE STATUS\G` on your replica cluster displays the correct master host, binlog position, etc. - [ ] Database backup retention is greater than 1 day to enable binlogs - [ ] You have setup [`hooks`][ghost_hooks] to issue RDS procedures for stopping and starting replication. (see [github/gh-ost#163][ghost_rds_issue_tracking] for examples) +- [ ] The parameter `aurora_enable_repl_bin_log_filtering` is set to 0 [new_issue]: https://github.com/github/gh-ost/issues/new [assume_rbr_docs]: https://github.com/github/gh-ost/blob/master/doc/command-line-flags.md#assume-rbr @@ -43,3 +52,4 @@ Before trying to run any `gh-ost` migrations you will want to confirm the follow [percona_toolkit_patch]: https://github.com/jacobbednarz/percona-toolkit/commit/0271ba6a094da446a5e5bb8d99b5c26f1777f2b9 [ghost_hooks]: https://github.com/github/gh-ost/blob/master/doc/hooks.md [ghost_rds_issue_tracking]: https://github.com/github/gh-ost/issues/163 +[aws_replication_docs_bin_log_filtering]: https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/AuroraMySQL.Replication.html#AuroraMySQL.Replication.Performance \ No newline at end of file diff --git a/go/base/context.go b/go/base/context.go index 35b9d99..1030463 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -120,6 +120,7 @@ type MigrationContext struct { ThrottleAdditionalFlagFile string throttleQuery string throttleHTTP string + IgnoreHTTPErrors bool ThrottleCommandedByUser int64 HibernateUntil int64 maxLoad LoadMap @@ -595,6 +596,13 @@ func (this *MigrationContext) SetThrottleHTTP(throttleHTTP string) { this.throttleHTTP = throttleHTTP } +func (this *MigrationContext) SetIgnoreHTTPErrors(ignoreHTTPErrors bool) { + this.throttleHTTPMutex.Lock() + defer this.throttleHTTPMutex.Unlock() + + this.IgnoreHTTPErrors = ignoreHTTPErrors +} + func (this *MigrationContext) GetMaxLoad() LoadMap { this.throttleMutex.Lock() defer this.throttleMutex.Unlock() diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index 6773c08..ee0c986 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -106,6 +106,7 @@ func main() { throttleControlReplicas := flag.String("throttle-control-replicas", "", "List of replicas on which to check for lag; comma delimited. Example: myhost1.com:3306,myhost2.com,myhost3.com:3307") throttleQuery := flag.String("throttle-query", "", "when given, issued (every second) to check if operation should throttle. Expecting to return zero for no-throttle, >0 for throttle. Query is issued on the migrated server. Make sure this query is lightweight") throttleHTTP := flag.String("throttle-http", "", "when given, gh-ost checks given URL via HEAD request; any response code other than 200 (OK) causes throttling; make sure it has low latency response") + ignoreHTTPErrors := flag.Bool("ignore-http-errors", false, "ignore HTTP connection errors during throttle check") heartbeatIntervalMillis := flag.Int64("heartbeat-interval-millis", 100, "how frequently would gh-ost inject a heartbeat value") flag.StringVar(&migrationContext.ThrottleFlagFile, "throttle-flag-file", "", "operation pauses when this file exists; hint: use a file that is specific to the table being altered") flag.StringVar(&migrationContext.ThrottleAdditionalFlagFile, "throttle-additional-flag-file", "/tmp/gh-ost.throttle", "operation pauses when this file exists; hint: keep default, use for throttling multiple gh-ost operations") @@ -259,6 +260,7 @@ func main() { migrationContext.SetMaxLagMillisecondsThrottleThreshold(*maxLagMillis) migrationContext.SetThrottleQuery(*throttleQuery) migrationContext.SetThrottleHTTP(*throttleHTTP) + migrationContext.SetIgnoreHTTPErrors(*ignoreHTTPErrors) migrationContext.SetDefaultNumRetries(*defaultRetries) migrationContext.ApplyCredentials() if err := migrationContext.SetupTLS(); err != nil { diff --git a/go/logic/throttler.go b/go/logic/throttler.go index 1fe413c..d234ea6 100644 --- a/go/logic/throttler.go +++ b/go/logic/throttler.go @@ -18,20 +18,22 @@ import ( ) var ( - httpStatusMessages map[int]string = map[int]string{ + httpStatusMessages = map[int]string{ 200: "OK", 404: "Not found", 417: "Expectation failed", 429: "Too many requests", 500: "Internal server error", + -1: "Connection error", } // See https://github.com/github/freno/blob/master/doc/http.md - httpStatusFrenoMessages map[int]string = map[int]string{ + httpStatusFrenoMessages = map[int]string{ 200: "OK", 404: "freno: unknown metric", 417: "freno: access forbidden", 429: "freno: threshold exceeded", 500: "freno: internal error", + -1: "freno: connection error", } ) @@ -83,6 +85,7 @@ func (this *Throttler) shouldThrottle() (result bool, reason string, reasonHint if statusCode != 0 && statusCode != http.StatusOK { return true, this.throttleHttpMessage(int(statusCode)), base.NoThrottleReasonHint } + // Replication lag throttle maxLagMillisecondsThrottleThreshold := atomic.LoadInt64(&this.migrationContext.MaxLagMillisecondsThrottleThreshold) lag := atomic.LoadInt64(&this.migrationContext.CurrentLag) @@ -287,7 +290,14 @@ func (this *Throttler) collectThrottleHTTPStatus(firstThrottlingCollected chan<- return false, nil } - collectFunc() + _, err := collectFunc() + if err != nil { + // If not told to ignore errors, we'll throttle on HTTP connection issues + if !this.migrationContext.IgnoreHTTPErrors { + atomic.StoreInt64(&this.migrationContext.ThrottleHTTPStatusCode, int64(-1)) + } + } + firstThrottlingCollected <- true ticker := time.Tick(100 * time.Millisecond) @@ -296,7 +306,15 @@ func (this *Throttler) collectThrottleHTTPStatus(firstThrottlingCollected chan<- return } - if sleep, _ := collectFunc(); sleep { + sleep, err := collectFunc() + if err != nil { + // If not told to ignore errors, we'll throttle on HTTP connection issues + if !this.migrationContext.IgnoreHTTPErrors { + atomic.StoreInt64(&this.migrationContext.ThrottleHTTPStatusCode, int64(-1)) + } + } + + if sleep { time.Sleep(1 * time.Second) } } diff --git a/localtests/bigint-change-nullable/create.sql b/localtests/bigint-change-nullable/create.sql new file mode 100644 index 0000000..f4f0548 --- /dev/null +++ b/localtests/bigint-change-nullable/create.sql @@ -0,0 +1,21 @@ +drop table if exists gh_ost_test; +create table gh_ost_test ( + id bigint auto_increment, + val bigint not null, + primary key(id) +) auto_increment=1; + +drop event if exists gh_ost_test; +delimiter ;; +create event gh_ost_test + on schedule every 1 second + starts current_timestamp + ends current_timestamp + interval 60 second + on completion not preserve + enable + do +begin + insert into gh_ost_test values (null, 18446744073709551615); + insert into gh_ost_test values (null, 18446744073709551614); + insert into gh_ost_test values (null, 18446744073709551613); +end ;; diff --git a/localtests/bigint-change-nullable/extra_args b/localtests/bigint-change-nullable/extra_args new file mode 100644 index 0000000..784d522 --- /dev/null +++ b/localtests/bigint-change-nullable/extra_args @@ -0,0 +1 @@ +--alter="change val val bigint" diff --git a/localtests/latin1text/create.sql b/localtests/latin1text/create.sql new file mode 100644 index 0000000..58837cb --- /dev/null +++ b/localtests/latin1text/create.sql @@ -0,0 +1,25 @@ +drop table if exists gh_ost_test; +create table gh_ost_test ( + id int auto_increment, + t text charset latin1 collate latin1_swedish_ci, + primary key(id) +) auto_increment=1 charset latin1 collate latin1_swedish_ci; + +drop event if exists gh_ost_test; +delimiter ;; +create event gh_ost_test + on schedule every 1 second + starts current_timestamp + ends current_timestamp + interval 60 second + on completion not preserve + enable + do +begin + insert into gh_ost_test values (null, md5(rand())); + insert into gh_ost_test values (null, 'átesting'); + insert into gh_ost_test values (null, 'ádelete'); + insert into gh_ost_test values (null, 'testátest'); + update gh_ost_test set t='áupdated' order by id desc limit 1; + update gh_ost_test set t='áupdated1' where t='áupdated' order by id desc limit 1; + delete from gh_ost_test where t='ádelete'; +end ;; diff --git a/localtests/swap-uk-uk/create.sql b/localtests/swap-uk-uk/create.sql index 5fcbf32..30c1542 100644 --- a/localtests/swap-uk-uk/create.sql +++ b/localtests/swap-uk-uk/create.sql @@ -1,8 +1,8 @@ drop table if exists gh_ost_test; create table gh_ost_test ( - id bigint, + id bigint not null, i int not null, - ts timestamp(6), + ts timestamp(6) not null, unique key id_uidx(id), unique key its_uidx(i, ts) ) ; diff --git a/localtests/varbinary/create.sql b/localtests/varbinary/create.sql new file mode 100644 index 0000000..9d85200 --- /dev/null +++ b/localtests/varbinary/create.sql @@ -0,0 +1,40 @@ +drop table if exists gh_ost_test; +create table gh_ost_test ( + id binary(16) NOT NULL, + info varchar(255) COLLATE utf8_unicode_ci NOT NULL, + data binary(8) NOT NULL, + primary key (id), + unique key info_uidx (info) +) auto_increment=1; + +drop event if exists gh_ost_test; +delimiter ;; +create event gh_ost_test + on schedule every 1 second + starts current_timestamp + ends current_timestamp + interval 60 second + on completion not preserve + enable + do +begin + replace into gh_ost_test (id, info, data) values (X'12ffffffffffffffffffffffffffff00', 'item 1a', X'12ffffffffffffff'); + replace into gh_ost_test (id, info, data) values (X'34ffffffffffffffffffffffffffffff', 'item 3a', X'34ffffffffffffff'); + replace into gh_ost_test (id, info, data) values (X'90ffffffffffffffffffffffffffffff', 'item 9a', X'90ffffffffffff00'); + + DELETE FROM gh_ost_test WHERE id = X'11ffffffffffffffffffffffffffff00'; + UPDATE gh_ost_test SET info = 'item 2++' WHERE id = X'22ffffffffffffffffffffffffffff00'; + UPDATE gh_ost_test SET info = 'item 3++', data = X'33ffffffffffff00' WHERE id = X'33ffffffffffffffffffffffffffffff'; + DELETE FROM gh_ost_test WHERE id = X'44ffffffffffffffffffffffffffffff'; + UPDATE gh_ost_test SET info = 'item 5++', data = X'55ffffffffffffee' WHERE id = X'55ffffffffffffffffffffffffffffff'; + INSERT INTO gh_ost_test (id, info, data) VALUES (X'66ffffffffffffffffffffffffffff00', 'item 6', X'66ffffffffffffff'); + INSERT INTO gh_ost_test (id, info, data) VALUES (X'77ffffffffffffffffffffffffffffff', 'item 7', X'77ffffffffffff00'); + INSERT INTO gh_ost_test (id, info, data) VALUES (X'88ffffffffffffffffffffffffffffff', 'item 8', X'88ffffffffffffff'); +end ;; + +INSERT INTO gh_ost_test (id, info, data) VALUES + (X'11ffffffffffffffffffffffffffff00', 'item 1', X'11ffffffffffffff'), -- id ends in 00 + (X'22ffffffffffffffffffffffffffff00', 'item 2', X'22ffffffffffffff'), -- id ends in 00 + (X'33ffffffffffffffffffffffffffffff', 'item 3', X'33ffffffffffffff'), + (X'44ffffffffffffffffffffffffffffff', 'item 4', X'44ffffffffffffff'), + (X'55ffffffffffffffffffffffffffffff', 'item 5', X'55ffffffffffffff'), + (X'99ffffffffffffffffffffffffffffff', 'item 9', X'99ffffffffffff00'); -- data ends in 00 diff --git a/script/cibuild-gh-ost-replica-tests b/script/cibuild-gh-ost-replica-tests index ab8c689..3de9e05 100755 --- a/script/cibuild-gh-ost-replica-tests +++ b/script/cibuild-gh-ost-replica-tests @@ -50,7 +50,8 @@ test_mysql_version() { export PATH="${PWD}/gh-ost-ci-env/bin/:${PATH}" - gh-ost-test-mysql-master -uroot -e "grant all on *.* to 'gh-ost'@'%' identified by 'gh-ost'" + gh-ost-test-mysql-master -uroot -e "create user 'gh-ost'@'%' identified by 'gh-ost'" + gh-ost-test-mysql-master -uroot -e "grant all on *.* to 'gh-ost'@'%'" echo "### Running gh-ost tests for $mysql_version" ./localtests/test.sh -b bin/gh-ost @@ -61,6 +62,9 @@ test_mysql_version() { echo "Building..." . script/build # Test all versions: +find gh-ost-ci-env/mysql-tarballs/ -name "*.tar.gz" | while read f ; do basename $f ".tar.gz" ; done | sort -r | while read mysql_version ; do + echo "found MySQL version: $mysql_version" +done find gh-ost-ci-env/mysql-tarballs/ -name "*.tar.gz" | while read f ; do basename $f ".tar.gz" ; done | sort -r | while read mysql_version ; do test_mysql_version "$mysql_version" done diff --git a/script/dock b/script/dock new file mode 100755 index 0000000..486061d --- /dev/null +++ b/script/dock @@ -0,0 +1,25 @@ +#!/bin/bash + +# Usage: +# dock [arg] +# dock test: build gh-ost & run unit and integration tests +# docker pkg [target-path]: build gh-ost release packages and copy to target path (default path: /tmp/gh-ost-release) + +command="$1" + +case "$command" in + "test") + docker_target="gh-ost-test" + docker build . -f Dockerfile.test -t "${docker_target}" && docker run --rm -it "${docker_target}:latest" + ;; + "pkg") + packages_path="${2:-/tmp/gh-ost-release}" + docker_target="gh-ost-packaging" + docker build . -f Dockerfile.packaging -t "${docker_target}" && docker run --rm -it -v "${packages_path}:/tmp/pkg" "${docker_target}:latest" bash -c 'find /tmp/gh-ost-release/ -maxdepth 1 -type f | xargs cp -t /tmp/pkg' + echo "packages generated on ${packages_path}:" + ls -l "${packages_path}" + ;; + *) + >&2 echo "Usage: dock dock [arg]" + exit 1 +esac