Compare commits
5 Commits
master
...
release-1-
Author | SHA1 | Date | |
---|---|---|---|
|
7c9c1f04ef | ||
|
62ac8974f5 | ||
|
dd919de4e3 | ||
|
0a87c7af58 | ||
|
d8ccc1152a |
6
.github/workflows/ci.yml
vendored
6
.github/workflows/ci.yml
vendored
@ -5,15 +5,15 @@ on: [pull_request]
|
||||
jobs:
|
||||
build:
|
||||
|
||||
runs-on: ubuntu-20.04
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
- name: Set up Go
|
||||
- name: Set up Go 1.16
|
||||
uses: actions/setup-go@v1
|
||||
with:
|
||||
go-version: 1.17
|
||||
go-version: 1.16
|
||||
|
||||
- name: Build
|
||||
run: script/cibuild
|
||||
|
25
.github/workflows/codeql.yml
vendored
25
.github/workflows/codeql.yml
vendored
@ -1,25 +0,0 @@
|
||||
name: "CodeQL analysis"
|
||||
|
||||
on:
|
||||
push:
|
||||
pull_request:
|
||||
schedule:
|
||||
- cron: '0 0 * * 0'
|
||||
|
||||
jobs:
|
||||
codeql:
|
||||
|
||||
strategy:
|
||||
fail-fast: false
|
||||
|
||||
runs-on: ubuntu-latest # windows-latest and ubuntu-latest are supported. macos-latest is not supported at this time.
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v2
|
||||
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@v1
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@v1
|
23
.github/workflows/golangci-lint.yml
vendored
23
.github/workflows/golangci-lint.yml
vendored
@ -1,23 +0,0 @@
|
||||
name: golangci-lint
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
pull_request:
|
||||
permissions:
|
||||
contents: read
|
||||
# Optional: allow read access to pull request. Use with `only-new-issues` option.
|
||||
# pull-requests: read
|
||||
jobs:
|
||||
golangci:
|
||||
name: lint
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/setup-go@v3
|
||||
with:
|
||||
go-version: 1.17
|
||||
- uses: actions/checkout@v3
|
||||
- name: golangci-lint
|
||||
uses: golangci/golangci-lint-action@v3
|
||||
with:
|
||||
version: v1.46.2
|
8
.github/workflows/replica-tests.yml
vendored
8
.github/workflows/replica-tests.yml
vendored
@ -5,18 +5,18 @@ on: [pull_request]
|
||||
jobs:
|
||||
build:
|
||||
|
||||
runs-on: ubuntu-20.04
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
version: [mysql-5.7.25,mysql-8.0.16,PerconaServer-8.0.21]
|
||||
version: [mysql-5.5.62,mysql-5.6.43,mysql-5.7.25,mysql-8.0.16]
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
- name: Set up Go
|
||||
- name: Set up Go 1.16
|
||||
uses: actions/setup-go@v1
|
||||
with:
|
||||
go-version: 1.17
|
||||
go-version: 1.16
|
||||
|
||||
- name: migration tests
|
||||
env:
|
||||
|
@ -1,30 +0,0 @@
|
||||
run:
|
||||
timeout: 5m
|
||||
linters:
|
||||
disable:
|
||||
- errcheck
|
||||
enable:
|
||||
- bodyclose
|
||||
- containedctx
|
||||
- contextcheck
|
||||
- dogsled
|
||||
- durationcheck
|
||||
- errname
|
||||
- errorlint
|
||||
- execinquery
|
||||
- gofmt
|
||||
- ifshort
|
||||
- misspell
|
||||
- nilerr
|
||||
- nilnil
|
||||
- noctx
|
||||
- nolintlint
|
||||
- nosprintfhostport
|
||||
- prealloc
|
||||
- rowserrcheck
|
||||
- sqlclosecheck
|
||||
- unconvert
|
||||
- unparam
|
||||
- unused
|
||||
- wastedassign
|
||||
- whitespace
|
@ -1,4 +1,4 @@
|
||||
FROM golang:1.17
|
||||
FROM golang:1.16.4
|
||||
|
||||
RUN apt-get update
|
||||
RUN apt-get install -y ruby ruby-dev rubygems build-essential
|
||||
|
@ -1,4 +1,4 @@
|
||||
FROM golang:1.17
|
||||
FROM golang:1.16.4
|
||||
LABEL maintainer="github@github.com"
|
||||
|
||||
RUN apt-get update
|
||||
|
@ -1 +1 @@
|
||||
1.1.2
|
||||
1.1.4
|
||||
|
19
build.sh
19
build.sh
@ -23,25 +23,25 @@ function build {
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Building ${osname}-${GOARCH} binary"
|
||||
# TODO: remove GO111MODULE once gh-ost uses Go modules
|
||||
echo "Building ${osname} binary"
|
||||
export GOOS
|
||||
export GOARCH
|
||||
go build -ldflags "$ldflags" -o $buildpath/$target go/cmd/gh-ost/main.go
|
||||
GO111MODULE=off go build -ldflags "$ldflags" -o $buildpath/$target go/cmd/gh-ost/main.go
|
||||
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Build failed for ${osname} ${GOARCH}."
|
||||
echo "Build failed for ${osname}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
(cd $buildpath && tar cfz ./gh-ost-binary-${osshort}-${GOARCH}-${timestamp}.tar.gz $target)
|
||||
(cd $buildpath && tar cfz ./gh-ost-binary-${osshort}-${timestamp}.tar.gz $target)
|
||||
|
||||
# build RPM and deb for Linux, x86-64 only
|
||||
if [ "$GOOS" == "linux" ] && [ "$GOARCH" == "amd64" ] ; then
|
||||
if [ "$GOOS" == "linux" ] ; then
|
||||
echo "Creating Distro full packages"
|
||||
builddir=$(setuptree)
|
||||
cp $buildpath/$target $builddir/gh-ost/usr/bin
|
||||
cd $buildpath
|
||||
fpm -v "${RELEASE_VERSION}" --epoch 1 -f -s dir -n gh-ost -m 'GitHub' --description "GitHub's Online Schema Migrations for MySQL " --url "https://github.com/github/gh-ost" --vendor "GitHub" --license "Apache 2.0" -C $builddir/gh-ost --prefix=/ -t rpm --rpm-rpmbuild-define "_build_id_links none" --rpm-os linux .
|
||||
fpm -v "${RELEASE_VERSION}" --epoch 1 -f -s dir -n gh-ost -m 'GitHub' --description "GitHub's Online Schema Migrations for MySQL " --url "https://github.com/github/gh-ost" --vendor "GitHub" --license "Apache 2.0" -C $builddir/gh-ost --prefix=/ -t rpm --rpm-rpmbuild-define "_build_id_links none" .
|
||||
fpm -v "${RELEASE_VERSION}" --epoch 1 -f -s dir -n gh-ost -m 'GitHub' --description "GitHub's Online Schema Migrations for MySQL " --url "https://github.com/github/gh-ost" --vendor "GitHub" --license "Apache 2.0" -C $builddir/gh-ost --prefix=/ -t deb --deb-no-default-config-files .
|
||||
cd -
|
||||
fi
|
||||
@ -64,9 +64,7 @@ main() {
|
||||
mkdir -p ${buildpath}
|
||||
rm -rf ${buildpath:?}/*
|
||||
build GNU/Linux linux linux amd64
|
||||
build GNU/Linux linux linux arm64
|
||||
build macOS osx darwin amd64
|
||||
build macOS osx darwin arm64
|
||||
|
||||
echo "Binaries found in:"
|
||||
find $buildpath/gh-ost* -type f -maxdepth 1
|
||||
@ -75,4 +73,7 @@ main() {
|
||||
(cd $buildpath && shasum -a256 gh-ost* 2>/dev/null)
|
||||
}
|
||||
|
||||
. script/bootstrap
|
||||
cd .gopath/src/github.com/github/gh-ost
|
||||
|
||||
main "$@"
|
||||
|
@ -5,7 +5,7 @@
|
||||
Getting started with gh-ost development is simple!
|
||||
|
||||
- First obtain the repository with `git clone` or `go get`.
|
||||
- From inside of the repository run `script/cibuild`.
|
||||
- From inside of the repository run `script/cibuild`
|
||||
- This will bootstrap the environment if needed, format the code, build the code, and then run the unit test.
|
||||
|
||||
## CI build workflow
|
||||
@ -14,12 +14,6 @@ Getting started with gh-ost development is simple!
|
||||
|
||||
If additional steps are needed, please add them into this workflow so that the workflow remains simple.
|
||||
|
||||
## `golang-ci` linter
|
||||
|
||||
To enfore best-practices, Pull Requests are automatically linted by [`golang-ci`](https://golangci-lint.run/). The linter config is located at [`.golangci.yml`](https://github.com/github/gh-ost/blob/master/.golangci.yml) and the `golangci-lint` GitHub Action is located at [`.github/workflows/golangci-lint.yml`](https://github.com/github/gh-ost/blob/master/.github/workflows/golangci-lint.yml).
|
||||
|
||||
To run the `golang-ci` linters locally _(recommended before push)_, use `script/lint`.
|
||||
|
||||
## Notes:
|
||||
|
||||
Currently, `script/ensure-go-installed` will install `go` for Mac OS X and Linux. We welcome PR's to add other platforms.
|
||||
|
@ -6,10 +6,6 @@ A more in-depth discussion of various `gh-ost` command line flags: implementatio
|
||||
|
||||
Add this flag when executing on Aliyun RDS.
|
||||
|
||||
### allow-zero-in-date
|
||||
|
||||
Allows the user to make schema changes that include a zero date or zero in date (e.g. adding a `datetime default '0000-00-00 00:00:00'` column), even if global `sql_mode` on MySQL has `NO_ZERO_IN_DATE,NO_ZERO_DATE`.
|
||||
|
||||
### azure
|
||||
|
||||
Add this flag when executing on Azure Database for MySQL.
|
||||
@ -45,25 +41,6 @@ If you happen to _know_ your servers use RBR (Row Based Replication, i.e. `binlo
|
||||
Skipping this step means `gh-ost` would not need the `SUPER` privilege in order to operate.
|
||||
You may want to use this on Amazon RDS.
|
||||
|
||||
### attempt-instant-ddl
|
||||
|
||||
MySQL 8.0 supports "instant DDL" for some operations. If an alter statement can be completed with instant DDL, only a metadata change is required internally. Instant operations include:
|
||||
|
||||
- Adding a column
|
||||
- Dropping a column
|
||||
- Dropping an index
|
||||
- Extending a varchar column
|
||||
- Adding a virtual generated column
|
||||
|
||||
It is not reliable to parse the `ALTER` statement to determine if it is instant or not. This is because the table might be in an older row format, or have some other incompatibility that is difficult to identify.
|
||||
|
||||
`--attempt-instant-ddl` is disabled by default, but the risks of enabling it are relatively minor: `gh-ost` may need to acquire a metadata lock at the start of the operation. This is not a problem for most scenarios, but it could be a problem for users that start the DDL during a period with long running transactions.
|
||||
|
||||
`gh-ost` will automatically fallback to the normal DDL process if the attempt to use instant DDL is unsuccessful.
|
||||
|
||||
### binlogsyncer-max-reconnect-attempts
|
||||
`--binlogsyncer-max-reconnect-attempts=0`, the maximum number of attempts to re-establish a broken inspector connection for sync binlog. `0` or `negative number` means infinite retry, default `0`
|
||||
|
||||
### conf
|
||||
|
||||
`--conf=/path/to/my.cnf`: file where credentials are specified. Should be in (or contain) the following format:
|
||||
@ -249,18 +226,6 @@ Allows `gh-ost` to connect to the MySQL servers using encrypted connections, but
|
||||
|
||||
`--ssl-key=/path/to/ssl-key.key`: SSL private key file (in PEM format).
|
||||
|
||||
### storage-engine
|
||||
Default is `innodb`, and `rocksdb` support is currently experimental. InnoDB and RocksDB are both transactional engines, supporting both shared and exclusive row locks.
|
||||
|
||||
But RocksDB currently lacks a few features support compared to InnoDB:
|
||||
- Gap Locks
|
||||
- Foreign Key
|
||||
- Generated Columns
|
||||
- Spatial
|
||||
- Geometry
|
||||
|
||||
When `--storage-engine=rocksdb`, `gh-ost` will make some changes necessary (e.g. sets isolation level to `READ_COMMITTED`) to support RocksDB.
|
||||
|
||||
### test-on-replica
|
||||
|
||||
Issue the migration on a replica; do not modify data on master. Useful for validating, testing and benchmarking. See [`testing-on-replica`](testing-on-replica.md)
|
||||
@ -277,14 +242,6 @@ Provide a command delimited list of replicas; `gh-ost` will throttle when any of
|
||||
|
||||
Provide an HTTP endpoint; `gh-ost` will issue `HEAD` requests on given URL and throttle whenever response status code is not `200`. The URL can be queried and updated dynamically via [interactive commands](interactive-commands.md). Empty URL disables the HTTP check.
|
||||
|
||||
### throttle-http-interval-millis
|
||||
|
||||
Defaults to 100. Configures the HTTP throttle check interval in milliseconds.
|
||||
|
||||
### throttle-http-timeout-millis
|
||||
|
||||
Defaults to 1000 (1 second). Configures the HTTP throttler check timeout in milliseconds.
|
||||
|
||||
### timestamp-old-table
|
||||
|
||||
Makes the _old_ table include a timestamp value. The _old_ table is what the original table is renamed to at the end of a successful migration. For example, if the table is `gh_ost_test`, then the _old_ table would normally be `_gh_ost_test_del`. With `--timestamp-old-table` it would be, for example, `_gh_ost_test_20170221103147_del`.
|
||||
|
@ -2,8 +2,6 @@
|
||||
|
||||
### Requirements
|
||||
|
||||
- `gh-ost` currently requires MySQL versions 5.7 and greater.
|
||||
|
||||
- You will need to have one server serving Row Based Replication (RBR) format binary logs. Right now `FULL` row image is supported. `MINIMAL` to be supported in the near future. `gh-ost` prefers to work with replicas. You may [still have your master configured with Statement Based Replication](migrating-with-sbr.md) (SBR).
|
||||
|
||||
- If you are using a replica, the table must have an identical schema between the master and replica.
|
||||
@ -20,8 +18,6 @@ The `SUPER` privilege is required for `STOP SLAVE`, `START SLAVE` operations. Th
|
||||
- Switching your `binlog_format` to `ROW`, in the case where it is _not_ `ROW` and you explicitly specified `--switch-to-rbr`
|
||||
- If your replication is already in RBR (`binlog_format=ROW`) you can specify `--assume-rbr` to avoid the `STOP SLAVE/START SLAVE` operations, hence no need for `SUPER`.
|
||||
|
||||
- `gh-ost` uses the `REPEATABLE_READ` transaction isolation level for all MySQL connections, regardless of the server default.
|
||||
|
||||
- Running `--test-on-replica`: before the cut-over phase, `gh-ost` stops replication so that you can compare the two tables and satisfy that the migration is sound.
|
||||
|
||||
### Limitations
|
||||
|
@ -29,7 +29,7 @@ CREATE TABLE tbl (
|
||||
|
||||
(This is also the definition of the _ghost_ table, except that that table would be called `_tbl_gho`).
|
||||
|
||||
In this migration, the _before_ and _after_ versions contain the same unique not-null key (the PRIMARY KEY). To run this migration, `gh-ost` would iterate through the `tbl` table using the primary key, copy rows from `tbl` to the _ghost_ table `_tbl_gho` in primary key order, while also applying the binlog event writes from `tbl` onto `_tbl_gho`.
|
||||
In this migration, the _before_ and _after_ versions contain the same unique not-null key (the PRIMARY KEY). To run this migration, `gh-ost` would iterate through the `tbl` table using the primary key, copy rows from `tbl` to the _ghost_ table `_tbl_gho` in primary key order, while also applying the binlog event writes from `tble` onto `_tbl_gho`.
|
||||
|
||||
The applying of the binlog events is what requires the shared unique key. For example, an `UPDATE` statement to `tbl` translates to a `REPLACE` statement which `gh-ost` applies to `_tbl_gho`. A `REPLACE` statement expects to insert or replace an existing row based on its row's values and the table's unique key constraints. In particular, if inserting that row would result in a unique key violation (e.g., a row with that primary key already exists), it would _replace_ that existing row with the new values.
|
||||
|
||||
|
@ -38,7 +38,7 @@ Note that you may dynamically change both `--max-lag-millis` and the `throttle-c
|
||||
|
||||
`--max-load='Threads_running=100,Threads_connected=500'`
|
||||
|
||||
Metrics must be valid, numeric [status variables](https://dev.mysql.com/doc/refman/5.7/en/server-status-variables.html)
|
||||
Metrics must be valid, numeric [status variables](http://dev.mysql.com/doc/refman/5.6/en/server-status-variables.html)
|
||||
|
||||
#### Throttle query
|
||||
|
||||
@ -97,7 +97,7 @@ Copy: 0/2915 0.0%; Applied: 0; Backlog: 0/100; Elapsed: 42s(copy), 42s(total); s
|
||||
|
||||
Throttling time is limited by the availability of the binary logs. When throttling begins, `gh-ost` suspends reading the binary logs, and expects to resume reading from same binary log where it paused.
|
||||
|
||||
Your availability of binary logs is typically determined by the [expire_logs_days](https://dev.mysql.com/doc/refman/5.7/en/replication-options-binary-log.html#sysvar_expire_logs_days) variable. If you have `expire_logs_days = 10` (or check `select @@global.expire_logs_days`), then you should be able to throttle for up to `10` days.
|
||||
Your availability of binary logs is typically determined by the [expire_logs_days](https://dev.mysql.com/doc/refman/5.6/en/server-system-variables.html#sysvar_expire_logs_days) variable. If you have `expire_logs_days = 10` (or check `select @@global.expire_logs_days`), then you should be able to throttle for up to `10` days.
|
||||
|
||||
Having said that, throttling for so long is far fetching, in that the `gh-ost` process itself must be kept alive during that time; and the amount of binary logs to process once it resumes will potentially take days to replay.
|
||||
|
||||
|
@ -7,7 +7,7 @@ Existing MySQL schema migration tools:
|
||||
- [LHM](https://github.com/soundcloud/lhm)
|
||||
- [oak-online-alter-table](https://github.com/shlomi-noach/openarkkit)
|
||||
|
||||
are all using [triggers](https://dev.mysql.com/doc/refman/5.7/en/triggers.html) to propagate live changes on your table onto a ghost/shadow table that is slowly being synchronized. The tools not all work the same: while most use a synchronous approach (all changes applied on the ghost table), the Facebook tool uses an asynchronous approach (changes are appended to a changelog table, later reviewed and applied on ghost table).
|
||||
are all using [triggers](http://dev.mysql.com/doc/refman/5.6/en/triggers.html) to propagate live changes on your table onto a ghost/shadow table that is slowly being synchronized. The tools not all work the same: while most use a synchronous approach (all changes applied on the ghost table), the Facebook tool uses an asynchronous approach (changes are appended to a changelog table, later reviewed and applied on ghost table).
|
||||
|
||||
Use of triggers simplifies a lot of the flow in doing a live table migration, but also poses some limitations or difficulties. Here are reasons why we choose to [design a triggerless solution](triggerless-design.md) to schema migrations.
|
||||
|
||||
|
27
go.mod
27
go.mod
@ -1,27 +0,0 @@
|
||||
module github.com/github/gh-ost
|
||||
|
||||
go 1.17
|
||||
|
||||
require (
|
||||
github.com/go-ini/ini v1.62.0
|
||||
github.com/go-mysql-org/go-mysql v1.3.0
|
||||
github.com/go-sql-driver/mysql v1.6.0
|
||||
github.com/openark/golib v0.0.0-20210531070646-355f37940af8
|
||||
github.com/satori/go.uuid v1.2.0
|
||||
golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83
|
||||
golang.org/x/net v0.0.0-20210224082022-3d97a244fca7
|
||||
golang.org/x/text v0.3.6
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/pingcap/errors v0.11.5-0.20201126102027-b0a155152ca3 // indirect
|
||||
github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24 // indirect
|
||||
github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 // indirect
|
||||
github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 // indirect
|
||||
github.com/smartystreets/goconvey v1.6.4 // indirect
|
||||
go.uber.org/atomic v1.7.0 // indirect
|
||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 // indirect
|
||||
golang.org/x/term v0.0.0-20220526004731-065cf7ba2467 // indirect
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
|
||||
gopkg.in/ini.v1 v1.62.0 // indirect
|
||||
)
|
136
go.sum
136
go.sum
@ -1,136 +0,0 @@
|
||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
github.com/cznic/golex v0.0.0-20181122101858-9c343928389c/go.mod h1:+bmmJDNmKlhWNG+gwWCkaBoTy39Fs+bzRxVBzoTQbIc=
|
||||
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM=
|
||||
github.com/cznic/parser v0.0.0-20160622100904-31edd927e5b1/go.mod h1:2B43mz36vGZNZEwkWi8ayRSSUXLfjL8OkbzwW4NcPMM=
|
||||
github.com/cznic/sortutil v0.0.0-20181122101858-f5f958428db8/go.mod h1:q2w6Bg5jeox1B+QkJ6Wp/+Vn0G/bo3f1uY7Fn3vivIQ=
|
||||
github.com/cznic/strutil v0.0.0-20171016134553-529a34b1c186/go.mod h1:AHHPPPXTw0h6pVabbcbyGRK1DckRn7r/STdZEeIDzZc=
|
||||
github.com/cznic/y v0.0.0-20170802143616-045f81c6662a/go.mod h1:1rk5VM7oSnA4vjp+hrLQ3HWHa+Y4yPCa3/CsJrcNnvs=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/go-ini/ini v1.62.0 h1:7VJT/ZXjzqSrvtraFp4ONq80hTcRQth1c9ZnQ3uNQvU=
|
||||
github.com/go-ini/ini v1.62.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
|
||||
github.com/go-mysql-org/go-mysql v1.3.0 h1:lpNqkwdPzIrYSZGdqt8HIgAXZaK6VxBNfr8f7Z4FgGg=
|
||||
github.com/go-mysql-org/go-mysql v1.3.0/go.mod h1:3lFZKf7l95Qo70+3XB2WpiSf9wu2s3na3geLMaIIrqQ=
|
||||
github.com/go-sql-driver/mysql v1.3.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
|
||||
github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
|
||||
github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE=
|
||||
github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
|
||||
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
|
||||
github.com/jmoiron/sqlx v1.3.3/go.mod h1:2BljVx/86SuTyjE+aPYlHCTNvZrnJXghYGpNiXLBMCQ=
|
||||
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
|
||||
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
|
||||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
|
||||
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
|
||||
github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
|
||||
github.com/openark/golib v0.0.0-20210531070646-355f37940af8 h1:9ciIHNuyFqRWi9NpMNw9sVLB6z1ItpP5ZhTY9Q1xVu4=
|
||||
github.com/openark/golib v0.0.0-20210531070646-355f37940af8/go.mod h1:1jj8x1eDVZxgc/Z4VyamX4qTbAdHPUQA6NeVtCd8Sl8=
|
||||
github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8 h1:USx2/E1bX46VG32FIw034Au6seQ2fY9NEILmNh/UlQg=
|
||||
github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8/go.mod h1:B1+S9LNcuMyLH/4HMTViQOJevkGiik3wW2AN9zb2fNQ=
|
||||
github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
|
||||
github.com/pingcap/errors v0.11.5-0.20201029093017-5a7df2af2ac7/go.mod h1:G7x87le1poQzLB/TqvTJI2ILrSgobnq4Ut7luOwvfvI=
|
||||
github.com/pingcap/errors v0.11.5-0.20201126102027-b0a155152ca3 h1:LllgC9eGfqzkfubMgjKIDyZYaa609nNWAyNZtpy2B3M=
|
||||
github.com/pingcap/errors v0.11.5-0.20201126102027-b0a155152ca3/go.mod h1:G7x87le1poQzLB/TqvTJI2ILrSgobnq4Ut7luOwvfvI=
|
||||
github.com/pingcap/log v0.0.0-20200511115504-543df19646ad/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8=
|
||||
github.com/pingcap/log v0.0.0-20210317133921-96f4fcab92a4/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8=
|
||||
github.com/pingcap/parser v0.0.0-20210415081931-48e7f467fd74/go.mod h1:xZC8I7bug4GJ5KtHhgAikjTfU4kBv1Sbo3Pf1MZ6lVw=
|
||||
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
|
||||
github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
|
||||
github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
|
||||
github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24 h1:pntxY8Ary0t43dCZ5dqY4YTJCObLY1kIXl0uzMv+7DE=
|
||||
github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
|
||||
github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 h1:xT+JlYxNGqyT+XcU8iUrN18JYed2TvG9yN5ULG2jATM=
|
||||
github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726/go.mod h1:3yhqj7WBBfRhbBlzyOC3gUxftwsU0u8gqevxwIHQpMw=
|
||||
github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 h1:oI+RNwuC9jF2g2lP0u0cVEEZrc/AYBCuFdvwrLWM/6Q=
|
||||
github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07/go.mod h1:yFdBgwXP24JziuRl2NMUahT7nGLNOKi1SIiFxMttVD4=
|
||||
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
|
||||
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
|
||||
github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
|
||||
github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
|
||||
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
||||
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
|
||||
go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
|
||||
go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
|
||||
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
|
||||
go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
|
||||
go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU=
|
||||
go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
|
||||
go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA=
|
||||
go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
|
||||
go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc=
|
||||
go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 h1:/ZScEX8SfEmUGRHs0gxpqteO5nfNW6axyZbBdw9A12g=
|
||||
golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
|
||||
golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||
golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
|
||||
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
|
||||
golang.org/x/net v0.0.0-20210224082022-3d97a244fca7 h1:OgUuv8lsRpBibGNbSizVwKWlysjaNzmC9gYMhPVfqFM=
|
||||
golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68 h1:nxC68pudNYkKU6jWhgrqdreuFiOQWj1Fs7T3VrH4Pjw=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 h1:SrN+KX8Art/Sf4HNj6Zcz06G7VEz+7w9tdXTPOZ7+l4=
|
||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20220526004731-065cf7ba2467 h1:CBpWXWQpIRjzmkkA+M7q9Fqnwd2mZr3AFqexg8YTfoM=
|
||||
golang.org/x/term v0.0.0-20220526004731-065cf7ba2467/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
|
||||
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
|
||||
golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20201125231158-b5590deeca9b/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
||||
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
|
||||
gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU=
|
||||
gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
|
||||
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
|
@ -1,5 +1,5 @@
|
||||
/*
|
||||
Copyright 2022 GitHub Inc.
|
||||
Copyright 2016 GitHub Inc.
|
||||
See https://github.com/github/gh-ost/blob/master/LICENSE
|
||||
*/
|
||||
|
||||
@ -15,13 +15,14 @@ import (
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
uuid "github.com/satori/go.uuid"
|
||||
"github.com/satori/go.uuid"
|
||||
|
||||
"github.com/github/gh-ost/go/mysql"
|
||||
"github.com/github/gh-ost/go/sql"
|
||||
"github.com/openark/golib/log"
|
||||
"github.com/outbrain/golib/log"
|
||||
|
||||
"github.com/go-ini/ini"
|
||||
"gopkg.in/gcfg.v1"
|
||||
gcfgscanner "gopkg.in/gcfg.v1/scanner"
|
||||
)
|
||||
|
||||
// RowsEstimateMethod is the type of row number estimation
|
||||
@ -82,8 +83,6 @@ type MigrationContext struct {
|
||||
AlterStatement string
|
||||
AlterStatementOptions string // anything following the 'ALTER TABLE [schema.]table' from AlterStatement
|
||||
|
||||
countMutex sync.Mutex
|
||||
countTableRowsCancelFunc func()
|
||||
CountTableRows bool
|
||||
ConcurrentCountTableRows bool
|
||||
AllowedRunningOnMaster bool
|
||||
@ -92,7 +91,6 @@ type MigrationContext struct {
|
||||
AssumeRBR bool
|
||||
SkipForeignKeyChecks bool
|
||||
SkipStrictMode bool
|
||||
AllowZeroInDate bool
|
||||
NullableUniqueKeyAllowed bool
|
||||
ApproveRenamedColumns bool
|
||||
SkipRenamedColumns bool
|
||||
@ -101,7 +99,6 @@ type MigrationContext struct {
|
||||
AliyunRDS bool
|
||||
GoogleCloudPlatform bool
|
||||
AzureMySQL bool
|
||||
AttemptInstantDDL bool
|
||||
|
||||
config ContextConfig
|
||||
configMutex *sync.Mutex
|
||||
@ -188,9 +185,7 @@ type MigrationContext struct {
|
||||
CurrentLag int64
|
||||
currentProgress uint64
|
||||
etaNanoseonds int64
|
||||
ThrottleHTTPIntervalMillis int64
|
||||
ThrottleHTTPStatusCode int64
|
||||
ThrottleHTTPTimeoutMillis int64
|
||||
controlReplicasLagResult mysql.ReplicationLagResult
|
||||
TotalRowsCopied int64
|
||||
TotalDMLEventsApplied int64
|
||||
@ -232,8 +227,6 @@ type MigrationContext struct {
|
||||
|
||||
recentBinlogCoordinates mysql.BinlogCoordinates
|
||||
|
||||
BinlogSyncerMaxReconnectAttempts int
|
||||
|
||||
Log Logger
|
||||
}
|
||||
|
||||
@ -292,19 +285,6 @@ func NewMigrationContext() *MigrationContext {
|
||||
}
|
||||
}
|
||||
|
||||
func (this *MigrationContext) SetConnectionConfig(storageEngine string) error {
|
||||
var transactionIsolation string
|
||||
switch storageEngine {
|
||||
case "rocksdb":
|
||||
transactionIsolation = "READ-COMMITTED"
|
||||
default:
|
||||
transactionIsolation = "REPEATABLE-READ"
|
||||
}
|
||||
this.InspectorConnectionConfig.TransactionIsolation = transactionIsolation
|
||||
this.ApplierConnectionConfig.TransactionIsolation = transactionIsolation
|
||||
return nil
|
||||
}
|
||||
|
||||
func getSafeTableName(baseName string, suffix string) string {
|
||||
name := fmt.Sprintf("_%s_%s", baseName, suffix)
|
||||
if len(name) <= mysql.MaxTableNameLength {
|
||||
@ -443,44 +423,10 @@ func (this *MigrationContext) IsTransactionalTable() bool {
|
||||
{
|
||||
return true
|
||||
}
|
||||
case "rocksdb":
|
||||
{
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// SetCountTableRowsCancelFunc sets the cancel function for the CountTableRows query context
|
||||
func (this *MigrationContext) SetCountTableRowsCancelFunc(f func()) {
|
||||
this.countMutex.Lock()
|
||||
defer this.countMutex.Unlock()
|
||||
|
||||
this.countTableRowsCancelFunc = f
|
||||
}
|
||||
|
||||
// IsCountingTableRows returns true if the migration has a table count query running
|
||||
func (this *MigrationContext) IsCountingTableRows() bool {
|
||||
this.countMutex.Lock()
|
||||
defer this.countMutex.Unlock()
|
||||
|
||||
return this.countTableRowsCancelFunc != nil
|
||||
}
|
||||
|
||||
// CancelTableRowsCount cancels the CountTableRows query context. It is safe to
|
||||
// call function even when IsCountingTableRows is false.
|
||||
func (this *MigrationContext) CancelTableRowsCount() {
|
||||
this.countMutex.Lock()
|
||||
defer this.countMutex.Unlock()
|
||||
|
||||
if this.countTableRowsCancelFunc == nil {
|
||||
return
|
||||
}
|
||||
|
||||
this.countTableRowsCancelFunc()
|
||||
this.countTableRowsCancelFunc = nil
|
||||
}
|
||||
|
||||
// ElapsedTime returns time since very beginning of the process
|
||||
func (this *MigrationContext) ElapsedTime() time.Duration {
|
||||
return time.Since(this.StartTime)
|
||||
@ -862,39 +808,10 @@ func (this *MigrationContext) ReadConfigFile() error {
|
||||
if this.ConfigFile == "" {
|
||||
return nil
|
||||
}
|
||||
cfg, err := ini.Load(this.ConfigFile)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if cfg.Section("client").HasKey("user") {
|
||||
this.config.Client.User = cfg.Section("client").Key("user").String()
|
||||
}
|
||||
|
||||
if cfg.Section("client").HasKey("password") {
|
||||
this.config.Client.Password = cfg.Section("client").Key("password").String()
|
||||
}
|
||||
|
||||
if cfg.Section("osc").HasKey("chunk_size") {
|
||||
this.config.Osc.Chunk_Size, err = cfg.Section("osc").Key("chunk_size").Int64()
|
||||
if err != nil {
|
||||
return fmt.Errorf("Unable to read osc chunk size: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
if cfg.Section("osc").HasKey("max_load") {
|
||||
this.config.Osc.Max_Load = cfg.Section("osc").Key("max_load").String()
|
||||
}
|
||||
|
||||
if cfg.Section("osc").HasKey("replication_lag_query") {
|
||||
this.config.Osc.Replication_Lag_Query = cfg.Section("osc").Key("replication_lag_query").String()
|
||||
}
|
||||
|
||||
if cfg.Section("osc").HasKey("max_lag_millis") {
|
||||
this.config.Osc.Max_Lag_Millis, err = cfg.Section("osc").Key("max_lag_millis").Int64()
|
||||
if err != nil {
|
||||
return fmt.Errorf("Unable to read max lag millis: %w", err)
|
||||
}
|
||||
gcfg.RelaxedParserMode = true
|
||||
gcfgscanner.RelaxedScannerMode = true
|
||||
if err := gcfg.ReadFileInto(&this.config, this.ConfigFile); err != nil {
|
||||
return fmt.Errorf("Error reading config file %s. Details: %s", this.ConfigFile, err.Error())
|
||||
}
|
||||
|
||||
// We accept user & password in the form "${SOME_ENV_VARIABLE}" in which case we pull
|
||||
|
@ -1,18 +1,16 @@
|
||||
/*
|
||||
Copyright 2022 GitHub Inc.
|
||||
Copyright 2016 GitHub Inc.
|
||||
See https://github.com/github/gh-ost/blob/master/LICENSE
|
||||
*/
|
||||
|
||||
package base
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/openark/golib/log"
|
||||
test "github.com/openark/golib/tests"
|
||||
"github.com/outbrain/golib/log"
|
||||
test "github.com/outbrain/golib/tests"
|
||||
)
|
||||
|
||||
func init() {
|
||||
@ -58,65 +56,3 @@ func TestGetTableNames(t *testing.T) {
|
||||
test.S(t).ExpectEquals(context.GetChangelogTableName(), "_tmp_ghc")
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadConfigFile(t *testing.T) {
|
||||
{
|
||||
context := NewMigrationContext()
|
||||
context.ConfigFile = "/does/not/exist"
|
||||
if err := context.ReadConfigFile(); err == nil {
|
||||
t.Fatal("Expected .ReadConfigFile() to return an error, got nil")
|
||||
}
|
||||
}
|
||||
{
|
||||
f, err := ioutil.TempFile("", t.Name())
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create tmp file: %v", err)
|
||||
}
|
||||
defer os.Remove(f.Name())
|
||||
|
||||
f.Write([]byte("[client]"))
|
||||
context := NewMigrationContext()
|
||||
context.ConfigFile = f.Name()
|
||||
if err := context.ReadConfigFile(); err != nil {
|
||||
t.Fatalf(".ReadConfigFile() failed: %v", err)
|
||||
}
|
||||
}
|
||||
{
|
||||
f, err := ioutil.TempFile("", t.Name())
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create tmp file: %v", err)
|
||||
}
|
||||
defer os.Remove(f.Name())
|
||||
|
||||
f.Write([]byte("[client]\nuser=test\npassword=123456"))
|
||||
context := NewMigrationContext()
|
||||
context.ConfigFile = f.Name()
|
||||
if err := context.ReadConfigFile(); err != nil {
|
||||
t.Fatalf(".ReadConfigFile() failed: %v", err)
|
||||
}
|
||||
|
||||
if context.config.Client.User != "test" {
|
||||
t.Fatalf("Expected client user %q, got %q", "test", context.config.Client.User)
|
||||
} else if context.config.Client.Password != "123456" {
|
||||
t.Fatalf("Expected client password %q, got %q", "123456", context.config.Client.Password)
|
||||
}
|
||||
}
|
||||
{
|
||||
f, err := ioutil.TempFile("", t.Name())
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create tmp file: %v", err)
|
||||
}
|
||||
defer os.Remove(f.Name())
|
||||
|
||||
f.Write([]byte("[osc]\nmax_load=10"))
|
||||
context := NewMigrationContext()
|
||||
context.ConfigFile = f.Name()
|
||||
if err := context.ReadConfigFile(); err != nil {
|
||||
t.Fatalf(".ReadConfigFile() failed: %v", err)
|
||||
}
|
||||
|
||||
if context.config.Osc.Max_Load != "10" {
|
||||
t.Fatalf("Expected osc 'max_load' %q, got %q", "10", context.config.Osc.Max_Load)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -1,12 +1,7 @@
|
||||
/*
|
||||
Copyright 2022 GitHub Inc.
|
||||
See https://github.com/github/gh-ost/blob/master/LICENSE
|
||||
*/
|
||||
|
||||
package base
|
||||
|
||||
import (
|
||||
"github.com/openark/golib/log"
|
||||
"github.com/outbrain/golib/log"
|
||||
)
|
||||
|
||||
type simpleLogger struct{}
|
||||
@ -17,18 +12,22 @@ func NewDefaultLogger() *simpleLogger {
|
||||
|
||||
func (*simpleLogger) Debug(args ...interface{}) {
|
||||
log.Debug(args[0].(string), args[1:])
|
||||
return
|
||||
}
|
||||
|
||||
func (*simpleLogger) Debugf(format string, args ...interface{}) {
|
||||
log.Debugf(format, args...)
|
||||
return
|
||||
}
|
||||
|
||||
func (*simpleLogger) Info(args ...interface{}) {
|
||||
log.Info(args[0].(string), args[1:])
|
||||
return
|
||||
}
|
||||
|
||||
func (*simpleLogger) Infof(format string, args ...interface{}) {
|
||||
log.Infof(format, args...)
|
||||
return
|
||||
}
|
||||
|
||||
func (*simpleLogger) Warning(args ...interface{}) error {
|
||||
@ -65,8 +64,10 @@ func (*simpleLogger) Fatale(err error) error {
|
||||
|
||||
func (*simpleLogger) SetLevel(level log.LogLevel) {
|
||||
log.SetLevel(level)
|
||||
return
|
||||
}
|
||||
|
||||
func (*simpleLogger) SetPrintStackTrace(printStackTraceFlag bool) {
|
||||
log.SetPrintStackTrace(printStackTraceFlag)
|
||||
return
|
||||
}
|
||||
|
@ -8,8 +8,8 @@ package base
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/openark/golib/log"
|
||||
test "github.com/openark/golib/tests"
|
||||
"github.com/outbrain/golib/log"
|
||||
test "github.com/outbrain/golib/tests"
|
||||
)
|
||||
|
||||
func init() {
|
||||
|
@ -1,5 +1,5 @@
|
||||
/*
|
||||
Copyright 2022 GitHub Inc.
|
||||
Copyright 2016 GitHub Inc.
|
||||
See https://github.com/github/gh-ost/blob/master/LICENSE
|
||||
*/
|
||||
|
||||
@ -25,7 +25,9 @@ func PrettifyDurationOutput(d time.Duration) string {
|
||||
if d < time.Second {
|
||||
return "0s"
|
||||
}
|
||||
return prettifyDurationRegexp.ReplaceAllString(d.String(), "")
|
||||
result := fmt.Sprintf("%s", d)
|
||||
result = prettifyDurationRegexp.ReplaceAllString(result, "")
|
||||
return result
|
||||
}
|
||||
|
||||
func FileExists(fileName string) bool {
|
||||
@ -69,7 +71,7 @@ func ValidateConnection(db *gosql.DB, connectionConfig *mysql.ConnectionConfig,
|
||||
return "", err
|
||||
}
|
||||
extraPortQuery := `select @@global.extra_port`
|
||||
if err := db.QueryRow(extraPortQuery).Scan(&extraPort); err != nil { // nolint:staticcheck
|
||||
if err := db.QueryRow(extraPortQuery).Scan(&extraPort); err != nil {
|
||||
// swallow this error. not all servers support extra_port
|
||||
}
|
||||
// AliyunRDS set users port to "NULL", replace it by gh-ost param
|
||||
|
@ -8,8 +8,8 @@ package base
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/openark/golib/log"
|
||||
test "github.com/openark/golib/tests"
|
||||
"github.com/outbrain/golib/log"
|
||||
test "github.com/outbrain/golib/tests"
|
||||
)
|
||||
|
||||
func init() {
|
||||
|
@ -1,5 +1,5 @@
|
||||
/*
|
||||
Copyright 2022 GitHub Inc.
|
||||
Copyright 2016 GitHub Inc.
|
||||
See https://github.com/github/gh-ost/blob/master/LICENSE
|
||||
*/
|
||||
|
||||
@ -13,8 +13,8 @@ import (
|
||||
"github.com/github/gh-ost/go/mysql"
|
||||
"github.com/github/gh-ost/go/sql"
|
||||
|
||||
gomysql "github.com/go-mysql-org/go-mysql/mysql"
|
||||
"github.com/go-mysql-org/go-mysql/replication"
|
||||
gomysql "github.com/siddontang/go-mysql/mysql"
|
||||
"github.com/siddontang/go-mysql/replication"
|
||||
"golang.org/x/net/context"
|
||||
)
|
||||
|
||||
@ -28,25 +28,31 @@ type GoMySQLReader struct {
|
||||
LastAppliedRowsEventHint mysql.BinlogCoordinates
|
||||
}
|
||||
|
||||
func NewGoMySQLReader(migrationContext *base.MigrationContext) *GoMySQLReader {
|
||||
connectionConfig := migrationContext.InspectorConnectionConfig
|
||||
return &GoMySQLReader{
|
||||
func NewGoMySQLReader(migrationContext *base.MigrationContext) (binlogReader *GoMySQLReader, err error) {
|
||||
binlogReader = &GoMySQLReader{
|
||||
migrationContext: migrationContext,
|
||||
connectionConfig: connectionConfig,
|
||||
connectionConfig: migrationContext.InspectorConnectionConfig,
|
||||
currentCoordinates: mysql.BinlogCoordinates{},
|
||||
currentCoordinatesMutex: &sync.Mutex{},
|
||||
binlogSyncer: replication.NewBinlogSyncer(replication.BinlogSyncerConfig{
|
||||
ServerID: uint32(migrationContext.ReplicaServerId),
|
||||
Flavor: gomysql.MySQLFlavor,
|
||||
Host: connectionConfig.Key.Hostname,
|
||||
Port: uint16(connectionConfig.Key.Port),
|
||||
User: connectionConfig.User,
|
||||
Password: connectionConfig.Password,
|
||||
TLSConfig: connectionConfig.TLSConfig(),
|
||||
UseDecimal: true,
|
||||
MaxReconnectAttempts: migrationContext.BinlogSyncerMaxReconnectAttempts,
|
||||
}),
|
||||
binlogSyncer: nil,
|
||||
binlogStreamer: nil,
|
||||
}
|
||||
|
||||
serverId := uint32(migrationContext.ReplicaServerId)
|
||||
|
||||
binlogSyncerConfig := replication.BinlogSyncerConfig{
|
||||
ServerID: serverId,
|
||||
Flavor: "mysql",
|
||||
Host: binlogReader.connectionConfig.Key.Hostname,
|
||||
Port: uint16(binlogReader.connectionConfig.Key.Port),
|
||||
User: binlogReader.connectionConfig.User,
|
||||
Password: binlogReader.connectionConfig.Password,
|
||||
TLSConfig: binlogReader.connectionConfig.TLSConfig(),
|
||||
UseDecimal: true,
|
||||
}
|
||||
binlogReader.binlogSyncer = replication.NewBinlogSyncer(binlogSyncerConfig)
|
||||
|
||||
return binlogReader, err
|
||||
}
|
||||
|
||||
// ConnectBinlogStreamer
|
||||
@ -58,10 +64,7 @@ func (this *GoMySQLReader) ConnectBinlogStreamer(coordinates mysql.BinlogCoordin
|
||||
this.currentCoordinates = coordinates
|
||||
this.migrationContext.Log.Infof("Connecting binlog streamer at %+v", this.currentCoordinates)
|
||||
// Start sync with specified binlog file and position
|
||||
this.binlogStreamer, err = this.binlogSyncer.StartSync(gomysql.Position{
|
||||
Name: this.currentCoordinates.LogFile,
|
||||
Pos: uint32(this.currentCoordinates.LogPos),
|
||||
})
|
||||
this.binlogStreamer, err = this.binlogSyncer.StartSync(gomysql.Position{this.currentCoordinates.LogFile, uint32(this.currentCoordinates.LogPos)})
|
||||
|
||||
return err
|
||||
}
|
||||
@ -139,17 +142,15 @@ func (this *GoMySQLReader) StreamEvents(canStopStreaming func() bool, entriesCha
|
||||
defer this.currentCoordinatesMutex.Unlock()
|
||||
this.currentCoordinates.LogPos = int64(ev.Header.LogPos)
|
||||
}()
|
||||
|
||||
switch binlogEvent := ev.Event.(type) {
|
||||
case *replication.RotateEvent:
|
||||
if rotateEvent, ok := ev.Event.(*replication.RotateEvent); ok {
|
||||
func() {
|
||||
this.currentCoordinatesMutex.Lock()
|
||||
defer this.currentCoordinatesMutex.Unlock()
|
||||
this.currentCoordinates.LogFile = string(binlogEvent.NextLogName)
|
||||
this.currentCoordinates.LogFile = string(rotateEvent.NextLogName)
|
||||
}()
|
||||
this.migrationContext.Log.Infof("rotate to next log from %s:%d to %s", this.currentCoordinates.LogFile, int64(ev.Header.LogPos), binlogEvent.NextLogName)
|
||||
case *replication.RowsEvent:
|
||||
if err := this.handleRowsEvent(ev, binlogEvent, entriesChannel); err != nil {
|
||||
this.migrationContext.Log.Infof("rotate to next log from %s:%d to %s", this.currentCoordinates.LogFile, int64(ev.Header.LogPos), rotateEvent.NextLogName)
|
||||
} else if rowsEvent, ok := ev.Event.(*replication.RowsEvent); ok {
|
||||
if err := this.handleRowsEvent(ev, rowsEvent, entriesChannel); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
@ -1,5 +1,5 @@
|
||||
/*
|
||||
Copyright 2022 GitHub Inc.
|
||||
Copyright 2016 GitHub Inc.
|
||||
See https://github.com/github/gh-ost/blob/master/LICENSE
|
||||
*/
|
||||
|
||||
@ -17,9 +17,9 @@ import (
|
||||
"github.com/github/gh-ost/go/logic"
|
||||
"github.com/github/gh-ost/go/sql"
|
||||
_ "github.com/go-sql-driver/mysql"
|
||||
"github.com/openark/golib/log"
|
||||
"github.com/outbrain/golib/log"
|
||||
|
||||
"golang.org/x/term"
|
||||
"golang.org/x/crypto/ssh/terminal"
|
||||
)
|
||||
|
||||
var AppVersion string
|
||||
@ -67,9 +67,6 @@ func main() {
|
||||
flag.StringVar(&migrationContext.DatabaseName, "database", "", "database name (mandatory)")
|
||||
flag.StringVar(&migrationContext.OriginalTableName, "table", "", "table name (mandatory)")
|
||||
flag.StringVar(&migrationContext.AlterStatement, "alter", "", "alter statement (mandatory)")
|
||||
flag.BoolVar(&migrationContext.AttemptInstantDDL, "attempt-instant-ddl", false, "Attempt to use instant DDL for this migration first")
|
||||
storageEngine := flag.String("storage-engine", "innodb", "Specify table storage engine (default: 'innodb'). When 'rocksdb': the session transaction isolation level is changed from REPEATABLE_READ to READ_COMMITTED.")
|
||||
|
||||
flag.BoolVar(&migrationContext.CountTableRows, "exact-rowcount", false, "actually count table rows as opposed to estimate them (results in more accurate progress estimation)")
|
||||
flag.BoolVar(&migrationContext.ConcurrentCountTableRows, "concurrent-rowcount", true, "(with --exact-rowcount), when true (default): count rows after row-copy begins, concurrently, and adjust row estimate later on; when false: first count rows, then start row copy")
|
||||
flag.BoolVar(&migrationContext.AllowedRunningOnMaster, "allow-on-master", false, "allow this migration to run directly on master. Preferably it would run on a replica")
|
||||
@ -81,7 +78,6 @@ func main() {
|
||||
flag.BoolVar(&migrationContext.DiscardForeignKeys, "discard-foreign-keys", false, "DANGER! This flag will migrate a table that has foreign keys and will NOT create foreign keys on the ghost table, thus your altered table will have NO foreign keys. This is useful for intentional dropping of foreign keys")
|
||||
flag.BoolVar(&migrationContext.SkipForeignKeyChecks, "skip-foreign-key-checks", false, "set to 'true' when you know for certain there are no foreign keys on your table, and wish to skip the time it takes for gh-ost to verify that")
|
||||
flag.BoolVar(&migrationContext.SkipStrictMode, "skip-strict-mode", false, "explicitly tell gh-ost binlog applier not to enforce strict sql mode")
|
||||
flag.BoolVar(&migrationContext.AllowZeroInDate, "allow-zero-in-date", false, "explicitly tell gh-ost binlog applier to ignore NO_ZERO_IN_DATE,NO_ZERO_DATE in sql_mode")
|
||||
flag.BoolVar(&migrationContext.AliyunRDS, "aliyun-rds", false, "set to 'true' when you execute on Aliyun RDS.")
|
||||
flag.BoolVar(&migrationContext.GoogleCloudPlatform, "gcp", false, "set to 'true' when you execute on a 1st generation Google Cloud Platform (GCP).")
|
||||
flag.BoolVar(&migrationContext.AzureMySQL, "azure", false, "set to 'true' when you execute on Azure Database on MySQL.")
|
||||
@ -114,8 +110,6 @@ func main() {
|
||||
throttleControlReplicas := flag.String("throttle-control-replicas", "", "List of replicas on which to check for lag; comma delimited. Example: myhost1.com:3306,myhost2.com,myhost3.com:3307")
|
||||
throttleQuery := flag.String("throttle-query", "", "when given, issued (every second) to check if operation should throttle. Expecting to return zero for no-throttle, >0 for throttle. Query is issued on the migrated server. Make sure this query is lightweight")
|
||||
throttleHTTP := flag.String("throttle-http", "", "when given, gh-ost checks given URL via HEAD request; any response code other than 200 (OK) causes throttling; make sure it has low latency response")
|
||||
flag.Int64Var(&migrationContext.ThrottleHTTPIntervalMillis, "throttle-http-interval-millis", 100, "Number of milliseconds to wait before triggering another HTTP throttle check")
|
||||
flag.Int64Var(&migrationContext.ThrottleHTTPTimeoutMillis, "throttle-http-timeout-millis", 1000, "Number of milliseconds to use as an HTTP throttle check timeout")
|
||||
ignoreHTTPErrors := flag.Bool("ignore-http-errors", false, "ignore HTTP connection errors during throttle check")
|
||||
heartbeatIntervalMillis := flag.Int64("heartbeat-interval-millis", 100, "how frequently would gh-ost inject a heartbeat value")
|
||||
flag.StringVar(&migrationContext.ThrottleFlagFile, "throttle-flag-file", "", "operation pauses when this file exists; hint: use a file that is specific to the table being altered")
|
||||
@ -134,7 +128,6 @@ func main() {
|
||||
flag.Int64Var(&migrationContext.HooksStatusIntervalSec, "hooks-status-interval", 60, "how many seconds to wait between calling onStatus hook")
|
||||
|
||||
flag.UintVar(&migrationContext.ReplicaServerId, "replica-server-id", 99999, "server id used by gh-ost process. Default: 99999")
|
||||
flag.IntVar(&migrationContext.BinlogSyncerMaxReconnectAttempts, "binlogsyncer-max-reconnect-attempts", 0, "when master node fails, the maximum number of binlog synchronization attempts to reconnect. 0 is unlimited")
|
||||
|
||||
maxLoad := flag.String("max-load", "", "Comma delimited status-name=threshold. e.g: 'Threads_running=100,Threads_connected=500'. When status exceeds threshold, app throttles writes")
|
||||
criticalLoad := flag.String("critical-load", "", "Comma delimited status-name=threshold, same format as --max-load. When status exceeds threshold, app panics and quits")
|
||||
@ -184,12 +177,8 @@ func main() {
|
||||
migrationContext.Log.SetLevel(log.ERROR)
|
||||
}
|
||||
|
||||
if err := migrationContext.SetConnectionConfig(*storageEngine); err != nil {
|
||||
migrationContext.Log.Fatale(err)
|
||||
}
|
||||
|
||||
if migrationContext.AlterStatement == "" {
|
||||
log.Fatal("--alter must be provided and statement must not be empty")
|
||||
log.Fatalf("--alter must be provided and statement must not be empty")
|
||||
}
|
||||
parser := sql.NewParserFromAlterStatement(migrationContext.AlterStatement)
|
||||
migrationContext.AlterStatementOptions = parser.GetAlterStatementOptions()
|
||||
@ -198,7 +187,7 @@ func main() {
|
||||
if parser.HasExplicitSchema() {
|
||||
migrationContext.DatabaseName = parser.GetExplicitSchema()
|
||||
} else {
|
||||
log.Fatal("--database must be provided and database name must not be empty, or --alter must specify database name")
|
||||
log.Fatalf("--database must be provided and database name must not be empty, or --alter must specify database name")
|
||||
}
|
||||
}
|
||||
|
||||
@ -210,51 +199,48 @@ func main() {
|
||||
if parser.HasExplicitTable() {
|
||||
migrationContext.OriginalTableName = parser.GetExplicitTable()
|
||||
} else {
|
||||
log.Fatal("--table must be provided and table name must not be empty, or --alter must specify table name")
|
||||
log.Fatalf("--table must be provided and table name must not be empty, or --alter must specify table name")
|
||||
}
|
||||
}
|
||||
migrationContext.Noop = !(*executeFlag)
|
||||
if migrationContext.AllowedRunningOnMaster && migrationContext.TestOnReplica {
|
||||
migrationContext.Log.Fatal("--allow-on-master and --test-on-replica are mutually exclusive")
|
||||
migrationContext.Log.Fatalf("--allow-on-master and --test-on-replica are mutually exclusive")
|
||||
}
|
||||
if migrationContext.AllowedRunningOnMaster && migrationContext.MigrateOnReplica {
|
||||
migrationContext.Log.Fatal("--allow-on-master and --migrate-on-replica are mutually exclusive")
|
||||
migrationContext.Log.Fatalf("--allow-on-master and --migrate-on-replica are mutually exclusive")
|
||||
}
|
||||
if migrationContext.MigrateOnReplica && migrationContext.TestOnReplica {
|
||||
migrationContext.Log.Fatal("--migrate-on-replica and --test-on-replica are mutually exclusive")
|
||||
migrationContext.Log.Fatalf("--migrate-on-replica and --test-on-replica are mutually exclusive")
|
||||
}
|
||||
if migrationContext.SwitchToRowBinlogFormat && migrationContext.AssumeRBR {
|
||||
migrationContext.Log.Fatal("--switch-to-rbr and --assume-rbr are mutually exclusive")
|
||||
migrationContext.Log.Fatalf("--switch-to-rbr and --assume-rbr are mutually exclusive")
|
||||
}
|
||||
if migrationContext.TestOnReplicaSkipReplicaStop {
|
||||
if !migrationContext.TestOnReplica {
|
||||
migrationContext.Log.Fatal("--test-on-replica-skip-replica-stop requires --test-on-replica to be enabled")
|
||||
migrationContext.Log.Fatalf("--test-on-replica-skip-replica-stop requires --test-on-replica to be enabled")
|
||||
}
|
||||
migrationContext.Log.Warning("--test-on-replica-skip-replica-stop enabled. We will not stop replication before cut-over. Ensure you have a plugin that does this.")
|
||||
}
|
||||
if migrationContext.CliMasterUser != "" && migrationContext.AssumeMasterHostname == "" {
|
||||
migrationContext.Log.Fatal("--master-user requires --assume-master-host")
|
||||
migrationContext.Log.Fatalf("--master-user requires --assume-master-host")
|
||||
}
|
||||
if migrationContext.CliMasterPassword != "" && migrationContext.AssumeMasterHostname == "" {
|
||||
migrationContext.Log.Fatal("--master-password requires --assume-master-host")
|
||||
migrationContext.Log.Fatalf("--master-password requires --assume-master-host")
|
||||
}
|
||||
if migrationContext.TLSCACertificate != "" && !migrationContext.UseTLS {
|
||||
migrationContext.Log.Fatal("--ssl-ca requires --ssl")
|
||||
migrationContext.Log.Fatalf("--ssl-ca requires --ssl")
|
||||
}
|
||||
if migrationContext.TLSCertificate != "" && !migrationContext.UseTLS {
|
||||
migrationContext.Log.Fatal("--ssl-cert requires --ssl")
|
||||
migrationContext.Log.Fatalf("--ssl-cert requires --ssl")
|
||||
}
|
||||
if migrationContext.TLSKey != "" && !migrationContext.UseTLS {
|
||||
migrationContext.Log.Fatal("--ssl-key requires --ssl")
|
||||
migrationContext.Log.Fatalf("--ssl-key requires --ssl")
|
||||
}
|
||||
if migrationContext.TLSAllowInsecure && !migrationContext.UseTLS {
|
||||
migrationContext.Log.Fatal("--ssl-allow-insecure requires --ssl")
|
||||
migrationContext.Log.Fatalf("--ssl-allow-insecure requires --ssl")
|
||||
}
|
||||
if *replicationLagQuery != "" {
|
||||
migrationContext.Log.Warning("--replication-lag-query is deprecated")
|
||||
}
|
||||
if *storageEngine == "rocksdb" {
|
||||
migrationContext.Log.Warning("RocksDB storage engine support is experimental")
|
||||
migrationContext.Log.Warningf("--replication-lag-query is deprecated")
|
||||
}
|
||||
|
||||
switch *cutOver {
|
||||
@ -282,7 +268,7 @@ func main() {
|
||||
}
|
||||
if *askPass {
|
||||
fmt.Println("Password:")
|
||||
bytePassword, err := term.ReadPassword(syscall.Stdin)
|
||||
bytePassword, err := terminal.ReadPassword(int(syscall.Stdin))
|
||||
if err != nil {
|
||||
migrationContext.Log.Fatale(err)
|
||||
}
|
||||
@ -311,10 +297,11 @@ func main() {
|
||||
log.Infof("starting gh-ost %+v", AppVersion)
|
||||
acceptSignals(migrationContext)
|
||||
|
||||
migrator := logic.NewMigrator(migrationContext, AppVersion)
|
||||
if err := migrator.Migrate(); err != nil {
|
||||
migrator := logic.NewMigrator(migrationContext)
|
||||
err := migrator.Migrate()
|
||||
if err != nil {
|
||||
migrator.ExecOnFailureHook()
|
||||
migrationContext.Log.Fatale(err)
|
||||
}
|
||||
fmt.Fprintln(os.Stdout, "# Done")
|
||||
fmt.Fprintf(os.Stdout, "# Done\n")
|
||||
}
|
||||
|
@ -1,5 +1,5 @@
|
||||
/*
|
||||
Copyright 2022 GitHub Inc.
|
||||
Copyright 2016 GitHub Inc.
|
||||
See https://github.com/github/gh-ost/blob/master/LICENSE
|
||||
*/
|
||||
|
||||
@ -8,7 +8,6 @@ package logic
|
||||
import (
|
||||
gosql "database/sql"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
@ -17,13 +16,12 @@ import (
|
||||
"github.com/github/gh-ost/go/mysql"
|
||||
"github.com/github/gh-ost/go/sql"
|
||||
|
||||
"github.com/openark/golib/log"
|
||||
"github.com/openark/golib/sqlutils"
|
||||
"github.com/outbrain/golib/sqlutils"
|
||||
"sync"
|
||||
)
|
||||
|
||||
const (
|
||||
GhostChangelogTableComment = "gh-ost changelog"
|
||||
atomicCutOverMagicHint = "ghost-cut-over-sentry"
|
||||
atomicCutOverMagicHint = "ghost-cut-over-sentry"
|
||||
)
|
||||
|
||||
type dmlBuildResult struct {
|
||||
@ -48,7 +46,7 @@ func newDmlBuildResultError(err error) *dmlBuildResult {
|
||||
}
|
||||
}
|
||||
|
||||
// Applier connects and writes the applier-server, which is the server where migration
|
||||
// Applier connects and writes the the applier-server, which is the server where migration
|
||||
// happens. This is typically the master, but could be a replica when `--test-on-replica` or
|
||||
// `--execute-on-replica` are given.
|
||||
// Applier is the one to actually write row data and apply binlog events onto the ghost table.
|
||||
@ -72,6 +70,7 @@ func NewApplier(migrationContext *base.MigrationContext) *Applier {
|
||||
}
|
||||
|
||||
func (this *Applier) InitDBConnections() (err error) {
|
||||
|
||||
applierUri := this.connectionConfig.GetDBUri(this.migrationContext.DatabaseName)
|
||||
if this.db, _, err = mysql.GetDB(this.migrationContext.Uuid, applierUri); err != nil {
|
||||
return err
|
||||
@ -117,33 +116,6 @@ func (this *Applier) validateAndReadTimeZone() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// generateSqlModeQuery return a `sql_mode = ...` query, to be wrapped with a `set session` or `set global`,
|
||||
// based on gh-ost configuration:
|
||||
// - User may skip strict mode
|
||||
// - User may allow zero dats or zero in dates
|
||||
func (this *Applier) generateSqlModeQuery() string {
|
||||
sqlModeAddendum := []string{`NO_AUTO_VALUE_ON_ZERO`}
|
||||
if !this.migrationContext.SkipStrictMode {
|
||||
sqlModeAddendum = append(sqlModeAddendum, `STRICT_ALL_TABLES`)
|
||||
}
|
||||
sqlModeQuery := fmt.Sprintf("CONCAT(@@session.sql_mode, ',%s')", strings.Join(sqlModeAddendum, ","))
|
||||
if this.migrationContext.AllowZeroInDate {
|
||||
sqlModeQuery = fmt.Sprintf("REPLACE(REPLACE(%s, 'NO_ZERO_IN_DATE', ''), 'NO_ZERO_DATE', '')", sqlModeQuery)
|
||||
}
|
||||
|
||||
return fmt.Sprintf("sql_mode = %s", sqlModeQuery)
|
||||
}
|
||||
|
||||
// generateInstantDDLQuery returns the SQL for this ALTER operation
|
||||
// with an INSTANT assertion (requires MySQL 8.0+)
|
||||
func (this *Applier) generateInstantDDLQuery() string {
|
||||
return fmt.Sprintf(`ALTER /* gh-ost */ TABLE %s.%s %s, ALGORITHM=INSTANT`,
|
||||
sql.EscapeName(this.migrationContext.DatabaseName),
|
||||
sql.EscapeName(this.migrationContext.OriginalTableName),
|
||||
this.migrationContext.AlterStatementOptions,
|
||||
)
|
||||
}
|
||||
|
||||
// readTableColumns reads table columns on applier
|
||||
func (this *Applier) readTableColumns() (err error) {
|
||||
this.migrationContext.Log.Infof("Examining table structure on applier")
|
||||
@ -197,27 +169,6 @@ func (this *Applier) ValidateOrDropExistingTables() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// AttemptInstantDDL attempts to use instant DDL (from MySQL 8.0, and earlier in Aurora and some others).
|
||||
// If successful, the operation is only a meta-data change so a lot of time is saved!
|
||||
// The risk of attempting to instant DDL when not supported is that a metadata lock may be acquired.
|
||||
// This is minor, since gh-ost will eventually require a metadata lock anyway, but at the cut-over stage.
|
||||
// Instant operations include:
|
||||
// - Adding a column
|
||||
// - Dropping a column
|
||||
// - Dropping an index
|
||||
// - Extending a VARCHAR column
|
||||
// - Adding a virtual generated column
|
||||
// It is not reliable to parse the `alter` statement to determine if it is instant or not.
|
||||
// This is because the table might be in an older row format, or have some other incompatibility
|
||||
// that is difficult to identify.
|
||||
func (this *Applier) AttemptInstantDDL() error {
|
||||
query := this.generateInstantDDLQuery()
|
||||
this.migrationContext.Log.Infof("INSTANT DDL query is: %s", query)
|
||||
// We don't need a trx, because for instant DDL the SQL mode doesn't matter.
|
||||
_, err := this.db.Exec(query)
|
||||
return err
|
||||
}
|
||||
|
||||
// CreateGhostTable creates the ghost table on the applier host
|
||||
func (this *Applier) CreateGhostTable() error {
|
||||
query := fmt.Sprintf(`create /* gh-ost */ table %s.%s like %s.%s`,
|
||||
@ -230,33 +181,11 @@ func (this *Applier) CreateGhostTable() error {
|
||||
sql.EscapeName(this.migrationContext.DatabaseName),
|
||||
sql.EscapeName(this.migrationContext.GetGhostTableName()),
|
||||
)
|
||||
|
||||
err := func() error {
|
||||
tx, err := this.db.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
|
||||
sessionQuery := fmt.Sprintf(`SET SESSION time_zone = '%s'`, this.migrationContext.ApplierTimeZone)
|
||||
sessionQuery = fmt.Sprintf("%s, %s", sessionQuery, this.generateSqlModeQuery())
|
||||
|
||||
if _, err := tx.Exec(sessionQuery); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.Exec(query); err != nil {
|
||||
return err
|
||||
}
|
||||
this.migrationContext.Log.Infof("Ghost table created")
|
||||
if err := tx.Commit(); err != nil {
|
||||
// Neither SET SESSION nor ALTER are really transactional, so strictly speaking
|
||||
// there's no need to commit; but let's do this the legit way anyway.
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}()
|
||||
|
||||
return err
|
||||
if _, err := sqlutils.ExecNoPrepare(this.db, query); err != nil {
|
||||
return err
|
||||
}
|
||||
this.migrationContext.Log.Infof("Ghost table created")
|
||||
return nil
|
||||
}
|
||||
|
||||
// AlterGhost applies `alter` statement on ghost table
|
||||
@ -271,33 +200,11 @@ func (this *Applier) AlterGhost() error {
|
||||
sql.EscapeName(this.migrationContext.GetGhostTableName()),
|
||||
)
|
||||
this.migrationContext.Log.Debugf("ALTER statement: %s", query)
|
||||
|
||||
err := func() error {
|
||||
tx, err := this.db.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
|
||||
sessionQuery := fmt.Sprintf(`SET SESSION time_zone = '%s'`, this.migrationContext.ApplierTimeZone)
|
||||
sessionQuery = fmt.Sprintf("%s, %s", sessionQuery, this.generateSqlModeQuery())
|
||||
|
||||
if _, err := tx.Exec(sessionQuery); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.Exec(query); err != nil {
|
||||
return err
|
||||
}
|
||||
this.migrationContext.Log.Infof("Ghost table altered")
|
||||
if err := tx.Commit(); err != nil {
|
||||
// Neither SET SESSION nor ALTER are really transactional, so strictly speaking
|
||||
// there's no need to commit; but let's do this the legit way anyway.
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}()
|
||||
|
||||
return err
|
||||
if _, err := sqlutils.ExecNoPrepare(this.db, query); err != nil {
|
||||
return err
|
||||
}
|
||||
this.migrationContext.Log.Infof("Ghost table altered")
|
||||
return nil
|
||||
}
|
||||
|
||||
// AlterGhost applies `alter` statement on ghost table
|
||||
@ -325,16 +232,16 @@ func (this *Applier) CreateChangelogTable() error {
|
||||
return err
|
||||
}
|
||||
query := fmt.Sprintf(`create /* gh-ost */ table %s.%s (
|
||||
id bigint unsigned auto_increment,
|
||||
id bigint auto_increment,
|
||||
last_update timestamp not null DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
hint varchar(64) charset ascii not null,
|
||||
value varchar(4096) charset ascii not null,
|
||||
primary key(id),
|
||||
unique key hint_uidx(hint)
|
||||
) auto_increment=256 comment='%s'`,
|
||||
) auto_increment=256
|
||||
`,
|
||||
sql.EscapeName(this.migrationContext.DatabaseName),
|
||||
sql.EscapeName(this.migrationContext.GetChangelogTableName()),
|
||||
GhostChangelogTableComment,
|
||||
)
|
||||
this.migrationContext.Log.Infof("Creating changelog table %s.%s",
|
||||
sql.EscapeName(this.migrationContext.DatabaseName),
|
||||
@ -436,9 +343,8 @@ func (this *Applier) InitiateHeartbeat() {
|
||||
}
|
||||
injectHeartbeat()
|
||||
|
||||
ticker := time.NewTicker(time.Duration(this.migrationContext.HeartbeatIntervalMilliseconds) * time.Millisecond)
|
||||
defer ticker.Stop()
|
||||
for range ticker.C {
|
||||
heartbeatTick := time.Tick(time.Duration(this.migrationContext.HeartbeatIntervalMilliseconds) * time.Millisecond)
|
||||
for range heartbeatTick {
|
||||
if atomic.LoadInt64(&this.finishedMigrating) > 0 {
|
||||
return
|
||||
}
|
||||
@ -468,20 +374,17 @@ func (this *Applier) ExecuteThrottleQuery() (int64, error) {
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// readMigrationMinValues returns the minimum values to be iterated on rowcopy
|
||||
func (this *Applier) readMigrationMinValues(tx *gosql.Tx, uniqueKey *sql.UniqueKey) error {
|
||||
// ReadMigrationMinValues returns the minimum values to be iterated on rowcopy
|
||||
func (this *Applier) ReadMigrationMinValues(uniqueKey *sql.UniqueKey) error {
|
||||
this.migrationContext.Log.Debugf("Reading migration range according to key: %s", uniqueKey.Name)
|
||||
query, err := sql.BuildUniqueKeyMinValuesPreparedQuery(this.migrationContext.DatabaseName, this.migrationContext.OriginalTableName, &uniqueKey.Columns)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
rows, err := tx.Query(query)
|
||||
rows, err := this.db.Query(query)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
for rows.Next() {
|
||||
this.migrationContext.MigrationRangeMinValues = sql.NewColumnValues(uniqueKey.Len())
|
||||
if err = rows.Scan(this.migrationContext.MigrationRangeMinValues.ValuesPointers...); err != nil {
|
||||
@ -490,23 +393,21 @@ func (this *Applier) readMigrationMinValues(tx *gosql.Tx, uniqueKey *sql.UniqueK
|
||||
}
|
||||
this.migrationContext.Log.Infof("Migration min values: [%s]", this.migrationContext.MigrationRangeMinValues)
|
||||
|
||||
return rows.Err()
|
||||
err = rows.Err()
|
||||
return err
|
||||
}
|
||||
|
||||
// readMigrationMaxValues returns the maximum values to be iterated on rowcopy
|
||||
func (this *Applier) readMigrationMaxValues(tx *gosql.Tx, uniqueKey *sql.UniqueKey) error {
|
||||
// ReadMigrationMaxValues returns the maximum values to be iterated on rowcopy
|
||||
func (this *Applier) ReadMigrationMaxValues(uniqueKey *sql.UniqueKey) error {
|
||||
this.migrationContext.Log.Debugf("Reading migration range according to key: %s", uniqueKey.Name)
|
||||
query, err := sql.BuildUniqueKeyMaxValuesPreparedQuery(this.migrationContext.DatabaseName, this.migrationContext.OriginalTableName, &uniqueKey.Columns)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
rows, err := tx.Query(query)
|
||||
rows, err := this.db.Query(query)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
for rows.Next() {
|
||||
this.migrationContext.MigrationRangeMaxValues = sql.NewColumnValues(uniqueKey.Len())
|
||||
if err = rows.Scan(this.migrationContext.MigrationRangeMaxValues.ValuesPointers...); err != nil {
|
||||
@ -515,45 +416,19 @@ func (this *Applier) readMigrationMaxValues(tx *gosql.Tx, uniqueKey *sql.UniqueK
|
||||
}
|
||||
this.migrationContext.Log.Infof("Migration max values: [%s]", this.migrationContext.MigrationRangeMaxValues)
|
||||
|
||||
return rows.Err()
|
||||
err = rows.Err()
|
||||
return err
|
||||
}
|
||||
|
||||
// ReadMigrationRangeValues reads min/max values that will be used for rowcopy.
|
||||
// Before read min/max, write a changelog state into the ghc table to avoid lost data in mysql two-phase commit.
|
||||
/*
|
||||
Detail description of the lost data in mysql two-phase commit issue by @Fanduzi:
|
||||
When using semi-sync and setting rpl_semi_sync_master_wait_point=AFTER_SYNC,
|
||||
if an INSERT statement is being committed but blocks due to an unmet ack count,
|
||||
the data inserted by the transaction is not visible to ReadMigrationRangeValues,
|
||||
so the copy of the existing data in the table does not include the new row inserted by the transaction.
|
||||
However, the binlog event for the transaction is already written to the binlog,
|
||||
so the addDMLEventsListener only captures the binlog event after the transaction,
|
||||
and thus the transaction's binlog event is not captured, resulting in data loss.
|
||||
|
||||
If write a changelog into ghc table before ReadMigrationRangeValues, and the transaction commit blocks
|
||||
because the ack is not met, then the changelog will not be able to write, so the ReadMigrationRangeValues
|
||||
will not be run. When the changelog writes successfully, the ReadMigrationRangeValues will read the
|
||||
newly inserted data, thus Avoiding data loss due to the above problem.
|
||||
*/
|
||||
// ReadMigrationRangeValues reads min/max values that will be used for rowcopy
|
||||
func (this *Applier) ReadMigrationRangeValues() error {
|
||||
if _, err := this.WriteChangelogState(string(ReadMigrationRangeValues)); err != nil {
|
||||
if err := this.ReadMigrationMinValues(this.migrationContext.UniqueKey); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
tx, err := this.db.Begin()
|
||||
if err != nil {
|
||||
if err := this.ReadMigrationMaxValues(this.migrationContext.UniqueKey); err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
|
||||
if err := this.readMigrationMinValues(tx, this.migrationContext.UniqueKey); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := this.readMigrationMaxValues(tx, this.migrationContext.UniqueKey); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return tx.Commit()
|
||||
return nil
|
||||
}
|
||||
|
||||
// CalculateNextIterationRangeEndValues reads the next-iteration-range-end unique key values,
|
||||
@ -583,13 +458,10 @@ func (this *Applier) CalculateNextIterationRangeEndValues() (hasFurtherRange boo
|
||||
if err != nil {
|
||||
return hasFurtherRange, err
|
||||
}
|
||||
|
||||
rows, err := this.db.Query(query, explodedArgs...)
|
||||
if err != nil {
|
||||
return hasFurtherRange, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
iterationRangeMaxValues := sql.NewColumnValues(this.migrationContext.UniqueKey.Len())
|
||||
for rows.Next() {
|
||||
if err = rows.Scan(iterationRangeMaxValues.ValuesPointers...); err != nil {
|
||||
@ -638,9 +510,12 @@ func (this *Applier) ApplyIterationInsertQuery() (chunkSize int64, rowsAffected
|
||||
return nil, err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
|
||||
sessionQuery := fmt.Sprintf(`SET SESSION time_zone = '%s'`, this.migrationContext.ApplierTimeZone)
|
||||
sessionQuery = fmt.Sprintf("%s, %s", sessionQuery, this.generateSqlModeQuery())
|
||||
sqlModeAddendum := `,NO_AUTO_VALUE_ON_ZERO`
|
||||
if !this.migrationContext.SkipStrictMode {
|
||||
sqlModeAddendum = fmt.Sprintf("%s,STRICT_ALL_TABLES", sqlModeAddendum)
|
||||
}
|
||||
sessionQuery = fmt.Sprintf("%s, sql_mode = CONCAT(@@session.sql_mode, ',%s')", sessionQuery, sqlModeAddendum)
|
||||
|
||||
if _, err := tx.Exec(sessionQuery); err != nil {
|
||||
return nil, err
|
||||
@ -934,7 +809,7 @@ func (this *Applier) CreateAtomicCutOverSentryTable() error {
|
||||
}
|
||||
|
||||
// AtomicCutOverMagicLock
|
||||
func (this *Applier) AtomicCutOverMagicLock(sessionIdChan chan int64, tableLocked chan<- error, okToUnlockTable <-chan bool, tableUnlocked chan<- error) error {
|
||||
func (this *Applier) AtomicCutOverMagicLock(sessionIdChan chan int64, tableLocked chan<- error, okToUnlockTable <-chan bool, tableUnlocked chan<- error, dropCutOverSentryTableOnce *sync.Once) error {
|
||||
tx, err := this.db.Begin()
|
||||
if err != nil {
|
||||
tableLocked <- err
|
||||
@ -945,7 +820,6 @@ func (this *Applier) AtomicCutOverMagicLock(sessionIdChan chan int64, tableLocke
|
||||
tableLocked <- fmt.Errorf("Unexpected error in AtomicCutOverMagicLock(), injected to release blocking channel reads")
|
||||
tableUnlocked <- fmt.Errorf("Unexpected error in AtomicCutOverMagicLock(), injected to release blocking channel reads")
|
||||
tx.Rollback()
|
||||
this.DropAtomicCutOverSentryTableIfExists()
|
||||
}()
|
||||
|
||||
var sessionId int64
|
||||
@ -1014,10 +888,12 @@ func (this *Applier) AtomicCutOverMagicLock(sessionIdChan chan int64, tableLocke
|
||||
sql.EscapeName(this.migrationContext.GetOldTableName()),
|
||||
)
|
||||
|
||||
if _, err := tx.Exec(query); err != nil {
|
||||
this.migrationContext.Log.Errore(err)
|
||||
// We DO NOT return here because we must `UNLOCK TABLES`!
|
||||
}
|
||||
dropCutOverSentryTableOnce.Do(func() {
|
||||
if _, err := tx.Exec(query); err != nil {
|
||||
this.migrationContext.Log.Errore(err)
|
||||
// We DO NOT return here because we must `UNLOCK TABLES`!
|
||||
}
|
||||
})
|
||||
|
||||
// Tables still locked
|
||||
this.migrationContext.Log.Infof("Releasing lock from %s.%s, %s.%s",
|
||||
@ -1137,6 +1013,7 @@ func (this *Applier) buildDMLEventQuery(dmlEvent *binlog.BinlogDMLEvent) (result
|
||||
|
||||
// ApplyDMLEventQueries applies multiple DML queries onto the _ghost_ table
|
||||
func (this *Applier) ApplyDMLEventQueries(dmlEvents [](*binlog.BinlogDMLEvent)) error {
|
||||
|
||||
var totalDelta int64
|
||||
|
||||
err := func() error {
|
||||
@ -1151,7 +1028,12 @@ func (this *Applier) ApplyDMLEventQueries(dmlEvents [](*binlog.BinlogDMLEvent))
|
||||
}
|
||||
|
||||
sessionQuery := "SET SESSION time_zone = '+00:00'"
|
||||
sessionQuery = fmt.Sprintf("%s, %s", sessionQuery, this.generateSqlModeQuery())
|
||||
|
||||
sqlModeAddendum := `,NO_AUTO_VALUE_ON_ZERO`
|
||||
if !this.migrationContext.SkipStrictMode {
|
||||
sqlModeAddendum = fmt.Sprintf("%s,STRICT_ALL_TABLES", sqlModeAddendum)
|
||||
}
|
||||
sessionQuery = fmt.Sprintf("%s, sql_mode = CONCAT(@@session.sql_mode, ',%s')", sessionQuery, sqlModeAddendum)
|
||||
|
||||
if _, err := tx.Exec(sessionQuery); err != nil {
|
||||
return rollback(err)
|
||||
@ -1161,20 +1043,11 @@ func (this *Applier) ApplyDMLEventQueries(dmlEvents [](*binlog.BinlogDMLEvent))
|
||||
if buildResult.err != nil {
|
||||
return rollback(buildResult.err)
|
||||
}
|
||||
result, err := tx.Exec(buildResult.query, buildResult.args...)
|
||||
if err != nil {
|
||||
err = fmt.Errorf("%w; query=%s; args=%+v", err, buildResult.query, buildResult.args)
|
||||
if _, err := tx.Exec(buildResult.query, buildResult.args...); err != nil {
|
||||
err = fmt.Errorf("%s; query=%s; args=%+v", err.Error(), buildResult.query, buildResult.args)
|
||||
return rollback(err)
|
||||
}
|
||||
|
||||
rowsAffected, err := result.RowsAffected()
|
||||
if err != nil {
|
||||
log.Warningf("error getting rows affected from DML event query: %s. i'm going to assume that the DML affected a single row, but this may result in inaccurate statistics", err)
|
||||
rowsAffected = 1
|
||||
}
|
||||
// each DML is either a single insert (delta +1), update (delta +0) or delete (delta -1).
|
||||
// multiplying by the rows actually affected (either 0 or 1) will give an accurate row delta for this DML event
|
||||
totalDelta += buildResult.rowsDelta * rowsAffected
|
||||
totalDelta += buildResult.rowsDelta
|
||||
}
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
|
@ -1,185 +0,0 @@
|
||||
/*
|
||||
Copyright 2022 GitHub Inc.
|
||||
See https://github.com/github/gh-ost/blob/master/LICENSE
|
||||
*/
|
||||
|
||||
package logic
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
test "github.com/openark/golib/tests"
|
||||
|
||||
"github.com/github/gh-ost/go/base"
|
||||
"github.com/github/gh-ost/go/binlog"
|
||||
"github.com/github/gh-ost/go/sql"
|
||||
)
|
||||
|
||||
func TestApplierGenerateSqlModeQuery(t *testing.T) {
|
||||
migrationContext := base.NewMigrationContext()
|
||||
applier := NewApplier(migrationContext)
|
||||
|
||||
{
|
||||
test.S(t).ExpectEquals(
|
||||
applier.generateSqlModeQuery(),
|
||||
`sql_mode = CONCAT(@@session.sql_mode, ',NO_AUTO_VALUE_ON_ZERO,STRICT_ALL_TABLES')`,
|
||||
)
|
||||
}
|
||||
{
|
||||
migrationContext.SkipStrictMode = true
|
||||
migrationContext.AllowZeroInDate = false
|
||||
test.S(t).ExpectEquals(
|
||||
applier.generateSqlModeQuery(),
|
||||
`sql_mode = CONCAT(@@session.sql_mode, ',NO_AUTO_VALUE_ON_ZERO')`,
|
||||
)
|
||||
}
|
||||
{
|
||||
migrationContext.SkipStrictMode = false
|
||||
migrationContext.AllowZeroInDate = true
|
||||
test.S(t).ExpectEquals(
|
||||
applier.generateSqlModeQuery(),
|
||||
`sql_mode = REPLACE(REPLACE(CONCAT(@@session.sql_mode, ',NO_AUTO_VALUE_ON_ZERO,STRICT_ALL_TABLES'), 'NO_ZERO_IN_DATE', ''), 'NO_ZERO_DATE', '')`,
|
||||
)
|
||||
}
|
||||
{
|
||||
migrationContext.SkipStrictMode = true
|
||||
migrationContext.AllowZeroInDate = true
|
||||
test.S(t).ExpectEquals(
|
||||
applier.generateSqlModeQuery(),
|
||||
`sql_mode = REPLACE(REPLACE(CONCAT(@@session.sql_mode, ',NO_AUTO_VALUE_ON_ZERO'), 'NO_ZERO_IN_DATE', ''), 'NO_ZERO_DATE', '')`,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplierUpdateModifiesUniqueKeyColumns(t *testing.T) {
|
||||
columns := sql.NewColumnList([]string{"id", "item_id"})
|
||||
columnValues := sql.ToColumnValues([]interface{}{123456, 42})
|
||||
|
||||
migrationContext := base.NewMigrationContext()
|
||||
migrationContext.OriginalTableColumns = columns
|
||||
migrationContext.UniqueKey = &sql.UniqueKey{
|
||||
Name: t.Name(),
|
||||
Columns: *columns,
|
||||
}
|
||||
|
||||
applier := NewApplier(migrationContext)
|
||||
|
||||
t.Run("unmodified", func(t *testing.T) {
|
||||
modifiedColumn, isModified := applier.updateModifiesUniqueKeyColumns(&binlog.BinlogDMLEvent{
|
||||
DatabaseName: "test",
|
||||
DML: binlog.UpdateDML,
|
||||
NewColumnValues: columnValues,
|
||||
WhereColumnValues: columnValues,
|
||||
})
|
||||
test.S(t).ExpectEquals(modifiedColumn, "")
|
||||
test.S(t).ExpectFalse(isModified)
|
||||
})
|
||||
|
||||
t.Run("modified", func(t *testing.T) {
|
||||
modifiedColumn, isModified := applier.updateModifiesUniqueKeyColumns(&binlog.BinlogDMLEvent{
|
||||
DatabaseName: "test",
|
||||
DML: binlog.UpdateDML,
|
||||
NewColumnValues: sql.ToColumnValues([]interface{}{123456, 24}),
|
||||
WhereColumnValues: columnValues,
|
||||
})
|
||||
test.S(t).ExpectEquals(modifiedColumn, "item_id")
|
||||
test.S(t).ExpectTrue(isModified)
|
||||
})
|
||||
}
|
||||
|
||||
func TestApplierBuildDMLEventQuery(t *testing.T) {
|
||||
columns := sql.NewColumnList([]string{"id", "item_id"})
|
||||
columnValues := sql.ToColumnValues([]interface{}{123456, 42})
|
||||
|
||||
migrationContext := base.NewMigrationContext()
|
||||
migrationContext.OriginalTableName = "test"
|
||||
migrationContext.OriginalTableColumns = columns
|
||||
migrationContext.SharedColumns = columns
|
||||
migrationContext.MappedSharedColumns = columns
|
||||
migrationContext.UniqueKey = &sql.UniqueKey{
|
||||
Name: t.Name(),
|
||||
Columns: *columns,
|
||||
}
|
||||
|
||||
applier := NewApplier(migrationContext)
|
||||
|
||||
t.Run("delete", func(t *testing.T) {
|
||||
binlogEvent := &binlog.BinlogDMLEvent{
|
||||
DatabaseName: "test",
|
||||
DML: binlog.DeleteDML,
|
||||
WhereColumnValues: columnValues,
|
||||
}
|
||||
|
||||
res := applier.buildDMLEventQuery(binlogEvent)
|
||||
test.S(t).ExpectEquals(len(res), 1)
|
||||
test.S(t).ExpectNil(res[0].err)
|
||||
test.S(t).ExpectEquals(strings.TrimSpace(res[0].query),
|
||||
`delete /* gh-ost `+"`test`.`_test_gho`"+` */
|
||||
from
|
||||
`+"`test`.`_test_gho`"+`
|
||||
where
|
||||
((`+"`id`"+` = ?) and (`+"`item_id`"+` = ?))`)
|
||||
|
||||
test.S(t).ExpectEquals(len(res[0].args), 2)
|
||||
test.S(t).ExpectEquals(res[0].args[0], 123456)
|
||||
test.S(t).ExpectEquals(res[0].args[1], 42)
|
||||
})
|
||||
|
||||
t.Run("insert", func(t *testing.T) {
|
||||
binlogEvent := &binlog.BinlogDMLEvent{
|
||||
DatabaseName: "test",
|
||||
DML: binlog.InsertDML,
|
||||
NewColumnValues: columnValues,
|
||||
}
|
||||
res := applier.buildDMLEventQuery(binlogEvent)
|
||||
test.S(t).ExpectEquals(len(res), 1)
|
||||
test.S(t).ExpectNil(res[0].err)
|
||||
test.S(t).ExpectEquals(strings.TrimSpace(res[0].query),
|
||||
`replace /* gh-ost `+"`test`.`_test_gho`"+` */ into
|
||||
`+"`test`.`_test_gho`"+`
|
||||
`+"(`id`, `item_id`)"+`
|
||||
values
|
||||
(?, ?)`)
|
||||
test.S(t).ExpectEquals(len(res[0].args), 2)
|
||||
test.S(t).ExpectEquals(res[0].args[0], 123456)
|
||||
test.S(t).ExpectEquals(res[0].args[1], 42)
|
||||
})
|
||||
|
||||
t.Run("update", func(t *testing.T) {
|
||||
binlogEvent := &binlog.BinlogDMLEvent{
|
||||
DatabaseName: "test",
|
||||
DML: binlog.UpdateDML,
|
||||
NewColumnValues: columnValues,
|
||||
WhereColumnValues: columnValues,
|
||||
}
|
||||
res := applier.buildDMLEventQuery(binlogEvent)
|
||||
test.S(t).ExpectEquals(len(res), 1)
|
||||
test.S(t).ExpectNil(res[0].err)
|
||||
test.S(t).ExpectEquals(strings.TrimSpace(res[0].query),
|
||||
`update /* gh-ost `+"`test`.`_test_gho`"+` */
|
||||
`+"`test`.`_test_gho`"+`
|
||||
set
|
||||
`+"`id`"+`=?, `+"`item_id`"+`=?
|
||||
where
|
||||
((`+"`id`"+` = ?) and (`+"`item_id`"+` = ?))`)
|
||||
test.S(t).ExpectEquals(len(res[0].args), 4)
|
||||
test.S(t).ExpectEquals(res[0].args[0], 123456)
|
||||
test.S(t).ExpectEquals(res[0].args[1], 42)
|
||||
test.S(t).ExpectEquals(res[0].args[2], 123456)
|
||||
test.S(t).ExpectEquals(res[0].args[3], 42)
|
||||
})
|
||||
}
|
||||
|
||||
func TestApplierInstantDDL(t *testing.T) {
|
||||
migrationContext := base.NewMigrationContext()
|
||||
migrationContext.DatabaseName = "test"
|
||||
migrationContext.OriginalTableName = "mytable"
|
||||
migrationContext.AlterStatementOptions = "ADD INDEX (foo)"
|
||||
applier := NewApplier(migrationContext)
|
||||
|
||||
t.Run("instantDDLstmt", func(t *testing.T) {
|
||||
stmt := applier.generateInstantDDLQuery()
|
||||
test.S(t).ExpectEquals(stmt, "ALTER /* gh-ost */ TABLE `test`.`mytable` ADD INDEX (foo), ALGORITHM=INSTANT")
|
||||
})
|
||||
}
|
@ -1,5 +1,6 @@
|
||||
/*
|
||||
Copyright 2022 GitHub Inc.
|
||||
/*
|
||||
Copyright 2016 GitHub Inc.
|
||||
See https://github.com/github/gh-ost/blob/master/LICENSE
|
||||
*/
|
||||
|
||||
@ -7,14 +8,13 @@ package logic
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/github/gh-ost/go/base"
|
||||
"github.com/openark/golib/log"
|
||||
"github.com/outbrain/golib/log"
|
||||
)
|
||||
|
||||
const (
|
||||
@ -35,16 +35,18 @@ const (
|
||||
|
||||
type HooksExecutor struct {
|
||||
migrationContext *base.MigrationContext
|
||||
writer io.Writer
|
||||
}
|
||||
|
||||
func NewHooksExecutor(migrationContext *base.MigrationContext) *HooksExecutor {
|
||||
return &HooksExecutor{
|
||||
migrationContext: migrationContext,
|
||||
writer: os.Stderr,
|
||||
}
|
||||
}
|
||||
|
||||
func (this *HooksExecutor) initHooks() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *HooksExecutor) applyEnvironmentVariables(extraVariables ...string) []string {
|
||||
env := os.Environ()
|
||||
env = append(env, fmt.Sprintf("GH_OST_DATABASE_NAME=%s", this.migrationContext.DatabaseName))
|
||||
@ -70,18 +72,20 @@ func (this *HooksExecutor) applyEnvironmentVariables(extraVariables ...string) [
|
||||
env = append(env, fmt.Sprintf("GH_OST_HOOKS_HINT_TOKEN=%s", this.migrationContext.HooksHintToken))
|
||||
env = append(env, fmt.Sprintf("GH_OST_DRY_RUN=%t", this.migrationContext.Noop))
|
||||
|
||||
env = append(env, extraVariables...)
|
||||
for _, variable := range extraVariables {
|
||||
env = append(env, variable)
|
||||
}
|
||||
return env
|
||||
}
|
||||
|
||||
// executeHook executes a command, and sets relevant environment variables
|
||||
// combined output & error are printed to the configured writer.
|
||||
// combined output & error are printed to gh-ost's standard error.
|
||||
func (this *HooksExecutor) executeHook(hook string, extraVariables ...string) error {
|
||||
cmd := exec.Command(hook)
|
||||
cmd.Env = this.applyEnvironmentVariables(extraVariables...)
|
||||
|
||||
combinedOutput, err := cmd.CombinedOutput()
|
||||
fmt.Fprintln(this.writer, string(combinedOutput))
|
||||
fmt.Fprintln(os.Stderr, string(combinedOutput))
|
||||
return log.Errore(err)
|
||||
}
|
||||
|
||||
|
@ -1,113 +0,0 @@
|
||||
/*
|
||||
Copyright 2022 GitHub Inc.
|
||||
See https://github.com/github/gh-ost/blob/master/LICENSE
|
||||
*/
|
||||
|
||||
package logic
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/openark/golib/tests"
|
||||
|
||||
"github.com/github/gh-ost/go/base"
|
||||
)
|
||||
|
||||
func TestHooksExecutorExecuteHooks(t *testing.T) {
|
||||
migrationContext := base.NewMigrationContext()
|
||||
migrationContext.AlterStatement = "ENGINE=InnoDB"
|
||||
migrationContext.DatabaseName = "test"
|
||||
migrationContext.Hostname = "test.example.com"
|
||||
migrationContext.OriginalTableName = "tablename"
|
||||
migrationContext.RowsDeltaEstimate = 1
|
||||
migrationContext.RowsEstimate = 122
|
||||
migrationContext.TotalRowsCopied = 123456
|
||||
migrationContext.SetETADuration(time.Minute)
|
||||
migrationContext.SetProgressPct(50)
|
||||
hooksExecutor := NewHooksExecutor(migrationContext)
|
||||
|
||||
writeTmpHookFunc := func(testName, hookName, script string) (path string, err error) {
|
||||
if path, err = os.MkdirTemp("", testName); err != nil {
|
||||
return path, err
|
||||
}
|
||||
err = os.WriteFile(filepath.Join(path, hookName), []byte(script), 0777)
|
||||
return path, err
|
||||
}
|
||||
|
||||
t.Run("does-not-exist", func(t *testing.T) {
|
||||
migrationContext.HooksPath = "/does/not/exist"
|
||||
tests.S(t).ExpectNil(hooksExecutor.executeHooks("test-hook"))
|
||||
})
|
||||
|
||||
t.Run("failed", func(t *testing.T) {
|
||||
var err error
|
||||
if migrationContext.HooksPath, err = writeTmpHookFunc(
|
||||
"TestHooksExecutorExecuteHooks-failed",
|
||||
"failed-hook",
|
||||
"#!/bin/sh\nexit 1",
|
||||
); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
defer os.RemoveAll(migrationContext.HooksPath)
|
||||
tests.S(t).ExpectNotNil(hooksExecutor.executeHooks("failed-hook"))
|
||||
})
|
||||
|
||||
t.Run("success", func(t *testing.T) {
|
||||
var err error
|
||||
if migrationContext.HooksPath, err = writeTmpHookFunc(
|
||||
"TestHooksExecutorExecuteHooks-success",
|
||||
"success-hook",
|
||||
"#!/bin/sh\nenv",
|
||||
); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
defer os.RemoveAll(migrationContext.HooksPath)
|
||||
|
||||
var buf bytes.Buffer
|
||||
hooksExecutor.writer = &buf
|
||||
tests.S(t).ExpectNil(hooksExecutor.executeHooks("success-hook", "TEST="+t.Name()))
|
||||
|
||||
scanner := bufio.NewScanner(&buf)
|
||||
for scanner.Scan() {
|
||||
split := strings.SplitN(scanner.Text(), "=", 2)
|
||||
switch split[0] {
|
||||
case "GH_OST_COPIED_ROWS":
|
||||
copiedRows, _ := strconv.ParseInt(split[1], 10, 64)
|
||||
tests.S(t).ExpectEquals(copiedRows, migrationContext.TotalRowsCopied)
|
||||
case "GH_OST_DATABASE_NAME":
|
||||
tests.S(t).ExpectEquals(split[1], migrationContext.DatabaseName)
|
||||
case "GH_OST_DDL":
|
||||
tests.S(t).ExpectEquals(split[1], migrationContext.AlterStatement)
|
||||
case "GH_OST_DRY_RUN":
|
||||
tests.S(t).ExpectEquals(split[1], "false")
|
||||
case "GH_OST_ESTIMATED_ROWS":
|
||||
estimatedRows, _ := strconv.ParseInt(split[1], 10, 64)
|
||||
tests.S(t).ExpectEquals(estimatedRows, int64(123))
|
||||
case "GH_OST_ETA_SECONDS":
|
||||
etaSeconds, _ := strconv.ParseInt(split[1], 10, 64)
|
||||
tests.S(t).ExpectEquals(etaSeconds, int64(60))
|
||||
case "GH_OST_EXECUTING_HOST":
|
||||
tests.S(t).ExpectEquals(split[1], migrationContext.Hostname)
|
||||
case "GH_OST_GHOST_TABLE_NAME":
|
||||
tests.S(t).ExpectEquals(split[1], fmt.Sprintf("_%s_gho", migrationContext.OriginalTableName))
|
||||
case "GH_OST_OLD_TABLE_NAME":
|
||||
tests.S(t).ExpectEquals(split[1], fmt.Sprintf("_%s_del", migrationContext.OriginalTableName))
|
||||
case "GH_OST_PROGRESS":
|
||||
progress, _ := strconv.ParseFloat(split[1], 64)
|
||||
tests.S(t).ExpectEquals(progress, 50.0)
|
||||
case "GH_OST_TABLE_NAME":
|
||||
tests.S(t).ExpectEquals(split[1], migrationContext.OriginalTableName)
|
||||
case "TEST":
|
||||
tests.S(t).ExpectEquals(split[1], t.Name())
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
@ -1,14 +1,12 @@
|
||||
/*
|
||||
Copyright 2022 GitHub Inc.
|
||||
Copyright 2016 GitHub Inc.
|
||||
See https://github.com/github/gh-ost/blob/master/LICENSE
|
||||
*/
|
||||
|
||||
package logic
|
||||
|
||||
import (
|
||||
"context"
|
||||
gosql "database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strings"
|
||||
@ -19,7 +17,7 @@ import (
|
||||
"github.com/github/gh-ost/go/mysql"
|
||||
"github.com/github/gh-ost/go/sql"
|
||||
|
||||
"github.com/openark/golib/sqlutils"
|
||||
"github.com/outbrain/golib/sqlutils"
|
||||
)
|
||||
|
||||
const startSlavePostWaitMilliseconds = 500 * time.Millisecond
|
||||
@ -133,7 +131,10 @@ func (this *Inspector) inspectOriginalAndGhostTables() (err error) {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
sharedUniqueKeys := this.getSharedUniqueKeys(this.migrationContext.OriginalTableUniqueKeys, this.migrationContext.GhostTableUniqueKeys)
|
||||
sharedUniqueKeys, err := this.getSharedUniqueKeys(this.migrationContext.OriginalTableUniqueKeys, this.migrationContext.GhostTableUniqueKeys)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for i, sharedUniqueKey := range sharedUniqueKeys {
|
||||
this.applyColumnTypes(this.migrationContext.DatabaseName, this.migrationContext.OriginalTableName, &sharedUniqueKey.Columns)
|
||||
uniqueKeyIsValid := true
|
||||
@ -190,9 +191,6 @@ func (this *Inspector) inspectOriginalAndGhostTables() (err error) {
|
||||
this.migrationContext.MappedSharedColumns.SetEnumToTextConversion(column.Name)
|
||||
this.migrationContext.MappedSharedColumns.SetEnumValues(column.Name, column.EnumValues)
|
||||
}
|
||||
if column.Name == mappedColumn.Name && column.Charset != mappedColumn.Charset {
|
||||
this.migrationContext.SharedColumns.SetCharsetConversion(column.Name, column.Charset, mappedColumn.Charset)
|
||||
}
|
||||
}
|
||||
|
||||
for _, column := range this.migrationContext.UniqueKey.Columns.Columns() {
|
||||
@ -285,7 +283,7 @@ func (this *Inspector) validateGrants() error {
|
||||
// It is entirely possible, for example, that the replication is using 'STATEMENT'
|
||||
// binlog format even as the variable says 'ROW'
|
||||
func (this *Inspector) restartReplication() error {
|
||||
this.migrationContext.Log.Infof("Restarting replication on %s to make sure binlog settings apply to replication thread", this.connectionConfig.Key.String())
|
||||
this.migrationContext.Log.Infof("Restarting replication on %s:%d to make sure binlog settings apply to replication thread", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port)
|
||||
|
||||
masterKey, _ := mysql.GetMasterKeyFromSlaveStatus(this.connectionConfig)
|
||||
if masterKey == nil {
|
||||
@ -344,13 +342,13 @@ func (this *Inspector) validateBinlogs() error {
|
||||
return err
|
||||
}
|
||||
if !hasBinaryLogs {
|
||||
return fmt.Errorf("%s must have binary logs enabled", this.connectionConfig.Key.String())
|
||||
return fmt.Errorf("%s:%d must have binary logs enabled", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port)
|
||||
}
|
||||
if this.migrationContext.RequiresBinlogFormatChange() {
|
||||
if !this.migrationContext.SwitchToRowBinlogFormat {
|
||||
return fmt.Errorf("You must be using ROW binlog format. I can switch it for you, provided --switch-to-rbr and that %s doesn't have replicas", this.connectionConfig.Key.String())
|
||||
return fmt.Errorf("You must be using ROW binlog format. I can switch it for you, provided --switch-to-rbr and that %s:%d doesn't have replicas", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port)
|
||||
}
|
||||
query := `show /* gh-ost */ slave hosts`
|
||||
query := fmt.Sprintf(`show /* gh-ost */ slave hosts`)
|
||||
countReplicas := 0
|
||||
err := sqlutils.QueryRowsMap(this.db, query, func(rowMap sqlutils.RowMap) error {
|
||||
countReplicas++
|
||||
@ -360,20 +358,21 @@ func (this *Inspector) validateBinlogs() error {
|
||||
return err
|
||||
}
|
||||
if countReplicas > 0 {
|
||||
return fmt.Errorf("%s has %s binlog_format, but I'm too scared to change it to ROW because it has replicas. Bailing out", this.connectionConfig.Key.String(), this.migrationContext.OriginalBinlogFormat)
|
||||
return fmt.Errorf("%s:%d has %s binlog_format, but I'm too scared to change it to ROW because it has replicas. Bailing out", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port, this.migrationContext.OriginalBinlogFormat)
|
||||
}
|
||||
this.migrationContext.Log.Infof("%s has %s binlog_format. I will change it to ROW, and will NOT change it back, even in the event of failure.", this.connectionConfig.Key.String(), this.migrationContext.OriginalBinlogFormat)
|
||||
this.migrationContext.Log.Infof("%s:%d has %s binlog_format. I will change it to ROW, and will NOT change it back, even in the event of failure.", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port, this.migrationContext.OriginalBinlogFormat)
|
||||
}
|
||||
query = `select @@global.binlog_row_image`
|
||||
if err := this.db.QueryRow(query).Scan(&this.migrationContext.OriginalBinlogRowImage); err != nil {
|
||||
return err
|
||||
// Only as of 5.6. We wish to support 5.5 as well
|
||||
this.migrationContext.OriginalBinlogRowImage = "FULL"
|
||||
}
|
||||
this.migrationContext.OriginalBinlogRowImage = strings.ToUpper(this.migrationContext.OriginalBinlogRowImage)
|
||||
if this.migrationContext.OriginalBinlogRowImage != "FULL" {
|
||||
return fmt.Errorf("%s has '%s' binlog_row_image, and only 'FULL' is supported. This operation cannot proceed. You may `set global binlog_row_image='full'` and try again", this.connectionConfig.Key.String(), this.migrationContext.OriginalBinlogRowImage)
|
||||
return fmt.Errorf("%s:%d has '%s' binlog_row_image, and only 'FULL' is supported. This operation cannot proceed. You may `set global binlog_row_image='full'` and try again", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port, this.migrationContext.OriginalBinlogRowImage)
|
||||
}
|
||||
|
||||
this.migrationContext.Log.Infof("binary logs validated on %s", this.connectionConfig.Key.String())
|
||||
this.migrationContext.Log.Infof("binary logs validated on %s:%d", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port)
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -386,25 +385,25 @@ func (this *Inspector) validateLogSlaveUpdates() error {
|
||||
}
|
||||
|
||||
if logSlaveUpdates {
|
||||
this.migrationContext.Log.Infof("log_slave_updates validated on %s", this.connectionConfig.Key.String())
|
||||
this.migrationContext.Log.Infof("log_slave_updates validated on %s:%d", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port)
|
||||
return nil
|
||||
}
|
||||
|
||||
if this.migrationContext.IsTungsten {
|
||||
this.migrationContext.Log.Warningf("log_slave_updates not found on %s, but --tungsten provided, so I'm proceeding", this.connectionConfig.Key.String())
|
||||
this.migrationContext.Log.Warningf("log_slave_updates not found on %s:%d, but --tungsten provided, so I'm proceeding", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port)
|
||||
return nil
|
||||
}
|
||||
|
||||
if this.migrationContext.TestOnReplica || this.migrationContext.MigrateOnReplica {
|
||||
return fmt.Errorf("%s must have log_slave_updates enabled for testing/migrating on replica", this.connectionConfig.Key.String())
|
||||
return fmt.Errorf("%s:%d must have log_slave_updates enabled for testing/migrating on replica", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port)
|
||||
}
|
||||
|
||||
if this.migrationContext.InspectorIsAlsoApplier() {
|
||||
this.migrationContext.Log.Warningf("log_slave_updates not found on %s, but executing directly on master, so I'm proceeding", this.connectionConfig.Key.String())
|
||||
this.migrationContext.Log.Warningf("log_slave_updates not found on %s:%d, but executing directly on master, so I'm proceeding", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port)
|
||||
return nil
|
||||
}
|
||||
|
||||
return fmt.Errorf("%s must have log_slave_updates enabled for executing migration", this.connectionConfig.Key.String())
|
||||
return fmt.Errorf("%s:%d must have log_slave_updates enabled for executing migration", this.connectionConfig.Key.Hostname, this.connectionConfig.Key.Port)
|
||||
}
|
||||
|
||||
// validateTable makes sure the table we need to operate on actually exists
|
||||
@ -535,37 +534,17 @@ func (this *Inspector) estimateTableRowsViaExplain() error {
|
||||
}
|
||||
|
||||
// CountTableRows counts exact number of rows on the original table
|
||||
func (this *Inspector) CountTableRows(ctx context.Context) error {
|
||||
func (this *Inspector) CountTableRows() error {
|
||||
atomic.StoreInt64(&this.migrationContext.CountingRowsFlag, 1)
|
||||
defer atomic.StoreInt64(&this.migrationContext.CountingRowsFlag, 0)
|
||||
|
||||
this.migrationContext.Log.Infof("As instructed, I'm issuing a SELECT COUNT(*) on the table. This may take a while")
|
||||
|
||||
conn, err := this.db.Conn(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer conn.Close()
|
||||
|
||||
var connectionID string
|
||||
if err := conn.QueryRowContext(ctx, `SELECT /* gh-ost */ CONNECTION_ID()`).Scan(&connectionID); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
query := fmt.Sprintf(`select /* gh-ost */ count(*) as count_rows from %s.%s`, sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName))
|
||||
var rowsEstimate int64
|
||||
if err := conn.QueryRowContext(ctx, query).Scan(&rowsEstimate); err != nil {
|
||||
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||
this.migrationContext.Log.Infof("exact row count cancelled (%s), likely because I'm about to cut over. I'm going to kill that query.", ctx.Err())
|
||||
return mysql.Kill(this.db, connectionID)
|
||||
}
|
||||
if err := this.db.QueryRow(query).Scan(&rowsEstimate); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// row count query finished. nil out the cancel func, so the main migration thread
|
||||
// doesn't bother calling it after row copy is done.
|
||||
this.migrationContext.SetCountTableRowsCancelFunc(nil)
|
||||
|
||||
atomic.StoreInt64(&this.migrationContext.RowsEstimate, rowsEstimate)
|
||||
this.migrationContext.UsedRowsEstimateMethod = base.CountRowsEstimate
|
||||
|
||||
@ -727,7 +706,7 @@ func (this *Inspector) getCandidateUniqueKeys(tableName string) (uniqueKeys [](*
|
||||
|
||||
// getSharedUniqueKeys returns the intersection of two given unique keys,
|
||||
// testing by list of columns
|
||||
func (this *Inspector) getSharedUniqueKeys(originalUniqueKeys, ghostUniqueKeys []*sql.UniqueKey) (uniqueKeys []*sql.UniqueKey) {
|
||||
func (this *Inspector) getSharedUniqueKeys(originalUniqueKeys, ghostUniqueKeys [](*sql.UniqueKey)) (uniqueKeys [](*sql.UniqueKey), err error) {
|
||||
// We actually do NOT rely on key name, just on the set of columns. This is because maybe
|
||||
// the ALTER is on the name itself...
|
||||
for _, originalUniqueKey := range originalUniqueKeys {
|
||||
@ -737,7 +716,7 @@ func (this *Inspector) getSharedUniqueKeys(originalUniqueKeys, ghostUniqueKeys [
|
||||
}
|
||||
}
|
||||
}
|
||||
return uniqueKeys
|
||||
return uniqueKeys, nil
|
||||
}
|
||||
|
||||
// getSharedColumns returns the intersection of two lists of columns in same order as the first list
|
||||
@ -826,4 +805,5 @@ func (this *Inspector) getReplicationLag() (replicationLag time.Duration, err er
|
||||
func (this *Inspector) Teardown() {
|
||||
this.db.Close()
|
||||
this.informationSchemaDb.Close()
|
||||
return
|
||||
}
|
||||
|
@ -1,31 +0,0 @@
|
||||
/*
|
||||
Copyright 2022 GitHub Inc.
|
||||
See https://github.com/github/gh-ost/blob/master/LICENSE
|
||||
*/
|
||||
|
||||
package logic
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
test "github.com/openark/golib/tests"
|
||||
|
||||
"github.com/github/gh-ost/go/sql"
|
||||
)
|
||||
|
||||
func TestInspectGetSharedUniqueKeys(t *testing.T) {
|
||||
origUniqKeys := []*sql.UniqueKey{
|
||||
{Columns: *sql.NewColumnList([]string{"id", "item_id"})},
|
||||
{Columns: *sql.NewColumnList([]string{"id", "org_id"})},
|
||||
}
|
||||
ghostUniqKeys := []*sql.UniqueKey{
|
||||
{Columns: *sql.NewColumnList([]string{"id", "item_id"})},
|
||||
{Columns: *sql.NewColumnList([]string{"id", "org_id"})},
|
||||
{Columns: *sql.NewColumnList([]string{"item_id", "user_id"})},
|
||||
}
|
||||
inspector := &Inspector{}
|
||||
sharedUniqKeys := inspector.getSharedUniqueKeys(origUniqKeys, ghostUniqKeys)
|
||||
test.S(t).ExpectEquals(len(sharedUniqKeys), 2)
|
||||
test.S(t).ExpectEquals(sharedUniqKeys[0].Columns.String(), "id,item_id")
|
||||
test.S(t).ExpectEquals(sharedUniqKeys[1].Columns.String(), "id,org_id")
|
||||
}
|
@ -1,18 +1,17 @@
|
||||
/*
|
||||
Copyright 2022 GitHub Inc.
|
||||
Copyright 2016 GitHub Inc.
|
||||
See https://github.com/github/gh-ost/blob/master/LICENSE
|
||||
*/
|
||||
|
||||
package logic
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
@ -22,17 +21,11 @@ import (
|
||||
"github.com/github/gh-ost/go/sql"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrMigratorUnsupportedRenameAlter = errors.New("ALTER statement seems to RENAME the table. This is not supported, and you should run your RENAME outside gh-ost.")
|
||||
)
|
||||
|
||||
type ChangelogState string
|
||||
|
||||
const (
|
||||
AllEventsUpToLockProcessed ChangelogState = "AllEventsUpToLockProcessed"
|
||||
GhostTableMigrated ChangelogState = "GhostTableMigrated"
|
||||
Migrated ChangelogState = "Migrated"
|
||||
ReadMigrationRangeValues ChangelogState = "ReadMigrationRangeValues"
|
||||
AllEventsUpToLockProcessed = "AllEventsUpToLockProcessed"
|
||||
)
|
||||
|
||||
func ReadChangelogState(s string) ChangelogState {
|
||||
@ -68,7 +61,6 @@ const (
|
||||
|
||||
// Migrator is the main schema migration flow manager.
|
||||
type Migrator struct {
|
||||
appVersion string
|
||||
parser *sql.AlterTableParser
|
||||
inspector *Inspector
|
||||
applier *Applier
|
||||
@ -94,10 +86,8 @@ type Migrator struct {
|
||||
finishedMigrating int64
|
||||
}
|
||||
|
||||
func NewMigrator(context *base.MigrationContext, appVersion string) *Migrator {
|
||||
func NewMigrator(context *base.MigrationContext) *Migrator {
|
||||
migrator := &Migrator{
|
||||
appVersion: appVersion,
|
||||
hooksExecutor: NewHooksExecutor(context),
|
||||
migrationContext: context,
|
||||
parser: sql.NewAlterTableParser(),
|
||||
ghostTableMigrated: make(chan bool),
|
||||
@ -113,6 +103,15 @@ func NewMigrator(context *base.MigrationContext, appVersion string) *Migrator {
|
||||
return migrator
|
||||
}
|
||||
|
||||
// initiateHooksExecutor
|
||||
func (this *Migrator) initiateHooksExecutor() (err error) {
|
||||
this.hooksExecutor = NewHooksExecutor(this.migrationContext)
|
||||
if err := this.hooksExecutor.initHooks(); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// sleepWhileTrue sleeps indefinitely until the given function returns 'false'
|
||||
// (or fails with error)
|
||||
func (this *Migrator) sleepWhileTrue(operation func() (bool, error)) error {
|
||||
@ -177,6 +176,16 @@ func (this *Migrator) retryOperationWithExponentialBackoff(operation func() erro
|
||||
return err
|
||||
}
|
||||
|
||||
// executeAndThrottleOnError executes a given function. If it errors, it
|
||||
// throttles.
|
||||
func (this *Migrator) executeAndThrottleOnError(operation func() error) (err error) {
|
||||
if err := operation(); err != nil {
|
||||
this.throttler.throttle(nil)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// consumeRowCopyComplete blocks on the rowCopyComplete channel once, and then
|
||||
// consumes and drops any further incoming events that may be left hanging.
|
||||
func (this *Migrator) consumeRowCopyComplete() {
|
||||
@ -216,25 +225,29 @@ func (this *Migrator) onChangelogStateEvent(dmlEvent *binlog.BinlogDMLEvent) (er
|
||||
changelogState := ReadChangelogState(changelogStateString)
|
||||
this.migrationContext.Log.Infof("Intercepted changelog state %s", changelogState)
|
||||
switch changelogState {
|
||||
case Migrated, ReadMigrationRangeValues:
|
||||
// no-op event
|
||||
case GhostTableMigrated:
|
||||
this.ghostTableMigrated <- true
|
||||
case AllEventsUpToLockProcessed:
|
||||
var applyEventFunc tableWriteFunc = func() error {
|
||||
this.allEventsUpToLockProcessed <- changelogStateString
|
||||
return nil
|
||||
{
|
||||
this.ghostTableMigrated <- true
|
||||
}
|
||||
case AllEventsUpToLockProcessed:
|
||||
{
|
||||
var applyEventFunc tableWriteFunc = func() error {
|
||||
this.allEventsUpToLockProcessed <- changelogStateString
|
||||
return nil
|
||||
}
|
||||
// at this point we know all events up to lock have been read from the streamer,
|
||||
// because the streamer works sequentially. So those events are either already handled,
|
||||
// or have event functions in applyEventsQueue.
|
||||
// So as not to create a potential deadlock, we write this func to applyEventsQueue
|
||||
// asynchronously, understanding it doesn't really matter.
|
||||
go func() {
|
||||
this.applyEventsQueue <- newApplyEventStructByFunc(&applyEventFunc)
|
||||
}()
|
||||
}
|
||||
// at this point we know all events up to lock have been read from the streamer,
|
||||
// because the streamer works sequentially. So those events are either already handled,
|
||||
// or have event functions in applyEventsQueue.
|
||||
// So as not to create a potential deadlock, we write this func to applyEventsQueue
|
||||
// asynchronously, understanding it doesn't really matter.
|
||||
go func() {
|
||||
this.applyEventsQueue <- newApplyEventStructByFunc(&applyEventFunc)
|
||||
}()
|
||||
default:
|
||||
return fmt.Errorf("Unknown changelog state: %+v", changelogState)
|
||||
{
|
||||
return fmt.Errorf("Unknown changelog state: %+v", changelogState)
|
||||
}
|
||||
}
|
||||
this.migrationContext.Log.Infof("Handled changelog state %s", changelogState)
|
||||
return nil
|
||||
@ -258,13 +271,13 @@ func (this *Migrator) listenOnPanicAbort() {
|
||||
this.migrationContext.Log.Fatale(err)
|
||||
}
|
||||
|
||||
// validateAlterStatement validates the `alter` statement meets criteria.
|
||||
// validateStatement validates the `alter` statement meets criteria.
|
||||
// At this time this means:
|
||||
// - column renames are approved
|
||||
// - no table rename allowed
|
||||
func (this *Migrator) validateAlterStatement() (err error) {
|
||||
func (this *Migrator) validateStatement() (err error) {
|
||||
if this.parser.IsRenameTable() {
|
||||
return ErrMigratorUnsupportedRenameAlter
|
||||
return fmt.Errorf("ALTER statement seems to RENAME the table. This is not supported, and you should run your RENAME outside gh-ost.")
|
||||
}
|
||||
if this.parser.HasNonTrivialRenames() && !this.migrationContext.SkipRenamedColumns {
|
||||
this.migrationContext.ColumnRenameMap = this.parser.GetNonTrivialRenames()
|
||||
@ -287,8 +300,8 @@ func (this *Migrator) countTableRows() (err error) {
|
||||
return nil
|
||||
}
|
||||
|
||||
countRowsFunc := func(ctx context.Context) error {
|
||||
if err := this.inspector.CountTableRows(ctx); err != nil {
|
||||
countRowsFunc := func() error {
|
||||
if err := this.inspector.CountTableRows(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := this.hooksExecutor.onRowCountComplete(); err != nil {
|
||||
@ -298,17 +311,12 @@ func (this *Migrator) countTableRows() (err error) {
|
||||
}
|
||||
|
||||
if this.migrationContext.ConcurrentCountTableRows {
|
||||
// store a cancel func so we can stop this query before a cut over
|
||||
rowCountContext, rowCountCancel := context.WithCancel(context.Background())
|
||||
this.migrationContext.SetCountTableRowsCancelFunc(rowCountCancel)
|
||||
|
||||
this.migrationContext.Log.Infof("As instructed, counting rows in the background; meanwhile I will use an estimated count, and will update it later on")
|
||||
go countRowsFunc(rowCountContext)
|
||||
|
||||
go countRowsFunc()
|
||||
// and we ignore errors, because this turns to be a background job
|
||||
return nil
|
||||
}
|
||||
return countRowsFunc(context.Background())
|
||||
return countRowsFunc()
|
||||
}
|
||||
|
||||
func (this *Migrator) createFlagFiles() (err error) {
|
||||
@ -333,13 +341,16 @@ func (this *Migrator) Migrate() (err error) {
|
||||
|
||||
go this.listenOnPanicAbort()
|
||||
|
||||
if err := this.initiateHooksExecutor(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := this.hooksExecutor.onStartup(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := this.parser.ParseAlterStatement(this.migrationContext.AlterStatement); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := this.validateAlterStatement(); err != nil {
|
||||
if err := this.validateStatement(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@ -359,17 +370,6 @@ func (this *Migrator) Migrate() (err error) {
|
||||
if err := this.createFlagFiles(); err != nil {
|
||||
return err
|
||||
}
|
||||
// In MySQL 8.0 (and possibly earlier) some DDL statements can be applied instantly.
|
||||
// Attempt to do this if AttemptInstantDDL is set.
|
||||
if this.migrationContext.AttemptInstantDDL {
|
||||
this.migrationContext.Log.Infof("Attempting to execute alter with ALGORITHM=INSTANT")
|
||||
if err := this.applier.AttemptInstantDDL(); err == nil {
|
||||
this.migrationContext.Log.Infof("Success! table %s.%s migrated instantly", sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName))
|
||||
return nil
|
||||
} else {
|
||||
this.migrationContext.Log.Infof("ALGORITHM=INSTANT not supported for this operation, proceeding with original algorithm: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
initialLag, _ := this.inspector.getReplicationLag()
|
||||
this.migrationContext.Log.Infof("Waiting for ghost table to be migrated. Current lag is %+v", initialLag)
|
||||
@ -401,9 +401,9 @@ func (this *Migrator) Migrate() (err error) {
|
||||
if err := this.applier.ReadMigrationRangeValues(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
this.initiateThrottler()
|
||||
|
||||
if err := this.initiateThrottler(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := this.hooksExecutor.onBeforeRowCopy(); err != nil {
|
||||
return err
|
||||
}
|
||||
@ -420,10 +420,6 @@ func (this *Migrator) Migrate() (err error) {
|
||||
}
|
||||
this.printStatus(ForcePrintStatusRule)
|
||||
|
||||
if this.migrationContext.IsCountingTableRows() {
|
||||
this.migrationContext.Log.Info("stopping query for exact row count, because that can accidentally lock out the cut over")
|
||||
this.migrationContext.CancelTableRowsCount()
|
||||
}
|
||||
if err := this.hooksExecutor.onBeforeCutOver(); err != nil {
|
||||
return err
|
||||
}
|
||||
@ -548,19 +544,19 @@ func (this *Migrator) cutOver() (err error) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
switch this.migrationContext.CutOverType {
|
||||
case base.CutOverAtomic:
|
||||
if this.migrationContext.CutOverType == base.CutOverAtomic {
|
||||
// Atomic solution: we use low timeout and multiple attempts. But for
|
||||
// each failed attempt, we throttle until replication lag is back to normal
|
||||
err = this.atomicCutOver()
|
||||
case base.CutOverTwoStep:
|
||||
err = this.cutOverTwoStep()
|
||||
default:
|
||||
return this.migrationContext.Log.Fatalf("Unknown cut-over type: %d; should never get here!", this.migrationContext.CutOverType)
|
||||
err := this.atomicCutOver()
|
||||
this.handleCutOverResult(err)
|
||||
return err
|
||||
}
|
||||
this.handleCutOverResult(err)
|
||||
return err
|
||||
if this.migrationContext.CutOverType == base.CutOverTwoStep {
|
||||
err := this.cutOverTwoStep()
|
||||
this.handleCutOverResult(err)
|
||||
return err
|
||||
}
|
||||
return this.migrationContext.Log.Fatalf("Unknown cut-over type: %d; should never get here!", this.migrationContext.CutOverType)
|
||||
}
|
||||
|
||||
// Inject the "AllEventsUpToLockProcessed" state hint, wait for it to appear in the binary logs,
|
||||
@ -638,8 +634,12 @@ func (this *Migrator) atomicCutOver() (err error) {
|
||||
defer atomic.StoreInt64(&this.migrationContext.InCutOverCriticalSectionFlag, 0)
|
||||
|
||||
okToUnlockTable := make(chan bool, 4)
|
||||
var dropCutOverSentryTableOnce sync.Once
|
||||
defer func() {
|
||||
okToUnlockTable <- true
|
||||
dropCutOverSentryTableOnce.Do(func() {
|
||||
this.applier.DropAtomicCutOverSentryTableIfExists()
|
||||
})
|
||||
}()
|
||||
|
||||
atomic.StoreInt64(&this.migrationContext.AllEventsUpToLockProcessedInjectedFlag, 0)
|
||||
@ -648,7 +648,7 @@ func (this *Migrator) atomicCutOver() (err error) {
|
||||
tableLocked := make(chan error, 2)
|
||||
tableUnlocked := make(chan error, 2)
|
||||
go func() {
|
||||
if err := this.applier.AtomicCutOverMagicLock(lockOriginalSessionIdChan, tableLocked, okToUnlockTable, tableUnlocked); err != nil {
|
||||
if err := this.applier.AtomicCutOverMagicLock(lockOriginalSessionIdChan, tableLocked, okToUnlockTable, tableUnlocked, &dropCutOverSentryTableOnce); err != nil {
|
||||
this.migrationContext.Log.Errore(err)
|
||||
}
|
||||
}()
|
||||
@ -804,16 +804,17 @@ func (this *Migrator) initiateInspector() (err error) {
|
||||
}
|
||||
|
||||
// initiateStatus sets and activates the printStatus() ticker
|
||||
func (this *Migrator) initiateStatus() {
|
||||
func (this *Migrator) initiateStatus() error {
|
||||
this.printStatus(ForcePrintStatusAndHintRule)
|
||||
ticker := time.NewTicker(time.Second)
|
||||
defer ticker.Stop()
|
||||
for range ticker.C {
|
||||
statusTick := time.Tick(1 * time.Second)
|
||||
for range statusTick {
|
||||
if atomic.LoadInt64(&this.finishedMigrating) > 0 {
|
||||
return
|
||||
return nil
|
||||
}
|
||||
go this.printStatus(HeuristicPrintStatusRule)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// printMigrationStatusHint prints a detailed configuration dump, that is useful
|
||||
@ -822,57 +823,57 @@ func (this *Migrator) initiateStatus() {
|
||||
// migration, and as response to the "status" interactive command.
|
||||
func (this *Migrator) printMigrationStatusHint(writers ...io.Writer) {
|
||||
w := io.MultiWriter(writers...)
|
||||
fmt.Fprintf(w, "# Migrating %s.%s; Ghost table is %s.%s\n",
|
||||
fmt.Fprintln(w, fmt.Sprintf("# Migrating %s.%s; Ghost table is %s.%s",
|
||||
sql.EscapeName(this.migrationContext.DatabaseName),
|
||||
sql.EscapeName(this.migrationContext.OriginalTableName),
|
||||
sql.EscapeName(this.migrationContext.DatabaseName),
|
||||
sql.EscapeName(this.migrationContext.GetGhostTableName()),
|
||||
)
|
||||
fmt.Fprintf(w, "# Migrating %+v; inspecting %+v; executing on %+v\n",
|
||||
))
|
||||
fmt.Fprintln(w, fmt.Sprintf("# Migrating %+v; inspecting %+v; executing on %+v",
|
||||
*this.applier.connectionConfig.ImpliedKey,
|
||||
*this.inspector.connectionConfig.ImpliedKey,
|
||||
this.migrationContext.Hostname,
|
||||
)
|
||||
fmt.Fprintf(w, "# Migration started at %+v\n",
|
||||
))
|
||||
fmt.Fprintln(w, fmt.Sprintf("# Migration started at %+v",
|
||||
this.migrationContext.StartTime.Format(time.RubyDate),
|
||||
)
|
||||
))
|
||||
maxLoad := this.migrationContext.GetMaxLoad()
|
||||
criticalLoad := this.migrationContext.GetCriticalLoad()
|
||||
fmt.Fprintf(w, "# chunk-size: %+v; max-lag-millis: %+vms; dml-batch-size: %+v; max-load: %s; critical-load: %s; nice-ratio: %f\n",
|
||||
fmt.Fprintln(w, fmt.Sprintf("# chunk-size: %+v; max-lag-millis: %+vms; dml-batch-size: %+v; max-load: %s; critical-load: %s; nice-ratio: %f",
|
||||
atomic.LoadInt64(&this.migrationContext.ChunkSize),
|
||||
atomic.LoadInt64(&this.migrationContext.MaxLagMillisecondsThrottleThreshold),
|
||||
atomic.LoadInt64(&this.migrationContext.DMLBatchSize),
|
||||
maxLoad.String(),
|
||||
criticalLoad.String(),
|
||||
this.migrationContext.GetNiceRatio(),
|
||||
)
|
||||
))
|
||||
if this.migrationContext.ThrottleFlagFile != "" {
|
||||
setIndicator := ""
|
||||
if base.FileExists(this.migrationContext.ThrottleFlagFile) {
|
||||
setIndicator = "[set]"
|
||||
}
|
||||
fmt.Fprintf(w, "# throttle-flag-file: %+v %+v\n",
|
||||
fmt.Fprintln(w, fmt.Sprintf("# throttle-flag-file: %+v %+v",
|
||||
this.migrationContext.ThrottleFlagFile, setIndicator,
|
||||
)
|
||||
))
|
||||
}
|
||||
if this.migrationContext.ThrottleAdditionalFlagFile != "" {
|
||||
setIndicator := ""
|
||||
if base.FileExists(this.migrationContext.ThrottleAdditionalFlagFile) {
|
||||
setIndicator = "[set]"
|
||||
}
|
||||
fmt.Fprintf(w, "# throttle-additional-flag-file: %+v %+v\n",
|
||||
fmt.Fprintln(w, fmt.Sprintf("# throttle-additional-flag-file: %+v %+v",
|
||||
this.migrationContext.ThrottleAdditionalFlagFile, setIndicator,
|
||||
)
|
||||
))
|
||||
}
|
||||
if throttleQuery := this.migrationContext.GetThrottleQuery(); throttleQuery != "" {
|
||||
fmt.Fprintf(w, "# throttle-query: %+v\n",
|
||||
fmt.Fprintln(w, fmt.Sprintf("# throttle-query: %+v",
|
||||
throttleQuery,
|
||||
)
|
||||
))
|
||||
}
|
||||
if throttleControlReplicaKeys := this.migrationContext.GetThrottleControlReplicaKeys(); throttleControlReplicaKeys.Len() > 0 {
|
||||
fmt.Fprintf(w, "# throttle-control-replicas count: %+v\n",
|
||||
fmt.Fprintln(w, fmt.Sprintf("# throttle-control-replicas count: %+v",
|
||||
throttleControlReplicaKeys.Len(),
|
||||
)
|
||||
))
|
||||
}
|
||||
|
||||
if this.migrationContext.PostponeCutOverFlagFile != "" {
|
||||
@ -880,111 +881,23 @@ func (this *Migrator) printMigrationStatusHint(writers ...io.Writer) {
|
||||
if base.FileExists(this.migrationContext.PostponeCutOverFlagFile) {
|
||||
setIndicator = "[set]"
|
||||
}
|
||||
fmt.Fprintf(w, "# postpone-cut-over-flag-file: %+v %+v\n",
|
||||
fmt.Fprintln(w, fmt.Sprintf("# postpone-cut-over-flag-file: %+v %+v",
|
||||
this.migrationContext.PostponeCutOverFlagFile, setIndicator,
|
||||
)
|
||||
))
|
||||
}
|
||||
if this.migrationContext.PanicFlagFile != "" {
|
||||
fmt.Fprintf(w, "# panic-flag-file: %+v\n",
|
||||
fmt.Fprintln(w, fmt.Sprintf("# panic-flag-file: %+v",
|
||||
this.migrationContext.PanicFlagFile,
|
||||
)
|
||||
))
|
||||
}
|
||||
fmt.Fprintf(w, "# Serving on unix socket: %+v\n",
|
||||
fmt.Fprintln(w, fmt.Sprintf("# Serving on unix socket: %+v",
|
||||
this.migrationContext.ServeSocketFile,
|
||||
)
|
||||
))
|
||||
if this.migrationContext.ServeTCPPort != 0 {
|
||||
fmt.Fprintf(w, "# Serving on TCP port: %+v\n", this.migrationContext.ServeTCPPort)
|
||||
fmt.Fprintln(w, fmt.Sprintf("# Serving on TCP port: %+v", this.migrationContext.ServeTCPPort))
|
||||
}
|
||||
}
|
||||
|
||||
// getProgressPercent returns an estimate of migration progess as a percent.
|
||||
func (this *Migrator) getProgressPercent(rowsEstimate int64) (progressPct float64) {
|
||||
progressPct = 100.0
|
||||
if rowsEstimate > 0 {
|
||||
progressPct *= float64(this.migrationContext.GetTotalRowsCopied()) / float64(rowsEstimate)
|
||||
}
|
||||
return progressPct
|
||||
}
|
||||
|
||||
// getMigrationETA returns the estimated duration of the migration
|
||||
func (this *Migrator) getMigrationETA(rowsEstimate int64) (eta string, duration time.Duration) {
|
||||
duration = time.Duration(base.ETAUnknown)
|
||||
progressPct := this.getProgressPercent(rowsEstimate)
|
||||
if progressPct >= 100.0 {
|
||||
duration = 0
|
||||
} else if progressPct >= 0.1 {
|
||||
totalRowsCopied := this.migrationContext.GetTotalRowsCopied()
|
||||
elapsedRowCopySeconds := this.migrationContext.ElapsedRowCopyTime().Seconds()
|
||||
totalExpectedSeconds := elapsedRowCopySeconds * float64(rowsEstimate) / float64(totalRowsCopied)
|
||||
etaSeconds := totalExpectedSeconds - elapsedRowCopySeconds
|
||||
if etaSeconds >= 0 {
|
||||
duration = time.Duration(etaSeconds) * time.Second
|
||||
} else {
|
||||
duration = 0
|
||||
}
|
||||
}
|
||||
|
||||
switch duration {
|
||||
case 0:
|
||||
eta = "due"
|
||||
case time.Duration(base.ETAUnknown):
|
||||
eta = "N/A"
|
||||
default:
|
||||
eta = base.PrettifyDurationOutput(duration)
|
||||
}
|
||||
|
||||
return eta, duration
|
||||
}
|
||||
|
||||
// getMigrationStateAndETA returns the state and eta of the migration.
|
||||
func (this *Migrator) getMigrationStateAndETA(rowsEstimate int64) (state, eta string, etaDuration time.Duration) {
|
||||
eta, etaDuration = this.getMigrationETA(rowsEstimate)
|
||||
state = "migrating"
|
||||
if atomic.LoadInt64(&this.migrationContext.CountingRowsFlag) > 0 && !this.migrationContext.ConcurrentCountTableRows {
|
||||
state = "counting rows"
|
||||
} else if atomic.LoadInt64(&this.migrationContext.IsPostponingCutOver) > 0 {
|
||||
eta = "due"
|
||||
state = "postponing cut-over"
|
||||
} else if isThrottled, throttleReason, _ := this.migrationContext.IsThrottled(); isThrottled {
|
||||
state = fmt.Sprintf("throttled, %s", throttleReason)
|
||||
}
|
||||
return state, eta, etaDuration
|
||||
}
|
||||
|
||||
// shouldPrintStatus returns true when the migrator is due to print status info.
|
||||
func (this *Migrator) shouldPrintStatus(rule PrintStatusRule, elapsedSeconds int64, etaDuration time.Duration) (shouldPrint bool) {
|
||||
if rule != HeuristicPrintStatusRule {
|
||||
return true
|
||||
}
|
||||
|
||||
etaSeconds := etaDuration.Seconds()
|
||||
if elapsedSeconds <= 60 {
|
||||
shouldPrint = true
|
||||
} else if etaSeconds <= 60 {
|
||||
shouldPrint = true
|
||||
} else if etaSeconds <= 180 {
|
||||
shouldPrint = (elapsedSeconds%5 == 0)
|
||||
} else if elapsedSeconds <= 180 {
|
||||
shouldPrint = (elapsedSeconds%5 == 0)
|
||||
} else if this.migrationContext.TimeSincePointOfInterest().Seconds() <= 60 {
|
||||
shouldPrint = (elapsedSeconds%5 == 0)
|
||||
} else {
|
||||
shouldPrint = (elapsedSeconds%30 == 0)
|
||||
}
|
||||
|
||||
return shouldPrint
|
||||
}
|
||||
|
||||
// shouldPrintMigrationStatus returns true when the migrator is due to print the migration status hint
|
||||
func (this *Migrator) shouldPrintMigrationStatusHint(rule PrintStatusRule, elapsedSeconds int64) (shouldPrint bool) {
|
||||
if elapsedSeconds%600 == 0 {
|
||||
shouldPrint = true
|
||||
} else if rule == ForcePrintStatusAndHintRule {
|
||||
shouldPrint = true
|
||||
}
|
||||
return shouldPrint
|
||||
}
|
||||
|
||||
// printStatus prints the progress status, and optionally additionally detailed
|
||||
// dump of configuration.
|
||||
// `rule` indicates the type of output expected.
|
||||
@ -1005,21 +918,81 @@ func (this *Migrator) printStatus(rule PrintStatusRule, writers ...io.Writer) {
|
||||
// and there is no further need to keep updating the value.
|
||||
rowsEstimate = totalRowsCopied
|
||||
}
|
||||
|
||||
var progressPct float64
|
||||
if rowsEstimate == 0 {
|
||||
progressPct = 100.0
|
||||
} else {
|
||||
progressPct = 100.0 * float64(totalRowsCopied) / float64(rowsEstimate)
|
||||
}
|
||||
// we take the opportunity to update migration context with progressPct
|
||||
progressPct := this.getProgressPercent(rowsEstimate)
|
||||
this.migrationContext.SetProgressPct(progressPct)
|
||||
|
||||
// Before status, let's see if we should print a nice reminder for what exactly we're doing here.
|
||||
if this.shouldPrintMigrationStatusHint(rule, elapsedSeconds) {
|
||||
shouldPrintMigrationStatusHint := (elapsedSeconds%600 == 0)
|
||||
if rule == ForcePrintStatusAndHintRule {
|
||||
shouldPrintMigrationStatusHint = true
|
||||
}
|
||||
if rule == ForcePrintStatusOnlyRule {
|
||||
shouldPrintMigrationStatusHint = false
|
||||
}
|
||||
if shouldPrintMigrationStatusHint {
|
||||
this.printMigrationStatusHint(writers...)
|
||||
}
|
||||
|
||||
// Get state + ETA
|
||||
state, eta, etaDuration := this.getMigrationStateAndETA(rowsEstimate)
|
||||
var etaSeconds float64 = math.MaxFloat64
|
||||
var etaDuration = time.Duration(base.ETAUnknown)
|
||||
if progressPct >= 100.0 {
|
||||
etaDuration = 0
|
||||
} else if progressPct >= 0.1 {
|
||||
elapsedRowCopySeconds := this.migrationContext.ElapsedRowCopyTime().Seconds()
|
||||
totalExpectedSeconds := elapsedRowCopySeconds * float64(rowsEstimate) / float64(totalRowsCopied)
|
||||
etaSeconds = totalExpectedSeconds - elapsedRowCopySeconds
|
||||
if etaSeconds >= 0 {
|
||||
etaDuration = time.Duration(etaSeconds) * time.Second
|
||||
} else {
|
||||
etaDuration = 0
|
||||
}
|
||||
}
|
||||
this.migrationContext.SetETADuration(etaDuration)
|
||||
var eta string
|
||||
switch etaDuration {
|
||||
case 0:
|
||||
eta = "due"
|
||||
case time.Duration(base.ETAUnknown):
|
||||
eta = "N/A"
|
||||
default:
|
||||
eta = base.PrettifyDurationOutput(etaDuration)
|
||||
}
|
||||
|
||||
if !this.shouldPrintStatus(rule, elapsedSeconds, etaDuration) {
|
||||
state := "migrating"
|
||||
if atomic.LoadInt64(&this.migrationContext.CountingRowsFlag) > 0 && !this.migrationContext.ConcurrentCountTableRows {
|
||||
state = "counting rows"
|
||||
} else if atomic.LoadInt64(&this.migrationContext.IsPostponingCutOver) > 0 {
|
||||
eta = "due"
|
||||
state = "postponing cut-over"
|
||||
} else if isThrottled, throttleReason, _ := this.migrationContext.IsThrottled(); isThrottled {
|
||||
state = fmt.Sprintf("throttled, %s", throttleReason)
|
||||
}
|
||||
|
||||
shouldPrintStatus := false
|
||||
if rule == HeuristicPrintStatusRule {
|
||||
if elapsedSeconds <= 60 {
|
||||
shouldPrintStatus = true
|
||||
} else if etaSeconds <= 60 {
|
||||
shouldPrintStatus = true
|
||||
} else if etaSeconds <= 180 {
|
||||
shouldPrintStatus = (elapsedSeconds%5 == 0)
|
||||
} else if elapsedSeconds <= 180 {
|
||||
shouldPrintStatus = (elapsedSeconds%5 == 0)
|
||||
} else if this.migrationContext.TimeSincePointOfInterest().Seconds() <= 60 {
|
||||
shouldPrintStatus = (elapsedSeconds%5 == 0)
|
||||
} else {
|
||||
shouldPrintStatus = (elapsedSeconds%30 == 0)
|
||||
}
|
||||
} else {
|
||||
// Not heuristic
|
||||
shouldPrintStatus = true
|
||||
}
|
||||
if !shouldPrintStatus {
|
||||
return
|
||||
}
|
||||
|
||||
@ -1038,14 +1011,12 @@ func (this *Migrator) printStatus(rule PrintStatusRule, writers ...io.Writer) {
|
||||
)
|
||||
this.applier.WriteChangelog(
|
||||
fmt.Sprintf("copy iteration %d at %d", this.migrationContext.GetIteration(), time.Now().Unix()),
|
||||
state,
|
||||
status,
|
||||
)
|
||||
w := io.MultiWriter(writers...)
|
||||
fmt.Fprintln(w, status)
|
||||
this.migrationContext.Log.Infof(status)
|
||||
|
||||
hooksStatusIntervalSec := this.migrationContext.HooksStatusIntervalSec
|
||||
if hooksStatusIntervalSec > 0 && elapsedSeconds%hooksStatusIntervalSec == 0 {
|
||||
if elapsedSeconds%this.migrationContext.HooksStatusIntervalSec == 0 {
|
||||
this.hooksExecutor.onStatus(status)
|
||||
}
|
||||
}
|
||||
@ -1075,9 +1046,8 @@ func (this *Migrator) initiateStreaming() error {
|
||||
}()
|
||||
|
||||
go func() {
|
||||
ticker := time.NewTicker(time.Second)
|
||||
defer ticker.Stop()
|
||||
for range ticker.C {
|
||||
ticker := time.Tick(1 * time.Second)
|
||||
for range ticker {
|
||||
if atomic.LoadInt64(&this.finishedMigrating) > 0 {
|
||||
return
|
||||
}
|
||||
@ -1103,8 +1073,8 @@ func (this *Migrator) addDMLEventsListener() error {
|
||||
}
|
||||
|
||||
// initiateThrottler kicks in the throttling collection and the throttling checks.
|
||||
func (this *Migrator) initiateThrottler() {
|
||||
this.throttler = NewThrottler(this.migrationContext, this.applier, this.inspector, this.appVersion)
|
||||
func (this *Migrator) initiateThrottler() error {
|
||||
this.throttler = NewThrottler(this.migrationContext, this.applier, this.inspector)
|
||||
|
||||
go this.throttler.initiateThrottlerCollection(this.firstThrottlingCollected)
|
||||
this.migrationContext.Log.Infof("Waiting for first throttle metrics to be collected")
|
||||
@ -1113,6 +1083,8 @@ func (this *Migrator) initiateThrottler() {
|
||||
<-this.firstThrottlingCollected // other, general metrics
|
||||
this.migrationContext.Log.Infof("First throttle metrics collected")
|
||||
go this.throttler.initiateThrottlerChecks()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *Migrator) initiateApplier() error {
|
||||
@ -1223,6 +1195,7 @@ func (this *Migrator) iterateChunks() error {
|
||||
// Enqueue copy operation; to be executed by executeWriteFuncs()
|
||||
this.copyRowsQueue <- copyRowsFunc
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *Migrator) onApplyEventStruct(eventStruct *applyEventStruct) error {
|
||||
@ -1313,7 +1286,7 @@ func (this *Migrator) executeWriteFuncs() error {
|
||||
if niceRatio := this.migrationContext.GetNiceRatio(); niceRatio > 0 {
|
||||
copyRowsDuration := time.Since(copyRowsStartTime)
|
||||
sleepTimeNanosecondFloat64 := niceRatio * float64(copyRowsDuration.Nanoseconds())
|
||||
sleepTime := time.Duration(int64(sleepTimeNanosecondFloat64)) * time.Nanosecond
|
||||
sleepTime := time.Duration(time.Duration(int64(sleepTimeNanosecondFloat64)) * time.Nanosecond)
|
||||
time.Sleep(sleepTime)
|
||||
}
|
||||
}
|
||||
@ -1328,17 +1301,13 @@ func (this *Migrator) executeWriteFuncs() error {
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// finalCleanup takes actions at very end of migration, dropping tables etc.
|
||||
func (this *Migrator) finalCleanup() error {
|
||||
atomic.StoreInt64(&this.migrationContext.CleanupImminentFlag, 1)
|
||||
|
||||
this.migrationContext.Log.Infof("Writing changelog state: %+v", Migrated)
|
||||
if _, err := this.applier.WriteChangelogState(string(Migrated)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if this.migrationContext.Noop {
|
||||
if createTableStatement, err := this.inspector.showCreateTable(this.migrationContext.GetGhostTableName()); err == nil {
|
||||
this.migrationContext.Log.Infof("New table structure follows")
|
||||
|
@ -1,256 +0,0 @@
|
||||
/*
|
||||
Copyright 2022 GitHub Inc.
|
||||
See https://github.com/github/gh-ost/blob/master/LICENSE
|
||||
*/
|
||||
|
||||
package logic
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/openark/golib/tests"
|
||||
|
||||
"github.com/github/gh-ost/go/base"
|
||||
"github.com/github/gh-ost/go/binlog"
|
||||
"github.com/github/gh-ost/go/sql"
|
||||
)
|
||||
|
||||
func TestMigratorOnChangelogEvent(t *testing.T) {
|
||||
migrationContext := base.NewMigrationContext()
|
||||
migrator := NewMigrator(migrationContext, "1.2.3")
|
||||
|
||||
t.Run("heartbeat", func(t *testing.T) {
|
||||
columnValues := sql.ToColumnValues([]interface{}{
|
||||
123,
|
||||
time.Now().Unix(),
|
||||
"heartbeat",
|
||||
"2022-08-16T00:45:10.52Z",
|
||||
})
|
||||
tests.S(t).ExpectNil(migrator.onChangelogEvent(&binlog.BinlogDMLEvent{
|
||||
DatabaseName: "test",
|
||||
DML: binlog.InsertDML,
|
||||
NewColumnValues: columnValues,
|
||||
}))
|
||||
})
|
||||
|
||||
t.Run("state-AllEventsUpToLockProcessed", func(t *testing.T) {
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(1)
|
||||
go func(wg *sync.WaitGroup) {
|
||||
defer wg.Done()
|
||||
es := <-migrator.applyEventsQueue
|
||||
tests.S(t).ExpectNotNil(es)
|
||||
tests.S(t).ExpectNotNil(es.writeFunc)
|
||||
}(&wg)
|
||||
|
||||
columnValues := sql.ToColumnValues([]interface{}{
|
||||
123,
|
||||
time.Now().Unix(),
|
||||
"state",
|
||||
AllEventsUpToLockProcessed,
|
||||
})
|
||||
tests.S(t).ExpectNil(migrator.onChangelogEvent(&binlog.BinlogDMLEvent{
|
||||
DatabaseName: "test",
|
||||
DML: binlog.InsertDML,
|
||||
NewColumnValues: columnValues,
|
||||
}))
|
||||
wg.Wait()
|
||||
})
|
||||
|
||||
t.Run("state-GhostTableMigrated", func(t *testing.T) {
|
||||
go func() {
|
||||
tests.S(t).ExpectTrue(<-migrator.ghostTableMigrated)
|
||||
}()
|
||||
|
||||
columnValues := sql.ToColumnValues([]interface{}{
|
||||
123,
|
||||
time.Now().Unix(),
|
||||
"state",
|
||||
GhostTableMigrated,
|
||||
})
|
||||
tests.S(t).ExpectNil(migrator.onChangelogEvent(&binlog.BinlogDMLEvent{
|
||||
DatabaseName: "test",
|
||||
DML: binlog.InsertDML,
|
||||
NewColumnValues: columnValues,
|
||||
}))
|
||||
})
|
||||
|
||||
t.Run("state-Migrated", func(t *testing.T) {
|
||||
columnValues := sql.ToColumnValues([]interface{}{
|
||||
123,
|
||||
time.Now().Unix(),
|
||||
"state",
|
||||
Migrated,
|
||||
})
|
||||
tests.S(t).ExpectNil(migrator.onChangelogEvent(&binlog.BinlogDMLEvent{
|
||||
DatabaseName: "test",
|
||||
DML: binlog.InsertDML,
|
||||
NewColumnValues: columnValues,
|
||||
}))
|
||||
})
|
||||
|
||||
t.Run("state-ReadMigrationRangeValues", func(t *testing.T) {
|
||||
columnValues := sql.ToColumnValues([]interface{}{
|
||||
123,
|
||||
time.Now().Unix(),
|
||||
"state",
|
||||
ReadMigrationRangeValues,
|
||||
})
|
||||
tests.S(t).ExpectNil(migrator.onChangelogEvent(&binlog.BinlogDMLEvent{
|
||||
DatabaseName: "test",
|
||||
DML: binlog.InsertDML,
|
||||
NewColumnValues: columnValues,
|
||||
}))
|
||||
})
|
||||
}
|
||||
|
||||
func TestMigratorValidateStatement(t *testing.T) {
|
||||
t.Run("add-column", func(t *testing.T) {
|
||||
migrationContext := base.NewMigrationContext()
|
||||
migrator := NewMigrator(migrationContext, "1.2.3")
|
||||
tests.S(t).ExpectNil(migrator.parser.ParseAlterStatement(`ALTER TABLE test ADD test_new VARCHAR(64) NOT NULL`))
|
||||
|
||||
tests.S(t).ExpectNil(migrator.validateAlterStatement())
|
||||
tests.S(t).ExpectEquals(len(migrator.migrationContext.DroppedColumnsMap), 0)
|
||||
})
|
||||
|
||||
t.Run("drop-column", func(t *testing.T) {
|
||||
migrationContext := base.NewMigrationContext()
|
||||
migrator := NewMigrator(migrationContext, "1.2.3")
|
||||
tests.S(t).ExpectNil(migrator.parser.ParseAlterStatement(`ALTER TABLE test DROP abc`))
|
||||
|
||||
tests.S(t).ExpectNil(migrator.validateAlterStatement())
|
||||
tests.S(t).ExpectEquals(len(migrator.migrationContext.DroppedColumnsMap), 1)
|
||||
_, exists := migrator.migrationContext.DroppedColumnsMap["abc"]
|
||||
tests.S(t).ExpectTrue(exists)
|
||||
})
|
||||
|
||||
t.Run("rename-column", func(t *testing.T) {
|
||||
migrationContext := base.NewMigrationContext()
|
||||
migrator := NewMigrator(migrationContext, "1.2.3")
|
||||
tests.S(t).ExpectNil(migrator.parser.ParseAlterStatement(`ALTER TABLE test CHANGE test123 test1234 bigint unsigned`))
|
||||
|
||||
err := migrator.validateAlterStatement()
|
||||
tests.S(t).ExpectNotNil(err)
|
||||
tests.S(t).ExpectTrue(strings.HasPrefix(err.Error(), "gh-ost believes the ALTER statement renames columns"))
|
||||
tests.S(t).ExpectEquals(len(migrator.migrationContext.DroppedColumnsMap), 0)
|
||||
})
|
||||
|
||||
t.Run("rename-column-approved", func(t *testing.T) {
|
||||
migrationContext := base.NewMigrationContext()
|
||||
migrator := NewMigrator(migrationContext, "1.2.3")
|
||||
migrator.migrationContext.ApproveRenamedColumns = true
|
||||
tests.S(t).ExpectNil(migrator.parser.ParseAlterStatement(`ALTER TABLE test CHANGE test123 test1234 bigint unsigned`))
|
||||
|
||||
tests.S(t).ExpectNil(migrator.validateAlterStatement())
|
||||
tests.S(t).ExpectEquals(len(migrator.migrationContext.DroppedColumnsMap), 0)
|
||||
})
|
||||
|
||||
t.Run("rename-table", func(t *testing.T) {
|
||||
migrationContext := base.NewMigrationContext()
|
||||
migrator := NewMigrator(migrationContext, "1.2.3")
|
||||
tests.S(t).ExpectNil(migrator.parser.ParseAlterStatement(`ALTER TABLE test RENAME TO test_new`))
|
||||
|
||||
err := migrator.validateAlterStatement()
|
||||
tests.S(t).ExpectNotNil(err)
|
||||
tests.S(t).ExpectTrue(errors.Is(err, ErrMigratorUnsupportedRenameAlter))
|
||||
tests.S(t).ExpectEquals(len(migrator.migrationContext.DroppedColumnsMap), 0)
|
||||
})
|
||||
}
|
||||
|
||||
func TestMigratorCreateFlagFiles(t *testing.T) {
|
||||
tmpdir, err := os.MkdirTemp("", t.Name())
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
defer os.RemoveAll(tmpdir)
|
||||
|
||||
migrationContext := base.NewMigrationContext()
|
||||
migrationContext.PostponeCutOverFlagFile = filepath.Join(tmpdir, "cut-over.flag")
|
||||
migrator := NewMigrator(migrationContext, "1.2.3")
|
||||
tests.S(t).ExpectNil(migrator.createFlagFiles())
|
||||
tests.S(t).ExpectNil(migrator.createFlagFiles()) // twice to test already-exists
|
||||
|
||||
_, err = os.Stat(migrationContext.PostponeCutOverFlagFile)
|
||||
tests.S(t).ExpectNil(err)
|
||||
}
|
||||
|
||||
func TestMigratorGetProgressPercent(t *testing.T) {
|
||||
migrationContext := base.NewMigrationContext()
|
||||
migrator := NewMigrator(migrationContext, "1.2.3")
|
||||
|
||||
{
|
||||
tests.S(t).ExpectEquals(migrator.getProgressPercent(0), float64(100.0))
|
||||
}
|
||||
{
|
||||
migrationContext.TotalRowsCopied = 250
|
||||
tests.S(t).ExpectEquals(migrator.getProgressPercent(1000), float64(25.0))
|
||||
}
|
||||
}
|
||||
|
||||
func TestMigratorGetMigrationStateAndETA(t *testing.T) {
|
||||
migrationContext := base.NewMigrationContext()
|
||||
migrator := NewMigrator(migrationContext, "1.2.3")
|
||||
now := time.Now()
|
||||
migrationContext.RowCopyStartTime = now.Add(-time.Minute)
|
||||
migrationContext.RowCopyEndTime = now
|
||||
|
||||
{
|
||||
migrationContext.TotalRowsCopied = 456
|
||||
state, eta, etaDuration := migrator.getMigrationStateAndETA(123456)
|
||||
tests.S(t).ExpectEquals(state, "migrating")
|
||||
tests.S(t).ExpectEquals(eta, "4h29m44s")
|
||||
tests.S(t).ExpectEquals(etaDuration.String(), "4h29m44s")
|
||||
}
|
||||
{
|
||||
migrationContext.TotalRowsCopied = 456
|
||||
state, eta, etaDuration := migrator.getMigrationStateAndETA(456)
|
||||
tests.S(t).ExpectEquals(state, "migrating")
|
||||
tests.S(t).ExpectEquals(eta, "due")
|
||||
tests.S(t).ExpectEquals(etaDuration.String(), "0s")
|
||||
}
|
||||
{
|
||||
migrationContext.TotalRowsCopied = 123456
|
||||
state, eta, etaDuration := migrator.getMigrationStateAndETA(456)
|
||||
tests.S(t).ExpectEquals(state, "migrating")
|
||||
tests.S(t).ExpectEquals(eta, "due")
|
||||
tests.S(t).ExpectEquals(etaDuration.String(), "0s")
|
||||
}
|
||||
{
|
||||
atomic.StoreInt64(&migrationContext.CountingRowsFlag, 1)
|
||||
state, eta, etaDuration := migrator.getMigrationStateAndETA(123456)
|
||||
tests.S(t).ExpectEquals(state, "counting rows")
|
||||
tests.S(t).ExpectEquals(eta, "due")
|
||||
tests.S(t).ExpectEquals(etaDuration.String(), "0s")
|
||||
}
|
||||
{
|
||||
atomic.StoreInt64(&migrationContext.CountingRowsFlag, 0)
|
||||
atomic.StoreInt64(&migrationContext.IsPostponingCutOver, 1)
|
||||
state, eta, etaDuration := migrator.getMigrationStateAndETA(123456)
|
||||
tests.S(t).ExpectEquals(state, "postponing cut-over")
|
||||
tests.S(t).ExpectEquals(eta, "due")
|
||||
tests.S(t).ExpectEquals(etaDuration.String(), "0s")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMigratorShouldPrintStatus(t *testing.T) {
|
||||
migrationContext := base.NewMigrationContext()
|
||||
migrator := NewMigrator(migrationContext, "1.2.3")
|
||||
|
||||
tests.S(t).ExpectTrue(migrator.shouldPrintStatus(NoPrintStatusRule, 10, time.Second)) // test 'rule != HeuristicPrintStatusRule' return
|
||||
tests.S(t).ExpectTrue(migrator.shouldPrintStatus(HeuristicPrintStatusRule, 10, time.Second)) // test 'etaDuration.Seconds() <= 60'
|
||||
tests.S(t).ExpectTrue(migrator.shouldPrintStatus(HeuristicPrintStatusRule, 90, time.Second)) // test 'etaDuration.Seconds() <= 60' again
|
||||
tests.S(t).ExpectTrue(migrator.shouldPrintStatus(HeuristicPrintStatusRule, 90, time.Minute)) // test 'etaDuration.Seconds() <= 180'
|
||||
tests.S(t).ExpectTrue(migrator.shouldPrintStatus(HeuristicPrintStatusRule, 60, 90*time.Second)) // test 'elapsedSeconds <= 180'
|
||||
tests.S(t).ExpectFalse(migrator.shouldPrintStatus(HeuristicPrintStatusRule, 61, 90*time.Second)) // test 'elapsedSeconds <= 180'
|
||||
tests.S(t).ExpectFalse(migrator.shouldPrintStatus(HeuristicPrintStatusRule, 99, 210*time.Second)) // test 'elapsedSeconds <= 180'
|
||||
tests.S(t).ExpectFalse(migrator.shouldPrintStatus(HeuristicPrintStatusRule, 12345, 86400*time.Second)) // test 'else'
|
||||
tests.S(t).ExpectTrue(migrator.shouldPrintStatus(HeuristicPrintStatusRule, 30030, 86400*time.Second)) // test 'else' again
|
||||
}
|
@ -1,5 +1,5 @@
|
||||
/*
|
||||
Copyright 2022 GitHub Inc.
|
||||
Copyright 2021 GitHub Inc.
|
||||
See https://github.com/github/gh-ost/blob/master/LICENSE
|
||||
*/
|
||||
|
||||
@ -122,6 +122,8 @@ func (this *Server) onServerCommand(command string, writer *bufio.Writer) (err e
|
||||
|
||||
// applyServerCommand parses and executes commands by user
|
||||
func (this *Server) applyServerCommand(command string, writer *bufio.Writer) (printStatusRule PrintStatusRule, err error) {
|
||||
printStatusRule = NoPrintStatusRule
|
||||
|
||||
tokens := strings.SplitN(command, "=", 2)
|
||||
command = strings.TrimSpace(tokens[0])
|
||||
arg := ""
|
||||
@ -132,7 +134,7 @@ func (this *Server) applyServerCommand(command string, writer *bufio.Writer) (pr
|
||||
}
|
||||
}
|
||||
argIsQuestion := (arg == "?")
|
||||
throttleHint := "# Note: you may only throttle for as long as your binary logs are not purged"
|
||||
throttleHint := "# Note: you may only throttle for as long as your binary logs are not purged\n"
|
||||
|
||||
if err := this.hooksExecutor.onInteractiveCommand(command); err != nil {
|
||||
return NoPrintStatusRule, err
|
||||
@ -280,7 +282,7 @@ help # This message
|
||||
return NoPrintStatusRule, nil
|
||||
}
|
||||
this.migrationContext.SetThrottleQuery(arg)
|
||||
fmt.Fprintln(writer, throttleHint)
|
||||
fmt.Fprintf(writer, throttleHint)
|
||||
return ForcePrintStatusAndHintRule, nil
|
||||
}
|
||||
case "throttle-http":
|
||||
@ -290,7 +292,7 @@ help # This message
|
||||
return NoPrintStatusRule, nil
|
||||
}
|
||||
this.migrationContext.SetThrottleHTTP(arg)
|
||||
fmt.Fprintln(writer, throttleHint)
|
||||
fmt.Fprintf(writer, throttleHint)
|
||||
return ForcePrintStatusAndHintRule, nil
|
||||
}
|
||||
case "throttle-control-replicas":
|
||||
@ -313,7 +315,7 @@ help # This message
|
||||
return NoPrintStatusRule, err
|
||||
}
|
||||
atomic.StoreInt64(&this.migrationContext.ThrottleCommandedByUser, 1)
|
||||
fmt.Fprintln(writer, throttleHint)
|
||||
fmt.Fprintf(writer, throttleHint)
|
||||
return ForcePrintStatusAndHintRule, nil
|
||||
}
|
||||
case "no-throttle", "unthrottle", "resume", "continue":
|
||||
|
@ -1,5 +1,5 @@
|
||||
/*
|
||||
Copyright 2022 GitHub Inc.
|
||||
Copyright 2016 GitHub Inc.
|
||||
See https://github.com/github/gh-ost/blob/master/LICENSE
|
||||
*/
|
||||
|
||||
@ -16,7 +16,7 @@ import (
|
||||
"github.com/github/gh-ost/go/binlog"
|
||||
"github.com/github/gh-ost/go/mysql"
|
||||
|
||||
"github.com/openark/golib/sqlutils"
|
||||
"github.com/outbrain/golib/sqlutils"
|
||||
)
|
||||
|
||||
type BinlogEventListener struct {
|
||||
@ -59,6 +59,7 @@ func NewEventsStreamer(migrationContext *base.MigrationContext) *EventsStreamer
|
||||
// AddListener registers a new listener for binlog events, on a per-table basis
|
||||
func (this *EventsStreamer) AddListener(
|
||||
async bool, databaseName string, tableName string, onDmlEvent func(event *binlog.BinlogDMLEvent) error) (err error) {
|
||||
|
||||
this.listenersMutex.Lock()
|
||||
defer this.listenersMutex.Unlock()
|
||||
|
||||
@ -86,10 +87,10 @@ func (this *EventsStreamer) notifyListeners(binlogEvent *binlog.BinlogDMLEvent)
|
||||
|
||||
for _, listener := range this.listeners {
|
||||
listener := listener
|
||||
if !strings.EqualFold(listener.databaseName, binlogEvent.DatabaseName) {
|
||||
if strings.ToLower(listener.databaseName) != strings.ToLower(binlogEvent.DatabaseName) {
|
||||
continue
|
||||
}
|
||||
if !strings.EqualFold(listener.tableName, binlogEvent.TableName) {
|
||||
if strings.ToLower(listener.tableName) != strings.ToLower(binlogEvent.TableName) {
|
||||
continue
|
||||
}
|
||||
if listener.async {
|
||||
@ -122,7 +123,10 @@ func (this *EventsStreamer) InitDBConnections() (err error) {
|
||||
|
||||
// initBinlogReader creates and connects the reader: we hook up to a MySQL server as a replica
|
||||
func (this *EventsStreamer) initBinlogReader(binlogCoordinates *mysql.BinlogCoordinates) error {
|
||||
goMySQLReader := binlog.NewGoMySQLReader(this.migrationContext)
|
||||
goMySQLReader, err := binlog.NewGoMySQLReader(this.migrationContext)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := goMySQLReader.ConnectBinlogStreamer(*binlogCoordinates); err != nil {
|
||||
return err
|
||||
}
|
||||
@ -216,4 +220,5 @@ func (this *EventsStreamer) Close() (err error) {
|
||||
|
||||
func (this *EventsStreamer) Teardown() {
|
||||
this.db.Close()
|
||||
return
|
||||
}
|
||||
|
@ -1,12 +1,11 @@
|
||||
/*
|
||||
Copyright 2022 GitHub Inc.
|
||||
Copyright 2016 GitHub Inc.
|
||||
See https://github.com/github/gh-ost/blob/master/LICENSE
|
||||
*/
|
||||
|
||||
package logic
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
@ -43,22 +42,16 @@ const frenoMagicHint = "freno"
|
||||
// Throttler collects metrics related to throttling and makes informed decision
|
||||
// whether throttling should take place.
|
||||
type Throttler struct {
|
||||
appVersion string
|
||||
migrationContext *base.MigrationContext
|
||||
applier *Applier
|
||||
httpClient *http.Client
|
||||
httpClientTimeout time.Duration
|
||||
inspector *Inspector
|
||||
finishedMigrating int64
|
||||
}
|
||||
|
||||
func NewThrottler(migrationContext *base.MigrationContext, applier *Applier, inspector *Inspector, appVersion string) *Throttler {
|
||||
func NewThrottler(migrationContext *base.MigrationContext, applier *Applier, inspector *Inspector) *Throttler {
|
||||
return &Throttler{
|
||||
appVersion: appVersion,
|
||||
migrationContext: migrationContext,
|
||||
applier: applier,
|
||||
httpClient: &http.Client{},
|
||||
httpClientTimeout: time.Duration(migrationContext.ThrottleHTTPTimeoutMillis) * time.Millisecond,
|
||||
inspector: inspector,
|
||||
finishedMigrating: 0,
|
||||
}
|
||||
@ -168,9 +161,8 @@ func (this *Throttler) collectReplicationLag(firstThrottlingCollected chan<- boo
|
||||
collectFunc()
|
||||
firstThrottlingCollected <- true
|
||||
|
||||
ticker := time.NewTicker(time.Duration(this.migrationContext.HeartbeatIntervalMilliseconds) * time.Millisecond)
|
||||
defer ticker.Stop()
|
||||
for range ticker.C {
|
||||
ticker := time.Tick(time.Duration(this.migrationContext.HeartbeatIntervalMilliseconds) * time.Millisecond)
|
||||
for range ticker {
|
||||
if atomic.LoadInt64(&this.finishedMigrating) > 0 {
|
||||
return
|
||||
}
|
||||
@ -180,6 +172,7 @@ func (this *Throttler) collectReplicationLag(firstThrottlingCollected chan<- boo
|
||||
|
||||
// collectControlReplicasLag polls all the control replicas to get maximum lag value
|
||||
func (this *Throttler) collectControlReplicasLag() {
|
||||
|
||||
if atomic.LoadInt64(&this.migrationContext.HibernateUntil) > 0 {
|
||||
return
|
||||
}
|
||||
@ -244,14 +237,12 @@ func (this *Throttler) collectControlReplicasLag() {
|
||||
}
|
||||
this.migrationContext.SetControlReplicasLagResult(readControlReplicasLag())
|
||||
}
|
||||
|
||||
aggressiveTicker := time.Tick(100 * time.Millisecond)
|
||||
relaxedFactor := 10
|
||||
counter := 0
|
||||
shouldReadLagAggressively := false
|
||||
|
||||
ticker := time.NewTicker(100 * time.Millisecond)
|
||||
defer ticker.Stop()
|
||||
for range ticker.C {
|
||||
for range aggressiveTicker {
|
||||
if atomic.LoadInt64(&this.finishedMigrating) > 0 {
|
||||
return
|
||||
}
|
||||
@ -294,22 +285,10 @@ func (this *Throttler) collectThrottleHTTPStatus(firstThrottlingCollected chan<-
|
||||
if url == "" {
|
||||
return true, nil
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), this.httpClientTimeout)
|
||||
defer cancel()
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodHead, url, nil)
|
||||
resp, err := http.Head(url)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
req.Header.Set("User-Agent", fmt.Sprintf("gh-ost/%s", this.appVersion))
|
||||
|
||||
resp, err := this.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
atomic.StoreInt64(&this.migrationContext.ThrottleHTTPStatusCode, int64(resp.StatusCode))
|
||||
return false, nil
|
||||
}
|
||||
@ -324,10 +303,8 @@ func (this *Throttler) collectThrottleHTTPStatus(firstThrottlingCollected chan<-
|
||||
|
||||
firstThrottlingCollected <- true
|
||||
|
||||
collectInterval := time.Duration(this.migrationContext.ThrottleHTTPIntervalMillis) * time.Millisecond
|
||||
ticker := time.NewTicker(collectInterval)
|
||||
defer ticker.Stop()
|
||||
for range ticker.C {
|
||||
ticker := time.Tick(100 * time.Millisecond)
|
||||
for range ticker {
|
||||
if atomic.LoadInt64(&this.finishedMigrating) > 0 {
|
||||
return
|
||||
}
|
||||
@ -446,9 +423,8 @@ func (this *Throttler) initiateThrottlerCollection(firstThrottlingCollected chan
|
||||
this.collectGeneralThrottleMetrics()
|
||||
firstThrottlingCollected <- true
|
||||
|
||||
ticker := time.NewTicker(time.Second)
|
||||
defer ticker.Stop()
|
||||
for range ticker.C {
|
||||
throttlerMetricsTick := time.Tick(1 * time.Second)
|
||||
for range throttlerMetricsTick {
|
||||
if atomic.LoadInt64(&this.finishedMigrating) > 0 {
|
||||
return
|
||||
}
|
||||
@ -459,7 +435,9 @@ func (this *Throttler) initiateThrottlerCollection(firstThrottlingCollected chan
|
||||
}
|
||||
|
||||
// initiateThrottlerChecks initiates the throttle ticker and sets the basic behavior of throttling.
|
||||
func (this *Throttler) initiateThrottlerChecks() {
|
||||
func (this *Throttler) initiateThrottlerChecks() error {
|
||||
throttlerTick := time.Tick(100 * time.Millisecond)
|
||||
|
||||
throttlerFunction := func() {
|
||||
alreadyThrottling, currentReason, _ := this.migrationContext.IsThrottled()
|
||||
shouldThrottle, throttleReason, throttleReasonHint := this.shouldThrottle()
|
||||
@ -476,15 +454,14 @@ func (this *Throttler) initiateThrottlerChecks() {
|
||||
this.migrationContext.SetThrottled(shouldThrottle, throttleReason, throttleReasonHint)
|
||||
}
|
||||
throttlerFunction()
|
||||
|
||||
ticker := time.NewTicker(100 * time.Millisecond)
|
||||
defer ticker.Stop()
|
||||
for range ticker.C {
|
||||
for range throttlerTick {
|
||||
if atomic.LoadInt64(&this.finishedMigrating) > 0 {
|
||||
return
|
||||
return nil
|
||||
}
|
||||
throttlerFunction()
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// throttle sees if throttling needs take place, and if so, continuously sleeps (blocks)
|
||||
|
@ -1,21 +1,36 @@
|
||||
/*
|
||||
Copyright 2015 Shlomi Noach, courtesy Booking.com
|
||||
Copyright 2022 GitHub Inc.
|
||||
See https://github.com/github/gh-ost/blob/master/LICENSE
|
||||
*/
|
||||
|
||||
package mysql
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var detachPattern *regexp.Regexp
|
||||
|
||||
func init() {
|
||||
detachPattern, _ = regexp.Compile(`//([^/:]+):([\d]+)`) // e.g. `//binlog.01234:567890`
|
||||
}
|
||||
|
||||
type BinlogType int
|
||||
|
||||
const (
|
||||
BinaryLog BinlogType = iota
|
||||
RelayLog
|
||||
)
|
||||
|
||||
// BinlogCoordinates described binary log coordinates in the form of log file & log position.
|
||||
type BinlogCoordinates struct {
|
||||
LogFile string
|
||||
LogPos int64
|
||||
Type BinlogType
|
||||
}
|
||||
|
||||
// ParseInstanceKey will parse an InstanceKey from a string representation such as 127.0.0.1:3306
|
||||
@ -47,7 +62,7 @@ func (this *BinlogCoordinates) Equals(other *BinlogCoordinates) bool {
|
||||
if other == nil {
|
||||
return false
|
||||
}
|
||||
return this.LogFile == other.LogFile && this.LogPos == other.LogPos
|
||||
return this.LogFile == other.LogFile && this.LogPos == other.LogPos && this.Type == other.Type
|
||||
}
|
||||
|
||||
// IsEmpty returns true if the log file is empty, unnamed
|
||||
@ -72,5 +87,76 @@ func (this *BinlogCoordinates) SmallerThanOrEquals(other *BinlogCoordinates) boo
|
||||
if this.SmallerThan(other) {
|
||||
return true
|
||||
}
|
||||
return this.LogFile == other.LogFile && this.LogPos == other.LogPos
|
||||
return this.LogFile == other.LogFile && this.LogPos == other.LogPos // No Type comparison
|
||||
}
|
||||
|
||||
// FileSmallerThan returns true if this coordinate's file is strictly smaller than the other's.
|
||||
func (this *BinlogCoordinates) FileSmallerThan(other *BinlogCoordinates) bool {
|
||||
return this.LogFile < other.LogFile
|
||||
}
|
||||
|
||||
// FileNumberDistance returns the numeric distance between this coordinate's file number and the other's.
|
||||
// Effectively it means "how many rotates/FLUSHes would make these coordinates's file reach the other's"
|
||||
func (this *BinlogCoordinates) FileNumberDistance(other *BinlogCoordinates) int {
|
||||
thisNumber, _ := this.FileNumber()
|
||||
otherNumber, _ := other.FileNumber()
|
||||
return otherNumber - thisNumber
|
||||
}
|
||||
|
||||
// FileNumber returns the numeric value of the file, and the length in characters representing the number in the filename.
|
||||
// Example: FileNumber() of mysqld.log.000789 is (789, 6)
|
||||
func (this *BinlogCoordinates) FileNumber() (int, int) {
|
||||
tokens := strings.Split(this.LogFile, ".")
|
||||
numPart := tokens[len(tokens)-1]
|
||||
numLen := len(numPart)
|
||||
fileNum, err := strconv.Atoi(numPart)
|
||||
if err != nil {
|
||||
return 0, 0
|
||||
}
|
||||
return fileNum, numLen
|
||||
}
|
||||
|
||||
// PreviousFileCoordinatesBy guesses the filename of the previous binlog/relaylog, by given offset (number of files back)
|
||||
func (this *BinlogCoordinates) PreviousFileCoordinatesBy(offset int) (BinlogCoordinates, error) {
|
||||
result := BinlogCoordinates{LogPos: 0, Type: this.Type}
|
||||
|
||||
fileNum, numLen := this.FileNumber()
|
||||
if fileNum == 0 {
|
||||
return result, errors.New("Log file number is zero, cannot detect previous file")
|
||||
}
|
||||
newNumStr := fmt.Sprintf("%d", (fileNum - offset))
|
||||
newNumStr = strings.Repeat("0", numLen-len(newNumStr)) + newNumStr
|
||||
|
||||
tokens := strings.Split(this.LogFile, ".")
|
||||
tokens[len(tokens)-1] = newNumStr
|
||||
result.LogFile = strings.Join(tokens, ".")
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// PreviousFileCoordinates guesses the filename of the previous binlog/relaylog
|
||||
func (this *BinlogCoordinates) PreviousFileCoordinates() (BinlogCoordinates, error) {
|
||||
return this.PreviousFileCoordinatesBy(1)
|
||||
}
|
||||
|
||||
// PreviousFileCoordinates guesses the filename of the previous binlog/relaylog
|
||||
func (this *BinlogCoordinates) NextFileCoordinates() (BinlogCoordinates, error) {
|
||||
result := BinlogCoordinates{LogPos: 0, Type: this.Type}
|
||||
|
||||
fileNum, numLen := this.FileNumber()
|
||||
newNumStr := fmt.Sprintf("%d", (fileNum + 1))
|
||||
newNumStr = strings.Repeat("0", numLen-len(newNumStr)) + newNumStr
|
||||
|
||||
tokens := strings.Split(this.LogFile, ".")
|
||||
tokens[len(tokens)-1] = newNumStr
|
||||
result.LogFile = strings.Join(tokens, ".")
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// FileSmallerThan returns true if this coordinate's file is strictly smaller than the other's.
|
||||
func (this *BinlogCoordinates) DetachedCoordinates() (isDetached bool, detachedLogFile string, detachedLogPos string) {
|
||||
detachedCoordinatesSubmatch := detachPattern.FindStringSubmatch(this.LogFile)
|
||||
if len(detachedCoordinatesSubmatch) == 0 {
|
||||
return false, "", ""
|
||||
}
|
||||
return true, detachedCoordinatesSubmatch[1], detachedCoordinatesSubmatch[2]
|
||||
}
|
||||
|
@ -8,8 +8,8 @@ package mysql
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/openark/golib/log"
|
||||
test "github.com/openark/golib/tests"
|
||||
"github.com/outbrain/golib/log"
|
||||
test "github.com/outbrain/golib/tests"
|
||||
)
|
||||
|
||||
func init() {
|
||||
@ -37,6 +37,57 @@ func TestBinlogCoordinates(t *testing.T) {
|
||||
test.S(t).ExpectTrue(c1.SmallerThanOrEquals(&c3))
|
||||
}
|
||||
|
||||
func TestBinlogNext(t *testing.T) {
|
||||
c1 := BinlogCoordinates{LogFile: "mysql-bin.00017", LogPos: 104}
|
||||
cres, err := c1.NextFileCoordinates()
|
||||
|
||||
test.S(t).ExpectNil(err)
|
||||
test.S(t).ExpectEquals(c1.Type, cres.Type)
|
||||
test.S(t).ExpectEquals(cres.LogFile, "mysql-bin.00018")
|
||||
|
||||
c2 := BinlogCoordinates{LogFile: "mysql-bin.00099", LogPos: 104}
|
||||
cres, err = c2.NextFileCoordinates()
|
||||
|
||||
test.S(t).ExpectNil(err)
|
||||
test.S(t).ExpectEquals(c1.Type, cres.Type)
|
||||
test.S(t).ExpectEquals(cres.LogFile, "mysql-bin.00100")
|
||||
|
||||
c3 := BinlogCoordinates{LogFile: "mysql.00.prod.com.00099", LogPos: 104}
|
||||
cres, err = c3.NextFileCoordinates()
|
||||
|
||||
test.S(t).ExpectNil(err)
|
||||
test.S(t).ExpectEquals(c1.Type, cres.Type)
|
||||
test.S(t).ExpectEquals(cres.LogFile, "mysql.00.prod.com.00100")
|
||||
}
|
||||
|
||||
func TestBinlogPrevious(t *testing.T) {
|
||||
c1 := BinlogCoordinates{LogFile: "mysql-bin.00017", LogPos: 104}
|
||||
cres, err := c1.PreviousFileCoordinates()
|
||||
|
||||
test.S(t).ExpectNil(err)
|
||||
test.S(t).ExpectEquals(c1.Type, cres.Type)
|
||||
test.S(t).ExpectEquals(cres.LogFile, "mysql-bin.00016")
|
||||
|
||||
c2 := BinlogCoordinates{LogFile: "mysql-bin.00100", LogPos: 104}
|
||||
cres, err = c2.PreviousFileCoordinates()
|
||||
|
||||
test.S(t).ExpectNil(err)
|
||||
test.S(t).ExpectEquals(c1.Type, cres.Type)
|
||||
test.S(t).ExpectEquals(cres.LogFile, "mysql-bin.00099")
|
||||
|
||||
c3 := BinlogCoordinates{LogFile: "mysql.00.prod.com.00100", LogPos: 104}
|
||||
cres, err = c3.PreviousFileCoordinates()
|
||||
|
||||
test.S(t).ExpectNil(err)
|
||||
test.S(t).ExpectEquals(c1.Type, cres.Type)
|
||||
test.S(t).ExpectEquals(cres.LogFile, "mysql.00.prod.com.00099")
|
||||
|
||||
c4 := BinlogCoordinates{LogFile: "mysql.00.prod.com.00000", LogPos: 104}
|
||||
_, err = c4.PreviousFileCoordinates()
|
||||
|
||||
test.S(t).ExpectNotNil(err)
|
||||
}
|
||||
|
||||
func TestBinlogCoordinatesAsKey(t *testing.T) {
|
||||
m := make(map[BinlogCoordinates]bool)
|
||||
|
||||
@ -52,3 +103,20 @@ func TestBinlogCoordinatesAsKey(t *testing.T) {
|
||||
|
||||
test.S(t).ExpectEquals(len(m), 3)
|
||||
}
|
||||
|
||||
func TestBinlogFileNumber(t *testing.T) {
|
||||
c1 := BinlogCoordinates{LogFile: "mysql-bin.00017", LogPos: 104}
|
||||
c2 := BinlogCoordinates{LogFile: "mysql-bin.00022", LogPos: 104}
|
||||
|
||||
test.S(t).ExpectEquals(c1.FileNumberDistance(&c1), 0)
|
||||
test.S(t).ExpectEquals(c1.FileNumberDistance(&c2), 5)
|
||||
test.S(t).ExpectEquals(c2.FileNumberDistance(&c1), -5)
|
||||
}
|
||||
|
||||
func TestBinlogFileNumberDistance(t *testing.T) {
|
||||
c1 := BinlogCoordinates{LogFile: "mysql-bin.00017", LogPos: 104}
|
||||
fileNum, numLen := c1.FileNumber()
|
||||
|
||||
test.S(t).ExpectEquals(fileNum, 17)
|
||||
test.S(t).ExpectEquals(numLen, 5)
|
||||
}
|
||||
|
@ -1,5 +1,5 @@
|
||||
/*
|
||||
Copyright 2022 GitHub Inc.
|
||||
Copyright 2016 GitHub Inc.
|
||||
See https://github.com/github/gh-ost/blob/master/LICENSE
|
||||
*/
|
||||
|
||||
@ -12,7 +12,6 @@ import (
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"net"
|
||||
"strings"
|
||||
|
||||
"github.com/go-sql-driver/mysql"
|
||||
)
|
||||
@ -23,13 +22,12 @@ const (
|
||||
|
||||
// ConnectionConfig is the minimal configuration required to connect to a MySQL server
|
||||
type ConnectionConfig struct {
|
||||
Key InstanceKey
|
||||
User string
|
||||
Password string
|
||||
ImpliedKey *InstanceKey
|
||||
tlsConfig *tls.Config
|
||||
Timeout float64
|
||||
TransactionIsolation string
|
||||
Key InstanceKey
|
||||
User string
|
||||
Password string
|
||||
ImpliedKey *InstanceKey
|
||||
tlsConfig *tls.Config
|
||||
Timeout float64
|
||||
}
|
||||
|
||||
func NewConnectionConfig() *ConnectionConfig {
|
||||
@ -43,12 +41,11 @@ func NewConnectionConfig() *ConnectionConfig {
|
||||
// DuplicateCredentials creates a new connection config with given key and with same credentials as this config
|
||||
func (this *ConnectionConfig) DuplicateCredentials(key InstanceKey) *ConnectionConfig {
|
||||
config := &ConnectionConfig{
|
||||
Key: key,
|
||||
User: this.User,
|
||||
Password: this.Password,
|
||||
tlsConfig: this.tlsConfig,
|
||||
Timeout: this.Timeout,
|
||||
TransactionIsolation: this.TransactionIsolation,
|
||||
Key: key,
|
||||
User: this.User,
|
||||
Password: this.Password,
|
||||
tlsConfig: this.tlsConfig,
|
||||
Timeout: this.Timeout,
|
||||
}
|
||||
config.ImpliedKey = &config.Key
|
||||
return config
|
||||
@ -115,23 +112,12 @@ func (this *ConnectionConfig) GetDBUri(databaseName string) string {
|
||||
// Wrap IPv6 literals in square brackets
|
||||
hostname = fmt.Sprintf("[%s]", hostname)
|
||||
}
|
||||
|
||||
interpolateParams := true
|
||||
// go-mysql-driver defaults to false if tls param is not provided; explicitly setting here to
|
||||
// simplify construction of the DSN below.
|
||||
tlsOption := "false"
|
||||
if this.tlsConfig != nil {
|
||||
tlsOption = TLS_CONFIG_KEY
|
||||
}
|
||||
connectionParams := []string{
|
||||
"autocommit=true",
|
||||
"charset=utf8mb4,utf8,latin1",
|
||||
"interpolateParams=true",
|
||||
fmt.Sprintf("tls=%s", tlsOption),
|
||||
fmt.Sprintf("transaction_isolation=%q", this.TransactionIsolation),
|
||||
fmt.Sprintf("timeout=%fs", this.Timeout),
|
||||
fmt.Sprintf("readTimeout=%fs", this.Timeout),
|
||||
fmt.Sprintf("writeTimeout=%fs", this.Timeout),
|
||||
}
|
||||
|
||||
return fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?%s", this.User, this.Password, hostname, this.Key.Port, databaseName, strings.Join(connectionParams, "&"))
|
||||
return fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?timeout=%fs&readTimeout=%fs&writeTimeout=%fs&interpolateParams=%t&autocommit=true&charset=utf8mb4,utf8,latin1&tls=%s", this.User, this.Password, hostname, this.Key.Port, databaseName, this.Timeout, this.Timeout, this.Timeout, interpolateParams, tlsOption)
|
||||
}
|
||||
|
@ -1,5 +1,5 @@
|
||||
/*
|
||||
Copyright 2022 GitHub Inc.
|
||||
Copyright 2016 GitHub Inc.
|
||||
See https://github.com/github/gh-ost/blob/master/LICENSE
|
||||
*/
|
||||
|
||||
@ -9,12 +9,8 @@ import (
|
||||
"crypto/tls"
|
||||
"testing"
|
||||
|
||||
"github.com/openark/golib/log"
|
||||
test "github.com/openark/golib/tests"
|
||||
)
|
||||
|
||||
const (
|
||||
transactionIsolation = "REPEATABLE-READ"
|
||||
"github.com/outbrain/golib/log"
|
||||
test "github.com/outbrain/golib/tests"
|
||||
)
|
||||
|
||||
func init() {
|
||||
@ -29,7 +25,6 @@ func TestNewConnectionConfig(t *testing.T) {
|
||||
test.S(t).ExpectEquals(c.ImpliedKey.Port, 0)
|
||||
test.S(t).ExpectEquals(c.User, "")
|
||||
test.S(t).ExpectEquals(c.Password, "")
|
||||
test.S(t).ExpectEquals(c.TransactionIsolation, "")
|
||||
}
|
||||
|
||||
func TestDuplicateCredentials(t *testing.T) {
|
||||
@ -41,7 +36,6 @@ func TestDuplicateCredentials(t *testing.T) {
|
||||
InsecureSkipVerify: true,
|
||||
ServerName: "feathers",
|
||||
}
|
||||
c.TransactionIsolation = transactionIsolation
|
||||
|
||||
dup := c.DuplicateCredentials(InstanceKey{Hostname: "otherhost", Port: 3310})
|
||||
test.S(t).ExpectEquals(dup.Key.Hostname, "otherhost")
|
||||
@ -51,7 +45,6 @@ func TestDuplicateCredentials(t *testing.T) {
|
||||
test.S(t).ExpectEquals(dup.User, "gromit")
|
||||
test.S(t).ExpectEquals(dup.Password, "penguin")
|
||||
test.S(t).ExpectEquals(dup.tlsConfig, c.tlsConfig)
|
||||
test.S(t).ExpectEquals(dup.TransactionIsolation, c.TransactionIsolation)
|
||||
}
|
||||
|
||||
func TestDuplicate(t *testing.T) {
|
||||
@ -59,7 +52,6 @@ func TestDuplicate(t *testing.T) {
|
||||
c.Key = InstanceKey{Hostname: "myhost", Port: 3306}
|
||||
c.User = "gromit"
|
||||
c.Password = "penguin"
|
||||
c.TransactionIsolation = transactionIsolation
|
||||
|
||||
dup := c.Duplicate()
|
||||
test.S(t).ExpectEquals(dup.Key.Hostname, "myhost")
|
||||
@ -68,7 +60,6 @@ func TestDuplicate(t *testing.T) {
|
||||
test.S(t).ExpectEquals(dup.ImpliedKey.Port, 3306)
|
||||
test.S(t).ExpectEquals(dup.User, "gromit")
|
||||
test.S(t).ExpectEquals(dup.Password, "penguin")
|
||||
test.S(t).ExpectEquals(dup.TransactionIsolation, transactionIsolation)
|
||||
}
|
||||
|
||||
func TestGetDBUri(t *testing.T) {
|
||||
@ -76,11 +67,9 @@ func TestGetDBUri(t *testing.T) {
|
||||
c.Key = InstanceKey{Hostname: "myhost", Port: 3306}
|
||||
c.User = "gromit"
|
||||
c.Password = "penguin"
|
||||
c.Timeout = 1.2345
|
||||
c.TransactionIsolation = transactionIsolation
|
||||
|
||||
uri := c.GetDBUri("test")
|
||||
test.S(t).ExpectEquals(uri, `gromit:penguin@tcp(myhost:3306)/test?autocommit=true&charset=utf8mb4,utf8,latin1&interpolateParams=true&tls=false&transaction_isolation="REPEATABLE-READ"&timeout=1.234500s&readTimeout=1.234500s&writeTimeout=1.234500s`)
|
||||
test.S(t).ExpectEquals(uri, "gromit:penguin@tcp(myhost:3306)/test?timeout=0.000000s&readTimeout=0.000000s&writeTimeout=0.000000s&interpolateParams=true&autocommit=true&charset=utf8mb4,utf8,latin1&tls=false")
|
||||
}
|
||||
|
||||
func TestGetDBUriWithTLSSetup(t *testing.T) {
|
||||
@ -88,10 +77,8 @@ func TestGetDBUriWithTLSSetup(t *testing.T) {
|
||||
c.Key = InstanceKey{Hostname: "myhost", Port: 3306}
|
||||
c.User = "gromit"
|
||||
c.Password = "penguin"
|
||||
c.Timeout = 1.2345
|
||||
c.tlsConfig = &tls.Config{}
|
||||
c.TransactionIsolation = transactionIsolation
|
||||
|
||||
uri := c.GetDBUri("test")
|
||||
test.S(t).ExpectEquals(uri, `gromit:penguin@tcp(myhost:3306)/test?autocommit=true&charset=utf8mb4,utf8,latin1&interpolateParams=true&tls=ghost&transaction_isolation="REPEATABLE-READ"&timeout=1.234500s&readTimeout=1.234500s&writeTimeout=1.234500s`)
|
||||
test.S(t).ExpectEquals(uri, "gromit:penguin@tcp(myhost:3306)/test?timeout=0.000000s&readTimeout=0.000000s&writeTimeout=0.000000s&interpolateParams=true&autocommit=true&charset=utf8mb4,utf8,latin1&tls=ghost")
|
||||
}
|
||||
|
@ -1,6 +1,5 @@
|
||||
/*
|
||||
Copyright 2015 Shlomi Noach, courtesy Booking.com
|
||||
Copyright 2022 GitHub Inc.
|
||||
See https://github.com/github/gh-ost/blob/master/LICENSE
|
||||
*/
|
||||
|
||||
@ -13,16 +12,15 @@ import (
|
||||
"strings"
|
||||
)
|
||||
|
||||
const DefaultInstancePort = 3306
|
||||
const (
|
||||
DefaultInstancePort = 3306
|
||||
)
|
||||
|
||||
var (
|
||||
ipv4HostPortRegexp = regexp.MustCompile("^([^:]+):([0-9]+)$")
|
||||
ipv4HostRegexp = regexp.MustCompile("^([^:]+)$")
|
||||
|
||||
// e.g. [2001:db8:1f70::999:de8:7648:6e8]:3308
|
||||
ipv6HostPortRegexp = regexp.MustCompile("^\\[([:0-9a-fA-F]+)\\]:([0-9]+)$") //nolint:gosimple
|
||||
// e.g. 2001:db8:1f70::999:de8:7648:6e8
|
||||
ipv6HostRegexp = regexp.MustCompile("^([:0-9a-fA-F]+)$")
|
||||
ipv6HostPortRegexp = regexp.MustCompile("^\\[([:0-9a-fA-F]+)\\]:([0-9]+)$") // e.g. [2001:db8:1f70::999:de8:7648:6e8]:3308
|
||||
ipv6HostRegexp = regexp.MustCompile("^([:0-9a-fA-F]+)$") // e.g. 2001:db8:1f70::999:de8:7648:6e8
|
||||
)
|
||||
|
||||
// InstanceKey is an instance indicator, identified by hostname and port
|
||||
@ -35,7 +33,8 @@ const detachHint = "//"
|
||||
|
||||
// ParseInstanceKey will parse an InstanceKey from a string representation such as 127.0.0.1:3306
|
||||
func NewRawInstanceKey(hostPort string) (*InstanceKey, error) {
|
||||
var hostname, port string
|
||||
hostname := ""
|
||||
port := ""
|
||||
if submatch := ipv4HostPortRegexp.FindStringSubmatch(hostPort); len(submatch) > 0 {
|
||||
hostname = submatch[1]
|
||||
port = submatch[2]
|
||||
|
@ -8,8 +8,8 @@ package mysql
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/openark/golib/log"
|
||||
test "github.com/openark/golib/tests"
|
||||
"github.com/outbrain/golib/log"
|
||||
test "github.com/outbrain/golib/tests"
|
||||
)
|
||||
|
||||
func init() {
|
||||
|
@ -14,8 +14,8 @@ import (
|
||||
|
||||
"github.com/github/gh-ost/go/sql"
|
||||
|
||||
"github.com/openark/golib/log"
|
||||
"github.com/openark/golib/sqlutils"
|
||||
"github.com/outbrain/golib/log"
|
||||
"github.com/outbrain/golib/sqlutils"
|
||||
)
|
||||
|
||||
const (
|
||||
@ -205,9 +205,3 @@ func GetTableColumns(db *gosql.DB, databaseName, tableName string) (*sql.ColumnL
|
||||
}
|
||||
return sql.NewColumnList(columnNames), sql.NewColumnList(virtualColumnNames), nil
|
||||
}
|
||||
|
||||
// Kill executes a KILL QUERY by connection id
|
||||
func Kill(db *gosql.DB, connectionID string) error {
|
||||
_, err := db.Exec(`KILL QUERY %s`, connectionID)
|
||||
return err
|
||||
}
|
||||
|
65
go/os/process.go
Normal file
65
go/os/process.go
Normal file
@ -0,0 +1,65 @@
|
||||
/*
|
||||
Copyright 2014 Outbrain Inc.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package os
|
||||
|
||||
import (
|
||||
"github.com/outbrain/golib/log"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"os/exec"
|
||||
)
|
||||
|
||||
func execCmd(commandText string, arguments ...string) (*exec.Cmd, string, error) {
|
||||
commandBytes := []byte(commandText)
|
||||
tmpFile, err := ioutil.TempFile("", "gh-ost-process-cmd-")
|
||||
if err != nil {
|
||||
return nil, "", log.Errore(err)
|
||||
}
|
||||
ioutil.WriteFile(tmpFile.Name(), commandBytes, 0644)
|
||||
log.Debugf("execCmd: %s", commandText)
|
||||
shellArguments := append([]string{}, tmpFile.Name())
|
||||
shellArguments = append(shellArguments, arguments...)
|
||||
log.Debugf("%+v", shellArguments)
|
||||
return exec.Command("bash", shellArguments...), tmpFile.Name(), nil
|
||||
}
|
||||
|
||||
// CommandRun executes a command
|
||||
func CommandRun(commandText string, arguments ...string) error {
|
||||
cmd, tmpFileName, err := execCmd(commandText, arguments...)
|
||||
defer os.Remove(tmpFileName)
|
||||
if err != nil {
|
||||
return log.Errore(err)
|
||||
}
|
||||
err = cmd.Run()
|
||||
return log.Errore(err)
|
||||
}
|
||||
|
||||
// RunCommandWithOutput executes a command and return output bytes
|
||||
func RunCommandWithOutput(commandText string) ([]byte, error) {
|
||||
cmd, tmpFileName, err := execCmd(commandText)
|
||||
defer os.Remove(tmpFileName)
|
||||
if err != nil {
|
||||
return nil, log.Errore(err)
|
||||
}
|
||||
|
||||
outputBytes, err := cmd.Output()
|
||||
if err != nil {
|
||||
return nil, log.Errore(err)
|
||||
}
|
||||
|
||||
return outputBytes, nil
|
||||
}
|
@ -1,5 +1,5 @@
|
||||
/*
|
||||
Copyright 2022 GitHub Inc.
|
||||
Copyright 2016 GitHub Inc.
|
||||
See https://github.com/github/gh-ost/blob/master/LICENSE
|
||||
*/
|
||||
|
||||
@ -33,7 +33,7 @@ func EscapeName(name string) string {
|
||||
}
|
||||
|
||||
func buildColumnsPreparedValues(columns *ColumnList) []string {
|
||||
values := make([]string, columns.Len())
|
||||
values := make([]string, columns.Len(), columns.Len())
|
||||
for i, column := range columns.Columns() {
|
||||
var token string
|
||||
if column.timezoneConversion != nil {
|
||||
@ -51,7 +51,7 @@ func buildColumnsPreparedValues(columns *ColumnList) []string {
|
||||
}
|
||||
|
||||
func buildPreparedValues(length int) []string {
|
||||
values := make([]string, length)
|
||||
values := make([]string, length, length)
|
||||
for i := 0; i < length; i++ {
|
||||
values[i] = "?"
|
||||
}
|
||||
@ -59,7 +59,7 @@ func buildPreparedValues(length int) []string {
|
||||
}
|
||||
|
||||
func duplicateNames(names []string) []string {
|
||||
duplicate := make([]string, len(names))
|
||||
duplicate := make([]string, len(names), len(names))
|
||||
copy(duplicate, names)
|
||||
return duplicate
|
||||
}
|
||||
@ -167,7 +167,7 @@ func BuildRangeComparison(columns []string, values []string, args []interface{},
|
||||
if includeEquals {
|
||||
comparison, err := BuildEqualsComparison(columns, values)
|
||||
if err != nil {
|
||||
return "", explodedArgs, err
|
||||
return "", explodedArgs, nil
|
||||
}
|
||||
comparisons = append(comparisons, comparison)
|
||||
explodedArgs = append(explodedArgs, args...)
|
||||
@ -261,8 +261,8 @@ func BuildUniqueKeyRangeEndPreparedQueryViaOffset(databaseName, tableName string
|
||||
explodedArgs = append(explodedArgs, rangeExplodedArgs...)
|
||||
|
||||
uniqueKeyColumnNames := duplicateNames(uniqueKeyColumns.Names())
|
||||
uniqueKeyColumnAscending := make([]string, len(uniqueKeyColumnNames))
|
||||
uniqueKeyColumnDescending := make([]string, len(uniqueKeyColumnNames))
|
||||
uniqueKeyColumnAscending := make([]string, len(uniqueKeyColumnNames), len(uniqueKeyColumnNames))
|
||||
uniqueKeyColumnDescending := make([]string, len(uniqueKeyColumnNames), len(uniqueKeyColumnNames))
|
||||
for i, column := range uniqueKeyColumns.Columns() {
|
||||
uniqueKeyColumnNames[i] = EscapeName(uniqueKeyColumnNames[i])
|
||||
if column.Type == EnumColumnType {
|
||||
@ -316,8 +316,8 @@ func BuildUniqueKeyRangeEndPreparedQueryViaTemptable(databaseName, tableName str
|
||||
explodedArgs = append(explodedArgs, rangeExplodedArgs...)
|
||||
|
||||
uniqueKeyColumnNames := duplicateNames(uniqueKeyColumns.Names())
|
||||
uniqueKeyColumnAscending := make([]string, len(uniqueKeyColumnNames))
|
||||
uniqueKeyColumnDescending := make([]string, len(uniqueKeyColumnNames))
|
||||
uniqueKeyColumnAscending := make([]string, len(uniqueKeyColumnNames), len(uniqueKeyColumnNames))
|
||||
uniqueKeyColumnDescending := make([]string, len(uniqueKeyColumnNames), len(uniqueKeyColumnNames))
|
||||
for i, column := range uniqueKeyColumns.Columns() {
|
||||
uniqueKeyColumnNames[i] = EscapeName(uniqueKeyColumnNames[i])
|
||||
if column.Type == EnumColumnType {
|
||||
@ -368,7 +368,7 @@ func buildUniqueKeyMinMaxValuesPreparedQuery(databaseName, tableName string, uni
|
||||
tableName = EscapeName(tableName)
|
||||
|
||||
uniqueKeyColumnNames := duplicateNames(uniqueKeyColumns.Names())
|
||||
uniqueKeyColumnOrder := make([]string, len(uniqueKeyColumnNames))
|
||||
uniqueKeyColumnOrder := make([]string, len(uniqueKeyColumnNames), len(uniqueKeyColumnNames))
|
||||
for i, column := range uniqueKeyColumns.Columns() {
|
||||
uniqueKeyColumnNames[i] = EscapeName(uniqueKeyColumnNames[i])
|
||||
if column.Type == EnumColumnType {
|
||||
@ -501,17 +501,14 @@ func BuildDMLUpdateQuery(databaseName, tableName string, tableColumns, sharedCol
|
||||
}
|
||||
|
||||
equalsComparison, err := BuildEqualsPreparedComparison(uniqueKeyColumns.Names())
|
||||
if err != nil {
|
||||
return "", sharedArgs, uniqueKeyArgs, err
|
||||
}
|
||||
result = fmt.Sprintf(`
|
||||
update /* gh-ost %s.%s */
|
||||
%s.%s
|
||||
update /* gh-ost %s.%s */
|
||||
%s.%s
|
||||
set
|
||||
%s
|
||||
where
|
||||
%s
|
||||
`, databaseName, tableName,
|
||||
%s
|
||||
`, databaseName, tableName,
|
||||
databaseName, tableName,
|
||||
setClause,
|
||||
equalsComparison,
|
||||
|
@ -12,8 +12,8 @@ import (
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/openark/golib/log"
|
||||
test "github.com/openark/golib/tests"
|
||||
"github.com/outbrain/golib/log"
|
||||
test "github.com/outbrain/golib/tests"
|
||||
)
|
||||
|
||||
var (
|
||||
|
@ -1,5 +1,5 @@
|
||||
/*
|
||||
Copyright 2022 GitHub Inc.
|
||||
Copyright 2016 GitHub Inc.
|
||||
See https://github.com/github/gh-ost/blob/master/LICENSE
|
||||
*/
|
||||
|
||||
@ -62,7 +62,7 @@ func NewParserFromAlterStatement(alterStatement string) *AlterTableParser {
|
||||
return parser
|
||||
}
|
||||
|
||||
func (this *AlterTableParser) tokenizeAlterStatement(alterStatement string) (tokens []string) {
|
||||
func (this *AlterTableParser) tokenizeAlterStatement(alterStatement string) (tokens []string, err error) {
|
||||
terminatingQuote := rune(0)
|
||||
f := func(c rune) bool {
|
||||
switch {
|
||||
@ -86,7 +86,7 @@ func (this *AlterTableParser) tokenizeAlterStatement(alterStatement string) (tok
|
||||
for i := range tokens {
|
||||
tokens[i] = strings.TrimSpace(tokens[i])
|
||||
}
|
||||
return tokens
|
||||
return tokens, nil
|
||||
}
|
||||
|
||||
func (this *AlterTableParser) sanitizeQuotesFromAlterStatement(alterStatement string) (strippedStatement string) {
|
||||
@ -95,7 +95,7 @@ func (this *AlterTableParser) sanitizeQuotesFromAlterStatement(alterStatement st
|
||||
return strippedStatement
|
||||
}
|
||||
|
||||
func (this *AlterTableParser) parseAlterToken(alterToken string) {
|
||||
func (this *AlterTableParser) parseAlterToken(alterToken string) (err error) {
|
||||
{
|
||||
// rename
|
||||
allStringSubmatch := renameColumnRegexp.FindAllStringSubmatch(alterToken, -1)
|
||||
@ -131,9 +131,11 @@ func (this *AlterTableParser) parseAlterToken(alterToken string) {
|
||||
this.isAutoIncrementDefined = true
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (this *AlterTableParser) ParseAlterStatement(alterStatement string) (err error) {
|
||||
|
||||
this.alterStatementOptions = alterStatement
|
||||
for _, alterTableRegexp := range alterTableExplicitSchemaTableRegexps {
|
||||
if submatch := alterTableRegexp.FindStringSubmatch(this.alterStatementOptions); len(submatch) > 0 {
|
||||
@ -150,7 +152,8 @@ func (this *AlterTableParser) ParseAlterStatement(alterStatement string) (err er
|
||||
break
|
||||
}
|
||||
}
|
||||
for _, alterToken := range this.tokenizeAlterStatement(this.alterStatementOptions) {
|
||||
alterTokens, _ := this.tokenizeAlterStatement(this.alterStatementOptions)
|
||||
for _, alterToken := range alterTokens {
|
||||
alterToken = this.sanitizeQuotesFromAlterStatement(alterToken)
|
||||
this.parseAlterToken(alterToken)
|
||||
this.alterTokens = append(this.alterTokens, alterToken)
|
||||
|
@ -1,5 +1,5 @@
|
||||
/*
|
||||
Copyright 2022 GitHub Inc.
|
||||
Copyright 2016 GitHub Inc.
|
||||
See https://github.com/github/gh-ost/blob/master/LICENSE
|
||||
*/
|
||||
|
||||
@ -9,8 +9,8 @@ import (
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/openark/golib/log"
|
||||
test "github.com/openark/golib/tests"
|
||||
"github.com/outbrain/golib/log"
|
||||
test "github.com/outbrain/golib/tests"
|
||||
)
|
||||
|
||||
func init() {
|
||||
@ -40,6 +40,7 @@ func TestParseAlterStatementTrivialRename(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestParseAlterStatementWithAutoIncrement(t *testing.T) {
|
||||
|
||||
statements := []string{
|
||||
"auto_increment=7",
|
||||
"auto_increment = 7",
|
||||
@ -99,37 +100,37 @@ func TestTokenizeAlterStatement(t *testing.T) {
|
||||
parser := NewAlterTableParser()
|
||||
{
|
||||
alterStatement := "add column t int"
|
||||
tokens := parser.tokenizeAlterStatement(alterStatement)
|
||||
tokens, _ := parser.tokenizeAlterStatement(alterStatement)
|
||||
test.S(t).ExpectTrue(reflect.DeepEqual(tokens, []string{"add column t int"}))
|
||||
}
|
||||
{
|
||||
alterStatement := "add column t int, change column i int"
|
||||
tokens := parser.tokenizeAlterStatement(alterStatement)
|
||||
tokens, _ := parser.tokenizeAlterStatement(alterStatement)
|
||||
test.S(t).ExpectTrue(reflect.DeepEqual(tokens, []string{"add column t int", "change column i int"}))
|
||||
}
|
||||
{
|
||||
alterStatement := "add column t int, change column i int 'some comment'"
|
||||
tokens := parser.tokenizeAlterStatement(alterStatement)
|
||||
tokens, _ := parser.tokenizeAlterStatement(alterStatement)
|
||||
test.S(t).ExpectTrue(reflect.DeepEqual(tokens, []string{"add column t int", "change column i int 'some comment'"}))
|
||||
}
|
||||
{
|
||||
alterStatement := "add column t int, change column i int 'some comment, with comma'"
|
||||
tokens := parser.tokenizeAlterStatement(alterStatement)
|
||||
tokens, _ := parser.tokenizeAlterStatement(alterStatement)
|
||||
test.S(t).ExpectTrue(reflect.DeepEqual(tokens, []string{"add column t int", "change column i int 'some comment, with comma'"}))
|
||||
}
|
||||
{
|
||||
alterStatement := "add column t int, add column d decimal(10,2)"
|
||||
tokens := parser.tokenizeAlterStatement(alterStatement)
|
||||
tokens, _ := parser.tokenizeAlterStatement(alterStatement)
|
||||
test.S(t).ExpectTrue(reflect.DeepEqual(tokens, []string{"add column t int", "add column d decimal(10,2)"}))
|
||||
}
|
||||
{
|
||||
alterStatement := "add column t int, add column e enum('a','b','c')"
|
||||
tokens := parser.tokenizeAlterStatement(alterStatement)
|
||||
tokens, _ := parser.tokenizeAlterStatement(alterStatement)
|
||||
test.S(t).ExpectTrue(reflect.DeepEqual(tokens, []string{"add column t int", "add column e enum('a','b','c')"}))
|
||||
}
|
||||
{
|
||||
alterStatement := "add column t int(11), add column e enum('a','b','c')"
|
||||
tokens := parser.tokenizeAlterStatement(alterStatement)
|
||||
tokens, _ := parser.tokenizeAlterStatement(alterStatement)
|
||||
test.S(t).ExpectTrue(reflect.DeepEqual(tokens, []string{"add column t int(11)", "add column e enum('a','b','c')"}))
|
||||
}
|
||||
}
|
||||
@ -149,6 +150,7 @@ func TestSanitizeQuotesFromAlterStatement(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestParseAlterStatementDroppedColumns(t *testing.T) {
|
||||
|
||||
{
|
||||
parser := NewAlterTableParser()
|
||||
statement := "drop column b"
|
||||
@ -188,6 +190,7 @@ func TestParseAlterStatementDroppedColumns(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestParseAlterStatementRenameTable(t *testing.T) {
|
||||
|
||||
{
|
||||
parser := NewAlterTableParser()
|
||||
statement := "drop column b"
|
||||
@ -227,6 +230,7 @@ func TestParseAlterStatementRenameTable(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestParseAlterStatementExplicitTable(t *testing.T) {
|
||||
|
||||
{
|
||||
parser := NewAlterTableParser()
|
||||
statement := "drop column b"
|
||||
|
@ -32,11 +32,6 @@ type TimezoneConversion struct {
|
||||
ToTimezone string
|
||||
}
|
||||
|
||||
type CharacterSetConversion struct {
|
||||
ToCharset string
|
||||
FromCharset string
|
||||
}
|
||||
|
||||
type Column struct {
|
||||
Name string
|
||||
IsUnsigned bool
|
||||
@ -48,22 +43,17 @@ type Column struct {
|
||||
// add Octet length for binary type, fix bytes with suffix "00" get clipped in mysql binlog.
|
||||
// https://github.com/github/gh-ost/issues/909
|
||||
BinaryOctetLength uint
|
||||
charsetConversion *CharacterSetConversion
|
||||
}
|
||||
|
||||
func (this *Column) convertArg(arg interface{}, isUniqueKeyColumn bool) interface{} {
|
||||
if s, ok := arg.(string); ok {
|
||||
arg2Bytes := []byte(s)
|
||||
// convert to bytes if character string without charsetConversion.
|
||||
if this.Charset != "" && this.charsetConversion == nil {
|
||||
arg = arg2Bytes
|
||||
} else {
|
||||
if encoding, ok := charsetEncodingMap[this.Charset]; ok {
|
||||
arg, _ = encoding.NewDecoder().String(s)
|
||||
}
|
||||
// string, charset conversion
|
||||
if encoding, ok := charsetEncodingMap[this.Charset]; ok {
|
||||
arg, _ = encoding.NewDecoder().String(s)
|
||||
}
|
||||
|
||||
if this.Type == BinaryColumnType && isUniqueKeyColumn {
|
||||
arg2Bytes := []byte(arg.(string))
|
||||
size := len(arg2Bytes)
|
||||
if uint(size) < this.BinaryOctetLength {
|
||||
buf := bytes.NewBuffer(arg2Bytes)
|
||||
@ -248,10 +238,6 @@ func (this *ColumnList) Len() int {
|
||||
return len(this.columns)
|
||||
}
|
||||
|
||||
func (this *ColumnList) SetCharsetConversion(columnName string, fromCharset string, toCharset string) {
|
||||
this.GetColumn(columnName).charsetConversion = &CharacterSetConversion{FromCharset: fromCharset, ToCharset: toCharset}
|
||||
}
|
||||
|
||||
// UniqueKey is the combination of a key's name and columns
|
||||
type UniqueKey struct {
|
||||
Name string
|
||||
|
@ -10,8 +10,8 @@ import (
|
||||
|
||||
"reflect"
|
||||
|
||||
"github.com/openark/golib/log"
|
||||
test "github.com/openark/golib/tests"
|
||||
"github.com/outbrain/golib/log"
|
||||
test "github.com/outbrain/golib/tests"
|
||||
)
|
||||
|
||||
func init() {
|
||||
|
@ -1,13 +0,0 @@
|
||||
drop table if exists gh_ost_test;
|
||||
create table gh_ost_test (
|
||||
id int auto_increment,
|
||||
i int not null,
|
||||
color varchar(32),
|
||||
primary key(id)
|
||||
) auto_increment=1;
|
||||
|
||||
drop event if exists gh_ost_test;
|
||||
|
||||
insert into gh_ost_test values (null, 11, 'red');
|
||||
insert into gh_ost_test values (null, 13, 'green');
|
||||
insert into gh_ost_test values (null, 17, 'blue');
|
@ -1 +0,0 @@
|
||||
--attempt-instant-ddl
|
@ -1,40 +0,0 @@
|
||||
drop table if exists gh_ost_test;
|
||||
create table gh_ost_test (
|
||||
id int auto_increment,
|
||||
i int not null,
|
||||
ts0 timestamp(6) default current_timestamp(6),
|
||||
updated tinyint unsigned default 0,
|
||||
primary key(id, ts0)
|
||||
) auto_increment=1;
|
||||
|
||||
drop event if exists gh_ost_test;
|
||||
delimiter ;;
|
||||
create event gh_ost_test
|
||||
on schedule every 1 second
|
||||
starts current_timestamp
|
||||
ends current_timestamp + interval 60 second
|
||||
on completion not preserve
|
||||
enable
|
||||
do
|
||||
begin
|
||||
insert into gh_ost_test values (null, 11, sysdate(6), 0);
|
||||
update gh_ost_test set updated = 1 where i = 11 order by id desc limit 1;
|
||||
|
||||
insert into gh_ost_test values (null, 13, sysdate(6), 0);
|
||||
update gh_ost_test set updated = 1 where i = 13 order by id desc limit 1;
|
||||
|
||||
insert into gh_ost_test values (null, 17, sysdate(6), 0);
|
||||
update gh_ost_test set updated = 1 where i = 17 order by id desc limit 1;
|
||||
|
||||
insert into gh_ost_test values (null, 19, sysdate(6), 0);
|
||||
update gh_ost_test set updated = 1 where i = 19 order by id desc limit 1;
|
||||
|
||||
insert into gh_ost_test values (null, 23, sysdate(6), 0);
|
||||
update gh_ost_test set updated = 1 where i = 23 order by id desc limit 1;
|
||||
|
||||
insert into gh_ost_test values (null, 29, sysdate(6), 0);
|
||||
insert into gh_ost_test values (null, 31, sysdate(6), 0);
|
||||
insert into gh_ost_test values (null, 37, sysdate(6), 0);
|
||||
insert into gh_ost_test values (null, 41, sysdate(6), 0);
|
||||
delete from gh_ost_test where i = 31 order by id desc limit 1;
|
||||
end ;;
|
@ -1,40 +0,0 @@
|
||||
drop table if exists gh_ost_test;
|
||||
create table gh_ost_test (
|
||||
id int auto_increment,
|
||||
i int not null,
|
||||
v varchar(128),
|
||||
updated tinyint unsigned default 0,
|
||||
primary key(id, v)
|
||||
) auto_increment=1;
|
||||
|
||||
drop event if exists gh_ost_test;
|
||||
delimiter ;;
|
||||
create event gh_ost_test
|
||||
on schedule every 1 second
|
||||
starts current_timestamp
|
||||
ends current_timestamp + interval 60 second
|
||||
on completion not preserve
|
||||
enable
|
||||
do
|
||||
begin
|
||||
insert into gh_ost_test values (null, 11, 'eleven', 0);
|
||||
update gh_ost_test set updated = 1 where i = 11 order by id desc limit 1;
|
||||
|
||||
insert into gh_ost_test values (null, 13, 'thirteen', 0);
|
||||
update gh_ost_test set updated = 1 where i = 13 order by id desc limit 1;
|
||||
|
||||
insert into gh_ost_test values (null, 17, 'seventeen', 0);
|
||||
update gh_ost_test set updated = 1 where i = 17 order by id desc limit 1;
|
||||
|
||||
insert into gh_ost_test values (null, 19, 'nineteen', 0);
|
||||
update gh_ost_test set updated = 1 where i = 19 order by id desc limit 1;
|
||||
|
||||
insert into gh_ost_test values (null, 23, 'twenty three', 0);
|
||||
update gh_ost_test set updated = 1 where i = 23 order by id desc limit 1;
|
||||
|
||||
insert into gh_ost_test values (null, 29, 'twenty nine', 0);
|
||||
insert into gh_ost_test values (null, 31, 'thirty one', 0);
|
||||
insert into gh_ost_test values (null, 37, 'thirty seven', 0);
|
||||
insert into gh_ost_test values (null, 41, 'forty one', 0);
|
||||
delete from gh_ost_test where i = 31 order by id desc limit 1;
|
||||
end ;;
|
@ -7,6 +7,9 @@ create table gh_ost_test (
|
||||
primary key(id)
|
||||
) auto_increment=1;
|
||||
|
||||
insert into gh_ost_test values (null, 'átesting');
|
||||
|
||||
|
||||
insert into gh_ost_test values (null, 'Hello world, Καλημέρα κόσμε, コンニチハ', 'átesting0', 'initial');
|
||||
|
||||
drop event if exists gh_ost_test;
|
||||
|
@ -0,0 +1 @@
|
||||
(5.5)
|
1
localtests/datetime-submillis/ignore_versions
Normal file
1
localtests/datetime-submillis/ignore_versions
Normal file
@ -0,0 +1 @@
|
||||
(5.5)
|
1
localtests/datetime-to-timestamp-pk-fail/ignore_versions
Normal file
1
localtests/datetime-to-timestamp-pk-fail/ignore_versions
Normal file
@ -0,0 +1 @@
|
||||
(5.5)
|
@ -1,20 +0,0 @@
|
||||
drop table if exists gh_ost_test;
|
||||
create table gh_ost_test (
|
||||
id int unsigned auto_increment,
|
||||
i int not null,
|
||||
dt datetime,
|
||||
primary key(id)
|
||||
) auto_increment=1;
|
||||
|
||||
drop event if exists gh_ost_test;
|
||||
delimiter ;;
|
||||
create event gh_ost_test
|
||||
on schedule every 1 second
|
||||
starts current_timestamp
|
||||
ends current_timestamp + interval 60 second
|
||||
on completion not preserve
|
||||
enable
|
||||
do
|
||||
begin
|
||||
insert into gh_ost_test values (null, 7, '2010-10-20 10:20:30');
|
||||
end ;;
|
@ -1 +0,0 @@
|
||||
--allow-zero-in-date --alter="change column dt dt datetime not null default '1970-00-00 00:00:00'"
|
1
localtests/datetime/ignore_versions
Normal file
1
localtests/datetime/ignore_versions
Normal file
@ -0,0 +1 @@
|
||||
(5.5)
|
@ -1 +0,0 @@
|
||||
Percona
|
@ -1,21 +0,0 @@
|
||||
set session sql_mode='';
|
||||
drop table if exists gh_ost_test;
|
||||
create table gh_ost_test (
|
||||
id int unsigned auto_increment,
|
||||
i int not null,
|
||||
dt datetime not null default '1970-00-00 00:00:00',
|
||||
primary key(id)
|
||||
) auto_increment=1;
|
||||
|
||||
drop event if exists gh_ost_test;
|
||||
delimiter ;;
|
||||
create event gh_ost_test
|
||||
on schedule every 1 second
|
||||
starts current_timestamp
|
||||
ends current_timestamp + interval 60 second
|
||||
on completion not preserve
|
||||
enable
|
||||
do
|
||||
begin
|
||||
insert into gh_ost_test values (null, 7, '2010-10-20 10:20:30');
|
||||
end ;;
|
@ -1 +0,0 @@
|
||||
--allow-zero-in-date --alter="engine=innodb"
|
@ -1,20 +0,0 @@
|
||||
drop table if exists gh_ost_test;
|
||||
create table gh_ost_test (
|
||||
id int unsigned auto_increment,
|
||||
i int not null,
|
||||
dt datetime,
|
||||
primary key(id)
|
||||
) auto_increment=1;
|
||||
|
||||
drop event if exists gh_ost_test;
|
||||
delimiter ;;
|
||||
create event gh_ost_test
|
||||
on schedule every 1 second
|
||||
starts current_timestamp
|
||||
ends current_timestamp + interval 60 second
|
||||
on completion not preserve
|
||||
enable
|
||||
do
|
||||
begin
|
||||
insert into gh_ost_test values (null, 7, '2010-10-20 10:20:30');
|
||||
end ;;
|
@ -1 +0,0 @@
|
||||
Invalid default value for 'dt'
|
@ -1 +0,0 @@
|
||||
--alter="change column dt dt datetime not null default '1970-00-00 00:00:00'"
|
@ -1,21 +0,0 @@
|
||||
set session sql_mode='';
|
||||
drop table if exists gh_ost_test;
|
||||
create table gh_ost_test (
|
||||
id int unsigned auto_increment,
|
||||
i int not null,
|
||||
dt datetime not null default '1970-00-00 00:00:00',
|
||||
primary key(id)
|
||||
) auto_increment=1;
|
||||
|
||||
drop event if exists gh_ost_test;
|
||||
delimiter ;;
|
||||
create event gh_ost_test
|
||||
on schedule every 1 second
|
||||
starts current_timestamp
|
||||
ends current_timestamp + interval 60 second
|
||||
on completion not preserve
|
||||
enable
|
||||
do
|
||||
begin
|
||||
insert into gh_ost_test values (null, 7, '2010-10-20 10:20:30');
|
||||
end ;;
|
@ -1 +0,0 @@
|
||||
Invalid default value for 'dt'
|
@ -1 +0,0 @@
|
||||
--alter="engine=innodb"
|
@ -1 +0,0 @@
|
||||
Percona
|
@ -1 +0,0 @@
|
||||
Percona
|
@ -1 +0,0 @@
|
||||
Percona
|
1
localtests/generated-columns-add57/ignore_versions
Normal file
1
localtests/generated-columns-add57/ignore_versions
Normal file
@ -0,0 +1 @@
|
||||
(5.5|5.6)
|
@ -1 +0,0 @@
|
||||
Percona
|
1
localtests/generated-columns-rename57/ignore_versions
Normal file
1
localtests/generated-columns-rename57/ignore_versions
Normal file
@ -0,0 +1 @@
|
||||
(5.5|5.6)
|
@ -1 +0,0 @@
|
||||
Percona
|
@ -1 +0,0 @@
|
||||
Percona
|
1
localtests/generated-columns57-unique/ignore_versions
Normal file
1
localtests/generated-columns57-unique/ignore_versions
Normal file
@ -0,0 +1 @@
|
||||
(5.5|5.6)
|
1
localtests/generated-columns57/ignore_versions
Normal file
1
localtests/generated-columns57/ignore_versions
Normal file
@ -0,0 +1 @@
|
||||
(5.5|5.6)
|
@ -1 +0,0 @@
|
||||
Percona
|
1
localtests/geometry57/ignore_versions
Normal file
1
localtests/geometry57/ignore_versions
Normal file
@ -0,0 +1 @@
|
||||
(5.5|5.6)
|
1
localtests/json57/ignore_versions
Normal file
1
localtests/json57/ignore_versions
Normal file
@ -0,0 +1 @@
|
||||
(5.5|5.6)
|
1
localtests/json57dml/ignore_versions
Normal file
1
localtests/json57dml/ignore_versions
Normal file
@ -0,0 +1 @@
|
||||
(5.5|5.6)
|
@ -1 +0,0 @@
|
||||
Percona
|
1
localtests/spatial57/ignore_versions
Normal file
1
localtests/spatial57/ignore_versions
Normal file
@ -0,0 +1 @@
|
||||
(5.5|5.6)
|
1
localtests/swap-pk-uk/ignore_versions
Normal file
1
localtests/swap-pk-uk/ignore_versions
Normal file
@ -0,0 +1 @@
|
||||
(5.5)
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user