diff --git a/RELEASE_VERSION b/RELEASE_VERSION index 45a1b3f..781dcb0 100644 --- a/RELEASE_VERSION +++ b/RELEASE_VERSION @@ -1 +1 @@ -1.1.2 +1.1.3 diff --git a/build.sh b/build.sh index ef24e81..653aeee 100755 --- a/build.sh +++ b/build.sh @@ -18,7 +18,7 @@ function build { GOOS=$3 GOARCH=$4 - if ! go version | egrep -q 'go(1\.1[56])' ; then + if ! go version | egrep -q 'go1\.(1[5-9]|[2-9][0-9]{1})' ; then echo "go version must be 1.15 or above" exit 1 fi @@ -41,8 +41,9 @@ function build { builddir=$(setuptree) cp $buildpath/$target $builddir/gh-ost/usr/bin cd $buildpath - fpm -v "${RELEASE_VERSION}" --epoch 1 -f -s dir -n gh-ost -m 'shlomi-noach ' --description "GitHub's Online Schema Migrations for MySQL " --url "https://github.com/github/gh-ost" --vendor "GitHub" --license "Apache 2.0" -C $builddir/gh-ost --prefix=/ -t rpm --rpm-rpmbuild-define "_build_id_links none" . - fpm -v "${RELEASE_VERSION}" --epoch 1 -f -s dir -n gh-ost -m 'shlomi-noach ' --description "GitHub's Online Schema Migrations for MySQL " --url "https://github.com/github/gh-ost" --vendor "GitHub" --license "Apache 2.0" -C $builddir/gh-ost --prefix=/ -t deb --deb-no-default-config-files . + fpm -v "${RELEASE_VERSION}" --epoch 1 -f -s dir -n gh-ost -m 'GitHub' --description "GitHub's Online Schema Migrations for MySQL " --url "https://github.com/github/gh-ost" --vendor "GitHub" --license "Apache 2.0" -C $builddir/gh-ost --prefix=/ -t rpm --rpm-rpmbuild-define "_build_id_links none" . + fpm -v "${RELEASE_VERSION}" --epoch 1 -f -s dir -n gh-ost -m 'GitHub' --description "GitHub's Online Schema Migrations for MySQL " --url "https://github.com/github/gh-ost" --vendor "GitHub" --license "Apache 2.0" -C $builddir/gh-ost --prefix=/ -t deb --deb-no-default-config-files . + cd - fi } @@ -63,10 +64,16 @@ main() { mkdir -p ${buildpath} rm -rf ${buildpath:?}/* build GNU/Linux linux linux amd64 - # build macOS osx darwin amd64 + build macOS osx darwin amd64 echo "Binaries found in:" find $buildpath/gh-ost* -type f -maxdepth 1 + + echo "Checksums:" + (cd $buildpath && shasum -a256 gh-ost* 2>/dev/null) } +. script/bootstrap +cd .gopath/src/github.com/github/gh-ost + main "$@" diff --git a/doc/command-line-flags.md b/doc/command-line-flags.md index 62d3d11..2818d10 100644 --- a/doc/command-line-flags.md +++ b/doc/command-line-flags.md @@ -22,7 +22,7 @@ If, for some reason, you do not wish `gh-ost` to connect to a replica, you may c ### approve-renamed-columns -When your migration issues a column rename (`change column old_name new_name ...`) `gh-ost` analyzes the statement to try and associate the old column name with new column name. Otherwise the new structure may also look like some column was dropped and another was added. +When your migration issues a column rename (`change column old_name new_name ...`) `gh-ost` analyzes the statement to try and associate the old column name with new column name. Otherwise, the new structure may also look like some column was dropped and another was added. `gh-ost` will print out what it thinks the _rename_ implied, but will not issue the migration unless you provide with `--approve-renamed-columns`. @@ -32,7 +32,7 @@ If you think `gh-ost` is mistaken and that there's actually no _rename_ involved `gh-ost` infers the identity of the master server by crawling up the replication topology. You may explicitly tell `gh-ost` the identity of the master host via `--assume-master-host=the.master.com`. This is useful in: -- _master-master_ topologies (together with [`--allow-master-master`](#allow-master-master)), where `gh-ost` can arbitrarily pick one of the co-masters and you prefer that it picks a specific one +- _master-master_ topologies (together with [`--allow-master-master`](#allow-master-master)), where `gh-ost` can arbitrarily pick one of the co-masters, and you prefer that it picks a specific one - _tungsten replicator_ topologies (together with [`--tungsten`](#tungsten)), where `gh-ost` is unable to crawl and detect the master ### assume-rbr @@ -61,7 +61,13 @@ Comma delimited status-name=threshold, same format as [`--max-load`](#max-load). `--critical-load` defines a threshold that, when met, `gh-ost` panics and bails out. The default behavior is to bail out immediately when meeting this threshold. -This may sometimes lead to migrations bailing out on a very short spike, that, while in itself is impacting production and is worth investigating, isn't reason enough to kill a 10 hour migration. +This may sometimes lead to migrations bailing out on a very short spike, that, while in itself is impacting production and is worth investigating, isn't reason enough to kill a 10-hour migration. + +### critical-load-hibernate-seconds + +When `--critical-load-hibernate-seconds` is non-zero (e.g. `--critical-load-hibernate-seconds=300`), `critical-load` does not panic and bail out; instead, `gh-ost` goes into hibernation for the specified duration. It will not read/write anything from/to any server during this time. Execution then continues upon waking from hibernation. + +If `critical-load` is met again, `gh-ost` will repeat this cycle, and never panic and bail out. ### critical-load-interval-millis @@ -98,7 +104,7 @@ Noteworthy is that setting `--dml-batch-size` to higher value _does not_ mean `g ### exact-rowcount -A `gh-ost` execution need to copy whatever rows you have in your existing table onto the ghost table. This can, and often be, a large number. Exactly what that number is? +A `gh-ost` execution need to copy whatever rows you have in your existing table onto the ghost table. This can and often will be, a large number. Exactly what that number is? `gh-ost` initially estimates the number of rows in your table by issuing an `explain select * from your_table`. This will use statistics on your table and return with a rough estimate. How rough? It might go as low as half or as high as double the actual number of rows in your table. This is the same method as used in [`pt-online-schema-change`](https://www.percona.com/doc/percona-toolkit/2.2/pt-online-schema-change.html). `gh-ost` also supports the `--exact-rowcount` flag. When this flag is given, two things happen: @@ -230,7 +236,7 @@ Provide a command delimited list of replicas; `gh-ost` will throttle when any of ### throttle-http -Provide a HTTP endpoint; `gh-ost` will issue `HEAD` requests on given URL and throttle whenever response status code is not `200`. The URL can be queried and updated dynamically via [interactive commands](interactive-commands.md). Empty URL disables the HTTP check. +Provide an HTTP endpoint; `gh-ost` will issue `HEAD` requests on given URL and throttle whenever response status code is not `200`. The URL can be queried and updated dynamically via [interactive commands](interactive-commands.md). Empty URL disables the HTTP check. ### timestamp-old-table diff --git a/doc/hooks.md b/doc/hooks.md index 91e1311..c1fe594 100644 --- a/doc/hooks.md +++ b/doc/hooks.md @@ -68,6 +68,7 @@ The following variables are available on all hooks: - `GH_OST_INSPECTED_LAG` - lag in seconds (floating point) of inspected server - `GH_OST_HEARTBEAT_LAG` - lag in seconds (floating point) of heartbeat - `GH_OST_PROGRESS` - progress pct ([0..100], floating point) of migration +- `GH_OST_ETA_SECONDS` - estimated duration until migration finishes in seconds - `GH_OST_MIGRATED_HOST` - `GH_OST_INSPECTED_HOST` - `GH_OST_EXECUTING_HOST` diff --git a/doc/triggerless-design.md b/doc/triggerless-design.md index 510a301..10a4203 100644 --- a/doc/triggerless-design.md +++ b/doc/triggerless-design.md @@ -112,7 +112,7 @@ It is also interesting to observe that `gh-ost` is the only application writing When `gh-ost` pauses (throttles), it issues no writes on the ghost table. Because there are no triggers, write workload is decoupled from the `gh-ost` write workload. And because we're using an asynchronous approach, the algorithm already handles a time difference between a master write time and the ghost apply time. A difference of a few microseconds is no different from a difference of minutes or hours. -When `gh-ost` [throttles](throttle.md), either by replication lag, `max-load` setting or and explicit [interactive user command](interactive-commands.md), the master is back to normal. It sees no more writes on the ghost table. +When `gh-ost` [throttles](throttle.md), either by replication lag, `max-load` setting or an explicit [interactive user command](interactive-commands.md), the master is back to normal. It sees no more writes on the ghost table. An exception is the ongoing heartbeat writes onto the changelog table, which we consider to be negligible. #### Testability diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index b8557f9..04d4eeb 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -8,6 +8,7 @@ package main import ( "flag" "fmt" + "net/url" "os" "os/signal" "syscall" @@ -130,7 +131,7 @@ func main() { maxLoad := flag.String("max-load", "", "Comma delimited status-name=threshold. e.g: 'Threads_running=100,Threads_connected=500'. When status exceeds threshold, app throttles writes") criticalLoad := flag.String("critical-load", "", "Comma delimited status-name=threshold, same format as --max-load. When status exceeds threshold, app panics and quits") flag.Int64Var(&migrationContext.CriticalLoadIntervalMilliseconds, "critical-load-interval-millis", 0, "When 0, migration immediately bails out upon meeting critical-load. When non-zero, a second check is done after given interval, and migration only bails out if 2nd check still meets critical load") - flag.Int64Var(&migrationContext.CriticalLoadHibernateSeconds, "critical-load-hibernate-seconds", 0, "When nonzero, critical-load does not panic and bail out; instead, gh-ost goes into hibernate for the specified duration. It will not read/write anything to from/to any server") + flag.Int64Var(&migrationContext.CriticalLoadHibernateSeconds, "critical-load-hibernate-seconds", 0, "When non-zero, critical-load does not panic and bail out; instead, gh-ost goes into hibernation for the specified duration. It will not read/write anything from/to any server") quiet := flag.Bool("quiet", false, "quiet") verbose := flag.Bool("verbose", false, "verbose") debug := flag.Bool("debug", false, "debug mode (very verbose)") @@ -188,6 +189,11 @@ func main() { log.Fatalf("--database must be provided and database name must not be empty, or --alter must specify database name") } } + + if err := flag.Set("database", url.QueryEscape(migrationContext.DatabaseName)); err != nil { + migrationContext.Log.Fatale(err) + } + if migrationContext.OriginalTableName == "" { if parser.HasExplicitTable() { migrationContext.OriginalTableName = parser.GetExplicitTable() diff --git a/script/bootstrap b/script/bootstrap index 573313a..2f5cd08 100755 --- a/script/bootstrap +++ b/script/bootstrap @@ -10,7 +10,6 @@ set -e # Since we want to be able to build this outside of GOPATH, we set it # up so it points back to us and go is none the wiser -set -x rm -rf .gopath mkdir -p .gopath/src/github.com/github ln -s "$PWD" .gopath/src/github.com/github/gh-ost diff --git a/script/ensure-go-installed b/script/ensure-go-installed index 473ba38..14c33ff 100755 --- a/script/ensure-go-installed +++ b/script/ensure-go-installed @@ -1,7 +1,7 @@ #!/bin/bash PREFERRED_GO_VERSION=go1.16.4 -SUPPORTED_GO_VERSIONS='go1.1[56]' +SUPPORTED_GO_VERSIONS='go1.1[567]' GO_PKG_DARWIN=${PREFERRED_GO_VERSION}.darwin-amd64.pkg GO_PKG_DARWIN_SHA=0f215de06019a054a3da46a0722989986c956d719c7a0a8fc38a5f3c216d6f6b @@ -35,7 +35,7 @@ if [ -z "$(which go)" ] || [ -z "$(go version | grep "$SUPPORTED_GO_VERSIONS")" curl -L -O https://dl.google.com/go/$GO_PKG_DARWIN shasum -a256 $GO_PKG_DARWIN | grep $GO_PKG_DARWIN_SHA xar -xf $GO_PKG_DARWIN - cpio -i < com.googlecode.go.pkg/Payload + cpio -i < org.golang.go.pkg/Payload else curl -L -O https://dl.google.com/go/$GO_PKG_LINUX shasum -a256 $GO_PKG_LINUX | grep $GO_PKG_LINUX_SHA diff --git a/vendor/github.com/siddontang/go-mysql/client/resp.go b/vendor/github.com/siddontang/go-mysql/client/resp.go index 71aa1bc..0cdff1d 100644 --- a/vendor/github.com/siddontang/go-mysql/client/resp.go +++ b/vendor/github.com/siddontang/go-mysql/client/resp.go @@ -1,6 +1,5 @@ package client -import "C" import ( "encoding/binary" diff --git a/vendor/github.com/siddontang/go-mysql/packet/conn.go b/vendor/github.com/siddontang/go-mysql/packet/conn.go index 41b1bf1..3785261 100644 --- a/vendor/github.com/siddontang/go-mysql/packet/conn.go +++ b/vendor/github.com/siddontang/go-mysql/packet/conn.go @@ -1,6 +1,5 @@ package packet -import "C" import ( "bytes" "io" @@ -127,7 +126,7 @@ func (c *Conn) WritePacket(data []byte) error { func (c *Conn) WriteClearAuthPacket(password string) error { // Calculate the packet length and add a tailing 0 pktLen := len(password) + 1 - data := make([]byte, 4 + pktLen) + data := make([]byte, 4+pktLen) // Add the clear password [null terminated string] copy(data[4:], password) @@ -140,7 +139,7 @@ func (c *Conn) WriteClearAuthPacket(password string) error { // http://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::AuthSwitchResponse func (c *Conn) WritePublicKeyAuthPacket(password string, cipher []byte) error { // request public key - data := make([]byte, 4 + 1) + data := make([]byte, 4+1) data[4] = 2 // cachingSha2PasswordRequestPublicKey c.WritePacket(data) @@ -163,7 +162,7 @@ func (c *Conn) WritePublicKeyAuthPacket(password string, cipher []byte) error { } sha1v := sha1.New() enc, _ := rsa.EncryptOAEP(sha1v, rand.Reader, pub.(*rsa.PublicKey), plain, nil) - data = make([]byte, 4 + len(enc)) + data = make([]byte, 4+len(enc)) copy(data[4:], enc) return c.WritePacket(data) }