From 577528a3bd43e94042a56fb852ccb20d10f43ea3 Mon Sep 17 00:00:00 2001 From: guineveresaenger Date: Fri, 12 Mar 2021 12:07:13 -0800 Subject: [PATCH 01/12] Adds doc entry for --serve-socket-file flag --- doc/command-line-flags.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/command-line-flags.md b/doc/command-line-flags.md index 22dccbd..62d3d11 100644 --- a/doc/command-line-flags.md +++ b/doc/command-line-flags.md @@ -181,6 +181,9 @@ Optionally involve the process ID, for example: `--replica-server-id=$((10000000 It's on you to choose a number that does not collide with another `gh-ost` or another running replica. See also: [`concurrent-migrations`](cheatsheet.md#concurrent-migrations) on the cheatsheet. +### serve-socket-file + +Defaults to an auto-determined and advertised upon startup file. Defines Unix socket file to serve on. ### skip-foreign-key-checks By default `gh-ost` verifies no foreign keys exist on the migrated table. On servers with large number of tables this check can take a long time. If you're absolutely certain no foreign keys exist (table does not reference other table nor is referenced by other tables) and wish to save the check time, provide with `--skip-foreign-key-checks`. From 2fb524f43a8d24e9b3f8e5e1f26624780112913f Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Fri, 2 Apr 2021 01:50:11 +0200 Subject: [PATCH 02/12] Adds 'hosts' command to server --- go/logic/server.go | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/go/logic/server.go b/go/logic/server.go index 1606884..aa5b43a 100644 --- a/go/logic/server.go +++ b/go/logic/server.go @@ -1,5 +1,5 @@ /* - Copyright 2016 GitHub Inc. + Copyright 2021 GitHub Inc. See https://github.com/github/gh-ost/blob/master/LICENSE */ @@ -146,7 +146,8 @@ func (this *Server) applyServerCommand(command string, writer *bufio.Writer) (pr fmt.Fprint(writer, `available commands: status # Print a detailed status message sup # Print a short status message -coordinates # Print the currently inspected coordinates +coordinates # Print the currently inspected coordinates +hosts # Print the list of hosts used to perform the migration (hostname, applier and migrator) chunk-size= # Set a new chunk-size dml-batch-size= # Set a new dml-batch-size nice-ratio= # Set a new nice-ratio, immediate sleep after each row-copy operation, float (examples: 0 is aggressive, 0.7 adds 70% runtime, 1.0 doubles runtime, 2.0 triples runtime, ...) @@ -177,6 +178,16 @@ help # This message } return NoPrintStatusRule, fmt.Errorf("coordinates are read-only") } + case "hosts": + fields := map[string]interface{}{ + "Applier": this.migrationContext.GetApplierHostname(), + "Hostname": this.migrationContext.Hostname, + "Inspector": this.migrationContext.GetInspectorHostname(), + } + for key, val := range fields { + fmt.Fprintf(writer, "%s: %v", key, val) + } + return NoPrintStatusRule, nil case "chunk-size": { if argIsQuestion { From 094d11d722eb0fd8fe69f01a0645ff2559b1623c Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Fri, 2 Apr 2021 01:58:06 +0200 Subject: [PATCH 03/12] Use a single line --- go/logic/server.go | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/go/logic/server.go b/go/logic/server.go index aa5b43a..749cc70 100644 --- a/go/logic/server.go +++ b/go/logic/server.go @@ -179,14 +179,11 @@ help # This message return NoPrintStatusRule, fmt.Errorf("coordinates are read-only") } case "hosts": - fields := map[string]interface{}{ - "Applier": this.migrationContext.GetApplierHostname(), - "Hostname": this.migrationContext.Hostname, - "Inspector": this.migrationContext.GetInspectorHostname(), - } - for key, val := range fields { - fmt.Fprintf(writer, "%s: %v", key, val) - } + fmt.Fprintf(writer, "Hostname: %s, Applier: %s, Inspector: %s\n", + this.migrationContext.GetApplierHostname(), + this.migrationContext.Hostname, + this.migrationContext.GetInspectorHostname(), + ) return NoPrintStatusRule, nil case "chunk-size": { From 7ea47cbfb5b02b08866b8c9638a37622b9be02e4 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Fri, 2 Apr 2021 01:58:59 +0200 Subject: [PATCH 04/12] Fix order --- go/logic/server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/logic/server.go b/go/logic/server.go index 749cc70..44e4c91 100644 --- a/go/logic/server.go +++ b/go/logic/server.go @@ -180,8 +180,8 @@ help # This message } case "hosts": fmt.Fprintf(writer, "Hostname: %s, Applier: %s, Inspector: %s\n", - this.migrationContext.GetApplierHostname(), this.migrationContext.Hostname, + this.migrationContext.GetApplierHostname(), this.migrationContext.GetInspectorHostname(), ) return NoPrintStatusRule, nil From cffb523badf2b98d3b5618f8e28c384ab483ac9f Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Fri, 2 Apr 2021 02:02:08 +0200 Subject: [PATCH 05/12] Fix help typo --- go/logic/server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/logic/server.go b/go/logic/server.go index 44e4c91..bf319ac 100644 --- a/go/logic/server.go +++ b/go/logic/server.go @@ -147,7 +147,7 @@ func (this *Server) applyServerCommand(command string, writer *bufio.Writer) (pr status # Print a detailed status message sup # Print a short status message coordinates # Print the currently inspected coordinates -hosts # Print the list of hosts used to perform the migration (hostname, applier and migrator) +hosts # Print the list of hosts used to perform the migration (hostname, applier and inspector) chunk-size= # Set a new chunk-size dml-batch-size= # Set a new dml-batch-size nice-ratio= # Set a new nice-ratio, immediate sleep after each row-copy operation, float (examples: 0 is aggressive, 0.7 adds 70% runtime, 1.0 doubles runtime, 2.0 triples runtime, ...) From 123b46f9bb091989aba9b8bdaa674bb4424774ec Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Fri, 2 Apr 2021 16:57:13 +0200 Subject: [PATCH 06/12] Split into 'applier' and 'inspector' commands --- doc/interactive-commands.md | 2 ++ go/logic/server.go | 14 +++++++------- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/doc/interactive-commands.md b/doc/interactive-commands.md index 591aa49..7ad44f1 100644 --- a/doc/interactive-commands.md +++ b/doc/interactive-commands.md @@ -18,6 +18,8 @@ Both interfaces may serve at the same time. Both respond to simple text command, - `status`: returns a detailed status summary of migration progress and configuration - `sup`: returns a brief status summary of migration progress - `coordinates`: returns recent (though not exactly up to date) binary log coordinates of the inspected server +- `applier`: returns the hostname of the applier +- `inspector`: returns the hostname of the inspector - `chunk-size=`: modify the `chunk-size`; applies on next running copy-iteration - `dml-batch-size=`: modify the `dml-batch-size`; applies on next applying of binary log events - `max-lag-millis=`: modify the maximum replication lag threshold (milliseconds, minimum value is `100`, i.e. `0.1` second) diff --git a/go/logic/server.go b/go/logic/server.go index bf319ac..a70c08f 100644 --- a/go/logic/server.go +++ b/go/logic/server.go @@ -147,7 +147,8 @@ func (this *Server) applyServerCommand(command string, writer *bufio.Writer) (pr status # Print a detailed status message sup # Print a short status message coordinates # Print the currently inspected coordinates -hosts # Print the list of hosts used to perform the migration (hostname, applier and inspector) +applier # Print the hostname of the applier +inspector # Print the hostname of the inspector chunk-size= # Set a new chunk-size dml-batch-size= # Set a new dml-batch-size nice-ratio= # Set a new nice-ratio, immediate sleep after each row-copy operation, float (examples: 0 is aggressive, 0.7 adds 70% runtime, 1.0 doubles runtime, 2.0 triples runtime, ...) @@ -178,12 +179,11 @@ help # This message } return NoPrintStatusRule, fmt.Errorf("coordinates are read-only") } - case "hosts": - fmt.Fprintf(writer, "Hostname: %s, Applier: %s, Inspector: %s\n", - this.migrationContext.Hostname, - this.migrationContext.GetApplierHostname(), - this.migrationContext.GetInspectorHostname(), - ) + case "applier": + fmt.Fprintln(writer, this.migrationContext.GetApplierHostname()) + return NoPrintStatusRule, nil + case "inspector": + fmt.Fprintln(writer, this.migrationContext.GetInspectorHostname()) return NoPrintStatusRule, nil case "chunk-size": { From 23a421cef7f6a2b3a90a5e4765678322e41b1dc0 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Sat, 3 Apr 2021 22:34:20 +0200 Subject: [PATCH 07/12] Add 'Hostname:' prefix to output --- go/logic/server.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go/logic/server.go b/go/logic/server.go index a70c08f..29d8318 100644 --- a/go/logic/server.go +++ b/go/logic/server.go @@ -180,10 +180,10 @@ help # This message return NoPrintStatusRule, fmt.Errorf("coordinates are read-only") } case "applier": - fmt.Fprintln(writer, this.migrationContext.GetApplierHostname()) + fmt.Fprintf(writer, "Hostname: %s\n", this.migrationContext.GetApplierHostname()) return NoPrintStatusRule, nil case "inspector": - fmt.Fprintln(writer, this.migrationContext.GetInspectorHostname()) + fmt.Fprintf(writer, "Hostname: %s\n", this.migrationContext.GetInspectorHostname()) return NoPrintStatusRule, nil case "chunk-size": { From 157dba920c1dddbb6b405dc6b7f2fdd5a1270b8b Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Sat, 3 Apr 2021 23:24:29 +0200 Subject: [PATCH 08/12] Add mysql port and version --- go/logic/server.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/go/logic/server.go b/go/logic/server.go index 29d8318..3d128b1 100644 --- a/go/logic/server.go +++ b/go/logic/server.go @@ -180,10 +180,20 @@ help # This message return NoPrintStatusRule, fmt.Errorf("coordinates are read-only") } case "applier": - fmt.Fprintf(writer, "Hostname: %s\n", this.migrationContext.GetApplierHostname()) + if this.migrationContext.ApplierConnectionConfig != nil && this.migrationContext.ApplierConnectionConfig.ImpliedKey != nil { + fmt.Fprintf(writer, "Host: %s, Version: %s\n", + this.migrationContext.ApplierConnectionConfig.ImpliedKey.String(), + this.migrationContext.ApplierMySQLVersion, + ) + } return NoPrintStatusRule, nil case "inspector": - fmt.Fprintf(writer, "Hostname: %s\n", this.migrationContext.GetInspectorHostname()) + if this.migrationContext.InspectorConnectionConfig != nil && this.migrationContext.InspectorConnectionConfig.ImpliedKey != nil { + fmt.Fprintf(writer, "Host: %s, Version: %s\n", + this.migrationContext.InspectorConnectionConfig.ImpliedKey.String(), + this.migrationContext.InspectorMySQLVersion, + ) + } return NoPrintStatusRule, nil case "chunk-size": { From c480ff133790d6558fcb6457c1a173ca62bdd906 Mon Sep 17 00:00:00 2001 From: Yakir Gibraltar Date: Tue, 6 Apr 2021 18:15:05 +0300 Subject: [PATCH 09/12] Remove build_id files from rpm --- build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sh b/build.sh index b5d4659..88ecf1e 100755 --- a/build.sh +++ b/build.sh @@ -40,7 +40,7 @@ function build { builddir=$(setuptree) cp $buildpath/$target $builddir/gh-ost/usr/bin cd $buildpath - fpm -v "${RELEASE_VERSION}" --epoch 1 -f -s dir -n gh-ost -m 'shlomi-noach ' --description "GitHub's Online Schema Migrations for MySQL " --url "https://github.com/github/gh-ost" --vendor "GitHub" --license "Apache 2.0" -C $builddir/gh-ost --prefix=/ -t rpm . + fpm -v "${RELEASE_VERSION}" --epoch 1 -f -s dir -n gh-ost -m 'shlomi-noach ' --description "GitHub's Online Schema Migrations for MySQL " --url "https://github.com/github/gh-ost" --vendor "GitHub" --license "Apache 2.0" -C $builddir/gh-ost --prefix=/ -t rpm --rpm-rpmbuild-define "_build_id_links none" . fpm -v "${RELEASE_VERSION}" --epoch 1 -f -s dir -n gh-ost -m 'shlomi-noach ' --description "GitHub's Online Schema Migrations for MySQL " --url "https://github.com/github/gh-ost" --vendor "GitHub" --license "Apache 2.0" -C $builddir/gh-ost --prefix=/ -t deb --deb-no-default-config-files . fi } From f40f14b9eeba6205fdcc303132548a6e29ce3318 Mon Sep 17 00:00:00 2001 From: "Fan()" <18501341937@163.com> Date: Thu, 29 Apr 2021 13:43:24 +0800 Subject: [PATCH 10/12] Update inspect.go fix https://github.com/github/gh-ost/issues/961 --- go/logic/inspect.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/logic/inspect.go b/go/logic/inspect.go index 0128010..3d101c0 100644 --- a/go/logic/inspect.go +++ b/go/logic/inspect.go @@ -528,7 +528,7 @@ func (this *Inspector) CountTableRows() error { this.migrationContext.Log.Infof("As instructed, I'm issuing a SELECT COUNT(*) on the table. This may take a while") - query := fmt.Sprintf(`select /* gh-ost */ count(*) as rows from %s.%s`, sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName)) + query := fmt.Sprintf(`select /* gh-ost */ count(*) as 'rows' from %s.%s`, sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName)) var rowsEstimate int64 if err := this.db.QueryRow(query).Scan(&rowsEstimate); err != nil { return err From 74f807103ee4391693054c892c845ea1b99edffc Mon Sep 17 00:00:00 2001 From: "Fan()" <18501341937@163.com> Date: Thu, 29 Apr 2021 14:33:10 +0800 Subject: [PATCH 11/12] Update inspect.go --- go/logic/inspect.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/logic/inspect.go b/go/logic/inspect.go index 3d101c0..584c56b 100644 --- a/go/logic/inspect.go +++ b/go/logic/inspect.go @@ -528,7 +528,7 @@ func (this *Inspector) CountTableRows() error { this.migrationContext.Log.Infof("As instructed, I'm issuing a SELECT COUNT(*) on the table. This may take a while") - query := fmt.Sprintf(`select /* gh-ost */ count(*) as 'rows' from %s.%s`, sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName)) + query := fmt.Sprintf(`select /* gh-ost */ count(*) as count_rows from %s.%s`, sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName)) var rowsEstimate int64 if err := this.db.QueryRow(query).Scan(&rowsEstimate); err != nil { return err From 38eeaa103675591246bd47a12c68b173c6948cc1 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Mon, 3 May 2021 16:30:42 +0200 Subject: [PATCH 12/12] Update RELEASE_VERSION to v1.1.1 --- RELEASE_VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE_VERSION b/RELEASE_VERSION index 9084fa2..524cb55 100644 --- a/RELEASE_VERSION +++ b/RELEASE_VERSION @@ -1 +1 @@ -1.1.0 +1.1.1