Merge branch 'master' into abort-missing-backend-tables

.github/workflows/ci.yml

@@ -0,0 +1,20 @@
+name: CI
+
+on: [pull_request]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@master
+
+    - name: Set up Go 1.12
+      uses: actions/setup-go@v1
+      with:
+        version: 1.12
+      id: go
+
+    - name: Build
+      run: script/cibuild

.github/workflows/replica-tests.yml

@@ -0,0 +1,20 @@
+name: migration tests
+
+on: [pull_request]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@master
+
+    - name: Set up Go 1.12
+      uses: actions/setup-go@v1
+      with:
+        version: 1.12
+      id: go
+
+    - name: migration tests
+      run:  script/cibuild-gh-ost-replica-tests

.travis.yml

@@ -2,12 +2,14 @@
 language: go
 
 go:
-  - "1.9"
+  - "1.12.x"
-  - "1.10"
 
 os:
   - linux
 
+services:
+  - mysql
+
 env:
 - MYSQL_USER=root
 - CURRENT_CI_ENV=travis

Dockerfile.test

@@ -0,0 +1,11 @@
+FROM golang:1.12.1
+LABEL maintainer="github@github.com"
+
+RUN apt-get update
+RUN apt-get install -y lsb-release
+RUN rm -rf /var/lib/apt/lists/*
+
+COPY . /go/src/github.com/github/gh-ost
+WORKDIR /go/src/github.com/github/gh-ost
+
+CMD ["script/test"]

README.md

@@ -94,7 +94,7 @@ Please see [Coding gh-ost](doc/coding-ghost.md) for a guide to getting started d
 
 [Download latest release here](https://github.com/github/gh-ost/releases/latest)
 
-`gh-ost` is a Go project; it is built with Go `1.9` and above. To build on your own, use either:
+`gh-ost` is a Go project; it is built with Go `1.12` and above. To build on your own, use either:
 - [script/build](https://github.com/github/gh-ost/blob/master/script/build) - this is the same build script used by CI hence the authoritative; artifact is `./bin/gh-ost` binary.
 - [build.sh](https://github.com/github/gh-ost/blob/master/build.sh) for building `tar.gz` artifacts in `/tmp/gh-ost`
 

build.sh

@@ -18,10 +18,8 @@ function build {
   GOOS=$3
   GOARCH=$4
 
-
+  if ! go version | egrep -q 'go(1\.1[234])' ; then
-
+    echo "go version must be 1.12 or above"
-  if ! go version | egrep -q 'go(1[.]9|1[.]1[0-9])' ; then
-    echo "go version is too low. Must use 1.9 or above"
     exit 1
   fi
 

docker-compose.yml

@@ -0,0 +1,7 @@
+version: "3.5"
+services:
+  app:
+    image: app
+    build:
+      context: .
+      dockerfile: Dockerfile.test

go/logic/server.go

@@ -144,7 +144,7 @@ func (this *Server) applyServerCommand(command string, writer *bufio.Writer) (pr
 	switch command {
 	case "help":
 		{
-			fmt.Fprintln(writer, `available commands:
+			fmt.Fprint(writer, `available commands:
 status                               # Print a detailed status message
 sup                                  # Print a short status message
 coordinates													 # Print the currently inspected coordinates
@@ -341,7 +341,7 @@ help                                 # This message
 				err := fmt.Errorf("User commanded 'panic' on %s, but migrated table is %s; ignoring request.", arg, this.migrationContext.OriginalTableName)
 				return NoPrintStatusRule, err
 			}
-			err := fmt.Errorf("User commanded 'panic'. I will now panic, without cleanup. PANIC!")
+			err := fmt.Errorf("User commanded 'panic'. The migration will be aborted without cleanup. Please drop the gh-ost tables before trying again.")
 			this.migrationContext.PanicAbortOnError(err)
 			return NoPrintStatusRule, err
 		}

localtests/test.sh

@@ -50,6 +50,9 @@ verify_master_and_replica() {
   original_sql_mode="$(gh-ost-test-mysql-master -e "select @@global.sql_mode" -s -s)"
   echo "sql_mode on master is ${original_sql_mode}"
 
+  echo "Gracefully sleeping for 3 seconds while replica is setting up..."
+  sleep 3
+
   if [ "$(gh-ost-test-mysql-replica -e "select 1" -ss)" != "1" ] ; then
     echo "Cannot verify gh-ost-test-mysql-replica"
     exit 1

script/bootstrap

@@ -4,6 +4,7 @@ set -e
 
 # Make sure we have the version of Go we want to depend on, either from the
 # system or one we grab ourselves.
+# If executing from within Dockerfile then this assumption is inherently true, since we use a `golang` docker image.
 . script/ensure-go-installed
 
 # Since we want to be able to build this outside of GOPATH, we set it

script/build-deploy-tarball

@@ -0,0 +1,35 @@
+#!/bin/sh
+
+set -e
+
+script/build
+
+# Get a fresh directory and make sure to delete it afterwards
+build_dir=tmp/build
+rm -rf $build_dir
+mkdir -p $build_dir
+trap "rm -rf $build_dir" EXIT
+
+commit_sha=$(git rev-parse HEAD)
+
+if [ $(uname -s) = "Darwin" ]; then
+    build_arch="$(uname -sr | tr -d ' ' | tr '[:upper:]' '[:lower:]')-$(uname -m)"
+else
+    build_arch="$(lsb_release -sc | tr -d ' ' | tr '[:upper:]' '[:lower:]')-$(uname -m)"
+fi
+
+tarball=$build_dir/${commit_sha}-${build_arch}.tar
+
+# Create the tarball
+tar cvf $tarball --mode="ugo=rx" bin/
+
+# Compress it and copy it to the directory for the CI to upload it
+gzip $tarball
+mkdir -p "$BUILD_ARTIFACT_DIR"/gh-ost
+cp ${tarball}.gz "$BUILD_ARTIFACT_DIR"/gh-ost/
+
+### HACK HACK HACK ###
+# blame @carlosmn and @mattr-
+# Allow builds on stretch to also be used for jessie
+jessie_tarball_name=$(echo $(basename "${tarball}") | sed s/-stretch-/-jessie-/)
+cp ${tarball}.gz "$BUILD_ARTIFACT_DIR/gh-ost/${jessie_tarball_name}.gz"

script/cibuild

@@ -1,17 +1,3 @@
 #!/bin/bash
 
-set -e
+script/test
-
-. script/bootstrap
-
-echo "Verifying code is formatted via 'gofmt -s -w  go/'"
-gofmt -s -w  go/
-git diff --exit-code --quiet
-
-echo "Building"
-script/build
-
-cd .gopath/src/github.com/github/gh-ost
-
-echo "Running unit tests"
-go test ./go/...

script/cibuild-gh-ost-build-deploy-tarball

@@ -1,37 +1,47 @@
-#!/bin/sh
+#!/bin/bash
 
-set -e
+output_fold() {
+  # Exit early if no label provided
+  if [ -z "$1" ]; then
+    echo "output_fold(): requires a label argument."
+    return
+  fi
 
-script/cibuild
+  exit_value=0  # exit_value is used to record exit status of the given command
+  label=$1      # human-readable label describing what's being folded up
+  shift 1       # having retrieved the output_fold()-specific arguments, strip them off $@
 
-# Get a fresh directory and make sure to delete it afterwards
+  # Only echo the tags when in CI_MODE
-build_dir=tmp/build
+  if [ "$CI_MODE" ]; then
-rm -rf $build_dir
+    echo "%%%FOLD {$label}%%%"
-mkdir -p $build_dir
+  fi
-trap "rm -rf $build_dir" EXIT
 
-commit_sha=$(git rev-parse HEAD)
+  # run the remaining arguments. If the command exits non-0, the `||` will
+  # prevent the `-e` flag from seeing the failure exit code, and we'll see
+  # the second echo execute
+  "$@" || exit_value=$?
 
-if [ $(uname -s) = "Darwin" ]; then
+  # Only echo the tags when in CI_MODE
-    build_arch="$(uname -sr | tr -d ' ' | tr '[:upper:]' '[:lower:]')-$(uname -m)"
+  if [ "$CI_MODE" ]; then
-else
+    echo "%%%END FOLD%%%"
-    build_arch="$(lsb_release -sc | tr -d ' ' | tr '[:upper:]' '[:lower:]')-$(uname -m)"
+  fi
-fi
 
-tarball=$build_dir/${commit_sha}-${build_arch}.tar
+  # preserve the exit code from the subcommand.
+  return $exit_value
+}
 
-# Create the tarball
+function cleanup() {
-tar cvf $tarball --mode="ugo=rx" bin/
+  echo
+  echo "%%%FOLD {Shutting down services...}%%%"
+  docker-compose down
+  echo "%%%END FOLD%%%"
+}
 
-# Compress it and copy it to the directory for the CI to upload it
+trap cleanup EXIT
-gzip $tarball
-mkdir -p "$BUILD_ARTIFACT_DIR"/gh-ost
-cp ${tarball}.gz "$BUILD_ARTIFACT_DIR"/gh-ost/
 
-### HACK HACK HACK ###
+export CI_MODE=true
-# Blame @carlosmn. In the good way.
+
-# We don't have any jessie machines for building, but a pure-Go binary depends
+output_fold "Bootstrapping container..." docker-compose build
-# on a version of libc and ld which are widely available, so we can copy the
+output_fold "Running tests..." docker-compose run --rm app
-# tarball over with jessie in its name so we can deploy it on jessie machines.
+
-jessie_tarball_name=$(echo $(basename "${tarball}") | sed s/-precise-/-jessie-/)
+docker-compose run -e BUILD_ARTIFACT_DIR=$BUILD_ARTIFACT_DIR -v $BUILD_ARTIFACT_DIR:$BUILD_ARTIFACT_DIR app script/build-deploy-tarball
-cp ${tarball}.gz "$BUILD_ARTIFACT_DIR/gh-ost/${jessie_tarball_name}.gz"

script/ensure-go-installed

@@ -1,20 +1,20 @@
 #!/bin/bash
 
-PREFERRED_GO_VERSION=go1.9.2
+PREFERRED_GO_VERSION=go1.12.6
-SUPPORTED_GO_VERSIONS='go1.[89]'
+SUPPORTED_GO_VERSIONS='go1.1[234]'
 
 GO_PKG_DARWIN=${PREFERRED_GO_VERSION}.darwin-amd64.pkg
-GO_PKG_DARWIN_SHA=8b4f6ae6deae1150d2e341d02c247fd18a99af387516540eeb84702ffd76d3a1
+GO_PKG_DARWIN_SHA=ea78245e43de2996fa0973033064b33f48820cfe39f4f3c6e953040925cc5815
 
 GO_PKG_LINUX=${PREFERRED_GO_VERSION}.linux-amd64.tar.gz
-GO_PKG_LINUX_SHA=de874549d9a8d8d8062be05808509c09a88a248e77ec14eb77453530829ac02b
+GO_PKG_LINUX_SHA=dbcf71a3c1ea53b8d54ef1b48c85a39a6c9a935d01fc8291ff2b92028e59913c
 
 export ROOTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )"
 cd $ROOTDIR
 
 # If Go isn't installed globally, setup environment variables for local install.
 if [ -z "$(which go)" ] || [ -z "$(go version | grep "$SUPPORTED_GO_VERSIONS")" ]; then
-  GODIR="$ROOTDIR/.vendor/go19"
+  GODIR="$ROOTDIR/.vendor/golocal"
 
   if [ $(uname -s) = "Darwin" ]; then
     export GOROOT="$GODIR/usr/local/go"
@@ -32,12 +32,12 @@ if [ -z "$(which go)" ] || [ -z "$(go version | grep "$SUPPORTED_GO_VERSIONS")"
   cd "$GODIR";
 
   if [ $(uname -s) = "Darwin" ]; then
-    curl -L -O https://storage.googleapis.com/golang/$GO_PKG_DARWIN
+    curl -L -O https://dl.google.com/go/$GO_PKG_DARWIN
     shasum -a256 $GO_PKG_DARWIN | grep $GO_PKG_DARWIN_SHA
     xar -xf $GO_PKG_DARWIN
     cpio -i < com.googlecode.go.pkg/Payload
   else
-    curl -L -O https://storage.googleapis.com/golang/$GO_PKG_LINUX
+    curl -L -O https://dl.google.com/go/$GO_PKG_LINUX
     shasum -a256 $GO_PKG_LINUX | grep $GO_PKG_LINUX_SHA
     tar xf $GO_PKG_LINUX
   fi

script/test

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+set -e
+
+. script/bootstrap
+
+echo "Verifying code is formatted via 'gofmt -s -w  go/'"
+gofmt -s -w  go/
+git diff --exit-code --quiet
+
+echo "Building"
+script/build
+
+cd .gopath/src/github.com/github/gh-ost
+
+echo "Running unit tests"
+go test ./go/...