From 90a38b05c23a10374721f446ee21ebc29ee4c58e Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Wed, 26 Oct 2016 17:00:27 +0200 Subject: [PATCH 1/5] working for automated deployment builds --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..63f0df9 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +/.gopath/ +/bin/ +/libexec/ +/.vendor/ From 02c9b97388b1a925a5f489c455ed5d594e64eba6 Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Wed, 26 Oct 2016 19:55:37 +0200 Subject: [PATCH 2/5] initial work towards automated builds and CI --- script/bootstrap | 16 +++++++ script/build | 21 +++++++++ script/cibuild | 12 +++++ script/cibuild-gh-ost-build-deploy-tarball | 29 ++++++++++++ script/ensure-go-installed | 51 ++++++++++++++++++++++ script/go | 11 +++++ script/godep | 18 ++++++++ script/shim | 22 ++++++++++ 8 files changed, 180 insertions(+) create mode 100755 script/bootstrap create mode 100755 script/build create mode 100755 script/cibuild create mode 100755 script/cibuild-gh-ost-build-deploy-tarball create mode 100755 script/ensure-go-installed create mode 100755 script/go create mode 100755 script/godep create mode 100755 script/shim diff --git a/script/bootstrap b/script/bootstrap new file mode 100755 index 0000000..6ac885b --- /dev/null +++ b/script/bootstrap @@ -0,0 +1,16 @@ +#!/bin/bash + +set -e + +# Make sure we have the version of Go we want to depend on, either from the +# system or one we grab ourselves. +. script/ensure-go-installed + +# Since we want to be able to build this outside of GOPATH, we set it +# up so it points back to us and go is none the wiser + +set -x +rm -rf .gopath +mkdir -p .gopath/src/github.com/github +ln -s "$PWD" .gopath/src/github.com/github/gh-ost +export GOPATH=$PWD/.gopath:$GOPATH diff --git a/script/build b/script/build new file mode 100755 index 0000000..ef04442 --- /dev/null +++ b/script/build @@ -0,0 +1,21 @@ +#!/bin/bash + +set -e + +. script/bootstrap + +mkdir -p bin +bindir="$PWD"/bin +libexecdir="$PWD"/libexec +scriptdir="$PWD"/script + +# We have a few binaries that we want to build, so let's put them into bin/ + +version=$(git rev-parse HEAD) +describe=$(git describe --tags --always --dirty) + +export GOPATH="$PWD/.gopath" +cd .gopath/src/github.com/github/gh-ost + +# We put the binaries into libexec/ and then put the shim with the appropriate name into bin/ +go build -o "$libexecdir/gh-ost" -ldflags "-X main.AppVersion=${version} -X main.BuildDescribe=${describe}" ./go/cmd/gh-ost/main.go diff --git a/script/cibuild b/script/cibuild new file mode 100755 index 0000000..e15c2ce --- /dev/null +++ b/script/cibuild @@ -0,0 +1,12 @@ +#!/bin/bash + +set -e + +. script/bootstrap + +script/build + +export GITBACKUPS_ENV=test + +cd .gopath/src/github.com/github/gh-ost +go test ./go/... diff --git a/script/cibuild-gh-ost-build-deploy-tarball b/script/cibuild-gh-ost-build-deploy-tarball new file mode 100755 index 0000000..b70a207 --- /dev/null +++ b/script/cibuild-gh-ost-build-deploy-tarball @@ -0,0 +1,29 @@ +#!/bin/sh + +set -e + +script/cibuild + +# Get a fresh directory and make sure to delete it afterwards +build_dir=tmp/build +rm -rf $build_dir +mkdir -p $build_dir +trap "rm -rf $build_dir" EXIT + +commit_sha=$(git rev-parse HEAD) + +if [ $(uname -s) = "Darwin" ]; then + build_arch="$(uname -sr | tr -d ' ' | tr '[:upper:]' '[:lower:]')-$(uname -m)" +else + build_arch="$(lsb_release -sc | tr -d ' ' | tr '[:upper:]' '[:lower:]')-$(uname -m)" +fi + +tarball=$build_dir/${commit_sha}-${build_arch}.tar + +# Create the tarball +tar cvf $tarball --mode="ugo=rx" libexec/ bin/ + +# Compress it and copy it to the directory for the CI to upload it +gzip $tarball +mkdir -p "$BUILD_ARTIFACT_DIR"/gh-ost +cp ${tarball}.gz "$BUILD_ARTIFACT_DIR"/gh-ost/ diff --git a/script/ensure-go-installed b/script/ensure-go-installed new file mode 100755 index 0000000..21c49e6 --- /dev/null +++ b/script/ensure-go-installed @@ -0,0 +1,51 @@ +#!/bin/bash + +GO_VERSION=go1.7 + +GO_PKG_DARWIN=${GO_VERSION}.darwin-amd64.pkg +GO_PKG_DARWIN_SHA=e7089843bc7148ffcc147759985b213604d22bb9fd19bd930b515aa981bf1b22 + +GO_PKG_LINUX=${GO_VERSION}.linux-amd64.tar.gz +GO_PKG_LINUX_SHA=702ad90f705365227e902b42d91dd1a40e48ca7f67a2f4b2fd052aaa4295cd95 + +export ROOTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )" +cd $ROOTDIR + +# If Go isn't installed globally, setup environment variables for local install. +if [ -z "$(which go)" ] || [ -z "$(go version | grep $GO_VERSION)" ]; then + GODIR="$ROOTDIR/.vendor/go17" + + if [ $(uname -s) = "Darwin" ]; then + export GOROOT="$GODIR/usr/local/go" + else + export GOROOT="$GODIR/go" + fi + + export PATH="$GOROOT/bin:$PATH" +fi + +# Check if local install exists, and install otherwise. +if [ -z "$(which go)" ] || [ -z "$(go version | grep $GO_VERSION)" ]; then + [ -d "$GODIR" ] && rm -rf $GODIR + mkdir -p "$GODIR" + cd "$GODIR"; + + if [ $(uname -s) = "Darwin" ]; then + curl -L -O https://storage.googleapis.com/golang/$GO_PKG_DARWIN + shasum -a256 $GO_PKG_DARWIN | grep $GO_PKG_DARWIN_SHA + xar -xf $GO_PKG_DARWIN + cpio -i < com.googlecode.go.pkg/Payload + else + curl -L -O https://storage.googleapis.com/golang/$GO_PKG_LINUX + shasum -a256 $GO_PKG_LINUX | grep $GO_PKG_LINUX_SHA + tar xf $GO_PKG_LINUX + fi + + # Prove we did something right + echo "$GO_VERSION installed in $GODIR: Go Binary: $(which go)" +fi + +cd $ROOTDIR + +# Configure the new go to be the first go found +export GOPATH=$ROOTDIR/.vendor diff --git a/script/go b/script/go new file mode 100755 index 0000000..309b591 --- /dev/null +++ b/script/go @@ -0,0 +1,11 @@ +#!/bin/bash + +set -e + +. script/bootstrap + +mkdir -p bin +bindir="$PWD"/bin + +cd .gopath/src/github.com/github/gh-ost +go "$@" diff --git a/script/godep b/script/godep new file mode 100755 index 0000000..b6d37a5 --- /dev/null +++ b/script/godep @@ -0,0 +1,18 @@ +#!/bin/bash + +set -e + +PREVGOPATH=$GOPATH + +. script/bootstrap + +mkdir -p bin +bindir="$PWD"/bin + +# Put the GOPATH for the user before our fake one so we can run `godep save ./...` +if [ -n "$PREVGOPATH" ]; then + export GOPATH=$PREVGOPATH:$GOPATH +fi + +cd .gopath/src/github.com/github/gh-ost +godep "$@" diff --git a/script/shim b/script/shim new file mode 100755 index 0000000..95f3968 --- /dev/null +++ b/script/shim @@ -0,0 +1,22 @@ +#!/bin/bash + +set -e + +# This is a shim to sit in front of the binaries and load the appropriate +# environment variables for the environment set in GITBACKUPS_ENV + +# Default to staging for now +GITBACKUPS_ENV=${GITBACKUPS_ENV:-"staging"} + +# The base directory, which includes bin/ and whatever else once deployed +base=$(cd $(dirname $(dirname ${BASH_SOURCE[0]})) && pwd) + +if [ -f "${base}/.app-config/${GITBACKUPS_ENV}.sh" ]; then + source "${base}/.app-config/${GITBACKUPS_ENV}.sh" +fi + +# Now that we have the env variables to let the binary know where to put the +# data, switch into it +myname=$(basename "$0") + +exec "${base}/libexec/${myname}" "$@" From b76b1c6f97f667655e0fc5ed7cd7dde5d8d808fe Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Thu, 27 Oct 2016 11:58:56 +0200 Subject: [PATCH 3/5] binary to be written directly to bindir --- script/build | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script/build b/script/build index ef04442..d8e8edd 100755 --- a/script/build +++ b/script/build @@ -17,5 +17,5 @@ describe=$(git describe --tags --always --dirty) export GOPATH="$PWD/.gopath" cd .gopath/src/github.com/github/gh-ost -# We put the binaries into libexec/ and then put the shim with the appropriate name into bin/ -go build -o "$libexecdir/gh-ost" -ldflags "-X main.AppVersion=${version} -X main.BuildDescribe=${describe}" ./go/cmd/gh-ost/main.go +# We put the binaries directly into the bindir, because we have no need for shim wrappers +go build -o "$bindir/gh-ost" -ldflags "-X main.AppVersion=${version} -X main.BuildDescribe=${describe}" ./go/cmd/gh-ost/main.go From ef5a535b244f7acfbb0e305686157aff4088942b Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Thu, 27 Oct 2016 12:02:08 +0200 Subject: [PATCH 4/5] removed shim, godep files, which are unused in this setup --- script/godep | 18 ------------------ script/shim | 22 ---------------------- 2 files changed, 40 deletions(-) delete mode 100755 script/godep delete mode 100755 script/shim diff --git a/script/godep b/script/godep deleted file mode 100755 index b6d37a5..0000000 --- a/script/godep +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -set -e - -PREVGOPATH=$GOPATH - -. script/bootstrap - -mkdir -p bin -bindir="$PWD"/bin - -# Put the GOPATH for the user before our fake one so we can run `godep save ./...` -if [ -n "$PREVGOPATH" ]; then - export GOPATH=$PREVGOPATH:$GOPATH -fi - -cd .gopath/src/github.com/github/gh-ost -godep "$@" diff --git a/script/shim b/script/shim deleted file mode 100755 index 95f3968..0000000 --- a/script/shim +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash - -set -e - -# This is a shim to sit in front of the binaries and load the appropriate -# environment variables for the environment set in GITBACKUPS_ENV - -# Default to staging for now -GITBACKUPS_ENV=${GITBACKUPS_ENV:-"staging"} - -# The base directory, which includes bin/ and whatever else once deployed -base=$(cd $(dirname $(dirname ${BASH_SOURCE[0]})) && pwd) - -if [ -f "${base}/.app-config/${GITBACKUPS_ENV}.sh" ]; then - source "${base}/.app-config/${GITBACKUPS_ENV}.sh" -fi - -# Now that we have the env variables to let the binary know where to put the -# data, switch into it -myname=$(basename "$0") - -exec "${base}/libexec/${myname}" "$@" From a97c25e9dff7ad526f20c1405da52e40c75c330b Mon Sep 17 00:00:00 2001 From: Shlomi Noach Date: Thu, 27 Oct 2016 12:03:24 +0200 Subject: [PATCH 5/5] removed libexec references --- script/build | 1 - script/cibuild-gh-ost-build-deploy-tarball | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/script/build b/script/build index d8e8edd..24be5d1 100755 --- a/script/build +++ b/script/build @@ -6,7 +6,6 @@ set -e mkdir -p bin bindir="$PWD"/bin -libexecdir="$PWD"/libexec scriptdir="$PWD"/script # We have a few binaries that we want to build, so let's put them into bin/ diff --git a/script/cibuild-gh-ost-build-deploy-tarball b/script/cibuild-gh-ost-build-deploy-tarball index b70a207..13560cc 100755 --- a/script/cibuild-gh-ost-build-deploy-tarball +++ b/script/cibuild-gh-ost-build-deploy-tarball @@ -21,7 +21,7 @@ fi tarball=$build_dir/${commit_sha}-${build_arch}.tar # Create the tarball -tar cvf $tarball --mode="ugo=rx" libexec/ bin/ +tar cvf $tarball --mode="ugo=rx" bin/ # Compress it and copy it to the directory for the CI to upload it gzip $tarball