From a4c814f9940a561fa8ff41fd4ee68d9086c76a78 Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Tue, 15 Sep 2020 23:18:28 +0530 Subject: [PATCH 1/3] fix: health check use shell script instead of python --- build/common/worker/healthcheck.sh | 27 +++++ build/common/worker/wait-for-it.sh | 183 +++++++++++++++++++++++++++++ build/frappe-worker/Dockerfile | 2 + 3 files changed, 212 insertions(+) create mode 100755 build/common/worker/healthcheck.sh create mode 100755 build/common/worker/wait-for-it.sh diff --git a/build/common/worker/healthcheck.sh b/build/common/worker/healthcheck.sh new file mode 100755 index 00000000..81f5e4e7 --- /dev/null +++ b/build/common/worker/healthcheck.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# Set DB Host and port +export DB_HOST=`cat ./common_site_config.json | awk '/db_host/ { gsub(/[",]/,"",$2); print $2}' | tr -d '\n'` +export DB_PORT=`cat ./common_site_config.json | awk '/db_port/ { gsub(/[",]/,"",$2); print $2}' | tr -d '\n'` +if [[ -z "$DB_PORT" ]]; then + export DB_PORT=3306 +fi + +# Set REDIS host:port +export REDIS_CACHE=`cat ./common_site_config.json | awk '/redis_cache/ { gsub(/[",]/,"",$2); print $2}' | tr -d '\n' | sed 's|redis://||g'` +export REDIS_QUEUE=`cat ./common_site_config.json | awk '/redis_queue/ { gsub(/[",]/,"",$2); print $2}' | tr -d '\n' | sed 's|redis://||g'` +export REDIS_SOCKETIO=`cat ./common_site_config.json | awk '/redis_socketio/ { gsub(/[",]/,"",$2); print $2}' | tr -d '\n' | sed 's|redis://||g'` + +echo "Check $DB_HOST:$DB_PORT" +wait-for-it.sh $DB_HOST:$DB_PORT -t 1 +echo "Check $REDIS_CACHE" +wait-for-it.sh $REDIS_CACHE -t 1 +echo "Check $REDIS_QUEUE" +wait-for-it.sh $REDIS_QUEUE -t 1 +echo "Check $REDIS_SOCKETIO" +wait-for-it.sh $REDIS_SOCKETIO -t 1 + +if [[ "$1" = "-p" ]] || [[ "$1" = "--ping-service" ]]; then + echo "Check $2" + wait-for-it.sh $2 -t 1 +fi diff --git a/build/common/worker/wait-for-it.sh b/build/common/worker/wait-for-it.sh new file mode 100755 index 00000000..64019e8d --- /dev/null +++ b/build/common/worker/wait-for-it.sh @@ -0,0 +1,183 @@ +#!/usr/bin/env bash +# https://github.com/vishnubob/wait-for-it +# Use this script to test if a given TCP host/port are available + +WAITFORIT_cmdname=${0##*/} + +echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi } + +usage() +{ + cat << USAGE >&2 +Usage: + $WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args] + -h HOST | --host=HOST Host or IP under test + -p PORT | --port=PORT TCP port under test + Alternatively, you specify the host and port as host:port + -s | --strict Only execute subcommand if the test succeeds + -q | --quiet Don't output any status messages + -t TIMEOUT | --timeout=TIMEOUT + Timeout in seconds, zero for no timeout + -- COMMAND ARGS Execute command with args after the test finishes +USAGE + exit 1 +} + +wait_for() +{ + if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then + echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" + else + echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout" + fi + WAITFORIT_start_ts=$(date +%s) + while : + do + if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then + nc -z $WAITFORIT_HOST $WAITFORIT_PORT + WAITFORIT_result=$? + else + (echo -n > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1 + WAITFORIT_result=$? + fi + if [[ $WAITFORIT_result -eq 0 ]]; then + WAITFORIT_end_ts=$(date +%s) + echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds" + break + fi + sleep 1 + done + return $WAITFORIT_result +} + +wait_for_wrapper() +{ + # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692 + if [[ $WAITFORIT_QUIET -eq 1 ]]; then + timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & + else + timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & + fi + WAITFORIT_PID=$! + trap "kill -INT -$WAITFORIT_PID" INT + wait $WAITFORIT_PID + WAITFORIT_RESULT=$? + if [[ $WAITFORIT_RESULT -ne 0 ]]; then + echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" + fi + return $WAITFORIT_RESULT +} + +# process arguments +while [[ $# -gt 0 ]] +do + case "$1" in + *:* ) + WAITFORIT_hostport=(${1//:/ }) + WAITFORIT_HOST=${WAITFORIT_hostport[0]} + WAITFORIT_PORT=${WAITFORIT_hostport[1]} + shift 1 + ;; + --child) + WAITFORIT_CHILD=1 + shift 1 + ;; + -q | --quiet) + WAITFORIT_QUIET=1 + shift 1 + ;; + -s | --strict) + WAITFORIT_STRICT=1 + shift 1 + ;; + -h) + WAITFORIT_HOST="$2" + if [[ $WAITFORIT_HOST == "" ]]; then break; fi + shift 2 + ;; + --host=*) + WAITFORIT_HOST="${1#*=}" + shift 1 + ;; + -p) + WAITFORIT_PORT="$2" + if [[ $WAITFORIT_PORT == "" ]]; then break; fi + shift 2 + ;; + --port=*) + WAITFORIT_PORT="${1#*=}" + shift 1 + ;; + -t) + WAITFORIT_TIMEOUT="$2" + if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi + shift 2 + ;; + --timeout=*) + WAITFORIT_TIMEOUT="${1#*=}" + shift 1 + ;; + --) + shift + WAITFORIT_CLI=("$@") + break + ;; + --help) + usage + ;; + *) + echoerr "Unknown argument: $1" + usage + ;; + esac +done + +if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then + echoerr "Error: you need to provide a host and port to test." + usage +fi + +WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15} +WAITFORIT_STRICT=${WAITFORIT_STRICT:-0} +WAITFORIT_CHILD=${WAITFORIT_CHILD:-0} +WAITFORIT_QUIET=${WAITFORIT_QUIET:-0} + +# Check to see if timeout is from busybox? +WAITFORIT_TIMEOUT_PATH=$(type -p timeout) +WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH) + +WAITFORIT_BUSYTIMEFLAG="" +if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then + WAITFORIT_ISBUSY=1 + # Check if busybox timeout uses -t flag + # (recent Alpine versions don't support -t anymore) + if timeout &>/dev/stdout | grep -q -e '-t '; then + WAITFORIT_BUSYTIMEFLAG="-t" + fi +else + WAITFORIT_ISBUSY=0 +fi + +if [[ $WAITFORIT_CHILD -gt 0 ]]; then + wait_for + WAITFORIT_RESULT=$? + exit $WAITFORIT_RESULT +else + if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then + wait_for_wrapper + WAITFORIT_RESULT=$? + else + wait_for + WAITFORIT_RESULT=$? + fi +fi + +if [[ $WAITFORIT_CLI != "" ]]; then + if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then + echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess" + exit $WAITFORIT_RESULT + fi + exec "${WAITFORIT_CLI[@]}" +else + exit $WAITFORIT_RESULT +fi diff --git a/build/frappe-worker/Dockerfile b/build/frappe-worker/Dockerfile index 27d77cfc..cff8d8e9 100644 --- a/build/frappe-worker/Dockerfile +++ b/build/frappe-worker/Dockerfile @@ -58,6 +58,8 @@ COPY build/common/commands/* /home/frappe/frappe-bench/commands/ COPY build/common/common_site_config.json.template /opt/frappe/common_site_config.json.template COPY build/common/worker/install_app.sh /usr/local/bin/install_app COPY build/common/worker/bench /usr/local/bin/bench +COPY build/common/worker/wait-for-it.sh /usr/local/bin/wait-for-it.sh +COPY build/common/worker/healthcheck.sh /usr/local/bin/healthcheck.sh # Setup docker-entrypoint COPY build/common/worker/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh From c43e9734819007ea8944dd97d3bdb759dbb1dcac Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Wed, 16 Sep 2020 05:06:07 +0530 Subject: [PATCH 2/3] fix(frappe-worker): install wait-for-it deb package --- build/common/worker/healthcheck.sh | 10 +- build/common/worker/wait-for-it.sh | 183 ----------------------------- build/frappe-worker/Dockerfile | 2 +- 3 files changed, 6 insertions(+), 189 deletions(-) delete mode 100755 build/common/worker/wait-for-it.sh diff --git a/build/common/worker/healthcheck.sh b/build/common/worker/healthcheck.sh index 81f5e4e7..32192f93 100755 --- a/build/common/worker/healthcheck.sh +++ b/build/common/worker/healthcheck.sh @@ -13,15 +13,15 @@ export REDIS_QUEUE=`cat ./common_site_config.json | awk '/redis_queue/ { gsub(/[ export REDIS_SOCKETIO=`cat ./common_site_config.json | awk '/redis_socketio/ { gsub(/[",]/,"",$2); print $2}' | tr -d '\n' | sed 's|redis://||g'` echo "Check $DB_HOST:$DB_PORT" -wait-for-it.sh $DB_HOST:$DB_PORT -t 1 +wait-for-it $DB_HOST:$DB_PORT -t 1 echo "Check $REDIS_CACHE" -wait-for-it.sh $REDIS_CACHE -t 1 +wait-for-it $REDIS_CACHE -t 1 echo "Check $REDIS_QUEUE" -wait-for-it.sh $REDIS_QUEUE -t 1 +wait-for-it $REDIS_QUEUE -t 1 echo "Check $REDIS_SOCKETIO" -wait-for-it.sh $REDIS_SOCKETIO -t 1 +wait-for-it $REDIS_SOCKETIO -t 1 if [[ "$1" = "-p" ]] || [[ "$1" = "--ping-service" ]]; then echo "Check $2" - wait-for-it.sh $2 -t 1 + wait-for-it $2 -t 1 fi diff --git a/build/common/worker/wait-for-it.sh b/build/common/worker/wait-for-it.sh deleted file mode 100755 index 64019e8d..00000000 --- a/build/common/worker/wait-for-it.sh +++ /dev/null @@ -1,183 +0,0 @@ -#!/usr/bin/env bash -# https://github.com/vishnubob/wait-for-it -# Use this script to test if a given TCP host/port are available - -WAITFORIT_cmdname=${0##*/} - -echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi } - -usage() -{ - cat << USAGE >&2 -Usage: - $WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args] - -h HOST | --host=HOST Host or IP under test - -p PORT | --port=PORT TCP port under test - Alternatively, you specify the host and port as host:port - -s | --strict Only execute subcommand if the test succeeds - -q | --quiet Don't output any status messages - -t TIMEOUT | --timeout=TIMEOUT - Timeout in seconds, zero for no timeout - -- COMMAND ARGS Execute command with args after the test finishes -USAGE - exit 1 -} - -wait_for() -{ - if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then - echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" - else - echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout" - fi - WAITFORIT_start_ts=$(date +%s) - while : - do - if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then - nc -z $WAITFORIT_HOST $WAITFORIT_PORT - WAITFORIT_result=$? - else - (echo -n > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1 - WAITFORIT_result=$? - fi - if [[ $WAITFORIT_result -eq 0 ]]; then - WAITFORIT_end_ts=$(date +%s) - echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds" - break - fi - sleep 1 - done - return $WAITFORIT_result -} - -wait_for_wrapper() -{ - # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692 - if [[ $WAITFORIT_QUIET -eq 1 ]]; then - timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & - else - timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & - fi - WAITFORIT_PID=$! - trap "kill -INT -$WAITFORIT_PID" INT - wait $WAITFORIT_PID - WAITFORIT_RESULT=$? - if [[ $WAITFORIT_RESULT -ne 0 ]]; then - echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" - fi - return $WAITFORIT_RESULT -} - -# process arguments -while [[ $# -gt 0 ]] -do - case "$1" in - *:* ) - WAITFORIT_hostport=(${1//:/ }) - WAITFORIT_HOST=${WAITFORIT_hostport[0]} - WAITFORIT_PORT=${WAITFORIT_hostport[1]} - shift 1 - ;; - --child) - WAITFORIT_CHILD=1 - shift 1 - ;; - -q | --quiet) - WAITFORIT_QUIET=1 - shift 1 - ;; - -s | --strict) - WAITFORIT_STRICT=1 - shift 1 - ;; - -h) - WAITFORIT_HOST="$2" - if [[ $WAITFORIT_HOST == "" ]]; then break; fi - shift 2 - ;; - --host=*) - WAITFORIT_HOST="${1#*=}" - shift 1 - ;; - -p) - WAITFORIT_PORT="$2" - if [[ $WAITFORIT_PORT == "" ]]; then break; fi - shift 2 - ;; - --port=*) - WAITFORIT_PORT="${1#*=}" - shift 1 - ;; - -t) - WAITFORIT_TIMEOUT="$2" - if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi - shift 2 - ;; - --timeout=*) - WAITFORIT_TIMEOUT="${1#*=}" - shift 1 - ;; - --) - shift - WAITFORIT_CLI=("$@") - break - ;; - --help) - usage - ;; - *) - echoerr "Unknown argument: $1" - usage - ;; - esac -done - -if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then - echoerr "Error: you need to provide a host and port to test." - usage -fi - -WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15} -WAITFORIT_STRICT=${WAITFORIT_STRICT:-0} -WAITFORIT_CHILD=${WAITFORIT_CHILD:-0} -WAITFORIT_QUIET=${WAITFORIT_QUIET:-0} - -# Check to see if timeout is from busybox? -WAITFORIT_TIMEOUT_PATH=$(type -p timeout) -WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH) - -WAITFORIT_BUSYTIMEFLAG="" -if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then - WAITFORIT_ISBUSY=1 - # Check if busybox timeout uses -t flag - # (recent Alpine versions don't support -t anymore) - if timeout &>/dev/stdout | grep -q -e '-t '; then - WAITFORIT_BUSYTIMEFLAG="-t" - fi -else - WAITFORIT_ISBUSY=0 -fi - -if [[ $WAITFORIT_CHILD -gt 0 ]]; then - wait_for - WAITFORIT_RESULT=$? - exit $WAITFORIT_RESULT -else - if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then - wait_for_wrapper - WAITFORIT_RESULT=$? - else - wait_for - WAITFORIT_RESULT=$? - fi -fi - -if [[ $WAITFORIT_CLI != "" ]]; then - if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then - echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess" - exit $WAITFORIT_RESULT - fi - exec "${WAITFORIT_CLI[@]}" -else - exit $WAITFORIT_RESULT -fi diff --git a/build/frappe-worker/Dockerfile b/build/frappe-worker/Dockerfile index cff8d8e9..948836cf 100644 --- a/build/frappe-worker/Dockerfile +++ b/build/frappe-worker/Dockerfile @@ -28,6 +28,7 @@ RUN install_packages \ xfonts-base \ # For psycopg2 libpq-dev \ + wait-for-it \ && wget https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.buster_amd64.deb \ && dpkg -i wkhtmltox_0.12.5-1.buster_amd64.deb && rm wkhtmltox_0.12.5-1.buster_amd64.deb \ && wget https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.3/install.sh \ @@ -58,7 +59,6 @@ COPY build/common/commands/* /home/frappe/frappe-bench/commands/ COPY build/common/common_site_config.json.template /opt/frappe/common_site_config.json.template COPY build/common/worker/install_app.sh /usr/local/bin/install_app COPY build/common/worker/bench /usr/local/bin/bench -COPY build/common/worker/wait-for-it.sh /usr/local/bin/wait-for-it.sh COPY build/common/worker/healthcheck.sh /usr/local/bin/healthcheck.sh # Setup docker-entrypoint From e6033719843f38cecb9d16341b15d19e548f7eea Mon Sep 17 00:00:00 2001 From: Revant Nandgaonkar Date: Wed, 16 Sep 2020 14:20:10 +0530 Subject: [PATCH 3/3] fix(frappe-worker): set absolute path of common_site_config.json in healthcheck.sh --- build/common/worker/healthcheck.sh | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/build/common/worker/healthcheck.sh b/build/common/worker/healthcheck.sh index 32192f93..f64d1a49 100755 --- a/build/common/worker/healthcheck.sh +++ b/build/common/worker/healthcheck.sh @@ -1,16 +1,18 @@ #!/bin/bash +export COMMON_SITE_CONFIG_JSON='/home/frappe/frappe-bench/sites/common_site_config.json' + # Set DB Host and port -export DB_HOST=`cat ./common_site_config.json | awk '/db_host/ { gsub(/[",]/,"",$2); print $2}' | tr -d '\n'` -export DB_PORT=`cat ./common_site_config.json | awk '/db_port/ { gsub(/[",]/,"",$2); print $2}' | tr -d '\n'` +export DB_HOST=`cat $COMMON_SITE_CONFIG_JSON | awk '/db_host/ { gsub(/[",]/,"",$2); print $2}' | tr -d '\n'` +export DB_PORT=`cat $COMMON_SITE_CONFIG_JSON | awk '/db_port/ { gsub(/[",]/,"",$2); print $2}' | tr -d '\n'` if [[ -z "$DB_PORT" ]]; then export DB_PORT=3306 fi # Set REDIS host:port -export REDIS_CACHE=`cat ./common_site_config.json | awk '/redis_cache/ { gsub(/[",]/,"",$2); print $2}' | tr -d '\n' | sed 's|redis://||g'` -export REDIS_QUEUE=`cat ./common_site_config.json | awk '/redis_queue/ { gsub(/[",]/,"",$2); print $2}' | tr -d '\n' | sed 's|redis://||g'` -export REDIS_SOCKETIO=`cat ./common_site_config.json | awk '/redis_socketio/ { gsub(/[",]/,"",$2); print $2}' | tr -d '\n' | sed 's|redis://||g'` +export REDIS_CACHE=`cat $COMMON_SITE_CONFIG_JSON | awk '/redis_cache/ { gsub(/[",]/,"",$2); print $2}' | tr -d '\n' | sed 's|redis://||g'` +export REDIS_QUEUE=`cat $COMMON_SITE_CONFIG_JSON | awk '/redis_queue/ { gsub(/[",]/,"",$2); print $2}' | tr -d '\n' | sed 's|redis://||g'` +export REDIS_SOCKETIO=`cat $COMMON_SITE_CONFIG_JSON | awk '/redis_socketio/ { gsub(/[",]/,"",$2); print $2}' | tr -d '\n' | sed 's|redis://||g'` echo "Check $DB_HOST:$DB_PORT" wait-for-it $DB_HOST:$DB_PORT -t 1