Added test by a shell script static analysis tool(ShellCheck)

This commit is contained in:
Takeshi Nakatani 2022-01-15 17:08:46 +00:00 committed by Andrew Gaul
parent 93d1c30d4d
commit 46014397d8
12 changed files with 876 additions and 738 deletions

View File

@ -101,6 +101,10 @@ jobs:
run: | run: |
make cppcheck make cppcheck
- name: Shellcheck
run: |
make shellcheck
- name: Test suite - name: Test suite
run: | run: |
make ALL_TESTS=1 check -C test || (test/filter-suite-log.sh test/test-suite.log; exit 1) make ALL_TESTS=1 check -C test || (test/filter-suite-log.sh test/test-suite.log; exit 1)
@ -131,7 +135,7 @@ jobs:
- name: Install brew other packages - name: Install brew other packages
run: | run: |
S3FS_BREW_PACKAGES='automake cppcheck python3 coreutils gnu-sed'; S3FS_BREW_PACKAGES='automake cppcheck python3 coreutils gnu-sed shellcheck';
for s3fs_brew_pkg in ${S3FS_BREW_PACKAGES}; do if brew list | grep -q ${s3fs_brew_pkg}; then if brew outdated | grep -q ${s3fs_brew_pkg}; then HOMEBREW_NO_AUTO_UPDATE=1 brew upgrade ${s3fs_brew_pkg}; fi; else HOMEBREW_NO_AUTO_UPDATE=1 brew install ${s3fs_brew_pkg}; fi; done; for s3fs_brew_pkg in ${S3FS_BREW_PACKAGES}; do if brew list | grep -q ${s3fs_brew_pkg}; then if brew outdated | grep -q ${s3fs_brew_pkg}; then HOMEBREW_NO_AUTO_UPDATE=1 brew upgrade ${s3fs_brew_pkg}; fi; else HOMEBREW_NO_AUTO_UPDATE=1 brew install ${s3fs_brew_pkg}; fi; done;
- name: Install awscli - name: Install awscli
@ -152,6 +156,10 @@ jobs:
run: | run: |
make cppcheck make cppcheck
- name: Shellcheck
run: |
make shellcheck
- name: Test suite - name: Test suite
run: | run: |
make check -C src make check -C src

View File

@ -24,7 +24,7 @@ echo "${PRGNAME} [INFO] Start Linux helper for installing packages."
#----------------------------------------------------------- #-----------------------------------------------------------
# Common variables # Common variables
#----------------------------------------------------------- #-----------------------------------------------------------
PRGNAME=`basename $0` PRGNAME=$(basename "$0")
#----------------------------------------------------------- #-----------------------------------------------------------
# Parameter check # Parameter check
@ -40,8 +40,10 @@ fi
# Container OS variables # Container OS variables
#----------------------------------------------------------- #-----------------------------------------------------------
CONTAINER_FULLNAME=$1 CONTAINER_FULLNAME=$1
CONTAINER_OSNAME=`echo ${CONTAINER_FULLNAME} | sed 's/:/ /g' | awk '{print $1}'` # shellcheck disable=SC2034
CONTAINER_OSVERSION=`echo ${CONTAINER_FULLNAME} | sed 's/:/ /g' | awk '{print $2}'` CONTAINER_OSNAME=$(echo "${CONTAINER_FULLNAME}" | sed 's/:/ /g' | awk '{print $1}')
# shellcheck disable=SC2034
CONTAINER_OSVERSION=$(echo "${CONTAINER_FULLNAME}" | sed 's/:/ /g' | awk '{print $2}')
#----------------------------------------------------------- #-----------------------------------------------------------
# Common variables for pip # Common variables for pip
@ -53,6 +55,7 @@ INSTALL_AWSCLI_PACKAGES="awscli"
#----------------------------------------------------------- #-----------------------------------------------------------
# Parameters for configure(set environments) # Parameters for configure(set environments)
#----------------------------------------------------------- #-----------------------------------------------------------
# shellcheck disable=SC2089
CONFIGURE_OPTIONS="CXXFLAGS='-O -std=c++03 -DS3FS_PTHREAD_ERRORCHECK=1' --prefix=/usr --with-openssl" CONFIGURE_OPTIONS="CXXFLAGS='-O -std=c++03 -DS3FS_PTHREAD_ERRORCHECK=1' --prefix=/usr --with-openssl"
#----------------------------------------------------------- #-----------------------------------------------------------
@ -63,63 +66,81 @@ if [ "${CONTAINER_FULLNAME}" = "ubuntu:21.10" ]; then
PACKAGE_UPDATE_OPTIONS="update -y -qq" PACKAGE_UPDATE_OPTIONS="update -y -qq"
INSTALL_PACKAGES="autoconf autotools-dev default-jre-headless fuse libfuse-dev libcurl4-openssl-dev libxml2-dev locales-all mime-support libtool pkg-config libssl-dev attr curl python3-pip" INSTALL_PACKAGES="autoconf autotools-dev default-jre-headless fuse libfuse-dev libcurl4-openssl-dev libxml2-dev locales-all mime-support libtool pkg-config libssl-dev attr curl python3-pip"
INSTALL_CPPCHECK_OPTIONS="" INSTALL_CHECKER_PKGS="cppcheck shellcheck"
INSTALL_CHECKER_PKG_OPTIONS=""
elif [ "${CONTAINER_FULLNAME}" = "ubuntu:20.04" ]; then elif [ "${CONTAINER_FULLNAME}" = "ubuntu:20.04" ]; then
PACKAGE_MANAGER_BIN="apt-get" PACKAGE_MANAGER_BIN="apt-get"
PACKAGE_UPDATE_OPTIONS="update -y -qq" PACKAGE_UPDATE_OPTIONS="update -y -qq"
INSTALL_PACKAGES="autoconf autotools-dev default-jre-headless fuse libfuse-dev libcurl4-openssl-dev libxml2-dev locales-all mime-support libtool pkg-config libssl-dev attr curl python3-pip" INSTALL_PACKAGES="autoconf autotools-dev default-jre-headless fuse libfuse-dev libcurl4-openssl-dev libxml2-dev locales-all mime-support libtool pkg-config libssl-dev attr curl python3-pip"
INSTALL_CPPCHECK_OPTIONS="" INSTALL_CHECKER_PKGS="cppcheck shellcheck"
INSTALL_CHECKER_PKG_OPTIONS=""
elif [ "${CONTAINER_FULLNAME}" = "ubuntu:18.04" ]; then elif [ "${CONTAINER_FULLNAME}" = "ubuntu:18.04" ]; then
PACKAGE_MANAGER_BIN="apt-get" PACKAGE_MANAGER_BIN="apt-get"
PACKAGE_UPDATE_OPTIONS="update -y -qq" PACKAGE_UPDATE_OPTIONS="update -y -qq"
INSTALL_PACKAGES="autoconf autotools-dev default-jre-headless fuse libfuse-dev libcurl4-openssl-dev libxml2-dev locales-all mime-support libtool pkg-config libssl-dev attr curl python3-pip" INSTALL_PACKAGES="autoconf autotools-dev default-jre-headless fuse libfuse-dev libcurl4-openssl-dev libxml2-dev locales-all mime-support libtool pkg-config libssl-dev attr curl python3-pip"
INSTALL_CPPCHECK_OPTIONS="" INSTALL_CHECKER_PKGS="cppcheck shellcheck"
INSTALL_CHECKER_PKG_OPTIONS=""
elif [ "${CONTAINER_FULLNAME}" = "ubuntu:16.04" ]; then elif [ "${CONTAINER_FULLNAME}" = "ubuntu:16.04" ]; then
PACKAGE_MANAGER_BIN="apt-get" PACKAGE_MANAGER_BIN="apt-get"
PACKAGE_UPDATE_OPTIONS="update -y -qq" PACKAGE_UPDATE_OPTIONS="update -y -qq"
INSTALL_PACKAGES="autoconf autotools-dev default-jre-headless fuse libfuse-dev libcurl4-openssl-dev libxml2-dev locales-all mime-support libtool pkg-config libssl-dev attr curl python3-pip" INSTALL_PACKAGES="autoconf autotools-dev default-jre-headless fuse libfuse-dev libcurl4-openssl-dev libxml2-dev locales-all mime-support libtool pkg-config libssl-dev attr curl python3-pip"
INSTALL_CPPCHECK_OPTIONS="" INSTALL_CHECKER_PKGS="cppcheck shellcheck"
INSTALL_CHECKER_PKG_OPTIONS=""
elif [ "${CONTAINER_FULLNAME}" = "debian:bullseye" ]; then elif [ "${CONTAINER_FULLNAME}" = "debian:bullseye" ]; then
PACKAGE_MANAGER_BIN="apt-get" PACKAGE_MANAGER_BIN="apt-get"
PACKAGE_UPDATE_OPTIONS="update -y -qq" PACKAGE_UPDATE_OPTIONS="update -y -qq"
INSTALL_PACKAGES="autoconf autotools-dev default-jre-headless fuse libfuse-dev libcurl4-openssl-dev libxml2-dev locales-all mime-support libtool pkg-config libssl-dev attr curl procps python3-pip" INSTALL_PACKAGES="autoconf autotools-dev default-jre-headless fuse libfuse-dev libcurl4-openssl-dev libxml2-dev locales-all mime-support libtool pkg-config libssl-dev attr curl procps python3-pip"
INSTALL_CPPCHECK_OPTIONS="" INSTALL_CHECKER_PKGS="cppcheck shellcheck"
INSTALL_CHECKER_PKG_OPTIONS=""
elif [ "${CONTAINER_FULLNAME}" = "debian:buster" ]; then elif [ "${CONTAINER_FULLNAME}" = "debian:buster" ]; then
PACKAGE_MANAGER_BIN="apt-get" PACKAGE_MANAGER_BIN="apt-get"
PACKAGE_UPDATE_OPTIONS="update -y -qq" PACKAGE_UPDATE_OPTIONS="update -y -qq"
INSTALL_PACKAGES="autoconf autotools-dev default-jre-headless fuse libfuse-dev libcurl4-openssl-dev libxml2-dev locales-all mime-support libtool pkg-config libssl-dev attr curl procps python3-pip" INSTALL_PACKAGES="autoconf autotools-dev default-jre-headless fuse libfuse-dev libcurl4-openssl-dev libxml2-dev locales-all mime-support libtool pkg-config libssl-dev attr curl procps python3-pip"
INSTALL_CPPCHECK_OPTIONS="" INSTALL_CHECKER_PKGS="cppcheck shellcheck"
INSTALL_CHECKER_PKG_OPTIONS=""
elif [ "${CONTAINER_FULLNAME}" = "debian:stretch" ]; then elif [ "${CONTAINER_FULLNAME}" = "debian:stretch" ]; then
PACKAGE_MANAGER_BIN="apt-get" PACKAGE_MANAGER_BIN="apt-get"
PACKAGE_UPDATE_OPTIONS="update -y -qq" PACKAGE_UPDATE_OPTIONS="update -y -qq"
INSTALL_PACKAGES="autoconf autotools-dev default-jre-headless fuse libfuse-dev libcurl4-openssl-dev libxml2-dev locales-all mime-support libtool pkg-config libssl-dev attr curl procps python3-pip" INSTALL_PACKAGES="autoconf autotools-dev default-jre-headless fuse libfuse-dev libcurl4-openssl-dev libxml2-dev locales-all mime-support libtool pkg-config libssl-dev attr curl procps python3-pip"
INSTALL_CPPCHECK_OPTIONS="" INSTALL_CHECKER_PKGS="cppcheck shellcheck"
INSTALL_CHECKER_PKG_OPTIONS=""
elif [ "${CONTAINER_FULLNAME}" = "centos:centos8" ]; then elif [ "${CONTAINER_FULLNAME}" = "centos:centos8" ]; then
PACKAGE_MANAGER_BIN="dnf" PACKAGE_MANAGER_BIN="dnf"
PACKAGE_UPDATE_OPTIONS="update -y -qq" PACKAGE_UPDATE_OPTIONS="update -y -qq"
# [NOTE]
# Installing ShellCheck on CentOS 8 is not easy.
# Give up to run ShellCheck on CentOS 8 as we don't have to run ShellChek on all operating systems.
#
INSTALL_PACKAGES="curl-devel fuse fuse-devel gcc libstdc++-devel gcc-c++ glibc-langpack-en java-11-openjdk-headless libxml2-devel mailcap git automake make openssl-devel attr diffutils curl python3" INSTALL_PACKAGES="curl-devel fuse fuse-devel gcc libstdc++-devel gcc-c++ glibc-langpack-en java-11-openjdk-headless libxml2-devel mailcap git automake make openssl-devel attr diffutils curl python3"
INSTALL_CPPCHECK_OPTIONS="--enablerepo=powertools" INSTALL_CHECKER_PKGS="cppcheck"
INSTALL_CHECKER_PKG_OPTIONS="--enablerepo=powertools"
elif [ "${CONTAINER_FULLNAME}" = "centos:centos7" ]; then elif [ "${CONTAINER_FULLNAME}" = "centos:centos7" ]; then
PACKAGE_MANAGER_BIN="yum" PACKAGE_MANAGER_BIN="yum"
PACKAGE_UPDATE_OPTIONS="update -y" PACKAGE_UPDATE_OPTIONS="update -y"
# [NOTE]
# ShellCheck version(0.3.8) is too low to check.
# And in this version, it cannot be passed due to following error.
# "shellcheck: ./test/integration-test-main.sh: hGetContents: invalid argument (invalid byte sequence)"
#
INSTALL_PACKAGES="curl-devel fuse fuse-devel gcc libstdc++-devel gcc-c++ glibc-langpack-en java-11-openjdk-headless libxml2-devel mailcap git automake make openssl-devel attr curl python3 epel-release" INSTALL_PACKAGES="curl-devel fuse fuse-devel gcc libstdc++-devel gcc-c++ glibc-langpack-en java-11-openjdk-headless libxml2-devel mailcap git automake make openssl-devel attr curl python3 epel-release"
INSTALL_CPPCHECK_OPTIONS="--enablerepo=epel" INSTALL_CHECKER_PKGS="cppcheck"
INSTALL_CHECKER_PKG_OPTIONS="--enablerepo=epel"
elif [ "${CONTAINER_FULLNAME}" = "fedora:35" ]; then elif [ "${CONTAINER_FULLNAME}" = "fedora:35" ]; then
PACKAGE_MANAGER_BIN="dnf" PACKAGE_MANAGER_BIN="dnf"
@ -127,14 +148,16 @@ elif [ "${CONTAINER_FULLNAME}" = "fedora:35" ]; then
# TODO: Cannot use java-latest-openjdk (17) due to modules issue in S3Proxy/jclouds/Guice # TODO: Cannot use java-latest-openjdk (17) due to modules issue in S3Proxy/jclouds/Guice
INSTALL_PACKAGES="curl-devel fuse fuse-devel gcc libstdc++-devel gcc-c++ glibc-langpack-en java-11-openjdk-headless libxml2-devel mailcap git automake make openssl-devel curl attr diffutils procps python3-pip" INSTALL_PACKAGES="curl-devel fuse fuse-devel gcc libstdc++-devel gcc-c++ glibc-langpack-en java-11-openjdk-headless libxml2-devel mailcap git automake make openssl-devel curl attr diffutils procps python3-pip"
INSTALL_CPPCHECK_OPTIONS="" INSTALL_CHECKER_PKGS="cppcheck ShellCheck"
INSTALL_CHECKER_PKG_OPTIONS=""
elif [ "${CONTAINER_FULLNAME}" = "opensuse/leap:15" ]; then elif [ "${CONTAINER_FULLNAME}" = "opensuse/leap:15" ]; then
PACKAGE_MANAGER_BIN="zypper" PACKAGE_MANAGER_BIN="zypper"
PACKAGE_UPDATE_OPTIONS="refresh" PACKAGE_UPDATE_OPTIONS="refresh"
INSTALL_PACKAGES="automake curl-devel fuse fuse-devel gcc-c++ java-11-openjdk-headless libxml2-devel make openssl-devel python3-pip curl attr" INSTALL_PACKAGES="automake curl-devel fuse fuse-devel gcc-c++ java-11-openjdk-headless libxml2-devel make openssl-devel python3-pip curl attr ShellCheck"
INSTALL_CPPCHECK_OPTIONS="" INSTALL_CHECKER_PKGS="cppcheck ShellCheck"
INSTALL_CHECKER_PKG_OPTIONS=""
else else
echo "No container configured for: ${CONTAINER_FULLNAME}" echo "No container configured for: ${CONTAINER_FULLNAME}"
@ -148,16 +171,16 @@ fi
# Update packages (ex. apt-get update -y -qq) # Update packages (ex. apt-get update -y -qq)
# #
echo "${PRGNAME} [INFO] Updates." echo "${PRGNAME} [INFO] Updates."
${PACKAGE_MANAGER_BIN} ${PACKAGE_UPDATE_OPTIONS} /bin/sh -c "${PACKAGE_MANAGER_BIN} ${PACKAGE_UPDATE_OPTIONS}"
# #
# Install packages ( with cppcheck ) # Install packages ( with cppcheck )
# #
echo "${PRGNAME} [INFO] Install packages." echo "${PRGNAME} [INFO] Install packages."
${PACKAGE_MANAGER_BIN} install -y ${INSTALL_PACKAGES} /bin/sh -c "${PACKAGE_MANAGER_BIN} install -y ${INSTALL_PACKAGES}"
echo "${PRGNAME} [INFO] Install cppcheck package." echo "${PRGNAME} [INFO] Install cppcheck package."
${PACKAGE_MANAGER_BIN} ${INSTALL_CPPCHECK_OPTIONS} install -y cppcheck /bin/sh -c "${PACKAGE_MANAGER_BIN} ${INSTALL_CHECKER_PKG_OPTIONS} install -y ${INSTALL_CHECKER_PKGS}"
# Check Java version # Check Java version
java -version java -version
@ -166,16 +189,19 @@ java -version
# Install awscli # Install awscli
# #
echo "${PRGNAME} [INFO] Install awscli package." echo "${PRGNAME} [INFO] Install awscli package."
${PIP_BIN} install ${PIP_OPTIONS} ${INSTALL_AWSCLI_PACKAGES} /bin/sh -c "${PIP_BIN} install ${PIP_OPTIONS} ${INSTALL_AWSCLI_PACKAGES}"
${PIP_BIN} install ${PIP_OPTIONS} rsa /bin/sh -c "${PIP_BIN} install ${PIP_OPTIONS} rsa"
#----------------------------------------------------------- #-----------------------------------------------------------
# Set environment for configure # Set environment for configure
#----------------------------------------------------------- #-----------------------------------------------------------
echo "${PRGNAME} [INFO] Set environment for configure options" echo "${PRGNAME} [INFO] Set environment for configure options"
# shellcheck disable=SC2090
export CONFIGURE_OPTIONS export CONFIGURE_OPTIONS
echo "${PRGNAME} [INFO] Finish Linux helper for installing packages." echo "${PRGNAME} [INFO] Finish Linux helper for installing packages."
exit 0 exit 0
# #

View File

@ -17,6 +17,7 @@
# along with this program; if not, write to the Free Software # along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
###################################################################### ######################################################################
SUBDIRS=src test doc SUBDIRS=src test doc
EXTRA_DIST=doc default_commit_hash EXTRA_DIST=doc default_commit_hash
@ -28,6 +29,8 @@ dist-hook:
release : dist ../utils/release.sh release : dist ../utils/release.sh
../utils/release.sh $(DIST_ARCHIVES) ../utils/release.sh $(DIST_ARCHIVES)
.PHONY: cppcheck shellcheck
cppcheck: cppcheck:
cppcheck --quiet --error-exitcode=1 \ cppcheck --quiet --error-exitcode=1 \
--inline-suppr \ --inline-suppr \
@ -43,6 +46,35 @@ cppcheck:
--suppress=unmatchedSuppression \ --suppress=unmatchedSuppression \
src/ test/ src/ test/
#
# ShellCheck
#
SHELLCHECK_CMD = shellcheck
SHELLCHECK_SH_OPT = --shell=sh
SHELLCHECK_BASH_OPT = --shell=bash
# [NOTE]
# To control error warnings as a whole, specify the "SC<number>" with the following variables.
#
SHELLCHECK_COMMON_IGN = --exclude=SC1091
SHELLCHECK_CUSTOM_IGN = --exclude=SC1091
shellcheck:
@if type shellcheck > /dev/null 2>&1; then \
echo "* ShellCheck version"; \
$(SHELLCHECK_CMD) --version; \
echo ""; \
echo "* Check all sh files with ShellCheck"; \
LC_ALL=C.UTF-8 $(SHELLCHECK_CMD) $(SHELLCHECK_SH_OPT) $(SHELLCHECK_COMMON_IGN) $$(grep '#![[:space:]]*/bin/sh' $$(find . -type f -name \*.sh) | sed -e 's|^\(.*\):#\!.*$$|\1|g') || exit 1; \
echo "-> No error was detected."; \
echo ""; \
echo "* Check all bash files with ShellCheck"; \
LC_ALL=C.UTF-8 $(SHELLCHECK_CMD) $(SHELLCHECK_BASH_OPT) $(SHELLCHECK_COMMON_IGN) $$(grep '#![[:space:]]*/bin/bash' $$(find . -type f -name \*.sh) | sed -e 's|^\(.*\):#\!.*$$|\1|g') || exit 1; \
echo "-> No error was detected."; \
else \
echo "* ShellCheck is not installed, so skip this."; \
fi
# #
# Local variables: # Local variables:
# tab-width: 4 # tab-width: 4

View File

@ -1,5 +1,5 @@
#!/bin/sh #!/bin/sh
#
# This file is part of S3FS. # This file is part of S3FS.
# #
# Copyright 2009, 2010 Free Software Foundation, Inc. # Copyright 2009, 2010 Free Software Foundation, Inc.
@ -22,14 +22,12 @@
echo "--- Make commit hash file -------" echo "--- Make commit hash file -------"
SHORTHASH="unknown" SHORTHASH="unknown"
type git > /dev/null 2>&1 if command -v git > /dev/null 2>&1 && test -d .git; then
if [ $? -eq 0 -a -d .git ]; then if RESULT=$(git rev-parse --short HEAD); then
RESULT=`git rev-parse --short HEAD` SHORTHASH="${RESULT}"
if [ $? -eq 0 ]; then
SHORTHASH=${RESULT}
fi fi
fi fi
echo ${SHORTHASH} > default_commit_hash echo "${SHORTHASH}" > default_commit_hash
echo "--- Finished commit hash file ---" echo "--- Finished commit hash file ---"

View File

@ -28,25 +28,25 @@ func_usage()
echo "" echo ""
} }
PRGNAME=`basename $0` PRGNAME=$(basename "$0")
SCRIPTDIR=`dirname $0` SCRIPTDIR=$(dirname "$0")
S3FSDIR=`cd ${SCRIPTDIR}/..; pwd` S3FSDIR=$(cd "${SCRIPTDIR}"/.. || exit 1; pwd)
TOPDIR=`cd ${S3FSDIR}/test; pwd` TOPDIR=$(cd "${S3FSDIR}"/test || exit 1; pwd)
SUITELOG="${TOPDIR}/test-suite.log" SUITELOG="${TOPDIR}/test-suite.log"
TMP_LINENO_FILE="/tmp/.lineno.tmp" TMP_LINENO_FILE="/tmp/.lineno.tmp"
while [ $# -ne 0 ]; do while [ $# -ne 0 ]; do
if [ "X$1" = "X" ]; then if [ "X$1" = "X" ]; then
break break
elif [ "X$1" = "X-h" -o "X$1" = "X-H" -o "X$1" = "X--help" -o "X$1" = "X--HELP" ]; then elif [ "X$1" = "X-h" ] || [ "X$1" = "X-H" ] || [ "X$1" = "X--help" ] || [ "X$1" = "X--HELP" ]; then
func_usage ${PRGNAME} func_usage "${PRGNAME}"
exit 0 exit 0
else else
SUITELOG=$1 SUITELOG=$1
fi fi
shift shift
done done
if [ ! -f ${SUITELOG} ]; then if [ ! -f "${SUITELOG}" ]; then
echo "[ERROR] not found ${SUITELOG} log file." echo "[ERROR] not found ${SUITELOG} log file."
exit 1 exit 1
fi fi
@ -59,75 +59,77 @@ fi
# 2 : passed line of end of one small test(specified in test-utils.sh) # 2 : passed line of end of one small test(specified in test-utils.sh)
# 3 : failed line of end of one small test(specified in test-utils.sh) # 3 : failed line of end of one small test(specified in test-utils.sh)
# #
grep -n -e 'test_.*: ".*"' -o -e 'test_.* passed' -o -e 'test_.* failed' ${SUITELOG} 2>/dev/null | sed 's/:test_.*: ".*"/ 1/g' | sed 's/:test_.* passed/ 2/g' | sed 's/:test_.* failed/ 3/g' > ${TMP_LINENO_FILE} grep -n -e 'test_.*: ".*"' -o -e 'test_.* passed' -o -e 'test_.* failed' "${SUITELOG}" 2>/dev/null | sed 's/:test_.*: ".*"/ 1/g' | sed 's/:test_.* passed/ 2/g' | sed 's/:test_.* failed/ 3/g' > "${TMP_LINENO_FILE}"
# #
# Loop for printing result # Loop for printing result
# #
prev_line_type=0 prev_line_type=0
prev_line_number=1 prev_line_number=1
while read line; do while read -r line; do
# line is "<line number> <line type>" # line is "<line number> <line type>"
number_type=($line) #
# shellcheck disable=SC2206
number_type=(${line})
head_line_cnt=`expr ${number_type[0]} - 1` head_line_cnt=$((number_type[0] - 1))
tail_line_cnt=`expr ${number_type[0]} - ${prev_line_number}` tail_line_cnt=$((number_type[0] - prev_line_number))
if [ ${number_type[1]} -eq 2 ]; then if [ "${number_type[1]}" -eq 2 ]; then
echo "" echo ""
fi fi
if [ ${prev_line_type} -eq 1 ]; then if [ "${prev_line_type}" -eq 1 ]; then
if [ ${number_type[1]} -eq 2 ]; then if [ "${number_type[1]}" -eq 2 ]; then
# if passed, cut s3fs information messages # if passed, cut s3fs information messages
head -${head_line_cnt} ${SUITELOG} | tail -${tail_line_cnt} | grep -v -e '[0-9]\+\%' | grep -v -e '^s3fs: ' -a -e '\[INF\]' head "-${head_line_cnt}" "${SUITELOG}" | tail "-${tail_line_cnt}" | grep -v -e '[0-9]\+\%' | grep -v -e '^s3fs: ' -a -e '\[INF\]'
elif [ ${number_type[1]} -eq 3 ]; then elif [ "${number_type[1]}" -eq 3 ]; then
# if failed, print all # if failed, print all
head -${head_line_cnt} ${SUITELOG} | tail -${tail_line_cnt} | grep -v -e '[0-9]\+\%' head "-${head_line_cnt}" "${SUITELOG}" | tail "-${tail_line_cnt}" | grep -v -e '[0-9]\+\%'
else else
# there is start keyword but not end keyword, so print all # there is start keyword but not end keyword, so print all
head -${head_line_cnt} ${SUITELOG} | tail -${tail_line_cnt} | grep -v -e '[0-9]\+\%' head "-${head_line_cnt}" "${SUITELOG}" | tail "-${tail_line_cnt}" | grep -v -e '[0-9]\+\%'
fi fi
elif [ ${prev_line_type} -eq 2 -o ${prev_line_type} -eq 3 ]; then elif [ "${prev_line_type}" -eq 2 ] || [ "${prev_line_type}" -eq 3 ]; then
if [ ${number_type[1]} -eq 2 -o ${number_type[1]} -eq 3 ]; then if [ "${number_type[1]}" -eq 2 ] || [ "${number_type[1]}" -eq 3 ]; then
# previous is end of chmpx, but this type is end of chmpx without start keyword. then print all # previous is end of chmpx, but this type is end of chmpx without start keyword. then print all
head -${head_line_cnt} ${SUITELOG} | tail -${tail_line_cnt} | grep -v -e '[0-9]\+\%' head "-${head_line_cnt}" "${SUITELOG}" | tail "-${tail_line_cnt}" | grep -v -e '[0-9]\+\%'
else else
# this area is not from start to end, cut s3fs information messages # this area is not from start to end, cut s3fs information messages
head -${head_line_cnt} ${SUITELOG} | tail -${tail_line_cnt} | grep -v -e '[0-9]\+\%' | grep -v -e '^s3fs: ' -a -e '\[INF\]' head "-${head_line_cnt}" "${SUITELOG}" | tail "-${tail_line_cnt}" | grep -v -e '[0-9]\+\%' | grep -v -e '^s3fs: ' -a -e '\[INF\]'
fi fi
else else
if [ ${number_type[1]} -eq 2 -o ${number_type[1]} -eq 3 ]; then if [ "${number_type[1]}" -eq 2 ] || [ "${number_type[1]}" -eq 3 ]; then
# previous is normal, but this type is end of chmpx without start keyword. then print all # previous is normal, but this type is end of chmpx without start keyword. then print all
head -${head_line_cnt} ${SUITELOG} | tail -${tail_line_cnt} | grep -v -e '[0-9]\+\%' head "-${head_line_cnt}" "${SUITELOG}" | tail "-${tail_line_cnt}" | grep -v -e '[0-9]\+\%'
else else
# this area is normal, cut s3fs information messages # this area is normal, cut s3fs information messages
head -${head_line_cnt} ${SUITELOG} | tail -${tail_line_cnt} | grep -v -e '[0-9]\+\%' | grep -v -e '^s3fs: ' -a -e '\[INF\]' head "-${head_line_cnt}" "${SUITELOG}" | tail "-${tail_line_cnt}" | grep -v -e '[0-9]\+\%' | grep -v -e '^s3fs: ' -a -e '\[INF\]'
fi fi
fi fi
if [ ${number_type[1]} -eq 3 ]; then if [ "${number_type[1]}" -eq 3 ]; then
echo "" echo ""
fi fi
prev_line_type=${number_type[1]} prev_line_type="${number_type[1]}"
prev_line_number=${number_type[0]} prev_line_number="${number_type[0]}"
done < ${TMP_LINENO_FILE} done < "${TMP_LINENO_FILE}"
# #
# Print rest lines # Print rest lines
# #
file_line_cnt=`wc -l ${SUITELOG} | awk '{print $1}'` file_line_cnt=$(wc -l "${SUITELOG}" | awk '{print $1}')
tail_line_cnt=`expr ${file_line_cnt} - ${prev_line_number}` tail_line_cnt=$((file_line_cnt - prev_line_number))
if [ ${prev_line_type} -eq 1 ]; then if [ "${prev_line_type}" -eq 1 ]; then
tail -${tail_line_cnt} ${SUITELOG} | grep -v -e '[0-9]\+\%' tail "-${tail_line_cnt}" "${SUITELOG}" | grep -v -e '[0-9]\+\%'
else else
tail -${tail_line_cnt} ${SUITELOG} | grep -v -e '[0-9]\+\%' | grep -v -e '^s3fs: ' -a -e '\[INF\]' tail "-${tail_line_cnt}" "${SUITELOG}" | grep -v -e '[0-9]\+\%' | grep -v -e '^s3fs: ' -a -e '\[INF\]'
fi fi
# #
# Remove temp file # Remove temp file
# #
rm -f ${TMP_LINENO_FILE} rm -f "${TMP_LINENO_FILE}"
exit 0 exit 0

View File

@ -66,59 +66,65 @@ set -o pipefail
S3FS=../src/s3fs S3FS=../src/s3fs
# Allow these defaulted values to be overridden # Allow these defaulted values to be overridden
: ${S3_URL:="https://127.0.0.1:8080"} : "${S3_URL:="https://127.0.0.1:8080"}"
: ${S3_ENDPOINT:="us-east-1"} : "${S3_ENDPOINT:="us-east-1"}"
: ${S3FS_CREDENTIALS_FILE:="passwd-s3fs"} : "${S3FS_CREDENTIALS_FILE:="passwd-s3fs"}"
: ${TEST_BUCKET_1:="s3fs-integration-test"} : "${TEST_BUCKET_1:="s3fs-integration-test"}"
export TEST_BUCKET_1 export TEST_BUCKET_1
export S3_URL export S3_URL
export S3_ENDPOINT export S3_ENDPOINT
export TEST_SCRIPT_DIR=`pwd` TEST_SCRIPT_DIR=$(pwd)
export TEST_SCRIPT_DIR
export TEST_BUCKET_MOUNT_POINT_1=${TEST_BUCKET_1} export TEST_BUCKET_MOUNT_POINT_1=${TEST_BUCKET_1}
S3PROXY_VERSION="1.9.0" S3PROXY_VERSION="1.9.0"
S3PROXY_BINARY=${S3PROXY_BINARY-"s3proxy-${S3PROXY_VERSION}"} S3PROXY_BINARY="${S3PROXY_BINARY-"s3proxy-${S3PROXY_VERSION}"}"
CHAOS_HTTP_PROXY_VERSION="1.1.0" CHAOS_HTTP_PROXY_VERSION="1.1.0"
CHAOS_HTTP_PROXY_BINARY="chaos-http-proxy-${CHAOS_HTTP_PROXY_VERSION}" CHAOS_HTTP_PROXY_BINARY="chaos-http-proxy-${CHAOS_HTTP_PROXY_VERSION}"
if [ ! -f "$S3FS_CREDENTIALS_FILE" ] if [ ! -f "$S3FS_CREDENTIALS_FILE" ]
then then
echo "Missing credentials file: $S3FS_CREDENTIALS_FILE" echo "Missing credentials file: ${S3FS_CREDENTIALS_FILE}"
exit 1 exit 1
fi fi
chmod 600 "$S3FS_CREDENTIALS_FILE" chmod 600 "${S3FS_CREDENTIALS_FILE}"
if [ -z "${S3FS_PROFILE}" ]; then if [ -z "${S3FS_PROFILE}" ]; then
export AWS_ACCESS_KEY_ID=$(cut -d: -f1 ${S3FS_CREDENTIALS_FILE}) AWS_ACCESS_KEY_ID=$(cut -d: -f1 "${S3FS_CREDENTIALS_FILE}")
export AWS_SECRET_ACCESS_KEY=$(cut -d: -f2 ${S3FS_CREDENTIALS_FILE}) export AWS_ACCESS_KEY_ID
AWS_SECRET_ACCESS_KEY=$(cut -d: -f2 "${S3FS_CREDENTIALS_FILE}")
export AWS_SECRET_ACCESS_KEY
fi fi
if [ ! -d $TEST_BUCKET_MOUNT_POINT_1 ] if [ ! -d "${TEST_BUCKET_MOUNT_POINT_1}" ]; then
then mkdir -p "${TEST_BUCKET_MOUNT_POINT_1}"
mkdir -p $TEST_BUCKET_MOUNT_POINT_1
fi fi
# This function execute the function parameters $1 times # This function execute the function parameters $1 times
# before giving up, with 1 second delays. # before giving up, with 1 second delays.
function retry { function retry {
local N=$1; shift; local N="$1"
shift
rc=0 rc=0
for i in $(seq $N); do for _ in $(seq "${N}"); do
echo "Trying: $*" echo "Trying: $*"
eval $@; rc=$? # shellcheck disable=SC2068,SC2294
if [ $rc = 0 ]; then eval $@
rc=$?
if [ "${rc}" -eq 0 ]; then
break break
fi fi
sleep 1 sleep 1
echo "Retrying: $*" echo "Retrying: $*"
done done
if [ $rc != 0 ]; then if [ "${rc}" -ne 0 ]; then
echo "timeout waiting for $*" echo "timeout waiting for $*"
fi fi
return $rc return "${rc}"
} }
# Proxy is not started if S3PROXY_BINARY is an empty string # Proxy is not started if S3PROXY_BINARY is an empty string
@ -145,7 +151,7 @@ function start_s3proxy {
echo -e 'password\npassword\n\n\n\n\n\n\nyes' | keytool -genkey -keystore /tmp/keystore.jks -keyalg RSA -keysize 2048 -validity 365 -ext SAN=IP:127.0.0.1 echo -e 'password\npassword\n\n\n\n\n\n\nyes' | keytool -genkey -keystore /tmp/keystore.jks -keyalg RSA -keysize 2048 -validity 365 -ext SAN=IP:127.0.0.1
echo password | keytool -exportcert -keystore /tmp/keystore.jks -rfc -file /tmp/keystore.pem echo password | keytool -exportcert -keystore /tmp/keystore.jks -rfc -file /tmp/keystore.pem
${STDBUF_BIN} -oL -eL java -jar "$S3PROXY_BINARY" --properties $S3PROXY_CONFIG & "${STDBUF_BIN}" -oL -eL java -jar "${S3PROXY_BINARY}" --properties "${S3PROXY_CONFIG}" &
S3PROXY_PID=$! S3PROXY_PID=$!
# wait for S3Proxy to start # wait for S3Proxy to start
@ -159,7 +165,7 @@ function start_s3proxy {
chmod +x "${CHAOS_HTTP_PROXY_BINARY}" chmod +x "${CHAOS_HTTP_PROXY_BINARY}"
fi fi
${STDBUF_BIN} -oL -eL java -jar ${CHAOS_HTTP_PROXY_BINARY} --properties chaos-http-proxy.conf & "${STDBUF_BIN}" -oL -eL java -jar "${CHAOS_HTTP_PROXY_BINARY}" --properties chaos-http-proxy.conf &
CHAOS_HTTP_PROXY_PID=$! CHAOS_HTTP_PROXY_PID=$!
# wait for Chaos HTTP Proxy to start # wait for Chaos HTTP Proxy to start
@ -170,12 +176,12 @@ function start_s3proxy {
function stop_s3proxy { function stop_s3proxy {
if [ -n "${S3PROXY_PID}" ] if [ -n "${S3PROXY_PID}" ]
then then
kill $S3PROXY_PID kill "${S3PROXY_PID}"
fi fi
if [ -n "${CHAOS_HTTP_PROXY_PID}" ] if [ -n "${CHAOS_HTTP_PROXY_PID}" ]
then then
kill $CHAOS_HTTP_PROXY_PID kill "${CHAOS_HTTP_PROXY_PID}"
fi fi
} }
@ -200,7 +206,7 @@ function start_s3fs {
fi fi
# On OSX only, we need to specify the direct_io and auto_cache flag. # On OSX only, we need to specify the direct_io and auto_cache flag.
if [ `uname` = "Darwin" ]; then if [ "$(uname)" = "Darwin" ]; then
local DIRECT_IO_OPT="-o direct_io -o auto_cache" local DIRECT_IO_OPT="-o direct_io -o auto_cache"
else else
local DIRECT_IO_OPT="" local DIRECT_IO_OPT=""
@ -215,7 +221,7 @@ function start_s3fs {
# Therefore, when it is macos, it is not executed via stdbuf. # Therefore, when it is macos, it is not executed via stdbuf.
# This patch may be temporary, but no other method has been found at this time. # This patch may be temporary, but no other method has been found at this time.
# #
if [ `uname` = "Darwin" ]; then if [ "$(uname)" = "Darwin" ]; then
local VIA_STDBUF_CMDLINE="" local VIA_STDBUF_CMDLINE=""
else else
local VIA_STDBUF_CMDLINE="${STDBUF_BIN} -oL -eL" local VIA_STDBUF_CMDLINE="${STDBUF_BIN} -oL -eL"
@ -238,15 +244,17 @@ function start_s3fs {
# #
# subshell with set -x to log exact invocation of s3fs-fuse # subshell with set -x to log exact invocation of s3fs-fuse
# shellcheck disable=SC2086
( (
set -x set -x
${VIA_STDBUF_CMDLINE} \ ${VIA_STDBUF_CMDLINE} \
${VALGRIND_EXEC} ${S3FS} \ ${VALGRIND_EXEC} \
$TEST_BUCKET_1 \ ${S3FS} \
$TEST_BUCKET_MOUNT_POINT_1 \ ${TEST_BUCKET_1} \
${TEST_BUCKET_MOUNT_POINT_1} \
-o use_path_request_style \ -o use_path_request_style \
-o url=${S3_URL} \ -o url="${S3_URL}" \
-o endpoint=${S3_ENDPOINT} \ -o endpoint="${S3_ENDPOINT}" \
-o no_check_certificate \ -o no_check_certificate \
-o ssl_verify_hostname=0 \ -o ssl_verify_hostname=0 \
-o use_xattr=1 \ -o use_xattr=1 \
@ -255,37 +263,39 @@ function start_s3fs {
${DIRECT_IO_OPT} \ ${DIRECT_IO_OPT} \
-o stat_cache_expire=1 \ -o stat_cache_expire=1 \
-o stat_cache_interval_expire=1 \ -o stat_cache_interval_expire=1 \
-o dbglevel=${DBGLEVEL:=info} \ -o dbglevel="${DBGLEVEL:=info}" \
-o no_time_stamp_msg \ -o no_time_stamp_msg \
-o retries=3 \ -o retries=3 \
-f \ -f \
"${@}" & "${@}" &
echo $! >&3 echo $! >&3
) 3>pid | ${STDBUF_BIN} -oL -eL ${SED_BIN} ${SED_BUFFER_FLAG} "s/^/s3fs: /" & ) 3>pid | "${STDBUF_BIN}" -oL -eL "${SED_BIN}" "${SED_BUFFER_FLAG}" "s/^/s3fs: /" &
sleep 1 sleep 1
export S3FS_PID=$(<pid) S3FS_PID=$(<pid)
export S3FS_PID
rm -f pid rm -f pid
if [ `uname` = "Darwin" ]; then if [ "$(uname)" = "Darwin" ]; then
local TRYCOUNT=0 local TRYCOUNT=0
while [ $TRYCOUNT -le ${RETRIES:=20} ]; do while [ "${TRYCOUNT}" -le "${RETRIES:=20}" ]; do
df | grep -q $TEST_BUCKET_MOUNT_POINT_1; rc=$? df | grep -q "${TEST_BUCKET_MOUNT_POINT_1}"
if [ $rc -eq 0 ]; then rc=$?
if [ "${rc}" -eq 0 ]; then
break; break;
fi fi
sleep 1 sleep 1
TRYCOUNT=`expr ${TRYCOUNT} + 1` TRYCOUNT=$((TRYCOUNT + 1))
done done
if [ $rc -ne 0 ]; then if [ "${rc}" -ne 0 ]; then
exit 1 exit 1
fi fi
else else
retry ${RETRIES:=20} grep -q $TEST_BUCKET_MOUNT_POINT_1 /proc/mounts || exit 1 retry "${RETRIES:=20}" grep -q "${TEST_BUCKET_MOUNT_POINT_1}" /proc/mounts || exit 1
fi fi
# Quick way to start system up for manual testing with options under test # Quick way to start system up for manual testing with options under test
if [[ -n ${INTERACT} ]]; then if [[ -n "${INTERACT}" ]]; then
echo "Mountpoint $TEST_BUCKET_MOUNT_POINT_1 is ready" echo "Mountpoint ${TEST_BUCKET_MOUNT_POINT_1} is ready"
echo "control-C to quit" echo "control-C to quit"
sleep infinity sleep infinity
exit 0 exit 0
@ -294,13 +304,13 @@ function start_s3fs {
function stop_s3fs { function stop_s3fs {
# Retry in case file system is in use # Retry in case file system is in use
if [ `uname` = "Darwin" ]; then if [ "$(uname)" = "Darwin" ]; then
if df | grep -q $TEST_BUCKET_MOUNT_POINT_1; then if df | grep -q "${TEST_BUCKET_MOUNT_POINT_1}"; then
retry 10 df "|" grep -q $TEST_BUCKET_MOUNT_POINT_1 "&&" umount $TEST_BUCKET_MOUNT_POINT_1 retry 10 df "|" grep -q "${TEST_BUCKET_MOUNT_POINT_1}" "&&" umount "${TEST_BUCKET_MOUNT_POINT_1}"
fi fi
else else
if grep -q $TEST_BUCKET_MOUNT_POINT_1 /proc/mounts; then if grep -q "${TEST_BUCKET_MOUNT_POINT_1}" /proc/mounts; then
retry 10 grep -q $TEST_BUCKET_MOUNT_POINT_1 /proc/mounts "&&" fusermount -u $TEST_BUCKET_MOUNT_POINT_1 retry 10 grep -q "${TEST_BUCKET_MOUNT_POINT_1}" /proc/mounts "&&" fusermount -u "${TEST_BUCKET_MOUNT_POINT_1}"
fi fi
fi fi
} }

File diff suppressed because it is too large Load Diff

View File

@ -40,3 +40,12 @@ int main(int argc, char *argv[])
} }
return 0; return 0;
} }
/*
* Local variables:
* tab-width: 4
* c-basic-offset: 4
* End:
* vim600: expandtab sw=4 ts=4 fdm=marker
* vim<600: expandtab sw=4 ts=4
*/

View File

@ -40,25 +40,25 @@ UsageFunction()
} }
### Check parameters ### Check parameters
WHOAMI=`whoami` WHOAMI=$(whoami)
OWNNAME=`basename $0` OWNNAME=$(basename "$0")
AUTOYES="no" AUTOYES="no"
ALLYES="no" ALLYES="no"
DIRPARAM="" DIRPARAM=""
while [ "$1" != "" ]; do while [ "$1" != "" ]; do
if [ "X$1" = "X-help" -o "X$1" = "X-h" -o "X$1" = "X-H" ]; then if [ "X$1" = "X-help" ] || [ "X$1" = "X-h" ] || [ "X$1" = "X-H" ]; then
UsageFunction $OWNNAME UsageFunction "${OWNNAME}"
exit 0 exit 0
elif [ "X$1" = "X-y" -o "X$1" = "X-Y" ]; then elif [ "X$1" = "X-y" ] || [ "X$1" = "X-Y" ]; then
AUTOYES="yes" AUTOYES="yes"
elif [ "X$1" = "X-all" -o "X$1" = "X-ALL" ]; then elif [ "X$1" = "X-all" ] || [ "X$1" = "X-ALL" ]; then
ALLYES="yes" ALLYES="yes"
else else
if [ "X$DIRPARAM" != "X" ]; then if [ "X$DIRPARAM" != "X" ]; then
echo "*** Input error." echo "*** Input error."
echo "" echo ""
UsageFunction $OWNNAME UsageFunction "${OWNNAME}"
exit 1 exit 1
fi fi
DIRPARAM=$1 DIRPARAM=$1
@ -68,7 +68,7 @@ done
if [ "X$DIRPARAM" = "X" ]; then if [ "X$DIRPARAM" = "X" ]; then
echo "*** Input error." echo "*** Input error."
echo "" echo ""
UsageFunction $OWNNAME UsageFunction "${OWNNAME}"
exit 1 exit 1
fi fi
@ -88,18 +88,17 @@ echo "Please execute this program by responsibility of your own."
echo "#############################################################################" echo "#############################################################################"
echo "" echo ""
DATE=`date +'%Y%m%d-%H%M%S'` DATE=$(date +'%Y%m%d-%H%M%S')
LOGFILE="$OWNNAME-$DATE.log" LOGFILE="${OWNNAME}-${DATE}.log"
echo -n "Start to merge directory object... [$DIRPARAM]" echo "Start to merge directory object... [${DIRPARAM}]"
echo "# Start to merge directory object... [$DIRPARAM]" >> $LOGFILE {
echo -n "# DATE : " >> $LOGFILE echo "# Start to merge directory object... [${DIRPARAM}]"
echo `date` >> $LOGFILE echo "# DATE : $(date)"
echo -n "# BASEDIR : " >> $LOGFILE echo "# BASEDIR : $(pwd)"
echo `pwd` >> $LOGFILE echo "# TARGET PATH : ${DIRPARAM}"
echo -n "# TARGET PATH : " >> $LOGFILE echo ""
echo $DIRPARAM >> $LOGFILE } > "${LOGFILE}"
echo "" >> $LOGFILE
if [ "$AUTOYES" = "yes" ]; then if [ "$AUTOYES" = "yes" ]; then
echo "(no confirmation)" echo "(no confirmation)"
@ -109,80 +108,84 @@ fi
echo "" echo ""
### Get Directory list ### Get Directory list
DIRLIST=`find $DIRPARAM -type d -print | grep -v ^\.$` DIRLIST=$(find "${DIRPARAM}" -type d -print | grep -v ^\.$)
# #
# Main loop # Main loop
# #
for DIR in $DIRLIST; do for DIR in $DIRLIST; do
### Skip "." and ".." directories ### Skip "." and ".." directories
BASENAME=`basename $DIR` BASENAME=$(basename "${DIR}")
if [ "$BASENAME" = "." -o "$BASENAME" = ".." ]; then if [ "${BASENAME}" = "." ] || [ "${BASENAME}" = ".." ]; then
continue continue
fi fi
if [ "$ALLYES" = "no" ]; then if [ "${ALLYES}" = "no" ]; then
### Skip "d---------" directories. ### Skip "d---------" directories.
### Other clients make directory object "dir/" which don't have ### Other clients make directory object "dir/" which don't have
### "x-amz-meta-mode" attribute. ### "x-amz-meta-mode" attribute.
### Then these directories is "d---------", it is target directory. ### Then these directories is "d---------", it is target directory.
DIRPERMIT=`ls -ld --time-style=+'%Y%m%d%H%M' $DIR | awk '{print $1}'`
if [ "$DIRPERMIT" != "d---------" ]; then # shellcheck disable=SC2012
DIRPERMIT=$(ls -ld --time-style=+'%Y%m%d%H%M' "${DIR}" | awk '{print $1}')
if [ "${DIRPERMIT}" != "d---------" ]; then
continue continue
fi fi
fi fi
### Confirm ### Confirm
ANSWER="" ANSWER=""
if [ "$AUTOYES" = "yes" ]; then if [ "${AUTOYES}" = "yes" ]; then
ANSWER="y" ANSWER="y"
fi fi
while [ "X$ANSWER" != "XY" -a "X$ANSWER" != "Xy" -a "X$ANSWER" != "XN" -a "X$ANSWER" != "Xn" ]; do while [ "X${ANSWER}" != "XY" ] && [ "X${ANSWER}" != "Xy" ] && [ "X${ANSWER}" != "XN" ] && [ "X${ANSWER}" != "Xn" ]; do
echo -n "Do you merge $DIR? (y/n): " printf "%s" "Do you merge ${DIR} ? (y/n): "
read ANSWER read -r ANSWER
done done
if [ "X$ANSWER" != "XY" -a "X$ANSWER" != "Xy" ]; then if [ "X${ANSWER}" != "XY" ] && [ "X${ANSWER}" != "Xy" ]; then
continue continue
fi fi
### Do ### Do
CHOWN=`ls -ld --time-style=+'%Y%m%d%H%M' $DIR | awk '{print $3":"$4" "$7}'` # shellcheck disable=SC2012
CHMOD=`ls -ld --time-style=+'%Y%m%d%H%M' $DIR | awk '{print $7}'` CHOWN=$(ls -ld --time-style=+'%Y%m%d%H%M' "${DIR}" | awk '{print $3":"$4" "$7}')
TOUCH=`ls -ld --time-style=+'%Y%m%d%H%M' $DIR | awk '{print $6" "$7}'` # shellcheck disable=SC2012
CHMOD=$(ls -ld --time-style=+'%Y%m%d%H%M' "${DIR}" | awk '{print $7}')
# shellcheck disable=SC2012
TOUCH=$(ls -ld --time-style=+'%Y%m%d%H%M' "${DIR}" | awk '{print $6" "$7}')
echo -n "*** Merge $DIR : " printf "%s" "*** Merge ${DIR} : "
echo -n " $DIR : " >> $LOGFILE printf "%s" " ${DIR} : " >> "${LOGFILE}"
chmod 755 $CHMOD > /dev/null 2>&1 chmod 755 "${CHMOD}" > /dev/null 2>&1
RESULT=$? RESULT=$?
if [ $RESULT -ne 0 ]; then if [ "${RESULT}" -ne 0 ]; then
echo "Failed(chmod)" echo "Failed(chmod)"
echo "Failed(chmod)" >> $LOGFILE echo "Failed(chmod)" >> "${LOGFILE}"
continue continue
fi fi
chown $CHOWN > /dev/null 2>&1 chown "${CHOWN}" > /dev/null 2>&1
RESULT=$? RESULT=$?
if [ $RESULT -ne 0 ]; then if [ "${RESULT}" -ne 0 ]; then
echo "Failed(chown)" echo "Failed(chown)"
echo "Failed(chown)" >> $LOGFILE echo "Failed(chown)" >> "${LOGFILE}"
continue continue
fi fi
touch -t $TOUCH > /dev/null 2>&1 touch -t "${TOUCH}" > /dev/null 2>&1
RESULT=$? RESULT=$?
if [ $RESULT -ne 0 ]; then if [ "${RESULT}" -ne 0 ]; then
echo "Failed(touch)" echo "Failed(touch)"
echo "Failed(touch)" >> $LOGFILE echo "Failed(touch)" >> "${LOGFILE}"
continue continue
fi fi
echo "Succeed" echo "Succeed"
echo "Succeed" >> $LOGFILE echo "Succeed" >> "${LOGFILE}"
done done
echo "" echo ""
echo "" >> $LOGFILE echo "" >> "${LOGFILE}"
echo "Finished." echo "Finished."
echo -n "# Finished : " >> $LOGFILE echo "# Finished : $(date)" >> "${LOGFILE}"
echo `date` >> $LOGFILE
# #
# Local variables: # Local variables:

View File

@ -46,33 +46,33 @@ func_usage()
echo "" echo ""
} }
PRGNAME=`basename $0` PRGNAME=$(basename "$0")
if [ "X$1" = "X-h" -o "X$1" = "X-H" ]; then if [ "X$1" = "X-h" ] || [ "X$1" = "X-H" ]; then
func_usage $PRGNAME func_usage "${PRGNAME}"
exit 0 exit 0
fi fi
if [ "X$1" = "X" -o "X$2" = "X" -o "X$3" = "X" ]; then if [ "X$1" = "X" ] || [ "X$2" = "X" ] || [ "X$3" = "X" ]; then
func_usage $PRGNAME func_usage "${PRGNAME}"
exit 1 exit 1
fi fi
BUCKET=$1 BUCKET="$1"
CDIR="$2" CDIR="$2"
LIMIT=$3 LIMIT="$3"
SILENT=0 SILENT=0
if [ "X$4" = "X-silent" ]; then if [ "X$4" = "X-silent" ]; then
SILENT=1 SILENT=1
fi fi
FILES_CDIR="${CDIR}/${BUCKET}" FILES_CDIR="${CDIR}/${BUCKET}"
STATS_CDIR="${CDIR}/.${BUCKET}.stat" STATS_CDIR="${CDIR}/.${BUCKET}.stat"
CURRENT_CACHE_SIZE=`du -sb "$FILES_CDIR" | awk '{print $1}'` CURRENT_CACHE_SIZE=$(du -sb "${FILES_CDIR}" | awk '{print $1}')
# #
# Check total size # Check total size
# #
if [ $LIMIT -ge $CURRENT_CACHE_SIZE ]; then if [ "${LIMIT}" -ge "${CURRENT_CACHE_SIZE}" ]; then
if [ $SILENT -ne 1 ]; then if [ $SILENT -ne 1 ]; then
echo "$FILES_CDIR ($CURRENT_CACHE_SIZE) is below allowed $LIMIT" echo "${FILES_CDIR} (${CURRENT_CACHE_SIZE}) is below allowed ${LIMIT}"
fi fi
exit 0 exit 0
fi fi
@ -86,37 +86,36 @@ TMP_CFILE=""
# #
# Make file list by sorted access time # Make file list by sorted access time
# #
find "$STATS_CDIR" -type f -exec stat -c "%X:%n" "{}" \; | sort | while read part find "${STATS_CDIR}" -type f -exec stat -c "%X:%n" "{}" \; | sort | while read -r part
do do
echo Looking at $part echo "Looking at ${part}"
TMP_ATIME=`echo "$part" | cut -d: -f1` TMP_ATIME=$(echo "${part}" | cut -d: -f1)
TMP_STATS="`echo "$part" | cut -d: -f2`" TMP_STATS=$(echo "${part}" | cut -d: -f2)
TMP_CFILE=`echo "$TMP_STATS" | sed s/\.$BUCKET\.stat/$BUCKET/` TMP_CFILE=$(echo "${TMP_STATS}" | sed -e "s/\\.${BUCKET}\\.stat/${BUCKET}/")
if [ `stat -c %X "$TMP_STATS"` -eq $TMP_ATIME ]; then if [ "$(stat -c %X "${TMP_STATS}")" -eq "${TMP_ATIME}" ]; then
rm -f "$TMP_STATS" "$TMP_CFILE" > /dev/null 2>&1 if ! rm "${TMP_STATS}" "${TMP_CFILE}" > /dev/null 2>&1; then
if [ $? -ne 0 ]; then if [ "${SILENT}" -ne 1 ]; then
if [ $SILENT -ne 1 ]; then echo "ERROR: Could not remove files(${TMP_STATS},${TMP_CFILE})"
echo "ERROR: Could not remove files($TMP_STATS,$TMP_CFILE)"
fi fi
exit 1 exit 1
else else
if [ $SILENT -ne 1 ]; then if [ "${SILENT}" -ne 1 ]; then
echo "remove file: $TMP_CFILE $TMP_STATS" echo "remove file: ${TMP_CFILE} ${TMP_STATS}"
fi fi
fi fi
fi fi
if [ $LIMIT -ge `du -sb "$FILES_CDIR" | awk '{print $1}'` ]; then if [ "${LIMIT}" -ge "$(du -sb "${FILES_CDIR}" | awk '{print $1}')" ]; then
if [ $SILENT -ne 1 ]; then if [ "${SILENT}" -ne 1 ]; then
echo "finish removing files" echo "finish removing files"
fi fi
break break
fi fi
done done
if [ $SILENT -ne 1 ]; then if [ "${SILENT}" -ne 1 ]; then
TOTAL_SIZE=`du -sb "$FILES_CDIR" | awk '{print $1}'` TOTAL_SIZE=$(du -sb "${FILES_CDIR}" | awk '{print $1}')
echo "Finish: $FILES_CDIR total size is $TOTAL_SIZE" echo "Finish: ${FILES_CDIR} total size is ${TOTAL_SIZE}"
fi fi
exit 0 exit 0

View File

@ -32,8 +32,9 @@ CACHE_DIR="/tmp/s3fs-cache"
rm -rf "${CACHE_DIR}" rm -rf "${CACHE_DIR}"
mkdir "${CACHE_DIR}" mkdir "${CACHE_DIR}"
#reserve 200MB for data cache
source test-utils.sh source test-utils.sh
#reserve 200MB for data cache
FAKE_FREE_DISK_SIZE=200 FAKE_FREE_DISK_SIZE=200
ENSURE_DISKFREE_SIZE=10 ENSURE_DISKFREE_SIZE=10
@ -44,13 +45,13 @@ if [ -n "${ALL_TESTS}" ]; then
"use_cache=${CACHE_DIR} -o ensure_diskfree=${ENSURE_DISKFREE_SIZE} -o fake_diskfree=${FAKE_FREE_DISK_SIZE}" "use_cache=${CACHE_DIR} -o ensure_diskfree=${ENSURE_DISKFREE_SIZE} -o fake_diskfree=${FAKE_FREE_DISK_SIZE}"
enable_content_md5 enable_content_md5
enable_noobj_cache enable_noobj_cache
max_stat_cache_size=100 "max_stat_cache_size=100"
nocopyapi nocopyapi
nomultipart nomultipart
notsup_compat_dir notsup_compat_dir
sigv2 sigv2
sigv4 sigv4
singlepart_copy_limit=10 # limit size to exercise multipart code paths "singlepart_copy_limit=10" # limit size to exercise multipart code paths
#use_sse # TODO: S3Proxy does not support SSE #use_sse # TODO: S3Proxy does not support SSE
) )
else else
@ -64,9 +65,10 @@ start_s3proxy
aws_cli s3 mb "s3://${TEST_BUCKET_1}" --region "${S3_ENDPOINT}" aws_cli s3 mb "s3://${TEST_BUCKET_1}" --region "${S3_ENDPOINT}"
for flag in "${FLAGS[@]}"; do for flag in "${FLAGS[@]}"; do
echo "testing s3fs flag: $flag" echo "testing s3fs flag: ${flag}"
start_s3fs -o $flag # shellcheck disable=SC2086
start_s3fs -o ${flag}
./integration-test-main.sh ./integration-test-main.sh

View File

@ -24,19 +24,29 @@
set -o errexit set -o errexit
set -o pipefail set -o pipefail
#
# Configuration # Configuration
#
TEST_TEXT="HELLO WORLD" TEST_TEXT="HELLO WORLD"
TEST_TEXT_FILE=test-s3fs.txt TEST_TEXT_FILE=test-s3fs.txt
TEST_DIR=testdir TEST_DIR=testdir
# shellcheck disable=SC2034
ALT_TEST_TEXT_FILE=test-s3fs-ALT.txt ALT_TEST_TEXT_FILE=test-s3fs-ALT.txt
# shellcheck disable=SC2034
TEST_TEXT_FILE_LENGTH=15 TEST_TEXT_FILE_LENGTH=15
# shellcheck disable=SC2034
BIG_FILE=big-file-s3fs.txt BIG_FILE=big-file-s3fs.txt
TEMP_DIR=${TMPDIR:-"/var/tmp"} # shellcheck disable=SC2034
TEMP_DIR="${TMPDIR:-"/var/tmp"}"
# /dev/urandom can only return 32 MB per block maximum # /dev/urandom can only return 32 MB per block maximum
BIG_FILE_BLOCK_SIZE=$((25 * 1024 * 1024)) BIG_FILE_BLOCK_SIZE=$((25 * 1024 * 1024))
BIG_FILE_COUNT=1 BIG_FILE_COUNT=1
# This should be greater than the multipart size # This should be greater than the multipart size
BIG_FILE_LENGTH=$(($BIG_FILE_BLOCK_SIZE * $BIG_FILE_COUNT)) # shellcheck disable=SC2034
BIG_FILE_LENGTH=$((BIG_FILE_BLOCK_SIZE * BIG_FILE_COUNT))
# Set locale because some tests check for English expressions # Set locale because some tests check for English expressions
export LC_ALL=en_US.UTF-8 export LC_ALL=en_US.UTF-8
export RUN_DIR export RUN_DIR
@ -48,7 +58,7 @@ export RUN_DIR
# and uses gnu commands(gstdbuf, gtruncate, gsed). # and uses gnu commands(gstdbuf, gtruncate, gsed).
# Set your PATH appropriately so that you can find these commands. # Set your PATH appropriately so that you can find these commands.
# #
if [ `uname` = "Darwin" ]; then if [ "$(uname)" = "Darwin" ]; then
export STDBUF_BIN="gstdbuf" export STDBUF_BIN="gstdbuf"
export TRUNCATE_BIN="gtruncate" export TRUNCATE_BIN="gtruncate"
export SED_BIN="gsed" export SED_BIN="gsed"
@ -62,7 +72,7 @@ fi
export SED_BUFFER_FLAG="--unbuffered" export SED_BUFFER_FLAG="--unbuffered"
function get_xattr() { function get_xattr() {
if [ `uname` = "Darwin" ]; then if [ "$(uname)" = "Darwin" ]; then
xattr -p "$1" "$2" xattr -p "$1" "$2"
else else
getfattr -n "$1" --only-values "$2" getfattr -n "$1" --only-values "$2"
@ -70,7 +80,7 @@ function get_xattr() {
} }
function set_xattr() { function set_xattr() {
if [ `uname` = "Darwin" ]; then if [ "$(uname)" = "Darwin" ]; then
xattr -w "$1" "$2" "$3" xattr -w "$1" "$2" "$3"
else else
setfattr -n "$1" -v "$2" "$3" setfattr -n "$1" -v "$2" "$3"
@ -78,7 +88,7 @@ function set_xattr() {
} }
function del_xattr() { function del_xattr() {
if [ `uname` = "Darwin" ]; then if [ "$(uname)" = "Darwin" ]; then
xattr -d "$1" "$2" xattr -d "$1" "$2"
else else
setfattr -x "$1" "$2" setfattr -x "$1" "$2"
@ -86,7 +96,7 @@ function del_xattr() {
} }
function get_size() { function get_size() {
if [ `uname` = "Darwin" ]; then if [ "$(uname)" = "Darwin" ]; then
stat -f "%z" "$1" stat -f "%z" "$1"
else else
stat -c %s "$1" stat -c %s "$1"
@ -98,45 +108,47 @@ function check_file_size() {
local EXPECTED_SIZE="$2" local EXPECTED_SIZE="$2"
# Verify file is zero length via metadata # Verify file is zero length via metadata
local size=$(get_size ${FILE_NAME}) local size
if [ $size -ne $EXPECTED_SIZE ] size=$(get_size "${FILE_NAME}")
if [ "${size}" -ne "${EXPECTED_SIZE}" ]
then then
echo "error: expected ${FILE_NAME} to be zero length" echo "error: expected ${FILE_NAME} to be zero length"
return 1 return 1
fi fi
# Verify file is zero length via data # Verify file is zero length via data
local size=$(cat ${FILE_NAME} | wc -c) size=$(wc -c < "${FILE_NAME}")
if [ $size -ne $EXPECTED_SIZE ] if [ "${size}" -ne "${EXPECTED_SIZE}" ]
then then
echo "error: expected ${FILE_NAME} to be $EXPECTED_SIZE length, got $size" echo "error: expected ${FILE_NAME} to be ${EXPECTED_SIZE} length, got ${size}"
return 1 return 1
fi fi
} }
function mk_test_file { function mk_test_file {
if [ $# = 0 ]; then if [ $# = 0 ]; then
local TEXT=$TEST_TEXT local TEXT="${TEST_TEXT}"
else else
local TEXT=$1 local TEXT="$1"
fi fi
echo $TEXT > $TEST_TEXT_FILE echo "${TEXT}" > "${TEST_TEXT_FILE}"
if [ ! -e $TEST_TEXT_FILE ] if [ ! -e "${TEST_TEXT_FILE}" ]
then then
echo "Could not create file ${TEST_TEXT_FILE}, it does not exist" echo "Could not create file ${TEST_TEXT_FILE}, it does not exist"
exit 1 exit 1
fi fi
# wait & check # wait & check
local BASE_TEXT_LENGTH=`echo $TEXT | wc -c | awk '{print $1}'` local BASE_TEXT_LENGTH; BASE_TEXT_LENGTH=$(echo "${TEXT}" | wc -c | awk '{print $1}')
local TRY_COUNT=10 local TRY_COUNT=10
while true; do while true; do
local MK_TEXT_LENGTH=`wc -c $TEST_TEXT_FILE | awk '{print $1}'` local MK_TEXT_LENGTH
if [ $BASE_TEXT_LENGTH -eq $MK_TEXT_LENGTH ]; then MK_TEXT_LENGTH=$(wc -c "${TEST_TEXT_FILE}" | awk '{print $1}')
if [ "${BASE_TEXT_LENGTH}" -eq "${MK_TEXT_LENGTH}" ]; then
break break
fi fi
local TRY_COUNT=`expr $TRY_COUNT - 1` local TRY_COUNT=$((TRY_COUNT - 1))
if [ $TRY_COUNT -le 0 ]; then if [ "${TRY_COUNT}" -le 0 ]; then
echo "Could not create file ${TEST_TEXT_FILE}, that file size is something wrong" echo "Could not create file ${TEST_TEXT_FILE}, that file size is something wrong"
fi fi
sleep 1 sleep 1
@ -145,13 +157,13 @@ function mk_test_file {
function rm_test_file { function rm_test_file {
if [ $# = 0 ]; then if [ $# = 0 ]; then
local FILE=$TEST_TEXT_FILE local FILE="${TEST_TEXT_FILE}"
else else
local FILE=$1 local FILE="$1"
fi fi
rm -f $FILE rm -f "${FILE}"
if [ -e $FILE ] if [ -e "${FILE}" ]
then then
echo "Could not cleanup file ${TEST_TEXT_FILE}" echo "Could not cleanup file ${TEST_TEXT_FILE}"
exit 1 exit 1
@ -159,17 +171,17 @@ function rm_test_file {
} }
function mk_test_dir { function mk_test_dir {
mkdir ${TEST_DIR} mkdir "${TEST_DIR}"
if [ ! -d ${TEST_DIR} ]; then if [ ! -d "${TEST_DIR}" ]; then
echo "Directory ${TEST_DIR} was not created" echo "Directory ${TEST_DIR} was not created"
exit 1 exit 1
fi fi
} }
function rm_test_dir { function rm_test_dir {
rmdir ${TEST_DIR} rmdir "${TEST_DIR}"
if [ -e $TEST_DIR ]; then if [ -e "${TEST_DIR}" ]; then
echo "Could not remove the test directory, it still exists: ${TEST_DIR}" echo "Could not remove the test directory, it still exists: ${TEST_DIR}"
exit 1 exit 1
fi fi
@ -178,18 +190,18 @@ function rm_test_dir {
# Create and cd to a unique directory for this test run # Create and cd to a unique directory for this test run
# Sets RUN_DIR to the name of the created directory # Sets RUN_DIR to the name of the created directory
function cd_run_dir { function cd_run_dir {
if [ "$TEST_BUCKET_MOUNT_POINT_1" = "" ]; then if [ "${TEST_BUCKET_MOUNT_POINT_1}" = "" ]; then
echo "TEST_BUCKET_MOUNT_POINT_1 variable not set" echo "TEST_BUCKET_MOUNT_POINT_1 variable not set"
exit 1 exit 1
fi fi
local RUN_DIR=${TEST_BUCKET_MOUNT_POINT_1}/${1} local RUN_DIR="${TEST_BUCKET_MOUNT_POINT_1}/${1}"
mkdir -p ${RUN_DIR} mkdir -p "${RUN_DIR}"
cd ${RUN_DIR} cd "${RUN_DIR}"
} }
function clean_run_dir { function clean_run_dir {
if [ -d ${RUN_DIR} ]; then if [ -d "${RUN_DIR}" ]; then
rm -rf ${RUN_DIR} || echo "Error removing ${RUN_DIR}" rm -rf "${RUN_DIR}" || echo "Error removing ${RUN_DIR}"
fi fi
} }
@ -204,14 +216,14 @@ function init_suite {
# report_pass TEST_NAME # report_pass TEST_NAME
function report_pass { function report_pass {
echo "$1 passed" echo "$1 passed"
TEST_PASSED_LIST+=($1) TEST_PASSED_LIST+=("$1")
} }
# Report a failing test case # Report a failing test case
# report_fail TEST_NAME # report_fail TEST_NAME
function report_fail { function report_fail {
echo "$1 failed" echo "$1 failed"
TEST_FAILED_LIST+=($1) TEST_FAILED_LIST+=("$1")
} }
# Add tests to the suite # Add tests to the suite
@ -231,37 +243,37 @@ function describe {
# directory in the bucket. An attempt to clean this directory is # directory in the bucket. An attempt to clean this directory is
# made after the test run. # made after the test run.
function run_suite { function run_suite {
orig_dir=$PWD orig_dir="${PWD}"
key_prefix="testrun-$RANDOM" key_prefix="testrun-${RANDOM}"
cd_run_dir $key_prefix cd_run_dir "${key_prefix}"
for t in "${TEST_LIST[@]}"; do for t in "${TEST_LIST[@]}"; do
# Ensure test input name differs every iteration # Ensure test input name differs every iteration
TEST_TEXT_FILE=test-s3fs.txt-$RANDOM TEST_TEXT_FILE="test-s3fs.txt-${RANDOM}"
TEST_DIR=testdir-$RANDOM TEST_DIR="testdir-${RANDOM}"
$t $key_prefix && rc=$? || rc=$? "${t}" "${key_prefix}" && rc=$? || rc=$?
if [[ $rc = 0 ]] ; then if [[ "${rc}" = 0 ]] ; then
report_pass $t report_pass "${t}"
else else
report_fail $t report_fail "${t}"
fi fi
done done
cd ${orig_dir} cd "${orig_dir}"
clean_run_dir clean_run_dir
for t in "${TEST_PASSED_LIST[@]}"; do for t in "${TEST_PASSED_LIST[@]}"; do
echo "PASS: $t" echo "PASS: ${t}"
done done
for t in "${TEST_FAILED_LIST[@]}"; do for t in "${TEST_FAILED_LIST[@]}"; do
echo "FAIL: $t" echo "FAIL: ${t}"
done done
local passed=${#TEST_PASSED_LIST[@]} local passed=${#TEST_PASSED_LIST[@]}
local failed=${#TEST_FAILED_LIST[@]} local failed=${#TEST_FAILED_LIST[@]}
echo "SUMMARY for $0: $passed tests passed. $failed tests failed." echo "SUMMARY for $0: ${passed} tests passed. ${failed} tests failed."
if [[ $failed != 0 ]]; then if [[ "${failed}" != 0 ]]; then
return 1 return 1
else else
return 0 return 0
@ -269,7 +281,7 @@ function run_suite {
} }
function get_ctime() { function get_ctime() {
if [ `uname` = "Darwin" ]; then if [ "$(uname)" = "Darwin" ]; then
stat -f "%c" "$1" stat -f "%c" "$1"
else else
stat -c "%Z" "$1" stat -c "%Z" "$1"
@ -277,7 +289,7 @@ function get_ctime() {
} }
function get_mtime() { function get_mtime() {
if [ `uname` = "Darwin" ]; then if [ "$(uname)" = "Darwin" ]; then
stat -f "%m" "$1" stat -f "%m" "$1"
else else
stat -c "%Y" "$1" stat -c "%Y" "$1"
@ -285,7 +297,7 @@ function get_mtime() {
} }
function get_atime() { function get_atime() {
if [ `uname` = "Darwin" ]; then if [ "$(uname)" = "Darwin" ]; then
stat -f "%a" "$1" stat -f "%a" "$1"
else else
stat -c "%X" "$1" stat -c "%X" "$1"
@ -293,7 +305,7 @@ function get_atime() {
} }
function get_permissions() { function get_permissions() {
if [ `uname` = "Darwin" ]; then if [ "$(uname)" = "Darwin" ]; then
stat -f "%p" "$1" stat -f "%p" "$1"
else else
stat -c "%a" "$1" stat -c "%a" "$1"
@ -301,7 +313,8 @@ function get_permissions() {
} }
function check_content_type() { function check_content_type() {
local INFO_STR=`aws_cli s3api head-object --bucket ${TEST_BUCKET_1} --key $1` local INFO_STR
INFO_STR=$(aws_cli s3api head-object --bucket "${TEST_BUCKET_1}" --key "$1")
if [[ "${INFO_STR}" != *"$2"* ]] if [[ "${INFO_STR}" != *"$2"* ]]
then then
echo "moved file content-type is not as expected expected:$2 got:${INFO_STR}" echo "moved file content-type is not as expected expected:$2 got:${INFO_STR}"
@ -310,8 +323,9 @@ function check_content_type() {
} }
function get_disk_avail_size() { function get_disk_avail_size() {
local DISK_AVAIL_SIZE=`BLOCKSIZE=$((1024 * 1024)) df $1 | awk '{print $4}' | tail -n 1` local DISK_AVAIL_SIZE
echo ${DISK_AVAIL_SIZE} DISK_AVAIL_SIZE=$(BLOCKSIZE=$((1024 * 1024)) df "$1" | awk '{print $4}' | tail -n 1)
echo "${DISK_AVAIL_SIZE}"
} }
function aws_cli() { function aws_cli() {
@ -319,12 +333,13 @@ function aws_cli() {
if [ -n "${S3FS_PROFILE}" ]; then if [ -n "${S3FS_PROFILE}" ]; then
FLAGS="--profile ${S3FS_PROFILE}" FLAGS="--profile ${S3FS_PROFILE}"
fi fi
aws $* --endpoint-url "${S3_URL}" --ca-bundle /tmp/keystore.pem $FLAGS # shellcheck disable=SC2086,SC2068
aws $@ --endpoint-url "${S3_URL}" --ca-bundle /tmp/keystore.pem ${FLAGS}
} }
function wait_for_port() { function wait_for_port() {
local PORT=$1 local PORT="$1"
for i in $(seq 30); do for _ in $(seq 30); do
if exec 3<>"/dev/tcp/127.0.0.1/${PORT}"; if exec 3<>"/dev/tcp/127.0.0.1/${PORT}";
then then
exec 3<&- # Close for read exec 3<&- # Close for read
@ -337,12 +352,12 @@ function wait_for_port() {
function make_random_string() { function make_random_string() {
if [ -n "$1" ]; then if [ -n "$1" ]; then
local END_POS=$1 local END_POS="$1"
else else
local END_POS=8 local END_POS=8
fi fi
${BASE64_BIN} --wrap=0 < /dev/urandom | tr -d /+ | head -c ${END_POS} "${BASE64_BIN}" --wrap=0 < /dev/urandom | tr -d /+ | head -c "${END_POS}"
return 0 return 0
} }