Added test by a shell script static analysis tool(ShellCheck)

This commit is contained in:
Takeshi Nakatani 2022-01-15 17:08:46 +00:00 committed by Andrew Gaul
parent 93d1c30d4d
commit 46014397d8
12 changed files with 876 additions and 738 deletions

View File

@ -101,6 +101,10 @@ jobs:
run: |
make cppcheck
- name: Shellcheck
run: |
make shellcheck
- name: Test suite
run: |
make ALL_TESTS=1 check -C test || (test/filter-suite-log.sh test/test-suite.log; exit 1)
@ -131,7 +135,7 @@ jobs:
- name: Install brew other packages
run: |
S3FS_BREW_PACKAGES='automake cppcheck python3 coreutils gnu-sed';
S3FS_BREW_PACKAGES='automake cppcheck python3 coreutils gnu-sed shellcheck';
for s3fs_brew_pkg in ${S3FS_BREW_PACKAGES}; do if brew list | grep -q ${s3fs_brew_pkg}; then if brew outdated | grep -q ${s3fs_brew_pkg}; then HOMEBREW_NO_AUTO_UPDATE=1 brew upgrade ${s3fs_brew_pkg}; fi; else HOMEBREW_NO_AUTO_UPDATE=1 brew install ${s3fs_brew_pkg}; fi; done;
- name: Install awscli
@ -152,6 +156,10 @@ jobs:
run: |
make cppcheck
- name: Shellcheck
run: |
make shellcheck
- name: Test suite
run: |
make check -C src

View File

@ -24,7 +24,7 @@ echo "${PRGNAME} [INFO] Start Linux helper for installing packages."
#-----------------------------------------------------------
# Common variables
#-----------------------------------------------------------
PRGNAME=`basename $0`
PRGNAME=$(basename "$0")
#-----------------------------------------------------------
# Parameter check
@ -40,8 +40,10 @@ fi
# Container OS variables
#-----------------------------------------------------------
CONTAINER_FULLNAME=$1
CONTAINER_OSNAME=`echo ${CONTAINER_FULLNAME} | sed 's/:/ /g' | awk '{print $1}'`
CONTAINER_OSVERSION=`echo ${CONTAINER_FULLNAME} | sed 's/:/ /g' | awk '{print $2}'`
# shellcheck disable=SC2034
CONTAINER_OSNAME=$(echo "${CONTAINER_FULLNAME}" | sed 's/:/ /g' | awk '{print $1}')
# shellcheck disable=SC2034
CONTAINER_OSVERSION=$(echo "${CONTAINER_FULLNAME}" | sed 's/:/ /g' | awk '{print $2}')
#-----------------------------------------------------------
# Common variables for pip
@ -53,6 +55,7 @@ INSTALL_AWSCLI_PACKAGES="awscli"
#-----------------------------------------------------------
# Parameters for configure(set environments)
#-----------------------------------------------------------
# shellcheck disable=SC2089
CONFIGURE_OPTIONS="CXXFLAGS='-O -std=c++03 -DS3FS_PTHREAD_ERRORCHECK=1' --prefix=/usr --with-openssl"
#-----------------------------------------------------------
@ -63,63 +66,81 @@ if [ "${CONTAINER_FULLNAME}" = "ubuntu:21.10" ]; then
PACKAGE_UPDATE_OPTIONS="update -y -qq"
INSTALL_PACKAGES="autoconf autotools-dev default-jre-headless fuse libfuse-dev libcurl4-openssl-dev libxml2-dev locales-all mime-support libtool pkg-config libssl-dev attr curl python3-pip"
INSTALL_CPPCHECK_OPTIONS=""
INSTALL_CHECKER_PKGS="cppcheck shellcheck"
INSTALL_CHECKER_PKG_OPTIONS=""
elif [ "${CONTAINER_FULLNAME}" = "ubuntu:20.04" ]; then
PACKAGE_MANAGER_BIN="apt-get"
PACKAGE_UPDATE_OPTIONS="update -y -qq"
INSTALL_PACKAGES="autoconf autotools-dev default-jre-headless fuse libfuse-dev libcurl4-openssl-dev libxml2-dev locales-all mime-support libtool pkg-config libssl-dev attr curl python3-pip"
INSTALL_CPPCHECK_OPTIONS=""
INSTALL_CHECKER_PKGS="cppcheck shellcheck"
INSTALL_CHECKER_PKG_OPTIONS=""
elif [ "${CONTAINER_FULLNAME}" = "ubuntu:18.04" ]; then
PACKAGE_MANAGER_BIN="apt-get"
PACKAGE_UPDATE_OPTIONS="update -y -qq"
INSTALL_PACKAGES="autoconf autotools-dev default-jre-headless fuse libfuse-dev libcurl4-openssl-dev libxml2-dev locales-all mime-support libtool pkg-config libssl-dev attr curl python3-pip"
INSTALL_CPPCHECK_OPTIONS=""
INSTALL_CHECKER_PKGS="cppcheck shellcheck"
INSTALL_CHECKER_PKG_OPTIONS=""
elif [ "${CONTAINER_FULLNAME}" = "ubuntu:16.04" ]; then
PACKAGE_MANAGER_BIN="apt-get"
PACKAGE_UPDATE_OPTIONS="update -y -qq"
INSTALL_PACKAGES="autoconf autotools-dev default-jre-headless fuse libfuse-dev libcurl4-openssl-dev libxml2-dev locales-all mime-support libtool pkg-config libssl-dev attr curl python3-pip"
INSTALL_CPPCHECK_OPTIONS=""
INSTALL_CHECKER_PKGS="cppcheck shellcheck"
INSTALL_CHECKER_PKG_OPTIONS=""
elif [ "${CONTAINER_FULLNAME}" = "debian:bullseye" ]; then
PACKAGE_MANAGER_BIN="apt-get"
PACKAGE_UPDATE_OPTIONS="update -y -qq"
INSTALL_PACKAGES="autoconf autotools-dev default-jre-headless fuse libfuse-dev libcurl4-openssl-dev libxml2-dev locales-all mime-support libtool pkg-config libssl-dev attr curl procps python3-pip"
INSTALL_CPPCHECK_OPTIONS=""
INSTALL_CHECKER_PKGS="cppcheck shellcheck"
INSTALL_CHECKER_PKG_OPTIONS=""
elif [ "${CONTAINER_FULLNAME}" = "debian:buster" ]; then
PACKAGE_MANAGER_BIN="apt-get"
PACKAGE_UPDATE_OPTIONS="update -y -qq"
INSTALL_PACKAGES="autoconf autotools-dev default-jre-headless fuse libfuse-dev libcurl4-openssl-dev libxml2-dev locales-all mime-support libtool pkg-config libssl-dev attr curl procps python3-pip"
INSTALL_CPPCHECK_OPTIONS=""
INSTALL_CHECKER_PKGS="cppcheck shellcheck"
INSTALL_CHECKER_PKG_OPTIONS=""
elif [ "${CONTAINER_FULLNAME}" = "debian:stretch" ]; then
PACKAGE_MANAGER_BIN="apt-get"
PACKAGE_UPDATE_OPTIONS="update -y -qq"
INSTALL_PACKAGES="autoconf autotools-dev default-jre-headless fuse libfuse-dev libcurl4-openssl-dev libxml2-dev locales-all mime-support libtool pkg-config libssl-dev attr curl procps python3-pip"
INSTALL_CPPCHECK_OPTIONS=""
INSTALL_CHECKER_PKGS="cppcheck shellcheck"
INSTALL_CHECKER_PKG_OPTIONS=""
elif [ "${CONTAINER_FULLNAME}" = "centos:centos8" ]; then
PACKAGE_MANAGER_BIN="dnf"
PACKAGE_UPDATE_OPTIONS="update -y -qq"
# [NOTE]
# Installing ShellCheck on CentOS 8 is not easy.
# Give up to run ShellCheck on CentOS 8 as we don't have to run ShellChek on all operating systems.
#
INSTALL_PACKAGES="curl-devel fuse fuse-devel gcc libstdc++-devel gcc-c++ glibc-langpack-en java-11-openjdk-headless libxml2-devel mailcap git automake make openssl-devel attr diffutils curl python3"
INSTALL_CPPCHECK_OPTIONS="--enablerepo=powertools"
INSTALL_CHECKER_PKGS="cppcheck"
INSTALL_CHECKER_PKG_OPTIONS="--enablerepo=powertools"
elif [ "${CONTAINER_FULLNAME}" = "centos:centos7" ]; then
PACKAGE_MANAGER_BIN="yum"
PACKAGE_UPDATE_OPTIONS="update -y"
# [NOTE]
# ShellCheck version(0.3.8) is too low to check.
# And in this version, it cannot be passed due to following error.
# "shellcheck: ./test/integration-test-main.sh: hGetContents: invalid argument (invalid byte sequence)"
#
INSTALL_PACKAGES="curl-devel fuse fuse-devel gcc libstdc++-devel gcc-c++ glibc-langpack-en java-11-openjdk-headless libxml2-devel mailcap git automake make openssl-devel attr curl python3 epel-release"
INSTALL_CPPCHECK_OPTIONS="--enablerepo=epel"
INSTALL_CHECKER_PKGS="cppcheck"
INSTALL_CHECKER_PKG_OPTIONS="--enablerepo=epel"
elif [ "${CONTAINER_FULLNAME}" = "fedora:35" ]; then
PACKAGE_MANAGER_BIN="dnf"
@ -127,14 +148,16 @@ elif [ "${CONTAINER_FULLNAME}" = "fedora:35" ]; then
# TODO: Cannot use java-latest-openjdk (17) due to modules issue in S3Proxy/jclouds/Guice
INSTALL_PACKAGES="curl-devel fuse fuse-devel gcc libstdc++-devel gcc-c++ glibc-langpack-en java-11-openjdk-headless libxml2-devel mailcap git automake make openssl-devel curl attr diffutils procps python3-pip"
INSTALL_CPPCHECK_OPTIONS=""
INSTALL_CHECKER_PKGS="cppcheck ShellCheck"
INSTALL_CHECKER_PKG_OPTIONS=""
elif [ "${CONTAINER_FULLNAME}" = "opensuse/leap:15" ]; then
PACKAGE_MANAGER_BIN="zypper"
PACKAGE_UPDATE_OPTIONS="refresh"
INSTALL_PACKAGES="automake curl-devel fuse fuse-devel gcc-c++ java-11-openjdk-headless libxml2-devel make openssl-devel python3-pip curl attr"
INSTALL_CPPCHECK_OPTIONS=""
INSTALL_PACKAGES="automake curl-devel fuse fuse-devel gcc-c++ java-11-openjdk-headless libxml2-devel make openssl-devel python3-pip curl attr ShellCheck"
INSTALL_CHECKER_PKGS="cppcheck ShellCheck"
INSTALL_CHECKER_PKG_OPTIONS=""
else
echo "No container configured for: ${CONTAINER_FULLNAME}"
@ -148,16 +171,16 @@ fi
# Update packages (ex. apt-get update -y -qq)
#
echo "${PRGNAME} [INFO] Updates."
${PACKAGE_MANAGER_BIN} ${PACKAGE_UPDATE_OPTIONS}
/bin/sh -c "${PACKAGE_MANAGER_BIN} ${PACKAGE_UPDATE_OPTIONS}"
#
# Install packages ( with cppcheck )
#
echo "${PRGNAME} [INFO] Install packages."
${PACKAGE_MANAGER_BIN} install -y ${INSTALL_PACKAGES}
/bin/sh -c "${PACKAGE_MANAGER_BIN} install -y ${INSTALL_PACKAGES}"
echo "${PRGNAME} [INFO] Install cppcheck package."
${PACKAGE_MANAGER_BIN} ${INSTALL_CPPCHECK_OPTIONS} install -y cppcheck
/bin/sh -c "${PACKAGE_MANAGER_BIN} ${INSTALL_CHECKER_PKG_OPTIONS} install -y ${INSTALL_CHECKER_PKGS}"
# Check Java version
java -version
@ -166,16 +189,19 @@ java -version
# Install awscli
#
echo "${PRGNAME} [INFO] Install awscli package."
${PIP_BIN} install ${PIP_OPTIONS} ${INSTALL_AWSCLI_PACKAGES}
${PIP_BIN} install ${PIP_OPTIONS} rsa
/bin/sh -c "${PIP_BIN} install ${PIP_OPTIONS} ${INSTALL_AWSCLI_PACKAGES}"
/bin/sh -c "${PIP_BIN} install ${PIP_OPTIONS} rsa"
#-----------------------------------------------------------
# Set environment for configure
#-----------------------------------------------------------
echo "${PRGNAME} [INFO] Set environment for configure options"
# shellcheck disable=SC2090
export CONFIGURE_OPTIONS
echo "${PRGNAME} [INFO] Finish Linux helper for installing packages."
exit 0
#

View File

@ -17,6 +17,7 @@
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
######################################################################
SUBDIRS=src test doc
EXTRA_DIST=doc default_commit_hash
@ -28,6 +29,8 @@ dist-hook:
release : dist ../utils/release.sh
../utils/release.sh $(DIST_ARCHIVES)
.PHONY: cppcheck shellcheck
cppcheck:
cppcheck --quiet --error-exitcode=1 \
--inline-suppr \
@ -43,6 +46,35 @@ cppcheck:
--suppress=unmatchedSuppression \
src/ test/
#
# ShellCheck
#
SHELLCHECK_CMD = shellcheck
SHELLCHECK_SH_OPT = --shell=sh
SHELLCHECK_BASH_OPT = --shell=bash
# [NOTE]
# To control error warnings as a whole, specify the "SC<number>" with the following variables.
#
SHELLCHECK_COMMON_IGN = --exclude=SC1091
SHELLCHECK_CUSTOM_IGN = --exclude=SC1091
shellcheck:
@if type shellcheck > /dev/null 2>&1; then \
echo "* ShellCheck version"; \
$(SHELLCHECK_CMD) --version; \
echo ""; \
echo "* Check all sh files with ShellCheck"; \
LC_ALL=C.UTF-8 $(SHELLCHECK_CMD) $(SHELLCHECK_SH_OPT) $(SHELLCHECK_COMMON_IGN) $$(grep '#![[:space:]]*/bin/sh' $$(find . -type f -name \*.sh) | sed -e 's|^\(.*\):#\!.*$$|\1|g') || exit 1; \
echo "-> No error was detected."; \
echo ""; \
echo "* Check all bash files with ShellCheck"; \
LC_ALL=C.UTF-8 $(SHELLCHECK_CMD) $(SHELLCHECK_BASH_OPT) $(SHELLCHECK_COMMON_IGN) $$(grep '#![[:space:]]*/bin/bash' $$(find . -type f -name \*.sh) | sed -e 's|^\(.*\):#\!.*$$|\1|g') || exit 1; \
echo "-> No error was detected."; \
else \
echo "* ShellCheck is not installed, so skip this."; \
fi
#
# Local variables:
# tab-width: 4

View File

@ -1,5 +1,5 @@
#!/bin/sh
#
# This file is part of S3FS.
#
# Copyright 2009, 2010 Free Software Foundation, Inc.
@ -22,14 +22,12 @@
echo "--- Make commit hash file -------"
SHORTHASH="unknown"
type git > /dev/null 2>&1
if [ $? -eq 0 -a -d .git ]; then
RESULT=`git rev-parse --short HEAD`
if [ $? -eq 0 ]; then
SHORTHASH=${RESULT}
if command -v git > /dev/null 2>&1 && test -d .git; then
if RESULT=$(git rev-parse --short HEAD); then
SHORTHASH="${RESULT}"
fi
fi
echo ${SHORTHASH} > default_commit_hash
echo "${SHORTHASH}" > default_commit_hash
echo "--- Finished commit hash file ---"

View File

@ -28,25 +28,25 @@ func_usage()
echo ""
}
PRGNAME=`basename $0`
SCRIPTDIR=`dirname $0`
S3FSDIR=`cd ${SCRIPTDIR}/..; pwd`
TOPDIR=`cd ${S3FSDIR}/test; pwd`
PRGNAME=$(basename "$0")
SCRIPTDIR=$(dirname "$0")
S3FSDIR=$(cd "${SCRIPTDIR}"/.. || exit 1; pwd)
TOPDIR=$(cd "${S3FSDIR}"/test || exit 1; pwd)
SUITELOG="${TOPDIR}/test-suite.log"
TMP_LINENO_FILE="/tmp/.lineno.tmp"
while [ $# -ne 0 ]; do
if [ "X$1" = "X" ]; then
break
elif [ "X$1" = "X-h" -o "X$1" = "X-H" -o "X$1" = "X--help" -o "X$1" = "X--HELP" ]; then
func_usage ${PRGNAME}
elif [ "X$1" = "X-h" ] || [ "X$1" = "X-H" ] || [ "X$1" = "X--help" ] || [ "X$1" = "X--HELP" ]; then
func_usage "${PRGNAME}"
exit 0
else
SUITELOG=$1
fi
shift
done
if [ ! -f ${SUITELOG} ]; then
if [ ! -f "${SUITELOG}" ]; then
echo "[ERROR] not found ${SUITELOG} log file."
exit 1
fi
@ -59,75 +59,77 @@ fi
# 2 : passed line of end of one small test(specified in test-utils.sh)
# 3 : failed line of end of one small test(specified in test-utils.sh)
#
grep -n -e 'test_.*: ".*"' -o -e 'test_.* passed' -o -e 'test_.* failed' ${SUITELOG} 2>/dev/null | sed 's/:test_.*: ".*"/ 1/g' | sed 's/:test_.* passed/ 2/g' | sed 's/:test_.* failed/ 3/g' > ${TMP_LINENO_FILE}
grep -n -e 'test_.*: ".*"' -o -e 'test_.* passed' -o -e 'test_.* failed' "${SUITELOG}" 2>/dev/null | sed 's/:test_.*: ".*"/ 1/g' | sed 's/:test_.* passed/ 2/g' | sed 's/:test_.* failed/ 3/g' > "${TMP_LINENO_FILE}"
#
# Loop for printing result
#
prev_line_type=0
prev_line_number=1
while read line; do
while read -r line; do
# line is "<line number> <line type>"
number_type=($line)
#
# shellcheck disable=SC2206
number_type=(${line})
head_line_cnt=`expr ${number_type[0]} - 1`
tail_line_cnt=`expr ${number_type[0]} - ${prev_line_number}`
head_line_cnt=$((number_type[0] - 1))
tail_line_cnt=$((number_type[0] - prev_line_number))
if [ ${number_type[1]} -eq 2 ]; then
if [ "${number_type[1]}" -eq 2 ]; then
echo ""
fi
if [ ${prev_line_type} -eq 1 ]; then
if [ ${number_type[1]} -eq 2 ]; then
if [ "${prev_line_type}" -eq 1 ]; then
if [ "${number_type[1]}" -eq 2 ]; then
# if passed, cut s3fs information messages
head -${head_line_cnt} ${SUITELOG} | tail -${tail_line_cnt} | grep -v -e '[0-9]\+\%' | grep -v -e '^s3fs: ' -a -e '\[INF\]'
elif [ ${number_type[1]} -eq 3 ]; then
head "-${head_line_cnt}" "${SUITELOG}" | tail "-${tail_line_cnt}" | grep -v -e '[0-9]\+\%' | grep -v -e '^s3fs: ' -a -e '\[INF\]'
elif [ "${number_type[1]}" -eq 3 ]; then
# if failed, print all
head -${head_line_cnt} ${SUITELOG} | tail -${tail_line_cnt} | grep -v -e '[0-9]\+\%'
head "-${head_line_cnt}" "${SUITELOG}" | tail "-${tail_line_cnt}" | grep -v -e '[0-9]\+\%'
else
# there is start keyword but not end keyword, so print all
head -${head_line_cnt} ${SUITELOG} | tail -${tail_line_cnt} | grep -v -e '[0-9]\+\%'
head "-${head_line_cnt}" "${SUITELOG}" | tail "-${tail_line_cnt}" | grep -v -e '[0-9]\+\%'
fi
elif [ ${prev_line_type} -eq 2 -o ${prev_line_type} -eq 3 ]; then
if [ ${number_type[1]} -eq 2 -o ${number_type[1]} -eq 3 ]; then
elif [ "${prev_line_type}" -eq 2 ] || [ "${prev_line_type}" -eq 3 ]; then
if [ "${number_type[1]}" -eq 2 ] || [ "${number_type[1]}" -eq 3 ]; then
# previous is end of chmpx, but this type is end of chmpx without start keyword. then print all
head -${head_line_cnt} ${SUITELOG} | tail -${tail_line_cnt} | grep -v -e '[0-9]\+\%'
head "-${head_line_cnt}" "${SUITELOG}" | tail "-${tail_line_cnt}" | grep -v -e '[0-9]\+\%'
else
# this area is not from start to end, cut s3fs information messages
head -${head_line_cnt} ${SUITELOG} | tail -${tail_line_cnt} | grep -v -e '[0-9]\+\%' | grep -v -e '^s3fs: ' -a -e '\[INF\]'
head "-${head_line_cnt}" "${SUITELOG}" | tail "-${tail_line_cnt}" | grep -v -e '[0-9]\+\%' | grep -v -e '^s3fs: ' -a -e '\[INF\]'
fi
else
if [ ${number_type[1]} -eq 2 -o ${number_type[1]} -eq 3 ]; then
if [ "${number_type[1]}" -eq 2 ] || [ "${number_type[1]}" -eq 3 ]; then
# previous is normal, but this type is end of chmpx without start keyword. then print all
head -${head_line_cnt} ${SUITELOG} | tail -${tail_line_cnt} | grep -v -e '[0-9]\+\%'
head "-${head_line_cnt}" "${SUITELOG}" | tail "-${tail_line_cnt}" | grep -v -e '[0-9]\+\%'
else
# this area is normal, cut s3fs information messages
head -${head_line_cnt} ${SUITELOG} | tail -${tail_line_cnt} | grep -v -e '[0-9]\+\%' | grep -v -e '^s3fs: ' -a -e '\[INF\]'
head "-${head_line_cnt}" "${SUITELOG}" | tail "-${tail_line_cnt}" | grep -v -e '[0-9]\+\%' | grep -v -e '^s3fs: ' -a -e '\[INF\]'
fi
fi
if [ ${number_type[1]} -eq 3 ]; then
if [ "${number_type[1]}" -eq 3 ]; then
echo ""
fi
prev_line_type=${number_type[1]}
prev_line_number=${number_type[0]}
prev_line_type="${number_type[1]}"
prev_line_number="${number_type[0]}"
done < ${TMP_LINENO_FILE}
done < "${TMP_LINENO_FILE}"
#
# Print rest lines
#
file_line_cnt=`wc -l ${SUITELOG} | awk '{print $1}'`
tail_line_cnt=`expr ${file_line_cnt} - ${prev_line_number}`
file_line_cnt=$(wc -l "${SUITELOG}" | awk '{print $1}')
tail_line_cnt=$((file_line_cnt - prev_line_number))
if [ ${prev_line_type} -eq 1 ]; then
tail -${tail_line_cnt} ${SUITELOG} | grep -v -e '[0-9]\+\%'
if [ "${prev_line_type}" -eq 1 ]; then
tail "-${tail_line_cnt}" "${SUITELOG}" | grep -v -e '[0-9]\+\%'
else
tail -${tail_line_cnt} ${SUITELOG} | grep -v -e '[0-9]\+\%' | grep -v -e '^s3fs: ' -a -e '\[INF\]'
tail "-${tail_line_cnt}" "${SUITELOG}" | grep -v -e '[0-9]\+\%' | grep -v -e '^s3fs: ' -a -e '\[INF\]'
fi
#
# Remove temp file
#
rm -f ${TMP_LINENO_FILE}
rm -f "${TMP_LINENO_FILE}"
exit 0

View File

@ -66,59 +66,65 @@ set -o pipefail
S3FS=../src/s3fs
# Allow these defaulted values to be overridden
: ${S3_URL:="https://127.0.0.1:8080"}
: ${S3_ENDPOINT:="us-east-1"}
: ${S3FS_CREDENTIALS_FILE:="passwd-s3fs"}
: ${TEST_BUCKET_1:="s3fs-integration-test"}
: "${S3_URL:="https://127.0.0.1:8080"}"
: "${S3_ENDPOINT:="us-east-1"}"
: "${S3FS_CREDENTIALS_FILE:="passwd-s3fs"}"
: "${TEST_BUCKET_1:="s3fs-integration-test"}"
export TEST_BUCKET_1
export S3_URL
export S3_ENDPOINT
export TEST_SCRIPT_DIR=`pwd`
TEST_SCRIPT_DIR=$(pwd)
export TEST_SCRIPT_DIR
export TEST_BUCKET_MOUNT_POINT_1=${TEST_BUCKET_1}
S3PROXY_VERSION="1.9.0"
S3PROXY_BINARY=${S3PROXY_BINARY-"s3proxy-${S3PROXY_VERSION}"}
S3PROXY_BINARY="${S3PROXY_BINARY-"s3proxy-${S3PROXY_VERSION}"}"
CHAOS_HTTP_PROXY_VERSION="1.1.0"
CHAOS_HTTP_PROXY_BINARY="chaos-http-proxy-${CHAOS_HTTP_PROXY_VERSION}"
if [ ! -f "$S3FS_CREDENTIALS_FILE" ]
then
echo "Missing credentials file: $S3FS_CREDENTIALS_FILE"
echo "Missing credentials file: ${S3FS_CREDENTIALS_FILE}"
exit 1
fi
chmod 600 "$S3FS_CREDENTIALS_FILE"
chmod 600 "${S3FS_CREDENTIALS_FILE}"
if [ -z "${S3FS_PROFILE}" ]; then
export AWS_ACCESS_KEY_ID=$(cut -d: -f1 ${S3FS_CREDENTIALS_FILE})
export AWS_SECRET_ACCESS_KEY=$(cut -d: -f2 ${S3FS_CREDENTIALS_FILE})
AWS_ACCESS_KEY_ID=$(cut -d: -f1 "${S3FS_CREDENTIALS_FILE}")
export AWS_ACCESS_KEY_ID
AWS_SECRET_ACCESS_KEY=$(cut -d: -f2 "${S3FS_CREDENTIALS_FILE}")
export AWS_SECRET_ACCESS_KEY
fi
if [ ! -d $TEST_BUCKET_MOUNT_POINT_1 ]
then
mkdir -p $TEST_BUCKET_MOUNT_POINT_1
if [ ! -d "${TEST_BUCKET_MOUNT_POINT_1}" ]; then
mkdir -p "${TEST_BUCKET_MOUNT_POINT_1}"
fi
# This function execute the function parameters $1 times
# before giving up, with 1 second delays.
function retry {
local N=$1; shift;
local N="$1"
shift
rc=0
for i in $(seq $N); do
for _ in $(seq "${N}"); do
echo "Trying: $*"
eval $@; rc=$?
if [ $rc = 0 ]; then
# shellcheck disable=SC2068,SC2294
eval $@
rc=$?
if [ "${rc}" -eq 0 ]; then
break
fi
sleep 1
echo "Retrying: $*"
done
if [ $rc != 0 ]; then
if [ "${rc}" -ne 0 ]; then
echo "timeout waiting for $*"
fi
return $rc
return "${rc}"
}
# Proxy is not started if S3PROXY_BINARY is an empty string
@ -145,7 +151,7 @@ function start_s3proxy {
echo -e 'password\npassword\n\n\n\n\n\n\nyes' | keytool -genkey -keystore /tmp/keystore.jks -keyalg RSA -keysize 2048 -validity 365 -ext SAN=IP:127.0.0.1
echo password | keytool -exportcert -keystore /tmp/keystore.jks -rfc -file /tmp/keystore.pem
${STDBUF_BIN} -oL -eL java -jar "$S3PROXY_BINARY" --properties $S3PROXY_CONFIG &
"${STDBUF_BIN}" -oL -eL java -jar "${S3PROXY_BINARY}" --properties "${S3PROXY_CONFIG}" &
S3PROXY_PID=$!
# wait for S3Proxy to start
@ -159,7 +165,7 @@ function start_s3proxy {
chmod +x "${CHAOS_HTTP_PROXY_BINARY}"
fi
${STDBUF_BIN} -oL -eL java -jar ${CHAOS_HTTP_PROXY_BINARY} --properties chaos-http-proxy.conf &
"${STDBUF_BIN}" -oL -eL java -jar "${CHAOS_HTTP_PROXY_BINARY}" --properties chaos-http-proxy.conf &
CHAOS_HTTP_PROXY_PID=$!
# wait for Chaos HTTP Proxy to start
@ -170,12 +176,12 @@ function start_s3proxy {
function stop_s3proxy {
if [ -n "${S3PROXY_PID}" ]
then
kill $S3PROXY_PID
kill "${S3PROXY_PID}"
fi
if [ -n "${CHAOS_HTTP_PROXY_PID}" ]
then
kill $CHAOS_HTTP_PROXY_PID
kill "${CHAOS_HTTP_PROXY_PID}"
fi
}
@ -200,7 +206,7 @@ function start_s3fs {
fi
# On OSX only, we need to specify the direct_io and auto_cache flag.
if [ `uname` = "Darwin" ]; then
if [ "$(uname)" = "Darwin" ]; then
local DIRECT_IO_OPT="-o direct_io -o auto_cache"
else
local DIRECT_IO_OPT=""
@ -215,7 +221,7 @@ function start_s3fs {
# Therefore, when it is macos, it is not executed via stdbuf.
# This patch may be temporary, but no other method has been found at this time.
#
if [ `uname` = "Darwin" ]; then
if [ "$(uname)" = "Darwin" ]; then
local VIA_STDBUF_CMDLINE=""
else
local VIA_STDBUF_CMDLINE="${STDBUF_BIN} -oL -eL"
@ -238,15 +244,17 @@ function start_s3fs {
#
# subshell with set -x to log exact invocation of s3fs-fuse
# shellcheck disable=SC2086
(
set -x
${VIA_STDBUF_CMDLINE} \
${VALGRIND_EXEC} ${S3FS} \
$TEST_BUCKET_1 \
$TEST_BUCKET_MOUNT_POINT_1 \
${VALGRIND_EXEC} \
${S3FS} \
${TEST_BUCKET_1} \
${TEST_BUCKET_MOUNT_POINT_1} \
-o use_path_request_style \
-o url=${S3_URL} \
-o endpoint=${S3_ENDPOINT} \
-o url="${S3_URL}" \
-o endpoint="${S3_ENDPOINT}" \
-o no_check_certificate \
-o ssl_verify_hostname=0 \
-o use_xattr=1 \
@ -255,37 +263,39 @@ function start_s3fs {
${DIRECT_IO_OPT} \
-o stat_cache_expire=1 \
-o stat_cache_interval_expire=1 \
-o dbglevel=${DBGLEVEL:=info} \
-o dbglevel="${DBGLEVEL:=info}" \
-o no_time_stamp_msg \
-o retries=3 \
-f \
"${@}" &
echo $! >&3
) 3>pid | ${STDBUF_BIN} -oL -eL ${SED_BIN} ${SED_BUFFER_FLAG} "s/^/s3fs: /" &
) 3>pid | "${STDBUF_BIN}" -oL -eL "${SED_BIN}" "${SED_BUFFER_FLAG}" "s/^/s3fs: /" &
sleep 1
export S3FS_PID=$(<pid)
S3FS_PID=$(<pid)
export S3FS_PID
rm -f pid
if [ `uname` = "Darwin" ]; then
if [ "$(uname)" = "Darwin" ]; then
local TRYCOUNT=0
while [ $TRYCOUNT -le ${RETRIES:=20} ]; do
df | grep -q $TEST_BUCKET_MOUNT_POINT_1; rc=$?
if [ $rc -eq 0 ]; then
while [ "${TRYCOUNT}" -le "${RETRIES:=20}" ]; do
df | grep -q "${TEST_BUCKET_MOUNT_POINT_1}"
rc=$?
if [ "${rc}" -eq 0 ]; then
break;
fi
sleep 1
TRYCOUNT=`expr ${TRYCOUNT} + 1`
TRYCOUNT=$((TRYCOUNT + 1))
done
if [ $rc -ne 0 ]; then
if [ "${rc}" -ne 0 ]; then
exit 1
fi
else
retry ${RETRIES:=20} grep -q $TEST_BUCKET_MOUNT_POINT_1 /proc/mounts || exit 1
retry "${RETRIES:=20}" grep -q "${TEST_BUCKET_MOUNT_POINT_1}" /proc/mounts || exit 1
fi
# Quick way to start system up for manual testing with options under test
if [[ -n ${INTERACT} ]]; then
echo "Mountpoint $TEST_BUCKET_MOUNT_POINT_1 is ready"
if [[ -n "${INTERACT}" ]]; then
echo "Mountpoint ${TEST_BUCKET_MOUNT_POINT_1} is ready"
echo "control-C to quit"
sleep infinity
exit 0
@ -294,13 +304,13 @@ function start_s3fs {
function stop_s3fs {
# Retry in case file system is in use
if [ `uname` = "Darwin" ]; then
if df | grep -q $TEST_BUCKET_MOUNT_POINT_1; then
retry 10 df "|" grep -q $TEST_BUCKET_MOUNT_POINT_1 "&&" umount $TEST_BUCKET_MOUNT_POINT_1
if [ "$(uname)" = "Darwin" ]; then
if df | grep -q "${TEST_BUCKET_MOUNT_POINT_1}"; then
retry 10 df "|" grep -q "${TEST_BUCKET_MOUNT_POINT_1}" "&&" umount "${TEST_BUCKET_MOUNT_POINT_1}"
fi
else
if grep -q $TEST_BUCKET_MOUNT_POINT_1 /proc/mounts; then
retry 10 grep -q $TEST_BUCKET_MOUNT_POINT_1 /proc/mounts "&&" fusermount -u $TEST_BUCKET_MOUNT_POINT_1
if grep -q "${TEST_BUCKET_MOUNT_POINT_1}" /proc/mounts; then
retry 10 grep -q "${TEST_BUCKET_MOUNT_POINT_1}" /proc/mounts "&&" fusermount -u "${TEST_BUCKET_MOUNT_POINT_1}"
fi
fi
}

File diff suppressed because it is too large Load Diff

View File

@ -40,3 +40,12 @@ int main(int argc, char *argv[])
}
return 0;
}
/*
* Local variables:
* tab-width: 4
* c-basic-offset: 4
* End:
* vim600: expandtab sw=4 ts=4 fdm=marker
* vim<600: expandtab sw=4 ts=4
*/

View File

@ -40,25 +40,25 @@ UsageFunction()
}
### Check parameters
WHOAMI=`whoami`
OWNNAME=`basename $0`
WHOAMI=$(whoami)
OWNNAME=$(basename "$0")
AUTOYES="no"
ALLYES="no"
DIRPARAM=""
while [ "$1" != "" ]; do
if [ "X$1" = "X-help" -o "X$1" = "X-h" -o "X$1" = "X-H" ]; then
UsageFunction $OWNNAME
if [ "X$1" = "X-help" ] || [ "X$1" = "X-h" ] || [ "X$1" = "X-H" ]; then
UsageFunction "${OWNNAME}"
exit 0
elif [ "X$1" = "X-y" -o "X$1" = "X-Y" ]; then
elif [ "X$1" = "X-y" ] || [ "X$1" = "X-Y" ]; then
AUTOYES="yes"
elif [ "X$1" = "X-all" -o "X$1" = "X-ALL" ]; then
elif [ "X$1" = "X-all" ] || [ "X$1" = "X-ALL" ]; then
ALLYES="yes"
else
if [ "X$DIRPARAM" != "X" ]; then
echo "*** Input error."
echo ""
UsageFunction $OWNNAME
UsageFunction "${OWNNAME}"
exit 1
fi
DIRPARAM=$1
@ -68,7 +68,7 @@ done
if [ "X$DIRPARAM" = "X" ]; then
echo "*** Input error."
echo ""
UsageFunction $OWNNAME
UsageFunction "${OWNNAME}"
exit 1
fi
@ -88,18 +88,17 @@ echo "Please execute this program by responsibility of your own."
echo "#############################################################################"
echo ""
DATE=`date +'%Y%m%d-%H%M%S'`
LOGFILE="$OWNNAME-$DATE.log"
DATE=$(date +'%Y%m%d-%H%M%S')
LOGFILE="${OWNNAME}-${DATE}.log"
echo -n "Start to merge directory object... [$DIRPARAM]"
echo "# Start to merge directory object... [$DIRPARAM]" >> $LOGFILE
echo -n "# DATE : " >> $LOGFILE
echo `date` >> $LOGFILE
echo -n "# BASEDIR : " >> $LOGFILE
echo `pwd` >> $LOGFILE
echo -n "# TARGET PATH : " >> $LOGFILE
echo $DIRPARAM >> $LOGFILE
echo "" >> $LOGFILE
echo "Start to merge directory object... [${DIRPARAM}]"
{
echo "# Start to merge directory object... [${DIRPARAM}]"
echo "# DATE : $(date)"
echo "# BASEDIR : $(pwd)"
echo "# TARGET PATH : ${DIRPARAM}"
echo ""
} > "${LOGFILE}"
if [ "$AUTOYES" = "yes" ]; then
echo "(no confirmation)"
@ -109,80 +108,84 @@ fi
echo ""
### Get Directory list
DIRLIST=`find $DIRPARAM -type d -print | grep -v ^\.$`
DIRLIST=$(find "${DIRPARAM}" -type d -print | grep -v ^\.$)
#
# Main loop
#
for DIR in $DIRLIST; do
### Skip "." and ".." directories
BASENAME=`basename $DIR`
if [ "$BASENAME" = "." -o "$BASENAME" = ".." ]; then
BASENAME=$(basename "${DIR}")
if [ "${BASENAME}" = "." ] || [ "${BASENAME}" = ".." ]; then
continue
fi
if [ "$ALLYES" = "no" ]; then
if [ "${ALLYES}" = "no" ]; then
### Skip "d---------" directories.
### Other clients make directory object "dir/" which don't have
### "x-amz-meta-mode" attribute.
### Then these directories is "d---------", it is target directory.
DIRPERMIT=`ls -ld --time-style=+'%Y%m%d%H%M' $DIR | awk '{print $1}'`
if [ "$DIRPERMIT" != "d---------" ]; then
# shellcheck disable=SC2012
DIRPERMIT=$(ls -ld --time-style=+'%Y%m%d%H%M' "${DIR}" | awk '{print $1}')
if [ "${DIRPERMIT}" != "d---------" ]; then
continue
fi
fi
### Confirm
ANSWER=""
if [ "$AUTOYES" = "yes" ]; then
if [ "${AUTOYES}" = "yes" ]; then
ANSWER="y"
fi
while [ "X$ANSWER" != "XY" -a "X$ANSWER" != "Xy" -a "X$ANSWER" != "XN" -a "X$ANSWER" != "Xn" ]; do
echo -n "Do you merge $DIR? (y/n): "
read ANSWER
while [ "X${ANSWER}" != "XY" ] && [ "X${ANSWER}" != "Xy" ] && [ "X${ANSWER}" != "XN" ] && [ "X${ANSWER}" != "Xn" ]; do
printf "%s" "Do you merge ${DIR} ? (y/n): "
read -r ANSWER
done
if [ "X$ANSWER" != "XY" -a "X$ANSWER" != "Xy" ]; then
if [ "X${ANSWER}" != "XY" ] && [ "X${ANSWER}" != "Xy" ]; then
continue
fi
### Do
CHOWN=`ls -ld --time-style=+'%Y%m%d%H%M' $DIR | awk '{print $3":"$4" "$7}'`
CHMOD=`ls -ld --time-style=+'%Y%m%d%H%M' $DIR | awk '{print $7}'`
TOUCH=`ls -ld --time-style=+'%Y%m%d%H%M' $DIR | awk '{print $6" "$7}'`
# shellcheck disable=SC2012
CHOWN=$(ls -ld --time-style=+'%Y%m%d%H%M' "${DIR}" | awk '{print $3":"$4" "$7}')
# shellcheck disable=SC2012
CHMOD=$(ls -ld --time-style=+'%Y%m%d%H%M' "${DIR}" | awk '{print $7}')
# shellcheck disable=SC2012
TOUCH=$(ls -ld --time-style=+'%Y%m%d%H%M' "${DIR}" | awk '{print $6" "$7}')
echo -n "*** Merge $DIR : "
echo -n " $DIR : " >> $LOGFILE
printf "%s" "*** Merge ${DIR} : "
printf "%s" " ${DIR} : " >> "${LOGFILE}"
chmod 755 $CHMOD > /dev/null 2>&1
chmod 755 "${CHMOD}" > /dev/null 2>&1
RESULT=$?
if [ $RESULT -ne 0 ]; then
if [ "${RESULT}" -ne 0 ]; then
echo "Failed(chmod)"
echo "Failed(chmod)" >> $LOGFILE
echo "Failed(chmod)" >> "${LOGFILE}"
continue
fi
chown $CHOWN > /dev/null 2>&1
chown "${CHOWN}" > /dev/null 2>&1
RESULT=$?
if [ $RESULT -ne 0 ]; then
if [ "${RESULT}" -ne 0 ]; then
echo "Failed(chown)"
echo "Failed(chown)" >> $LOGFILE
echo "Failed(chown)" >> "${LOGFILE}"
continue
fi
touch -t $TOUCH > /dev/null 2>&1
touch -t "${TOUCH}" > /dev/null 2>&1
RESULT=$?
if [ $RESULT -ne 0 ]; then
if [ "${RESULT}" -ne 0 ]; then
echo "Failed(touch)"
echo "Failed(touch)" >> $LOGFILE
echo "Failed(touch)" >> "${LOGFILE}"
continue
fi
echo "Succeed"
echo "Succeed" >> $LOGFILE
echo "Succeed" >> "${LOGFILE}"
done
echo ""
echo "" >> $LOGFILE
echo "" >> "${LOGFILE}"
echo "Finished."
echo -n "# Finished : " >> $LOGFILE
echo `date` >> $LOGFILE
echo "# Finished : $(date)" >> "${LOGFILE}"
#
# Local variables:

View File

@ -46,33 +46,33 @@ func_usage()
echo ""
}
PRGNAME=`basename $0`
PRGNAME=$(basename "$0")
if [ "X$1" = "X-h" -o "X$1" = "X-H" ]; then
func_usage $PRGNAME
if [ "X$1" = "X-h" ] || [ "X$1" = "X-H" ]; then
func_usage "${PRGNAME}"
exit 0
fi
if [ "X$1" = "X" -o "X$2" = "X" -o "X$3" = "X" ]; then
func_usage $PRGNAME
if [ "X$1" = "X" ] || [ "X$2" = "X" ] || [ "X$3" = "X" ]; then
func_usage "${PRGNAME}"
exit 1
fi
BUCKET=$1
BUCKET="$1"
CDIR="$2"
LIMIT=$3
LIMIT="$3"
SILENT=0
if [ "X$4" = "X-silent" ]; then
SILENT=1
fi
FILES_CDIR="${CDIR}/${BUCKET}"
STATS_CDIR="${CDIR}/.${BUCKET}.stat"
CURRENT_CACHE_SIZE=`du -sb "$FILES_CDIR" | awk '{print $1}'`
CURRENT_CACHE_SIZE=$(du -sb "${FILES_CDIR}" | awk '{print $1}')
#
# Check total size
#
if [ $LIMIT -ge $CURRENT_CACHE_SIZE ]; then
if [ "${LIMIT}" -ge "${CURRENT_CACHE_SIZE}" ]; then
if [ $SILENT -ne 1 ]; then
echo "$FILES_CDIR ($CURRENT_CACHE_SIZE) is below allowed $LIMIT"
echo "${FILES_CDIR} (${CURRENT_CACHE_SIZE}) is below allowed ${LIMIT}"
fi
exit 0
fi
@ -86,37 +86,36 @@ TMP_CFILE=""
#
# Make file list by sorted access time
#
find "$STATS_CDIR" -type f -exec stat -c "%X:%n" "{}" \; | sort | while read part
find "${STATS_CDIR}" -type f -exec stat -c "%X:%n" "{}" \; | sort | while read -r part
do
echo Looking at $part
TMP_ATIME=`echo "$part" | cut -d: -f1`
TMP_STATS="`echo "$part" | cut -d: -f2`"
TMP_CFILE=`echo "$TMP_STATS" | sed s/\.$BUCKET\.stat/$BUCKET/`
echo "Looking at ${part}"
TMP_ATIME=$(echo "${part}" | cut -d: -f1)
TMP_STATS=$(echo "${part}" | cut -d: -f2)
TMP_CFILE=$(echo "${TMP_STATS}" | sed -e "s/\\.${BUCKET}\\.stat/${BUCKET}/")
if [ `stat -c %X "$TMP_STATS"` -eq $TMP_ATIME ]; then
rm -f "$TMP_STATS" "$TMP_CFILE" > /dev/null 2>&1
if [ $? -ne 0 ]; then
if [ $SILENT -ne 1 ]; then
echo "ERROR: Could not remove files($TMP_STATS,$TMP_CFILE)"
if [ "$(stat -c %X "${TMP_STATS}")" -eq "${TMP_ATIME}" ]; then
if ! rm "${TMP_STATS}" "${TMP_CFILE}" > /dev/null 2>&1; then
if [ "${SILENT}" -ne 1 ]; then
echo "ERROR: Could not remove files(${TMP_STATS},${TMP_CFILE})"
fi
exit 1
else
if [ $SILENT -ne 1 ]; then
echo "remove file: $TMP_CFILE $TMP_STATS"
if [ "${SILENT}" -ne 1 ]; then
echo "remove file: ${TMP_CFILE} ${TMP_STATS}"
fi
fi
fi
if [ $LIMIT -ge `du -sb "$FILES_CDIR" | awk '{print $1}'` ]; then
if [ $SILENT -ne 1 ]; then
if [ "${LIMIT}" -ge "$(du -sb "${FILES_CDIR}" | awk '{print $1}')" ]; then
if [ "${SILENT}" -ne 1 ]; then
echo "finish removing files"
fi
break
fi
done
if [ $SILENT -ne 1 ]; then
TOTAL_SIZE=`du -sb "$FILES_CDIR" | awk '{print $1}'`
echo "Finish: $FILES_CDIR total size is $TOTAL_SIZE"
if [ "${SILENT}" -ne 1 ]; then
TOTAL_SIZE=$(du -sb "${FILES_CDIR}" | awk '{print $1}')
echo "Finish: ${FILES_CDIR} total size is ${TOTAL_SIZE}"
fi
exit 0

View File

@ -32,8 +32,9 @@ CACHE_DIR="/tmp/s3fs-cache"
rm -rf "${CACHE_DIR}"
mkdir "${CACHE_DIR}"
#reserve 200MB for data cache
source test-utils.sh
#reserve 200MB for data cache
FAKE_FREE_DISK_SIZE=200
ENSURE_DISKFREE_SIZE=10
@ -44,13 +45,13 @@ if [ -n "${ALL_TESTS}" ]; then
"use_cache=${CACHE_DIR} -o ensure_diskfree=${ENSURE_DISKFREE_SIZE} -o fake_diskfree=${FAKE_FREE_DISK_SIZE}"
enable_content_md5
enable_noobj_cache
max_stat_cache_size=100
"max_stat_cache_size=100"
nocopyapi
nomultipart
notsup_compat_dir
sigv2
sigv4
singlepart_copy_limit=10 # limit size to exercise multipart code paths
"singlepart_copy_limit=10" # limit size to exercise multipart code paths
#use_sse # TODO: S3Proxy does not support SSE
)
else
@ -64,9 +65,10 @@ start_s3proxy
aws_cli s3 mb "s3://${TEST_BUCKET_1}" --region "${S3_ENDPOINT}"
for flag in "${FLAGS[@]}"; do
echo "testing s3fs flag: $flag"
echo "testing s3fs flag: ${flag}"
start_s3fs -o $flag
# shellcheck disable=SC2086
start_s3fs -o ${flag}
./integration-test-main.sh

View File

@ -24,19 +24,29 @@
set -o errexit
set -o pipefail
#
# Configuration
#
TEST_TEXT="HELLO WORLD"
TEST_TEXT_FILE=test-s3fs.txt
TEST_DIR=testdir
# shellcheck disable=SC2034
ALT_TEST_TEXT_FILE=test-s3fs-ALT.txt
# shellcheck disable=SC2034
TEST_TEXT_FILE_LENGTH=15
# shellcheck disable=SC2034
BIG_FILE=big-file-s3fs.txt
TEMP_DIR=${TMPDIR:-"/var/tmp"}
# shellcheck disable=SC2034
TEMP_DIR="${TMPDIR:-"/var/tmp"}"
# /dev/urandom can only return 32 MB per block maximum
BIG_FILE_BLOCK_SIZE=$((25 * 1024 * 1024))
BIG_FILE_COUNT=1
# This should be greater than the multipart size
BIG_FILE_LENGTH=$(($BIG_FILE_BLOCK_SIZE * $BIG_FILE_COUNT))
# shellcheck disable=SC2034
BIG_FILE_LENGTH=$((BIG_FILE_BLOCK_SIZE * BIG_FILE_COUNT))
# Set locale because some tests check for English expressions
export LC_ALL=en_US.UTF-8
export RUN_DIR
@ -48,7 +58,7 @@ export RUN_DIR
# and uses gnu commands(gstdbuf, gtruncate, gsed).
# Set your PATH appropriately so that you can find these commands.
#
if [ `uname` = "Darwin" ]; then
if [ "$(uname)" = "Darwin" ]; then
export STDBUF_BIN="gstdbuf"
export TRUNCATE_BIN="gtruncate"
export SED_BIN="gsed"
@ -62,7 +72,7 @@ fi
export SED_BUFFER_FLAG="--unbuffered"
function get_xattr() {
if [ `uname` = "Darwin" ]; then
if [ "$(uname)" = "Darwin" ]; then
xattr -p "$1" "$2"
else
getfattr -n "$1" --only-values "$2"
@ -70,7 +80,7 @@ function get_xattr() {
}
function set_xattr() {
if [ `uname` = "Darwin" ]; then
if [ "$(uname)" = "Darwin" ]; then
xattr -w "$1" "$2" "$3"
else
setfattr -n "$1" -v "$2" "$3"
@ -78,7 +88,7 @@ function set_xattr() {
}
function del_xattr() {
if [ `uname` = "Darwin" ]; then
if [ "$(uname)" = "Darwin" ]; then
xattr -d "$1" "$2"
else
setfattr -x "$1" "$2"
@ -86,7 +96,7 @@ function del_xattr() {
}
function get_size() {
if [ `uname` = "Darwin" ]; then
if [ "$(uname)" = "Darwin" ]; then
stat -f "%z" "$1"
else
stat -c %s "$1"
@ -98,45 +108,47 @@ function check_file_size() {
local EXPECTED_SIZE="$2"
# Verify file is zero length via metadata
local size=$(get_size ${FILE_NAME})
if [ $size -ne $EXPECTED_SIZE ]
local size
size=$(get_size "${FILE_NAME}")
if [ "${size}" -ne "${EXPECTED_SIZE}" ]
then
echo "error: expected ${FILE_NAME} to be zero length"
return 1
fi
# Verify file is zero length via data
local size=$(cat ${FILE_NAME} | wc -c)
if [ $size -ne $EXPECTED_SIZE ]
size=$(wc -c < "${FILE_NAME}")
if [ "${size}" -ne "${EXPECTED_SIZE}" ]
then
echo "error: expected ${FILE_NAME} to be $EXPECTED_SIZE length, got $size"
echo "error: expected ${FILE_NAME} to be ${EXPECTED_SIZE} length, got ${size}"
return 1
fi
}
function mk_test_file {
if [ $# = 0 ]; then
local TEXT=$TEST_TEXT
local TEXT="${TEST_TEXT}"
else
local TEXT=$1
local TEXT="$1"
fi
echo $TEXT > $TEST_TEXT_FILE
if [ ! -e $TEST_TEXT_FILE ]
echo "${TEXT}" > "${TEST_TEXT_FILE}"
if [ ! -e "${TEST_TEXT_FILE}" ]
then
echo "Could not create file ${TEST_TEXT_FILE}, it does not exist"
exit 1
fi
# wait & check
local BASE_TEXT_LENGTH=`echo $TEXT | wc -c | awk '{print $1}'`
local BASE_TEXT_LENGTH; BASE_TEXT_LENGTH=$(echo "${TEXT}" | wc -c | awk '{print $1}')
local TRY_COUNT=10
while true; do
local MK_TEXT_LENGTH=`wc -c $TEST_TEXT_FILE | awk '{print $1}'`
if [ $BASE_TEXT_LENGTH -eq $MK_TEXT_LENGTH ]; then
local MK_TEXT_LENGTH
MK_TEXT_LENGTH=$(wc -c "${TEST_TEXT_FILE}" | awk '{print $1}')
if [ "${BASE_TEXT_LENGTH}" -eq "${MK_TEXT_LENGTH}" ]; then
break
fi
local TRY_COUNT=`expr $TRY_COUNT - 1`
if [ $TRY_COUNT -le 0 ]; then
local TRY_COUNT=$((TRY_COUNT - 1))
if [ "${TRY_COUNT}" -le 0 ]; then
echo "Could not create file ${TEST_TEXT_FILE}, that file size is something wrong"
fi
sleep 1
@ -145,13 +157,13 @@ function mk_test_file {
function rm_test_file {
if [ $# = 0 ]; then
local FILE=$TEST_TEXT_FILE
local FILE="${TEST_TEXT_FILE}"
else
local FILE=$1
local FILE="$1"
fi
rm -f $FILE
rm -f "${FILE}"
if [ -e $FILE ]
if [ -e "${FILE}" ]
then
echo "Could not cleanup file ${TEST_TEXT_FILE}"
exit 1
@ -159,17 +171,17 @@ function rm_test_file {
}
function mk_test_dir {
mkdir ${TEST_DIR}
mkdir "${TEST_DIR}"
if [ ! -d ${TEST_DIR} ]; then
if [ ! -d "${TEST_DIR}" ]; then
echo "Directory ${TEST_DIR} was not created"
exit 1
fi
}
function rm_test_dir {
rmdir ${TEST_DIR}
if [ -e $TEST_DIR ]; then
rmdir "${TEST_DIR}"
if [ -e "${TEST_DIR}" ]; then
echo "Could not remove the test directory, it still exists: ${TEST_DIR}"
exit 1
fi
@ -178,18 +190,18 @@ function rm_test_dir {
# Create and cd to a unique directory for this test run
# Sets RUN_DIR to the name of the created directory
function cd_run_dir {
if [ "$TEST_BUCKET_MOUNT_POINT_1" = "" ]; then
if [ "${TEST_BUCKET_MOUNT_POINT_1}" = "" ]; then
echo "TEST_BUCKET_MOUNT_POINT_1 variable not set"
exit 1
fi
local RUN_DIR=${TEST_BUCKET_MOUNT_POINT_1}/${1}
mkdir -p ${RUN_DIR}
cd ${RUN_DIR}
local RUN_DIR="${TEST_BUCKET_MOUNT_POINT_1}/${1}"
mkdir -p "${RUN_DIR}"
cd "${RUN_DIR}"
}
function clean_run_dir {
if [ -d ${RUN_DIR} ]; then
rm -rf ${RUN_DIR} || echo "Error removing ${RUN_DIR}"
if [ -d "${RUN_DIR}" ]; then
rm -rf "${RUN_DIR}" || echo "Error removing ${RUN_DIR}"
fi
}
@ -204,14 +216,14 @@ function init_suite {
# report_pass TEST_NAME
function report_pass {
echo "$1 passed"
TEST_PASSED_LIST+=($1)
TEST_PASSED_LIST+=("$1")
}
# Report a failing test case
# report_fail TEST_NAME
function report_fail {
echo "$1 failed"
TEST_FAILED_LIST+=($1)
TEST_FAILED_LIST+=("$1")
}
# Add tests to the suite
@ -231,37 +243,37 @@ function describe {
# directory in the bucket. An attempt to clean this directory is
# made after the test run.
function run_suite {
orig_dir=$PWD
key_prefix="testrun-$RANDOM"
cd_run_dir $key_prefix
orig_dir="${PWD}"
key_prefix="testrun-${RANDOM}"
cd_run_dir "${key_prefix}"
for t in "${TEST_LIST[@]}"; do
# Ensure test input name differs every iteration
TEST_TEXT_FILE=test-s3fs.txt-$RANDOM
TEST_DIR=testdir-$RANDOM
$t $key_prefix && rc=$? || rc=$?
TEST_TEXT_FILE="test-s3fs.txt-${RANDOM}"
TEST_DIR="testdir-${RANDOM}"
"${t}" "${key_prefix}" && rc=$? || rc=$?
if [[ $rc = 0 ]] ; then
report_pass $t
if [[ "${rc}" = 0 ]] ; then
report_pass "${t}"
else
report_fail $t
report_fail "${t}"
fi
done
cd ${orig_dir}
cd "${orig_dir}"
clean_run_dir
for t in "${TEST_PASSED_LIST[@]}"; do
echo "PASS: $t"
echo "PASS: ${t}"
done
for t in "${TEST_FAILED_LIST[@]}"; do
echo "FAIL: $t"
echo "FAIL: ${t}"
done
local passed=${#TEST_PASSED_LIST[@]}
local failed=${#TEST_FAILED_LIST[@]}
echo "SUMMARY for $0: $passed tests passed. $failed tests failed."
echo "SUMMARY for $0: ${passed} tests passed. ${failed} tests failed."
if [[ $failed != 0 ]]; then
if [[ "${failed}" != 0 ]]; then
return 1
else
return 0
@ -269,7 +281,7 @@ function run_suite {
}
function get_ctime() {
if [ `uname` = "Darwin" ]; then
if [ "$(uname)" = "Darwin" ]; then
stat -f "%c" "$1"
else
stat -c "%Z" "$1"
@ -277,7 +289,7 @@ function get_ctime() {
}
function get_mtime() {
if [ `uname` = "Darwin" ]; then
if [ "$(uname)" = "Darwin" ]; then
stat -f "%m" "$1"
else
stat -c "%Y" "$1"
@ -285,7 +297,7 @@ function get_mtime() {
}
function get_atime() {
if [ `uname` = "Darwin" ]; then
if [ "$(uname)" = "Darwin" ]; then
stat -f "%a" "$1"
else
stat -c "%X" "$1"
@ -293,7 +305,7 @@ function get_atime() {
}
function get_permissions() {
if [ `uname` = "Darwin" ]; then
if [ "$(uname)" = "Darwin" ]; then
stat -f "%p" "$1"
else
stat -c "%a" "$1"
@ -301,7 +313,8 @@ function get_permissions() {
}
function check_content_type() {
local INFO_STR=`aws_cli s3api head-object --bucket ${TEST_BUCKET_1} --key $1`
local INFO_STR
INFO_STR=$(aws_cli s3api head-object --bucket "${TEST_BUCKET_1}" --key "$1")
if [[ "${INFO_STR}" != *"$2"* ]]
then
echo "moved file content-type is not as expected expected:$2 got:${INFO_STR}"
@ -310,8 +323,9 @@ function check_content_type() {
}
function get_disk_avail_size() {
local DISK_AVAIL_SIZE=`BLOCKSIZE=$((1024 * 1024)) df $1 | awk '{print $4}' | tail -n 1`
echo ${DISK_AVAIL_SIZE}
local DISK_AVAIL_SIZE
DISK_AVAIL_SIZE=$(BLOCKSIZE=$((1024 * 1024)) df "$1" | awk '{print $4}' | tail -n 1)
echo "${DISK_AVAIL_SIZE}"
}
function aws_cli() {
@ -319,12 +333,13 @@ function aws_cli() {
if [ -n "${S3FS_PROFILE}" ]; then
FLAGS="--profile ${S3FS_PROFILE}"
fi
aws $* --endpoint-url "${S3_URL}" --ca-bundle /tmp/keystore.pem $FLAGS
# shellcheck disable=SC2086,SC2068
aws $@ --endpoint-url "${S3_URL}" --ca-bundle /tmp/keystore.pem ${FLAGS}
}
function wait_for_port() {
local PORT=$1
for i in $(seq 30); do
local PORT="$1"
for _ in $(seq 30); do
if exec 3<>"/dev/tcp/127.0.0.1/${PORT}";
then
exec 3<&- # Close for read
@ -337,12 +352,12 @@ function wait_for_port() {
function make_random_string() {
if [ -n "$1" ]; then
local END_POS=$1
local END_POS="$1"
else
local END_POS=8
fi
${BASE64_BIN} --wrap=0 < /dev/urandom | tr -d /+ | head -c ${END_POS}
"${BASE64_BIN}" --wrap=0 < /dev/urandom | tr -d /+ | head -c "${END_POS}"
return 0
}