From 4d0daddad4d4b5659fd5aa9a59a7ee459e09da36 Mon Sep 17 00:00:00 2001 From: Takeshi Nakatani Date: Wed, 23 Dec 2020 19:34:04 +0900 Subject: [PATCH] Fixed about Github Actions failure(centos8 and macos) (#1498) --- .github/workflows/ci.yml | 12 ++---------- .github/workflows/linux-ci-helper.sh | 2 +- .travis.yml | 2 +- test/integration-test-common.sh | 17 ++++++++++++++--- test/integration-test-main.sh | 18 +++++++++--------- test/test-utils.sh | 16 ++++++++++++++-- 6 files changed, 41 insertions(+), 26 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4726a88..e192624 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -143,7 +143,7 @@ jobs: run: | TAPS="$(brew --repository)/Library/Taps"; if [ -e "$TAPS/caskroom/homebrew-cask" ]; then rm -rf "$TAPS/caskroom/homebrew-cask"; fi; - if [ ! -f $HOME/.osx_cache/cached ]; then HOMEBREW_NO_AUTO_UPDATE=1 brew tap homebrew/homebrew-cask; else HOMEBREW_NO_AUTO_UPDATE=1 brew tap homebrew/homebrew-cask; fi + HOMEBREW_NO_AUTO_UPDATE=1 brew tap homebrew/homebrew-cask - name: Install osxfuse run: | @@ -151,7 +151,7 @@ jobs: - name: Install brew other packages run: | - S3FS_BREW_PACKAGES='automake cppcheck python3 coreutils'; + S3FS_BREW_PACKAGES='automake cppcheck python3 coreutils gnu-sed'; for s3fs_brew_pkg in ${S3FS_BREW_PACKAGES}; do if brew list | grep -q ${s3fs_brew_pkg}; then if brew outdated | grep -q ${s3fs_brew_pkg}; then HOMEBREW_NO_AUTO_UPDATE=1 brew upgrade ${s3fs_brew_pkg}; fi; else HOMEBREW_NO_AUTO_UPDATE=1 brew install ${s3fs_brew_pkg}; fi; done; - name: Install awscli @@ -162,14 +162,6 @@ jobs: run: | if [ -f /Library/Filesystems/osxfusefs.fs/Support/load_osxfusefs ]; then sudo chmod +s /Library/Filesystems/osxfusefs.fs/Support/load_osxfusefs; elif [ -f /Library/Filesystems/osxfuse.fs/Contents/Resources/load_osxfuse ]; then sudo chmod +s /Library/Filesystems/osxfuse.fs/Contents/Resources/load_osxfuse; else exit 1; fi - - name: Check truncate - run: | - if [ ! -f /usr/local/bin/truncate ]; then sudo ln -s /usr/local/opt/coreutils/bin/gtruncate /usr/local/bin/truncate; fi - - - name: Check stdbuf - run: | - if [ ! -f /usr/local/bin/stdbuf ]; then sudo ln -s /usr/local/opt/coreutils/bin/gstdbuf /usr/local/bin/stdbuf; fi - - name: Build run: | ./autogen.sh diff --git a/.github/workflows/linux-ci-helper.sh b/.github/workflows/linux-ci-helper.sh index 8c5d624..d61c46f 100755 --- a/.github/workflows/linux-ci-helper.sh +++ b/.github/workflows/linux-ci-helper.sh @@ -103,7 +103,7 @@ elif [ "${CONTAINER_FULLNAME}" = "centos:centos8" ]; then PACKAGE_UPDATE_OPTIONS="update -y -qq" INSTALL_PACKAGES="gcc libstdc++-devel gcc-c++ fuse fuse-devel curl-devel libxml2-devel mailcap git automake make openssl-devel attr diffutils wget python2 python3" - INSTALL_CPPCHECK_OPTIONS="--enablerepo=PowerTools" + INSTALL_CPPCHECK_OPTIONS="--enablerepo=powertools" INSTALL_JDK_PACKAGES="java-1.8.0-openjdk" # [NOTE] diff --git a/.travis.yml b/.travis.yml index dea6d96..b9defa4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -70,7 +70,7 @@ jobs: HOMEBREW_NO_AUTO_UPDATE=1 brew tap homebrew/homebrew-cask; fi - HOMEBREW_NO_AUTO_UPDATE=1 brew cask install osxfuse - - S3FS_BREW_PACKAGES='cppcheck python3'; + - S3FS_BREW_PACKAGES='cppcheck python3 gnu-sed'; for s3fs_brew_pkg in ${S3FS_BREW_PACKAGES}; do if brew list | grep -q ${s3fs_brew_pkg}; then if brew outdated | grep -q ${s3fs_brew_pkg}; then diff --git a/test/integration-test-common.sh b/test/integration-test-common.sh index f9c28dd..8a3ed4b 100644 --- a/test/integration-test-common.sh +++ b/test/integration-test-common.sh @@ -138,7 +138,7 @@ function start_s3proxy { chmod +x "${S3PROXY_BINARY}" fi - stdbuf -oL -eL java -jar "$S3PROXY_BINARY" --properties $S3PROXY_CONFIG & + ${STDBUF_BIN} -oL -eL java -jar "$S3PROXY_BINARY" --properties $S3PROXY_CONFIG & S3PROXY_PID=$! # wait for S3Proxy to start @@ -189,6 +189,17 @@ function start_s3fs { DIRECT_IO_OPT="" fi + # [NOTE] + # On macos, running s3fs via stdbuf will result in no response. + # Therefore, when it is macos, it is not executed via stdbuf. + # This patch may be temporary, but no other method has been found at this time. + # + if [ `uname` = "Darwin" ]; then + VIA_STDBUF_CMDLINE="" + else + VIA_STDBUF_CMDLINE="${STDBUF_BIN} -oL -eL" + fi + # Common s3fs options: # # TODO: Allow all these options to be overridden with env variables @@ -211,7 +222,7 @@ function start_s3fs { # subshell with set -x to log exact invocation of s3fs-fuse ( set -x - stdbuf -oL -eL \ + ${VIA_STDBUF_CMDLINE} \ ${VALGRIND_EXEC} ${S3FS} \ $TEST_BUCKET_1 \ $TEST_BUCKET_MOUNT_POINT_1 \ @@ -231,7 +242,7 @@ function start_s3fs { -f \ "${@}" & echo $! >&3 - ) 3>pid | stdbuf -oL -eL sed $SED_BUFFER_FLAG "s/^/s3fs: /" & + ) 3>pid | ${STDBUF_BIN} -oL -eL ${SED_BIN} ${SED_BUFFER_FLAG} "s/^/s3fs: /" & sleep 1 export S3FS_PID=$(> $TEST_TEXT_FILE - LINE1=`sed -n '1,1p' $TEST_TEXT_FILE` - LINE2=`sed -n '2,2p' $TEST_TEXT_FILE` + LINE1=`${SED_BIN} -n '1,1p' $TEST_TEXT_FILE` + LINE2=`${SED_BIN} -n '2,2p' $TEST_TEXT_FILE` if [ ${LINE1} != "XYZ" ]; then echo "LINE1 was not as expected, got ${LINE1}, expected XYZ" @@ -918,7 +918,7 @@ function test_concurrent_directory_updates { for i in `seq 5`; do echo foo > $i; done for process in `seq 10`; do for i in `seq 5`; do - file=$(ls `seq 5` | sed -n "$(($RANDOM % 5 + 1))p") + file=$(ls `seq 5` | ${SED_BIN} -n "$(($RANDOM % 5 + 1))p") cat $file >/dev/null || true rm -f $file echo foo > $file || true @@ -1066,8 +1066,8 @@ function test_cache_file_stat() { # # get lines from cache stat file # - CACHE_FILE_STAT_LINE_1=$(sed -n 1p ${CACHE_DIR}/.${TEST_BUCKET_1}.stat/${CACHE_TESTRUN_DIR}/${BIG_FILE}) - CACHE_FILE_STAT_LINE_2=$(sed -n 2p ${CACHE_DIR}/.${TEST_BUCKET_1}.stat/${CACHE_TESTRUN_DIR}/${BIG_FILE}) + CACHE_FILE_STAT_LINE_1=$(${SED_BIN} -n 1p ${CACHE_DIR}/.${TEST_BUCKET_1}.stat/${CACHE_TESTRUN_DIR}/${BIG_FILE}) + CACHE_FILE_STAT_LINE_2=$(${SED_BIN} -n 2p ${CACHE_DIR}/.${TEST_BUCKET_1}.stat/${CACHE_TESTRUN_DIR}/${BIG_FILE}) if [ -z ${CACHE_FILE_STAT_LINE_1} ] || [ -z ${CACHE_FILE_STAT_LINE_2} ]; then echo "could not get first or second line from cache file stat: ${CACHE_DIR}/.${TEST_BUCKET_1}.stat/${CACHE_TESTRUN_DIR}/${BIG_FILE}" return 1; @@ -1109,7 +1109,7 @@ function test_cache_file_stat() { # # get lines from cache stat file # - CACHE_FILE_STAT_LINE_1=$(sed -n 1p ${CACHE_DIR}/.${TEST_BUCKET_1}.stat/${CACHE_TESTRUN_DIR}/${BIG_FILE}) + CACHE_FILE_STAT_LINE_1=$(${SED_BIN} -n 1p ${CACHE_DIR}/.${TEST_BUCKET_1}.stat/${CACHE_TESTRUN_DIR}/${BIG_FILE}) CACHE_FILE_STAT_LINE_E=$(tail -1 ${CACHE_DIR}/.${TEST_BUCKET_1}.stat/${CACHE_TESTRUN_DIR}/${BIG_FILE} 2>/dev/null) if [ -z ${CACHE_FILE_STAT_LINE_1} ] || [ -z ${CACHE_FILE_STAT_LINE_E} ]; then echo "could not get first or end line from cache file stat: ${CACHE_DIR}/.${TEST_BUCKET_1}.stat/${CACHE_TESTRUN_DIR}/${BIG_FILE}" @@ -1148,7 +1148,7 @@ function test_upload_sparsefile { # # Make all HOLE file # - truncate ${BIG_FILE} -s ${BIG_FILE_LENGTH} + ${TRUNCATE_BIN} ${BIG_FILE} -s ${BIG_FILE_LENGTH} # # Write some bytes to ABOUT middle in the file diff --git a/test/test-utils.sh b/test/test-utils.sh index c77209a..0ce28fb 100644 --- a/test/test-utils.sh +++ b/test/test-utils.sh @@ -34,11 +34,23 @@ BIG_FILE=big-file-s3fs.txt BIG_FILE_LENGTH=$((25 * 1024 * 1024)) export RUN_DIR +# [NOTE] +# stdbuf, truncate and sed installed on macos do not work as +# expected(not compatible with Linux). +# Therefore, macos installs a brew package such as coreutils +# and uses gnu commands(gstdbuf, gtruncate, gsed). +# Set your PATH appropriately so that you can find these commands. +# if [ `uname` = "Darwin" ]; then - export SED_BUFFER_FLAG="-l" + export STDBUF_BIN="gstdbuf" + export TRUNCATE_BIN="gtruncate" + export SED_BIN="gsed" else - export SED_BUFFER_FLAG="--unbuffered" + export STDBUF_BIN="stdbuf" + export TRUNCATE_BIN="truncate" + export SED_BIN="sed" fi +export SED_BUFFER_FLAG="--unbuffered" function get_xattr() { if [ `uname` = "Darwin" ]; then