Fixed about Github Actions failure(centos8 and macos) (#1498)

2024-09-21 22:09:02 +00:00 · 2020-12-23 19:34:04 +09:00 · 2020-12-23 19:34:04 +09:00 · 4d0daddad4
commit 4d0daddad4
parent d246b9e8bf
6 changed files with 41 additions and 26 deletions
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@ -143,7 +143,7 @@ jobs:
        run: |
          TAPS="$(brew --repository)/Library/Taps";
          if [ -e "$TAPS/caskroom/homebrew-cask" ]; then rm -rf "$TAPS/caskroom/homebrew-cask"; fi;
-          if [ ! -f $HOME/.osx_cache/cached ]; then HOMEBREW_NO_AUTO_UPDATE=1 brew tap homebrew/homebrew-cask; else HOMEBREW_NO_AUTO_UPDATE=1 brew tap homebrew/homebrew-cask; fi
+          HOMEBREW_NO_AUTO_UPDATE=1 brew tap homebrew/homebrew-cask

      - name: Install osxfuse
        run: |
@ -151,7 +151,7 @@ jobs:

      - name: Install brew other packages
        run: |
-          S3FS_BREW_PACKAGES='automake cppcheck python3 coreutils';
+          S3FS_BREW_PACKAGES='automake cppcheck python3 coreutils gnu-sed';
          for s3fs_brew_pkg in ${S3FS_BREW_PACKAGES}; do if brew list | grep -q ${s3fs_brew_pkg}; then if brew outdated | grep -q ${s3fs_brew_pkg}; then HOMEBREW_NO_AUTO_UPDATE=1 brew upgrade ${s3fs_brew_pkg}; fi; else HOMEBREW_NO_AUTO_UPDATE=1 brew install ${s3fs_brew_pkg}; fi; done;

      - name: Install awscli
@ -162,14 +162,6 @@ jobs:
        run: |
          if [ -f /Library/Filesystems/osxfusefs.fs/Support/load_osxfusefs ]; then sudo chmod +s /Library/Filesystems/osxfusefs.fs/Support/load_osxfusefs; elif [ -f /Library/Filesystems/osxfuse.fs/Contents/Resources/load_osxfuse ]; then sudo chmod +s /Library/Filesystems/osxfuse.fs/Contents/Resources/load_osxfuse; else exit 1; fi

-      - name: Check truncate
-        run: |
-          if [ ! -f /usr/local/bin/truncate ]; then sudo ln -s /usr/local/opt/coreutils/bin/gtruncate /usr/local/bin/truncate; fi
-
-      - name: Check stdbuf
-        run: |
-          if [ ! -f /usr/local/bin/stdbuf ]; then sudo ln -s /usr/local/opt/coreutils/bin/gstdbuf /usr/local/bin/stdbuf; fi
-
      - name: Build
        run: |
          ./autogen.sh
--- a/.github/workflows/linux-ci-helper.sh
+++ b/.github/workflows/linux-ci-helper.sh
@ -103,7 +103,7 @@ elif [ "${CONTAINER_FULLNAME}" = "centos:centos8" ]; then
    PACKAGE_UPDATE_OPTIONS="update -y -qq"

    INSTALL_PACKAGES="gcc libstdc++-devel gcc-c++ fuse fuse-devel curl-devel libxml2-devel mailcap git automake make openssl-devel attr diffutils wget python2 python3"
-    INSTALL_CPPCHECK_OPTIONS="--enablerepo=PowerTools"
+    INSTALL_CPPCHECK_OPTIONS="--enablerepo=powertools"
    INSTALL_JDK_PACKAGES="java-1.8.0-openjdk"

    # [NOTE]
--- a/.travis.yml
+++ b/.travis.yml
@ -70,7 +70,7 @@ jobs:
            HOMEBREW_NO_AUTO_UPDATE=1 brew tap homebrew/homebrew-cask;
          fi
        - HOMEBREW_NO_AUTO_UPDATE=1 brew cask install osxfuse
-        - S3FS_BREW_PACKAGES='cppcheck python3';
+        - S3FS_BREW_PACKAGES='cppcheck python3 gnu-sed';
          for s3fs_brew_pkg in ${S3FS_BREW_PACKAGES}; do
            if brew list | grep -q ${s3fs_brew_pkg}; then
              if brew outdated | grep -q ${s3fs_brew_pkg}; then
--- a/test/integration-test-common.sh
+++ b/test/integration-test-common.sh
@ -138,7 +138,7 @@ function start_s3proxy {
            chmod +x "${S3PROXY_BINARY}"
        fi

-        stdbuf -oL -eL java -jar "$S3PROXY_BINARY" --properties $S3PROXY_CONFIG &
+        ${STDBUF_BIN} -oL -eL java -jar "$S3PROXY_BINARY" --properties $S3PROXY_CONFIG &
        S3PROXY_PID=$!

        # wait for S3Proxy to start
@ -189,6 +189,17 @@ function start_s3fs {
       DIRECT_IO_OPT=""
    fi

+    # [NOTE]
+    # On macos, running s3fs via stdbuf will result in no response.
+    # Therefore, when it is macos, it is not executed via stdbuf.
+    # This patch may be temporary, but no other method has been found at this time.
+    #
+    if [ `uname` = "Darwin" ]; then
+        VIA_STDBUF_CMDLINE=""
+    else
+        VIA_STDBUF_CMDLINE="${STDBUF_BIN} -oL -eL"
+    fi
+
    # Common s3fs options:
    #
    # TODO: Allow all these options to be overridden with env variables
@ -211,7 +222,7 @@ function start_s3fs {
    # subshell with set -x to log exact invocation of s3fs-fuse
    (
        set -x 
-        stdbuf -oL -eL \
+        ${VIA_STDBUF_CMDLINE} \
            ${VALGRIND_EXEC} ${S3FS} \
            $TEST_BUCKET_1 \
            $TEST_BUCKET_MOUNT_POINT_1 \
@ -231,7 +242,7 @@ function start_s3fs {
            -f \
            "${@}" &
        echo $! >&3
-    ) 3>pid | stdbuf -oL -eL sed $SED_BUFFER_FLAG "s/^/s3fs: /" &
+    ) 3>pid | ${STDBUF_BIN} -oL -eL ${SED_BIN} ${SED_BUFFER_FLAG} "s/^/s3fs: /" &
    sleep 1
    export S3FS_PID=$(<pid)
    rm -f pid
--- a/test/integration-test-main.sh
+++ b/test/integration-test-main.sh
@ -60,7 +60,7 @@ function test_truncate_upload {

    rm_test_file ${BIG_FILE}

-    truncate ${BIG_FILE} -s ${BIG_FILE_LENGTH}
+    ${TRUNCATE_BIN} ${BIG_FILE} -s ${BIG_FILE_LENGTH}

    rm_test_file ${BIG_FILE}
 }
@ -72,7 +72,7 @@ function test_truncate_empty_file {

    # Truncate the file to 1024 length
    t_size=1024
-    truncate ${TEST_TEXT_FILE} -s $t_size
+    ${TRUNCATE_BIN} ${TEST_TEXT_FILE} -s $t_size

    check_file_size "${TEST_TEXT_FILE}" $t_size

@ -194,8 +194,8 @@ function test_redirects {

    echo 123456 >> $TEST_TEXT_FILE

-    LINE1=`sed -n '1,1p' $TEST_TEXT_FILE`
-    LINE2=`sed -n '2,2p' $TEST_TEXT_FILE`
+    LINE1=`${SED_BIN} -n '1,1p' $TEST_TEXT_FILE`
+    LINE2=`${SED_BIN} -n '2,2p' $TEST_TEXT_FILE`

    if [ ${LINE1} != "XYZ" ]; then
       echo "LINE1 was not as expected, got ${LINE1}, expected XYZ"
@ -918,7 +918,7 @@ function test_concurrent_directory_updates {
    for i in `seq 5`; do echo foo > $i; done
    for process in `seq 10`; do
        for i in `seq 5`; do
-            file=$(ls `seq 5` | sed -n "$(($RANDOM % 5 + 1))p")
+            file=$(ls `seq 5` | ${SED_BIN} -n "$(($RANDOM % 5 + 1))p")
            cat $file >/dev/null || true
            rm -f $file
            echo foo > $file || true
@ -1066,8 +1066,8 @@ function test_cache_file_stat() {
    #
    # get lines from cache stat file
    #
-    CACHE_FILE_STAT_LINE_1=$(sed -n 1p ${CACHE_DIR}/.${TEST_BUCKET_1}.stat/${CACHE_TESTRUN_DIR}/${BIG_FILE})
-    CACHE_FILE_STAT_LINE_2=$(sed -n 2p ${CACHE_DIR}/.${TEST_BUCKET_1}.stat/${CACHE_TESTRUN_DIR}/${BIG_FILE})
+    CACHE_FILE_STAT_LINE_1=$(${SED_BIN} -n 1p ${CACHE_DIR}/.${TEST_BUCKET_1}.stat/${CACHE_TESTRUN_DIR}/${BIG_FILE})
+    CACHE_FILE_STAT_LINE_2=$(${SED_BIN} -n 2p ${CACHE_DIR}/.${TEST_BUCKET_1}.stat/${CACHE_TESTRUN_DIR}/${BIG_FILE})
    if [ -z ${CACHE_FILE_STAT_LINE_1} ] || [ -z ${CACHE_FILE_STAT_LINE_2} ]; then
        echo "could not get first or second line from cache file stat: ${CACHE_DIR}/.${TEST_BUCKET_1}.stat/${CACHE_TESTRUN_DIR}/${BIG_FILE}"
        return 1;
@ -1109,7 +1109,7 @@ function test_cache_file_stat() {
    #
    # get lines from cache stat file
    #
-    CACHE_FILE_STAT_LINE_1=$(sed -n 1p ${CACHE_DIR}/.${TEST_BUCKET_1}.stat/${CACHE_TESTRUN_DIR}/${BIG_FILE})
+    CACHE_FILE_STAT_LINE_1=$(${SED_BIN} -n 1p ${CACHE_DIR}/.${TEST_BUCKET_1}.stat/${CACHE_TESTRUN_DIR}/${BIG_FILE})
    CACHE_FILE_STAT_LINE_E=$(tail -1 ${CACHE_DIR}/.${TEST_BUCKET_1}.stat/${CACHE_TESTRUN_DIR}/${BIG_FILE} 2>/dev/null)
    if [ -z ${CACHE_FILE_STAT_LINE_1} ] || [ -z ${CACHE_FILE_STAT_LINE_E} ]; then
        echo "could not get first or end line from cache file stat: ${CACHE_DIR}/.${TEST_BUCKET_1}.stat/${CACHE_TESTRUN_DIR}/${BIG_FILE}"
@ -1148,7 +1148,7 @@ function test_upload_sparsefile {
    #
    # Make all HOLE file
    #
-    truncate ${BIG_FILE} -s ${BIG_FILE_LENGTH}
+    ${TRUNCATE_BIN} ${BIG_FILE} -s ${BIG_FILE_LENGTH}

    #
    # Write some bytes to ABOUT middle in the file
--- a/test/test-utils.sh
+++ b/test/test-utils.sh
@ -34,11 +34,23 @@ BIG_FILE=big-file-s3fs.txt
 BIG_FILE_LENGTH=$((25 * 1024 * 1024))
 export RUN_DIR

+# [NOTE]
+# stdbuf, truncate and sed installed on macos do not work as
+# expected(not compatible with Linux).
+# Therefore, macos installs a brew package such as coreutils
+# and uses gnu commands(gstdbuf, gtruncate, gsed).
+# Set your PATH appropriately so that you can find these commands.
+#
 if [ `uname` = "Darwin" ]; then
-    export SED_BUFFER_FLAG="-l"
+    export STDBUF_BIN="gstdbuf"
+    export TRUNCATE_BIN="gtruncate"
+    export SED_BIN="gsed"
 else
-    export SED_BUFFER_FLAG="--unbuffered"
+    export STDBUF_BIN="stdbuf"
+    export TRUNCATE_BIN="truncate"
+    export SED_BIN="sed"
 fi
+export SED_BUFFER_FLAG="--unbuffered"

 function get_xattr() {
    if [ `uname` = "Darwin" ]; then