diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fcf7db6..ee1344d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -141,9 +141,11 @@ jobs: S3FS_BREW_PACKAGES='automake cppcheck python3 coreutils gnu-sed shellcheck'; for s3fs_brew_pkg in ${S3FS_BREW_PACKAGES}; do if brew list | grep -q ${s3fs_brew_pkg}; then if brew outdated | grep -q ${s3fs_brew_pkg}; then HOMEBREW_NO_AUTO_UPDATE=1 brew upgrade ${s3fs_brew_pkg}; fi; else HOMEBREW_NO_AUTO_UPDATE=1 brew install ${s3fs_brew_pkg}; fi; done; - - name: Install awscli + - name: Install awscli2 run: | - if pip3 --version; then pip3 install awscli; else curl https://bootstrap.pypa.io/get-pip.py | sudo python; pip install awscli --ignore-installed matplotlib; fi + cd /tmp + curl "https://awscli.amazonaws.com/AWSCLIV2.pkg" -o "AWSCLIV2.pkg" + sudo installer -pkg AWSCLIV2.pkg -target / - name: Check osxfuse permission run: | diff --git a/.github/workflows/linux-ci-helper.sh b/.github/workflows/linux-ci-helper.sh index f93c4ee..f73b023 100755 --- a/.github/workflows/linux-ci-helper.sh +++ b/.github/workflows/linux-ci-helper.sh @@ -50,11 +50,10 @@ CONTAINER_OSNAME=$(echo "${CONTAINER_FULLNAME}" | sed 's/:/ /g' | awk '{print $1 CONTAINER_OSVERSION=$(echo "${CONTAINER_FULLNAME}" | sed 's/:/ /g' | awk '{print $2}') #----------------------------------------------------------- -# Common variables for pip +# Common variables for awscli2 #----------------------------------------------------------- -PIP_BIN="pip3" -PIP_OPTIONS="--upgrade" -INSTALL_AWSCLI_PACKAGES="awscli" +AWSCLI_URI="https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" +AWSCLI_ZIP_FILE="awscliv2.zip" #----------------------------------------------------------- # Parameters for configure(set environments) @@ -69,7 +68,7 @@ if [ "${CONTAINER_FULLNAME}" = "ubuntu:21.10" ]; then PACKAGE_MANAGER_BIN="apt-get" PACKAGE_UPDATE_OPTIONS="update -y -qq" - INSTALL_PACKAGES="autoconf autotools-dev default-jre-headless fuse libfuse-dev libcurl4-openssl-dev libxml2-dev locales-all mime-support libtool pkg-config libssl-dev attr curl python3-pip" + INSTALL_PACKAGES="autoconf autotools-dev default-jre-headless fuse libfuse-dev libcurl4-openssl-dev libxml2-dev locales-all mime-support libtool pkg-config libssl-dev attr curl python3-pip unzip" INSTALL_CHECKER_PKGS="cppcheck shellcheck" INSTALL_CHECKER_PKG_OPTIONS="" @@ -77,7 +76,7 @@ elif [ "${CONTAINER_FULLNAME}" = "ubuntu:20.04" ]; then PACKAGE_MANAGER_BIN="apt-get" PACKAGE_UPDATE_OPTIONS="update -y -qq" - INSTALL_PACKAGES="autoconf autotools-dev default-jre-headless fuse libfuse-dev libcurl4-openssl-dev libxml2-dev locales-all mime-support libtool pkg-config libssl-dev attr curl python3-pip" + INSTALL_PACKAGES="autoconf autotools-dev default-jre-headless fuse libfuse-dev libcurl4-openssl-dev libxml2-dev locales-all mime-support libtool pkg-config libssl-dev attr curl python3-pip unzip" INSTALL_CHECKER_PKGS="cppcheck shellcheck" INSTALL_CHECKER_PKG_OPTIONS="" @@ -85,7 +84,7 @@ elif [ "${CONTAINER_FULLNAME}" = "ubuntu:18.04" ]; then PACKAGE_MANAGER_BIN="apt-get" PACKAGE_UPDATE_OPTIONS="update -y -qq" - INSTALL_PACKAGES="autoconf autotools-dev default-jre-headless fuse libfuse-dev libcurl4-openssl-dev libxml2-dev locales-all mime-support libtool pkg-config libssl-dev attr curl python3-pip" + INSTALL_PACKAGES="autoconf autotools-dev default-jre-headless fuse libfuse-dev libcurl4-openssl-dev libxml2-dev locales-all mime-support libtool pkg-config libssl-dev attr curl python3-pip unzip" INSTALL_CHECKER_PKGS="cppcheck shellcheck" INSTALL_CHECKER_PKG_OPTIONS="" @@ -93,7 +92,7 @@ elif [ "${CONTAINER_FULLNAME}" = "ubuntu:16.04" ]; then PACKAGE_MANAGER_BIN="apt-get" PACKAGE_UPDATE_OPTIONS="update -y -qq" - INSTALL_PACKAGES="autoconf autotools-dev default-jre-headless fuse libfuse-dev libcurl4-openssl-dev libxml2-dev locales-all mime-support libtool pkg-config libssl-dev attr curl python3-pip" + INSTALL_PACKAGES="autoconf autotools-dev default-jre-headless fuse libfuse-dev libcurl4-openssl-dev libxml2-dev locales-all mime-support libtool pkg-config libssl-dev attr curl python3-pip unzip" INSTALL_CHECKER_PKGS="cppcheck shellcheck" INSTALL_CHECKER_PKG_OPTIONS="" @@ -101,7 +100,7 @@ elif [ "${CONTAINER_FULLNAME}" = "debian:bullseye" ]; then PACKAGE_MANAGER_BIN="apt-get" PACKAGE_UPDATE_OPTIONS="update -y -qq" - INSTALL_PACKAGES="autoconf autotools-dev default-jre-headless fuse libfuse-dev libcurl4-openssl-dev libxml2-dev locales-all mime-support libtool pkg-config libssl-dev attr curl procps python3-pip" + INSTALL_PACKAGES="autoconf autotools-dev default-jre-headless fuse libfuse-dev libcurl4-openssl-dev libxml2-dev locales-all mime-support libtool pkg-config libssl-dev attr curl procps python3-pip unzip" INSTALL_CHECKER_PKGS="cppcheck shellcheck" INSTALL_CHECKER_PKG_OPTIONS="" @@ -109,7 +108,7 @@ elif [ "${CONTAINER_FULLNAME}" = "debian:buster" ]; then PACKAGE_MANAGER_BIN="apt-get" PACKAGE_UPDATE_OPTIONS="update -y -qq" - INSTALL_PACKAGES="autoconf autotools-dev default-jre-headless fuse libfuse-dev libcurl4-openssl-dev libxml2-dev locales-all mime-support libtool pkg-config libssl-dev attr curl procps python3-pip" + INSTALL_PACKAGES="autoconf autotools-dev default-jre-headless fuse libfuse-dev libcurl4-openssl-dev libxml2-dev locales-all mime-support libtool pkg-config libssl-dev attr curl procps python3-pip unzip" INSTALL_CHECKER_PKGS="cppcheck shellcheck" INSTALL_CHECKER_PKG_OPTIONS="" @@ -117,7 +116,7 @@ elif [ "${CONTAINER_FULLNAME}" = "debian:stretch" ]; then PACKAGE_MANAGER_BIN="apt-get" PACKAGE_UPDATE_OPTIONS="update -y -qq" - INSTALL_PACKAGES="autoconf autotools-dev default-jre-headless fuse libfuse-dev libcurl4-openssl-dev libxml2-dev locales-all mime-support libtool pkg-config libssl-dev attr curl procps python3-pip" + INSTALL_PACKAGES="autoconf autotools-dev default-jre-headless fuse libfuse-dev libcurl4-openssl-dev libxml2-dev locales-all mime-support libtool pkg-config libssl-dev attr curl procps python3-pip unzip" INSTALL_CHECKER_PKGS="cppcheck shellcheck" INSTALL_CHECKER_PKG_OPTIONS="" @@ -129,7 +128,7 @@ elif [ "${CONTAINER_FULLNAME}" = "rockylinux:8" ]; then # Installing ShellCheck on Rocky Linux is not easy. # Give up to run ShellCheck on Rocky Linux as we don't have to run ShellChek on all operating systems. # - INSTALL_PACKAGES="curl-devel fuse fuse-devel gcc libstdc++-devel gcc-c++ glibc-langpack-en java-11-openjdk-headless libxml2-devel mailcap git automake make openssl-devel attr diffutils curl python3" + INSTALL_PACKAGES="curl-devel fuse fuse-devel gcc libstdc++-devel gcc-c++ glibc-langpack-en java-11-openjdk-headless libxml2-devel mailcap git automake make openssl-devel attr diffutils curl python3 unzip" INSTALL_CHECKER_PKGS="cppcheck" INSTALL_CHECKER_PKG_OPTIONS="--enablerepo=powertools" @@ -142,7 +141,7 @@ elif [ "${CONTAINER_FULLNAME}" = "centos:centos7" ]; then # And in this version, it cannot be passed due to following error. # "shellcheck: ./test/integration-test-main.sh: hGetContents: invalid argument (invalid byte sequence)" # - INSTALL_PACKAGES="curl-devel fuse fuse-devel gcc libstdc++-devel gcc-c++ glibc-langpack-en java-11-openjdk-headless libxml2-devel mailcap git automake make openssl-devel attr curl python3 epel-release" + INSTALL_PACKAGES="curl-devel fuse fuse-devel gcc libstdc++-devel gcc-c++ glibc-langpack-en java-11-openjdk-headless libxml2-devel mailcap git automake make openssl-devel attr curl python3 epel-release unzip" INSTALL_CHECKER_PKGS="cppcheck" INSTALL_CHECKER_PKG_OPTIONS="--enablerepo=epel" @@ -151,7 +150,7 @@ elif [ "${CONTAINER_FULLNAME}" = "fedora:35" ]; then PACKAGE_UPDATE_OPTIONS="update -y -qq" # TODO: Cannot use java-latest-openjdk (17) due to modules issue in S3Proxy/jclouds/Guice - INSTALL_PACKAGES="curl-devel fuse fuse-devel gcc libstdc++-devel gcc-c++ glibc-langpack-en java-11-openjdk-headless libxml2-devel mailcap git automake make openssl-devel curl attr diffutils procps python3-pip" + INSTALL_PACKAGES="curl-devel fuse fuse-devel gcc libstdc++-devel gcc-c++ glibc-langpack-en java-11-openjdk-headless libxml2-devel mailcap git automake make openssl-devel curl attr diffutils procps python3-pip unzip" INSTALL_CHECKER_PKGS="cppcheck ShellCheck" INSTALL_CHECKER_PKG_OPTIONS="" @@ -159,7 +158,7 @@ elif [ "${CONTAINER_FULLNAME}" = "opensuse/leap:15" ]; then PACKAGE_MANAGER_BIN="zypper" PACKAGE_UPDATE_OPTIONS="refresh" - INSTALL_PACKAGES="automake curl-devel fuse fuse-devel gcc-c++ java-11-openjdk-headless libxml2-devel make openssl-devel python3-pip curl attr ShellCheck" + INSTALL_PACKAGES="automake curl-devel fuse fuse-devel gcc-c++ java-11-openjdk-headless libxml2-devel make openssl-devel python3-pip curl attr ShellCheck unzip" INSTALL_CHECKER_PKGS="cppcheck ShellCheck" INSTALL_CHECKER_PKG_OPTIONS="" @@ -192,9 +191,13 @@ java -version # # Install awscli # -echo "${PRGNAME} [INFO] Install awscli package." -/bin/sh -c "${PIP_BIN} install ${PIP_OPTIONS} ${INSTALL_AWSCLI_PACKAGES}" -/bin/sh -c "${PIP_BIN} install ${PIP_OPTIONS} rsa" +echo "${PRGNAME} [INFO] Install awscli2 package." +CURRENT_DIR=$(pwd) +cd /tmp +curl "${AWSCLI_URI}" -o "${AWSCLI_ZIP_FILE}" +unzip "${AWSCLI_ZIP_FILE}" +./aws/install +cd "${CURRENT_DIR}" #----------------------------------------------------------- # Set environment for configure diff --git a/test/test-utils.sh b/test/test-utils.sh index cdf77da..e453dc6 100644 --- a/test/test-utils.sh +++ b/test/test-utils.sh @@ -336,8 +336,10 @@ function aws_cli() { if [ -n "${S3FS_PROFILE}" ]; then FLAGS="--profile ${S3FS_PROFILE}" fi + # [NOTE] + # AWS_EC2_METADATA_DISABLED for preventing the metadata service(to 169.254.169.254). # shellcheck disable=SC2086,SC2068 - aws $@ --endpoint-url "${S3_URL}" --ca-bundle /tmp/keystore.pem ${FLAGS} + AWS_EC2_METADATA_DISABLED=true aws $@ --endpoint-url "${S3_URL}" --ca-bundle /tmp/keystore.pem ${FLAGS} --debug } function wait_for_port() {