From 3a6af3858276346064524ac78f88fdcd2cf35414 Mon Sep 17 00:00:00 2001 From: Andrew Gaul Date: Tue, 15 Aug 2023 21:23:59 +0900 Subject: [PATCH] Tighten up Content-Type checking (#2258) --- .github/workflows/linux-ci-helper.sh | 20 ++++++++++---------- test/integration-test-main.sh | 24 ++++-------------------- test/test-utils.sh | 8 ++++---- 3 files changed, 18 insertions(+), 34 deletions(-) diff --git a/.github/workflows/linux-ci-helper.sh b/.github/workflows/linux-ci-helper.sh index 274317e..45fd966 100755 --- a/.github/workflows/linux-ci-helper.sh +++ b/.github/workflows/linux-ci-helper.sh @@ -85,7 +85,7 @@ if [ "${CONTAINER_FULLNAME}" = "ubuntu:23.04" ]; then PACKAGE_UPDATE_OPTIONS="update -y -qq" PACKAGE_INSTALL_OPTIONS="install -y" - INSTALL_PACKAGES="autoconf autotools-dev openjdk-17-jre-headless fuse libfuse-dev libcurl4-openssl-dev libxml2-dev locales-all mime-support libtool pkg-config libssl-dev attr curl python3-pip unzip" + INSTALL_PACKAGES="autoconf autotools-dev openjdk-17-jre-headless fuse jq libfuse-dev libcurl4-openssl-dev libxml2-dev locales-all mime-support libtool pkg-config libssl-dev attr curl python3-pip unzip" INSTALL_CHECKER_PKGS="cppcheck shellcheck" INSTALL_CHECKER_PKG_OPTIONS="" @@ -94,7 +94,7 @@ elif [ "${CONTAINER_FULLNAME}" = "ubuntu:22.04" ]; then PACKAGE_UPDATE_OPTIONS="update -y -qq" PACKAGE_INSTALL_OPTIONS="install -y" - INSTALL_PACKAGES="autoconf autotools-dev openjdk-17-jre-headless fuse libfuse-dev libcurl4-openssl-dev libxml2-dev locales-all mime-support libtool pkg-config libssl-dev attr curl python3-pip unzip" + INSTALL_PACKAGES="autoconf autotools-dev openjdk-17-jre-headless fuse jq libfuse-dev libcurl4-openssl-dev libxml2-dev locales-all mime-support libtool pkg-config libssl-dev attr curl python3-pip unzip" INSTALL_CHECKER_PKGS="cppcheck shellcheck" INSTALL_CHECKER_PKG_OPTIONS="" @@ -103,7 +103,7 @@ elif [ "${CONTAINER_FULLNAME}" = "ubuntu:20.04" ]; then PACKAGE_UPDATE_OPTIONS="update -y -qq" PACKAGE_INSTALL_OPTIONS="install -y" - INSTALL_PACKAGES="autoconf autotools-dev openjdk-17-jre-headless fuse libfuse-dev libcurl4-openssl-dev libxml2-dev locales-all mime-support libtool pkg-config libssl-dev attr curl python3-pip unzip" + INSTALL_PACKAGES="autoconf autotools-dev openjdk-17-jre-headless fuse jq libfuse-dev libcurl4-openssl-dev libxml2-dev locales-all mime-support libtool pkg-config libssl-dev attr curl python3-pip unzip" INSTALL_CHECKER_PKGS="cppcheck shellcheck" INSTALL_CHECKER_PKG_OPTIONS="" @@ -112,7 +112,7 @@ elif [ "${CONTAINER_FULLNAME}" = "debian:bullseye" ]; then PACKAGE_UPDATE_OPTIONS="update -y -qq" PACKAGE_INSTALL_OPTIONS="install -y" - INSTALL_PACKAGES="autoconf autotools-dev openjdk-17-jre-headless fuse libfuse-dev libcurl4-openssl-dev libxml2-dev locales-all mime-support libtool pkg-config libssl-dev attr curl procps python3-pip unzip" + INSTALL_PACKAGES="autoconf autotools-dev openjdk-17-jre-headless fuse jq libfuse-dev libcurl4-openssl-dev libxml2-dev locales-all mime-support libtool pkg-config libssl-dev attr curl procps python3-pip unzip" INSTALL_CHECKER_PKGS="cppcheck shellcheck" INSTALL_CHECKER_PKG_OPTIONS="" @@ -121,7 +121,7 @@ elif [ "${CONTAINER_FULLNAME}" = "debian:buster" ]; then PACKAGE_UPDATE_OPTIONS="update -y -qq" PACKAGE_INSTALL_OPTIONS="install -y" - INSTALL_PACKAGES="autoconf autotools-dev default-jre-headless fuse libfuse-dev libcurl4-openssl-dev libxml2-dev locales-all mime-support libtool pkg-config libssl-dev attr curl procps python3-pip unzip" + INSTALL_PACKAGES="autoconf autotools-dev default-jre-headless fuse jq libfuse-dev libcurl4-openssl-dev libxml2-dev locales-all mime-support libtool pkg-config libssl-dev attr curl procps python3-pip unzip" INSTALL_CHECKER_PKGS="cppcheck shellcheck" INSTALL_CHECKER_PKG_OPTIONS="" @@ -171,7 +171,7 @@ elif [ "${CONTAINER_FULLNAME}" = "centos:centos7" ]; then # "shellcheck: ./test/integration-test-main.sh: hGetContents: invalid argument (invalid byte sequence)" # INSTALL_PACKAGES="curl-devel fuse fuse-devel gcc libstdc++-devel gcc-c++ glibc-langpack-en java-11-openjdk-headless libxml2-devel mailcap git automake make openssl openssl-devel attr curl python3 epel-release unzip" - INSTALL_CHECKER_PKGS="cppcheck" + INSTALL_CHECKER_PKGS="cppcheck jq" INSTALL_CHECKER_PKG_OPTIONS="--enablerepo=epel" elif [ "${CONTAINER_FULLNAME}" = "fedora:38" ]; then @@ -179,7 +179,7 @@ elif [ "${CONTAINER_FULLNAME}" = "fedora:38" ]; then PACKAGE_UPDATE_OPTIONS="update -y -qq" PACKAGE_INSTALL_OPTIONS="install -y" - INSTALL_PACKAGES="curl-devel fuse fuse-devel gcc libstdc++-devel gcc-c++ glibc-langpack-en java-latest-openjdk-headless libxml2-devel mailcap git automake make openssl openssl-devel curl attr diffutils procps python3-pip unzip" + INSTALL_PACKAGES="curl-devel fuse fuse-devel gcc libstdc++-devel gcc-c++ glibc-langpack-en java-latest-openjdk-headless jq libxml2-devel mailcap git automake make openssl openssl-devel curl attr diffutils procps python3-pip unzip" INSTALL_CHECKER_PKGS="cppcheck ShellCheck" INSTALL_CHECKER_PKG_OPTIONS="" @@ -188,7 +188,7 @@ elif [ "${CONTAINER_FULLNAME}" = "fedora:37" ]; then PACKAGE_UPDATE_OPTIONS="update -y -qq" PACKAGE_INSTALL_OPTIONS="install -y" - INSTALL_PACKAGES="curl-devel fuse fuse-devel gcc libstdc++-devel gcc-c++ glibc-langpack-en java-latest-openjdk-headless libxml2-devel mailcap git automake make openssl openssl-devel curl attr diffutils procps python3-pip unzip" + INSTALL_PACKAGES="curl-devel fuse fuse-devel gcc libstdc++-devel gcc-c++ glibc-langpack-en java-latest-openjdk-headless jq libxml2-devel mailcap git automake make openssl openssl-devel curl attr diffutils procps python3-pip unzip" INSTALL_CHECKER_PKGS="cppcheck ShellCheck" INSTALL_CHECKER_PKG_OPTIONS="" @@ -197,7 +197,7 @@ elif [ "${CONTAINER_FULLNAME}" = "opensuse/leap:15" ]; then PACKAGE_UPDATE_OPTIONS="refresh" PACKAGE_INSTALL_OPTIONS="install -y" - INSTALL_PACKAGES="automake curl-devel fuse fuse-devel gcc-c++ java-17-openjdk-headless libxml2-devel make openssl openssl-devel python3-pip curl attr ShellCheck unzip" + INSTALL_PACKAGES="automake curl-devel fuse fuse-devel gcc-c++ java-17-openjdk-headless jq libxml2-devel make openssl openssl-devel python3-pip curl attr ShellCheck unzip" INSTALL_CHECKER_PKGS="cppcheck ShellCheck" INSTALL_CHECKER_PKG_OPTIONS="" @@ -206,7 +206,7 @@ elif [ "${CONTAINER_FULLNAME}" = "alpine:3.17" ]; then PACKAGE_UPDATE_OPTIONS="update --no-progress" PACKAGE_INSTALL_OPTIONS="add --no-progress --no-cache" - INSTALL_PACKAGES="bash curl g++ make automake autoconf libtool git curl-dev fuse-dev libxml2-dev openssl coreutils procps attr sed mailcap openjdk17 aws-cli" + INSTALL_PACKAGES="bash curl g++ make automake autoconf libtool git curl-dev fuse-dev jq libxml2-dev openssl coreutils procps attr sed mailcap openjdk17 aws-cli" INSTALL_CHECKER_PKGS="cppcheck shellcheck" INSTALL_CHECKER_PKG_OPTIONS="" diff --git a/test/integration-test-main.sh b/test/integration-test-main.sh index 63959e9..2b15968 100755 --- a/test/integration-test-main.sh +++ b/test/integration-test-main.sh @@ -1933,32 +1933,16 @@ function test_content_type() { local DIR_NAME; DIR_NAME=$(basename "${PWD}") touch "test.txt" - local CONTENT_TYPE; CONTENT_TYPE=$(aws_cli s3api head-object --bucket "${TEST_BUCKET_1}" --key "${DIR_NAME}/test.txt" | grep "ContentType") - if ! echo "${CONTENT_TYPE}" | grep -q "text/plain"; then - echo "Unexpected Content-Type: ${CONTENT_TYPE}" - return 1; - fi + check_content_type "${DIR_NAME}/test.txt" "text/plain" touch "test.jpg" - CONTENT_TYPE=$(aws_cli s3api head-object --bucket "${TEST_BUCKET_1}" --key "${DIR_NAME}/test.jpg" | grep "ContentType") - if ! echo "${CONTENT_TYPE}" | grep -q "image/jpeg"; then - echo "Unexpected Content-Type: ${CONTENT_TYPE}" - return 1; - fi + check_content_type "${DIR_NAME}/test.jpg" "image/jpeg" touch "test.bin" - CONTENT_TYPE=$(aws_cli s3api head-object --bucket "${TEST_BUCKET_1}" --key "${DIR_NAME}/test.bin" | grep "ContentType") - if ! echo "${CONTENT_TYPE}" | grep -q "application/octet-stream"; then - echo "Unexpected Content-Type: ${CONTENT_TYPE}" - return 1; - fi + check_content_type "${DIR_NAME}/test.bin" "application/octet-stream" mkdir "test.dir" - CONTENT_TYPE=$(aws_cli s3api head-object --bucket "${TEST_BUCKET_1}" --key "${DIR_NAME}/test.dir/" | grep "ContentType") - if ! echo "${CONTENT_TYPE}" | grep -q "application/x-directory"; then - echo "Unexpected Content-Type: ${CONTENT_TYPE}" - return 1; - fi + check_content_type "${DIR_NAME}/test.dir/" "application/x-directory" rm -f test.txt rm -f test.jpg diff --git a/test/test-utils.sh b/test/test-utils.sh index 07dfd17..531c4a0 100644 --- a/test/test-utils.sh +++ b/test/test-utils.sh @@ -315,11 +315,11 @@ function get_permissions() { function check_content_type() { local INFO_STR - INFO_STR=$(aws_cli s3api head-object --bucket "${TEST_BUCKET_1}" --key "$1") - if [[ "${INFO_STR}" != *"$2"* ]] + INFO_STR=$(aws_cli s3api head-object --bucket "${TEST_BUCKET_1}" --key "$1" | jq -r .ContentType) + if [ "${INFO_STR}" != "$2" ] then - echo "moved file content-type is not as expected expected:$2 got:${INFO_STR}" - exit 1 + echo "Expected Content-Type: $2 but got: ${INFO_STR}" + return 1 fi }