octoleo/s3fs-fuse
1
0
Fork 0
mirror of https://github.com/s3fs-fuse/s3fs-fuse.git synced 2024-12-22 08:48:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add AWS CLI config for use_sse=custom (#2230)

Browse Source
This commit is contained in:
Andrew Gaul 2023-07-25 23:31:20 +09:00 committed by GitHub
parent cb3dc28e6e
commit e14a2eb94b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 27 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
.github/workflows/linux-ci-helper.sh vendored
View File

@ -137,7 +137,7 @@ elif [ "${CONTAINER_FULLNAME}" = "rockylinux:9" ]; then
# #
PACKAGE_INSTALL_ADDITIONAL_OPTIONS="--allowerasing" PACKAGE_INSTALL_ADDITIONAL_OPTIONS="--allowerasing"
INSTALL_PACKAGES="curl-devel fuse fuse-devel gcc libstdc++-devel gcc-c++ glibc-langpack-en java-17-openjdk-headless libxml2-devel mailcap git automake make openssl-devel attr diffutils curl python3 procps unzip xz https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm" INSTALL_PACKAGES="curl-devel fuse fuse-devel gcc libstdc++-devel gcc-c++ glibc-langpack-en java-17-openjdk-headless libxml2-devel mailcap git automake make openssl openssl-devel attr diffutils curl python3 procps unzip xz https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm"
INSTALL_CHECKER_PKGS="cppcheck" INSTALL_CHECKER_PKGS="cppcheck"
INSTALL_CHECKER_PKG_OPTIONS="--enablerepo=epel" INSTALL_CHECKER_PKG_OPTIONS="--enablerepo=epel"
@ -151,7 +151,7 @@ elif [ "${CONTAINER_FULLNAME}" = "rockylinux:8" ]; then
PACKAGE_UPDATE_OPTIONS="update -y -qq" PACKAGE_UPDATE_OPTIONS="update -y -qq"
PACKAGE_INSTALL_OPTIONS="install -y" PACKAGE_INSTALL_OPTIONS="install -y"
INSTALL_PACKAGES="curl-devel fuse fuse-devel gcc libstdc++-devel gcc-c++ glibc-langpack-en java-17-openjdk-headless libxml2-devel mailcap git automake make openssl-devel attr diffutils curl python3 unzip" INSTALL_PACKAGES="curl-devel fuse fuse-devel gcc libstdc++-devel gcc-c++ glibc-langpack-en java-17-openjdk-headless libxml2-devel mailcap git automake make openssl openssl-devel attr diffutils curl python3 unzip"
INSTALL_CHECKER_PKGS="cppcheck" INSTALL_CHECKER_PKGS="cppcheck"
INSTALL_CHECKER_PKG_OPTIONS="--enablerepo=powertools" INSTALL_CHECKER_PKG_OPTIONS="--enablerepo=powertools"
@ -170,7 +170,7 @@ elif [ "${CONTAINER_FULLNAME}" = "centos:centos7" ]; then
# And in this version, it cannot be passed due to following error. # And in this version, it cannot be passed due to following error.
# "shellcheck: ./test/integration-test-main.sh: hGetContents: invalid argument (invalid byte sequence)" # "shellcheck: ./test/integration-test-main.sh: hGetContents: invalid argument (invalid byte sequence)"
# #
INSTALL_PACKAGES="curl-devel fuse fuse-devel gcc libstdc++-devel gcc-c++ glibc-langpack-en java-11-openjdk-headless libxml2-devel mailcap git automake make openssl-devel attr curl python3 epel-release unzip" INSTALL_PACKAGES="curl-devel fuse fuse-devel gcc libstdc++-devel gcc-c++ glibc-langpack-en java-11-openjdk-headless libxml2-devel mailcap git automake make openssl openssl-devel attr curl python3 epel-release unzip"
INSTALL_CHECKER_PKGS="cppcheck" INSTALL_CHECKER_PKGS="cppcheck"
INSTALL_CHECKER_PKG_OPTIONS="--enablerepo=epel" INSTALL_CHECKER_PKG_OPTIONS="--enablerepo=epel"
@ -179,7 +179,7 @@ elif [ "${CONTAINER_FULLNAME}" = "fedora:38" ]; then
PACKAGE_UPDATE_OPTIONS="update -y -qq" PACKAGE_UPDATE_OPTIONS="update -y -qq"
PACKAGE_INSTALL_OPTIONS="install -y" PACKAGE_INSTALL_OPTIONS="install -y"
INSTALL_PACKAGES="curl-devel fuse fuse-devel gcc libstdc++-devel gcc-c++ glibc-langpack-en java-latest-openjdk-headless libxml2-devel mailcap git automake make openssl-devel curl attr diffutils procps python3-pip unzip" INSTALL_PACKAGES="curl-devel fuse fuse-devel gcc libstdc++-devel gcc-c++ glibc-langpack-en java-latest-openjdk-headless libxml2-devel mailcap git automake make openssl openssl-devel curl attr diffutils procps python3-pip unzip"
INSTALL_CHECKER_PKGS="cppcheck ShellCheck" INSTALL_CHECKER_PKGS="cppcheck ShellCheck"
INSTALL_CHECKER_PKG_OPTIONS="" INSTALL_CHECKER_PKG_OPTIONS=""
@ -188,7 +188,7 @@ elif [ "${CONTAINER_FULLNAME}" = "fedora:37" ]; then
PACKAGE_UPDATE_OPTIONS="update -y -qq" PACKAGE_UPDATE_OPTIONS="update -y -qq"
PACKAGE_INSTALL_OPTIONS="install -y" PACKAGE_INSTALL_OPTIONS="install -y"
INSTALL_PACKAGES="curl-devel fuse fuse-devel gcc libstdc++-devel gcc-c++ glibc-langpack-en java-latest-openjdk-headless libxml2-devel mailcap git automake make openssl-devel curl attr diffutils procps python3-pip unzip" INSTALL_PACKAGES="curl-devel fuse fuse-devel gcc libstdc++-devel gcc-c++ glibc-langpack-en java-latest-openjdk-headless libxml2-devel mailcap git automake make openssl openssl-devel curl attr diffutils procps python3-pip unzip"
INSTALL_CHECKER_PKGS="cppcheck ShellCheck" INSTALL_CHECKER_PKGS="cppcheck ShellCheck"
INSTALL_CHECKER_PKG_OPTIONS="" INSTALL_CHECKER_PKG_OPTIONS=""
@ -197,7 +197,7 @@ elif [ "${CONTAINER_FULLNAME}" = "opensuse/leap:15" ]; then
PACKAGE_UPDATE_OPTIONS="refresh" PACKAGE_UPDATE_OPTIONS="refresh"
PACKAGE_INSTALL_OPTIONS="install -y" PACKAGE_INSTALL_OPTIONS="install -y"
INSTALL_PACKAGES="automake curl-devel fuse fuse-devel gcc-c++ java-17-openjdk-headless libxml2-devel make openssl-devel python3-pip curl attr ShellCheck unzip" INSTALL_PACKAGES="automake curl-devel fuse fuse-devel gcc-c++ java-17-openjdk-headless libxml2-devel make openssl openssl-devel python3-pip curl attr ShellCheck unzip"
INSTALL_CHECKER_PKGS="cppcheck ShellCheck" INSTALL_CHECKER_PKGS="cppcheck ShellCheck"
INSTALL_CHECKER_PKG_OPTIONS="" INSTALL_CHECKER_PKG_OPTIONS=""
@ -206,7 +206,7 @@ elif [ "${CONTAINER_FULLNAME}" = "alpine:3.17" ]; then
PACKAGE_UPDATE_OPTIONS="update --no-progress" PACKAGE_UPDATE_OPTIONS="update --no-progress"
PACKAGE_INSTALL_OPTIONS="add --no-progress --no-cache" PACKAGE_INSTALL_OPTIONS="add --no-progress --no-cache"
INSTALL_PACKAGES="bash curl g++ make automake autoconf libtool git curl-dev fuse-dev libxml2-dev coreutils procps attr sed mailcap openjdk17 aws-cli" INSTALL_PACKAGES="bash curl g++ make automake autoconf libtool git curl-dev fuse-dev libxml2-dev openssl coreutils procps attr sed mailcap openjdk17 aws-cli"
INSTALL_CHECKER_PKGS="cppcheck shellcheck" INSTALL_CHECKER_PKGS="cppcheck shellcheck"
INSTALL_CHECKER_PKG_OPTIONS="" INSTALL_CHECKER_PKG_OPTIONS=""

7
test/small-integration-test.sh
View File

@ -38,6 +38,12 @@ source test-utils.sh
FAKE_FREE_DISK_SIZE=200 FAKE_FREE_DISK_SIZE=200
ENSURE_DISKFREE_SIZE=10 ENSURE_DISKFREE_SIZE=10
# set up client-side encryption keys
head -c 32 < /dev/urandom > /tmp/ssekey.bin
base64 < /tmp/ssekey.bin > /tmp/ssekey
openssl md5 -binary < /tmp/ssekey.bin | base64 > /tmp/ssekeymd5
chmod 600 /tmp/ssekey /tmp/ssekey.bin /tmp/ssekeymd5
export CACHE_DIR export CACHE_DIR
export ENSURE_DISKFREE_SIZE export ENSURE_DISKFREE_SIZE
if [ -n "${ALL_TESTS}" ]; then if [ -n "${ALL_TESTS}" ]; then
@ -52,6 +58,7 @@ if [ -n "${ALL_TESTS}" ]; then
sigv4 sigv4
"singlepart_copy_limit=10" # limit size to exercise multipart code paths "singlepart_copy_limit=10" # limit size to exercise multipart code paths
#use_sse # TODO: S3Proxy does not support SSE #use_sse # TODO: S3Proxy does not support SSE
#use_sse=custom:/tmp/ssekey # TODO: S3Proxy does not support SSE
"use_cache=${CACHE_DIR} -o ensure_diskfree=${ENSURE_DISKFREE_SIZE} -o fake_diskfree=${FAKE_FREE_DISK_SIZE} -o streamupload" "use_cache=${CACHE_DIR} -o ensure_diskfree=${ENSURE_DISKFREE_SIZE} -o fake_diskfree=${FAKE_FREE_DISK_SIZE} -o streamupload"
) )
else else

13
test/test-utils.sh
View File

@ -334,6 +334,19 @@ function aws_cli() {
if [ -n "${S3FS_PROFILE}" ]; then if [ -n "${S3FS_PROFILE}" ]; then
FLAGS="--profile ${S3FS_PROFILE}" FLAGS="--profile ${S3FS_PROFILE}"
fi fi
if [ "$1" = "s3" ] && [ "$2" != "ls" ] && [ "$2" != "mb" ]; then
# shellcheck disable=SC2009
if ps u -p "${S3FS_PID}" | grep -q use_sse=custom; then
FLAGS="${FLAGS} --sse-c AES256 --sse-c-key fileb:///tmp/ssekey.bin"
fi
elif [ "$1" = "s3api" ] && [ "$2" != "head-bucket" ]; then
# shellcheck disable=SC2009
if ps u -p "${S3FS_PID}" | grep -q use_sse=custom; then
FLAGS="${FLAGS} --sse-customer-algorithm AES256 --sse-customer-key $(cat /tmp/ssekey) --sse-customer-key-md5 $(cat /tmp/ssekeymd5)"
fi
fi
# [NOTE] # [NOTE]
# AWS_EC2_METADATA_DISABLED for preventing the metadata service(to 169.254.169.254). # AWS_EC2_METADATA_DISABLED for preventing the metadata service(to 169.254.169.254).
# shellcheck disable=SC2086,SC2068 # shellcheck disable=SC2086,SC2068