#!/bin/bash # # s3fs - FUSE-based file system backed by Amazon S3 # # Copyright 2007-2008 Randy Rizun # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; either version 2 # of the License, or (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. # # # Common code for starting an s3fs-fuse mountpoint and an S3Proxy instance # to run tests against S3Proxy locally. # # To run against an Amazon S3 or other S3 provider, specify the following # environment variables: # # S3FS_CREDENTIALS_FILE=keyfile s3fs format key file # S3FS_PROFILE=name s3fs profile to use (overrides key file) # TEST_BUCKET_1=bucketname Name of bucket to use # S3PROXY_BINARY="" Specify empty string to skip S3Proxy start # S3_URL="https://s3.amazonaws.com" Specify Amazon AWS as the S3 provider # S3_ENDPOINT="us-east-1" Specify region # TMPDIR="/var/tmp" Set to use a temporary directory different # from /var/tmp # CHAOS_HTTP_PROXY=1 Test proxy(environment) by CHAOS HTTP PROXY # CHAOS_HTTP_PROXY_OPT=1 Test proxy(option) by CHAOS HTTP PROXY # # Example of running against Amazon S3 using a bucket named "bucket": # # S3FS_CREDENTIALS_FILE=keyfile TEST_BUCKET_1=bucket S3PROXY_BINARY="" S3_URL="https://s3.amazonaws.com" ./small-integration-test.sh # # To change the s3fs-fuse debug level: # # DBGLEVEL=debug ./small-integration-test.sh # # To stop and wait after the mount point is up for manual interaction. This allows you to # explore the mounted file system exactly as it would have been started for the test case # # INTERACT=1 DBGLEVEL=debug ./small-integration-test.sh # # Run all of the tests from the makefile # # S3FS_CREDENTIALS_FILE=keyfile TEST_BUCKET_1=bucket S3PROXY_BINARY="" S3_URL="https://s3.amazonaws.com" make check # # Run the tests with request auth turned off in both S3Proxy and s3fs-fuse. This can be # useful for poking around with plain old curl # # PUBLIC=1 INTERACT=1 ./small-integration-test.sh # # A valgrind tool can be specified # eg: VALGRIND="--tool=memcheck --leak-check=full" ./small-integration-test.sh set -o errexit set -o pipefail S3FS=../src/s3fs # Allow these defaulted values to be overridden # # [NOTE] # CHAOS HTTP PROXY does not support HTTPS. # if [ -z "${CHAOS_HTTP_PROXY}" ] && [ -z "${CHAOS_HTTP_PROXY_OPT}" ]; then : "${S3_URL:="https://127.0.0.1:8080"}" else : "${S3_URL:="http://127.0.0.1:8080"}" fi : "${S3_ENDPOINT:="us-east-1"}" : "${S3FS_CREDENTIALS_FILE:="passwd-s3fs"}" : "${TEST_BUCKET_1:="s3fs-integration-test"}" export TEST_BUCKET_1 export S3_URL export S3_ENDPOINT TEST_SCRIPT_DIR=$(pwd) export TEST_SCRIPT_DIR export TEST_BUCKET_MOUNT_POINT_1=${TEST_BUCKET_1} S3PROXY_VERSION="2.0.0" S3PROXY_BINARY="${S3PROXY_BINARY-"s3proxy-${S3PROXY_VERSION}"}" CHAOS_HTTP_PROXY_VERSION="1.1.0" CHAOS_HTTP_PROXY_BINARY="chaos-http-proxy-${CHAOS_HTTP_PROXY_VERSION}" if [ ! -f "$S3FS_CREDENTIALS_FILE" ] then echo "Missing credentials file: ${S3FS_CREDENTIALS_FILE}" exit 1 fi chmod 600 "${S3FS_CREDENTIALS_FILE}" if [ -z "${S3FS_PROFILE}" ]; then AWS_ACCESS_KEY_ID=$(cut -d: -f1 "${S3FS_CREDENTIALS_FILE}") export AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY=$(cut -d: -f2 "${S3FS_CREDENTIALS_FILE}") export AWS_SECRET_ACCESS_KEY fi if [ ! -d "${TEST_BUCKET_MOUNT_POINT_1}" ]; then mkdir -p "${TEST_BUCKET_MOUNT_POINT_1}" fi # This function execute the function parameters $1 times # before giving up, with 1 second delays. function retry { local N="$1" shift rc=0 for _ in $(seq "${N}"); do echo "Trying: $*" # shellcheck disable=SC2068,SC2294 eval $@ rc=$? if [ "${rc}" -eq 0 ]; then break fi sleep 1 echo "Retrying: $*" done if [ "${rc}" -ne 0 ]; then echo "timeout waiting for $*" fi return "${rc}" } # Proxy is not started if S3PROXY_BINARY is an empty string # PUBLIC unset: use s3proxy.conf # PUBLIC=1: use s3proxy-noauth.conf (no request signing) # function start_s3proxy { if [ -n "${PUBLIC}" ]; then local S3PROXY_CONFIG="s3proxy-noauth.conf" else if [ -z "${CHAOS_HTTP_PROXY}" ] && [ -z "${CHAOS_HTTP_PROXY_OPT}" ]; then local S3PROXY_CONFIG="s3proxy.conf" else local S3PROXY_CONFIG="s3proxy_http.conf" fi fi if [ -n "${S3PROXY_BINARY}" ] then if [ ! -e "${S3PROXY_BINARY}" ]; then curl "https://github.com/gaul/s3proxy/releases/download/s3proxy-${S3PROXY_VERSION}/s3proxy" \ --fail --location --silent --output "${S3PROXY_BINARY}" chmod +x "${S3PROXY_BINARY}" fi # generate self-signed SSL certificate # # [NOTE] # The PROXY test is HTTP only, so do not create CA certificates. # if [ -z "${CHAOS_HTTP_PROXY}" ] && [ -z "${CHAOS_HTTP_PROXY_OPT}" ]; then S3PROXY_CACERT_FILE="/tmp/keystore.pem" rm -f /tmp/keystore.jks "${S3PROXY_CACERT_FILE}" printf 'password\npassword\n\n\n\n\n\n\ny' | keytool -genkey -keystore /tmp/keystore.jks -keyalg RSA -keysize 2048 -validity 365 -ext SAN=IP:127.0.0.1 echo password | keytool -exportcert -keystore /tmp/keystore.jks -rfc -file "${S3PROXY_CACERT_FILE}" else S3PROXY_CACERT_FILE="" fi "${STDBUF_BIN}" -oL -eL java -jar "${S3PROXY_BINARY}" --properties "${S3PROXY_CONFIG}" & S3PROXY_PID=$! # wait for S3Proxy to start wait_for_port 8080 fi if [ -n "${CHAOS_HTTP_PROXY}" ] || [ -n "${CHAOS_HTTP_PROXY_OPT}" ]; then if [ ! -e "${CHAOS_HTTP_PROXY_BINARY}" ]; then curl "https://github.com/bouncestorage/chaos-http-proxy/releases/download/chaos-http-proxy-${CHAOS_HTTP_PROXY_VERSION}/chaos-http-proxy" \ --fail --location --silent --output "${CHAOS_HTTP_PROXY_BINARY}" chmod +x "${CHAOS_HTTP_PROXY_BINARY}" fi "${STDBUF_BIN}" -oL -eL java -jar "${CHAOS_HTTP_PROXY_BINARY}" --properties chaos-http-proxy.conf & CHAOS_HTTP_PROXY_PID=$! # wait for Chaos HTTP Proxy to start wait_for_port 1080 fi } function stop_s3proxy { if [ -n "${S3PROXY_PID}" ] then kill "${S3PROXY_PID}" fi if [ -n "${CHAOS_HTTP_PROXY_PID}" ] then kill "${CHAOS_HTTP_PROXY_PID}" fi } # Mount the bucket, function arguments passed to s3fs in addition to # a set of common arguments. function start_s3fs { # Public bucket if PUBLIC is set if [ -n "${PUBLIC}" ]; then local AUTH_OPT="-o public_bucket=1" elif [ -n "${S3FS_PROFILE}" ]; then local AUTH_OPT="-o profile=${S3FS_PROFILE}" else local AUTH_OPT="-o passwd_file=${S3FS_CREDENTIALS_FILE}" fi # If VALGRIND is set, pass it as options to valgrind. # start valgrind-listener in another shell. # eg: VALGRIND="--tool=memcheck --leak-check=full" ./small-integration-test.sh # Start valgrind-listener (default port is 1500) if [ -n "${VALGRIND}" ]; then VALGRIND_EXEC="valgrind ${VALGRIND} --log-socket=127.0.1.1" fi # On OSX only, we need to specify the direct_io and auto_cache flag. # # And Turn off creation and reference of spotlight index. # (Leaving spotlight ON will result in a lot of wasted requests, # which will affect test execution time) # if [ "$(uname)" = "Darwin" ]; then local DIRECT_IO_OPT="-o direct_io -o auto_cache" # disable spotlight sudo mdutil -a -i off else local DIRECT_IO_OPT="" fi # Set environment variables or options for proxy. # And the PROXY test is HTTP only and does not set CA certificates. # if [ -n "${CHAOS_HTTP_PROXY}" ]; then export http_proxy="127.0.0.1:1080" S3FS_HTTP_PROXY_OPT="" elif [ -n "${CHAOS_HTTP_PROXY_OPT}" ]; then S3FS_HTTP_PROXY_OPT="-o proxy=http://127.0.0.1:1080" else S3FS_HTTP_PROXY_OPT="" fi # [NOTE] # For macos fuse-t, we need to specify the "noattrcache" option to # disable NFS caching. # if [ "$(uname)" = "Darwin" ]; then local FUSE_T_ATTRCACHE_OPT="-o noattrcache" else local FUSE_T_ATTRCACHE_OPT="" fi # [NOTE] # On macOS we may get a VERIFY error for the self-signed certificate used by s3proxy. # We can specify NO_CHECK_CERT=1 to avoid this. # if [ -n "${NO_CHECK_CERT}" ] && [ "${NO_CHECK_CERT}" -eq 1 ]; then local NO_CHECK_CERT_OPT="-o no_check_certificate" else local NO_CHECK_CERT_OPT="" fi # Common s3fs options: # # TODO: Allow all these options to be overridden with env variables # # use_path_request_style # The test env doesn't have virtual hosts # $AUTH_OPT # Will be either "-o public_bucket=1" # or # "-o passwd_file=${S3FS_CREDENTIALS_FILE}" # dbglevel # error by default. override with DBGLEVEL env variable # -f # Keep s3fs in foreground instead of daemonizing # # subshell with set -x to log exact invocation of s3fs-fuse # shellcheck disable=SC2086 ( set -x CURL_CA_BUNDLE="${S3PROXY_CACERT_FILE}" \ ${STDBUF_BIN} -oL -eL \ ${VALGRIND_EXEC} \ ${S3FS} \ ${TEST_BUCKET_1} \ ${TEST_BUCKET_MOUNT_POINT_1} \ -o use_path_request_style \ -o url="${S3_URL}" \ -o endpoint="${S3_ENDPOINT}" \ -o use_xattr=1 \ -o enable_unsigned_payload \ ${AUTH_OPT} \ ${DIRECT_IO_OPT} \ ${S3FS_HTTP_PROXY_OPT} \ ${NO_CHECK_CERT_OPT} \ ${FUSE_T_ATTRCACHE_OPT} \ -o stat_cache_expire=1 \ -o stat_cache_interval_expire=1 \ -o dbglevel="${DBGLEVEL:=info}" \ -o no_time_stamp_msg \ -o retries=3 \ -f \ "${@}" & echo $! >&3 ) 3>pid | "${STDBUF_BIN}" -oL -eL "${SED_BIN}" "${SED_BUFFER_FLAG}" "s/^/s3fs: /" & sleep 1 S3FS_PID=$(/dev/null) if echo "${_DF_RESULT}" | grep -q "${TEST_BUCKET_MOUNT_POINT_1}"; then break; fi sleep 1 TRYCOUNT=$((TRYCOUNT + 1)) done if [ "${TRYCOUNT}" -gt "${RETRIES}" ]; then echo "Waited ${TRYCOUNT} seconds, but it could not be mounted." exit 1 fi else retry "${RETRIES:=20}" grep -q "${TEST_BUCKET_MOUNT_POINT_1}" /proc/mounts || exit 1 fi # Quick way to start system up for manual testing with options under test if [[ -n "${INTERACT}" ]]; then echo "Mountpoint ${TEST_BUCKET_MOUNT_POINT_1} is ready" echo "control-C to quit" sleep infinity exit 0 fi } function stop_s3fs { # Retry in case file system is in use if [ "$(uname)" = "Darwin" ]; then if df | grep -q "${TEST_BUCKET_MOUNT_POINT_1}"; then retry 10 df "|" grep -q "${TEST_BUCKET_MOUNT_POINT_1}" "&&" umount "${TEST_BUCKET_MOUNT_POINT_1}" fi else if grep -q "${TEST_BUCKET_MOUNT_POINT_1}" /proc/mounts; then retry 10 grep -q "${TEST_BUCKET_MOUNT_POINT_1}" /proc/mounts "&&" fusermount -u "${TEST_BUCKET_MOUNT_POINT_1}" fi fi } # trap handlers do not stack. If a test sets its own, the new handler should call common_exit_handler function common_exit_handler { stop_s3fs stop_s3proxy } trap common_exit_handler EXIT # # Local variables: # tab-width: 4 # c-basic-offset: 4 # End: # vim600: expandtab sw=4 ts=4 fdm=marker # vim<600: expandtab sw=4 ts=4 #