#!/bin/bash # # s3fs - FUSE-based file system backed by Amazon S3 # # Copyright 2007-2008 Randy Rizun # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; either version 2 # of the License, or (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. # # # Common code for starting an s3fs-fuse mountpoint and an S3Proxy instance # to run tests against S3Proxy locally. # # To run against an Amazon S3 or other S3 provider, specify the following # environment variables: # # S3FS_CREDENTIALS_FILE=keyfile s3fs format key file # S3FS_PROFILE=name s3fs profile to use (overrides key file) # TEST_BUCKET_1=bucketname Name of bucket to use # S3PROXY_BINARY="" Specify empty string to skip S3Proxy start # S3_URL="https://s3.amazonaws.com" Specify Amazon AWS as the S3 provider # S3_ENDPOINT="us-east-1" Specify region # # Example of running against Amazon S3 using a bucket named "bucket": # # S3FS_CREDENTIALS_FILE=keyfile TEST_BUCKET_1=bucket S3PROXY_BINARY="" S3_URL="https://s3.amazonaws.com" ./small-integration-test.sh # # To change the s3fs-fuse debug level: # # DBGLEVEL=debug ./small-integration-test.sh # # To stop and wait after the mount point is up for manual interaction. This allows you to # explore the mounted file system exactly as it would have been started for the test case # # INTERACT=1 DBGLEVEL=debug ./small-integration-test.sh # # Run all of the tests from the makefile # # S3FS_CREDENTIALS_FILE=keyfile TEST_BUCKET_1=bucket S3PROXY_BINARY="" S3_URL="https://s3.amazonaws.com" make check # # Run the tests with request auth turned off in both S3Proxy and s3fs-fuse. This can be # useful for poking around with plain old curl # # PUBLIC=1 INTERACT=1 ./small-integration-test.sh # # A valgrind tool can be specified # eg: VALGRIND="--tool=memcheck --leak-check=full" ./small-integration-test.sh set -o errexit set -o pipefail S3FS=../src/s3fs # Allow these defaulted values to be overridden : ${S3_URL:="https://127.0.0.1:8080"} : ${S3_ENDPOINT:="us-east-1"} : ${S3FS_CREDENTIALS_FILE:="passwd-s3fs"} : ${TEST_BUCKET_1:="s3fs-integration-test"} export TEST_BUCKET_1 export S3_URL export S3_ENDPOINT export TEST_SCRIPT_DIR=`pwd` export TEST_BUCKET_MOUNT_POINT_1=${TEST_BUCKET_1} S3PROXY_VERSION="1.7.1" S3PROXY_BINARY=${S3PROXY_BINARY-"s3proxy-${S3PROXY_VERSION}"} CHAOS_HTTP_PROXY_VERSION="1.1.0" CHAOS_HTTP_PROXY_BINARY="chaos-http-proxy-${CHAOS_HTTP_PROXY_VERSION}" if [ ! -f "$S3FS_CREDENTIALS_FILE" ] then echo "Missing credentials file: $S3FS_CREDENTIALS_FILE" exit 1 fi chmod 600 "$S3FS_CREDENTIALS_FILE" if [ -z "${S3FS_PROFILE}" ]; then export AWS_ACCESS_KEY_ID=$(cut -d: -f1 ${S3FS_CREDENTIALS_FILE}) export AWS_SECRET_ACCESS_KEY=$(cut -d: -f2 ${S3FS_CREDENTIALS_FILE}) fi if [ ! -d $TEST_BUCKET_MOUNT_POINT_1 ] then mkdir -p $TEST_BUCKET_MOUNT_POINT_1 fi # This function execute the function parameters $1 times # before giving up, with 1 second delays. function retry { set +o errexit N=$1; shift; status=0 for i in $(seq $N); do echo "Trying: $*" eval $@ status=$? if [ $status == 0 ]; then break fi sleep 1 echo "Retrying: $*" done if [ $status != 0 ]; then echo "timeout waiting for $*" fi set -o errexit return $status } # Proxy is not started if S3PROXY_BINARY is an empty string # PUBLIC unset: use s3proxy.conf # PUBLIC=1: use s3proxy-noauth.conf (no request signing) # function start_s3proxy { if [ -n "${PUBLIC}" ]; then S3PROXY_CONFIG="s3proxy-noauth.conf" else S3PROXY_CONFIG="s3proxy.conf" fi if [ -n "${S3PROXY_BINARY}" ] then if [ ! -e "${S3PROXY_BINARY}" ]; then wget "https://github.com/andrewgaul/s3proxy/releases/download/s3proxy-${S3PROXY_VERSION}/s3proxy" \ --quiet -O "${S3PROXY_BINARY}" chmod +x "${S3PROXY_BINARY}" fi ${STDBUF_BIN} -oL -eL java -jar "$S3PROXY_BINARY" --properties $S3PROXY_CONFIG & S3PROXY_PID=$! # wait for S3Proxy to start wait_for_port 8080 fi if [ -n "${CHAOS_HTTP_PROXY}" ]; then if [ ! -e "${CHAOS_HTTP_PROXY_BINARY}" ]; then wget "https://github.com/bouncestorage/chaos-http-proxy/releases/download/chaos-http-proxy-${CHAOS_HTTP_PROXY_VERSION}/chaos-http-proxy" \ --quiet -O "${CHAOS_HTTP_PROXY_BINARY}" chmod +x "${CHAOS_HTTP_PROXY_BINARY}" fi ${STDBUF_BIN} -oL -eL java -jar ${CHAOS_HTTP_PROXY_BINARY} --properties chaos-http-proxy.conf & CHAOS_HTTP_PROXY_PID=$! # wait for Chaos HTTP Proxy to start wait_for_port 1080 fi } function stop_s3proxy { if [ -n "${S3PROXY_PID}" ] then kill $S3PROXY_PID fi if [ -n "${CHAOS_HTTP_PROXY_PID}" ] then kill $CHAOS_HTTP_PROXY_PID fi } # Mount the bucket, function arguments passed to s3fs in addition to # a set of common arguments. function start_s3fs { # Public bucket if PUBLIC is set if [ -n "${PUBLIC}" ]; then AUTH_OPT="-o public_bucket=1" elif [ -n "${S3FS_PROFILE}" ]; then AUTH_OPT="-o profile=${S3FS_PROFILE}" else AUTH_OPT="-o passwd_file=${S3FS_CREDENTIALS_FILE}" fi # If VALGRIND is set, pass it as options to valgrind. # start valgrind-listener in another shell. # eg: VALGRIND="--tool=memcheck --leak-check=full" ./small-integration-test.sh # Start valgrind-listener (default port is 1500) if [ -n "${VALGRIND}" ]; then VALGRIND_EXEC="valgrind ${VALGRIND} --log-socket=127.0.1.1" fi # On OSX only, we need to specify the direct_io and auto_cache flag. if [ `uname` = "Darwin" ]; then DIRECT_IO_OPT="-o direct_io -o auto_cache" else DIRECT_IO_OPT="" fi if [ -n "${CHAOS_HTTP_PROXY}" ]; then export http_proxy="127.0.0.1:1080" fi # [NOTE] # On macos, running s3fs via stdbuf will result in no response. # Therefore, when it is macos, it is not executed via stdbuf. # This patch may be temporary, but no other method has been found at this time. # if [ `uname` = "Darwin" ]; then VIA_STDBUF_CMDLINE="" else VIA_STDBUF_CMDLINE="${STDBUF_BIN} -oL -eL" fi # Common s3fs options: # # TODO: Allow all these options to be overridden with env variables # # use_path_request_style # The test env doesn't have virtual hosts # createbucket # S3Proxy always starts with no buckets, this tests the s3fs-fuse # automatic bucket creation path. # $AUTH_OPT # Will be either "-o public_bucket=1" # or # "-o passwd_file=${S3FS_CREDENTIALS_FILE}" # dbglevel # error by default. override with DBGLEVEL env variable # -f # Keep s3fs in foreground instead of daemonizing # # subshell with set -x to log exact invocation of s3fs-fuse ( set -x ${VIA_STDBUF_CMDLINE} \ ${VALGRIND_EXEC} ${S3FS} \ $TEST_BUCKET_1 \ $TEST_BUCKET_MOUNT_POINT_1 \ -o use_path_request_style \ -o url=${S3_URL} \ -o endpoint=${S3_ENDPOINT} \ -o no_check_certificate \ -o ssl_verify_hostname=0 \ -o use_xattr=1 \ -o createbucket \ ${AUTH_OPT} \ ${DIRECT_IO_OPT} \ -o stat_cache_expire=1 \ -o stat_cache_interval_expire=1 \ -o dbglevel=${DBGLEVEL:=info} \ -o retries=3 \ -f \ "${@}" & echo $! >&3 ) 3>pid | ${STDBUF_BIN} -oL -eL ${SED_BIN} ${SED_BUFFER_FLAG} "s/^/s3fs: /" & sleep 1 export S3FS_PID=$(