octoleo/s3fs-fuse
1
0
Fork 0
mirror of https://github.com/s3fs-fuse/s3fs-fuse.git synced 2024-12-22 08:48:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add configuration for Chaos HTTP Proxy (#1508)

Browse Source 
This can find errors in retry logic.  Chaos HTTP Proxy does not
support SSL bouncestorage/chaos-http-proxy#1 so users must set
s3proxy.endpoint and run via:

CHAOS_HTTP_PROXY=1 S3_URL=http://127.0.0.1:8080 make check -C test

It can also be helpful to increase retries and reduce sleep times.
References #1504.
This commit is contained in:
Andrew Gaul 2021-01-04 23:32:04 +09:00 committed by GitHub
parent 168e588ac7
commit b04bca37a5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 43 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -75,6 +75,7 @@ default_commit_hash
src/s3fs src/s3fs
src/test_curl_util src/test_curl_util
src/test_string_util src/test_string_util
test/chaos-http-proxy-*
test/s3proxy-* test/s3proxy-*
# #

2
test/chaos-http-proxy.conf Normal file
View File

@ -0,0 +1,2 @@
com.bouncestorage.chaoshttpproxy.http_503=1
com.bouncestorage.chaoshttpproxy.success=9

37
test/integration-test-common.sh
View File

@ -78,6 +78,9 @@ export TEST_BUCKET_MOUNT_POINT_1=${TEST_BUCKET_1}
S3PROXY_VERSION="1.7.1" S3PROXY_VERSION="1.7.1"
S3PROXY_BINARY=${S3PROXY_BINARY-"s3proxy-${S3PROXY_VERSION}"} S3PROXY_BINARY=${S3PROXY_BINARY-"s3proxy-${S3PROXY_VERSION}"}
CHAOS_HTTP_PROXY_VERSION="1.1.0"
CHAOS_HTTP_PROXY_BINARY="chaos-http-proxy-${CHAOS_HTTP_PROXY_VERSION}"
if [ ! -f "$S3FS_CREDENTIALS_FILE" ] if [ ! -f "$S3FS_CREDENTIALS_FILE" ]
then then
echo "Missing credentials file: $S3FS_CREDENTIALS_FILE" echo "Missing credentials file: $S3FS_CREDENTIALS_FILE"
@ -142,16 +145,21 @@ function start_s3proxy {
S3PROXY_PID=$! S3PROXY_PID=$!
# wait for S3Proxy to start # wait for S3Proxy to start
for i in $(seq 30); wait_for_port 8080
do fi
if exec 3<>"/dev/tcp/127.0.0.1/8080";
then if [ -n "${CHAOS_HTTP_PROXY}" ]; then
exec 3<&- # Close for read if [ ! -e "${CHAOS_HTTP_PROXY_BINARY}" ]; then
exec 3>&- # Close for write wget "https://github.com/bouncestorage/chaos-http-proxy/releases/download/chaos-http-proxy-${CHAOS_HTTP_PROXY_VERSION}/chaos-http-proxy" \
break --quiet -O "${CHAOS_HTTP_PROXY_BINARY}"
fi chmod +x "${CHAOS_HTTP_PROXY_BINARY}"
sleep 1 fi
done
${STDBUF_BIN} -oL -eL java -jar ${CHAOS_HTTP_PROXY_BINARY} --properties chaos-http-proxy.conf &
CHAOS_HTTP_PROXY_PID=$!
# wait for Chaos HTTP Proxy to start
wait_for_port 1080
fi fi
} }
@ -160,6 +168,11 @@ function stop_s3proxy {
then then
kill $S3PROXY_PID kill $S3PROXY_PID
fi fi
if [ -n "${CHAOS_HTTP_PROXY_PID}" ]
then
kill $CHAOS_HTTP_PROXY_PID
fi
} }
# Mount the bucket, function arguments passed to s3fs in addition to # Mount the bucket, function arguments passed to s3fs in addition to
@ -189,6 +202,10 @@ function start_s3fs {
DIRECT_IO_OPT="" DIRECT_IO_OPT=""
fi fi
if [ -n "${CHAOS_HTTP_PROXY}" ]; then
export http_proxy="127.0.0.1:1080"
fi
# [NOTE] # [NOTE]
# On macos, running s3fs via stdbuf will result in no response. # On macos, running s3fs via stdbuf will result in no response.
# Therefore, when it is macos, it is not executed via stdbuf. # Therefore, when it is macos, it is not executed via stdbuf.

13
test/test-utils.sh
View File

@ -319,6 +319,19 @@ function aws_cli() {
aws $* --endpoint-url "${S3_URL}" --no-verify-ssl $FLAGS aws $* --endpoint-url "${S3_URL}" --no-verify-ssl $FLAGS
} }
function wait_for_port() {
PORT=$1
for i in $(seq 30); do
if exec 3<>"/dev/tcp/127.0.0.1/${PORT}";
then
exec 3<&- # Close for read
exec 3>&- # Close for write
break
fi
sleep 1
done
}
# #
# Local variables: # Local variables:
# tab-width: 4 # tab-width: 4