Add configuration for Chaos HTTP Proxy (#1508)

This can find errors in retry logic.  Chaos HTTP Proxy does not
support SSL bouncestorage/chaos-http-proxy#1 so users must set
s3proxy.endpoint and run via:

CHAOS_HTTP_PROXY=1 S3_URL=http://127.0.0.1:8080 make check -C test

It can also be helpful to increase retries and reduce sleep times.
References #1504.

.gitignore

@@ -75,6 +75,7 @@ default_commit_hash
 src/s3fs
 src/test_curl_util
 src/test_string_util
+test/chaos-http-proxy-*
 test/s3proxy-*
 
 #

test/chaos-http-proxy.conf

@@ -0,0 +1,2 @@
+com.bouncestorage.chaoshttpproxy.http_503=1
+com.bouncestorage.chaoshttpproxy.success=9

test/integration-test-common.sh

@@ -78,6 +78,9 @@ export TEST_BUCKET_MOUNT_POINT_1=${TEST_BUCKET_1}
 S3PROXY_VERSION="1.7.1"
 S3PROXY_BINARY=${S3PROXY_BINARY-"s3proxy-${S3PROXY_VERSION}"}
 
+CHAOS_HTTP_PROXY_VERSION="1.1.0"
+CHAOS_HTTP_PROXY_BINARY="chaos-http-proxy-${CHAOS_HTTP_PROXY_VERSION}"
+
 if [ ! -f "$S3FS_CREDENTIALS_FILE" ]
 then
 	echo "Missing credentials file: $S3FS_CREDENTIALS_FILE"
@@ -142,16 +145,21 @@ function start_s3proxy {
         S3PROXY_PID=$!
 
         # wait for S3Proxy to start
-        for i in $(seq 30);
-        do
-            if exec 3<>"/dev/tcp/127.0.0.1/8080";
-            then
-                exec 3<&-  # Close for read
-                exec 3>&-  # Close for write
-                break
-            fi
-            sleep 1
-        done
+        wait_for_port 8080
+    fi
+
+    if [ -n "${CHAOS_HTTP_PROXY}" ]; then
+        if [ ! -e "${CHAOS_HTTP_PROXY_BINARY}" ]; then
+            wget "https://github.com/bouncestorage/chaos-http-proxy/releases/download/chaos-http-proxy-${CHAOS_HTTP_PROXY_VERSION}/chaos-http-proxy" \
+                --quiet -O "${CHAOS_HTTP_PROXY_BINARY}"
+            chmod +x "${CHAOS_HTTP_PROXY_BINARY}"
+        fi
+
+        ${STDBUF_BIN} -oL -eL java -jar ${CHAOS_HTTP_PROXY_BINARY} --properties chaos-http-proxy.conf &
+        CHAOS_HTTP_PROXY_PID=$!
+
+        # wait for Chaos HTTP Proxy to start
+        wait_for_port 1080
     fi
 }
 
@@ -160,6 +168,11 @@ function stop_s3proxy {
     then
         kill $S3PROXY_PID
     fi
+
+    if [ -n "${CHAOS_HTTP_PROXY_PID}" ]
+    then
+        kill $CHAOS_HTTP_PROXY_PID
+    fi
 }
 
 # Mount the bucket, function arguments passed to s3fs in addition to
@@ -189,6 +202,10 @@ function start_s3fs {
        DIRECT_IO_OPT=""
     fi
 
+    if [ -n "${CHAOS_HTTP_PROXY}" ]; then
+        export http_proxy="127.0.0.1:1080"
+    fi
+
     # [NOTE]
     # On macos, running s3fs via stdbuf will result in no response.
     # Therefore, when it is macos, it is not executed via stdbuf.

test/test-utils.sh

@@ -319,6 +319,19 @@ function aws_cli() {
     aws $* --endpoint-url "${S3_URL}" --no-verify-ssl $FLAGS
 }
 
+function wait_for_port() {
+    PORT=$1
+    for i in $(seq 30); do
+        if exec 3<>"/dev/tcp/127.0.0.1/${PORT}";
+        then
+            exec 3<&-  # Close for read
+            exec 3>&-  # Close for write
+            break
+        fi
+        sleep 1
+    done
+}
+
 #
 # Local variables:
 # tab-width: 4