Merge pull request #263 from RobbKistler/aws

Allow integration testing against Amazon S3

test/integration-test-common.sh

@@ -1,10 +1,10 @@
 #!/bin/bash -e
-
+set -x
 S3FS=../src/s3fs
 
-S3FS_CREDENTIALS_FILE="passwd-s3fs"
+: ${S3FS_CREDENTIALS_FILE:="passwd-s3fs"}
 
-TEST_BUCKET_1="s3fs-integration-test"
+: ${TEST_BUCKET_1:="s3fs-integration-test"}
 TEST_BUCKET_MOUNT_POINT_1=${TEST_BUCKET_1}
 
 if [ ! -f "$S3FS_CREDENTIALS_FILE" ]
@@ -15,8 +15,8 @@ fi
 chmod 600 "$S3FS_CREDENTIALS_FILE"
 
 S3PROXY_VERSION="1.4.0"
-S3PROXY_BINARY="s3proxy-${S3PROXY_VERSION}"
-if [ ! -e "${S3PROXY_BINARY}" ]; then
+S3PROXY_BINARY=${S3PROXY_BINARY-"s3proxy-${S3PROXY_VERSION}"}
+if [ -n "${S3PROXY_BINARY}" ] && [ ! -e "${S3PROXY_BINARY}" ]; then
     wget "https://github.com/andrewgaul/s3proxy/releases/download/s3proxy-${S3PROXY_VERSION}/s3proxy" \
             --quiet -O "${S3PROXY_BINARY}"
     chmod +x "${S3PROXY_BINARY}"

test/small-integration-test.sh

@@ -1,8 +1,24 @@
 #!/bin/bash
 
+#
+# By default tests run against a local s3proxy instance.  To run against 
+# Amazon S3, specify the following variables:
+#
+# S3FS_CREDENTIALS_FILE=keyfile      s3fs format key file
+# TEST_BUCKET_1=bucket               Name of bucket to use 
+# S3PROXY_BINARY=""                  Leave empty 
+# S3_URL="http://s3.amazonaws.com"   Specify Amazon server
+#
+# Example: 
+#
+# S3FS_CREDENTIALS_FILE=keyfile TEST_BUCKET_1=bucket S3PROXY_BINARY="" S3_URL="http://s3.amazonaws.com" ./small-integration-test.sh
+#
+
 set -o xtrace
 set -o errexit
 
+: ${S3_URL:="http://127.0.0.1:8080"}
+
 # Require root
 REQUIRE_ROOT=require-root.sh
 #source $REQUIRE_ROOT
@@ -29,26 +45,32 @@ function retry {
 }
 
 function exit_handler {
-    kill $S3PROXY_PID
+    if [ -n "${S3PROXY_PID}" ]
+    then
+        kill $S3PROXY_PID
+    fi
     retry 30 fusermount -u $TEST_BUCKET_MOUNT_POINT_1
 }
 trap exit_handler EXIT
 
-stdbuf -oL -eL java -jar "$S3PROXY_BINARY" --properties s3proxy.conf | stdbuf -oL -eL sed -u "s/^/s3proxy: /" &
+if [ -n "${S3PROXY_BINARY}" ]
+then
+    stdbuf -oL -eL java -jar "$S3PROXY_BINARY" --properties s3proxy.conf | stdbuf -oL -eL sed -u "s/^/s3proxy: /" &
 
-# wait for S3Proxy to start
-for i in $(seq 30);
-do
-    if exec 3<>"/dev/tcp/localhost/8080";
-    then
-        exec 3<&-  # Close for read
-        exec 3>&-  # Close for write
-        break
-    fi
-    sleep 1
-done
+    # wait for S3Proxy to start
+    for i in $(seq 30);
+    do
+        if exec 3<>"/dev/tcp/localhost/8080";
+        then
+            exec 3<&-  # Close for read
+            exec 3>&-  # Close for write
+            break
+        fi
+        sleep 1
+    done
 
-S3PROXY_PID=$(netstat -lpnt | grep :8080 | awk '{ print $7 }' | sed -u 's|/java||')
+    S3PROXY_PID=$(netstat -lpnt | grep :8080 | awk '{ print $7 }' | sed -u 's|/java||')
+fi
 
 # Mount the bucket
 if [ ! -d $TEST_BUCKET_MOUNT_POINT_1 ]
@@ -61,7 +83,7 @@ stdbuf -oL -eL $S3FS $TEST_BUCKET_1 $TEST_BUCKET_MOUNT_POINT_1 \
     -o passwd_file=$S3FS_CREDENTIALS_FILE \
     -o sigv2 \
     -o singlepart_copy_limit=$((10 * 1024)) \
-    -o url=http://127.0.0.1:8080 \
+    -o url=${S3_URL} \
     -o use_path_request_style -f -o f2 -d -d |& stdbuf -oL -eL sed -u "s/^/s3fs: /" &
 
 retry 30 grep $TEST_BUCKET_MOUNT_POINT_1 /proc/mounts || exit 1