mirror of
https://github.com/s3fs-fuse/s3fs-fuse.git
synced 2024-11-04 20:07:54 +00:00
Merge pull request #348 from RobbKistler/pushtests
Integration test summary, continue on error
This commit is contained in:
Takeshi Nakatani
commit
f548e8ad5e
@ -1,11 +1,57 @@
|
||||
#!/bin/bash -e
|
||||
set -x
|
||||
#!/bin/bash
|
||||
|
||||
#
|
||||
# Common code for starting an s3fs-fuse mountpoint and an S3Proxy instance
|
||||
# to run tests against S3Proxy locally.
|
||||
#
|
||||
# To run against an Amazon S3 or other S3 provider, specify the following
|
||||
# environment variables:
|
||||
#
|
||||
# S3FS_CREDENTIALS_FILE=keyfile s3fs format key file
|
||||
# TEST_BUCKET_1=bucketname Name of bucket to use
|
||||
# S3PROXY_BINARY="" Specify empty string to skip S3Proxy start
|
||||
# S3_URL="http://s3.amazonaws.com" Specify Amazon AWS as the S3 provider
|
||||
#
|
||||
# Example of running against Amazon S3 using a bucket named "bucket:
|
||||
#
|
||||
# S3FS_CREDENTIALS_FILE=keyfile TEST_BUCKET_1=bucket S3PROXY_BINARY="" S3_URL="http://s3.amazonaws.com" ./small-integration-test.sh
|
||||
#
|
||||
# To change the s3fs-fuse debug level:
|
||||
#
|
||||
# DBGLEVEL=debug ./small-integration-test.sh
|
||||
#
|
||||
# To stop and wait after the mount point is up for manual interaction. This allows you to
|
||||
# explore the mounted file system exactly as it would have been started for the test case
|
||||
#
|
||||
# INTERACT=1 DBGLEVEL=debug ./small-integration-test.sh
|
||||
#
|
||||
# Run all of the tests from the makefile
|
||||
#
|
||||
# S3FS_CREDENTIALS_FILE=keyfile TEST_BUCKET_1=bucket S3PROXY_BINARY="" S3_URL="http://s3.amazonaws.com" make check
|
||||
#
|
||||
# Run the tests with request auth turned off in both S3Proxy and s3fs-fuse. This can be
|
||||
# useful for poking around with plain old curl
|
||||
#
|
||||
# PUBLIC=1 INTERACT=1 ./small-integration-test.sh
|
||||
#
|
||||
# A valgrind tool can be specified
|
||||
# eg: VALGRIND="--tool=memcheck --leak-check=full" ./small-integration-test.sh
|
||||
|
||||
set -o errexit
|
||||
S3FS=../src/s3fs
|
||||
|
||||
# Allow these defaulted values to be overridden
|
||||
: ${S3_URL:="http://127.0.0.1:8080"}
|
||||
: ${S3FS_CREDENTIALS_FILE:="passwd-s3fs"}
|
||||
|
||||
: ${TEST_BUCKET_1:="s3fs-integration-test"}
|
||||
TEST_BUCKET_MOUNT_POINT_1=${TEST_BUCKET_1}
|
||||
|
||||
export TEST_BUCKET_1
|
||||
export S3_URL
|
||||
export TEST_SCRIPT_DIR=`pwd`
|
||||
export TEST_BUCKET_MOUNT_POINT_1=${TEST_BUCKET_1}
|
||||
|
||||
S3PROXY_VERSION="1.4.0"
|
||||
S3PROXY_BINARY=${S3PROXY_BINARY-"s3proxy-${S3PROXY_VERSION}"}
|
||||
|
||||
if [ ! -f "$S3FS_CREDENTIALS_FILE" ]
|
||||
then
|
||||
@ -14,10 +60,159 @@ then
|
||||
fi
|
||||
chmod 600 "$S3FS_CREDENTIALS_FILE"
|
||||
|
||||
S3PROXY_VERSION="1.4.0"
|
||||
S3PROXY_BINARY=${S3PROXY_BINARY-"s3proxy-${S3PROXY_VERSION}"}
|
||||
if [ -n "${S3PROXY_BINARY}" ] && [ ! -e "${S3PROXY_BINARY}" ]; then
|
||||
wget "https://github.com/andrewgaul/s3proxy/releases/download/s3proxy-${S3PROXY_VERSION}/s3proxy" \
|
||||
--quiet -O "${S3PROXY_BINARY}"
|
||||
chmod +x "${S3PROXY_BINARY}"
|
||||
if [ ! -d $TEST_BUCKET_MOUNT_POINT_1 ]
|
||||
then
|
||||
mkdir -p $TEST_BUCKET_MOUNT_POINT_1
|
||||
fi
|
||||
|
||||
# This function execute the function parameters $1 times
|
||||
# before giving up, with 1 second delays.
|
||||
function retry {
|
||||
set +o errexit
|
||||
N=$1; shift;
|
||||
status=0
|
||||
for i in $(seq $N); do
|
||||
echo "Trying: $@"
|
||||
$@
|
||||
status=$?
|
||||
if [ $status == 0 ]; then
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
echo "Retrying: $@"
|
||||
done
|
||||
|
||||
if [ $status != 0 ]; then
|
||||
echo "timeout waiting for $@"
|
||||
fi
|
||||
set -o errexit
|
||||
return $status
|
||||
}
|
||||
|
||||
# Proxy is not started if S3PROXY_BINARY is an empty string
|
||||
# PUBLIC unset: use s3proxy.conf
|
||||
# PUBLIC=1: use s3proxy-noauth.conf (no request signing)
|
||||
#
|
||||
function start_s3proxy {
|
||||
if [ -n "${PUBLIC}" ]; then
|
||||
S3PROXY_CONFIG="s3proxy-noauth.conf"
|
||||
else
|
||||
S3PROXY_CONFIG="s3proxy.conf"
|
||||
fi
|
||||
|
||||
if [ -n "${S3PROXY_BINARY}" ]
|
||||
then
|
||||
if [ ! -e "${S3PROXY_BINARY}" ]; then
|
||||
wget "https://github.com/andrewgaul/s3proxy/releases/download/s3proxy-${S3PROXY_VERSION}/s3proxy" \
|
||||
--quiet -O "${S3PROXY_BINARY}"
|
||||
chmod +x "${S3PROXY_BINARY}"
|
||||
fi
|
||||
|
||||
stdbuf -oL -eL java -jar "$S3PROXY_BINARY" --properties $S3PROXY_CONFIG | stdbuf -oL -eL sed -u "s/^/s3proxy: /" &
|
||||
|
||||
# wait for S3Proxy to start
|
||||
for i in $(seq 30);
|
||||
do
|
||||
if exec 3<>"/dev/tcp/127.0.0.1/8080";
|
||||
then
|
||||
exec 3<&- # Close for read
|
||||
exec 3>&- # Close for write
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
S3PROXY_PID=$(netstat -lpnt | grep :8080 | awk '{ print $7 }' | sed -u 's|/java||')
|
||||
fi
|
||||
}
|
||||
|
||||
function stop_s3proxy {
|
||||
if [ -n "${S3PROXY_PID}" ]
|
||||
then
|
||||
kill $S3PROXY_PID
|
||||
wait $S3PROXY_PID
|
||||
fi
|
||||
}
|
||||
|
||||
# Mount the bucket, function arguments passed to s3fs in addition to
|
||||
# a set of common arguments.
|
||||
function start_s3fs {
|
||||
|
||||
# Public bucket if PUBLIC is set
|
||||
if [ -n "${PUBLIC}" ]; then
|
||||
AUTH_OPT="-o public_bucket=1"
|
||||
else
|
||||
AUTH_OPT="-o passwd_file=${S3FS_CREDENTIALS_FILE}"
|
||||
fi
|
||||
|
||||
# If VALGRIND is set, pass it as options to valgrind.
|
||||
# start valgrind-listener in another shell.
|
||||
# eg: VALGRIND="--tool=memcheck --leak-check=full" ./small-integration-test.sh
|
||||
# Start valgind-listener (default port is 1500)
|
||||
if [ -n "${VALGRIND}" ]; then
|
||||
VALGRIND_EXEC="valgrind ${VALGRIND} --log-socket=127.0.1.1"
|
||||
fi
|
||||
|
||||
# Common s3fs options:
|
||||
#
|
||||
# TODO: Allow all these options to be overriden with env variables
|
||||
#
|
||||
# sigv2
|
||||
# Historically because S3Proxy only supports sigv2.
|
||||
# use_path_request_style
|
||||
# The test env doesn't have virtual hosts
|
||||
# createbucket
|
||||
# S3Proxy always starts with no buckets, this tests the s3fs-fuse
|
||||
# automatic bucket creation path.
|
||||
# $AUTH_OPT
|
||||
# Will be either "-o public_bucket=1"
|
||||
# or
|
||||
# "-o passwd_file=${S3FS_CREDENTIALS_FILE}"
|
||||
# dbglevel
|
||||
# error by default. override with DBGLEVEL env variable
|
||||
# -f
|
||||
# Keep s3fs in foreground instead of daemonizing
|
||||
#
|
||||
|
||||
# subshell with set -x to log exact invocation of s3fs-fuse
|
||||
(
|
||||
set -x
|
||||
stdbuf -oL -eL \
|
||||
${VALGRIND_EXEC} ${S3FS} \
|
||||
$TEST_BUCKET_1 \
|
||||
$TEST_BUCKET_MOUNT_POINT_1 \
|
||||
-o sigv2 \
|
||||
-o use_path_request_style \
|
||||
-o url=${S3_URL} \
|
||||
-o createbucket \
|
||||
${AUTH_OPT} \
|
||||
-o dbglevel=${DBGLEVEL:=info} \
|
||||
-f \
|
||||
${@} \
|
||||
|& stdbuf -oL -eL sed -u "s/^/s3fs: /" &
|
||||
)
|
||||
|
||||
retry 5 grep -q $TEST_BUCKET_MOUNT_POINT_1 /proc/mounts || exit 1
|
||||
|
||||
# Quick way to start system up for manual testing with options under test
|
||||
if [[ -n ${INTERACT} ]]; then
|
||||
echo "Mountpoint $TEST_BUCKET_MOUNT_POINT_1 is ready"
|
||||
echo "control-C to quit"
|
||||
sleep infinity
|
||||
exit 0
|
||||
fi
|
||||
}
|
||||
|
||||
function stop_s3fs {
|
||||
# Retry in case file system is in use
|
||||
if grep -q $TEST_BUCKET_MOUNT_POINT_1 /proc/mounts; then
|
||||
retry 10 grep -q $TEST_BUCKET_MOUNT_POINT_1 /proc/mounts && fusermount -u $TEST_BUCKET_MOUNT_POINT_1
|
||||
fi
|
||||
}
|
||||
|
||||
# trap handlers do not stack. If a test sets its own, the new handler should call common_exit_handler
|
||||
function common_exit_handler {
|
||||
stop_s3proxy
|
||||
stop_s3fs
|
||||
}
|
||||
trap common_exit_handler EXIT
|
||||
|
||||
@ -1,68 +1,11 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -o xtrace
|
||||
set -o errexit
|
||||
|
||||
COMMON=integration-test-common.sh
|
||||
source $COMMON
|
||||
|
||||
# Configuration
|
||||
TEST_TEXT="HELLO WORLD"
|
||||
TEST_TEXT_FILE=test-s3fs.txt
|
||||
TEST_DIR=testdir
|
||||
ALT_TEST_TEXT_FILE=test-s3fs-ALT.txt
|
||||
TEST_TEXT_FILE_LENGTH=15
|
||||
BIG_FILE=big-file-s3fs.txt
|
||||
BIG_FILE_LENGTH=$((25 * 1024 * 1024))
|
||||
|
||||
function mk_test_file {
|
||||
if [ $# == 0 ]; then
|
||||
TEXT=$TEST_TEXT
|
||||
else
|
||||
TEXT=$1
|
||||
fi
|
||||
echo $TEXT > $TEST_TEXT_FILE
|
||||
if [ ! -e $TEST_TEXT_FILE ]
|
||||
then
|
||||
echo "Could not create file ${TEST_TEXT_FILE}, it does not exist"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
function rm_test_file {
|
||||
if [ $# == 0 ]; then
|
||||
FILE=$TEST_TEXT_FILE
|
||||
else
|
||||
FILE=$1
|
||||
fi
|
||||
rm -f $FILE
|
||||
|
||||
if [ -e $FILE ]
|
||||
then
|
||||
echo "Could not cleanup file ${TEST_TEXT_FILE}"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
function mk_test_dir {
|
||||
mkdir ${TEST_DIR}
|
||||
|
||||
if [ ! -d ${TEST_DIR} ]; then
|
||||
echo "Directory ${TEST_DIR} was not created"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
function rm_test_dir {
|
||||
rmdir ${TEST_DIR}
|
||||
if [ -e $TEST_DIR ]; then
|
||||
echo "Could not remove the test directory, it still exists: ${TEST_DIR}"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
source test-utils.sh
|
||||
|
||||
function test_append_file {
|
||||
echo "Testing append to file ..."
|
||||
describe "Testing append to file ..."
|
||||
# Write a small test file
|
||||
for x in `seq 1 $TEST_TEXT_FILE_LENGTH`
|
||||
do
|
||||
@ -75,33 +18,32 @@ function test_append_file {
|
||||
if [ "$FILE_LENGTH" -ne "$TEST_TEXT_FILE_LENGTH" ]
|
||||
then
|
||||
echo "error: expected $TEST_TEXT_FILE_LENGTH , got $FILE_LENGTH"
|
||||
exit 1
|
||||
return 1
|
||||
fi
|
||||
|
||||
rm_test_file
|
||||
}
|
||||
|
||||
function test_truncate_file {
|
||||
echo "Testing truncate file ..."
|
||||
describe "Testing truncate file ..."
|
||||
# Write a small test file
|
||||
echo "${TEST_TEXT}" > ${TEST_TEXT_FILE}
|
||||
|
||||
|
||||
# Truncate file to 0 length. This should trigger open(path, O_RDWR | O_TRUNC...)
|
||||
: > ${TEST_TEXT_FILE}
|
||||
|
||||
|
||||
# Verify file is zero length
|
||||
if [ -s ${TEST_TEXT_FILE} ]
|
||||
then
|
||||
echo "error: expected ${TEST_TEXT_FILE} to be zero length"
|
||||
exit 1
|
||||
return 1
|
||||
fi
|
||||
rm_test_file
|
||||
}
|
||||
|
||||
|
||||
function test_mv_file {
|
||||
echo "Testing mv file function ..."
|
||||
|
||||
describe "Testing mv file function ..."
|
||||
# if the rename file exists, delete it
|
||||
if [ -e $ALT_TEST_TEXT_FILE ]
|
||||
then
|
||||
@ -111,7 +53,7 @@ function test_mv_file {
|
||||
if [ -e $ALT_TEST_TEXT_FILE ]
|
||||
then
|
||||
echo "Could not delete file ${ALT_TEST_TEXT_FILE}, it still exists"
|
||||
exit 1
|
||||
return 1
|
||||
fi
|
||||
|
||||
# create the test file again
|
||||
@ -122,7 +64,7 @@ function test_mv_file {
|
||||
if [ ! -e $ALT_TEST_TEXT_FILE ]
|
||||
then
|
||||
echo "Could not move file"
|
||||
exit 1
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Check the contents of the alt file
|
||||
@ -131,7 +73,7 @@ function test_mv_file {
|
||||
if [ "$ALT_FILE_LENGTH" -ne "$ALT_TEXT_LENGTH" ]
|
||||
then
|
||||
echo "moved file length is not as expected expected: $ALT_TEXT_LENGTH got: $ALT_FILE_LENGTH"
|
||||
exit 1
|
||||
return 1
|
||||
fi
|
||||
|
||||
# clean up
|
||||
@ -139,10 +81,10 @@ function test_mv_file {
|
||||
}
|
||||
|
||||
function test_mv_directory {
|
||||
echo "Testing mv directory function ..."
|
||||
describe "Testing mv directory function ..."
|
||||
if [ -e $TEST_DIR ]; then
|
||||
echo "Unexpected, this file/directory exists: ${TEST_DIR}"
|
||||
exit 1
|
||||
return 1
|
||||
fi
|
||||
|
||||
mk_test_dir
|
||||
@ -151,26 +93,26 @@ function test_mv_directory {
|
||||
|
||||
if [ ! -d "${TEST_DIR}_rename" ]; then
|
||||
echo "Directory ${TEST_DIR} was not renamed"
|
||||
exit 1
|
||||
return 1
|
||||
fi
|
||||
|
||||
rmdir ${TEST_DIR}_rename
|
||||
if [ -e "${TEST_DIR}_rename" ]; then
|
||||
echo "Could not remove the test directory, it still exists: ${TEST_DIR}_rename"
|
||||
exit 1
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
function test_redirects {
|
||||
echo "Testing redirects ..."
|
||||
describe "Testing redirects ..."
|
||||
|
||||
mk_test_file ABCDEF
|
||||
|
||||
CONTENT=`cat $TEST_TEXT_FILE`
|
||||
|
||||
if [ ${CONTENT} != "ABCDEF" ]; then
|
||||
if [ "${CONTENT}" != "ABCDEF" ]; then
|
||||
echo "CONTENT read is unexpected, got ${CONTENT}, expected ABCDEF"
|
||||
exit 1
|
||||
return 1
|
||||
fi
|
||||
|
||||
echo XYZ > $TEST_TEXT_FILE
|
||||
@ -179,7 +121,7 @@ function test_redirects {
|
||||
|
||||
if [ ${CONTENT} != "XYZ" ]; then
|
||||
echo "CONTENT read is unexpected, got ${CONTENT}, expected XYZ"
|
||||
exit 1
|
||||
return 1
|
||||
fi
|
||||
|
||||
echo 123456 >> $TEST_TEXT_FILE
|
||||
@ -189,12 +131,12 @@ function test_redirects {
|
||||
|
||||
if [ ${LINE1} != "XYZ" ]; then
|
||||
echo "LINE1 was not as expected, got ${LINE1}, expected XYZ"
|
||||
exit 1
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [ ${LINE2} != "123456" ]; then
|
||||
echo "LINE2 was not as expected, got ${LINE2}, expected 123456"
|
||||
exit 1
|
||||
return 1
|
||||
fi
|
||||
|
||||
# clean up
|
||||
@ -202,11 +144,11 @@ function test_redirects {
|
||||
}
|
||||
|
||||
function test_mkdir_rmdir {
|
||||
echo "Testing creation/removal of a directory"
|
||||
describe "Testing creation/removal of a directory"
|
||||
|
||||
if [ -e $TEST_DIR ]; then
|
||||
echo "Unexpected, this file/directory exists: ${TEST_DIR}"
|
||||
exit 1
|
||||
return 1
|
||||
fi
|
||||
|
||||
mk_test_dir
|
||||
@ -214,7 +156,7 @@ function test_mkdir_rmdir {
|
||||
}
|
||||
|
||||
function test_chmod {
|
||||
echo "Testing chmod file function ..."
|
||||
describe "Testing chmod file function ..."
|
||||
|
||||
# create the test file again
|
||||
mk_test_file
|
||||
@ -227,7 +169,7 @@ function test_chmod {
|
||||
if [ $(stat --format=%a $TEST_TEXT_FILE) == $ORIGINAL_PERMISSIONS ]
|
||||
then
|
||||
echo "Could not modify $TEST_TEXT_FILE permissions"
|
||||
exit 1
|
||||
return 1
|
||||
fi
|
||||
|
||||
# clean up
|
||||
@ -235,7 +177,7 @@ function test_chmod {
|
||||
}
|
||||
|
||||
function test_chown {
|
||||
echo "Testing chown file function ..."
|
||||
describe "Testing chown file function ..."
|
||||
|
||||
# create the test file again
|
||||
mk_test_file
|
||||
@ -248,7 +190,7 @@ function test_chown {
|
||||
if [ $(stat --format=%u:%g $TEST_TEXT_FILE) == $ORIGINAL_PERMISSIONS ]
|
||||
then
|
||||
echo "Could not modify $TEST_TEXT_FILE ownership"
|
||||
exit 1
|
||||
return 1
|
||||
fi
|
||||
|
||||
# clean up
|
||||
@ -256,14 +198,14 @@ function test_chown {
|
||||
}
|
||||
|
||||
function test_list {
|
||||
echo "Testing list"
|
||||
describe "Testing list"
|
||||
mk_test_file
|
||||
mk_test_dir
|
||||
|
||||
file_cnt=$(ls -1 | wc -l)
|
||||
if [ $file_cnt != 2 ]; then
|
||||
echo "Expected 2 file but got $file_cnt"
|
||||
exit 1
|
||||
return 1
|
||||
fi
|
||||
|
||||
rm_test_file
|
||||
@ -271,7 +213,7 @@ function test_list {
|
||||
}
|
||||
|
||||
function test_remove_nonempty_directory {
|
||||
echo "Testing removing a non-empty directory"
|
||||
describe "Testing removing a non-empty directory"
|
||||
mk_test_dir
|
||||
touch "${TEST_DIR}/file"
|
||||
rmdir "${TEST_DIR}" 2>&1 | grep -q "Directory not empty"
|
||||
@ -280,7 +222,7 @@ function test_remove_nonempty_directory {
|
||||
}
|
||||
|
||||
function test_rename_before_close {
|
||||
echo "Testing rename before close ..."
|
||||
describe "Testing rename before close ..."
|
||||
(
|
||||
echo foo
|
||||
mv $TEST_TEXT_FILE ${TEST_TEXT_FILE}.new
|
||||
@ -288,7 +230,7 @@ function test_rename_before_close {
|
||||
|
||||
if ! cmp <(echo foo) ${TEST_TEXT_FILE}.new; then
|
||||
echo "rename before close failed"
|
||||
exit 1
|
||||
return 1
|
||||
fi
|
||||
|
||||
rm_test_file ${TEST_TEXT_FILE}.new
|
||||
@ -296,7 +238,7 @@ function test_rename_before_close {
|
||||
}
|
||||
|
||||
function test_multipart_upload {
|
||||
echo "Testing multi-part upload ..."
|
||||
describe "Testing multi-part upload ..."
|
||||
dd if=/dev/urandom of="/tmp/${BIG_FILE}" bs=$BIG_FILE_LENGTH count=1
|
||||
dd if="/tmp/${BIG_FILE}" of="${BIG_FILE}" bs=$BIG_FILE_LENGTH count=1
|
||||
|
||||
@ -304,7 +246,7 @@ function test_multipart_upload {
|
||||
echo "Comparing test file"
|
||||
if ! cmp "/tmp/${BIG_FILE}" "${BIG_FILE}"
|
||||
then
|
||||
exit 1
|
||||
return 1
|
||||
fi
|
||||
|
||||
rm -f "/tmp/${BIG_FILE}"
|
||||
@ -312,7 +254,7 @@ function test_multipart_upload {
|
||||
}
|
||||
|
||||
function test_multipart_copy {
|
||||
echo "Testing multi-part copy ..."
|
||||
describe "Testing multi-part copy ..."
|
||||
dd if=/dev/urandom of="/tmp/${BIG_FILE}" bs=$BIG_FILE_LENGTH count=1
|
||||
dd if="/tmp/${BIG_FILE}" of="${BIG_FILE}" bs=$BIG_FILE_LENGTH count=1
|
||||
mv "${BIG_FILE}" "${BIG_FILE}-copy"
|
||||
@ -321,7 +263,7 @@ function test_multipart_copy {
|
||||
echo "Comparing test file"
|
||||
if ! cmp "/tmp/${BIG_FILE}" "${BIG_FILE}-copy"
|
||||
then
|
||||
exit 1
|
||||
return 1
|
||||
fi
|
||||
|
||||
rm -f "/tmp/${BIG_FILE}"
|
||||
@ -329,7 +271,7 @@ function test_multipart_copy {
|
||||
}
|
||||
|
||||
function test_special_characters {
|
||||
echo "Testing special characters ..."
|
||||
describe "Testing special characters ..."
|
||||
|
||||
ls 'special' 2>&1 | grep -q 'No such file or directory'
|
||||
ls 'special?' 2>&1 | grep -q 'No such file or directory'
|
||||
@ -339,7 +281,7 @@ function test_special_characters {
|
||||
}
|
||||
|
||||
function test_symlink {
|
||||
echo "Testing symlinks ..."
|
||||
describe "Testing symlinks ..."
|
||||
|
||||
rm -f $TEST_TEXT_FILE
|
||||
rm -f $ALT_TEST_TEXT_FILE
|
||||
@ -358,7 +300,7 @@ function test_extended_attributes {
|
||||
command -v setfattr >/dev/null 2>&1 || \
|
||||
{ echo "Skipping extended attribute tests" ; return; }
|
||||
|
||||
echo "Testing extended attributes ..."
|
||||
describe "Testing extended attributes ..."
|
||||
|
||||
rm -f $TEST_TEXT_FILE
|
||||
touch $TEST_TEXT_FILE
|
||||
@ -379,7 +321,7 @@ function test_extended_attributes {
|
||||
}
|
||||
|
||||
function test_mtime_file {
|
||||
echo "Testing mtime preservation function ..."
|
||||
describe "Testing mtime preservation function ..."
|
||||
|
||||
# if the rename file exists, delete it
|
||||
if [ -e $ALT_TEST_TEXT_FILE -o -L $ALT_TEST_TEXT_FILE ]
|
||||
@ -390,7 +332,7 @@ function test_mtime_file {
|
||||
if [ -e $ALT_TEST_TEXT_FILE ]
|
||||
then
|
||||
echo "Could not delete file ${ALT_TEST_TEXT_FILE}, it still exists"
|
||||
exit 1
|
||||
return 1
|
||||
fi
|
||||
|
||||
# create the test file again
|
||||
@ -404,48 +346,49 @@ function test_mtime_file {
|
||||
if [ "$testmtime" -ne "$altmtime" ]
|
||||
then
|
||||
echo "File times do not match: $testmtime != $altmtime"
|
||||
exit 1
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
function run_all_tests {
|
||||
test_append_file
|
||||
test_truncate_file
|
||||
test_mv_file
|
||||
test_mv_directory
|
||||
test_redirects
|
||||
test_mkdir_rmdir
|
||||
test_chmod
|
||||
test_chown
|
||||
test_list
|
||||
test_remove_nonempty_directory
|
||||
# TODO: broken: https://github.com/s3fs-fuse/s3fs-fuse/issues/145
|
||||
#test_rename_before_close
|
||||
test_multipart_upload
|
||||
# TODO: test disabled until S3Proxy 1.5.0 is released
|
||||
#test_multipart_copy
|
||||
test_special_characters
|
||||
test_symlink
|
||||
test_extended_attributes
|
||||
test_mtime_file
|
||||
function test_rm_rf_dir {
|
||||
describe "Test that rm -rf will remove directory with contents"
|
||||
# Create a dir with some files and directories
|
||||
mkdir dir1
|
||||
mkdir dir1/dir2
|
||||
touch dir1/file1
|
||||
touch dir1/dir2/file2
|
||||
|
||||
# Remove the dir with recursive rm
|
||||
rm -rf dir1
|
||||
|
||||
if [ -e dir1 ]; then
|
||||
echo "rm -rf did not remove $PWD/dir1"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Mount the bucket
|
||||
CUR_DIR=`pwd`
|
||||
TEST_BUCKET_MOUNT_POINT_1=$1
|
||||
if [ "$TEST_BUCKET_MOUNT_POINT_1" == "" ]; then
|
||||
echo "Mountpoint missing"
|
||||
exit 1
|
||||
fi
|
||||
cd $TEST_BUCKET_MOUNT_POINT_1
|
||||
function add_all_tests {
|
||||
add_tests test_append_file
|
||||
add_tests test_truncate_file
|
||||
add_tests test_mv_file
|
||||
add_tests test_mv_directory
|
||||
add_tests test_redirects
|
||||
add_tests test_mkdir_rmdir
|
||||
add_tests test_chmod
|
||||
add_tests test_chown
|
||||
add_tests test_list
|
||||
add_tests test_remove_nonempty_directory
|
||||
# TODO: broken: https://github.com/s3fs-fuse/s3fs-fuse/issues/145
|
||||
#add_tests test_rename_before_close
|
||||
add_tests test_multipart_upload
|
||||
# TODO: test disabled until S3Proxy 1.5.0 is released
|
||||
#add_tests test_multipart_copy
|
||||
add_tests test_special_characters
|
||||
add_tests test_symlink
|
||||
add_tests test_extended_attributes
|
||||
add_tests test_rm_rf_dir
|
||||
}
|
||||
|
||||
if [ -e $TEST_TEXT_FILE ]
|
||||
then
|
||||
rm -f $TEST_TEXT_FILE
|
||||
fi
|
||||
|
||||
run_all_tests
|
||||
|
||||
# Unmount the bucket
|
||||
cd $CUR_DIR
|
||||
echo "All tests complete."
|
||||
init_suite
|
||||
add_all_tests
|
||||
run_suite
|
||||
|
||||
@ -1,94 +1,30 @@
|
||||
#!/bin/bash
|
||||
|
||||
#
|
||||
# By default tests run against a local s3proxy instance. To run against
|
||||
# Amazon S3, specify the following variables:
|
||||
#
|
||||
# S3FS_CREDENTIALS_FILE=keyfile s3fs format key file
|
||||
# TEST_BUCKET_1=bucket Name of bucket to use
|
||||
# S3PROXY_BINARY="" Leave empty
|
||||
# S3_URL="http://s3.amazonaws.com" Specify Amazon server
|
||||
#
|
||||
# Example:
|
||||
#
|
||||
# S3FS_CREDENTIALS_FILE=keyfile TEST_BUCKET_1=bucket S3PROXY_BINARY="" S3_URL="http://s3.amazonaws.com" ./small-integration-test.sh
|
||||
# Test s3fs-fuse file system operations with
|
||||
#
|
||||
|
||||
set -o xtrace
|
||||
set -o errexit
|
||||
|
||||
: ${S3_URL:="http://127.0.0.1:8080"}
|
||||
|
||||
# Require root
|
||||
REQUIRE_ROOT=require-root.sh
|
||||
#source $REQUIRE_ROOT
|
||||
|
||||
source integration-test-common.sh
|
||||
|
||||
function retry {
|
||||
set +o errexit
|
||||
N=$1; shift;
|
||||
status=0
|
||||
for i in $(seq $N); do
|
||||
$@
|
||||
status=$?
|
||||
if [ $status == 0 ]; then
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
start_s3proxy
|
||||
|
||||
if [ $status != 0 ]; then
|
||||
echo "timeout waiting for $@"
|
||||
fi
|
||||
set -o errexit
|
||||
return $status
|
||||
}
|
||||
#
|
||||
# enable_content_md5
|
||||
# Causes s3fs to validate file contents. This isn't included in the common
|
||||
# options used by start_s3fs because tests may be performance tests
|
||||
# singlepart_copy_limit
|
||||
# Appeared in upstream s3fs-fuse tests, possibly a limitation of S3Proxy
|
||||
# TODO: github archaeology to see why it was added.
|
||||
#
|
||||
start_s3fs -o enable_content_md5 \
|
||||
-o singlepart_copy_limit=$((10 * 1024))
|
||||
|
||||
function exit_handler {
|
||||
if [ -n "${S3PROXY_PID}" ]
|
||||
then
|
||||
kill $S3PROXY_PID
|
||||
fi
|
||||
retry 30 grep $TEST_BUCKET_MOUNT_POINT_1 /proc/mounts && fusermount -u $TEST_BUCKET_MOUNT_POINT_1
|
||||
}
|
||||
trap exit_handler EXIT
|
||||
./integration-test-main.sh
|
||||
|
||||
if [ -n "${S3PROXY_BINARY}" ]
|
||||
then
|
||||
stdbuf -oL -eL java -jar "$S3PROXY_BINARY" --properties s3proxy.conf | stdbuf -oL -eL sed -u "s/^/s3proxy: /" &
|
||||
|
||||
# wait for S3Proxy to start
|
||||
for i in $(seq 30);
|
||||
do
|
||||
if exec 3<>"/dev/tcp/127.0.0.1/8080";
|
||||
then
|
||||
exec 3<&- # Close for read
|
||||
exec 3>&- # Close for write
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
S3PROXY_PID=$(netstat -lpnt | grep :8080 | awk '{ print $7 }' | sed -u 's|/java||')
|
||||
fi
|
||||
|
||||
# Mount the bucket
|
||||
if [ ! -d $TEST_BUCKET_MOUNT_POINT_1 ]
|
||||
then
|
||||
mkdir -p $TEST_BUCKET_MOUNT_POINT_1
|
||||
fi
|
||||
stdbuf -oL -eL $S3FS $TEST_BUCKET_1 $TEST_BUCKET_MOUNT_POINT_1 \
|
||||
-o createbucket \
|
||||
-o enable_content_md5 \
|
||||
-o passwd_file=$S3FS_CREDENTIALS_FILE \
|
||||
-o sigv2 \
|
||||
-o singlepart_copy_limit=$((10 * 1024)) \
|
||||
-o url=${S3_URL} \
|
||||
-o use_path_request_style \
|
||||
-o dbglevel=info -f |& stdbuf -oL -eL sed -u "s/^/s3fs: /" &
|
||||
|
||||
retry 30 grep $TEST_BUCKET_MOUNT_POINT_1 /proc/mounts || exit 1
|
||||
|
||||
./integration-test-main.sh $TEST_BUCKET_MOUNT_POINT_1
|
||||
|
||||
echo "All tests complete."
|
||||
echo "$0: tests complete."
|
||||
|
||||
156
test/test-utils.sh
Normal file
156
test/test-utils.sh
Normal file
@ -0,0 +1,156 @@
|
||||
#### Test utils
|
||||
|
||||
set -o errexit
|
||||
|
||||
# Configuration
|
||||
TEST_TEXT="HELLO WORLD"
|
||||
TEST_TEXT_FILE=test-s3fs.txt
|
||||
TEST_DIR=testdir
|
||||
ALT_TEST_TEXT_FILE=test-s3fs-ALT.txt
|
||||
TEST_TEXT_FILE_LENGTH=15
|
||||
BIG_FILE=big-file-s3fs.txt
|
||||
BIG_FILE_LENGTH=$((25 * 1024 * 1024))
|
||||
export RUN_DIR
|
||||
|
||||
function mk_test_file {
|
||||
if [ $# == 0 ]; then
|
||||
TEXT=$TEST_TEXT
|
||||
else
|
||||
TEXT=$1
|
||||
fi
|
||||
echo $TEXT > $TEST_TEXT_FILE
|
||||
if [ ! -e $TEST_TEXT_FILE ]
|
||||
then
|
||||
echo "Could not create file ${TEST_TEXT_FILE}, it does not exist"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
function rm_test_file {
|
||||
if [ $# == 0 ]; then
|
||||
FILE=$TEST_TEXT_FILE
|
||||
else
|
||||
FILE=$1
|
||||
fi
|
||||
rm -f $FILE
|
||||
|
||||
if [ -e $FILE ]
|
||||
then
|
||||
echo "Could not cleanup file ${TEST_TEXT_FILE}"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
function mk_test_dir {
|
||||
mkdir ${TEST_DIR}
|
||||
|
||||
if [ ! -d ${TEST_DIR} ]; then
|
||||
echo "Directory ${TEST_DIR} was not created"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
function rm_test_dir {
|
||||
rmdir ${TEST_DIR}
|
||||
if [ -e $TEST_DIR ]; then
|
||||
echo "Could not remove the test directory, it still exists: ${TEST_DIR}"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Create and cd to a unique directory for this test run
|
||||
# Sets RUN_DIR to the name of the created directory
|
||||
function cd_run_dir {
|
||||
if [ "$TEST_BUCKET_MOUNT_POINT_1" == "" ]; then
|
||||
echo "TEST_BUCKET_MOUNT_POINT variable not set"
|
||||
exit 1
|
||||
fi
|
||||
RUN_DIR=$(mktemp --directory ${TEST_BUCKET_MOUNT_POINT_1}/testrun-XXXXXX)
|
||||
cd ${RUN_DIR}
|
||||
}
|
||||
|
||||
function clean_run_dir {
|
||||
if [ -d ${RUN_DIR} ]; then
|
||||
rm -rf ${RUN_DIR} || echo "Error removing ${RUN_DIR}"
|
||||
fi
|
||||
}
|
||||
|
||||
# Resets test suite
|
||||
function init_suite {
|
||||
TEST_LIST=()
|
||||
TEST_FAILED_LIST=()
|
||||
TEST_PASSED_LIST=()
|
||||
}
|
||||
|
||||
# Report a passing test case
|
||||
# report_pass TEST_NAME
|
||||
function report_pass {
|
||||
echo "$1 passed"
|
||||
TEST_PASSED_LIST+=($1)
|
||||
}
|
||||
|
||||
# Report a failing test case
|
||||
# report_fail TEST_NAME
|
||||
function report_fail {
|
||||
echo "$1 failed"
|
||||
TEST_FAILED_LIST+=($1)
|
||||
}
|
||||
|
||||
# Add tests to the suite
|
||||
# add_tests TEST_NAME...
|
||||
function add_tests {
|
||||
TEST_LIST+=("$@")
|
||||
}
|
||||
|
||||
# Log test name and description
|
||||
# describe [DESCRIPTION]
|
||||
function describe {
|
||||
echo "${FUNCNAME[1]}: "$@""
|
||||
}
|
||||
|
||||
# Runs each test in a suite and summarizes results. The list of
|
||||
# tests added by add_tests() is called with CWD set to a tmp
|
||||
# directory in the bucket. An attempt to clean this directory is
|
||||
# made after the test run.
|
||||
function run_suite {
|
||||
orig_dir=$PWD
|
||||
cd_run_dir
|
||||
for t in "${TEST_LIST[@]}"; do
|
||||
# The following sequence runs tests in a subshell to allow continuation
|
||||
# on test failure, but still allowing errexit to be in effect during
|
||||
# the test.
|
||||
#
|
||||
# See:
|
||||
# https://groups.google.com/d/msg/gnu.bash.bug/NCK_0GmIv2M/dkeZ9MFhPOIJ
|
||||
# Other ways of trying to capture the return value will also disable
|
||||
# errexit in the function due to bash... compliance with POSIX?
|
||||
set +o errexit
|
||||
(set -o errexit; $t)
|
||||
if [[ $? == 0 ]]; then
|
||||
report_pass $t
|
||||
else
|
||||
report_fail $t
|
||||
fi
|
||||
set -o errexit
|
||||
done
|
||||
cd ${orig_dir}
|
||||
clean_run_dir
|
||||
|
||||
for t in "${TEST_PASSED_LIST[@]}"; do
|
||||
echo "PASS: $t"
|
||||
done
|
||||
for t in "${TEST_FAILED_LIST[@]}"; do
|
||||
echo "FAIL: $t"
|
||||
done
|
||||
|
||||
passed=${#TEST_PASSED_LIST[@]}
|
||||
failed=${#TEST_FAILED_LIST[@]}
|
||||
|
||||
echo "SUMMARY for $0: $passed tests passed. $failed tests failed."
|
||||
|
||||
if [[ $failed != 0 ]]; then
|
||||
return 1
|
||||
else
|
||||
return 0
|
||||
fi
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user