s3fs-fuse/test/integration-test-main.sh
Andrew Gaul 771bbfeac5
Do not create zero-byte object when creating file (#1640)
Previously s3fs created this object to store metadata and overwrote it
when flushing.  This prevented use with object stores which do not
allow overwrites like HDS.  Instead only create an in-memory
representation which reduces the time to create small files.
Fixes #1013.
2021-04-30 19:56:33 +09:00

1514 lines
46 KiB
Bash
Executable File

#!/bin/bash
#
# s3fs - FUSE-based file system backed by Amazon S3
#
# Copyright 2007-2008 Randy Rizun <rrizun@gmail.com>
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#
set -o errexit
set -o pipefail
source test-utils.sh
function test_create_empty_file {
describe "Testing creating an empty file ..."
OBJECT_NAME="$(basename $PWD)/${TEST_TEXT_FILE}"
touch ${TEST_TEXT_FILE}
check_file_size "${TEST_TEXT_FILE}" 0
aws_cli s3api head-object --bucket "${TEST_BUCKET_1}" --key "${OBJECT_NAME}"
rm_test_file
}
function test_append_file {
describe "Testing append to file ..."
TEST_INPUT="echo ${TEST_TEXT} to ${TEST_TEXT_FILE}"
# Write a small test file
for x in `seq 1 $TEST_TEXT_FILE_LENGTH`
do
echo $TEST_INPUT
done > ${TEST_TEXT_FILE}
check_file_size "${TEST_TEXT_FILE}" $(($TEST_TEXT_FILE_LENGTH * $(echo $TEST_INPUT | wc -c)))
rm_test_file
}
function test_truncate_file {
describe "Testing truncate file ..."
# Write a small test file
echo "${TEST_TEXT}" > ${TEST_TEXT_FILE}
# Truncate file to 0 length. This should trigger open(path, O_RDWR | O_TRUNC...)
: > ${TEST_TEXT_FILE}
check_file_size "${TEST_TEXT_FILE}" 0
rm_test_file
}
function test_truncate_upload {
describe "Testing truncate file for uploading ..."
# This file size uses multipart, mix upload when uploading.
# We will test these cases.
rm_test_file ${BIG_FILE}
${TRUNCATE_BIN} ${BIG_FILE} -s ${BIG_FILE_LENGTH}
rm_test_file ${BIG_FILE}
}
function test_truncate_empty_file {
describe "Testing truncate empty file ..."
# Write an empty test file
touch ${TEST_TEXT_FILE}
# Truncate the file to 1024 length
t_size=1024
${TRUNCATE_BIN} ${TEST_TEXT_FILE} -s $t_size
check_file_size "${TEST_TEXT_FILE}" $t_size
rm_test_file
}
function test_mv_file {
describe "Testing mv file function ..."
# if the rename file exists, delete it
if [ -e $ALT_TEST_TEXT_FILE ]
then
rm $ALT_TEST_TEXT_FILE
fi
if [ -e $ALT_TEST_TEXT_FILE ]
then
echo "Could not delete file ${ALT_TEST_TEXT_FILE}, it still exists"
return 1
fi
# create the test file again
mk_test_file
# save file length
ALT_TEXT_LENGTH=`wc -c $TEST_TEXT_FILE | awk '{print $1}'`
#rename the test file
mv $TEST_TEXT_FILE $ALT_TEST_TEXT_FILE
if [ ! -e $ALT_TEST_TEXT_FILE ]
then
echo "Could not move file"
return 1
fi
#check the renamed file content-type
if [ -f "/etc/mime.types" ]
then
check_content_type "$1/$ALT_TEST_TEXT_FILE" "text/plain"
fi
# Check the contents of the alt file
ALT_FILE_LENGTH=`wc -c $ALT_TEST_TEXT_FILE | awk '{print $1}'`
if [ "$ALT_FILE_LENGTH" -ne "$ALT_TEXT_LENGTH" ]
then
echo "moved file length is not as expected expected: $ALT_TEXT_LENGTH got: $ALT_FILE_LENGTH"
return 1
fi
# clean up
rm_test_file $ALT_TEST_TEXT_FILE
}
function test_mv_empty_directory {
describe "Testing mv directory function ..."
if [ -e $TEST_DIR ]; then
echo "Unexpected, this file/directory exists: ${TEST_DIR}"
return 1
fi
mk_test_dir
mv ${TEST_DIR} ${TEST_DIR}_rename
if [ ! -d "${TEST_DIR}_rename" ]; then
echo "Directory ${TEST_DIR} was not renamed"
return 1
fi
rmdir ${TEST_DIR}_rename
if [ -e "${TEST_DIR}_rename" ]; then
echo "Could not remove the test directory, it still exists: ${TEST_DIR}_rename"
return 1
fi
}
function test_mv_nonempty_directory {
describe "Testing mv directory function ..."
if [ -e $TEST_DIR ]; then
echo "Unexpected, this file/directory exists: ${TEST_DIR}"
return 1
fi
mk_test_dir
touch ${TEST_DIR}/file
mv ${TEST_DIR} ${TEST_DIR}_rename
if [ ! -d "${TEST_DIR}_rename" ]; then
echo "Directory ${TEST_DIR} was not renamed"
return 1
fi
rm -r ${TEST_DIR}_rename
if [ -e "${TEST_DIR}_rename" ]; then
echo "Could not remove the test directory, it still exists: ${TEST_DIR}_rename"
return 1
fi
}
function test_redirects {
describe "Testing redirects ..."
mk_test_file ABCDEF
CONTENT=`cat $TEST_TEXT_FILE`
if [ "${CONTENT}" != "ABCDEF" ]; then
echo "CONTENT read is unexpected, got ${CONTENT}, expected ABCDEF"
return 1
fi
echo XYZ > $TEST_TEXT_FILE
CONTENT=`cat $TEST_TEXT_FILE`
if [ ${CONTENT} != "XYZ" ]; then
echo "CONTENT read is unexpected, got ${CONTENT}, expected XYZ"
return 1
fi
echo 123456 >> $TEST_TEXT_FILE
LINE1=`${SED_BIN} -n '1,1p' $TEST_TEXT_FILE`
LINE2=`${SED_BIN} -n '2,2p' $TEST_TEXT_FILE`
if [ ${LINE1} != "XYZ" ]; then
echo "LINE1 was not as expected, got ${LINE1}, expected XYZ"
return 1
fi
if [ ${LINE2} != "123456" ]; then
echo "LINE2 was not as expected, got ${LINE2}, expected 123456"
return 1
fi
# clean up
rm_test_file
}
function test_mkdir_rmdir {
describe "Testing creation/removal of a directory ..."
if [ -e $TEST_DIR ]; then
echo "Unexpected, this file/directory exists: ${TEST_DIR}"
return 1
fi
mk_test_dir
rm_test_dir
}
function test_chmod {
describe "Testing chmod file function ..."
# create the test file again
mk_test_file
ORIGINAL_PERMISSIONS=$(get_permissions $TEST_TEXT_FILE)
chmod 777 $TEST_TEXT_FILE;
# if they're the same, we have a problem.
CHANGED_PERMISSIONS=$(get_permissions $TEST_TEXT_FILE)
if [ $CHANGED_PERMISSIONS == $ORIGINAL_PERMISSIONS ]
then
echo "Could not modify $TEST_TEXT_FILE permissions"
return 1
fi
# clean up
rm_test_file
}
function test_chown {
describe "Testing chown file function ..."
# create the test file again
mk_test_file
if [ `uname` = "Darwin" ]; then
ORIGINAL_PERMISSIONS=$(stat -f "%u:%g" $TEST_TEXT_FILE)
else
ORIGINAL_PERMISSIONS=$(stat --format=%u:%g $TEST_TEXT_FILE)
fi
# [NOTE]
# Prevents test interruptions due to permission errors, etc.
# If the chown command fails, an error will occur with the
# following judgment statement. So skip the chown command error.
# '|| true' was added due to a problem with Travis CI and MacOS
# and ensure_diskfree option.
#
chown 1000:1000 $TEST_TEXT_FILE || true
# if they're the same, we have a problem.
if [ `uname` = "Darwin" ]; then
CHANGED_PERMISSIONS=$(stat -f "%u:%g" $TEST_TEXT_FILE)
else
CHANGED_PERMISSIONS=$(stat --format=%u:%g $TEST_TEXT_FILE)
fi
if [ $CHANGED_PERMISSIONS == $ORIGINAL_PERMISSIONS ]
then
if [ $ORIGINAL_PERMISSIONS == "1000:1000" ]
then
echo "Could not be strict check because original file permission 1000:1000"
else
echo "Could not modify $TEST_TEXT_FILE ownership($ORIGINAL_PERMISSIONS to 1000:1000)"
return 1
fi
fi
# clean up
rm_test_file
}
function test_list {
describe "Testing list ..."
mk_test_file
mk_test_dir
file_cnt=$(ls -1 | wc -l)
if [ $file_cnt != 2 ]; then
echo "Expected 2 file but got $file_cnt"
return 1
fi
rm_test_file
rm_test_dir
}
function test_remove_nonempty_directory {
describe "Testing removing a non-empty directory ..."
mk_test_dir
touch "${TEST_DIR}/file"
(
set +o pipefail
rmdir "${TEST_DIR}" 2>&1 | grep -q "Directory not empty"
)
rm "${TEST_DIR}/file"
rm_test_dir
}
function test_external_directory_creation {
describe "Test external directory creation ..."
OBJECT_NAME="$(basename $PWD)/directory/${TEST_TEXT_FILE}"
echo "data" | aws_cli s3 cp - "s3://${TEST_BUCKET_1}/${OBJECT_NAME}"
ls | grep -q directory
get_permissions directory | grep -q 750$
ls directory
cmp <(echo "data") directory/${TEST_TEXT_FILE}
rm -f directory/${TEST_TEXT_FILE}
}
function test_external_modification {
describe "Test external modification to an object ..."
echo "old" > ${TEST_TEXT_FILE}
OBJECT_NAME="$(basename $PWD)/${TEST_TEXT_FILE}"
sleep 2
echo "new new" | aws_cli s3 cp - "s3://${TEST_BUCKET_1}/${OBJECT_NAME}"
cmp ${TEST_TEXT_FILE} <(echo "new new")
rm -f ${TEST_TEXT_FILE}
}
function test_read_external_object() {
describe "create objects via aws CLI and read via s3fs ..."
OBJECT_NAME="$(basename $PWD)/${TEST_TEXT_FILE}"
sleep 3
echo "test" | aws_cli s3 cp - "s3://${TEST_BUCKET_1}/${OBJECT_NAME}"
cmp ${TEST_TEXT_FILE} <(echo "test")
rm -f ${TEST_TEXT_FILE}
}
function test_update_metadata_external_small_object() {
describe "update meta to small file after created file by aws cli"
# [NOTE]
# Use the only filename in the test to avoid being affected by noobjcache.
#
TEST_FILE_EXT=`make_random_string`
TEST_CHMOD_FILE="${TEST_TEXT_FILE}_chmod.${TEST_FILE_EXT}"
TEST_CHOWN_FILE="${TEST_TEXT_FILE}_chown.${TEST_FILE_EXT}"
TEST_UTIMENS_FILE="${TEST_TEXT_FILE}_utimens.${TEST_FILE_EXT}"
TEST_SETXATTR_FILE="${TEST_TEXT_FILE}_xattr.${TEST_FILE_EXT}"
TEST_RMXATTR_FILE="${TEST_TEXT_FILE}_xattr.${TEST_FILE_EXT}"
TEST_INPUT="TEST_STRING_IN_SMALL_FILE"
#
# chmod
#
OBJECT_NAME="$(basename $PWD)/${TEST_CHMOD_FILE}"
echo "${TEST_INPUT}" | aws_cli s3 cp - "s3://${TEST_BUCKET_1}/${OBJECT_NAME}"
chmod +x ${TEST_CHMOD_FILE}
cmp ${TEST_CHMOD_FILE} <(echo "${TEST_INPUT}")
#
# chown
#
OBJECT_NAME="$(basename $PWD)/${TEST_CHOWN_FILE}"
echo "${TEST_INPUT}" | aws_cli s3 cp - "s3://${TEST_BUCKET_1}/${OBJECT_NAME}"
chown $UID ${TEST_CHOWN_FILE}
cmp ${TEST_CHOWN_FILE} <(echo "${TEST_INPUT}")
#
# utimens
#
OBJECT_NAME="$(basename $PWD)/${TEST_UTIMENS_FILE}"
echo "${TEST_INPUT}" | aws_cli s3 cp - "s3://${TEST_BUCKET_1}/${OBJECT_NAME}"
touch ${TEST_UTIMENS_FILE}
cmp ${TEST_UTIMENS_FILE} <(echo "${TEST_INPUT}")
#
# set xattr
#
OBJECT_NAME="$(basename $PWD)/${TEST_SETXATTR_FILE}"
echo "${TEST_INPUT}" | aws_cli s3 cp - "s3://${TEST_BUCKET_1}/${OBJECT_NAME}"
set_xattr key value ${TEST_SETXATTR_FILE}
cmp ${TEST_SETXATTR_FILE} <(echo "${TEST_INPUT}")
#
# remove xattr
#
# "%7B%22key%22%3A%22dmFsdWU%3D%22%7D" = {"key":"value"}
#
OBJECT_NAME="$(basename $PWD)/${TEST_RMXATTR_FILE}"
echo "${TEST_INPUT}" | aws_cli s3 cp - "s3://${TEST_BUCKET_1}/${OBJECT_NAME}" --metadata xattr=%7B%22key%22%3A%22dmFsdWU%3D%22%7D
del_xattr key ${TEST_RMXATTR_FILE}
cmp ${TEST_RMXATTR_FILE} <(echo "${TEST_INPUT}")
rm -f ${TEST_CHMOD_FILE}
rm -f ${TEST_CHOWN_FILE}
rm -f ${TEST_UTIMENS_FILE}
rm -f ${TEST_SETXATTR_FILE}
rm -f ${TEST_RMXATTR_FILE}
}
function test_update_metadata_external_large_object() {
describe "update meta to large file after created file by aws cli"
# [NOTE]
# Use the only filename in the test to avoid being affected by noobjcache.
#
TEST_FILE_EXT=`make_random_string`
TEST_CHMOD_FILE="${TEST_TEXT_FILE}_chmod.${TEST_FILE_EXT}"
TEST_CHOWN_FILE="${TEST_TEXT_FILE}_chown.${TEST_FILE_EXT}"
TEST_UTIMENS_FILE="${TEST_TEXT_FILE}_utimens.${TEST_FILE_EXT}"
TEST_SETXATTR_FILE="${TEST_TEXT_FILE}_xattr.${TEST_FILE_EXT}"
TEST_RMXATTR_FILE="${TEST_TEXT_FILE}_xattr.${TEST_FILE_EXT}"
dd if=/dev/urandom of="${TEMP_DIR}/${BIG_FILE}" bs=$BIG_FILE_BLOCK_SIZE count=$BIG_FILE_COUNT
#
# chmod
#
OBJECT_NAME="$(basename $PWD)/${TEST_CHMOD_FILE}"
aws_cli s3 cp ${TEMP_DIR}/${BIG_FILE} "s3://${TEST_BUCKET_1}/${OBJECT_NAME}" --no-progress
chmod +x ${TEST_CHMOD_FILE}
cmp ${TEST_CHMOD_FILE} ${TEMP_DIR}/${BIG_FILE}
#
# chown
#
OBJECT_NAME="$(basename $PWD)/${TEST_CHOWN_FILE}"
aws_cli s3 cp ${TEMP_DIR}/${BIG_FILE} "s3://${TEST_BUCKET_1}/${OBJECT_NAME}" --no-progress
chown $UID ${TEST_CHOWN_FILE}
cmp ${TEST_CHOWN_FILE} ${TEMP_DIR}/${BIG_FILE}
#
# utimens
#
OBJECT_NAME="$(basename $PWD)/${TEST_UTIMENS_FILE}"
aws_cli s3 cp ${TEMP_DIR}/${BIG_FILE} "s3://${TEST_BUCKET_1}/${OBJECT_NAME}" --no-progress
touch ${TEST_UTIMENS_FILE}
cmp ${TEST_UTIMENS_FILE} ${TEMP_DIR}/${BIG_FILE}
#
# set xattr
#
OBJECT_NAME="$(basename $PWD)/${TEST_SETXATTR_FILE}"
aws_cli s3 cp ${TEMP_DIR}/${BIG_FILE} "s3://${TEST_BUCKET_1}/${OBJECT_NAME}" --no-progress
set_xattr key value ${TEST_SETXATTR_FILE}
cmp ${TEST_SETXATTR_FILE} ${TEMP_DIR}/${BIG_FILE}
#
# remove xattr
#
# "%7B%22key%22%3A%22dmFsdWU%3D%22%7D" = {"key":"value"}
#
OBJECT_NAME="$(basename $PWD)/${TEST_RMXATTR_FILE}"
aws_cli s3 cp ${TEMP_DIR}/${BIG_FILE} "s3://${TEST_BUCKET_1}/${OBJECT_NAME}" --no-progress --metadata xattr=%7B%22key%22%3A%22dmFsdWU%3D%22%7D
del_xattr key ${TEST_RMXATTR_FILE}
cmp ${TEST_RMXATTR_FILE} ${TEMP_DIR}/${BIG_FILE}
rm -f ${TEMP_DIR}/${BIG_FILE}
rm -f ${TEST_CHMOD_FILE}
rm -f ${TEST_CHOWN_FILE}
rm -f ${TEST_UTIMENS_FILE}
rm -f ${TEST_SETXATTR_FILE}
rm -f ${TEST_RMXATTR_FILE}
}
function test_rename_before_close {
describe "Testing rename before close ..."
(
echo foo
mv $TEST_TEXT_FILE ${TEST_TEXT_FILE}.new
) > $TEST_TEXT_FILE
if ! cmp <(echo foo) ${TEST_TEXT_FILE}.new; then
echo "rename before close failed"
return 1
fi
rm_test_file ${TEST_TEXT_FILE}.new
rm -f ${TEST_TEXT_FILE}
}
function test_multipart_upload {
describe "Testing multi-part upload ..."
dd if=/dev/urandom of="${TEMP_DIR}/${BIG_FILE}" bs=$BIG_FILE_BLOCK_SIZE count=$BIG_FILE_COUNT
dd if="${TEMP_DIR}/${BIG_FILE}" of="${BIG_FILE}" bs=$BIG_FILE_BLOCK_SIZE count=$BIG_FILE_COUNT
# Verify contents of file
echo "Comparing test file"
if ! cmp "${TEMP_DIR}/${BIG_FILE}" "${BIG_FILE}"
then
return 1
fi
rm -f "${TEMP_DIR}/${BIG_FILE}"
rm_test_file "${BIG_FILE}"
}
function test_multipart_copy {
describe "Testing multi-part copy ..."
dd if=/dev/urandom of="${TEMP_DIR}/${BIG_FILE}" bs=$BIG_FILE_BLOCK_SIZE count=$BIG_FILE_COUNT
dd if="${TEMP_DIR}/${BIG_FILE}" of="${BIG_FILE}" bs=$BIG_FILE_BLOCK_SIZE count=$BIG_FILE_COUNT
mv "${BIG_FILE}" "${BIG_FILE}-copy"
# Verify contents of file
echo "Comparing test file"
if ! cmp "${TEMP_DIR}/${BIG_FILE}" "${BIG_FILE}-copy"
then
return 1
fi
#check the renamed file content-type
check_content_type "$1/${BIG_FILE}-copy" "application/octet-stream"
rm -f "${TEMP_DIR}/${BIG_FILE}"
rm_test_file "${BIG_FILE}-copy"
}
function test_multipart_mix {
describe "Testing multi-part mix ..."
if [ `uname` = "Darwin" ]; then
cat /dev/null > $BIG_FILE
fi
dd if=/dev/urandom of="${TEMP_DIR}/${BIG_FILE}" bs=$BIG_FILE_BLOCK_SIZE count=$BIG_FILE_COUNT
dd if="${TEMP_DIR}/${BIG_FILE}" of="${BIG_FILE}" bs=$BIG_FILE_BLOCK_SIZE count=$BIG_FILE_COUNT
# (1) Edit the middle of an existing file
# modify directly(seek 7.5MB offset)
# In the case of nomultipart and nocopyapi,
# it makes no sense, but copying files is because it leaves no cache.
#
cp ${TEMP_DIR}/${BIG_FILE} ${TEMP_DIR}/${BIG_FILE}-mix
cp ${BIG_FILE} ${BIG_FILE}-mix
MODIFY_START_BLOCK=$((15*1024*1024/2/4))
echo -n "0123456789ABCDEF" | dd of="${BIG_FILE}-mix" bs=4 count=4 seek=$MODIFY_START_BLOCK conv=notrunc
echo -n "0123456789ABCDEF" | dd of="${TEMP_DIR}/${BIG_FILE}-mix" bs=4 count=4 seek=$MODIFY_START_BLOCK conv=notrunc
# Verify contents of file
echo "Comparing test file (1)"
if ! cmp "${TEMP_DIR}/${BIG_FILE}-mix" "${BIG_FILE}-mix"
then
return 1
fi
# (2) Write to an area larger than the size of the existing file
# modify directly(over file end offset)
#
cp ${TEMP_DIR}/${BIG_FILE} ${TEMP_DIR}/${BIG_FILE}-mix
cp ${BIG_FILE} ${BIG_FILE}-mix
OVER_FILE_BLOCK_POS=$((26*1024*1024/4))
echo -n "0123456789ABCDEF" | dd of="${BIG_FILE}-mix" bs=4 count=4 seek=$OVER_FILE_BLOCK_POS conv=notrunc
echo -n "0123456789ABCDEF" | dd of="${TEMP_DIR}/${BIG_FILE}-mix" bs=4 count=4 seek=$OVER_FILE_BLOCK_POS conv=notrunc
# Verify contents of file
echo "Comparing test file (2)"
if ! cmp "${TEMP_DIR}/${BIG_FILE}-mix" "${BIG_FILE}-mix"
then
return 1
fi
# (3) Writing from the 0th byte
#
cp ${TEMP_DIR}/${BIG_FILE} ${TEMP_DIR}/${BIG_FILE}-mix
cp ${BIG_FILE} ${BIG_FILE}-mix
echo -n "0123456789ABCDEF" | dd of="${BIG_FILE}-mix" bs=4 count=4 seek=0 conv=notrunc
echo -n "0123456789ABCDEF" | dd of="${TEMP_DIR}/${BIG_FILE}-mix" bs=4 count=4 seek=0 conv=notrunc
# Verify contents of file
echo "Comparing test file (3)"
if ! cmp "${TEMP_DIR}/${BIG_FILE}-mix" "${BIG_FILE}-mix"
then
return 1
fi
# (4) Write to the area within 5MB from the top
# modify directly(seek 1MB offset)
#
cp ${TEMP_DIR}/${BIG_FILE} ${TEMP_DIR}/${BIG_FILE}-mix
cp ${BIG_FILE} ${BIG_FILE}-mix
MODIFY_START_BLOCK=$((1*1024*1024))
echo -n "0123456789ABCDEF" | dd of="${BIG_FILE}-mix" bs=4 count=4 seek=$MODIFY_START_BLOCK conv=notrunc
echo -n "0123456789ABCDEF" | dd of="${TEMP_DIR}/${BIG_FILE}-mix" bs=4 count=4 seek=$MODIFY_START_BLOCK conv=notrunc
# Verify contents of file
echo "Comparing test file (4)"
if ! cmp "${TEMP_DIR}/${BIG_FILE}-mix" "${BIG_FILE}-mix"
then
return 1
fi
rm -f "${TEMP_DIR}/${BIG_FILE}"
rm -f "${TEMP_DIR}/${BIG_FILE}-mix"
rm_test_file "${BIG_FILE}"
rm_test_file "${BIG_FILE}-mix"
}
function test_special_characters {
describe "Testing special characters ..."
(
set +o pipefail
ls 'special' 2>&1 | grep -q 'No such file or directory'
ls 'special?' 2>&1 | grep -q 'No such file or directory'
ls 'special*' 2>&1 | grep -q 'No such file or directory'
ls 'special~' 2>&1 | grep -q 'No such file or directory'
ls 'specialµ' 2>&1 | grep -q 'No such file or directory'
)
mkdir "TOYOTA TRUCK 8.2.2"
rm -rf "TOYOTA TRUCK 8.2.2"
}
function test_hardlink {
describe "Testing hardlinks ..."
rm -f $TEST_TEXT_FILE
rm -f $ALT_TEST_TEXT_FILE
echo foo > $TEST_TEXT_FILE
(
set +o pipefail
ln $TEST_TEXT_FILE $ALT_TEST_TEXT_FILE 2>&1 | grep -q 'Operation not supported'
)
rm_test_file
rm_test_file $ALT_TEST_TEXT_FILE
}
function test_symlink {
describe "Testing symlinks ..."
rm -f $TEST_TEXT_FILE
rm -f $ALT_TEST_TEXT_FILE
echo foo > $TEST_TEXT_FILE
ln -s $TEST_TEXT_FILE $ALT_TEST_TEXT_FILE
cmp $TEST_TEXT_FILE $ALT_TEST_TEXT_FILE
rm -f $TEST_TEXT_FILE
[ -L $ALT_TEST_TEXT_FILE ]
[ ! -f $ALT_TEST_TEXT_FILE ]
rm -f $ALT_TEST_TEXT_FILE
}
function test_extended_attributes {
describe "Testing extended attributes ..."
rm -f $TEST_TEXT_FILE
touch $TEST_TEXT_FILE
# set value
set_xattr key1 value1 $TEST_TEXT_FILE
get_xattr key1 $TEST_TEXT_FILE | grep -q '^value1$'
# append value
set_xattr key2 value2 $TEST_TEXT_FILE
get_xattr key1 $TEST_TEXT_FILE | grep -q '^value1$'
get_xattr key2 $TEST_TEXT_FILE | grep -q '^value2$'
# remove value
del_xattr key1 $TEST_TEXT_FILE
! get_xattr key1 $TEST_TEXT_FILE
get_xattr key2 $TEST_TEXT_FILE | grep -q '^value2$'
rm_test_file
}
function test_mtime_file {
describe "Testing mtime preservation function ..."
# if the rename file exists, delete it
if [ -e $ALT_TEST_TEXT_FILE -o -L $ALT_TEST_TEXT_FILE ]
then
rm $ALT_TEST_TEXT_FILE
fi
if [ -e $ALT_TEST_TEXT_FILE ]
then
echo "Could not delete file ${ALT_TEST_TEXT_FILE}, it still exists"
return 1
fi
# create the test file again
mk_test_file
sleep 2 # allow for some time to pass to compare the timestamps between test & alt
#copy the test file with preserve mode
cp -p $TEST_TEXT_FILE $ALT_TEST_TEXT_FILE
testmtime=`get_mtime $TEST_TEXT_FILE`
altmtime=`get_mtime $ALT_TEST_TEXT_FILE`
if [ "$testmtime" -ne "$altmtime" ]
then
echo "File times do not match: $testmtime != $altmtime"
return 1
fi
rm_test_file
rm_test_file $ALT_TEST_TEXT_FILE
}
# [NOTE]
# If it mounted with relatime or noatime options , the "touch -a"
# command may not update the atime.
# In ubuntu:xenial, atime was updated even if relatime was granted.
# However, it was not updated in bionic/focal.
# We can probably update atime by explicitly specifying the strictatime
# option and running the "touch -a" command. However, the strictatime
# option cannot be set.
# Therefore, if the relatime option is set, the test with the "touch -a"
# command is bypassed.
# We do not know why atime is not updated may or not be affected by
# these options.(can't say for sure)
# However, if atime has not been updated, the s3fs_utimens entry point
# will not be called from FUSE library. We added this bypass because
# the test became unstable.
#
function test_update_time() {
describe "Testing update time function ..."
#
# create the test
#
mk_test_file
base_atime=`get_atime $TEST_TEXT_FILE`
base_ctime=`get_ctime $TEST_TEXT_FILE`
base_mtime=`get_mtime $TEST_TEXT_FILE`
sleep 2
#
# chmod -> update only ctime
#
chmod +x $TEST_TEXT_FILE
atime=`get_atime $TEST_TEXT_FILE`
ctime=`get_ctime $TEST_TEXT_FILE`
mtime=`get_mtime $TEST_TEXT_FILE`
if [ $base_atime -ne $atime -o $base_ctime -eq $ctime -o $base_mtime -ne $mtime ]; then
echo "chmod expected updated ctime: $base_ctime != $ctime and same mtime: $base_mtime == $mtime, atime: $base_atime == $atime"
return 1
fi
base_ctime=$ctime
sleep 2
#
# chown -> update only ctime
#
chown $UID $TEST_TEXT_FILE
atime=`get_atime $TEST_TEXT_FILE`
ctime=`get_ctime $TEST_TEXT_FILE`
mtime=`get_mtime $TEST_TEXT_FILE`
if [ $base_atime -ne $atime -o $base_ctime -eq $ctime -o $base_mtime -ne $mtime ]; then
echo "chown expected updated ctime: $base_ctime != $ctime and same mtime: $base_mtime == $mtime, atime: $base_atime == $atime"
return 1
fi
base_ctime=$ctime
sleep 2
#
# set_xattr -> update only ctime
#
set_xattr key value $TEST_TEXT_FILE
atime=`get_atime $TEST_TEXT_FILE`
ctime=`get_ctime $TEST_TEXT_FILE`
mtime=`get_mtime $TEST_TEXT_FILE`
if [ $base_atime -ne $atime -o $base_ctime -eq $ctime -o $base_mtime -ne $mtime ]; then
echo "set_xattr expected updated ctime: $base_ctime != $ctime and same mtime: $base_mtime == $mtime, atime: $base_atime == $atime"
return 1
fi
base_ctime=$ctime
sleep 2
#
# touch -> update ctime/atime/mtime
#
touch $TEST_TEXT_FILE
atime=`get_atime $TEST_TEXT_FILE`
ctime=`get_ctime $TEST_TEXT_FILE`
mtime=`get_mtime $TEST_TEXT_FILE`
if [ $base_atime -eq $atime -o $base_ctime -eq $ctime -o $base_mtime -eq $mtime ]; then
echo "touch expected updated ctime: $base_ctime != $ctime, mtime: $base_mtime != $mtime, atime: $base_atime != $atime"
return 1
fi
base_atime=$atime
base_mtime=$mtime
base_ctime=$ctime
sleep 2
#
# "touch -a" -> update ctime/atime, not update mtime
#
if ! cat /proc/mounts | grep "^s3fs " | grep "$TEST_BUCKET_MOUNT_POINT_1 " | grep -e noatime -e relatime >/dev/null; then
touch -a $TEST_TEXT_FILE
atime=`get_atime $TEST_TEXT_FILE`
ctime=`get_ctime $TEST_TEXT_FILE`
mtime=`get_mtime $TEST_TEXT_FILE`
if [ $base_atime -eq $atime -o $base_ctime -eq $ctime -o $base_mtime -ne $mtime ]; then
echo "touch with -a option expected updated ctime: $base_ctime != $ctime, atime: $base_atime != $atime and same mtime: $base_mtime == $mtime"
return 1
fi
base_atime=$atime
base_ctime=$ctime
sleep 2
fi
#
# append -> update ctime/mtime, not update atime
#
echo foo >> $TEST_TEXT_FILE
atime=`get_atime $TEST_TEXT_FILE`
ctime=`get_ctime $TEST_TEXT_FILE`
mtime=`get_mtime $TEST_TEXT_FILE`
if [ $base_atime -ne $atime -o $base_ctime -eq $ctime -o $base_mtime -eq $mtime ]; then
echo "append expected updated ctime: $base_ctime != $ctime, mtime: $base_mtime != $mtime and same atime: $base_atime == $atime"
return 1
fi
base_mtime=$mtime
base_ctime=$ctime
sleep 2
#
# cp -p -> update ctime, not update atime/mtime
#
TIME_TEST_TEXT_FILE=test-s3fs-time.txt
cp -p $TEST_TEXT_FILE $TIME_TEST_TEXT_FILE
atime=`get_atime $TIME_TEST_TEXT_FILE`
ctime=`get_ctime $TIME_TEST_TEXT_FILE`
mtime=`get_mtime $TIME_TEST_TEXT_FILE`
if [ $base_atime -ne $atime -o $base_ctime -eq $ctime -o $base_mtime -ne $mtime ]; then
echo "cp with -p option expected updated ctime: $base_ctime != $ctime and same mtime: $base_mtime == $mtime, atime: $base_atime == $atime"
return 1
fi
sleep 2
#
# mv -> update ctime, not update atime/mtime
#
TIME2_TEST_TEXT_FILE=test-s3fs-time2.txt
mv $TEST_TEXT_FILE $TIME2_TEST_TEXT_FILE
atime=`get_atime $TIME2_TEST_TEXT_FILE`
ctime=`get_ctime $TIME2_TEST_TEXT_FILE`
mtime=`get_mtime $TIME2_TEST_TEXT_FILE`
if [ $base_atime -ne $atime -o $base_ctime -eq $ctime -o $base_mtime -ne $mtime ]; then
echo "mv expected updated ctime: $base_ctime != $ctime and same mtime: $base_mtime == $mtime, atime: $base_atime == $atime"
return 1
fi
rm_test_file $TIME_TEST_TEXT_FILE
rm_test_file $TIME2_TEST_TEXT_FILE
}
# [NOTE]
# See the description of test_update_time () for notes about the
# "touch -a" command and atime.
#
function test_update_directory_time() {
describe "Testing update time for directory function ..."
#
# create the directory and sub-directory and a file in directory
#
TIME_TEST_SUBDIR="$TEST_DIR/testsubdir"
TIME_TEST_FILE_INDIR="$TEST_DIR/testfile"
mk_test_dir
mkdir $TIME_TEST_SUBDIR
touch $TIME_TEST_FILE_INDIR
base_atime=`get_atime $TEST_DIR`
base_ctime=`get_ctime $TEST_DIR`
base_mtime=`get_mtime $TEST_DIR`
subdir_atime=`get_atime $TIME_TEST_SUBDIR`
subdir_ctime=`get_ctime $TIME_TEST_SUBDIR`
subdir_mtime=`get_mtime $TIME_TEST_SUBDIR`
subfile_atime=`get_atime $TIME_TEST_FILE_INDIR`
subfile_ctime=`get_ctime $TIME_TEST_FILE_INDIR`
subfile_mtime=`get_mtime $TIME_TEST_FILE_INDIR`
sleep 2
#
# chmod -> update only ctime
#
chmod 0777 $TEST_DIR
atime=`get_atime $TEST_DIR`
ctime=`get_ctime $TEST_DIR`
mtime=`get_mtime $TEST_DIR`
if [ $base_atime -ne $atime -o $base_ctime -eq $ctime -o $base_mtime -ne $mtime ]; then
echo "chmod expected updated ctime: $base_ctime != $ctime and same mtime: $base_mtime == $mtime, atime: $base_atime == $atime"
return 1
fi
base_ctime=$ctime
sleep 2
#
# chown -> update only ctime
#
chown $UID $TEST_DIR
atime=`get_atime $TEST_DIR`
ctime=`get_ctime $TEST_DIR`
mtime=`get_mtime $TEST_DIR`
if [ $base_atime -ne $atime -o $base_ctime -eq $ctime -o $base_mtime -ne $mtime ]; then
echo "chown expected updated ctime: $base_ctime != $ctime and same mtime: $base_mtime == $mtime, atime: $base_atime == $atime"
return 1
fi
base_ctime=$ctime
sleep 2
#
# set_xattr -> update only ctime
#
set_xattr key value $TEST_DIR
atime=`get_atime $TEST_DIR`
ctime=`get_ctime $TEST_DIR`
mtime=`get_mtime $TEST_DIR`
if [ $base_atime -ne $atime -o $base_ctime -eq $ctime -o $base_mtime -ne $mtime ]; then
echo "set_xattr expected updated ctime: $base_ctime != $ctime and same mtime: $base_mtime == $mtime, atime: $base_atime == $atime"
return 1
fi
base_ctime=$ctime
sleep 2
#
# touch -> update ctime/atime/mtime
#
touch $TEST_DIR
atime=`get_atime $TEST_DIR`
ctime=`get_ctime $TEST_DIR`
mtime=`get_mtime $TEST_DIR`
if [ $base_atime -eq $atime -o $base_ctime -eq $ctime -o $base_mtime -eq $mtime ]; then
echo "touch expected updated ctime: $base_ctime != $ctime, mtime: $base_mtime != $mtime, atime: $base_atime != $atime"
return 1
fi
base_atime=$atime
base_mtime=$mtime
base_ctime=$ctime
sleep 2
#
# "touch -a" -> update ctime/atime, not update mtime
#
if ! cat /proc/mounts | grep "^s3fs " | grep "$TEST_BUCKET_MOUNT_POINT_1 " | grep -e noatime -e relatime >/dev/null; then
touch -a $TEST_DIR
atime=`get_atime $TEST_DIR`
ctime=`get_ctime $TEST_DIR`
mtime=`get_mtime $TEST_DIR`
if [ $base_atime -eq $atime -o $base_ctime -eq $ctime -o $base_mtime -ne $mtime ]; then
echo "touch with -a option expected updated ctime: $base_ctime != $ctime, atime: $base_atime != $atime and same mtime: $base_mtime == $mtime"
return 1
fi
base_atime=$atime
base_ctime=$ctime
sleep 2
fi
#
# mv -> update ctime, not update atime/mtime for taget directory
# not update any for sub-directory and a file
#
TIME_TEST_DIR=timetestdir
TIME2_TEST_SUBDIR="$TIME_TEST_DIR/testsubdir"
TIME2_TEST_FILE_INDIR="$TIME_TEST_DIR/testfile"
mv $TEST_DIR $TIME_TEST_DIR
atime=`get_atime $TIME_TEST_DIR`
ctime=`get_ctime $TIME_TEST_DIR`
mtime=`get_mtime $TIME_TEST_DIR`
if [ $base_atime -ne $atime -o $base_ctime -eq $ctime -o $base_mtime -ne $mtime ]; then
echo "mv expected updated ctime: $base_ctime != $ctime and same mtime: $base_mtime == $mtime, atime: $base_atime == $atime"
return 1
fi
atime=`get_atime $TIME2_TEST_SUBDIR`
ctime=`get_ctime $TIME2_TEST_SUBDIR`
mtime=`get_mtime $TIME2_TEST_SUBDIR`
if [ $subdir_atime -ne $atime -o $subdir_ctime -ne $ctime -o $subdir_mtime -ne $mtime ]; then
echo "mv for sub-directory expected same ctime: $subdir_ctime == $ctime, mtime: $subdir_mtime == $mtime, atime: $subdir_atime == $atime"
return 1
fi
atime=`get_atime $TIME2_TEST_FILE_INDIR`
ctime=`get_ctime $TIME2_TEST_FILE_INDIR`
mtime=`get_mtime $TIME2_TEST_FILE_INDIR`
if [ $subfile_atime -ne $atime -o $subfile_ctime -ne $ctime -o $subfile_mtime -ne $mtime ]; then
echo "mv for a file in directory expected same ctime: $subfile_ctime == $ctime, mtime: $subfile_mtime == $mtime, atime: $subfile_atime == $atime"
return 1
fi
rm -rf $TIME_TEST_SUBDIR
rm -rf $TIME_TEST_DIR
}
function test_rm_rf_dir {
describe "Test that rm -rf will remove directory with contents ..."
# Create a dir with some files and directories
mkdir dir1
mkdir dir1/dir2
touch dir1/file1
touch dir1/dir2/file2
# Remove the dir with recursive rm
rm -rf dir1
if [ -e dir1 ]; then
echo "rm -rf did not remove $PWD/dir1"
return 1
fi
}
function test_copy_file {
describe "Test simple copy ..."
dd if=/dev/urandom of=/tmp/simple_file bs=1024 count=1
cp /tmp/simple_file copied_simple_file
cmp /tmp/simple_file copied_simple_file
rm_test_file /tmp/simple_file
rm_test_file copied_simple_file
}
function test_write_after_seek_ahead {
describe "Test writes succeed after a seek ahead ..."
dd if=/dev/zero of=testfile seek=1 count=1 bs=1024
rm_test_file testfile
}
function test_overwrite_existing_file_range {
describe "Test overwrite range succeeds ..."
dd if=<(seq 1000) of=${TEST_TEXT_FILE}
dd if=/dev/zero of=${TEST_TEXT_FILE} seek=1 count=1 bs=1024 conv=notrunc
cmp ${TEST_TEXT_FILE} <(
seq 1000 | head -c 1024
dd if=/dev/zero count=1 bs=1024
seq 1000 | tail -c +2049
)
rm_test_file
}
function test_concurrent_directory_updates {
describe "Test concurrent updates to a directory ..."
for i in `seq 5`; do echo foo > $i; done
for process in `seq 10`; do
for i in `seq 5`; do
file=$(ls `seq 5` | ${SED_BIN} -n "$(($RANDOM % 5 + 1))p")
cat $file >/dev/null || true
rm -f $file
echo foo > $file || true
done &
done
wait
rm -f `seq 5`
}
function test_concurrent_reads {
describe "Test concurrent reads from a file ..."
dd if=/dev/urandom of="${TEST_TEXT_FILE}" bs=$BIG_FILE_BLOCK_SIZE count=$BIG_FILE_COUNT
for process in `seq 10`; do
dd if=${TEST_TEXT_FILE} of=/dev/null seek=$(($RANDOM % $BIG_FILE_LENGTH)) count=16 bs=1024 &
done
wait
rm_test_file
}
function test_concurrent_writes {
describe "Test concurrent writes to a file ..."
dd if=/dev/urandom of="${TEST_TEXT_FILE}" bs=$BIG_FILE_BLOCK_SIZE count=$BIG_FILE_COUNT
for process in `seq 10`; do
dd if=/dev/zero of=${TEST_TEXT_FILE} seek=$(($RANDOM % $BIG_FILE_LENGTH)) count=16 bs=1024 conv=notrunc &
done
wait
rm_test_file
}
function test_open_second_fd {
describe "read from an open fd ..."
rm_test_file second_fd_file
RESULT=$( (echo foo ; wc -c < second_fd_file >&2) 2>& 1>second_fd_file)
if [ "$RESULT" -ne 4 ]; then
echo "size mismatch, expected: 4, was: ${RESULT}"
return 1
fi
rm_test_file second_fd_file
}
function test_write_multiple_offsets {
describe "test writing to multiple offsets ..."
../../write_multiple_offsets.py ${TEST_TEXT_FILE} 1024 1 $((16 * 1024 * 1024)) 1 $((18 * 1024 * 1024)) 1
rm_test_file ${TEST_TEXT_FILE}
}
function test_write_multiple_offsets_backwards {
describe "test writing to multiple offsets ..."
../../write_multiple_offsets.py ${TEST_TEXT_FILE} $((20 * 1024 * 1024 + 1)) 1 $((10 * 1024 * 1024)) 1
rm_test_file ${TEST_TEXT_FILE}
}
function test_clean_up_cache() {
describe "Test clean up cache ..."
dir="many_files"
count=25
mkdir -p $dir
for x in $(seq $count); do
dd if=/dev/urandom of=$dir/file-$x bs=10485760 count=1
done
file_cnt=$(ls $dir | wc -l)
if [ $file_cnt != $count ]; then
echo "Expected $count files but got $file_cnt"
rm -rf $dir
return 1
fi
CACHE_DISK_AVAIL_SIZE=`get_disk_avail_size $CACHE_DIR`
if [ "$CACHE_DISK_AVAIL_SIZE" -lt "$ENSURE_DISKFREE_SIZE" ];then
echo "Cache disk avail size:$CACHE_DISK_AVAIL_SIZE less than ensure_diskfree size:$ENSURE_DISKFREE_SIZE"
rm -rf $dir
return 1
fi
rm -rf $dir
}
function test_content_type() {
describe "Test Content-Type detection ..."
DIR_NAME="$(basename $PWD)"
touch "test.txt"
CONTENT_TYPE=$(aws_cli s3api head-object --bucket "${TEST_BUCKET_1}" --key "${DIR_NAME}/test.txt" | grep "ContentType")
if ! echo $CONTENT_TYPE | grep -q "text/plain"; then
echo "Unexpected Content-Type: $CONTENT_TYPE"
return 1;
fi
touch "test.jpg"
CONTENT_TYPE=$(aws_cli s3api head-object --bucket "${TEST_BUCKET_1}" --key "${DIR_NAME}/test.jpg" | grep "ContentType")
if ! echo $CONTENT_TYPE | grep -q "image/jpeg"; then
echo "Unexpected Content-Type: $CONTENT_TYPE"
return 1;
fi
touch "test.bin"
CONTENT_TYPE=$(aws_cli s3api head-object --bucket "${TEST_BUCKET_1}" --key "${DIR_NAME}/test.bin" | grep "ContentType")
if ! echo $CONTENT_TYPE | grep -q "application/octet-stream"; then
echo "Unexpected Content-Type: $CONTENT_TYPE"
return 1;
fi
mkdir "test.dir"
CONTENT_TYPE=$(aws_cli s3api head-object --bucket "${TEST_BUCKET_1}" --key "${DIR_NAME}/test.dir/" | grep "ContentType")
if ! echo $CONTENT_TYPE | grep -q "application/x-directory"; then
echo "Unexpected Content-Type: $CONTENT_TYPE"
return 1;
fi
rm -f test.txt
rm -f test.jpg
rm -f test.bin
rm -rf test.dir
}
# create more files than -o max_stat_cache_size
function test_truncate_cache() {
describe "Test make cache files over max cache file size ..."
for dir in $(seq 2); do
mkdir $dir
for file in $(seq 75); do
touch $dir/$file
done
ls $dir
done
rm -rf `seq 2`
}
function test_cache_file_stat() {
describe "Test cache file stat ..."
dd if=/dev/urandom of="${BIG_FILE}" bs=$BIG_FILE_BLOCK_SIZE count=$BIG_FILE_COUNT
#
# get "testrun-xxx" directory name
#
CACHE_TESTRUN_DIR=$(ls -1 ${CACHE_DIR}/${TEST_BUCKET_1}/ 2>/dev/null | grep testrun 2>/dev/null)
#
# get cache file inode number
#
CACHE_FILE_INODE=$(ls -i ${CACHE_DIR}/${TEST_BUCKET_1}/${CACHE_TESTRUN_DIR}/${BIG_FILE} 2>/dev/null | awk '{print $1}')
if [ -z ${CACHE_FILE_INODE} ]; then
echo "Not found cache file or failed to get inode: ${CACHE_DIR}/${TEST_BUCKET_1}/${CACHE_TESTRUN_DIR}/${BIG_FILE}"
return 1;
fi
#
# get lines from cache stat file
#
CACHE_FILE_STAT_LINE_1=$(${SED_BIN} -n 1p ${CACHE_DIR}/.${TEST_BUCKET_1}.stat/${CACHE_TESTRUN_DIR}/${BIG_FILE})
CACHE_FILE_STAT_LINE_2=$(${SED_BIN} -n 2p ${CACHE_DIR}/.${TEST_BUCKET_1}.stat/${CACHE_TESTRUN_DIR}/${BIG_FILE})
if [ -z ${CACHE_FILE_STAT_LINE_1} ] || [ -z ${CACHE_FILE_STAT_LINE_2} ]; then
echo "could not get first or second line from cache file stat: ${CACHE_DIR}/.${TEST_BUCKET_1}.stat/${CACHE_TESTRUN_DIR}/${BIG_FILE}"
return 1;
fi
#
# compare
#
if [ "${CACHE_FILE_STAT_LINE_1}" != "${CACHE_FILE_INODE}:${BIG_FILE_LENGTH}" ]; then
echo "first line(cache file stat) is different: \"${CACHE_FILE_STAT_LINE_1}\" != \"${CACHE_FILE_INODE}:${BIG_FILE_LENGTH}\""
return 1;
fi
if [ "${CACHE_FILE_STAT_LINE_2}" != "0:${BIG_FILE_LENGTH}:1:0" ]; then
echo "last line(cache file stat) is different: \"${CACHE_FILE_STAT_LINE_2}\" != \"0:${BIG_FILE_LENGTH}:1:0\""
return 1;
fi
#
# remove cache files directly
#
rm -f ${CACHE_DIR}/${TEST_BUCKET_1}/${CACHE_TESTRUN_DIR}/${BIG_FILE}
rm -f ${CACHE_DIR}/.${TEST_BUCKET_1}.stat/${CACHE_TESTRUN_DIR}/${BIG_FILE}
#
# write a byte into the middle(not the boundary) of the file
#
CHECK_UPLOAD_OFFSET=$((10 * 1024 * 1024 + 17))
dd if=/dev/urandom of="${BIG_FILE}" bs=1 count=1 seek=${CHECK_UPLOAD_OFFSET} conv=notrunc
#
# get cache file inode number
#
CACHE_FILE_INODE=$(ls -i ${CACHE_DIR}/${TEST_BUCKET_1}/${CACHE_TESTRUN_DIR}/${BIG_FILE} 2>/dev/null | awk '{print $1}')
if [ -z ${CACHE_FILE_INODE} ]; then
echo "Not found cache file or failed to get inode: ${CACHE_DIR}/${TEST_BUCKET_1}/${CACHE_TESTRUN_DIR}/${BIG_FILE}"
return 1;
fi
#
# get lines from cache stat file
#
CACHE_FILE_STAT_LINE_1=$(${SED_BIN} -n 1p ${CACHE_DIR}/.${TEST_BUCKET_1}.stat/${CACHE_TESTRUN_DIR}/${BIG_FILE})
CACHE_FILE_STAT_LINE_E=$(tail -1 ${CACHE_DIR}/.${TEST_BUCKET_1}.stat/${CACHE_TESTRUN_DIR}/${BIG_FILE} 2>/dev/null)
if [ -z ${CACHE_FILE_STAT_LINE_1} ] || [ -z ${CACHE_FILE_STAT_LINE_E} ]; then
echo "could not get first or end line from cache file stat: ${CACHE_DIR}/.${TEST_BUCKET_1}.stat/${CACHE_TESTRUN_DIR}/${BIG_FILE}"
return 1;
fi
#
# check first and cache file length from last line
#
# we should check all stat lines, but there are cases where the value
# differs depending on the processing system etc., then the cache file
# size is calculated and compared.
#
CACHE_LAST_OFFSET=$(echo ${CACHE_FILE_STAT_LINE_E} | cut -d ":" -f1)
CACHE_LAST_SIZE=$(echo ${CACHE_FILE_STAT_LINE_E} | cut -d ":" -f2)
CACHE_TOTAL_SIZE=$((${CACHE_LAST_OFFSET} + ${CACHE_LAST_SIZE}))
if [ "${CACHE_FILE_STAT_LINE_1}" != "${CACHE_FILE_INODE}:${BIG_FILE_LENGTH}" ]; then
echo "first line(cache file stat) is different: \"${CACHE_FILE_STAT_LINE_1}\" != \"${CACHE_FILE_INODE}:${BIG_FILE_LENGTH}\""
return 1;
fi
if [ ${BIG_FILE_LENGTH} -ne ${CACHE_TOTAL_SIZE} ]; then
echo "the file size indicated by the cache stat file is different: \"${BIG_FILE_LENGTH}\" != \"${CACHE_TOTAL_SIZE}\""
return 1;
fi
rm_test_file "${BIG_FILE}"
}
function test_upload_sparsefile {
describe "Testing upload sparse file ..."
rm_test_file ${BIG_FILE}
rm -f ${TEMP_DIR}/${BIG_FILE}
#
# Make all HOLE file
#
${TRUNCATE_BIN} ${BIG_FILE} -s ${BIG_FILE_LENGTH}
#
# Write some bytes to ABOUT middle in the file
# (Dare to remove the block breaks)
#
WRITE_POS=$((${BIG_FILE_LENGTH} / 2 - 128))
echo -n "0123456789ABCDEF" | dd of="${TEMP_DIR}/${BIG_FILE}" bs=1 count=16 seek=${WRITE_POS} conv=notrunc
#
# copy(upload) the file
#
cp ${TEMP_DIR}/${BIG_FILE} ${BIG_FILE}
#
# check
#
cmp ${TEMP_DIR}/${BIG_FILE} ${BIG_FILE}
rm_test_file ${BIG_FILE}
rm -f ${TEMP_DIR}/${BIG_FILE}
}
function test_mix_upload_entities() {
describe "Testing upload sparse files ..."
#
# Make test file
#
dd if=/dev/urandom of="${BIG_FILE}" bs=$BIG_FILE_BLOCK_SIZE count=$BIG_FILE_COUNT
#
# If the cache option is enabled, delete the cache of uploaded files.
#
if [ -f ${CACHE_DIR}/${TEST_BUCKET_1}/${BIG_FILE} ]; then
rm -f ${CACHE_DIR}/${TEST_BUCKET_1}/${BIG_FILE}
fi
if [ -f ${CACHE_DIR}/.${TEST_BUCKET_1}.stat/${BIG_FILE} ]; then
rm -f ${CACHE_DIR}/.${TEST_BUCKET_1}.stat/${BIG_FILE}
fi
#
# Do a partial write to the file.
#
echo -n "0123456789ABCDEF" | dd of=${BIG_FILE} bs=1 count=16 seek=0 conv=notrunc
echo -n "0123456789ABCDEF" | dd of=${BIG_FILE} bs=1 count=16 seek=8192 conv=notrunc
echo -n "0123456789ABCDEF" | dd of=${BIG_FILE} bs=1 count=16 seek=1073152 conv=notrunc
echo -n "0123456789ABCDEF" | dd of=${BIG_FILE} bs=1 count=16 seek=26214400 conv=notrunc
echo -n "0123456789ABCDEF" | dd of=${BIG_FILE} bs=1 count=16 seek=26222592 conv=notrunc
rm_test_file "${BIG_FILE}"
}
#
# [NOTE]
# This test runs last because it uses up disk space and may not recover.
# This may be a problem, especially on MacOS. (See the comment near the definition
# line for the ENSURE_DISKFREE_SIZE variable)
#
function test_ensurespace_move_file() {
describe "Testing upload(mv) file when diskspace is not enough ..."
#
# Make test file which is not under mountpoint
#
mkdir -p ${CACHE_DIR}/.s3fs_test_tmpdir
dd if=/dev/urandom of="${CACHE_DIR}/.s3fs_test_tmpdir/${BIG_FILE}" bs=$BIG_FILE_BLOCK_SIZE count=$BIG_FILE_COUNT
#
# Backup file stat
#
if [ `uname` = "Darwin" ]; then
ORIGINAL_PERMISSIONS=$(stat -f "%u:%g" ${CACHE_DIR}/.s3fs_test_tmpdir/$BIG_FILE)
else
ORIGINAL_PERMISSIONS=$(stat --format=%u:%g ${CACHE_DIR}/.s3fs_test_tmpdir/$BIG_FILE)
fi
#
# Fill the disk size
#
NOW_CACHE_DISK_AVAIL_SIZE=`get_disk_avail_size ${CACHE_DIR}`
TMP_FILE_NO=0
while true; do
ALLOWED_USING_SIZE=$((NOW_CACHE_DISK_AVAIL_SIZE - ENSURE_DISKFREE_SIZE))
if [ ${ALLOWED_USING_SIZE} -gt ${BIG_FILE_LENGTH} ]; then
cp -p ${CACHE_DIR}/.s3fs_test_tmpdir/${BIG_FILE} ${CACHE_DIR}/.s3fs_test_tmpdir/${BIG_FILE}_${TMP_FILE_NO}
TMP_FILE_NO=$((TMP_FILE_NO + 1))
else
break;
fi
done
#
# move file
#
mv "${CACHE_DIR}/.s3fs_test_tmpdir/${BIG_FILE}" "${BIG_FILE}"
#
# file stat
#
if [ `uname` = "Darwin" ]; then
MOVED_PERMISSIONS=$(stat -f "%u:%g" $BIG_FILE)
else
MOVED_PERMISSIONS=$(stat --format=%u:%g $BIG_FILE)
fi
MOVED_FILE_LENGTH=$(ls -l $BIG_FILE | awk '{print $5}')
#
# check
#
if [ "${MOVED_PERMISSIONS}" != "${ORIGINAL_PERMISSIONS}" ]; then
echo "Failed to move file with permission"
return 1
fi
if [ ${MOVED_FILE_LENGTH} -ne ${BIG_FILE_LENGTH} ]; then
echo "Failed to move file with file length: ${MOVED_FILE_LENGTH} ${BIG_FILE_LENGTH}"
return 1
fi
rm_test_file "${BIG_FILE}"
rm -rf ${CACHE_DIR}/.s3fs_test_tmpdir
}
function test_ut_ossfs {
describe "Testing ossfs python ut..."
export TEST_BUCKET_MOUNT_POINT=$TEST_BUCKET_MOUNT_POINT_1
../../ut_test.py
}
function add_all_tests {
if ps u $S3FS_PID | grep -q use_cache; then
add_tests test_cache_file_stat
fi
if ! ps u $S3FS_PID | grep -q ensure_diskfree && ! uname | grep -q Darwin; then
add_tests test_clean_up_cache
fi
add_tests test_create_empty_file
add_tests test_append_file
add_tests test_truncate_file
add_tests test_truncate_upload
add_tests test_truncate_empty_file
add_tests test_mv_file
add_tests test_mv_empty_directory
add_tests test_mv_nonempty_directory
add_tests test_redirects
add_tests test_mkdir_rmdir
add_tests test_chmod
add_tests test_chown
add_tests test_list
add_tests test_remove_nonempty_directory
if ! ps u $S3FS_PID | grep -q notsup_compat_dir; then
# TODO: investigate why notsup_compat_dir fails
add_tests test_external_directory_creation
fi
add_tests test_external_modification
add_tests test_read_external_object
add_tests test_update_metadata_external_small_object
if ! uname | grep -q Darwin; then
# [NOTE]
# This test is very time consuming on OSX and will not run.
# And this test should be no different between OSX and
# other OSs. so skip this test on OSX.
#
add_tests test_update_metadata_external_large_object
fi
add_tests test_rename_before_close
add_tests test_multipart_upload
add_tests test_multipart_copy
add_tests test_multipart_mix
add_tests test_special_characters
add_tests test_hardlink
add_tests test_symlink
add_tests test_extended_attributes
add_tests test_mtime_file
add_tests test_update_time
add_tests test_update_directory_time
add_tests test_rm_rf_dir
add_tests test_copy_file
add_tests test_write_after_seek_ahead
add_tests test_overwrite_existing_file_range
add_tests test_concurrent_directory_updates
add_tests test_concurrent_reads
add_tests test_concurrent_writes
add_tests test_open_second_fd
add_tests test_write_multiple_offsets
add_tests test_write_multiple_offsets_backwards
add_tests test_content_type
add_tests test_truncate_cache
add_tests test_upload_sparsefile
add_tests test_mix_upload_entities
add_tests test_ut_ossfs
if ! ps u $S3FS_PID | grep -q ensure_diskfree && ! uname | grep -q Darwin; then
add_tests test_ensurespace_move_file
fi
}
init_suite
add_all_tests
run_suite
#
# Local variables:
# tab-width: 4
# c-basic-offset: 4
# End:
# vim600: expandtab sw=4 ts=4 fdm=marker
# vim<600: expandtab sw=4 ts=4
#