From 34f89e5936c711188332f604e8e42b79baae69c7 Mon Sep 17 00:00:00 2001 From: Takeshi Nakatani Date: Thu, 22 Jul 2021 05:35:54 +0000 Subject: [PATCH] Added fake_diskfree option to deceive free disk space for test --- src/fdcache.cpp | 20 +++++++++++++++++++- src/fdcache.h | 4 +++- src/s3fs.cpp | 13 +++++++++++++ test/small-integration-test.sh | 17 +++-------------- 4 files changed, 38 insertions(+), 16 deletions(-) diff --git a/src/fdcache.cpp b/src/fdcache.cpp index 2a9b796..0bbc0ed 100644 --- a/src/fdcache.cpp +++ b/src/fdcache.cpp @@ -82,6 +82,7 @@ bool FdManager::is_lock_init(false); std::string FdManager::cache_dir; bool FdManager::check_cache_dir_exist(false); off_t FdManager::free_disk_space = 0; +off_t FdManager::fake_used_disk_space = 0; std::string FdManager::check_cache_output; bool FdManager::checked_lseek(false); bool FdManager::have_lseek_hole(false); @@ -254,6 +255,20 @@ off_t FdManager::SetEnsureFreeDiskSpace(off_t size) return old; } +bool FdManager::InitFakeUsedDiskSize(off_t fake_freesize) +{ + FdManager::fake_used_disk_space = 0; // At first, clear this value because this value is used in GetFreeDiskSpace. + + off_t actual_freesize = FdManager::GetFreeDiskSpace(NULL); + + if(fake_freesize < actual_freesize){ + FdManager::fake_used_disk_space = actual_freesize - fake_freesize; + }else{ + FdManager::fake_used_disk_space = 0; + } + return true; +} + off_t FdManager::GetFreeDiskSpace(const char* path) { struct statvfs vfsbuf; @@ -276,7 +291,10 @@ off_t FdManager::GetFreeDiskSpace(const char* path) S3FS_PRN_ERR("could not get vfs stat by errno(%d)", errno); return 0; } - return (vfsbuf.f_bavail * vfsbuf.f_frsize); + + off_t actual_freesize = vfsbuf.f_bavail * vfsbuf.f_frsize; + + return (FdManager::fake_used_disk_space < actual_freesize ? (actual_freesize - FdManager::fake_used_disk_space) : 0); } bool FdManager::IsSafeDiskSpace(const char* path, off_t size) diff --git a/src/fdcache.h b/src/fdcache.h index d0ba55b..efde83c 100644 --- a/src/fdcache.h +++ b/src/fdcache.h @@ -36,7 +36,8 @@ class FdManager static bool is_lock_init; static std::string cache_dir; static bool check_cache_dir_exist; - static off_t free_disk_space; // limit free disk space + static off_t free_disk_space; // limit free disk space + static off_t fake_used_disk_space; // difference between fake free disk space and actual at startup(for test/debug) static std::string check_cache_output; static bool checked_lseek; static bool have_lseek_hole; @@ -72,6 +73,7 @@ class FdManager static bool HasOpenEntityFd(const char* path); static off_t GetEnsureFreeDiskSpace(); static off_t SetEnsureFreeDiskSpace(off_t size); + static bool InitFakeUsedDiskSize(off_t fake_freesize); static bool IsSafeDiskSpace(const char* path, off_t size); static void FreeReservedDiskSpace(off_t size); static bool ReserveDiskSpace(off_t size); diff --git a/src/s3fs.cpp b/src/s3fs.cpp index 27419e8..af1be97 100644 --- a/src/s3fs.cpp +++ b/src/s3fs.cpp @@ -98,6 +98,7 @@ static bool support_compat_dir = true;// default supports compatibility direc static int max_keys_list_object = 1000;// default is 1000 static off_t max_dirty_data = 5LL * 1024LL * 1024LL * 1024LL; static bool use_wtf8 = false; +static off_t fake_diskfree_size = -1; // default is not set(-1) static const std::string allbucket_fields_type; // special key for mapping(This name is absolutely not used as a bucket name) static const std::string keyval_fields_type = "\t"; // special key for mapping(This name is absolutely not used as a bucket name) @@ -4620,6 +4621,13 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar FdManager::SetEnsureFreeDiskSpace(dfsize); return 0; } + if(is_prefix(arg, "fake_diskfree=")){ + S3FS_PRN_WARN("The fake_diskfree option was specified. Use this option for testing or debugging."); + + // [NOTE] This value is used for initializing to FdManager after parsing all options. + fake_diskfree_size = cvt_strtoofft(strchr(arg, '=') + sizeof(char), /*base=*/ 10) * 1024 * 1024; + return 0; + } if(is_prefix(arg, "multipart_threshold=")){ multipart_threshold = static_cast(cvt_strtoofft(strchr(arg, '=') + sizeof(char), /*base=*/ 10)) * 1024 * 1024; if(multipart_threshold <= MIN_MULTIPART_SIZE){ @@ -5128,6 +5136,11 @@ int main(int argc, char* argv[]) exit(EXIT_FAILURE); } + // set fake free disk space + if(-1 != fake_diskfree_size){ + FdManager::InitFakeUsedDiskSize(fake_diskfree_size); + } + // check IBM IAM requirements if(is_ibm_iam_auth){ // check that default ACL is either public-read or private diff --git a/test/small-integration-test.sh b/test/small-integration-test.sh index d4648c9..de90dc6 100755 --- a/test/small-integration-test.sh +++ b/test/small-integration-test.sh @@ -38,25 +38,14 @@ mkdir "${CACHE_DIR}" #reserve 200MB for data cache source test-utils.sh -CACHE_DISK_AVAIL_SIZE=`get_disk_avail_size $CACHE_DIR` -if [ `uname` = "Darwin" ]; then - # [FIXME] - # Only on MacOS, there are cases where process or system - # other than the s3fs cache uses disk space. - # We can imagine that this is caused by Timemachine, but - # there is no workaround, so s3fs cache size is set +1gb - # for error bypass. - # - ENSURE_DISKFREE_SIZE=$((CACHE_DISK_AVAIL_SIZE - 1200)) -else - ENSURE_DISKFREE_SIZE=$((CACHE_DISK_AVAIL_SIZE - 200)) -fi +FAKE_FREE_DISK_SIZE=200 +ENSURE_DISKFREE_SIZE=10 export CACHE_DIR export ENSURE_DISKFREE_SIZE if [ -n "${ALL_TESTS}" ]; then FLAGS=( - "use_cache=${CACHE_DIR} -o ensure_diskfree=${ENSURE_DISKFREE_SIZE}" + "use_cache=${CACHE_DIR} -o ensure_diskfree=${ENSURE_DISKFREE_SIZE} -o fake_diskfree=${FAKE_FREE_DISK_SIZE}" enable_content_md5 enable_noobj_cache max_stat_cache_size=100