From 8b657eee412251039b5d7d0d3ea1e728b712be6a Mon Sep 17 00:00:00 2001 From: Or Ozeri Date: Mon, 29 Jan 2018 13:19:39 +0200 Subject: [PATCH] add disk space reservation --- doc/man/s3fs.1 | 2 +- src/fdcache.cpp | 143 ++++++++++++++++++++++++++++++---------------- src/fdcache.h | 6 +- src/s3fs.cpp | 5 +- src/s3fs_util.cpp | 2 +- 5 files changed, 102 insertions(+), 56 deletions(-) diff --git a/doc/man/s3fs.1 b/doc/man/s3fs.1 index fec888c..d4b60dc 100644 --- a/doc/man/s3fs.1 +++ b/doc/man/s3fs.1 @@ -183,7 +183,7 @@ number of one part size in multipart uploading request. The default size is 10MB(10485760byte), minimum value is 5MB(5242880byte). Specify number of MB and over 5(MB). .TP -\fB\-o\fR ensure_diskfree(default the same as multipart_size value) +\fB\-o\fR ensure_diskfree(default 0) sets MB to ensure disk free space. This option means the threshold of free space size on disk which is used for the cache file by s3fs. s3fs makes file for downloading, and uploading and caching files. If the disk free space is smaller than this value, s3fs do not use diskspace as possible in exchange for the performance. diff --git a/src/fdcache.cpp b/src/fdcache.cpp index a0b5f42..b2cdf8e 100644 --- a/src/fdcache.cpp +++ b/src/fdcache.cpp @@ -1429,7 +1429,7 @@ int FdEntity::NoCacheCompleteMultipartPost(void) int FdEntity::RowFlush(const char* tpath, bool force_sync) { - int result; + int result = 0; S3FS_PRN_INFO3("[tpath=%s][path=%s][fd=%d]", SAFESTRPTR(tpath), path.c_str(), fd); @@ -1448,10 +1448,12 @@ int FdEntity::RowFlush(const char* tpath, bool force_sync) if(0 < restsize){ if(0 == upload_id.length()){ // check disk space - if(FdManager::IsSafeDiskSpace(NULL, restsize)){ + if(ReserveDiskSpace(restsize)){ // enough disk space // Load all uninitialized area - if(0 != (result = Load())){ + result = Load(); + FdManager::get()->FreeReservedDiskSpace(restsize); + if(0 != result){ S3FS_PRN_ERR("failed to upload all area(errno=%d)", result); return static_cast(result); } @@ -1554,6 +1556,32 @@ int FdEntity::RowFlush(const char* tpath, bool force_sync) return result; } +// [NOTICE] +// Need to lock before calling this method. +bool FdEntity::ReserveDiskSpace(size_t size) +{ + if(FdManager::get()->ReserveDiskSpace(size)){ + return true; + } + + if(!is_modify){ + // try to clear all cache for this fd. + pagelist.Init(pagelist.Size(), false); + if(-1 == ftruncate(fd, 0) || -1 == ftruncate(fd, pagelist.Size())){ + S3FS_PRN_ERR("failed to truncate temporary file(%d).", fd); + return false; + } + + if(FdManager::get()->ReserveDiskSpace(size)){ + return true; + } + } + + FdManager::get()->CleanupCacheDir(); + + return FdManager::get()->ReserveDiskSpace(size); +} + ssize_t FdEntity::Read(char* bytes, off_t start, size_t size, bool force_load) { S3FS_PRN_DBG("[path=%s][fd=%d][offset=%jd][size=%zu]", path.c_str(), fd, (intmax_t)start, size); @@ -1561,10 +1589,6 @@ ssize_t FdEntity::Read(char* bytes, off_t start, size_t size, bool force_load) if(-1 == fd){ return -EBADF; } - // check if not enough disk space left BEFORE locking fd - if(FdManager::IsCacheDir() && !FdManager::IsSafeDiskSpace(NULL, size)){ - FdManager::get()->CleanupCacheDir(); - } AutoLock auto_lock(&fdent_lock); if(force_load){ @@ -1575,23 +1599,6 @@ ssize_t FdEntity::Read(char* bytes, off_t start, size_t size, bool force_load) // check disk space if(0 < pagelist.GetTotalUnloadedPageSize(start, size)){ - if(!FdManager::IsSafeDiskSpace(NULL, size)){ - // [NOTE] - // If the area of this entity fd used can be released, try to do it. - // But If file data is updated, we can not even release of fd. - // Fundamentally, this method will fail as long as the disk capacity - // is not ensured. - // - if(!is_modify){ - // try to clear all cache for this fd. - pagelist.Init(pagelist.Size(), false); - if(-1 == ftruncate(fd, 0) || -1 == ftruncate(fd, pagelist.Size())){ - S3FS_PRN_ERR("failed to truncate temporary file(%d).", fd); - return -ENOSPC; - } - } - } - // load size(for prefetch) size_t load_size = size; if(static_cast(start + size) < pagelist.Size()){ @@ -1603,9 +1610,25 @@ ssize_t FdEntity::Read(char* bytes, off_t start, size_t size, bool force_load) load_size = static_cast(pagelist.Size() - start); } } + + if(!ReserveDiskSpace(load_size)){ + S3FS_PRN_WARN("could not reserve disk space for pre-fetch download"); + load_size = size; + if(!ReserveDiskSpace(load_size)){ + S3FS_PRN_ERR("could not reserve disk space for pre-fetch download"); + return -ENOSPC; + } + } + // Loading - int result; - if(0 < size && 0 != (result = Load(start, load_size))){ + int result = 0; + if(0 < size){ + result = Load(start, load_size); + } + + FdManager::get()->FreeReservedDiskSpace(load_size); + + if(0 != result){ S3FS_PRN_ERR("could not download. start(%jd), size(%zu), errno(%d)", (intmax_t)start, size, result); return -EIO; } @@ -1642,17 +1665,21 @@ ssize_t FdEntity::Write(const char* bytes, off_t start, size_t size) pagelist.SetPageLoadedStatus(static_cast(pagelist.Size()), static_cast(start) - pagelist.Size(), false); } - int result; + int result = 0; ssize_t wsize; if(0 == upload_id.length()){ // check disk space size_t restsize = pagelist.GetTotalUnloadedPageSize(0, start) + size; - if(FdManager::IsSafeDiskSpace(NULL, restsize)){ + if(ReserveDiskSpace(restsize)){ // enough disk space // Load uninitialized area which starts from 0 to (start + size) before writing. - if(0 < start && 0 != (result = Load(0, static_cast(start)))){ + if(0 < start){ + result = Load(0, static_cast(start)); + } + FdManager::get()->FreeReservedDiskSpace(restsize); + if(0 != result){ S3FS_PRN_ERR("failed to load uninitialized area before writing(errno=%d)", result); return static_cast(result); } @@ -1750,6 +1777,7 @@ void FdEntity::CleanupCache() FdManager FdManager::singleton; pthread_mutex_t FdManager::fd_manager_lock; pthread_mutex_t FdManager::cache_cleanup_lock; +pthread_mutex_t FdManager::reserved_diskspace_lock; bool FdManager::is_lock_init(false); string FdManager::cache_dir(""); bool FdManager::check_cache_dir_exist(false); @@ -1901,19 +1929,7 @@ bool FdManager::CheckCacheDirExist(void) size_t FdManager::SetEnsureFreeDiskSpace(size_t size) { size_t old = FdManager::free_disk_space; - if(0 == size){ - if(0 == FdManager::free_disk_space){ - FdManager::free_disk_space = static_cast(S3fsCurl::GetMultipartSize() * S3fsCurl::GetMaxParallelCount()); - } - }else{ - if(0 == FdManager::free_disk_space){ - FdManager::free_disk_space = max(size, static_cast(S3fsCurl::GetMultipartSize() * S3fsCurl::GetMaxParallelCount())); - }else{ - if(static_cast(S3fsCurl::GetMultipartSize() * S3fsCurl::GetMaxParallelCount()) <= size){ - FdManager::free_disk_space = size; - } - } - } + FdManager::free_disk_space = size; return old; } @@ -1957,6 +1973,7 @@ FdManager::FdManager() try{ pthread_mutex_init(&FdManager::fd_manager_lock, NULL); pthread_mutex_init(&FdManager::cache_cleanup_lock, NULL); + pthread_mutex_init(&FdManager::reserved_diskspace_lock, NULL); FdManager::is_lock_init = true; }catch(exception& e){ FdManager::is_lock_init = false; @@ -1980,6 +1997,7 @@ FdManager::~FdManager() try{ pthread_mutex_destroy(&FdManager::fd_manager_lock); pthread_mutex_destroy(&FdManager::cache_cleanup_lock); + pthread_mutex_destroy(&FdManager::reserved_diskspace_lock); }catch(exception& e){ S3FS_PRN_CRIT("failed to init mutex"); } @@ -2181,17 +2199,22 @@ bool FdManager::ChangeEntityToTempPath(FdEntity* ent, const char* path) void FdManager::CleanupCacheDir() { - if (!FdManager::IsCacheDir()) { + S3FS_PRN_INFO("cache cleanup requested"); + + if(!FdManager::IsCacheDir()){ return; } - AutoLock auto_lock(&FdManager::cache_cleanup_lock, true); + AutoLock auto_lock_no_wait(&FdManager::cache_cleanup_lock, true); - if (!auto_lock.isLockAcquired()) { - return; + if(auto_lock_no_wait.isLockAcquired()){ + S3FS_PRN_INFO("cache cleanup started"); + CleanupCacheDirInternal(""); + S3FS_PRN_INFO("cache cleanup ended"); + }else{ + // wait for other thread to finish cache cleanup + AutoLock auto_lock(&FdManager::cache_cleanup_lock); } - - CleanupCacheDirInternal(""); } void FdManager::CleanupCacheDirInternal(const std::string &path) @@ -2224,16 +2247,38 @@ void FdManager::CleanupCacheDirInternal(const std::string &path) }else{ FdEntity* ent; if(NULL == (ent = FdManager::get()->Open(next_path.c_str(), NULL, -1, -1, false, true, true))){ + S3FS_PRN_DBG("skipping locked file: %s", next_path.c_str()); continue; } - ent->CleanupCache(); + if(ent->IsMultiOpened()){ + S3FS_PRN_DBG("skipping opened file: %s", next_path.c_str()); + }else{ + ent->CleanupCache(); + S3FS_PRN_DBG("cleaned up: %s", next_path.c_str()); + } Close(ent); } } closedir(dp); } +bool FdManager::ReserveDiskSpace(size_t size) +{ + AutoLock auto_lock(&FdManager::reserved_diskspace_lock); + if(IsSafeDiskSpace(NULL, size)){ + free_disk_space += size; + return true; + } + return false; +} + +void FdManager::FreeReservedDiskSpace(size_t size) +{ + AutoLock auto_lock(&FdManager::reserved_diskspace_lock); + free_disk_space -= size; +} + /* * Local variables: * tab-width: 4 diff --git a/src/fdcache.h b/src/fdcache.h index 13f5bcf..92cccab 100644 --- a/src/fdcache.h +++ b/src/fdcache.h @@ -144,6 +144,7 @@ class FdEntity void Close(void); bool IsOpen(void) const { return (-1 != fd); } + bool IsMultiOpened(void) const { return refcnt > 1; } int Open(headers_t* pmeta = NULL, ssize_t size = -1, time_t time = -1, bool no_fd_lock_wait = false); bool OpenAndLoadAll(headers_t* pmeta = NULL, size_t* size = NULL, bool force_load = false); int Dup(bool no_fd_lock_wait = false); @@ -173,6 +174,7 @@ class FdEntity ssize_t Read(char* bytes, off_t start, size_t size, bool force_load = false); ssize_t Write(const char* bytes, off_t start, size_t size); + bool ReserveDiskSpace(size_t size); void CleanupCache(); }; typedef std::map fdent_map_t; // key=path, value=FdEntity* @@ -186,6 +188,7 @@ class FdManager static FdManager singleton; static pthread_mutex_t fd_manager_lock; static pthread_mutex_t cache_cleanup_lock; + static pthread_mutex_t reserved_diskspace_lock; static bool is_lock_init; static std::string cache_dir; static bool check_cache_dir_exist; @@ -217,8 +220,9 @@ class FdManager static size_t GetEnsureFreeDiskSpace(void) { return FdManager::free_disk_space; } static size_t SetEnsureFreeDiskSpace(size_t size); - static size_t InitEnsureFreeDiskSpace(void) { return SetEnsureFreeDiskSpace(0); } static bool IsSafeDiskSpace(const char* path, size_t size); + static void FreeReservedDiskSpace(size_t size); + bool ReserveDiskSpace(size_t size); FdEntity* GetFdEntity(const char* path, int existfd = -1); FdEntity* Open(const char* path, headers_t* pmeta = NULL, ssize_t size = -1, time_t time = -1, bool force_tmpfile = false, bool is_create = true, bool no_fd_lock_wait = false); diff --git a/src/s3fs.cpp b/src/s3fs.cpp index ab1ce02..92376d0 100644 --- a/src/s3fs.cpp +++ b/src/s3fs.cpp @@ -4671,8 +4671,6 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar S3FS_PRN_EXIT("multipart_size option must be at least 5 MB."); return -1; } - // update ensure free disk space if it is not set. - FdManager::InitEnsureFreeDiskSpace(); return 0; } if(0 == STR2NCMP(arg, "ensure_diskfree=")){ @@ -5034,8 +5032,7 @@ int main(int argc, char* argv[]) } // check free disk space - FdManager::InitEnsureFreeDiskSpace(); - if(!FdManager::IsSafeDiskSpace(NULL, S3fsCurl::GetMultipartSize())){ + if(!FdManager::IsSafeDiskSpace(NULL, S3fsCurl::GetMultipartSize() * S3fsCurl::GetMaxParallelCount())){ S3FS_PRN_EXIT("There is no enough disk space for used as cache(or temporary) directory by s3fs."); exit(EXIT_FAILURE); } diff --git a/src/s3fs_util.cpp b/src/s3fs_util.cpp index 3d05136..2c977be 100644 --- a/src/s3fs_util.cpp +++ b/src/s3fs_util.cpp @@ -1125,7 +1125,7 @@ void show_help (void) " multipart_size (default=\"10\")\n" " - part size, in MB, for each multipart request.\n" "\n" - " ensure_diskfree (default same multipart_size value)\n" + " ensure_diskfree (default 0)\n" " - sets MB to ensure disk free space. s3fs makes file for\n" " downloading, uploading and caching files. If the disk free\n" " space is smaller than this value, s3fs do not use diskspace\n"