From 95578cad4355470154c25c60e2a4164f0e020e6b Mon Sep 17 00:00:00 2001 From: Or Ozeri Date: Sun, 2 Apr 2017 10:22:12 +0300 Subject: [PATCH] cleanup cache directory when running out of disk space --- src/fdcache.cpp | 101 +++++++++++++++++++++++++++++++++++++++++++--- src/fdcache.h | 11 +++-- src/s3fs_util.cpp | 17 ++++++-- src/s3fs_util.h | 4 +- 4 files changed, 120 insertions(+), 13 deletions(-) diff --git a/src/fdcache.cpp b/src/fdcache.cpp index 453599d..24b6266 100644 --- a/src/fdcache.cpp +++ b/src/fdcache.cpp @@ -31,6 +31,7 @@ #include #include #include +#include #include #include #include @@ -724,12 +725,15 @@ void FdEntity::Close(void) } } -int FdEntity::Dup(void) +int FdEntity::Dup(bool no_fd_lock_wait) { S3FS_PRN_DBG("[path=%s][fd=%d][refcnt=%d]", path.c_str(), fd, (-1 != fd ? refcnt + 1 : refcnt)); if(-1 != fd){ - AutoLock auto_lock(&fdent_lock); + AutoLock auto_lock(&fdent_lock, no_fd_lock_wait); + if (!auto_lock.isLockAcquired()) { + return -1; + } refcnt++; } return fd; @@ -781,13 +785,16 @@ int FdEntity::OpenMirrorFile(void) // This method does not lock fdent_lock, because FdManager::fd_manager_lock // is locked before calling. // -int FdEntity::Open(headers_t* pmeta, ssize_t size, time_t time) +int FdEntity::Open(headers_t* pmeta, ssize_t size, time_t time, bool no_fd_lock_wait) { S3FS_PRN_DBG("[path=%s][fd=%d][size=%jd][time=%jd]", path.c_str(), fd, (intmax_t)size, (intmax_t)time); if(-1 != fd){ // already opened, needs to increment refcnt. - Dup(); + if (fd != Dup(no_fd_lock_wait)) { + // had to wait for fd lock, return + return -EIO; + } // check only file size(do not need to save cfs and time. if(0 <= size && pagelist.Size() != static_cast(size)){ @@ -1545,6 +1552,10 @@ ssize_t FdEntity::Read(char* bytes, off_t start, size_t size, bool force_load) if(-1 == fd){ return -EBADF; } + // check if not enough disk space left BEFORE locking fd + if(FdManager::IsCacheDir() && !FdManager::IsSafeDiskSpace(NULL, size)){ + FdManager::get()->CleanupCacheDir(); + } AutoLock auto_lock(&fdent_lock); if(force_load){ @@ -1605,6 +1616,10 @@ ssize_t FdEntity::Write(const char* bytes, off_t start, size_t size) if(-1 == fd){ return -EBADF; } + // check if not enough disk space left BEFORE locking fd + if(FdManager::IsCacheDir() && !FdManager::IsSafeDiskSpace(NULL, size)){ + FdManager::get()->CleanupCacheDir(); + } AutoLock auto_lock(&fdent_lock); // check file size @@ -1686,6 +1701,22 @@ ssize_t FdEntity::Write(const char* bytes, off_t start, size_t size) return wsize; } +void FdEntity::CleanupCache() +{ + AutoLock auto_lock(&fdent_lock, true); + + if (!auto_lock.isLockAcquired()) { + return; + } + + if (is_modify) { + // cache is not commited to s3, cannot cleanup + return; + } + + FdManager::DeleteCacheFile(path.c_str()); +} + //------------------------------------------------ // FdManager symbol //------------------------------------------------ @@ -1709,6 +1740,7 @@ ssize_t FdEntity::Write(const char* bytes, off_t start, size_t size) //------------------------------------------------ FdManager FdManager::singleton; pthread_mutex_t FdManager::fd_manager_lock; +pthread_mutex_t FdManager::cache_cleanup_lock; bool FdManager::is_lock_init(false); string FdManager::cache_dir(""); size_t FdManager::free_disk_space = 0; @@ -1892,6 +1924,7 @@ FdManager::FdManager() if(this == FdManager::get()){ try{ pthread_mutex_init(&FdManager::fd_manager_lock, NULL); + pthread_mutex_init(&FdManager::cache_cleanup_lock, NULL); FdManager::is_lock_init = true; }catch(exception& e){ FdManager::is_lock_init = false; @@ -1914,6 +1947,7 @@ FdManager::~FdManager() if(FdManager::is_lock_init){ try{ pthread_mutex_destroy(&FdManager::fd_manager_lock); + pthread_mutex_destroy(&FdManager::cache_cleanup_lock); }catch(exception& e){ S3FS_PRN_CRIT("failed to init mutex"); } @@ -1954,7 +1988,7 @@ FdEntity* FdManager::GetFdEntity(const char* path, int existfd) return NULL; } -FdEntity* FdManager::Open(const char* path, headers_t* pmeta, ssize_t size, time_t time, bool force_tmpfile, bool is_create) +FdEntity* FdManager::Open(const char* path, headers_t* pmeta, ssize_t size, time_t time, bool force_tmpfile, bool is_create, bool no_fd_lock_wait) { S3FS_PRN_DBG("[path=%s][size=%jd][time=%jd]", SAFESTRPTR(path), (intmax_t)size, (intmax_t)time); @@ -2014,7 +2048,7 @@ FdEntity* FdManager::Open(const char* path, headers_t* pmeta, ssize_t size, time } // open - if(-1 == ent->Open(pmeta, size, time)){ + if(0 != ent->Open(pmeta, size, time, no_fd_lock_wait)){ return NULL; } return ent; @@ -2112,6 +2146,61 @@ bool FdManager::ChangeEntityToTempPath(FdEntity* ent, const char* path) return false; } +void FdManager::CleanupCacheDir() +{ + if (!FdManager::IsCacheDir()) { + return; + } + + AutoLock auto_lock(&FdManager::cache_cleanup_lock, true); + + if (!auto_lock.isLockAcquired()) { + return; + } + + CleanupCacheDirInternal(""); +} + +void FdManager::CleanupCacheDirInternal(const std::string &path) +{ + DIR* dp; + struct dirent* dent; + std::string abs_path = cache_dir + "/" + bucket + path; + + if(NULL == (dp = opendir(abs_path.c_str()))){ + S3FS_PRN_ERR("could not open cache dir(%s) - errno(%d)", abs_path.c_str(), errno); + return; + } + + for(dent = readdir(dp); dent; dent = readdir(dp)){ + if(0 == strcmp(dent->d_name, "..") || 0 == strcmp(dent->d_name, ".")){ + continue; + } + string fullpath = abs_path; + fullpath += "/"; + fullpath += dent->d_name; + struct stat st; + if(0 != lstat(fullpath.c_str(), &st)){ + S3FS_PRN_ERR("could not get stats of file(%s) - errno(%d)", fullpath.c_str(), errno); + closedir(dp); + return; + } + string next_path = path + "/" + dent->d_name; + if(S_ISDIR(st.st_mode)){ + CleanupCacheDirInternal(next_path); + }else{ + FdEntity* ent; + if(NULL == (ent = FdManager::get()->Open(next_path.c_str(), NULL, -1, -1, false, true, true))){ + continue; + } + + ent->CleanupCache(); + Close(ent); + } + } + closedir(dp); +} + /* * Local variables: * tab-width: 4 diff --git a/src/fdcache.h b/src/fdcache.h index 2c5c9a3..7cd6f22 100644 --- a/src/fdcache.h +++ b/src/fdcache.h @@ -144,9 +144,9 @@ class FdEntity void Close(void); bool IsOpen(void) const { return (-1 != fd); } - int Open(headers_t* pmeta = NULL, ssize_t size = -1, time_t time = -1); + int Open(headers_t* pmeta = NULL, ssize_t size = -1, time_t time = -1, bool no_fd_lock_wait = false); bool OpenAndLoadAll(headers_t* pmeta = NULL, size_t* size = NULL, bool force_load = false); - int Dup(void); + int Dup(bool no_fd_lock_wait = false); const char* GetPath(void) const { return path.c_str(); } void SetPath(const std::string &newpath) { path = newpath; } @@ -172,6 +172,8 @@ class FdEntity ssize_t Read(char* bytes, off_t start, size_t size, bool force_load = false); ssize_t Write(const char* bytes, off_t start, size_t size); + + void CleanupCache(); }; typedef std::map fdent_map_t; // key=path, value=FdEntity* @@ -183,6 +185,7 @@ class FdManager private: static FdManager singleton; static pthread_mutex_t fd_manager_lock; + static pthread_mutex_t cache_cleanup_lock; static bool is_lock_init; static std::string cache_dir; static size_t free_disk_space; // limit free disk space @@ -191,6 +194,7 @@ class FdManager private: static fsblkcnt_t GetFreeDiskSpace(const char* path); + void CleanupCacheDirInternal(const std::string &path = ""); public: FdManager(); @@ -214,11 +218,12 @@ class FdManager static bool IsSafeDiskSpace(const char* path, size_t size); FdEntity* GetFdEntity(const char* path, int existfd = -1); - FdEntity* Open(const char* path, headers_t* pmeta = NULL, ssize_t size = -1, time_t time = -1, bool force_tmpfile = false, bool is_create = true); + FdEntity* Open(const char* path, headers_t* pmeta = NULL, ssize_t size = -1, time_t time = -1, bool force_tmpfile = false, bool is_create = true, bool no_fd_lock_wait = false); FdEntity* ExistOpen(const char* path, int existfd = -1, bool ignore_existfd = false); void Rename(const std::string &from, const std::string &to); bool Close(FdEntity* ent); bool ChangeEntityToTempPath(FdEntity* ent, const char* path); + void CleanupCacheDir(); }; #endif // FD_CACHE_H_ diff --git a/src/s3fs_util.cpp b/src/s3fs_util.cpp index 2915861..abdf098 100644 --- a/src/s3fs_util.cpp +++ b/src/s3fs_util.cpp @@ -425,14 +425,25 @@ void free_mvnodes(MVNODE *head) //------------------------------------------------------------------- // Class AutoLock //------------------------------------------------------------------- -AutoLock::AutoLock(pthread_mutex_t* pmutex) : auto_mutex(pmutex) +AutoLock::AutoLock(pthread_mutex_t* pmutex, bool no_wait) : auto_mutex(pmutex) { - pthread_mutex_lock(auto_mutex); + if (no_wait) { + is_lock_acquired = pthread_mutex_trylock(auto_mutex) == 0; + } else { + is_lock_acquired = pthread_mutex_lock(auto_mutex) == 0; + } +} + +bool AutoLock::isLockAcquired() const +{ + return is_lock_acquired; } AutoLock::~AutoLock() { - pthread_mutex_unlock(auto_mutex); + if (is_lock_acquired) { + pthread_mutex_unlock(auto_mutex); + } } //------------------------------------------------------------------- diff --git a/src/s3fs_util.h b/src/s3fs_util.h index f16e31a..ace782d 100644 --- a/src/s3fs_util.h +++ b/src/s3fs_util.h @@ -88,9 +88,11 @@ class AutoLock { private: pthread_mutex_t* auto_mutex; + bool is_lock_acquired; public: - explicit AutoLock(pthread_mutex_t* pmutex); + explicit AutoLock(pthread_mutex_t* pmutex, bool no_wait = false); + bool isLockAcquired() const; ~AutoLock(); };