cleanup cache directory when running out of disk space

src/fdcache.cpp

@@ -31,6 +31,7 @@
 #include <errno.h>
 #include <string.h>
 #include <assert.h>
+#include <dirent.h>
 #include <curl/curl.h>
 #include <string>
 #include <iostream>
@@ -724,12 +725,15 @@ void FdEntity::Close(void)
   }
 }
 
-int FdEntity::Dup(void)
+int FdEntity::Dup(bool no_fd_lock_wait)
 {
   S3FS_PRN_DBG("[path=%s][fd=%d][refcnt=%d]", path.c_str(), fd, (-1 != fd ? refcnt + 1 : refcnt));
 
   if(-1 != fd){
-    AutoLock auto_lock(&fdent_lock);
+    AutoLock auto_lock(&fdent_lock, no_fd_lock_wait);
+    if (!auto_lock.isLockAcquired()) {
+      return -1;
+    }
     refcnt++;
   }
   return fd;
@@ -781,13 +785,16 @@ int FdEntity::OpenMirrorFile(void)
 // This method does not lock fdent_lock, because FdManager::fd_manager_lock
 // is locked before calling.
 //
-int FdEntity::Open(headers_t* pmeta, ssize_t size, time_t time)
+int FdEntity::Open(headers_t* pmeta, ssize_t size, time_t time, bool no_fd_lock_wait)
 {
   S3FS_PRN_DBG("[path=%s][fd=%d][size=%jd][time=%jd]", path.c_str(), fd, (intmax_t)size, (intmax_t)time);
 
   if(-1 != fd){
     // already opened, needs to increment refcnt.
-    Dup();
+    if (fd != Dup(no_fd_lock_wait)) {
+      // had to wait for fd lock, return
+      return -EIO;
+    }
 
     // check only file size(do not need to save cfs and time.
     if(0 <= size && pagelist.Size() != static_cast<size_t>(size)){
@@ -1545,6 +1552,10 @@ ssize_t FdEntity::Read(char* bytes, off_t start, size_t size, bool force_load)
   if(-1 == fd){
     return -EBADF;
   }
+  // check if not enough disk space left BEFORE locking fd
+  if(FdManager::IsCacheDir() && !FdManager::IsSafeDiskSpace(NULL, size)){
+    FdManager::get()->CleanupCacheDir();
+  }
   AutoLock auto_lock(&fdent_lock);
 
   if(force_load){
@@ -1605,6 +1616,10 @@ ssize_t FdEntity::Write(const char* bytes, off_t start, size_t size)
   if(-1 == fd){
     return -EBADF;
   }
+  // check if not enough disk space left BEFORE locking fd
+  if(FdManager::IsCacheDir() && !FdManager::IsSafeDiskSpace(NULL, size)){
+    FdManager::get()->CleanupCacheDir();
+  }
   AutoLock auto_lock(&fdent_lock);
 
   // check file size
@@ -1686,6 +1701,22 @@ ssize_t FdEntity::Write(const char* bytes, off_t start, size_t size)
   return wsize;
 }
 
+void FdEntity::CleanupCache()
+{
+  AutoLock auto_lock(&fdent_lock, true);
+
+  if (!auto_lock.isLockAcquired()) {
+    return;
+  }
+
+  if (is_modify) {
+    // cache is not commited to s3, cannot cleanup
+    return;
+  }
+
+  FdManager::DeleteCacheFile(path.c_str());
+}
+
 //------------------------------------------------
 // FdManager symbol
 //------------------------------------------------
@@ -1709,6 +1740,7 @@ ssize_t FdEntity::Write(const char* bytes, off_t start, size_t size)
 //------------------------------------------------
 FdManager       FdManager::singleton;
 pthread_mutex_t FdManager::fd_manager_lock;
+pthread_mutex_t FdManager::cache_cleanup_lock;
 bool            FdManager::is_lock_init(false);
 string          FdManager::cache_dir("");
 size_t          FdManager::free_disk_space = 0;
@@ -1892,6 +1924,7 @@ FdManager::FdManager()
   if(this == FdManager::get()){
     try{
       pthread_mutex_init(&FdManager::fd_manager_lock, NULL);
+      pthread_mutex_init(&FdManager::cache_cleanup_lock, NULL);
       FdManager::is_lock_init = true;
     }catch(exception& e){
       FdManager::is_lock_init = false;
@@ -1914,6 +1947,7 @@ FdManager::~FdManager()
     if(FdManager::is_lock_init){
       try{
         pthread_mutex_destroy(&FdManager::fd_manager_lock);
+        pthread_mutex_destroy(&FdManager::cache_cleanup_lock);
       }catch(exception& e){
         S3FS_PRN_CRIT("failed to init mutex");
       }
@@ -1954,7 +1988,7 @@ FdEntity* FdManager::GetFdEntity(const char* path, int existfd)
   return NULL;
 }
 
-FdEntity* FdManager::Open(const char* path, headers_t* pmeta, ssize_t size, time_t time, bool force_tmpfile, bool is_create)
+FdEntity* FdManager::Open(const char* path, headers_t* pmeta, ssize_t size, time_t time, bool force_tmpfile, bool is_create, bool no_fd_lock_wait)
 {
   S3FS_PRN_DBG("[path=%s][size=%jd][time=%jd]", SAFESTRPTR(path), (intmax_t)size, (intmax_t)time);
 
@@ -2014,7 +2048,7 @@ FdEntity* FdManager::Open(const char* path, headers_t* pmeta, ssize_t size, time
   }
 
   // open
-  if(-1 == ent->Open(pmeta, size, time)){
+  if(0 != ent->Open(pmeta, size, time, no_fd_lock_wait)){
     return NULL;
   }
   return ent;
@@ -2112,6 +2146,61 @@ bool FdManager::ChangeEntityToTempPath(FdEntity* ent, const char* path)
   return false;
 }
 
+void FdManager::CleanupCacheDir()
+{
+  if (!FdManager::IsCacheDir()) {
+    return;
+  }
+
+  AutoLock auto_lock(&FdManager::cache_cleanup_lock, true);
+
+  if (!auto_lock.isLockAcquired()) {
+    return;
+  }
+
+  CleanupCacheDirInternal("");
+}
+
+void FdManager::CleanupCacheDirInternal(const std::string &path)
+{
+  DIR*           dp;
+  struct dirent* dent;
+  std::string    abs_path = cache_dir + "/" + bucket + path;
+
+  if(NULL == (dp = opendir(abs_path.c_str()))){
+    S3FS_PRN_ERR("could not open cache dir(%s) - errno(%d)", abs_path.c_str(), errno);
+    return;
+  }
+
+  for(dent = readdir(dp); dent; dent = readdir(dp)){
+    if(0 == strcmp(dent->d_name, "..") || 0 == strcmp(dent->d_name, ".")){
+      continue;
+    }
+    string   fullpath = abs_path;
+    fullpath         += "/";
+    fullpath         += dent->d_name;
+    struct stat st;
+    if(0 != lstat(fullpath.c_str(), &st)){
+      S3FS_PRN_ERR("could not get stats of file(%s) - errno(%d)", fullpath.c_str(), errno);
+      closedir(dp);
+      return;
+    }
+    string next_path = path + "/" + dent->d_name;
+    if(S_ISDIR(st.st_mode)){
+      CleanupCacheDirInternal(next_path);
+    }else{
+      FdEntity* ent;
+      if(NULL == (ent = FdManager::get()->Open(next_path.c_str(), NULL, -1, -1, false, true, true))){
+        continue;
+      }
+
+      ent->CleanupCache();
+      Close(ent);
+    }
+  }
+  closedir(dp);
+}
+
 /*
 * Local variables:
 * tab-width: 4

src/fdcache.h

@@ -144,9 +144,9 @@ class FdEntity
 
     void Close(void);
     bool IsOpen(void) const { return (-1 != fd); }
-    int Open(headers_t* pmeta = NULL, ssize_t size = -1, time_t time = -1);
+    int Open(headers_t* pmeta = NULL, ssize_t size = -1, time_t time = -1, bool no_fd_lock_wait = false);
     bool OpenAndLoadAll(headers_t* pmeta = NULL, size_t* size = NULL, bool force_load = false);
-    int Dup(void);
+    int Dup(bool no_fd_lock_wait = false);
 
     const char* GetPath(void) const { return path.c_str(); }
     void SetPath(const std::string &newpath) { path = newpath; }
@@ -172,6 +172,8 @@ class FdEntity
 
     ssize_t Read(char* bytes, off_t start, size_t size, bool force_load = false);
     ssize_t Write(const char* bytes, off_t start, size_t size);
+
+    void CleanupCache();
 };
 typedef std::map<std::string, class FdEntity*> fdent_map_t;   // key=path, value=FdEntity*
 
@@ -183,6 +185,7 @@ class FdManager
   private:
     static FdManager       singleton;
     static pthread_mutex_t fd_manager_lock;
+    static pthread_mutex_t cache_cleanup_lock;
     static bool            is_lock_init;
     static std::string     cache_dir;
     static size_t          free_disk_space; // limit free disk space
@@ -191,6 +194,7 @@ class FdManager
 
   private:
     static fsblkcnt_t GetFreeDiskSpace(const char* path);
+    void CleanupCacheDirInternal(const std::string &path = "");
 
   public:
     FdManager();
@@ -214,11 +218,12 @@ class FdManager
     static bool IsSafeDiskSpace(const char* path, size_t size);
 
     FdEntity* GetFdEntity(const char* path, int existfd = -1);
-    FdEntity* Open(const char* path, headers_t* pmeta = NULL, ssize_t size = -1, time_t time = -1, bool force_tmpfile = false, bool is_create = true);
+    FdEntity* Open(const char* path, headers_t* pmeta = NULL, ssize_t size = -1, time_t time = -1, bool force_tmpfile = false, bool is_create = true, bool no_fd_lock_wait = false);
     FdEntity* ExistOpen(const char* path, int existfd = -1, bool ignore_existfd = false);
     void Rename(const std::string &from, const std::string &to);
     bool Close(FdEntity* ent);
     bool ChangeEntityToTempPath(FdEntity* ent, const char* path);
+    void CleanupCacheDir();
 };
 
 #endif // FD_CACHE_H_

src/s3fs_util.cpp

@@ -425,14 +425,25 @@ void free_mvnodes(MVNODE *head)
 //-------------------------------------------------------------------
 // Class AutoLock
 //-------------------------------------------------------------------
-AutoLock::AutoLock(pthread_mutex_t* pmutex) : auto_mutex(pmutex)
+AutoLock::AutoLock(pthread_mutex_t* pmutex, bool no_wait) : auto_mutex(pmutex)
 {
-  pthread_mutex_lock(auto_mutex);
+  if (no_wait) {
+    is_lock_acquired = pthread_mutex_trylock(auto_mutex) == 0;
+  } else {
+    is_lock_acquired = pthread_mutex_lock(auto_mutex) == 0;
+  }
+}
+
+bool AutoLock::isLockAcquired() const
+{
+  return is_lock_acquired;
 }
 
 AutoLock::~AutoLock()
 {
-  pthread_mutex_unlock(auto_mutex);
+  if (is_lock_acquired) {
+    pthread_mutex_unlock(auto_mutex);
+  }
 }
 
 //-------------------------------------------------------------------

src/s3fs_util.h

@@ -88,9 +88,11 @@ class AutoLock
 {
   private:
     pthread_mutex_t* auto_mutex;
+    bool is_lock_acquired;
 
   public:
-    explicit AutoLock(pthread_mutex_t* pmutex);
+    explicit AutoLock(pthread_mutex_t* pmutex, bool no_wait = false);
+    bool isLockAcquired() const;
     ~AutoLock();
 };