Merge pull request #710 from orozery/disk_space_reservation

add disk space reservation

doc/man/s3fs.1

@@ -183,7 +183,7 @@ number of one part size in multipart uploading request.
 The default size is 10MB(10485760byte), minimum value is 5MB(5242880byte).
 Specify number of MB and over 5(MB).
 .TP
-\fB\-o\fR ensure_diskfree(default the same as multipart_size value)
+\fB\-o\fR ensure_diskfree(default 0)
 sets MB to ensure disk free space. This option means the threshold of free space size on disk which is used for the cache file by s3fs.
 s3fs makes file for downloading, and uploading and caching files.
 If the disk free space is smaller than this value, s3fs do not use diskspace as possible in exchange for the performance.

src/fdcache.cpp

@@ -1429,7 +1429,7 @@ int FdEntity::NoCacheCompleteMultipartPost(void)
 
 int FdEntity::RowFlush(const char* tpath, bool force_sync)
 {
-  int result;
+  int result = 0;
 
   S3FS_PRN_INFO3("[tpath=%s][path=%s][fd=%d]", SAFESTRPTR(tpath), path.c_str(), fd);
 
@@ -1448,10 +1448,12 @@ int FdEntity::RowFlush(const char* tpath, bool force_sync)
   if(0 < restsize){
     if(0 == upload_id.length()){
       // check disk space
-      if(FdManager::IsSafeDiskSpace(NULL, restsize)){
+      if(ReserveDiskSpace(restsize)){
         // enough disk space
         // Load all uninitialized area
-        if(0 != (result = Load())){
+        result = Load();
+        FdManager::get()->FreeReservedDiskSpace(restsize);
+        if(0 != result){
           S3FS_PRN_ERR("failed to upload all area(errno=%d)", result);
           return static_cast<ssize_t>(result);
         }
@@ -1554,6 +1556,32 @@ int FdEntity::RowFlush(const char* tpath, bool force_sync)
   return result;
 }
 
+// [NOTICE]
+// Need to lock before calling this method.
+bool FdEntity::ReserveDiskSpace(size_t size)
+{
+  if(FdManager::get()->ReserveDiskSpace(size)){
+    return true;
+  }
+
+  if(!is_modify){
+    // try to clear all cache for this fd.
+    pagelist.Init(pagelist.Size(), false);
+    if(-1 == ftruncate(fd, 0) || -1 == ftruncate(fd, pagelist.Size())){
+      S3FS_PRN_ERR("failed to truncate temporary file(%d).", fd);
+      return false;
+    }
+
+    if(FdManager::get()->ReserveDiskSpace(size)){
+      return true;
+    }
+  }
+
+  FdManager::get()->CleanupCacheDir();
+
+  return FdManager::get()->ReserveDiskSpace(size);
+}
+
 ssize_t FdEntity::Read(char* bytes, off_t start, size_t size, bool force_load)
 {
   S3FS_PRN_DBG("[path=%s][fd=%d][offset=%jd][size=%zu]", path.c_str(), fd, (intmax_t)start, size);
@@ -1561,10 +1589,6 @@ ssize_t FdEntity::Read(char* bytes, off_t start, size_t size, bool force_load)
   if(-1 == fd){
     return -EBADF;
   }
-  // check if not enough disk space left BEFORE locking fd
-  if(FdManager::IsCacheDir() && !FdManager::IsSafeDiskSpace(NULL, size)){
-    FdManager::get()->CleanupCacheDir();
-  }
   AutoLock auto_lock(&fdent_lock);
 
   if(force_load){
@@ -1575,23 +1599,6 @@ ssize_t FdEntity::Read(char* bytes, off_t start, size_t size, bool force_load)
 
   // check disk space
   if(0 < pagelist.GetTotalUnloadedPageSize(start, size)){
-    if(!FdManager::IsSafeDiskSpace(NULL, size)){
-      // [NOTE]
-      // If the area of this entity fd used can be released, try to do it.
-      // But If file data is updated, we can not even release of fd.
-      // Fundamentally, this method will fail as long as the disk capacity
-      // is not ensured.
-      //
-      if(!is_modify){
-        // try to clear all cache for this fd.
-        pagelist.Init(pagelist.Size(), false);
-        if(-1 == ftruncate(fd, 0) || -1 == ftruncate(fd, pagelist.Size())){
-          S3FS_PRN_ERR("failed to truncate temporary file(%d).", fd);
-          return -ENOSPC;
-        }
-      }
-    }
-
     // load size(for prefetch)
     size_t load_size = size;
     if(static_cast<size_t>(start + size) < pagelist.Size()){
@@ -1603,9 +1610,25 @@ ssize_t FdEntity::Read(char* bytes, off_t start, size_t size, bool force_load)
         load_size = static_cast<size_t>(pagelist.Size() - start);
       }
     }
+
+    if(!ReserveDiskSpace(load_size)){
+      S3FS_PRN_WARN("could not reserve disk space for pre-fetch download");
+      load_size = size;
+      if(!ReserveDiskSpace(load_size)){
+        S3FS_PRN_ERR("could not reserve disk space for pre-fetch download");
+        return -ENOSPC;
+      }
+    }
+
     // Loading
-    int result;
+    int result = 0;
-    if(0 < size && 0 != (result = Load(start, load_size))){
+    if(0 < size){
+      result = Load(start, load_size);
+    }
+
+    FdManager::get()->FreeReservedDiskSpace(load_size);
+
+    if(0 != result){
       S3FS_PRN_ERR("could not download. start(%jd), size(%zu), errno(%d)", (intmax_t)start, size, result);
       return -EIO;
     }
@@ -1642,17 +1665,21 @@ ssize_t FdEntity::Write(const char* bytes, off_t start, size_t size)
     pagelist.SetPageLoadedStatus(static_cast<off_t>(pagelist.Size()), static_cast<size_t>(start) - pagelist.Size(), false);
   }
 
-  int     result;
+  int     result = 0;
   ssize_t wsize;
 
   if(0 == upload_id.length()){
     // check disk space
     size_t restsize = pagelist.GetTotalUnloadedPageSize(0, start) + size;
-    if(FdManager::IsSafeDiskSpace(NULL, restsize)){
+    if(ReserveDiskSpace(restsize)){
       // enough disk space
 
       // Load uninitialized area which starts from 0 to (start + size) before writing.
-      if(0 < start && 0 != (result = Load(0, static_cast<size_t>(start)))){
+      if(0 < start){
+        result = Load(0, static_cast<size_t>(start));
+      }
+      FdManager::get()->FreeReservedDiskSpace(restsize);
+      if(0 != result){
         S3FS_PRN_ERR("failed to load uninitialized area before writing(errno=%d)", result);
         return static_cast<ssize_t>(result);
       }
@@ -1750,6 +1777,7 @@ void FdEntity::CleanupCache()
 FdManager       FdManager::singleton;
 pthread_mutex_t FdManager::fd_manager_lock;
 pthread_mutex_t FdManager::cache_cleanup_lock;
+pthread_mutex_t FdManager::reserved_diskspace_lock;
 bool            FdManager::is_lock_init(false);
 string          FdManager::cache_dir("");
 bool            FdManager::check_cache_dir_exist(false);
@@ -1901,19 +1929,7 @@ bool FdManager::CheckCacheDirExist(void)
 size_t FdManager::SetEnsureFreeDiskSpace(size_t size)
 {
   size_t old = FdManager::free_disk_space;
-  if(0 == size){
-    if(0 == FdManager::free_disk_space){
-      FdManager::free_disk_space = static_cast<size_t>(S3fsCurl::GetMultipartSize() * S3fsCurl::GetMaxParallelCount());
-    }
-  }else{
-    if(0 == FdManager::free_disk_space){
-      FdManager::free_disk_space = max(size, static_cast<size_t>(S3fsCurl::GetMultipartSize() * S3fsCurl::GetMaxParallelCount()));
-    }else{
-      if(static_cast<size_t>(S3fsCurl::GetMultipartSize() * S3fsCurl::GetMaxParallelCount()) <= size){
   FdManager::free_disk_space = size;
-      }
-    }
-  }
   return old;
 }
 
@@ -1957,6 +1973,7 @@ FdManager::FdManager()
     try{
       pthread_mutex_init(&FdManager::fd_manager_lock, NULL);
       pthread_mutex_init(&FdManager::cache_cleanup_lock, NULL);
+      pthread_mutex_init(&FdManager::reserved_diskspace_lock, NULL);
       FdManager::is_lock_init = true;
     }catch(exception& e){
       FdManager::is_lock_init = false;
@@ -1980,6 +1997,7 @@ FdManager::~FdManager()
       try{
         pthread_mutex_destroy(&FdManager::fd_manager_lock);
         pthread_mutex_destroy(&FdManager::cache_cleanup_lock);
+        pthread_mutex_destroy(&FdManager::reserved_diskspace_lock);
       }catch(exception& e){
         S3FS_PRN_CRIT("failed to init mutex");
       }
@@ -2181,17 +2199,22 @@ bool FdManager::ChangeEntityToTempPath(FdEntity* ent, const char* path)
 
 void FdManager::CleanupCacheDir()
 {
+  S3FS_PRN_INFO("cache cleanup requested");
+
   if(!FdManager::IsCacheDir()){
     return;
   }
 
-  AutoLock auto_lock(&FdManager::cache_cleanup_lock, true);
+  AutoLock auto_lock_no_wait(&FdManager::cache_cleanup_lock, true);
-
-  if (!auto_lock.isLockAcquired()) {
-    return;
-  }
 
+  if(auto_lock_no_wait.isLockAcquired()){
+    S3FS_PRN_INFO("cache cleanup started");
     CleanupCacheDirInternal("");
+    S3FS_PRN_INFO("cache cleanup ended");
+  }else{
+    // wait for other thread to finish cache cleanup
+    AutoLock auto_lock(&FdManager::cache_cleanup_lock);
+  }
 }
 
 void FdManager::CleanupCacheDirInternal(const std::string &path)
@@ -2224,16 +2247,38 @@ void FdManager::CleanupCacheDirInternal(const std::string &path)
     }else{
       FdEntity* ent;
       if(NULL == (ent = FdManager::get()->Open(next_path.c_str(), NULL, -1, -1, false, true, true))){
+        S3FS_PRN_DBG("skipping locked file: %s", next_path.c_str());
         continue;
       }
 
+      if(ent->IsMultiOpened()){
+        S3FS_PRN_DBG("skipping opened file: %s", next_path.c_str());
+      }else{
         ent->CleanupCache();
+        S3FS_PRN_DBG("cleaned up: %s", next_path.c_str());
+      }
       Close(ent);
     }
   }
   closedir(dp);
 }
 
+bool FdManager::ReserveDiskSpace(size_t size)
+{
+  AutoLock auto_lock(&FdManager::reserved_diskspace_lock);
+  if(IsSafeDiskSpace(NULL, size)){
+    free_disk_space += size;
+    return true;
+  }
+  return false;
+}
+
+void FdManager::FreeReservedDiskSpace(size_t size)
+{
+  AutoLock auto_lock(&FdManager::reserved_diskspace_lock);
+  free_disk_space -= size;
+}
+
 /*
 * Local variables:
 * tab-width: 4

src/fdcache.h

@@ -144,6 +144,7 @@ class FdEntity
 
     void Close(void);
     bool IsOpen(void) const { return (-1 != fd); }
+    bool IsMultiOpened(void) const { return refcnt > 1; }
     int Open(headers_t* pmeta = NULL, ssize_t size = -1, time_t time = -1, bool no_fd_lock_wait = false);
     bool OpenAndLoadAll(headers_t* pmeta = NULL, size_t* size = NULL, bool force_load = false);
     int Dup(bool no_fd_lock_wait = false);
@@ -173,6 +174,7 @@ class FdEntity
     ssize_t Read(char* bytes, off_t start, size_t size, bool force_load = false);
     ssize_t Write(const char* bytes, off_t start, size_t size);
 
+    bool ReserveDiskSpace(size_t size);
     void CleanupCache();
 };
 typedef std::map<std::string, class FdEntity*> fdent_map_t;   // key=path, value=FdEntity*
@@ -186,6 +188,7 @@ class FdManager
     static FdManager       singleton;
     static pthread_mutex_t fd_manager_lock;
     static pthread_mutex_t cache_cleanup_lock;
+    static pthread_mutex_t reserved_diskspace_lock;
     static bool            is_lock_init;
     static std::string     cache_dir;
     static bool            check_cache_dir_exist;
@@ -217,8 +220,9 @@ class FdManager
 
     static size_t GetEnsureFreeDiskSpace(void) { return FdManager::free_disk_space; }
     static size_t SetEnsureFreeDiskSpace(size_t size);
-    static size_t InitEnsureFreeDiskSpace(void) { return SetEnsureFreeDiskSpace(0); }
     static bool IsSafeDiskSpace(const char* path, size_t size);
+    static void FreeReservedDiskSpace(size_t size);
+    bool ReserveDiskSpace(size_t size);
 
     FdEntity* GetFdEntity(const char* path, int existfd = -1);
     FdEntity* Open(const char* path, headers_t* pmeta = NULL, ssize_t size = -1, time_t time = -1, bool force_tmpfile = false, bool is_create = true, bool no_fd_lock_wait = false);

src/s3fs.cpp

@@ -4671,8 +4671,6 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
         S3FS_PRN_EXIT("multipart_size option must be at least 5 MB.");
         return -1;
       }
-      // update ensure free disk space if it is not set.
-      FdManager::InitEnsureFreeDiskSpace();
       return 0;
     }
     if(0 == STR2NCMP(arg, "ensure_diskfree=")){
@@ -5034,8 +5032,7 @@ int main(int argc, char* argv[])
   }
 
   // check free disk space
-  FdManager::InitEnsureFreeDiskSpace();
+  if(!FdManager::IsSafeDiskSpace(NULL, S3fsCurl::GetMultipartSize() * S3fsCurl::GetMaxParallelCount())){
-  if(!FdManager::IsSafeDiskSpace(NULL, S3fsCurl::GetMultipartSize())){
     S3FS_PRN_EXIT("There is no enough disk space for used as cache(or temporary) directory by s3fs.");
     exit(EXIT_FAILURE);
   }

src/s3fs_util.cpp

@@ -1125,7 +1125,7 @@ void show_help (void)
     "   multipart_size (default=\"10\")\n"
     "      - part size, in MB, for each multipart request.\n"
     "\n"
-    "   ensure_diskfree (default same multipart_size value)\n"
+    "   ensure_diskfree (default 0)\n"
     "      - sets MB to ensure disk free space. s3fs makes file for\n"
     "        downloading, uploading and caching files. If the disk free\n"
     "        space is smaller than this value, s3fs do not use diskspace\n"