Fixed bugs about a object larger than free disk space

doc/man/s3fs.1

@@ -161,8 +161,8 @@ It is necessary to set this value depending on a CPU and a network band.
 .TP
 \fB\-o\fR multipart_size(default="10"(10MB))
 number of one part size in multipart uploading request.
-The default size is 10MB(10485760byte), this value is minimum size.
+The default size is 10MB(10485760byte), minimum value is 5MB(5242880byte).
-Specify number of MB and over 10(MB).
+Specify number of MB and over 5(MB).
 .TP
 \fB\-o\fR ensure_diskfree(default the same as multipart_size value)
 sets MB to ensure disk free space. This option means the threshold of free space size on disk which is used for the cache file by s3fs.

src/curl.cpp

@@ -1025,7 +1025,7 @@ string S3fsCurl::SetIAMRole(const char* role)
 bool S3fsCurl::SetMultipartSize(off_t size)
 {
   size = size * 1024 * 1024;
-  if(size < MULTIPART_SIZE){
+  if(size < MIN_MULTIPART_SIZE){
     return false;
   }
   S3fsCurl::multipart_size = size;

src/curl.h

@@ -20,6 +20,11 @@
 #ifndef S3FS_CURL_H_
 #define S3FS_CURL_H_
 
+//----------------------------------------------
+// Symbols
+//----------------------------------------------
+#define MIN_MULTIPART_SIZE          5242880           // 5MB
+
 //----------------------------------------------
 // class BodyData
 //----------------------------------------------

src/fdcache.cpp

@@ -317,17 +317,15 @@ bool PageList::Resize(size_t size, bool is_loaded)
 
   }else if(size < total){
     // cut area
-    fdpage_list_t::iterator iter;
+    for(fdpage_list_t::iterator iter = pages.begin(); iter != pages.end(); ){
-    for(fdpage_list_t::reverse_iterator riter = pages.rbegin(); riter != pages.rend(); ){
+      if(static_cast<size_t>((*iter)->next()) <= size){
-      if(size < static_cast<size_t>((*riter)->offset)){
+        ++iter;
-        // cut over area
+      }else{
-        iter = riter.base();
+        if(size <= static_cast<size_t>((*iter)->offset)){
-        ++riter;
+          iter = pages.erase(iter);
-        pages.erase(iter);
+        }else{
-      }else{    // size < static_cast<size_t>((*riter)->offset + (*riter)->bytes)
+          (*iter)->bytes = size - static_cast<size_t>((*iter)->offset);
-        // change size of last area
+        }
-        (*riter)->bytes = size - static_cast<size_t>((*riter)->offset);
-        break;
       }
     }
   }else{    // total == size
@@ -408,45 +406,62 @@ bool PageList::FindUnloadedPage(off_t start, off_t& resstart, size_t& ressize) c
 size_t PageList::GetTotalUnloadedPageSize(off_t start, size_t size) const
 {
   size_t restsize = 0;
+  off_t  next     = static_cast<off_t>(start + size);
   for(fdpage_list_t::const_iterator iter = pages.begin(); iter != pages.end(); ++iter){
-    if(start <= (*iter)->end()){
+    if((*iter)->next() <= start){
-      if(0 != size && static_cast<size_t>(start + size) <= static_cast<size_t>((*iter)->offset)){
+      continue;
-        // reach to end
+    }
+    if(next <= (*iter)->offset){
       break;
     }
-      // after start pos
+    if((*iter)->loaded){
-      if(!(*iter)->loaded){
+      continue;
-        // found unloadedialized area
+    }
-        restsize += (*iter)->bytes;
+    size_t tmpsize;
+    if((*iter)->offset <= start){
+      if((*iter)->next() <= next){
+        tmpsize = static_cast<size_t>((*iter)->next() - start);
+      }else{
+        tmpsize = static_cast<size_t>(next - start);                         // = size
+      }
+    }else{
+      if((*iter)->next() <= next){
+        tmpsize = static_cast<size_t>((*iter)->next() - (*iter)->offset);   // = (*iter)->bytes
+      }else{
+        tmpsize = static_cast<size_t>(next - (*iter)->offset);
       }
     }
+    restsize += tmpsize;
   }
   return restsize;
 }
 
 int PageList::GetUnloadedPages(fdpage_list_t& unloaded_list, off_t start, size_t size) const
 {
-  for(fdpage_list_t::const_iterator iter = pages.begin(); iter != pages.end(); ++iter){
+  // If size is 0, it means loading to end.
-    if(start < (*iter)->offset){
+  if(0 == size){
-      continue;
+    if(static_cast<size_t>(start) < Size()){
+      size = static_cast<size_t>(Size() - start);
     }
+  }
+  off_t next = static_cast<off_t>(start + size);
+
+  for(fdpage_list_t::const_iterator iter = pages.begin(); iter != pages.end(); ++iter){
     if((*iter)->next() <= start){
       continue;
     }
-    if(0 != size && static_cast<size_t>(start + size) <= static_cast<size_t>((*iter)->offset)){
+    if(next <= (*iter)->offset){
-      break;    // reach end
+      break;
     }
     if((*iter)->loaded){
       continue; // already loaded
     }
+
     // page area
     off_t  page_start = max((*iter)->offset, start);
-    size_t page_size;
+    off_t  page_next  = min((*iter)->next(), next);
-    if(0 == size || (*iter)->next() < static_cast<off_t>(start + size)){
+    size_t page_size  = static_cast<size_t>(page_next - page_start);
-      page_size = static_cast<size_t>((*iter)->next() - page_start);
+
-    }else{
-      page_size = static_cast<size_t>((start + size) - page_start);
-    }
     // add list
     fdpage_list_t::reverse_iterator riter = unloaded_list.rbegin();
     if(riter != unloaded_list.rend() && (*riter)->next() == page_start){
@@ -1096,6 +1111,22 @@ int FdEntity::NoCacheLoadAndPost(off_t start, size_t size)
       off_t offset    = (*iter)->offset + totalread;
       oneread         = min(((*iter)->bytes - totalread), static_cast<size_t>(S3fsCurl::GetMultipartSize()));
 
+      // check rest size is over minimum part size
+      //
+      // [NOTE]
+      // If the final part size is smaller than 5MB, it is not allowed by S3 API.
+      // For this case, if the previous part of the final part is not over 5GB,
+      // we incorporate the final part to the previous part. If the previous part
+      // is over 5GB, we want to even out the last part and the previous part.
+      //
+      if(((*iter)->bytes - totalread - oneread) < MIN_MULTIPART_SIZE){
+        if(FIVE_GB < ((*iter)->bytes - totalread)){
+          oneread = ((*iter)->bytes - totalread) / 2;
+        }else{
+          oneread = ((*iter)->bytes - totalread);
+        }
+      }
+
       if(!(*iter)->loaded){
         //
         // loading or initializing
@@ -1405,22 +1436,24 @@ ssize_t FdEntity::Read(char* bytes, off_t start, size_t size, bool force_load)
         }
       }
     }
-  }
+
     // load size(for prefetch)
     size_t load_size = size;
     if(static_cast<size_t>(start + size) < pagelist.Size()){
-    if(static_cast<size_t>(start + S3fsCurl::GetMultipartSize()) < pagelist.Size()){
+      size_t prefetch_max_size = max(size, static_cast<size_t>(S3fsCurl::GetMultipartSize()));
-      load_size = S3fsCurl::GetMultipartSize();
+
+      if(static_cast<size_t>(start + prefetch_max_size) < pagelist.Size()){
+        load_size = prefetch_max_size;
       }else{
         load_size = static_cast<size_t>(pagelist.Size() - start);
       }
     }
-
     // Loading
     if(0 < size && 0 != (result = Load(start, load_size))){
       S3FS_PRN_ERR("could not download. start(%jd), size(%zu), errno(%d)", (intmax_t)start, size, result);
       return -EIO;
     }
+  }
   // Reading
   if(-1 == (rsize = pread(fd, bytes, size, start))){
     S3FS_PRN_ERR("pread failed. errno(%d)", errno);
@@ -1446,6 +1479,7 @@ ssize_t FdEntity::Write(const char* bytes, off_t start, size_t size)
     size_t restsize = pagelist.GetTotalUnloadedPageSize(0, start) + size;
     if(FdManager::IsSafeDiskSpace(NULL, restsize)){
       // enough disk space
+
       // Load unitialized area which starts from 0 to (start + size) before writing.
       if(0 < start && 0 != (result = Load(0, static_cast<size_t>(start)))){
         S3FS_PRN_ERR("failed to load uninitialized area before writing(errno=%d)", result);
@@ -1641,9 +1675,17 @@ size_t FdManager::SetEnsureFreeDiskSpace(size_t size)
 {
   size_t old = FdManager::free_disk_space;
   if(0 == size){
+    if(0 == FdManager::free_disk_space){
       FdManager::free_disk_space = static_cast<size_t>(S3fsCurl::GetMultipartSize());
+    }
   }else{
+    if(0 == FdManager::free_disk_space){
       FdManager::free_disk_space = max(size, static_cast<size_t>(S3fsCurl::GetMultipartSize()));
+    }else{
+      if(static_cast<size_t>(S3fsCurl::GetMultipartSize()) <= size){
+        FdManager::free_disk_space = size;
+      }
+    }
   }
   return old;
 }

src/fdcache.h

@@ -206,7 +206,6 @@ class FdManager
 
     static size_t GetEnsureFreeDiskSpace(void) { return FdManager::free_disk_space; }
     static size_t SetEnsureFreeDiskSpace(size_t size);
-    static size_t SetEnsureFreeDiskSpace(void) { return FdManager::SetEnsureFreeDiskSpace(FdManager::free_disk_space); }
     static size_t InitEnsureFreeDiskSpace(void) { return FdManager::SetEnsureFreeDiskSpace(0); }
     static bool IsSafeDiskSpace(const char* path, size_t size);
 

src/s3fs.cpp

@@ -4469,11 +4469,11 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
     if(0 == STR2NCMP(arg, "multipart_size=")){
       off_t size = static_cast<off_t>(s3fs_strtoofft(strchr(arg, '=') + sizeof(char)));
       if(!S3fsCurl::SetMultipartSize(size)){
-        S3FS_PRN_EXIT("multipart_size option must be at least 10 MB.");
+        S3FS_PRN_EXIT("multipart_size option must be at least 5 MB.");
         return -1;
       }
-      // update ensure free disk space
+      // update ensure free disk space if it is not set.
-      FdManager::SetEnsureFreeDiskSpace();
+      FdManager::InitEnsureFreeDiskSpace();
       return 0;
     }
     if(0 == STR2NCMP(arg, "ensure_diskfree=")){