Merge pull request #138 from s3fs-fuse/issue#97

Fixed bugs, not turn use_cache off and ty to load to end - issue#97

src/fdcache.cpp

@@ -19,6 +19,7 @@
  */
 
 #include <stdio.h>
+#include <stdlib.h>
 #include <sys/stat.h>
 #include <sys/types.h>
 #include <sys/time.h>
@@ -51,7 +52,7 @@ using namespace std;
 // Symbols
 //------------------------------------------------
 #define MAX_MULTIPART_CNT   10000                   // S3 multipart max count
-#define	FDPAGE_SIZE	    (50 * 1024 * 1024)      // 50MB(parallel uploading is 5 parallel(default) * 10 MB)
+#define FDPAGE_SIZE         (50 * 1024 * 1024)      // 50MB(parallel uploading is 5 parallel(default) * 10 MB)
 
 //------------------------------------------------
 // CacheFileStat class methods
@@ -344,10 +345,14 @@ bool PageList::FindUninitPage(off_t start, off_t& resstart, size_t& ressize)
   return false;
 }
 
-int PageList::GetUninitPages(fdpage_list_t& uninit_list, off_t start)
+int PageList::GetUninitPages(fdpage_list_t& uninit_list, off_t start, off_t size)
 {
   for(fdpage_list_t::iterator iter = pages.begin(); iter != pages.end(); iter++){
     if(start <= (*iter)->end()){
+      if((start + size) <= (*iter)->offset){
+        // reach to end
+        break;
+      }
       // after start pos
       if(!(*iter)->init){
         // found uninitialized area
@@ -790,7 +795,7 @@ int FdEntity::Load(off_t start, off_t size)
 
   // check loaded area & load
   fdpage_list_t uninit_list;
-  if(0 < pagelist.GetUninitPages(uninit_list, start)){
+  if(0 < pagelist.GetUninitPages(uninit_list, start, size)){
     for(fdpage_list_t::iterator iter = uninit_list.begin(); iter != uninit_list.end(); iter++){
       if(-1 != size && (start + size) <= (*iter)->offset){
         break;
@@ -989,6 +994,24 @@ ssize_t FdEntity::Write(const char* bytes, off_t start, size_t size)
   return wsize;
 }
 
+//------------------------------------------------
+// FdManager symbol
+//------------------------------------------------
+// [NOTE]
+// NOCACHE_PATH_PREFIX symbol needs for not using cache mode.
+// Now s3fs I/F functions in s3fs.cpp has left the processing
+// to FdManager and FdEntity class. FdManager class manages
+// the list of local file stat and file discriptor in conjunction
+// with the FdEntity class.
+// When s3fs is not using local cache, it means FdManager must
+// return new temporary file discriptor at each opening it.
+// Then FdManager caches fd by key which is dummy file path
+// instead of real file path.
+// This process may not be complete, but it is easy way can
+// be realized.
+//
+#define NOCACHE_PATH_PREFIX_FORM    " __S3FS_UNEXISTED_PATH_%lx__ / "      // important space words for simply
+
 //------------------------------------------------
 // FdManager class valiable
 //------------------------------------------------
@@ -1082,6 +1105,16 @@ bool FdManager::MakeCachePath(const char* path, string& cache_path, bool is_crea
   return true;
 }
 
+bool FdManager::MakeRandomTempPath(const char* path, string& tmppath)
+{
+  char szBuff[64];
+
+  sprintf(szBuff, NOCACHE_PATH_PREFIX_FORM, random());     // warry for performance, but maybe don't warry.
+  tmppath  = szBuff;
+  tmppath += path ? path : "";
+  return true;
+}
+
 //------------------------------------------------
 // FdManager methods
 //------------------------------------------------
@@ -1122,9 +1155,9 @@ FdManager::~FdManager()
   }
 }
 
-FdEntity* FdManager::GetFdEntity(const char* path)
+FdEntity* FdManager::GetFdEntity(const char* path, int existfd)
 {
-  FPRNINFO("[path=%s]", SAFESTRPTR(path));
+  FPRNINFO("[path=%s][fd=%d]", SAFESTRPTR(path), existfd);
 
   if(!path || '\0' == path[0]){
     return NULL;
@@ -1132,10 +1165,24 @@ FdEntity* FdManager::GetFdEntity(const char* path)
   AutoLock auto_lock(&FdManager::fd_manager_lock);
 
   fdent_map_t::iterator iter = fent.find(string(path));
-  if(fent.end() == iter){
+  if(fent.end() != iter && (-1 == existfd || (*iter).second->GetFd() == existfd)){
-    return NULL;
+    return (*iter).second;
   }
-  return (*iter).second;
+
+  if(-1 != existfd){
+    for(iter = fent.begin(); iter != fent.end(); iter++){
+      if((*iter).second && (*iter).second->GetFd() == existfd){
+        // found opend fd in map
+        if(0 == strcmp((*iter).second->GetPath(), path)){
+          return (*iter).second;
+        }
+        // found fd, but it is used another file(file discriptor is recycled)
+        // so returns NULL.
+        break;
+      }
+    }
+  }
+  return NULL;
 }
 
 FdEntity* FdManager::Open(const char* path, off_t size, time_t time, bool force_tmpfile, bool is_create)
@@ -1164,8 +1211,22 @@ FdEntity* FdManager::Open(const char* path, off_t size, time_t time, bool force_
     }
     // make new obj
     ent = new FdEntity(path, cache_path.c_str());
-    fent[string(path)] = ent;
 
+    if(0 < cache_path.size()){
+      // using cache
+      fent[string(path)] = ent;
+    }else{
+      // not using cache, so the key of fdentity is set not really existsing path.
+      // (but not strictly unexisting path.)
+      //
+      // [NOTE]
+      // The reason why this process here, please look at the definition of the
+      // comments of NOCACHE_PATH_PREFIX_FORM symbol.
+      //
+      string tmppath("");
+      FdManager::MakeRandomTempPath(path, tmppath);
+      fent[tmppath] = ent;
+    }
   }else{
     return NULL;
   }
@@ -1177,6 +1238,33 @@ FdEntity* FdManager::Open(const char* path, off_t size, time_t time, bool force_
   return ent;
 }
 
+FdEntity* FdManager::ExistOpen(const char* path, int existfd)
+{
+  FPRNINFO("[path=%s][fd=%d]", SAFESTRPTR(path), existfd);
+
+  // search by real path
+  FdEntity* ent = Open(path, -1, -1, false, false);
+
+  if(!ent && -1 != existfd){
+    // search from all fdentity because of not using cache.
+    AutoLock auto_lock(&FdManager::fd_manager_lock);
+
+    for(fdent_map_t::iterator iter = fent.begin(); iter != fent.end(); iter++){
+      if((*iter).second && (*iter).second->GetFd() == existfd && (*iter).second->IsOpen()){
+        // found opend fd in map
+        if(0 == strcmp((*iter).second->GetPath(), path)){
+          ent = (*iter).second;
+        }else{
+          // found fd, but it is used another file(file discriptor is recycled)
+          // so returns NULL.
+        }
+        break;
+      }
+    }
+  }
+  return ent;
+}
+
 void FdManager::Rename(const std::string &from, const std::string &to)
 {
   fdent_map_t::iterator iter = fent.find(from);

src/fdcache.h

@@ -85,7 +85,7 @@ class PageList
     bool IsInit(off_t start, off_t size);
     bool SetInit(off_t start, off_t size, bool is_init = true);
     bool FindUninitPage(off_t start, off_t& resstart, size_t& ressize);
-    int GetUninitPages(fdpage_list_t& uninit_list, off_t start = 0);
+    int GetUninitPages(fdpage_list_t& uninit_list, off_t start = 0, off_t size = -1);
     bool Serialize(CacheFileStat& file, bool is_output);
     void Dump(void);
 };
@@ -166,10 +166,11 @@ class FdManager
     static size_t SetPageSize(size_t size);
     static size_t GetPageSize(void) { return FdManager::page_size; }
     static bool MakeCachePath(const char* path, std::string& cache_path, bool is_create_dir = true);
+    static bool MakeRandomTempPath(const char* path, std::string& tmppath);
 
-    FdEntity* GetFdEntity(const char* path);
+    FdEntity* GetFdEntity(const char* path, int existfd = -1);
     FdEntity* Open(const char* path, off_t size = -1, time_t time = -1, bool force_tmpfile = false, bool is_create = true);
-    FdEntity* ExistOpen(const char* path) { return Open(path, -1, -1, false, false); }
+    FdEntity* ExistOpen(const char* path, int existfd = -1);
     void Rename(const std::string &from, const std::string &to);
     bool Close(FdEntity* ent);
 };

src/s3fs.cpp

@@ -1953,7 +1953,7 @@ static int s3fs_read(const char* path, char* buf, size_t size, off_t offset, str
   FPRNINFO("[path=%s][size=%zu][offset=%jd][fd=%llu]", path, size, (intmax_t)offset, (unsigned long long)(fi->fh));
 
   FdEntity* ent;
-  if(NULL == (ent = FdManager::get()->ExistOpen(path))){
+  if(NULL == (ent = FdManager::get()->ExistOpen(path, static_cast<int>(fi->fh)))){
     DPRN("could not find opened fd(%s)", path);
     return -EIO;
   }
@@ -1984,7 +1984,7 @@ static int s3fs_write(const char* path, const char* buf, size_t size, off_t offs
   FPRNINFO("[path=%s][size=%zu][offset=%jd][fd=%llu]", path, size, (intmax_t)offset, (unsigned long long)(fi->fh));
 
   FdEntity* ent;
-  if(NULL == (ent = FdManager::get()->ExistOpen(path))){
+  if(NULL == (ent = FdManager::get()->ExistOpen(path, static_cast<int>(fi->fh)))){
     DPRN("could not find opened fd(%s)", path);
     return -EIO;
   }
@@ -2030,7 +2030,7 @@ static int s3fs_flush(const char* path, struct fuse_file_info* fi)
   }
 
   FdEntity* ent;
-  if(NULL != (ent = FdManager::get()->ExistOpen(path))){
+  if(NULL != (ent = FdManager::get()->ExistOpen(path, static_cast<int>(fi->fh)))){
     headers_t meta;
     if(0 != (result = get_object_attribute(path, NULL, &meta))){
       FdManager::get()->Close(ent);
@@ -2067,7 +2067,7 @@ static int s3fs_release(const char* path, struct fuse_file_info* fi)
   }
 
   FdEntity* ent;
-  if(NULL == (ent = FdManager::get()->GetFdEntity(path))){
+  if(NULL == (ent = FdManager::get()->GetFdEntity(path, static_cast<int>(fi->fh)))){
     DPRN("could not find fd(file=%s)", path);
     return -EIO;
   }
@@ -2078,7 +2078,7 @@ static int s3fs_release(const char* path, struct fuse_file_info* fi)
 
   // check - for debug
   if(debug){
-    if(NULL != (ent = FdManager::get()->GetFdEntity(path))){
+    if(NULL != (ent = FdManager::get()->GetFdEntity(path, static_cast<int>(fi->fh)))){
       DPRNNN("Warning - file(%s),fd(%d) is still opened.", path, ent->GetFd());
     }
   }