Fixed issue 291, and Adds "disable_noobj_cache" option.

1) Man file has wrong permissions for passwd file(Issue 291)
    Fixes man page for wrong permissions of passwd file.

2) Fixes a bug and Strictly checks passwd file permission.
    * Fixes a bug about checking passwd file permission.
       A bug is that s3fs continues to run after s3fs finds invalid passwd 
      file permission.
    * Checks passwd file strictly.
       Before this revision, s3fs allows executable permission for a 
       passwd file and  allows group writable permission for a passwd 
       file(which is not "/etc/passwd-s3fs").
       New s3fs checks permission strictly, that is  /etc/passwd-s3fs is 
       allowed owner readable/writable and group readable, and  the 
       passwd file(which is not "/etc/passwd-s3fs") is allowed only owner 
       readable/writable.

3) Adds disable_noobj_cache option for no-existing object.
    s3fs v1.68 always has to check whether file(or sub directory) exists 
    under object(path)  when s3fs does some command, since s3fs has 
    recognized a directory which does not exist and has files or sub 
    directories under itself.
    It increases ListBucket request and makes performance bad.
    For performance if the disable_noobj_cache option is specified, s3fs 
    memorizes in stat cache that the object(file or directory) does not exist.




git-svn-id: http://s3fs.googlecode.com/svn/trunk@420 df820570-a93a-0410-bd06-b72b767a4274

doc/man/s3fs.1

@@ -23,8 +23,8 @@ If you have more than one set of credentials, this syntax is also recognized:
 .PP
 Password files can be stored in two locations:
 .RS 4
- \fB/etc/passwd-s3fs\fP     [0600]
+ \fB/etc/passwd-s3fs\fP     [0640]
- \fB$HOME/.passwd-s3fs\fP   [0640]
+ \fB$HOME/.passwd-s3fs\fP   [0600]
 .RE
 .SH OPTIONS
 .SS "general options"
@@ -80,6 +80,12 @@ maximum number of entries in the stat cache
 \fB\-o\fR stat_cache_expire (default is no expire)
 specify expire time(seconds) for entries in the stat cache
 .TP
+\fB\-o\fR enable_noobj_cache (default is disable)
+enable cache entries for the object which does not exist.
+s3fs always has to check whether file(or sub directory) exists under object(path) when s3fs does some command, since s3fs has recognized a directory which does not exist and has files or sub directories under itself.
+It increases ListBucket request and makes performance bad.
+You can specify this option for performance, s3fs memorizes in stat cache that the object(file or directory) does not exist.
+.TP
 \fB\-o\fR url (default="http://s3.amazonaws.com")
 sets the url to use to access Amazon S3. If you want to use HTTPS, then you can set url=https://s3.amazonaws.com
 .TP

src/cache.cpp

@@ -51,9 +51,10 @@ StatCache::StatCache()
   }else{
     assert(false);
   }
-  CacheSize    = 1000;
+  CacheSize       = 1000;
-  ExpireTime   = 0;
+  ExpireTime      = 0;
-  IsExpireTime = false;
+  IsExpireTime    = false;
+  IsCacheNoObject = false;
 }
 
 StatCache::~StatCache()
@@ -101,6 +102,13 @@ time_t StatCache::UnsetExpireTime(void)
   return old;
 }
 
+bool StatCache::SetCacheNoObject(bool flag)
+{
+  bool old = IsCacheNoObject;
+  IsCacheNoObject = flag;
+  return old;
+}
+
 bool StatCache::GetStat(string& key, struct stat* pst, headers_t* meta, bool overcheck, const char* petag, bool* pisforce)
 {
   bool is_delete_cache = false;
@@ -120,6 +128,16 @@ bool StatCache::GetStat(string& key, struct stat* pst, headers_t* meta, bool ove
 
   if(iter != stat_cache.end()) {
     if(!IsExpireTime|| ((*iter).second.cache_date + ExpireTime) >= time(NULL)){
+      if((*iter).second.noobjcache){
+        pthread_mutex_unlock(&StatCache::stat_cache_lock);
+        if(!IsCacheNoObject){
+          // need to delete this cache.
+          DelStat(strpath);
+        }else{
+          // noobjcache = true means no object.
+        }
+        return false;
+      }
       // hit without checking etag
       if(petag){
         string stretag = (*iter).second.meta["ETag"];
@@ -165,6 +183,47 @@ bool StatCache::GetStat(string& key, struct stat* pst, headers_t* meta, bool ove
   return false;
 }
 
+bool StatCache::IsNoObjectCache(string& key, bool overcheck)
+{
+  bool is_delete_cache = false;
+  string strpath = key;
+
+  if(!IsCacheNoObject){
+    return false;
+  }
+
+  pthread_mutex_lock(&StatCache::stat_cache_lock);
+
+  stat_cache_t::iterator iter = stat_cache.end();
+  if(overcheck && '/' != strpath[strpath.length() - 1]){
+    strpath += "/";
+    iter = stat_cache.find(strpath.c_str());
+  }
+  if(iter == stat_cache.end()){
+    strpath = key;
+    iter = stat_cache.find(strpath.c_str());
+  }
+
+  if(iter != stat_cache.end()) {
+    if(!IsExpireTime|| ((*iter).second.cache_date + ExpireTime) >= time(NULL)){
+      if((*iter).second.noobjcache){
+        // noobjcache = true means no object.
+        pthread_mutex_unlock(&StatCache::stat_cache_lock);
+        return true;
+      }
+    }else{
+      // timeout
+      is_delete_cache = true;
+    }
+  }
+  pthread_mutex_unlock(&StatCache::stat_cache_lock);
+
+  if(is_delete_cache){
+    DelStat(strpath);
+  }
+  return false;
+}
+
 bool StatCache::AddStat(std::string& key, headers_t& meta, bool forcedir)
 {
   if(CacheSize< 1){
@@ -188,6 +247,7 @@ bool StatCache::AddStat(std::string& key, headers_t& meta, bool forcedir)
   stat_cache[key].hit_count  = 0;
   stat_cache[key].cache_date = time(NULL); // Set time.
   stat_cache[key].isforce    = forcedir;
+  stat_cache[key].noobjcache = false;
 
   //copy only some keys
   for (headers_t::iterator iter = meta.begin(); iter != meta.end(); ++iter) {
@@ -216,6 +276,36 @@ bool StatCache::AddStat(std::string& key, headers_t& meta, bool forcedir)
   return true;
 }
 
+bool StatCache::AddNoObjectCache(string& key)
+{
+  if(!IsCacheNoObject){
+    return true;    // pretend successful
+  }
+  if(CacheSize < 1){
+    return true;
+  }
+  FGPRINT("    add_stat_cache_entry - noobjcache[path=%s]\n", key.c_str());
+
+  if(stat_cache.size() > CacheSize){
+    if(!TruncateCache()){
+      return false;
+    }
+  }
+
+  struct stat st;
+  memset(&st, 0, sizeof(struct stat));
+
+  pthread_mutex_lock(&StatCache::stat_cache_lock);
+  stat_cache[key].stbuf      = st;
+  stat_cache[key].hit_count  = 0;
+  stat_cache[key].cache_date = time(NULL); // Set time.
+  stat_cache[key].isforce    = false;
+  stat_cache[key].noobjcache = true;
+  pthread_mutex_unlock(&StatCache::stat_cache_lock);
+
+  return true;
+}
+
 bool StatCache::TruncateCache(void)
 {
   string path_to_delete;

src/cache.h

@@ -12,8 +12,9 @@ struct stat_cache_entry {
   time_t        cache_date;
   headers_t     meta;
   bool          isforce;
+  bool          noobjcache;  // Flag: cache is no object for no listing.
 
-  stat_cache_entry() : hit_count(0), cache_date(0), isforce(false) {
+  stat_cache_entry() : hit_count(0), cache_date(0), isforce(false), noobjcache(false) {
     memset(&stbuf, 0, sizeof(struct stat));
     meta.clear();
   }
@@ -33,6 +34,7 @@ class StatCache
     bool IsExpireTime;
     time_t ExpireTime;
     unsigned long CacheSize;
+    bool IsCacheNoObject;
 
   private:
     bool GetStat(std::string& key, struct stat* pst, headers_t* meta, bool overcheck, const char* petag, bool* pisforce);
@@ -54,6 +56,16 @@ class StatCache
     time_t GetExpireTime(void) const;
     time_t SetExpireTime(time_t expire);
     time_t UnsetExpireTime(void);
+    bool SetCacheNoObject(bool flag);
+    bool EnableCacheNoObject(void) {
+      return SetCacheNoObject(true);
+    }
+    bool DisableCacheNoObject(void) {
+      return SetCacheNoObject(false);
+    }
+    bool GetCacheNoObject(void) const {
+      return IsCacheNoObject;
+    }
 
     // Get stat cache
     bool GetStat(std::string& key, struct stat* pst, headers_t* meta, bool overcheck = true, bool* pisforce = NULL) {
@@ -72,6 +84,10 @@ class StatCache
       return GetStat(key, NULL, NULL, overcheck, etag, NULL);
     }
 
+    // Cache For no object
+    bool IsNoObjectCache(std::string& key, bool overcheck = true);
+    bool AddNoObjectCache(std::string& key);
+
     // Add stat cache
     bool AddStat(std::string& key, headers_t& meta, bool forcedir = false);
 

src/s3fs.cpp

@@ -334,6 +334,10 @@ static int get_object_attribute(const char *path, struct stat *pstbuf, headers_t
   if(StatCache::getStatCacheData()->GetStat(strpath, pstat, pheader, overcheck, pisforce)){
     return 0;
   }
+  if(StatCache::getStatCacheData()->IsNoObjectCache(strpath)){
+    // there is the path in the cache for no object, it is no object.
+    return -ENOENT;
+  }
 
   // At first, check "object/".
   strpath = path;
@@ -374,6 +378,9 @@ static int get_object_attribute(const char *path, struct stat *pstbuf, headers_t
               (*pisforce) = true;
             }
           }else{
+            // Add no object cache.
+            strpath = path;  // reset original
+            StatCache::getStatCacheData()->AddNoObjectCache(strpath);
             return result;
           }
         }
@@ -1712,9 +1719,10 @@ static int s3fs_create(const char *path, mode_t mode, struct fuse_file_info *fi)
     return result;
   }
   result = create_file_object(path, mode, pcxt->uid, pcxt->gid);
-
+  StatCache::getStatCacheData()->DelStat(path);
-  if(result != 0)
+  if(result != 0){
     return result;
+  }
 
   // object created, open it
   if((fi->fh = get_local_fd(path)) <= 0)
@@ -1804,7 +1812,9 @@ static int s3fs_mkdir(const char *path, mode_t mode)
     return result;
   }
 
-  return create_directory_object(path, mode, time(NULL), pcxt->uid, pcxt->gid);
+  result = create_directory_object(path, mode, time(NULL), pcxt->uid, pcxt->gid);
+  StatCache::getStatCacheData()->DelStat(path);
+  return result;
 }
 
 static int s3fs_unlink(const char *path) {
@@ -3781,6 +3791,18 @@ static int check_passwd_file_perms (void) {
         program_name.c_str(), passwd_file.c_str());
       return EXIT_FAILURE;
     }
+  }else{
+    // "/etc/passwd-s3fs" does not allow group write.
+    if((info.st_mode & S_IWGRP)){
+      fprintf (stderr, "%s: credentials file %s should not have group writable permissions\n", 
+        program_name.c_str(), passwd_file.c_str());
+      return EXIT_FAILURE;
+    }
+  }
+  if((info.st_mode & S_IXUSR) || (info.st_mode & S_IXGRP)){
+    fprintf (stderr, "%s: credentials file %s should not have executable permissions\n", 
+      program_name.c_str(), passwd_file.c_str());
+    return EXIT_FAILURE;
   }
   return EXIT_SUCCESS;
 }
@@ -3812,7 +3834,9 @@ static int read_passwd_file (void) {
   // if you got here, the password file
   // exists and is readable by the
   // current user, check for permissions
-  check_passwd_file_perms();
+  if(EXIT_SUCCESS != check_passwd_file_perms()){
+    return EXIT_FAILURE;
+  }
 
   aws_format = check_for_aws_format();
   if(1 == aws_format){
@@ -4151,6 +4175,10 @@ static int my_fuse_opt_proc(void *data, const char *arg, int key, struct fuse_ar
       StatCache::getStatCacheData()->SetExpireTime(expr_time);
       return 0;
     }
+    if(strstr(arg, "enable_noobj_cache") != 0) {
+      StatCache::getStatCacheData()->EnableCacheNoObject();
+      return 0;
+    }
     if(strstr(arg, "noxmlns") != 0) {
       noxmlns = true;
       return 0;

src/s3fs_util.cpp

@@ -594,6 +594,16 @@ void show_help (void)
     "   stat_cache_expire (default is no expire)\n"
     "      - specify expire time(seconds) for entries in the stat cache.\n"
     "\n"
+    "   enable_noobj_cache (default is disable)\n"
+    "      - enable cache entries for the object which does not exist.\n"
+    "      s3fs always has to check whether file(or sub directory) exists \n"
+    "      under object(path) when s3fs does some command, since s3fs has \n"
+    "      recognized a directory which does not exist and has files or \n"
+    "      sub directories under itself. It increases ListBucket request \n"
+    "      and makes performance bad.\n"
+    "      You can specify this option for performance, s3fs memorizes \n"
+    "      in stat cache that the object(file or directory) does not exist.\n"
+    "\n"
     "   url (default=\"http://s3.amazonaws.com\")\n"
     "      - sets the url to use to access amazon s3\n"
     "\n"