Fixed bugs(overflow)

1) Overflow
   About over 4GB file, when st_size which is member stat structure,
   the value is overflow.
   Fixed this bug and fixed like this bug in all sources. 

2) Changed retrying request
   If s3fs gets 500 HTTP status for multipart request, s3fs retry
   to send same request.





git-svn-id: http://s3fs.googlecode.com/svn/trunk@495 df820570-a93a-0410-bd06-b72b767a4274

src/curl.cpp

@@ -3085,20 +3085,24 @@ int S3fsMultiCurl::MultiRead(void)
           if(400 > responseCode){
             // add into stat cache
             if(SuccessCallback && !SuccessCallback(s3fscurl)){
-              DPRNNN("error from callback function(%s).", s3fscurl->base_path.c_str());
+              DPRN("error from callback function(%s).", s3fscurl->url.c_str());
             }
           }else if(400 == responseCode){
             // as possibly in multipart
-            DPRNNN("failed a request(%ld: %s)", responseCode, s3fscurl->base_path.c_str());
+            DPRN("failed a request(%ld: %s)", responseCode, s3fscurl->url.c_str());
             isRetry = true;
           }else{
-            DPRNNN("failed a request(%ld: %s)", responseCode, s3fscurl->base_path.c_str());
+            // case of all other result, do retry.(11/13/2013)
+            // because it was found that s3fs got 500 error from S3, but could success
+            // to retry it.
+            DPRN("failed a request(%ld: %s)", responseCode, s3fscurl->url.c_str());
+            isRetry = true;
           }
         }else{
-          DPRNNN("failed a request(Unknown respons code: %s)", s3fscurl->base_path.c_str());
+          DPRN("failed a request(Unknown respons code: %s)", s3fscurl->url.c_str());
         }
       }else{
-        DPRNNN("failed to read(remaining: %d code: %d  msg: %s), so retry this.",
+        DPRN("failed to read(remaining: %d code: %d  msg: %s), so retry this.",
               remaining_messages, msg->data.result, curl_easy_strerror(msg->data.result));
         isRetry = true;
       }

src/fdcache.cpp

@@ -427,7 +427,7 @@ bool PageList::Serialize(CacheFileStat& file, bool is_output)
       free(ptmp);
       return false;
     }
-    size_t total = static_cast<size_t>(atoi(oneline.c_str()));
+    size_t total = s3fs_strtoul(oneline.c_str());
 
     // load each part
     bool is_err = false;
@@ -439,19 +439,19 @@ bool PageList::Serialize(CacheFileStat& file, bool is_output)
         is_err = true;
         break;
       }
-      off_t offset = static_cast<off_t>(atoi(part.c_str()));
+      off_t offset = static_cast<off_t>(s3fs_strtoul(part.c_str()));
       // size
       if(!getline(ssparts, part, ':')){
         is_err = true;
         break;
       }
-      ssize_t size = static_cast<ssize_t>(atoi(part.c_str()));
+      ssize_t size = static_cast<ssize_t>(s3fs_strtoul(part.c_str()));
       // init
       if(!getline(ssparts, part, ':')){
         is_err = true;
         break;
       }
-      bool is_init = (1 == atoi(part.c_str()) ? true : false);
+      bool is_init = (1 == s3fs_strtoul(part.c_str()) ? true : false);
       // add new area
       SetInit(offset, size, is_init);
     }

src/s3fs.cpp

@@ -3396,7 +3396,7 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
 
   }else if(key == FUSE_OPT_KEY_OPT){
     if(0 == STR2NCMP(arg, "uid=")){
-      s3fs_uid = strtoul(strchr(arg, '=') + sizeof(char), 0, 10);
+      s3fs_uid = get_uid(strchr(arg, '=') + sizeof(char));
       if(0 != geteuid() && 0 == s3fs_uid){
         fprintf(stderr, "%s: root user can only specify uid=0.\n", program_name.c_str());
         return -1;
@@ -3405,7 +3405,7 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
       return 1; // continue for fuse option
     }
     if(0 == STR2NCMP(arg, "gid=")){
-      s3fs_gid = strtoul(strchr(arg, '=') + sizeof(char), 0, 10);
+      s3fs_gid = get_gid(strchr(arg, '=') + sizeof(char));
       if(0 != getegid() && 0 == s3fs_gid){
         fprintf(stderr, "%s: root user can only specify gid=0.\n", program_name.c_str());
         return -1;
@@ -3414,7 +3414,7 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
       return 1; // continue for fuse option
     }
     if(0 == STR2NCMP(arg, "umask=")){
-      s3fs_umask = (mode_t)strtoul(strchr(arg, '=') + sizeof(char), 0, 8);
+      s3fs_umask = get_mode(strchr(arg, '=') + sizeof(char));
       s3fs_umask &= (S_IRWXU | S_IRWXG | S_IRWXO);
       is_s3fs_umask = true;
       return 1; // continue for fuse option
@@ -3429,7 +3429,7 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
       return 0;
     }
     if(0 == STR2NCMP(arg, "retries=")){
-      S3fsCurl::SetRetries(atoi(strchr(arg, '=') + sizeof(char)));
+      S3fsCurl::SetRetries(static_cast<int>(s3fs_strtoul(strchr(arg, '=') + sizeof(char))));
       return 0;
     }
     if(0 == STR2NCMP(arg, "use_cache=")){
@@ -3441,7 +3441,7 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
       return 0;
     }
     if(0 == STR2NCMP(arg, "multireq_max=")){
-      long maxreq = (long)atoi(strchr(arg, '=') + sizeof(char));
+      long maxreq = static_cast<long>(s3fs_strtoul(strchr(arg, '=') + sizeof(char)));
       S3fsMultiCurl::SetMaxMultiRequest(maxreq);
       return 0;
     }
@@ -3454,10 +3454,10 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
       return 0;
     }
     if(0 == strcmp(arg, "use_rrs") || 0 == STR2NCMP(arg, "use_rrs=")){
-      int rrs = 1;
+      size_t rrs = 1;
       // for an old format.
       if(0 == STR2NCMP(arg, "use_rrs=")){
-        rrs = atoi(strchr(arg, '=') + sizeof(char));
+        rrs = s3fs_strtoul(strchr(arg, '=') + sizeof(char));
       }
       if(0 == rrs){
         S3fsCurl::SetUseRrs(false);
@@ -3474,10 +3474,10 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
       return 0;
     }
     if(0 == strcmp(arg, "use_sse") || 0 == STR2NCMP(arg, "use_sse=")){
-      int sse = 1;
+      size_t sse = 1;
       // for an old format.
       if(0 == STR2NCMP(arg, "use_sse=")){
-        sse = atoi(strchr(arg, '=') + sizeof(char));
+        sse = s3fs_strtoul(strchr(arg, '=') + sizeof(char));
       }
       if(0 == sse){
         S3fsCurl::SetUseSse(false);
@@ -3494,7 +3494,7 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
       return 0;
     }
     if(0 == STR2NCMP(arg, "ssl_verify_hostname=")){
-      long sslvh = strtol(strchr(arg, '=') + sizeof(char), 0, 10);
+      long sslvh = static_cast<long>(s3fs_strtoul(strchr(arg, '=') + sizeof(char)));
       if(-1 == S3fsCurl::SetSslVerifyHostname(sslvh)){
         fprintf(stderr, "%s: poorly formed argument to option: ssl_verify_hostname\n", 
                 program_name.c_str());
@@ -3512,7 +3512,7 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
       return 0;
     }
     if(0 == STR2NCMP(arg, "public_bucket=")){
-      long pubbucket = strtol(strchr(arg, '=') + sizeof(char), 0, 10);
+      size_t pubbucket = s3fs_strtoul(strchr(arg, '=') + sizeof(char));
       if(1 == pubbucket){
         S3fsCurl::SetPublicBucket(true);
       }else if(0 == pubbucket){
@@ -3533,22 +3533,22 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
       return 0;
     }
     if(0 == STR2NCMP(arg, "connect_timeout=")){
-      long contimeout = strtol(strchr(arg, '=') + sizeof(char), 0, 10);
+      long contimeout = static_cast<long>(s3fs_strtoul(strchr(arg, '=') + sizeof(char)));
       S3fsCurl::SetConnectTimeout(contimeout);
       return 0;
     }
     if(0 == STR2NCMP(arg, "readwrite_timeout=")){
-      time_t rwtimeout = (time_t)strtoul(strchr(arg, '=') + sizeof(char), 0, 10);
+      time_t rwtimeout = static_cast<time_t>(s3fs_strtoul(strchr(arg, '=') + sizeof(char)));
       S3fsCurl::SetReadwriteTimeout(rwtimeout);
       return 0;
     }
     if(0 == STR2NCMP(arg, "max_stat_cache_size=")){
-      unsigned long cache_size = strtoul(strchr(arg, '=') + sizeof(char), 0, 10);
+      unsigned long cache_size = static_cast<unsigned long>(s3fs_strtoul(strchr(arg, '=') + sizeof(char)));
       StatCache::getStatCacheData()->SetCacheSize(cache_size);
       return 0;
     }
     if(0 == STR2NCMP(arg, "stat_cache_expire=")){
-      time_t expr_time = strtoul(strchr(arg, '=') + sizeof(char), 0, 10);
+      time_t expr_time = static_cast<time_t>(s3fs_strtoul(strchr(arg, '=') + sizeof(char)));
       StatCache::getStatCacheData()->SetExpireTime(expr_time);
       return 0;
     }
@@ -3565,7 +3565,7 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
       return 0;
     }
     if(0 == STR2NCMP(arg, "parallel_count=") || 0 == STR2NCMP(arg, "parallel_upload=")){
-      int maxpara = (int)strtoul(strchr(arg, '=') + sizeof(char), 0, 10);
+      int maxpara = static_cast<int>(s3fs_strtoul(strchr(arg, '=') + sizeof(char)));
       if(0 >= maxpara){
         fprintf(stderr, "%s: argument should be over 1: parallel_count\n", 
            program_name.c_str());
@@ -3575,7 +3575,7 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
       return 0;
     }
     if(0 == STR2NCMP(arg, "fd_page_size=")){
-      ssize_t pagesize = static_cast<ssize_t>(strtoul(strchr(arg, '=') + sizeof(char), 0, 10));
+      ssize_t pagesize = static_cast<ssize_t>(s3fs_strtoul(strchr(arg, '=') + sizeof(char)));
       if((1024 * 1024) >= pagesize){
         fprintf(stderr, "%s: argument should be over 1MB: fd_page_size\n", 
            program_name.c_str());

src/s3fs_util.cpp

@@ -618,9 +618,52 @@ bool delete_files_in_dir(const char* dir, bool is_remove_own)
 //-------------------------------------------------------------------
 // Utility functions for convert
 //-------------------------------------------------------------------
+size_t s3fs_strtoul(const char* str, bool is_base_16)
+{
+  if(!str || '\0' == *str){
+    return 0;
+  }
+  size_t result;
+  bool   chk_space;
+  bool   chk_base16_prefix;
+  for(result = 0, chk_space = false, chk_base16_prefix = false; '\0' != *str; str++){
+    // check head space
+    if(!chk_space && isspace(*str)){
+      continue;
+    }else if(!chk_space){
+      chk_space = true;
+    }
+    // check prefix for base 16
+    if(!chk_base16_prefix){
+      chk_base16_prefix = true;
+      if('0' == *str && ('x' == str[1] || 'X' == str[1])){
+        is_base_16 = true;
+        str++;
+        continue;
+      }
+    }
+    // check like isalnum and set data
+    result *= (is_base_16 ? 16 : 10);
+    if('0' <= *str || '9' < *str){
+      result += static_cast<size_t>(*str - '0');
+    }else if(is_base_16){
+      if('A' <= *str && *str <= 'F'){
+        result += static_cast<size_t>(*str - 'A' + 0x0a);
+      }else if('a' <= *str && *str <= 'f'){
+        result += static_cast<size_t>(*str - 'a' + 0x0a);
+      }else{
+        return 0;
+      }
+    }else{
+      return 0;
+    }
+  }
+  return result;
+}
+
 time_t get_mtime(const char *s)
 {
-  return (time_t) strtoul(s, (char **) NULL, 10);
+  return static_cast<time_t>(s3fs_strtoul(s));
 }
 
 time_t get_mtime(headers_t& meta, bool overcheck)
@@ -637,7 +680,7 @@ time_t get_mtime(headers_t& meta, bool overcheck)
 
 off_t get_size(const char *s)
 {
-  return (off_t) strtoul(s, (char **) NULL, 10);
+  return static_cast<off_t>(s3fs_strtoul(s));
 }
 
 off_t get_size(headers_t& meta)
@@ -651,7 +694,7 @@ off_t get_size(headers_t& meta)
 
 mode_t get_mode(const char *s)
 {
-  return (mode_t) strtoul(s, (char **) NULL, 10);
+  return static_cast<mode_t>(s3fs_strtoul(s));
 }
 
 mode_t get_mode(headers_t& meta, const char* path, bool checkdir, bool forcedir)
@@ -708,7 +751,7 @@ mode_t get_mode(headers_t& meta, const char* path, bool checkdir, bool forcedir)
 
 uid_t get_uid(const char *s)
 {
-  return (uid_t) strtoul(s, (char **) NULL, 10);
+  return static_cast<uid_t>(s3fs_strtoul(s));
 }
 
 uid_t get_uid(headers_t& meta)
@@ -724,7 +767,7 @@ uid_t get_uid(headers_t& meta)
 
 gid_t get_gid(const char *s)
 {
-  return (gid_t) strtoul(s, (char **) NULL, 10);
+  return static_cast<gid_t>(s3fs_strtoul(s));
 }
 
 gid_t get_gid(headers_t& meta)

src/s3fs_util.h

@@ -95,6 +95,7 @@ std::string mybasename(std::string path);
 int mkdirp(const std::string& path, mode_t mode);
 bool delete_files_in_dir(const char* dir, bool is_remove_own);
 
+size_t s3fs_strtoul(const char* str, bool is_base_16 = false);
 time_t get_mtime(const char *s);
 time_t get_mtime(headers_t& meta, bool overcheck = true);
 off_t get_size(const char *s);