mirror of
https://github.com/s3fs-fuse/s3fs-fuse.git
synced 2024-12-22 16:58:55 +00:00
Fixed bugs(overflow)
1) Overflow About over 4GB file, when st_size which is member stat structure, the value is overflow. Fixed this bug and fixed like this bug in all sources. 2) Changed retrying request If s3fs gets 500 HTTP status for multipart request, s3fs retry to send same request. git-svn-id: http://s3fs.googlecode.com/svn/trunk@495 df820570-a93a-0410-bd06-b72b767a4274
This commit is contained in:
ggtakec@gmail.com
parent
c785be917f
commit
882f13020e
14
src/curl.cpp
14
src/curl.cpp
@ -3085,20 +3085,24 @@ int S3fsMultiCurl::MultiRead(void)
|
||||
if(400 > responseCode){
|
||||
// add into stat cache
|
||||
if(SuccessCallback && !SuccessCallback(s3fscurl)){
|
||||
DPRNNN("error from callback function(%s).", s3fscurl->base_path.c_str());
|
||||
DPRN("error from callback function(%s).", s3fscurl->url.c_str());
|
||||
}
|
||||
}else if(400 == responseCode){
|
||||
// as possibly in multipart
|
||||
DPRNNN("failed a request(%ld: %s)", responseCode, s3fscurl->base_path.c_str());
|
||||
DPRN("failed a request(%ld: %s)", responseCode, s3fscurl->url.c_str());
|
||||
isRetry = true;
|
||||
}else{
|
||||
DPRNNN("failed a request(%ld: %s)", responseCode, s3fscurl->base_path.c_str());
|
||||
// case of all other result, do retry.(11/13/2013)
|
||||
// because it was found that s3fs got 500 error from S3, but could success
|
||||
// to retry it.
|
||||
DPRN("failed a request(%ld: %s)", responseCode, s3fscurl->url.c_str());
|
||||
isRetry = true;
|
||||
}
|
||||
}else{
|
||||
DPRNNN("failed a request(Unknown respons code: %s)", s3fscurl->base_path.c_str());
|
||||
DPRN("failed a request(Unknown respons code: %s)", s3fscurl->url.c_str());
|
||||
}
|
||||
}else{
|
||||
DPRNNN("failed to read(remaining: %d code: %d msg: %s), so retry this.",
|
||||
DPRN("failed to read(remaining: %d code: %d msg: %s), so retry this.",
|
||||
remaining_messages, msg->data.result, curl_easy_strerror(msg->data.result));
|
||||
isRetry = true;
|
||||
}
|
||||
|
||||
@ -427,7 +427,7 @@ bool PageList::Serialize(CacheFileStat& file, bool is_output)
|
||||
free(ptmp);
|
||||
return false;
|
||||
}
|
||||
size_t total = static_cast<size_t>(atoi(oneline.c_str()));
|
||||
size_t total = s3fs_strtoul(oneline.c_str());
|
||||
|
||||
// load each part
|
||||
bool is_err = false;
|
||||
@ -439,19 +439,19 @@ bool PageList::Serialize(CacheFileStat& file, bool is_output)
|
||||
is_err = true;
|
||||
break;
|
||||
}
|
||||
off_t offset = static_cast<off_t>(atoi(part.c_str()));
|
||||
off_t offset = static_cast<off_t>(s3fs_strtoul(part.c_str()));
|
||||
// size
|
||||
if(!getline(ssparts, part, ':')){
|
||||
is_err = true;
|
||||
break;
|
||||
}
|
||||
ssize_t size = static_cast<ssize_t>(atoi(part.c_str()));
|
||||
ssize_t size = static_cast<ssize_t>(s3fs_strtoul(part.c_str()));
|
||||
// init
|
||||
if(!getline(ssparts, part, ':')){
|
||||
is_err = true;
|
||||
break;
|
||||
}
|
||||
bool is_init = (1 == atoi(part.c_str()) ? true : false);
|
||||
bool is_init = (1 == s3fs_strtoul(part.c_str()) ? true : false);
|
||||
// add new area
|
||||
SetInit(offset, size, is_init);
|
||||
}
|
||||
|
||||
34
src/s3fs.cpp
34
src/s3fs.cpp
@ -3396,7 +3396,7 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
|
||||
|
||||
}else if(key == FUSE_OPT_KEY_OPT){
|
||||
if(0 == STR2NCMP(arg, "uid=")){
|
||||
s3fs_uid = strtoul(strchr(arg, '=') + sizeof(char), 0, 10);
|
||||
s3fs_uid = get_uid(strchr(arg, '=') + sizeof(char));
|
||||
if(0 != geteuid() && 0 == s3fs_uid){
|
||||
fprintf(stderr, "%s: root user can only specify uid=0.\n", program_name.c_str());
|
||||
return -1;
|
||||
@ -3405,7 +3405,7 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
|
||||
return 1; // continue for fuse option
|
||||
}
|
||||
if(0 == STR2NCMP(arg, "gid=")){
|
||||
s3fs_gid = strtoul(strchr(arg, '=') + sizeof(char), 0, 10);
|
||||
s3fs_gid = get_gid(strchr(arg, '=') + sizeof(char));
|
||||
if(0 != getegid() && 0 == s3fs_gid){
|
||||
fprintf(stderr, "%s: root user can only specify gid=0.\n", program_name.c_str());
|
||||
return -1;
|
||||
@ -3414,7 +3414,7 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
|
||||
return 1; // continue for fuse option
|
||||
}
|
||||
if(0 == STR2NCMP(arg, "umask=")){
|
||||
s3fs_umask = (mode_t)strtoul(strchr(arg, '=') + sizeof(char), 0, 8);
|
||||
s3fs_umask = get_mode(strchr(arg, '=') + sizeof(char));
|
||||
s3fs_umask &= (S_IRWXU | S_IRWXG | S_IRWXO);
|
||||
is_s3fs_umask = true;
|
||||
return 1; // continue for fuse option
|
||||
@ -3429,7 +3429,7 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
|
||||
return 0;
|
||||
}
|
||||
if(0 == STR2NCMP(arg, "retries=")){
|
||||
S3fsCurl::SetRetries(atoi(strchr(arg, '=') + sizeof(char)));
|
||||
S3fsCurl::SetRetries(static_cast<int>(s3fs_strtoul(strchr(arg, '=') + sizeof(char))));
|
||||
return 0;
|
||||
}
|
||||
if(0 == STR2NCMP(arg, "use_cache=")){
|
||||
@ -3441,7 +3441,7 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
|
||||
return 0;
|
||||
}
|
||||
if(0 == STR2NCMP(arg, "multireq_max=")){
|
||||
long maxreq = (long)atoi(strchr(arg, '=') + sizeof(char));
|
||||
long maxreq = static_cast<long>(s3fs_strtoul(strchr(arg, '=') + sizeof(char)));
|
||||
S3fsMultiCurl::SetMaxMultiRequest(maxreq);
|
||||
return 0;
|
||||
}
|
||||
@ -3454,10 +3454,10 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
|
||||
return 0;
|
||||
}
|
||||
if(0 == strcmp(arg, "use_rrs") || 0 == STR2NCMP(arg, "use_rrs=")){
|
||||
int rrs = 1;
|
||||
size_t rrs = 1;
|
||||
// for an old format.
|
||||
if(0 == STR2NCMP(arg, "use_rrs=")){
|
||||
rrs = atoi(strchr(arg, '=') + sizeof(char));
|
||||
rrs = s3fs_strtoul(strchr(arg, '=') + sizeof(char));
|
||||
}
|
||||
if(0 == rrs){
|
||||
S3fsCurl::SetUseRrs(false);
|
||||
@ -3474,10 +3474,10 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
|
||||
return 0;
|
||||
}
|
||||
if(0 == strcmp(arg, "use_sse") || 0 == STR2NCMP(arg, "use_sse=")){
|
||||
int sse = 1;
|
||||
size_t sse = 1;
|
||||
// for an old format.
|
||||
if(0 == STR2NCMP(arg, "use_sse=")){
|
||||
sse = atoi(strchr(arg, '=') + sizeof(char));
|
||||
sse = s3fs_strtoul(strchr(arg, '=') + sizeof(char));
|
||||
}
|
||||
if(0 == sse){
|
||||
S3fsCurl::SetUseSse(false);
|
||||
@ -3494,7 +3494,7 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
|
||||
return 0;
|
||||
}
|
||||
if(0 == STR2NCMP(arg, "ssl_verify_hostname=")){
|
||||
long sslvh = strtol(strchr(arg, '=') + sizeof(char), 0, 10);
|
||||
long sslvh = static_cast<long>(s3fs_strtoul(strchr(arg, '=') + sizeof(char)));
|
||||
if(-1 == S3fsCurl::SetSslVerifyHostname(sslvh)){
|
||||
fprintf(stderr, "%s: poorly formed argument to option: ssl_verify_hostname\n",
|
||||
program_name.c_str());
|
||||
@ -3512,7 +3512,7 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
|
||||
return 0;
|
||||
}
|
||||
if(0 == STR2NCMP(arg, "public_bucket=")){
|
||||
long pubbucket = strtol(strchr(arg, '=') + sizeof(char), 0, 10);
|
||||
size_t pubbucket = s3fs_strtoul(strchr(arg, '=') + sizeof(char));
|
||||
if(1 == pubbucket){
|
||||
S3fsCurl::SetPublicBucket(true);
|
||||
}else if(0 == pubbucket){
|
||||
@ -3533,22 +3533,22 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
|
||||
return 0;
|
||||
}
|
||||
if(0 == STR2NCMP(arg, "connect_timeout=")){
|
||||
long contimeout = strtol(strchr(arg, '=') + sizeof(char), 0, 10);
|
||||
long contimeout = static_cast<long>(s3fs_strtoul(strchr(arg, '=') + sizeof(char)));
|
||||
S3fsCurl::SetConnectTimeout(contimeout);
|
||||
return 0;
|
||||
}
|
||||
if(0 == STR2NCMP(arg, "readwrite_timeout=")){
|
||||
time_t rwtimeout = (time_t)strtoul(strchr(arg, '=') + sizeof(char), 0, 10);
|
||||
time_t rwtimeout = static_cast<time_t>(s3fs_strtoul(strchr(arg, '=') + sizeof(char)));
|
||||
S3fsCurl::SetReadwriteTimeout(rwtimeout);
|
||||
return 0;
|
||||
}
|
||||
if(0 == STR2NCMP(arg, "max_stat_cache_size=")){
|
||||
unsigned long cache_size = strtoul(strchr(arg, '=') + sizeof(char), 0, 10);
|
||||
unsigned long cache_size = static_cast<unsigned long>(s3fs_strtoul(strchr(arg, '=') + sizeof(char)));
|
||||
StatCache::getStatCacheData()->SetCacheSize(cache_size);
|
||||
return 0;
|
||||
}
|
||||
if(0 == STR2NCMP(arg, "stat_cache_expire=")){
|
||||
time_t expr_time = strtoul(strchr(arg, '=') + sizeof(char), 0, 10);
|
||||
time_t expr_time = static_cast<time_t>(s3fs_strtoul(strchr(arg, '=') + sizeof(char)));
|
||||
StatCache::getStatCacheData()->SetExpireTime(expr_time);
|
||||
return 0;
|
||||
}
|
||||
@ -3565,7 +3565,7 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
|
||||
return 0;
|
||||
}
|
||||
if(0 == STR2NCMP(arg, "parallel_count=") || 0 == STR2NCMP(arg, "parallel_upload=")){
|
||||
int maxpara = (int)strtoul(strchr(arg, '=') + sizeof(char), 0, 10);
|
||||
int maxpara = static_cast<int>(s3fs_strtoul(strchr(arg, '=') + sizeof(char)));
|
||||
if(0 >= maxpara){
|
||||
fprintf(stderr, "%s: argument should be over 1: parallel_count\n",
|
||||
program_name.c_str());
|
||||
@ -3575,7 +3575,7 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
|
||||
return 0;
|
||||
}
|
||||
if(0 == STR2NCMP(arg, "fd_page_size=")){
|
||||
ssize_t pagesize = static_cast<ssize_t>(strtoul(strchr(arg, '=') + sizeof(char), 0, 10));
|
||||
ssize_t pagesize = static_cast<ssize_t>(s3fs_strtoul(strchr(arg, '=') + sizeof(char)));
|
||||
if((1024 * 1024) >= pagesize){
|
||||
fprintf(stderr, "%s: argument should be over 1MB: fd_page_size\n",
|
||||
program_name.c_str());
|
||||
|
||||
@ -618,9 +618,52 @@ bool delete_files_in_dir(const char* dir, bool is_remove_own)
|
||||
//-------------------------------------------------------------------
|
||||
// Utility functions for convert
|
||||
//-------------------------------------------------------------------
|
||||
size_t s3fs_strtoul(const char* str, bool is_base_16)
|
||||
{
|
||||
if(!str || '\0' == *str){
|
||||
return 0;
|
||||
}
|
||||
size_t result;
|
||||
bool chk_space;
|
||||
bool chk_base16_prefix;
|
||||
for(result = 0, chk_space = false, chk_base16_prefix = false; '\0' != *str; str++){
|
||||
// check head space
|
||||
if(!chk_space && isspace(*str)){
|
||||
continue;
|
||||
}else if(!chk_space){
|
||||
chk_space = true;
|
||||
}
|
||||
// check prefix for base 16
|
||||
if(!chk_base16_prefix){
|
||||
chk_base16_prefix = true;
|
||||
if('0' == *str && ('x' == str[1] || 'X' == str[1])){
|
||||
is_base_16 = true;
|
||||
str++;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
// check like isalnum and set data
|
||||
result *= (is_base_16 ? 16 : 10);
|
||||
if('0' <= *str || '9' < *str){
|
||||
result += static_cast<size_t>(*str - '0');
|
||||
}else if(is_base_16){
|
||||
if('A' <= *str && *str <= 'F'){
|
||||
result += static_cast<size_t>(*str - 'A' + 0x0a);
|
||||
}else if('a' <= *str && *str <= 'f'){
|
||||
result += static_cast<size_t>(*str - 'a' + 0x0a);
|
||||
}else{
|
||||
return 0;
|
||||
}
|
||||
}else{
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
time_t get_mtime(const char *s)
|
||||
{
|
||||
return (time_t) strtoul(s, (char **) NULL, 10);
|
||||
return static_cast<time_t>(s3fs_strtoul(s));
|
||||
}
|
||||
|
||||
time_t get_mtime(headers_t& meta, bool overcheck)
|
||||
@ -637,7 +680,7 @@ time_t get_mtime(headers_t& meta, bool overcheck)
|
||||
|
||||
off_t get_size(const char *s)
|
||||
{
|
||||
return (off_t) strtoul(s, (char **) NULL, 10);
|
||||
return static_cast<off_t>(s3fs_strtoul(s));
|
||||
}
|
||||
|
||||
off_t get_size(headers_t& meta)
|
||||
@ -651,7 +694,7 @@ off_t get_size(headers_t& meta)
|
||||
|
||||
mode_t get_mode(const char *s)
|
||||
{
|
||||
return (mode_t) strtoul(s, (char **) NULL, 10);
|
||||
return static_cast<mode_t>(s3fs_strtoul(s));
|
||||
}
|
||||
|
||||
mode_t get_mode(headers_t& meta, const char* path, bool checkdir, bool forcedir)
|
||||
@ -708,7 +751,7 @@ mode_t get_mode(headers_t& meta, const char* path, bool checkdir, bool forcedir)
|
||||
|
||||
uid_t get_uid(const char *s)
|
||||
{
|
||||
return (uid_t) strtoul(s, (char **) NULL, 10);
|
||||
return static_cast<uid_t>(s3fs_strtoul(s));
|
||||
}
|
||||
|
||||
uid_t get_uid(headers_t& meta)
|
||||
@ -724,7 +767,7 @@ uid_t get_uid(headers_t& meta)
|
||||
|
||||
gid_t get_gid(const char *s)
|
||||
{
|
||||
return (gid_t) strtoul(s, (char **) NULL, 10);
|
||||
return static_cast<gid_t>(s3fs_strtoul(s));
|
||||
}
|
||||
|
||||
gid_t get_gid(headers_t& meta)
|
||||
|
||||
@ -95,6 +95,7 @@ std::string mybasename(std::string path);
|
||||
int mkdirp(const std::string& path, mode_t mode);
|
||||
bool delete_files_in_dir(const char* dir, bool is_remove_own);
|
||||
|
||||
size_t s3fs_strtoul(const char* str, bool is_base_16 = false);
|
||||
time_t get_mtime(const char *s);
|
||||
time_t get_mtime(headers_t& meta, bool overcheck = true);
|
||||
off_t get_size(const char *s);
|
||||
|
||||
Loading…
Reference in New Issue
Block a user