Changes codes

1) Changed buffer size for file size
   Changes a internal buffer size from size_t to offt_t.
   It is a bug for 32 bit OS enviroment.



git-svn-id: http://s3fs.googlecode.com/svn/trunk@496 df820570-a93a-0410-bd06-b72b767a4274
This commit is contained in:
ggtakec@gmail.com 2013-11-17 08:50:41 +00:00
parent 882f13020e
commit 40b9f0a408
7 changed files with 113 additions and 110 deletions

View File

@ -43,6 +43,7 @@
#include "fdcache.h"
#include "s3fs.h"
#include "s3fs_util.h"
#include "string_util.h"
#include "curl.h"
using namespace std;
@ -200,7 +201,7 @@ void PageList::FreeList(fdpage_list_t& list)
list.clear();
}
PageList::PageList(size_t size, bool is_init)
PageList::PageList(off_t size, bool is_init)
{
Init(size, is_init);
}
@ -210,7 +211,7 @@ PageList::~PageList()
Clear();
}
size_t PageList::Size(void) const
off_t PageList::Size(void) const
{
if(0 == pages.size()){
return 0;
@ -219,36 +220,36 @@ size_t PageList::Size(void) const
return ((*riter)->offset + (*riter)->bytes);
}
int PageList::Resize(size_t size, bool is_init)
int PageList::Resize(off_t size, bool is_init)
{
size_t total = Size();
off_t total = Size();
if(0 == total){
Init(size, is_init);
}else if(total < size){
size_t remain = size - total; // remaining bytes
off_t remain = size - total; // remaining bytes
fdpage_list_t::reverse_iterator riter = pages.rbegin();
if((*riter)->bytes < FdManager::GetPageSize()){
// resize last area
remain += (*riter)->bytes; // remaining bytes(without last page)
(*riter)->bytes = remain > FdManager::GetPageSize() ? FdManager::GetPageSize() : remain; // reset page size
(*riter)->bytes = remain > static_cast<off_t>(FdManager::GetPageSize()) ? FdManager::GetPageSize() : static_cast<size_t>(remain); // reset page size
remain -= (*riter)->bytes; // remaining bytes(after last page)
(*riter)->init = is_init;
}
// add new area
for(off_t next = (*riter)->next(); 0 < remain; remain -= size, next += size){
size = remain > FdManager::GetPageSize() ? FdManager::GetPageSize() : remain;
size = remain > static_cast<off_t>(FdManager::GetPageSize()) ? static_cast<off_t>(FdManager::GetPageSize()) : remain;
fdpage* page = new fdpage(next, size, is_init);
pages.push_back(page);
}
}else if(total > size){
for(fdpage_list_t::reverse_iterator riter = pages.rbegin(); riter != pages.rend(); riter++){
if(static_cast<size_t>((*riter)->offset) < size){
(*riter)->bytes = size - (*riter)->offset;
if((*riter)->offset < size){
(*riter)->bytes = static_cast<size_t>(size - (*riter)->offset);
break;
}
}
@ -261,18 +262,18 @@ void PageList::Clear(void)
PageList::FreeList(pages);
}
int PageList::Init(size_t size, bool is_init)
int PageList::Init(off_t size, bool is_init)
{
Clear();
for(size_t total = 0; total < size; total += FdManager::GetPageSize()){
size_t areasize = (total + FdManager::GetPageSize()) < size ? FdManager::GetPageSize() : (size - total);
for(off_t total = 0; total < size; total += FdManager::GetPageSize()){
size_t areasize = (total + FdManager::GetPageSize()) < size ? FdManager::GetPageSize() : static_cast<size_t>(size - total);
fdpage* page = new fdpage(total, areasize, is_init);
pages.push_back(page);
}
return pages.size();
}
bool PageList::IsInit(off_t start, size_t size)
bool PageList::IsInit(off_t start, off_t size)
{
off_t next = start + size;
@ -301,7 +302,7 @@ bool PageList::IsInit(off_t start, size_t size)
return true;
}
bool PageList::SetInit(off_t start, size_t size, bool is_init)
bool PageList::SetInit(off_t start, off_t size, bool is_init)
{
// check size & resize
if(Size() < (start + size)){
@ -427,7 +428,7 @@ bool PageList::Serialize(CacheFileStat& file, bool is_output)
free(ptmp);
return false;
}
size_t total = s3fs_strtoul(oneline.c_str());
off_t total = s3fs_strtoofft(oneline.c_str());
// load each part
bool is_err = false;
@ -439,19 +440,19 @@ bool PageList::Serialize(CacheFileStat& file, bool is_output)
is_err = true;
break;
}
off_t offset = static_cast<off_t>(s3fs_strtoul(part.c_str()));
off_t offset = s3fs_strtoofft(part.c_str());
// size
if(!getline(ssparts, part, ':')){
is_err = true;
break;
}
ssize_t size = static_cast<ssize_t>(s3fs_strtoul(part.c_str()));
off_t size = s3fs_strtoofft(part.c_str());
// init
if(!getline(ssparts, part, ':')){
is_err = true;
break;
}
bool is_init = (1 == s3fs_strtoul(part.c_str()) ? true : false);
bool is_init = (1 == s3fs_strtoofft(part.c_str()) ? true : false);
// add new area
SetInit(offset, size, is_init);
}
@ -464,7 +465,7 @@ bool PageList::Serialize(CacheFileStat& file, bool is_output)
// check size
if(total != Size()){
DPRN("different size(%zu - %zu).", total, Size());
DPRN("different size(%jd - %jd).", (intmax_t)total, (intmax_t)Size());
Clear();
return false;
}
@ -568,14 +569,14 @@ int FdEntity::Dup(void)
return fd;
}
int FdEntity::Open(ssize_t size, time_t time)
int FdEntity::Open(off_t size, time_t time)
{
bool already_opened = false; // already opened fd
bool is_csf_loaded = false; // loaded by cache stat file
bool is_truncate = false; // need to truncate
bool init_value = false; // value for pagelist
FPRNINFO("[path=%s][fd=%d][size=%zd][time=%jd]", path.c_str(), fd, size, (intmax_t)time);
FPRNINFO("[path=%s][fd=%d][size=%jd][time=%jd]", path.c_str(), fd, (intmax_t)size, (intmax_t)time);
if(-1 != fd){
// already opened, needs to increment refcnt.
@ -602,7 +603,7 @@ int FdEntity::Open(ssize_t size, time_t time)
fd = -1;
return (0 == errno ? -EIO : -errno);
}
if((-1 != size && static_cast<size_t>(size) != pagelist.Size()) || static_cast<size_t>(st.st_size) != pagelist.Size()){
if((-1 != size && size != pagelist.Size()) || st.st_size != pagelist.Size()){
is_csf_loaded = false; // reinitializing
if(-1 == size){
size = st.st_size;
@ -719,7 +720,7 @@ int FdEntity::SetMtime(time_t time)
return 0;
}
bool FdEntity::GetSize(size_t& size)
bool FdEntity::GetSize(off_t& size)
{
if(-1 == fd){
return false;
@ -778,11 +779,11 @@ bool FdEntity::SetAllStatus(bool is_enable)
return true;
}
int FdEntity::Load(off_t start, ssize_t size)
int FdEntity::Load(off_t start, off_t size)
{
int result = 0;
FPRNINFO("[path=%s][fd=%d][offset=%jd][size=%zd]", path.c_str(), fd, (intmax_t)start, size);
FPRNINFO("[path=%s][fd=%d][offset=%jd][size=%jd]", path.c_str(), fd, (intmax_t)start, (intmax_t)size);
if(-1 == fd){
return -EBADF;
@ -818,14 +819,14 @@ int FdEntity::Load(off_t start, ssize_t size)
}
// Set init flag
pagelist.SetInit((*iter)->offset, (*iter)->bytes, true);
pagelist.SetInit((*iter)->offset, static_cast<off_t>((*iter)->bytes), true);
}
PageList::FreeList(uninit_list);
}
return result;
}
bool FdEntity::LoadFull(size_t* size, bool force_load)
bool FdEntity::LoadFull(off_t* size, bool force_load)
{
int result;
@ -936,7 +937,7 @@ ssize_t FdEntity::Read(char* bytes, off_t start, size_t size, bool force_load)
}
if(force_load){
AutoLock auto_lock(&fdent_lock);
pagelist.SetInit(start, size, false);
pagelist.SetInit(start, static_cast<off_t>(size), false);
}
// Loading
if(0 != (result = Load(start, size))){
@ -984,7 +985,7 @@ ssize_t FdEntity::Write(const char* bytes, off_t start, size_t size)
is_modify = true;
}
if(0 < wsize){
pagelist.SetInit(start, wsize, true);
pagelist.SetInit(start, static_cast<off_t>(wsize), true);
}
}
return wsize;
@ -1139,11 +1140,11 @@ FdEntity* FdManager::GetFdEntity(const char* path)
return (*iter).second;
}
FdEntity* FdManager::Open(const char* path, ssize_t size, time_t time, bool force_tmpfile, bool is_create)
FdEntity* FdManager::Open(const char* path, off_t size, time_t time, bool force_tmpfile, bool is_create)
{
FdEntity* ent;
FPRNINFO("[path=%s][size=%zd][time=%jd]", SAFESTRPTR(path), size, (intmax_t)time);
FPRNINFO("[path=%s][size=%jd][time=%jd]", SAFESTRPTR(path), (intmax_t)size, (intmax_t)time);
if(!path || '\0' == path[0]){
return NULL;

View File

@ -1,4 +1,4 @@
#ifndef FD_CACHE_H_
#ifndef FD_CACHE_H_
#define FD_CACHE_H_
//------------------------------------------------
@ -57,14 +57,14 @@ class PageList
public:
static void FreeList(fdpage_list_t& list);
PageList(size_t size = 0, bool is_init = false);
PageList(off_t size = 0, bool is_init = false);
~PageList();
size_t Size(void) const;
int Resize(size_t size, bool is_init);
int Init(size_t size, bool is_init);
bool IsInit(off_t start, size_t size);
bool SetInit(off_t start, size_t size, bool is_init = true);
off_t Size(void) const;
int Resize(off_t size, bool is_init);
int Init(off_t size, bool is_init);
bool IsInit(off_t start, off_t size);
bool SetInit(off_t start, off_t size, bool is_init = true);
bool FindUninitPage(off_t start, off_t& resstart, size_t& ressize);
int GetUninitPages(fdpage_list_t& uninit_list, off_t start = 0);
bool Serialize(CacheFileStat& file, bool is_output);
@ -98,18 +98,18 @@ class FdEntity
void Close(void);
bool IsOpen(void) const { return (-1 != fd); }
int Open(ssize_t size = -1, time_t time = -1);
int Open(off_t size = -1, time_t time = -1);
const char* GetPath(void) const { return path.c_str(); }
int GetFd(void) const { return fd; }
int SetMtime(time_t time);
bool GetSize(size_t& size);
bool GetSize(off_t& size);
bool GetMtime(time_t& time);
bool GetStats(struct stat& st);
bool SetAllEnable(void) { return SetAllStatus(true); }
bool SetAllDisable(void) { return SetAllStatus(false); }
bool LoadFull(size_t* size = NULL, bool force_load = false);
int Load(off_t start, ssize_t size);
bool LoadFull(off_t* size = NULL, bool force_load = false);
int Load(off_t start, off_t size);
int RowFlush(const char* tpath, headers_t& meta, bool ow_sse_flg, bool force_sync = false);
int Flush(headers_t& meta, bool ow_sse_flg, bool force_sync = false) { return RowFlush(NULL, meta, ow_sse_flg, force_sync); }
ssize_t Read(char* bytes, off_t start, size_t size, bool force_load = false);
@ -148,7 +148,7 @@ class FdManager
static bool MakeCachePath(const char* path, std::string& cache_path, bool is_create_dir = true);
FdEntity* GetFdEntity(const char* path);
FdEntity* Open(const char* path, ssize_t size = -1, time_t time = -1, bool force_tmpfile = false, bool is_create = true);
FdEntity* Open(const char* path, off_t size = -1, time_t time = -1, bool force_tmpfile = false, bool is_create = true);
FdEntity* ExistOpen(const char* path) { return Open(path, -1, -1, false, false); }
bool Close(FdEntity* ent);
};

View File

@ -727,18 +727,18 @@ static int s3fs_readlink(const char* path, char* buf, size_t size)
return -EIO;
}
// Get size
size_t readsize;
off_t readsize;
if(!ent->GetSize(readsize)){
DPRN("could not get file size(file=%s)", path);
FdManager::get()->Close(ent);
return -EIO;
}
if(size <= readsize){
if(static_cast<off_t>(size) <= readsize){
readsize = size - 1;
}
// Read
ssize_t ressize;
if(0 > (ressize = ent->Read(buf, 0, readsize))){
if(0 > (ressize = ent->Read(buf, 0, static_cast<size_t>(readsize)))){
DPRN("could not read file(file=%s, errno=%zd)", path, ressize);
FdManager::get()->Close(ent);
return static_cast<int>(ressize);
@ -1917,7 +1917,7 @@ static int s3fs_read(const char* path, char* buf, size_t size, off_t offset, str
}
// check real file size
size_t realsize = 0;
off_t realsize = 0;
if(!ent->GetSize(realsize) || 0 >= realsize){
DPRNINFO("file size is 0, so break to read.");
FdManager::get()->Close(ent);
@ -3429,7 +3429,7 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
return 0;
}
if(0 == STR2NCMP(arg, "retries=")){
S3fsCurl::SetRetries(static_cast<int>(s3fs_strtoul(strchr(arg, '=') + sizeof(char))));
S3fsCurl::SetRetries(static_cast<int>(s3fs_strtoofft(strchr(arg, '=') + sizeof(char))));
return 0;
}
if(0 == STR2NCMP(arg, "use_cache=")){
@ -3441,7 +3441,7 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
return 0;
}
if(0 == STR2NCMP(arg, "multireq_max=")){
long maxreq = static_cast<long>(s3fs_strtoul(strchr(arg, '=') + sizeof(char)));
long maxreq = static_cast<long>(s3fs_strtoofft(strchr(arg, '=') + sizeof(char)));
S3fsMultiCurl::SetMaxMultiRequest(maxreq);
return 0;
}
@ -3454,10 +3454,10 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
return 0;
}
if(0 == strcmp(arg, "use_rrs") || 0 == STR2NCMP(arg, "use_rrs=")){
size_t rrs = 1;
off_t rrs = 1;
// for an old format.
if(0 == STR2NCMP(arg, "use_rrs=")){
rrs = s3fs_strtoul(strchr(arg, '=') + sizeof(char));
rrs = s3fs_strtoofft(strchr(arg, '=') + sizeof(char));
}
if(0 == rrs){
S3fsCurl::SetUseRrs(false);
@ -3474,10 +3474,10 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
return 0;
}
if(0 == strcmp(arg, "use_sse") || 0 == STR2NCMP(arg, "use_sse=")){
size_t sse = 1;
off_t sse = 1;
// for an old format.
if(0 == STR2NCMP(arg, "use_sse=")){
sse = s3fs_strtoul(strchr(arg, '=') + sizeof(char));
sse = s3fs_strtoofft(strchr(arg, '=') + sizeof(char));
}
if(0 == sse){
S3fsCurl::SetUseSse(false);
@ -3494,7 +3494,7 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
return 0;
}
if(0 == STR2NCMP(arg, "ssl_verify_hostname=")){
long sslvh = static_cast<long>(s3fs_strtoul(strchr(arg, '=') + sizeof(char)));
long sslvh = static_cast<long>(s3fs_strtoofft(strchr(arg, '=') + sizeof(char)));
if(-1 == S3fsCurl::SetSslVerifyHostname(sslvh)){
fprintf(stderr, "%s: poorly formed argument to option: ssl_verify_hostname\n",
program_name.c_str());
@ -3512,7 +3512,7 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
return 0;
}
if(0 == STR2NCMP(arg, "public_bucket=")){
size_t pubbucket = s3fs_strtoul(strchr(arg, '=') + sizeof(char));
off_t pubbucket = s3fs_strtoofft(strchr(arg, '=') + sizeof(char));
if(1 == pubbucket){
S3fsCurl::SetPublicBucket(true);
}else if(0 == pubbucket){
@ -3533,22 +3533,22 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
return 0;
}
if(0 == STR2NCMP(arg, "connect_timeout=")){
long contimeout = static_cast<long>(s3fs_strtoul(strchr(arg, '=') + sizeof(char)));
long contimeout = static_cast<long>(s3fs_strtoofft(strchr(arg, '=') + sizeof(char)));
S3fsCurl::SetConnectTimeout(contimeout);
return 0;
}
if(0 == STR2NCMP(arg, "readwrite_timeout=")){
time_t rwtimeout = static_cast<time_t>(s3fs_strtoul(strchr(arg, '=') + sizeof(char)));
time_t rwtimeout = static_cast<time_t>(s3fs_strtoofft(strchr(arg, '=') + sizeof(char)));
S3fsCurl::SetReadwriteTimeout(rwtimeout);
return 0;
}
if(0 == STR2NCMP(arg, "max_stat_cache_size=")){
unsigned long cache_size = static_cast<unsigned long>(s3fs_strtoul(strchr(arg, '=') + sizeof(char)));
unsigned long cache_size = static_cast<unsigned long>(s3fs_strtoofft(strchr(arg, '=') + sizeof(char)));
StatCache::getStatCacheData()->SetCacheSize(cache_size);
return 0;
}
if(0 == STR2NCMP(arg, "stat_cache_expire=")){
time_t expr_time = static_cast<time_t>(s3fs_strtoul(strchr(arg, '=') + sizeof(char)));
time_t expr_time = static_cast<time_t>(s3fs_strtoofft(strchr(arg, '=') + sizeof(char)));
StatCache::getStatCacheData()->SetExpireTime(expr_time);
return 0;
}
@ -3565,7 +3565,7 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
return 0;
}
if(0 == STR2NCMP(arg, "parallel_count=") || 0 == STR2NCMP(arg, "parallel_upload=")){
int maxpara = static_cast<int>(s3fs_strtoul(strchr(arg, '=') + sizeof(char)));
int maxpara = static_cast<int>(s3fs_strtoofft(strchr(arg, '=') + sizeof(char)));
if(0 >= maxpara){
fprintf(stderr, "%s: argument should be over 1: parallel_count\n",
program_name.c_str());
@ -3575,7 +3575,7 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
return 0;
}
if(0 == STR2NCMP(arg, "fd_page_size=")){
ssize_t pagesize = static_cast<ssize_t>(s3fs_strtoul(strchr(arg, '=') + sizeof(char)));
size_t pagesize = static_cast<size_t>(s3fs_strtoofft(strchr(arg, '=') + sizeof(char)));
if((1024 * 1024) >= pagesize){
fprintf(stderr, "%s: argument should be over 1MB: fd_page_size\n",
program_name.c_str());

View File

@ -39,6 +39,7 @@
#include "common.h"
#include "s3fs_util.h"
#include "string_util.h"
#include "s3fs.h"
using namespace std;
@ -618,52 +619,9 @@ bool delete_files_in_dir(const char* dir, bool is_remove_own)
//-------------------------------------------------------------------
// Utility functions for convert
//-------------------------------------------------------------------
size_t s3fs_strtoul(const char* str, bool is_base_16)
{
if(!str || '\0' == *str){
return 0;
}
size_t result;
bool chk_space;
bool chk_base16_prefix;
for(result = 0, chk_space = false, chk_base16_prefix = false; '\0' != *str; str++){
// check head space
if(!chk_space && isspace(*str)){
continue;
}else if(!chk_space){
chk_space = true;
}
// check prefix for base 16
if(!chk_base16_prefix){
chk_base16_prefix = true;
if('0' == *str && ('x' == str[1] || 'X' == str[1])){
is_base_16 = true;
str++;
continue;
}
}
// check like isalnum and set data
result *= (is_base_16 ? 16 : 10);
if('0' <= *str || '9' < *str){
result += static_cast<size_t>(*str - '0');
}else if(is_base_16){
if('A' <= *str && *str <= 'F'){
result += static_cast<size_t>(*str - 'A' + 0x0a);
}else if('a' <= *str && *str <= 'f'){
result += static_cast<size_t>(*str - 'a' + 0x0a);
}else{
return 0;
}
}else{
return 0;
}
}
return result;
}
time_t get_mtime(const char *s)
{
return static_cast<time_t>(s3fs_strtoul(s));
return static_cast<time_t>(s3fs_strtoofft(s));
}
time_t get_mtime(headers_t& meta, bool overcheck)
@ -680,7 +638,7 @@ time_t get_mtime(headers_t& meta, bool overcheck)
off_t get_size(const char *s)
{
return static_cast<off_t>(s3fs_strtoul(s));
return s3fs_strtoofft(s);
}
off_t get_size(headers_t& meta)
@ -694,7 +652,7 @@ off_t get_size(headers_t& meta)
mode_t get_mode(const char *s)
{
return static_cast<mode_t>(s3fs_strtoul(s));
return static_cast<mode_t>(s3fs_strtoofft(s));
}
mode_t get_mode(headers_t& meta, const char* path, bool checkdir, bool forcedir)
@ -751,7 +709,7 @@ mode_t get_mode(headers_t& meta, const char* path, bool checkdir, bool forcedir)
uid_t get_uid(const char *s)
{
return static_cast<uid_t>(s3fs_strtoul(s));
return static_cast<uid_t>(s3fs_strtoofft(s));
}
uid_t get_uid(headers_t& meta)
@ -767,7 +725,7 @@ uid_t get_uid(headers_t& meta)
gid_t get_gid(const char *s)
{
return static_cast<gid_t>(s3fs_strtoul(s));
return static_cast<gid_t>(s3fs_strtoofft(s));
}
gid_t get_gid(headers_t& meta)

View File

@ -95,7 +95,6 @@ std::string mybasename(std::string path);
int mkdirp(const std::string& path, mode_t mode);
bool delete_files_in_dir(const char* dir, bool is_remove_own);
size_t s3fs_strtoul(const char* str, bool is_base_16 = false);
time_t get_mtime(const char *s);
time_t get_mtime(headers_t& meta, bool overcheck = true);
off_t get_size(const char *s);

View File

@ -32,6 +32,49 @@ using namespace std;
static const char hexAlphabet[] = "0123456789ABCDEF";
off_t s3fs_strtoofft(const char* str, bool is_base_16)
{
if(!str || '\0' == *str){
return 0;
}
off_t result;
bool chk_space;
bool chk_base16_prefix;
for(result = 0, chk_space = false, chk_base16_prefix = false; '\0' != *str; str++){
// check head space
if(!chk_space && isspace(*str)){
continue;
}else if(!chk_space){
chk_space = true;
}
// check prefix for base 16
if(!chk_base16_prefix){
chk_base16_prefix = true;
if('0' == *str && ('x' == str[1] || 'X' == str[1])){
is_base_16 = true;
str++;
continue;
}
}
// check like isalnum and set data
result *= (is_base_16 ? 16 : 10);
if('0' <= *str || '9' < *str){
result += static_cast<off_t>(*str - '0');
}else if(is_base_16){
if('A' <= *str && *str <= 'F'){
result += static_cast<off_t>(*str - 'A' + 0x0a);
}else if('a' <= *str && *str <= 'f'){
result += static_cast<off_t>(*str - 'a' + 0x0a);
}else{
return 0;
}
}else{
return 0;
}
}
return result;
}
string lower(string s)
{
// change each character of the string to lower case

View File

@ -19,6 +19,8 @@ template<typename T> std::string str(T value) {
return s.str();
}
off_t s3fs_strtoofft(const char* str, bool is_base_16 = false);
std::string trim_left(const std::string &s, const std::string &t = SPACES);
std::string trim_right(const std::string &s, const std::string &t = SPACES);
std::string trim(const std::string &s, const std::string &t = SPACES);