octoleo/s3fs-fuse
1
0
Fork 0
mirror of https://github.com/s3fs-fuse/s3fs-fuse.git synced 2025-02-03 02:58:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

Make fdpage a value type in fdpage_list_t

Browse Source 
This simplifies memory management.
This commit is contained in:
Andrew Gaul 2019-06-16 18:19:44 -07:00
parent 511d223468
commit 455e29cbea
2 changed files with 73 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

149
src/fdcache.cpp
View File

@ -241,9 +241,6 @@ bool CacheFileStat::Release()
//------------------------------------------------ //------------------------------------------------
void PageList::FreeList(fdpage_list_t& list) void PageList::FreeList(fdpage_list_t& list)
{ {
for(fdpage_list_t::iterator iter = list.begin(); iter != list.end(); iter = list.erase(iter)){
delete (*iter);
}
list.clear(); list.clear();
} }
@ -265,7 +262,7 @@ void PageList::Clear()
bool PageList::Init(off_t size, bool is_loaded) bool PageList::Init(off_t size, bool is_loaded)
{ {
Clear(); Clear();
fdpage* page = new fdpage(0, size, is_loaded); fdpage page(0, size, is_loaded);
pages.push_back(page); pages.push_back(page);
return true; return true;
} }
@ -276,7 +273,7 @@ off_t PageList::Size() const
return 0; return 0;
} }
fdpage_list_t::const_reverse_iterator riter = pages.rbegin(); fdpage_list_t::const_reverse_iterator riter = pages.rbegin();
return (*riter)->next(); return riter->next();
} }
bool PageList::Compress() bool PageList::Compress()
@ -286,17 +283,16 @@ bool PageList::Compress()
for(fdpage_list_t::iterator iter = pages.begin(); iter != pages.end(); ){ for(fdpage_list_t::iterator iter = pages.begin(); iter != pages.end(); ){
if(is_first){ if(is_first){
is_first = false; is_first = false;
is_last_loaded = (*iter)->loaded; is_last_loaded = iter->loaded;
++iter; ++iter;
}else{ }else{
if(is_last_loaded == (*iter)->loaded){ if(is_last_loaded == iter->loaded){
fdpage_list_t::iterator biter = iter; fdpage_list_t::iterator biter = iter;
--biter; --biter;
(*biter)->bytes += (*iter)->bytes; biter->bytes += iter->bytes;
delete *iter;
iter = pages.erase(iter); iter = pages.erase(iter);
}else{ }else{
is_last_loaded = (*iter)->loaded; is_last_loaded = iter->loaded;
++iter; ++iter;
} }
} }
@ -307,13 +303,13 @@ bool PageList::Compress()
bool PageList::Parse(off_t new_pos) bool PageList::Parse(off_t new_pos)
{ {
for(fdpage_list_t::iterator iter = pages.begin(); iter != pages.end(); ++iter){ for(fdpage_list_t::iterator iter = pages.begin(); iter != pages.end(); ++iter){
if(new_pos == (*iter)->offset){ if(new_pos == iter->offset){
// nothing to do // nothing to do
return true; return true;
}else if((*iter)->offset < new_pos && new_pos < (*iter)->next()){ }else if(iter->offset < new_pos && new_pos < iter->next()){
fdpage* page = new fdpage((*iter)->offset, new_pos - (*iter)->offset, (*iter)->loaded); fdpage page(iter->offset, new_pos - iter->offset, iter->loaded);
(*iter)->bytes -= (new_pos - (*iter)->offset); iter->bytes -= (new_pos - iter->offset);
(*iter)->offset = new_pos; iter->offset = new_pos;
pages.insert(iter, page); pages.insert(iter, page);
return true; return true;
} }
@ -330,20 +326,19 @@ bool PageList::Resize(off_t size, bool is_loaded)
}else if(total < size){ }else if(total < size){
// add new area // add new area
fdpage* page = new fdpage(total, (size - total), is_loaded); fdpage page(total, (size - total), is_loaded);
pages.push_back(page); pages.push_back(page);
}else if(size < total){ }else if(size < total){
// cut area // cut area
for(fdpage_list_t::iterator iter = pages.begin(); iter != pages.end(); ){ for(fdpage_list_t::iterator iter = pages.begin(); iter != pages.end(); ){
if((*iter)->next() <= size){ if(iter->next() <= size){
++iter; ++iter;
}else{ }else{
if(size <= (*iter)->offset){ if(size <= iter->offset){
delete *iter;
iter = pages.erase(iter); iter = pages.erase(iter);
}else{ }else{
(*iter)->bytes = size - (*iter)->offset; iter->bytes = size - iter->offset;
} }
} }
} }
@ -357,13 +352,13 @@ bool PageList::Resize(off_t size, bool is_loaded)
bool PageList::IsPageLoaded(off_t start, off_t size) const bool PageList::IsPageLoaded(off_t start, off_t size) const
{ {
for(fdpage_list_t::const_iterator iter = pages.begin(); iter != pages.end(); ++iter){ for(fdpage_list_t::const_iterator iter = pages.begin(); iter != pages.end(); ++iter){
if((*iter)->end() < start){ if(iter->end() < start){
continue; continue;
} }
if(!(*iter)->loaded){ if(!iter->loaded){
return false; return false;
} }
if(0 != size && start + size <= (*iter)->next()){ if(0 != size && start + size <= iter->next()){
break; break;
} }
} }
@ -395,12 +390,12 @@ bool PageList::SetPageLoadedStatus(off_t start, off_t size, bool is_loaded, bool
// set loaded flag // set loaded flag
for(fdpage_list_t::iterator iter = pages.begin(); iter != pages.end(); ++iter){ for(fdpage_list_t::iterator iter = pages.begin(); iter != pages.end(); ++iter){
if((*iter)->end() < start){ if(iter->end() < start){
continue; continue;
}else if(start + size <= (*iter)->offset){ }else if(start + size <= iter->offset){
break; break;
}else{ }else{
(*iter)->loaded = is_loaded; iter->loaded = is_loaded;
} }
} }
} }
@ -411,10 +406,10 @@ bool PageList::SetPageLoadedStatus(off_t start, off_t size, bool is_loaded, bool
bool PageList::FindUnloadedPage(off_t start, off_t& resstart, off_t& ressize) const bool PageList::FindUnloadedPage(off_t start, off_t& resstart, off_t& ressize) const
{ {
for(fdpage_list_t::const_iterator iter = pages.begin(); iter != pages.end(); ++iter){ for(fdpage_list_t::const_iterator iter = pages.begin(); iter != pages.end(); ++iter){
if(start <= (*iter)->end()){ if(start <= iter->end()){
if(!(*iter)->loaded){ if(!iter->loaded){
resstart = (*iter)->offset; resstart = iter->offset;
ressize = (*iter)->bytes; ressize = iter->bytes;
return true; return true;
} }
} }
@ -427,27 +422,27 @@ off_t PageList::GetTotalUnloadedPageSize(off_t start, off_t size) const
off_t restsize = 0; off_t restsize = 0;
off_t next = start + size; off_t next = start + size;
for(fdpage_list_t::const_iterator iter = pages.begin(); iter != pages.end(); ++iter){ for(fdpage_list_t::const_iterator iter = pages.begin(); iter != pages.end(); ++iter){
if((*iter)->next() <= start){ if(iter->next() <= start){
continue; continue;
} }
if(next <= (*iter)->offset){ if(next <= iter->offset){
break; break;
} }
if((*iter)->loaded){ if(iter->loaded){
continue; continue;
} }
off_t tmpsize; off_t tmpsize;
if((*iter)->offset <= start){ if(iter->offset <= start){
if((*iter)->next() <= next){ if(iter->next() <= next){
tmpsize = ((*iter)->next() - start); tmpsize = (iter->next() - start);
}else{ }else{
tmpsize = next - start; // = size tmpsize = next - start; // = size
} }
}else{ }else{
if((*iter)->next() <= next){ if(iter->next() <= next){
tmpsize = (*iter)->next() - (*iter)->offset; // = (*iter)->bytes tmpsize = iter->next() - iter->offset; // = iter->bytes
}else{ }else{
tmpsize = next - (*iter)->offset; tmpsize = next - iter->offset;
} }
} }
restsize += tmpsize; restsize += tmpsize;
@ -466,28 +461,28 @@ int PageList::GetUnloadedPages(fdpage_list_t& unloaded_list, off_t start, off_t
off_t next = start + size; off_t next = start + size;
for(fdpage_list_t::const_iterator iter = pages.begin(); iter != pages.end(); ++iter){ for(fdpage_list_t::const_iterator iter = pages.begin(); iter != pages.end(); ++iter){
if((*iter)->next() <= start){ if(iter->next() <= start){
continue; continue;
} }
if(next <= (*iter)->offset){ if(next <= iter->offset){
break; break;
} }
if((*iter)->loaded){ if(iter->loaded){
continue; // already loaded continue; // already loaded
} }
// page area // page area
off_t page_start = max((*iter)->offset, start); off_t page_start = max(iter->offset, start);
off_t page_next = min((*iter)->next(), next); off_t page_next = min(iter->next(), next);
off_t page_size = page_next - page_start; off_t page_size = page_next - page_start;
// add list // add list
fdpage_list_t::reverse_iterator riter = unloaded_list.rbegin(); fdpage_list_t::reverse_iterator riter = unloaded_list.rbegin();
if(riter != unloaded_list.rend() && (*riter)->next() == page_start){ if(riter != unloaded_list.rend() && riter->next() == page_start){
// merge to before page // merge to before page
(*riter)->bytes += page_size; riter->bytes += page_size;
}else{ }else{
fdpage* page = new fdpage(page_start, page_size, false); fdpage page(page_start, page_size, false);
unloaded_list.push_back(page); unloaded_list.push_back(page);
} }
} }
@ -507,7 +502,7 @@ bool PageList::Serialize(CacheFileStat& file, bool is_output)
ssall << Size(); ssall << Size();
for(fdpage_list_t::iterator iter = pages.begin(); iter != pages.end(); ++iter){ for(fdpage_list_t::iterator iter = pages.begin(); iter != pages.end(); ++iter){
ssall << "\n" << (*iter)->offset << ":" << (*iter)->bytes << ":" << ((*iter)->loaded ? "1" : "0"); ssall << "\n" << iter->offset << ":" << iter->bytes << ":" << (iter->loaded ? "1" : "0");
} }
string strall = ssall.str(); string strall = ssall.str();
@ -602,7 +597,7 @@ void PageList::Dump()
S3FS_PRN_DBG("pages = {"); S3FS_PRN_DBG("pages = {");
for(fdpage_list_t::iterator iter = pages.begin(); iter != pages.end(); ++iter, ++cnt){ for(fdpage_list_t::iterator iter = pages.begin(); iter != pages.end(); ++iter, ++cnt){
S3FS_PRN_DBG(" [%08d] -> {%014lld - %014lld : %s}", cnt, static_cast<long long int>((*iter)->offset), static_cast<long long int>((*iter)->bytes), (*iter)->loaded ? "true" : "false"); S3FS_PRN_DBG(" [%08d] -> {%014lld - %014lld : %s}", cnt, static_cast<long long int>(iter->offset), static_cast<long long int>(iter->bytes), iter->loaded ? "true" : "false");
} }
S3FS_PRN_DBG("}"); S3FS_PRN_DBG("}");
} }
@ -1186,17 +1181,17 @@ int FdEntity::Load(off_t start, off_t size)
fdpage_list_t unloaded_list; fdpage_list_t unloaded_list;
if(0 < pagelist.GetUnloadedPages(unloaded_list, start, size)){ if(0 < pagelist.GetUnloadedPages(unloaded_list, start, size)){
for(fdpage_list_t::iterator iter = unloaded_list.begin(); iter != unloaded_list.end(); ++iter){ for(fdpage_list_t::iterator iter = unloaded_list.begin(); iter != unloaded_list.end(); ++iter){
if(0 != size && start + size <= (*iter)->offset){ if(0 != size && start + size <= iter->offset){
// reached end // reached end
break; break;
} }
// check loading size // check loading size
off_t need_load_size = 0; off_t need_load_size = 0;
if((*iter)->offset < size_orgmeta){ if(iter->offset < size_orgmeta){
// original file size(on S3) is smaller than request. // original file size(on S3) is smaller than request.
need_load_size = ((*iter)->next() <= size_orgmeta ? (*iter)->bytes : (size_orgmeta - (*iter)->offset)); need_load_size = (iter->next() <= size_orgmeta ? iter->bytes : (size_orgmeta - iter->offset));
} }
off_t over_size = (*iter)->bytes - need_load_size; off_t over_size = iter->bytes - need_load_size;
// download // download
if(2 * S3fsCurl::GetMultipartSize() <= need_load_size && !nomultipart){ // default 20MB if(2 * S3fsCurl::GetMultipartSize() <= need_load_size && !nomultipart){ // default 20MB
@ -1206,7 +1201,7 @@ int FdEntity::Load(off_t start, off_t size)
if(120 > S3fsCurl::GetReadwriteTimeout()){ if(120 > S3fsCurl::GetReadwriteTimeout()){
backup = S3fsCurl::SetReadwriteTimeout(120); backup = S3fsCurl::SetReadwriteTimeout(120);
} }
result = S3fsCurl::ParallelGetObjectRequest(path.c_str(), fd, (*iter)->offset, need_load_size); result = S3fsCurl::ParallelGetObjectRequest(path.c_str(), fd, iter->offset, need_load_size);
if(0 != backup){ if(0 != backup){
S3fsCurl::SetReadwriteTimeout(backup); S3fsCurl::SetReadwriteTimeout(backup);
} }
@ -1214,7 +1209,7 @@ int FdEntity::Load(off_t start, off_t size)
// single request // single request
if(0 < need_load_size){ if(0 < need_load_size){
S3fsCurl s3fscurl; S3fsCurl s3fscurl;
result = s3fscurl.GetObjectRequest(path.c_str(), fd, (*iter)->offset, need_load_size); result = s3fscurl.GetObjectRequest(path.c_str(), fd, iter->offset, need_load_size);
}else{ }else{
result = 0; result = 0;
} }
@ -1225,7 +1220,7 @@ int FdEntity::Load(off_t start, off_t size)
// initialize for the area of over original size // initialize for the area of over original size
if(0 < over_size){ if(0 < over_size){
if(0 != (result = FdEntity::FillFile(fd, 0, over_size, (*iter)->offset + need_load_size))){ if(0 != (result = FdEntity::FillFile(fd, 0, over_size, iter->offset + need_load_size))){
S3FS_PRN_ERR("failed to fill rest bytes for fd(%d). errno(%d)", fd, result); S3FS_PRN_ERR("failed to fill rest bytes for fd(%d). errno(%d)", fd, result);
break; break;
} }
@ -1234,7 +1229,7 @@ int FdEntity::Load(off_t start, off_t size)
} }
// Set loaded flag // Set loaded flag
pagelist.SetPageLoadedStatus((*iter)->offset, (*iter)->bytes, true); pagelist.SetPageLoadedStatus(iter->offset, iter->bytes, true);
} }
PageList::FreeList(unloaded_list); PageList::FreeList(unloaded_list);
} }
@ -1286,17 +1281,17 @@ int FdEntity::NoCacheLoadAndPost(off_t start, off_t size)
// loop uploading by multipart // loop uploading by multipart
for(fdpage_list_t::iterator iter = pagelist.pages.begin(); iter != pagelist.pages.end(); ++iter){ for(fdpage_list_t::iterator iter = pagelist.pages.begin(); iter != pagelist.pages.end(); ++iter){
if((*iter)->end() < start){ if(iter->end() < start){
continue; continue;
} }
if(0 != size && start + size <= (*iter)->offset){ if(0 != size && start + size <= iter->offset){
break; break;
} }
// download each multipart size(default 10MB) in unit // download each multipart size(default 10MB) in unit
for(off_t oneread = 0, totalread = ((*iter)->offset < start ? start : 0); totalread < static_cast<off_t>((*iter)->bytes); totalread += oneread){ for(off_t oneread = 0, totalread = (iter->offset < start ? start : 0); totalread < static_cast<off_t>(iter->bytes); totalread += oneread){
int upload_fd = fd; int upload_fd = fd;
off_t offset = (*iter)->offset + totalread; off_t offset = iter->offset + totalread;
oneread = min(static_cast<off_t>((*iter)->bytes) - totalread, S3fsCurl::GetMultipartSize()); oneread = min(static_cast<off_t>(iter->bytes) - totalread, S3fsCurl::GetMultipartSize());
// check rest size is over minimum part size // check rest size is over minimum part size
// //
@ -1306,15 +1301,15 @@ int FdEntity::NoCacheLoadAndPost(off_t start, off_t size)
// we incorporate the final part to the previous part. If the previous part // we incorporate the final part to the previous part. If the previous part
// is over 5GB, we want to even out the last part and the previous part. // is over 5GB, we want to even out the last part and the previous part.
// //
if(((*iter)->bytes - totalread - oneread) < MIN_MULTIPART_SIZE){ if((iter->bytes - totalread - oneread) < MIN_MULTIPART_SIZE){
if(FIVE_GB < ((*iter)->bytes - totalread)){ if(FIVE_GB < iter->bytes - totalread){
oneread = ((*iter)->bytes - totalread) / 2; oneread = (iter->bytes - totalread) / 2;
}else{ }else{
oneread = ((*iter)->bytes - totalread); oneread = iter->bytes - totalread;
} }
} }
if(!(*iter)->loaded){ if(!iter->loaded){
// //
// loading or initializing // loading or initializing
// //
@ -1378,20 +1373,20 @@ int FdEntity::NoCacheLoadAndPost(off_t start, off_t size)
} }
// set loaded flag // set loaded flag
if(!(*iter)->loaded){ if(!iter->loaded){
if((*iter)->offset < start){ if(iter->offset < start){
fdpage* page = new fdpage(((*iter)->offset, start - (*iter)->offset), (*iter)->loaded); fdpage page(iter->offset, start - iter->offset, iter->loaded);
(*iter)->bytes -= (start - (*iter)->offset); iter->bytes -= (start - iter->offset);
(*iter)->offset = start; iter->offset = start;
pagelist.pages.insert(iter, page); pagelist.pages.insert(iter, page);
} }
if(0 != size && start + size < (*iter)->next()){ if(0 != size && start + size < iter->next()){
fdpage* page = new fdpage(((*iter)->offset, (start + size - (*iter)->offset), true)); fdpage page(iter->offset, start + size - iter->offset, true);
(*iter)->bytes -= (start + size - (*iter)->offset); iter->bytes -= (start + size - iter->offset);
(*iter)->offset = start + size; iter->offset = start + size;
pagelist.pages.insert(iter, page); pagelist.pages.insert(iter, page);
}else{ }else{
(*iter)->loaded = true; iter->loaded = true;
} }
} }
} }

2
src/fdcache.h
View File

@ -65,7 +65,7 @@ struct fdpage
off_t next(void) const { return (offset + bytes); } off_t next(void) const { return (offset + bytes); }
off_t end(void) const { return (0 < bytes ? offset + bytes - 1 : 0); } off_t end(void) const { return (0 < bytes ? offset + bytes - 1 : 0); }
}; };
typedef std::list<struct fdpage*> fdpage_list_t; typedef std::list<struct fdpage> fdpage_list_t;
class FdEntity; class FdEntity;