diff --git a/src/fdcache_entity.cpp b/src/fdcache_entity.cpp index 4ae67bd..33cb0cd 100644 --- a/src/fdcache_entity.cpp +++ b/src/fdcache_entity.cpp @@ -1359,7 +1359,7 @@ int FdEntity::NoCacheCompleteMultipartPost(PseudoFdInfo* pseudo_obj) // clear multipart upload info pseudo_obj->ClearUntreated(); - pseudo_obj->ClearUploadInfo(false); + pseudo_obj->ClearUploadInfo(); return 0; } @@ -1421,7 +1421,7 @@ int FdEntity::RowFlush(int fd, const char* tpath, bool force_sync) // No multipart upload result = RowFlushNoMultipart(pseudo_obj, tpath); }else if(FdEntity::streamupload){ - // Stream maultipart upload + // Stream multipart upload result = RowFlushStreamMultipart(pseudo_obj, tpath); }else if(FdEntity::mixmultipart){ // Mix multipart upload @@ -1793,8 +1793,7 @@ int FdEntity::RowFlushStreamMultipart(PseudoFdInfo* pseudo_obj, const char* tpat // // Check total size for downloading and Download // - total_mp_part_list mptoal; - off_t total_download_size = mptoal(to_download_list); + off_t total_download_size = total_mp_part_list(to_download_list); if(0 < total_download_size){ // // Check if there is enough free disk space for the total download size @@ -1868,13 +1867,13 @@ int FdEntity::RowFlushStreamMultipart(PseudoFdInfo* pseudo_obj, const char* tpat if(!pseudo_obj->ParallelMultipartUploadAll(path.c_str(), to_upload_list, to_copy_list, result)){ S3FS_PRN_ERR("Failed to upload multipart parts."); pseudo_obj->ClearUntreated(); - pseudo_obj->ClearUploadInfo(false); // clear multipart upload info + pseudo_obj->ClearUploadInfo(); // clear multipart upload info return -EIO; } if(0 != result){ S3FS_PRN_ERR("An error(%d) occurred in some threads that were uploading parallel multiparts, but continue to clean up..", result); pseudo_obj->ClearUntreated(); - pseudo_obj->ClearUploadInfo(false); // clear multipart upload info + pseudo_obj->ClearUploadInfo(); // clear multipart upload info return result; } @@ -1886,20 +1885,20 @@ int FdEntity::RowFlushStreamMultipart(PseudoFdInfo* pseudo_obj, const char* tpat if(!pseudo_obj->GetUploadId(upload_id) || !pseudo_obj->GetEtaglist(etaglist)){ S3FS_PRN_ERR("There is no upload id or etag list."); pseudo_obj->ClearUntreated(); - pseudo_obj->ClearUploadInfo(false); // clear multipart upload info + pseudo_obj->ClearUploadInfo(); // clear multipart upload info return -EIO; }else{ S3fsCurl s3fscurl(true); if(0 != (result = s3fscurl.CompleteMultipartPostRequest(path.c_str(), upload_id, etaglist))){ S3FS_PRN_ERR("failed to complete multipart upload by errno(%d)", result); pseudo_obj->ClearUntreated(); - pseudo_obj->ClearUploadInfo(false); // clear multipart upload info + pseudo_obj->ClearUploadInfo(); // clear multipart upload info return result; } s3fscurl.DestroyCurlHandle(); } pseudo_obj->ClearUntreated(); - pseudo_obj->ClearUploadInfo(false); // clear multipart upload info + pseudo_obj->ClearUploadInfo(); // clear multipart upload info // put pending headers if(0 != (result = UploadPendingMeta())){ diff --git a/src/fdcache_fdinfo.cpp b/src/fdcache_fdinfo.cpp index 00a717c..5dab4b1 100644 --- a/src/fdcache_fdinfo.cpp +++ b/src/fdcache_fdinfo.cpp @@ -30,7 +30,6 @@ #include "s3fs.h" #include "fdcache_fdinfo.h" #include "fdcache_pseudofd.h" -#include "autolock.h" #include "curl.h" #include "string_util.h" #include "threadpoolman.h" @@ -162,23 +161,23 @@ bool PseudoFdInfo::Clear() pseudo_fd = -1; physical_fd = -1; - CloseUploadFd(true); // [NOTE] already destroy mutex, then do not lock it. + CloseUploadFd(AutoLock::ALREADY_LOCKED); // [NOTE] already destroy mutex, then do not lock it. return true; } -void PseudoFdInfo::CloseUploadFd(bool lock_already_held) +void PseudoFdInfo::CloseUploadFd(AutoLock::Type type) { - AutoLock auto_lock(&upload_list_lock, lock_already_held ? AutoLock::ALREADY_LOCKED : AutoLock::NONE); + AutoLock auto_lock(&upload_list_lock, type); if(-1 != upload_fd){ close(upload_fd); } } -bool PseudoFdInfo::OpenUploadFd(bool lock_already_held) +bool PseudoFdInfo::OpenUploadFd(AutoLock::Type type) { - AutoLock auto_lock(&upload_list_lock, lock_already_held ? AutoLock::ALREADY_LOCKED : AutoLock::NONE); + AutoLock auto_lock(&upload_list_lock, type); if(-1 != upload_fd){ // already initialized @@ -239,9 +238,9 @@ bool PseudoFdInfo::Readable() const return true; } -bool PseudoFdInfo::ClearUploadInfo(bool is_cancel_mp, bool lock_already_held) +bool PseudoFdInfo::ClearUploadInfo(bool is_cancel_mp, AutoLock::Type type) { - AutoLock auto_lock(&upload_list_lock, lock_already_held ? AutoLock::ALREADY_LOCKED : AutoLock::NONE); + AutoLock auto_lock(&upload_list_lock, type); if(is_cancel_mp){ // [TODO] @@ -259,11 +258,11 @@ bool PseudoFdInfo::ClearUploadInfo(bool is_cancel_mp, bool lock_already_held) return true; } -bool PseudoFdInfo::InitialUploadInfo(const std::string& id, bool lock_already_held) +bool PseudoFdInfo::InitialUploadInfo(const std::string& id, AutoLock::Type type) { - AutoLock auto_lock(&upload_list_lock, lock_already_held ? AutoLock::ALREADY_LOCKED : AutoLock::NONE); + AutoLock auto_lock(&upload_list_lock, type); - if(!ClearUploadInfo(true, true)){ + if(!ClearUploadInfo(true, AutoLock::ALREADY_LOCKED)){ return false; } upload_id = id; @@ -342,9 +341,9 @@ bool PseudoFdInfo::AppendUploadPart(off_t start, off_t size, bool is_copy, etagp return true; } -void PseudoFdInfo::ClearUntreated(bool lock_already_held) +void PseudoFdInfo::ClearUntreated(AutoLock::Type type) { - AutoLock auto_lock(&upload_list_lock, lock_already_held ? AutoLock::ALREADY_LOCKED : AutoLock::NONE); + AutoLock auto_lock(&upload_list_lock, type); untreated_list.ClearAll(); } @@ -413,7 +412,7 @@ static bool filepart_partnum_compare(const filepart& src1, const filepart& src2) return (src1.get_part_number() <= src2.get_part_number()); } -bool PseudoFdInfo::InsertUploadPart(off_t start, off_t size, int part_num, bool is_copy, etagpair** ppetag, bool lock_already_held) +bool PseudoFdInfo::InsertUploadPart(off_t start, off_t size, int part_num, bool is_copy, etagpair** ppetag, AutoLock::Type type) { //S3FS_PRN_DBG("[start=%lld][size=%lld][part_num=%d][is_copy=%s]", static_cast(start), static_cast(size), part_num, (is_copy ? "true" : "false")); @@ -426,7 +425,7 @@ bool PseudoFdInfo::InsertUploadPart(off_t start, off_t size, int part_num, bool return false; } - AutoLock auto_lock(&upload_list_lock, lock_already_held ? AutoLock::ALREADY_LOCKED : AutoLock::NONE); + AutoLock auto_lock(&upload_list_lock, type); // insert new part etag_entities.push_back(etagpair(NULL, part_num)); @@ -455,14 +454,14 @@ bool PseudoFdInfo::ParallelMultipartUpload(const char* path, const mp_part_list_ // nothing to do return true; } - if(!OpenUploadFd(true)){ + if(!OpenUploadFd(AutoLock::ALREADY_LOCKED)){ return false; } for(mp_part_list_t::const_iterator iter = mplist.begin(); iter != mplist.end(); ++iter){ // Insert upload part etagpair* petag = NULL; - if(!InsertUploadPart(iter->start, iter->size, iter->part_num, is_copy, &petag, true)){ + if(!InsertUploadPart(iter->start, iter->size, iter->part_num, is_copy, &petag, AutoLock::ALREADY_LOCKED)){ S3FS_PRN_ERR("Failed to insert insert upload part(path=%s, start=%lld, size=%lld, part=%d, copy=%s) to mplist", SAFESTRPTR(path), static_cast(iter->start), static_cast(iter->size), iter->part_num, (is_copy ? "true" : "false")); return false; } @@ -503,7 +502,7 @@ bool PseudoFdInfo::ParallelMultipartUploadAll(const char* path, const mp_part_li result = 0; - if(!OpenUploadFd(true)){ + if(!OpenUploadFd(AutoLock::ALREADY_LOCKED)){ return false; } @@ -610,7 +609,7 @@ ssize_t PseudoFdInfo::UploadBoundaryLastUntreatedArea(const char* path, headers_ S3FS_PRN_ERR("failed to setup multipart upload(create upload id) by errno(%d)", result); return result; } - if(!InitialUploadInfo(upload_id, true)){ + if(!InitialUploadInfo(upload_id, AutoLock::ALREADY_LOCKED)){ S3FS_PRN_ERR("failed to setup multipart upload(set upload id to object)"); return result; } @@ -897,7 +896,7 @@ bool PseudoFdInfo::ExtractUploadPartsFromAllArea(mp_part_list_t& to_upload_list, S3FS_PRN_ERR("The uploaded list may not be the boundary for the maximum multipart upload size. No further processing is possible."); return false; } - // Set this iterator to ovrelap iter + // Set this iterator to overlap iter overlap_uploaded_iter = uploaded_iter; }else if((cur_start + cur_size - 1) < uploaded_iter->startpos){ @@ -954,7 +953,7 @@ bool PseudoFdInfo::ExtractUploadPartsFromAllArea(mp_part_list_t& to_upload_list, // S3FS_PRN_DBG("Cancel upload: start=%lld, size=%lld", static_cast(overlap_uploaded_iter->startpos), static_cast(overlap_uploaded_iter->size)); cancel_upload_list.push_back(*overlap_uploaded_iter); // add this uploaded area to cancel_upload_list - upload_list.erase(overlap_uploaded_iter); // remove it from upload_list + uploaded_iter = upload_list.erase(overlap_uploaded_iter); // remove it from upload_list S3FS_PRN_DBG("To upload: start=%lld, size=%lld", static_cast(cur_start), static_cast(cur_size)); to_upload_list.push_back(mp_part(cur_start, cur_size, part_num)); // add new uploading area to list diff --git a/src/fdcache_fdinfo.h b/src/fdcache_fdinfo.h index 86b064d..c738673 100644 --- a/src/fdcache_fdinfo.h +++ b/src/fdcache_fdinfo.h @@ -24,6 +24,7 @@ #include "fdcache_untreated.h" #include "psemaphore.h" #include "metaheader.h" +#include "autolock.h" //------------------------------------------------ // Structure of parameters to pass to thread @@ -73,12 +74,12 @@ class PseudoFdInfo static void* MultipartUploadThreadWorker(void* arg); bool Clear(); - void CloseUploadFd(bool lock_already_held = false); - bool OpenUploadFd(bool lock_already_held = false); + void CloseUploadFd(AutoLock::Type type = AutoLock::NONE); + bool OpenUploadFd(AutoLock::Type type = AutoLock::NONE); bool GetLastUpdateUntreatedPart(off_t& start, off_t& size); bool ReplaceLastUpdateUntreatedPart(off_t front_start, off_t front_size, off_t behind_start, off_t behind_size); bool ParallelMultipartUpload(const char* path, const mp_part_list_t& mplist, bool is_copy); - bool InsertUploadPart(off_t start, off_t size, int part_num, bool is_copy, etagpair** ppetag, bool lock_already_held = false); + bool InsertUploadPart(off_t start, off_t size, int part_num, bool is_copy, etagpair** ppetag, AutoLock::Type type = AutoLock::NONE); int WaitAllThreadsExit(); bool ExtractUploadPartsFromUntreatedArea(off_t& untreated_start, off_t& untreated_size, mp_part_list_t& to_upload_list, filepart_list_t& cancel_upload_list, off_t max_mp_size); @@ -93,8 +94,8 @@ class PseudoFdInfo bool Readable() const; bool Set(int fd, int open_flags); - bool ClearUploadInfo(bool is_clear_part = false, bool lock_already_held = false); - bool InitialUploadInfo(const std::string& id, bool lock_already_held = false); + bool ClearUploadInfo(bool is_clear_part = false, AutoLock::Type type = AutoLock::NONE); + bool InitialUploadInfo(const std::string& id, AutoLock::Type type = AutoLock::NONE); bool IsUploading() const { return !upload_id.empty(); } bool GetUploadId(std::string& id) const; @@ -102,7 +103,7 @@ class PseudoFdInfo bool AppendUploadPart(off_t start, off_t size, bool is_copy = false, etagpair** ppetag = NULL); - void ClearUntreated(bool lock_already_held = false); + void ClearUntreated(AutoLock::Type type = AutoLock::NONE); bool ClearUntreated(off_t start, off_t size); bool GetLastUntreated(off_t& start, off_t& size, off_t max_size, off_t min_size = MIN_MULTIPART_SIZE); bool AddUntreated(off_t start, off_t size); diff --git a/src/fdcache_untreated.cpp b/src/fdcache_untreated.cpp index 87bbd2f..433c156 100644 --- a/src/fdcache_untreated.cpp +++ b/src/fdcache_untreated.cpp @@ -224,7 +224,7 @@ bool UntreatedParts::ReplaceLastUpdatePart(off_t start, off_t size) iter->start = start; iter->size = size; }else{ - untreated_list.erase(iter); + iter = untreated_list.erase(iter); } return true; } diff --git a/src/types.h b/src/types.h index 2372a30..1d30060 100644 --- a/src/types.h +++ b/src/types.h @@ -326,17 +326,14 @@ struct mp_part typedef std::list mp_part_list_t; -struct total_mp_part_list +inline off_t total_mp_part_list(const mp_part_list_t& mplist) { - off_t operator()(const mp_part_list_t& mplist) const - { - off_t size = 0; - for(mp_part_list_t::const_iterator iter = mplist.begin(); iter != mplist.end(); ++iter){ - size += iter->size; - } - return size; + off_t size = 0; + for(mp_part_list_t::const_iterator iter = mplist.begin(); iter != mplist.end(); ++iter){ + size += iter->size; } -}; + return size; +} //------------------------------------------------------------------- // mimes_t