octoleo/s3fs-fuse
1
0
Fork 0
mirror of https://github.com/s3fs-fuse/s3fs-fuse.git synced 2024-12-22 16:58:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

Reflected the result of the review in the code

Browse Source
This commit is contained in:
Takeshi Nakatani 2022-01-14 15:02:49 +00:00 committed by Andrew Gaul
parent d22e1dc018
commit b0eeaa6679
5 changed files with 42 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
src/fdcache_entity.cpp
View File

@ -1359,7 +1359,7 @@ int FdEntity::NoCacheCompleteMultipartPost(PseudoFdInfo* pseudo_obj)
// clear multipart upload info
pseudo_obj->ClearUntreated();
pseudo_obj->ClearUploadInfo(false);
pseudo_obj->ClearUploadInfo();
return 0;
}
@ -1421,7 +1421,7 @@ int FdEntity::RowFlush(int fd, const char* tpath, bool force_sync)
// No multipart upload
result = RowFlushNoMultipart(pseudo_obj, tpath);
}else if(FdEntity::streamupload){
// Stream maultipart upload
// Stream multipart upload
result = RowFlushStreamMultipart(pseudo_obj, tpath);
}else if(FdEntity::mixmultipart){
// Mix multipart upload
@ -1793,8 +1793,7 @@ int FdEntity::RowFlushStreamMultipart(PseudoFdInfo* pseudo_obj, const char* tpat
//
// Check total size for downloading and Download
//
total_mp_part_list mptoal;
off_t total_download_size = mptoal(to_download_list);
off_t total_download_size = total_mp_part_list(to_download_list);
if(0 < total_download_size){
//
// Check if there is enough free disk space for the total download size
@ -1868,13 +1867,13 @@ int FdEntity::RowFlushStreamMultipart(PseudoFdInfo* pseudo_obj, const char* tpat
if(!pseudo_obj->ParallelMultipartUploadAll(path.c_str(), to_upload_list, to_copy_list, result)){
S3FS_PRN_ERR("Failed to upload multipart parts.");
pseudo_obj->ClearUntreated();
pseudo_obj->ClearUploadInfo(false); // clear multipart upload info
pseudo_obj->ClearUploadInfo(); // clear multipart upload info
return -EIO;
}
if(0 != result){
S3FS_PRN_ERR("An error(%d) occurred in some threads that were uploading parallel multiparts, but continue to clean up..", result);
pseudo_obj->ClearUntreated();
pseudo_obj->ClearUploadInfo(false); // clear multipart upload info
pseudo_obj->ClearUploadInfo(); // clear multipart upload info
return result;
}
@ -1886,20 +1885,20 @@ int FdEntity::RowFlushStreamMultipart(PseudoFdInfo* pseudo_obj, const char* tpat
if(!pseudo_obj->GetUploadId(upload_id) || !pseudo_obj->GetEtaglist(etaglist)){
S3FS_PRN_ERR("There is no upload id or etag list.");
pseudo_obj->ClearUntreated();
pseudo_obj->ClearUploadInfo(false); // clear multipart upload info
pseudo_obj->ClearUploadInfo(); // clear multipart upload info
return -EIO;
}else{
S3fsCurl s3fscurl(true);
if(0 != (result = s3fscurl.CompleteMultipartPostRequest(path.c_str(), upload_id, etaglist))){
S3FS_PRN_ERR("failed to complete multipart upload by errno(%d)", result);
pseudo_obj->ClearUntreated();
pseudo_obj->ClearUploadInfo(false); // clear multipart upload info
pseudo_obj->ClearUploadInfo(); // clear multipart upload info
return result;
}
s3fscurl.DestroyCurlHandle();
}
pseudo_obj->ClearUntreated();
pseudo_obj->ClearUploadInfo(false); // clear multipart upload info
pseudo_obj->ClearUploadInfo(); // clear multipart upload info
// put pending headers
if(0 != (result = UploadPendingMeta())){

41
src/fdcache_fdinfo.cpp
View File

@ -30,7 +30,6 @@
#include "s3fs.h"
#include "fdcache_fdinfo.h"
#include "fdcache_pseudofd.h"
#include "autolock.h"
#include "curl.h"
#include "string_util.h"
#include "threadpoolman.h"
@ -162,23 +161,23 @@ bool PseudoFdInfo::Clear()
pseudo_fd = -1;
physical_fd = -1;
CloseUploadFd(true); // [NOTE] already destroy mutex, then do not lock it.
CloseUploadFd(AutoLock::ALREADY_LOCKED); // [NOTE] already destroy mutex, then do not lock it.
return true;
}
void PseudoFdInfo::CloseUploadFd(bool lock_already_held)
void PseudoFdInfo::CloseUploadFd(AutoLock::Type type)
{
AutoLock auto_lock(&upload_list_lock, lock_already_held ? AutoLock::ALREADY_LOCKED : AutoLock::NONE);
AutoLock auto_lock(&upload_list_lock, type);
if(-1 != upload_fd){
close(upload_fd);
}
}
bool PseudoFdInfo::OpenUploadFd(bool lock_already_held)
bool PseudoFdInfo::OpenUploadFd(AutoLock::Type type)
{
AutoLock auto_lock(&upload_list_lock, lock_already_held ? AutoLock::ALREADY_LOCKED : AutoLock::NONE);
AutoLock auto_lock(&upload_list_lock, type);
if(-1 != upload_fd){
// already initialized
@ -239,9 +238,9 @@ bool PseudoFdInfo::Readable() const
return true;
}
bool PseudoFdInfo::ClearUploadInfo(bool is_cancel_mp, bool lock_already_held)
bool PseudoFdInfo::ClearUploadInfo(bool is_cancel_mp, AutoLock::Type type)
{
AutoLock auto_lock(&upload_list_lock, lock_already_held ? AutoLock::ALREADY_LOCKED : AutoLock::NONE);
AutoLock auto_lock(&upload_list_lock, type);
if(is_cancel_mp){
// [TODO]
@ -259,11 +258,11 @@ bool PseudoFdInfo::ClearUploadInfo(bool is_cancel_mp, bool lock_already_held)
return true;
}
bool PseudoFdInfo::InitialUploadInfo(const std::string& id, bool lock_already_held)
bool PseudoFdInfo::InitialUploadInfo(const std::string& id, AutoLock::Type type)
{
AutoLock auto_lock(&upload_list_lock, lock_already_held ? AutoLock::ALREADY_LOCKED : AutoLock::NONE);
AutoLock auto_lock(&upload_list_lock, type);
if(!ClearUploadInfo(true, true)){
if(!ClearUploadInfo(true, AutoLock::ALREADY_LOCKED)){
return false;
}
upload_id = id;
@ -342,9 +341,9 @@ bool PseudoFdInfo::AppendUploadPart(off_t start, off_t size, bool is_copy, etagp
return true;
}
void PseudoFdInfo::ClearUntreated(bool lock_already_held)
void PseudoFdInfo::ClearUntreated(AutoLock::Type type)
{
AutoLock auto_lock(&upload_list_lock, lock_already_held ? AutoLock::ALREADY_LOCKED : AutoLock::NONE);
AutoLock auto_lock(&upload_list_lock, type);
untreated_list.ClearAll();
}
@ -413,7 +412,7 @@ static bool filepart_partnum_compare(const filepart& src1, const filepart& src2)
return (src1.get_part_number() <= src2.get_part_number());
}
bool PseudoFdInfo::InsertUploadPart(off_t start, off_t size, int part_num, bool is_copy, etagpair** ppetag, bool lock_already_held)
bool PseudoFdInfo::InsertUploadPart(off_t start, off_t size, int part_num, bool is_copy, etagpair** ppetag, AutoLock::Type type)
{
//S3FS_PRN_DBG("[start=%lld][size=%lld][part_num=%d][is_copy=%s]", static_cast<long long int>(start), static_cast<long long int>(size), part_num, (is_copy ? "true" : "false"));
@ -426,7 +425,7 @@ bool PseudoFdInfo::InsertUploadPart(off_t start, off_t size, int part_num, bool
return false;
}
AutoLock auto_lock(&upload_list_lock, lock_already_held ? AutoLock::ALREADY_LOCKED : AutoLock::NONE);
AutoLock auto_lock(&upload_list_lock, type);
// insert new part
etag_entities.push_back(etagpair(NULL, part_num));
@ -455,14 +454,14 @@ bool PseudoFdInfo::ParallelMultipartUpload(const char* path, const mp_part_list_
// nothing to do
return true;
}
if(!OpenUploadFd(true)){
if(!OpenUploadFd(AutoLock::ALREADY_LOCKED)){
return false;
}
for(mp_part_list_t::const_iterator iter = mplist.begin(); iter != mplist.end(); ++iter){
// Insert upload part
etagpair* petag = NULL;
if(!InsertUploadPart(iter->start, iter->size, iter->part_num, is_copy, &petag, true)){
if(!InsertUploadPart(iter->start, iter->size, iter->part_num, is_copy, &petag, AutoLock::ALREADY_LOCKED)){
S3FS_PRN_ERR("Failed to insert insert upload part(path=%s, start=%lld, size=%lld, part=%d, copy=%s) to mplist", SAFESTRPTR(path), static_cast<long long int>(iter->start), static_cast<long long int>(iter->size), iter->part_num, (is_copy ? "true" : "false"));
return false;
}
@ -503,7 +502,7 @@ bool PseudoFdInfo::ParallelMultipartUploadAll(const char* path, const mp_part_li
result = 0;
if(!OpenUploadFd(true)){
if(!OpenUploadFd(AutoLock::ALREADY_LOCKED)){
return false;
}
@ -610,7 +609,7 @@ ssize_t PseudoFdInfo::UploadBoundaryLastUntreatedArea(const char* path, headers_
S3FS_PRN_ERR("failed to setup multipart upload(create upload id) by errno(%d)", result);
return result;
}
if(!InitialUploadInfo(upload_id, true)){
if(!InitialUploadInfo(upload_id, AutoLock::ALREADY_LOCKED)){
S3FS_PRN_ERR("failed to setup multipart upload(set upload id to object)");
return result;
}
@ -897,7 +896,7 @@ bool PseudoFdInfo::ExtractUploadPartsFromAllArea(mp_part_list_t& to_upload_list,
S3FS_PRN_ERR("The uploaded list may not be the boundary for the maximum multipart upload size. No further processing is possible.");
return false;
}
// Set this iterator to ovrelap iter
// Set this iterator to overlap iter
overlap_uploaded_iter = uploaded_iter;
}else if((cur_start + cur_size - 1) < uploaded_iter->startpos){
@ -954,7 +953,7 @@ bool PseudoFdInfo::ExtractUploadPartsFromAllArea(mp_part_list_t& to_upload_list,
//
S3FS_PRN_DBG("Cancel upload: start=%lld, size=%lld", static_cast<long long int>(overlap_uploaded_iter->startpos), static_cast<long long int>(overlap_uploaded_iter->size));
cancel_upload_list.push_back(*overlap_uploaded_iter); // add this uploaded area to cancel_upload_list
upload_list.erase(overlap_uploaded_iter); // remove it from upload_list
uploaded_iter = upload_list.erase(overlap_uploaded_iter); // remove it from upload_list
S3FS_PRN_DBG("To upload: start=%lld, size=%lld", static_cast<long long int>(cur_start), static_cast<long long int>(cur_size));
to_upload_list.push_back(mp_part(cur_start, cur_size, part_num)); // add new uploading area to list

13
src/fdcache_fdinfo.h
View File

@ -24,6 +24,7 @@
#include "fdcache_untreated.h"
#include "psemaphore.h"
#include "metaheader.h"
#include "autolock.h"
//------------------------------------------------
// Structure of parameters to pass to thread
@ -73,12 +74,12 @@ class PseudoFdInfo
static void* MultipartUploadThreadWorker(void* arg);
bool Clear();
void CloseUploadFd(bool lock_already_held = false);
bool OpenUploadFd(bool lock_already_held = false);
void CloseUploadFd(AutoLock::Type type = AutoLock::NONE);
bool OpenUploadFd(AutoLock::Type type = AutoLock::NONE);
bool GetLastUpdateUntreatedPart(off_t& start, off_t& size);
bool ReplaceLastUpdateUntreatedPart(off_t front_start, off_t front_size, off_t behind_start, off_t behind_size);
bool ParallelMultipartUpload(const char* path, const mp_part_list_t& mplist, bool is_copy);
bool InsertUploadPart(off_t start, off_t size, int part_num, bool is_copy, etagpair** ppetag, bool lock_already_held = false);
bool InsertUploadPart(off_t start, off_t size, int part_num, bool is_copy, etagpair** ppetag, AutoLock::Type type = AutoLock::NONE);
int WaitAllThreadsExit();
bool ExtractUploadPartsFromUntreatedArea(off_t& untreated_start, off_t& untreated_size, mp_part_list_t& to_upload_list, filepart_list_t& cancel_upload_list, off_t max_mp_size);
@ -93,8 +94,8 @@ class PseudoFdInfo
bool Readable() const;
bool Set(int fd, int open_flags);
bool ClearUploadInfo(bool is_clear_part = false, bool lock_already_held = false);
bool InitialUploadInfo(const std::string& id, bool lock_already_held = false);
bool ClearUploadInfo(bool is_clear_part = false, AutoLock::Type type = AutoLock::NONE);
bool InitialUploadInfo(const std::string& id, AutoLock::Type type = AutoLock::NONE);
bool IsUploading() const { return !upload_id.empty(); }
bool GetUploadId(std::string& id) const;
@ -102,7 +103,7 @@ class PseudoFdInfo
bool AppendUploadPart(off_t start, off_t size, bool is_copy = false, etagpair** ppetag = NULL);
void ClearUntreated(bool lock_already_held = false);
void ClearUntreated(AutoLock::Type type = AutoLock::NONE);
bool ClearUntreated(off_t start, off_t size);
bool GetLastUntreated(off_t& start, off_t& size, off_t max_size, off_t min_size = MIN_MULTIPART_SIZE);
bool AddUntreated(off_t start, off_t size);

2
src/fdcache_untreated.cpp
View File

@ -224,7 +224,7 @@ bool UntreatedParts::ReplaceLastUpdatePart(off_t start, off_t size)
iter->start = start;
iter->size = size;
}else{
untreated_list.erase(iter);
iter = untreated_list.erase(iter);
}
return true;
}

15
src/types.h
View File

@ -326,17 +326,14 @@ struct mp_part
typedef std::list<struct mp_part> mp_part_list_t;
struct total_mp_part_list
inline off_t total_mp_part_list(const mp_part_list_t& mplist)
{
off_t operator()(const mp_part_list_t& mplist) const
{
off_t size = 0;
for(mp_part_list_t::const_iterator iter = mplist.begin(); iter != mplist.end(); ++iter){
size += iter->size;
}
return size;
off_t size = 0;
for(mp_part_list_t::const_iterator iter = mplist.begin(); iter != mplist.end(); ++iter){
size += iter->size;
}
};
return size;
}
//-------------------------------------------------------------------
// mimes_t