mirror of
https://github.com/s3fs-fuse/s3fs-fuse.git
synced 2024-06-07 19:40:50 +00:00
Changed etaglist_t from string list to new structure etagpairs list
This commit is contained in:
Takeshi Nakatani
Andrew Gaul
parent
cd466ebdd4
commit
a868c0656e
47
src/curl.cpp
47
src/curl.cpp
|
|
@ -1336,10 +1336,10 @@ int S3fsCurl::ParallelMultipartUploadRequest(const char* tpath, headers_t& meta,
|
|||
s3fscurl_para->partdata.size = chunk;
|
||||
s3fscurl_para->b_partdata_startpos = s3fscurl_para->partdata.startpos;
|
||||
s3fscurl_para->b_partdata_size = s3fscurl_para->partdata.size;
|
||||
s3fscurl_para->partdata.add_etag_list(&list);
|
||||
s3fscurl_para->partdata.add_etag_list(list);
|
||||
|
||||
// initiate upload part for parallel
|
||||
if(0 != (result = s3fscurl_para->UploadMultipartPostSetup(tpath, static_cast<int>(list.size()), upload_id))){
|
||||
if(0 != (result = s3fscurl_para->UploadMultipartPostSetup(tpath, s3fscurl_para->partdata.get_part_number(), upload_id))){
|
||||
S3FS_PRN_ERR("failed uploading part setup(%d)", result);
|
||||
close(fd2);
|
||||
delete s3fscurl_para;
|
||||
|
|
@ -1431,12 +1431,12 @@ int S3fsCurl::ParallelMixMultipartUploadRequest(const char* tpath, headers_t& me
|
|||
s3fscurl_para->partdata.size = iter->bytes;
|
||||
s3fscurl_para->b_partdata_startpos = s3fscurl_para->partdata.startpos;
|
||||
s3fscurl_para->b_partdata_size = s3fscurl_para->partdata.size;
|
||||
s3fscurl_para->partdata.add_etag_list(&list);
|
||||
s3fscurl_para->partdata.add_etag_list(list);
|
||||
|
||||
S3FS_PRN_INFO3("Upload Part [tpath=%s][start=%lld][size=%lld][part=%zu]", SAFESTRPTR(tpath), static_cast<long long>(iter->offset), static_cast<long long>(iter->bytes), list.size());
|
||||
S3FS_PRN_INFO3("Upload Part [tpath=%s][start=%lld][size=%lld][part=%d]", SAFESTRPTR(tpath), static_cast<long long>(iter->offset), static_cast<long long>(iter->bytes), s3fscurl_para->partdata.get_part_number());
|
||||
|
||||
// initiate upload part for parallel
|
||||
if(0 != (result = s3fscurl_para->UploadMultipartPostSetup(tpath, static_cast<int>(list.size()), upload_id))){
|
||||
if(0 != (result = s3fscurl_para->UploadMultipartPostSetup(tpath, s3fscurl_para->partdata.get_part_number(), upload_id))){
|
||||
S3FS_PRN_ERR("failed uploading part setup(%d)", result);
|
||||
close(fd2);
|
||||
delete s3fscurl_para;
|
||||
|
|
@ -1462,12 +1462,12 @@ int S3fsCurl::ParallelMixMultipartUploadRequest(const char* tpath, headers_t& me
|
|||
|
||||
s3fscurl_para->b_from = SAFESTRPTR(tpath);
|
||||
s3fscurl_para->b_meta = meta;
|
||||
s3fscurl_para->partdata.add_etag_list(&list);
|
||||
s3fscurl_para->partdata.add_etag_list(list);
|
||||
|
||||
S3FS_PRN_INFO3("Copy Part [tpath=%s][start=%lld][size=%lld][part=%zu]", SAFESTRPTR(tpath), static_cast<long long>(iter->offset + i), static_cast<long long>(bytes), list.size());
|
||||
S3FS_PRN_INFO3("Copy Part [tpath=%s][start=%lld][size=%lld][part=%d]", SAFESTRPTR(tpath), static_cast<long long>(iter->offset + i), static_cast<long long>(bytes), s3fscurl_para->partdata.get_part_number());
|
||||
|
||||
// initiate upload part for parallel
|
||||
if(0 != (result = s3fscurl_para->CopyMultipartPostSetup(tpath, tpath, static_cast<int>(list.size()), upload_id, meta))){
|
||||
if(0 != (result = s3fscurl_para->CopyMultipartPostSetup(tpath, tpath, s3fscurl_para->partdata.get_part_number(), upload_id, meta))){
|
||||
S3FS_PRN_ERR("failed uploading part setup(%d)", result);
|
||||
close(fd2);
|
||||
delete s3fscurl_para;
|
||||
|
|
@ -3583,15 +3583,14 @@ int S3fsCurl::CompleteMultipartPostRequest(const char* tpath, const std::string&
|
|||
// make contents
|
||||
std::string postContent;
|
||||
postContent += "<CompleteMultipartUpload>\n";
|
||||
int cnt = 0;
|
||||
for(etaglist_t::iterator it = parts.begin(); it != parts.end(); ++it, ++cnt){
|
||||
if(it->empty()){
|
||||
S3FS_PRN_ERR("%d file part is not finished uploading.", cnt + 1);
|
||||
for(etaglist_t::iterator it = parts.begin(); it != parts.end(); ++it){
|
||||
if(it->etag.empty()){
|
||||
S3FS_PRN_ERR("%d file part is not finished uploading.", it->part_num);
|
||||
return -EIO;
|
||||
}
|
||||
postContent += "<Part>\n";
|
||||
postContent += " <PartNumber>" + str(cnt + 1) + "</PartNumber>\n";
|
||||
postContent += " <ETag>" + *it + "</ETag>\n";
|
||||
postContent += " <PartNumber>" + str(it->part_num) + "</PartNumber>\n";
|
||||
postContent += " <ETag>" + it->etag + "</ETag>\n";
|
||||
postContent += "</Part>\n";
|
||||
}
|
||||
postContent += "</CompleteMultipartUpload>\n";
|
||||
|
|
@ -3887,7 +3886,7 @@ bool S3fsCurl::UploadMultipartPostComplete()
|
|||
return false;
|
||||
}
|
||||
}
|
||||
(*partdata.petag) = it->second;
|
||||
partdata.petag->etag = it->second;
|
||||
partdata.uploaded = true;
|
||||
|
||||
return true;
|
||||
|
|
@ -3910,7 +3909,7 @@ bool S3fsCurl::CopyMultipartPostComplete()
|
|||
etag.erase(etag.size() - 1);
|
||||
etag.erase(0, 1);
|
||||
}
|
||||
(*partdata.petag) = etag;
|
||||
partdata.petag->etag = etag;
|
||||
|
||||
bodydata.Clear();
|
||||
headdata.Clear();
|
||||
|
|
@ -3960,10 +3959,10 @@ int S3fsCurl::MultipartHeadRequest(const char* tpath, off_t size, headers_t& met
|
|||
S3fsCurl* s3fscurl_para = new S3fsCurl(true);
|
||||
s3fscurl_para->b_from = SAFESTRPTR(tpath);
|
||||
s3fscurl_para->b_meta = meta;
|
||||
s3fscurl_para->partdata.add_etag_list(&list);
|
||||
s3fscurl_para->partdata.add_etag_list(list);
|
||||
|
||||
// initiate upload part for parallel
|
||||
if(0 != (result = s3fscurl_para->CopyMultipartPostSetup(tpath, tpath, static_cast<int>(list.size()), upload_id, meta))){
|
||||
if(0 != (result = s3fscurl_para->CopyMultipartPostSetup(tpath, tpath, s3fscurl_para->partdata.get_part_number(), upload_id, meta))){
|
||||
S3FS_PRN_ERR("failed uploading part setup(%d)", result);
|
||||
delete s3fscurl_para;
|
||||
return result;
|
||||
|
|
@ -3996,7 +3995,7 @@ int S3fsCurl::MultipartHeadRequest(const char* tpath, off_t size, headers_t& met
|
|||
return 0;
|
||||
}
|
||||
|
||||
int S3fsCurl::MultipartUploadRequest(const std::string& upload_id, const char* tpath, int fd, off_t offset, off_t size, int part_num, std::string* petag)
|
||||
int S3fsCurl::MultipartUploadRequest(const std::string& upload_id, const char* tpath, int fd, off_t offset, off_t size, etagpair* petagpair)
|
||||
{
|
||||
S3FS_PRN_INFO3("[upload_id=%s][tpath=%s][fd=%d][offset=%lld][size=%lld]", upload_id.c_str(), SAFESTRPTR(tpath), fd, static_cast<long long int>(offset), static_cast<long long int>(size));
|
||||
|
||||
|
|
@ -4016,12 +4015,12 @@ int S3fsCurl::MultipartUploadRequest(const std::string& upload_id, const char* t
|
|||
partdata.size = size;
|
||||
b_partdata_startpos = partdata.startpos;
|
||||
b_partdata_size = partdata.size;
|
||||
partdata.add_etag(petag);
|
||||
partdata.set_etag(petagpair);
|
||||
|
||||
// upload part
|
||||
int result;
|
||||
if(0 != (result = UploadMultipartPostRequest(tpath, part_num, upload_id))){
|
||||
S3FS_PRN_ERR("failed uploading %d part by error(%d)", part_num, result);
|
||||
if(0 != (result = UploadMultipartPostRequest(tpath, petagpair->part_num, upload_id))){
|
||||
S3FS_PRN_ERR("failed uploading %d part by error(%d)", petagpair->part_num, result);
|
||||
close(fd2);
|
||||
return result;
|
||||
}
|
||||
|
|
@ -4069,10 +4068,10 @@ int S3fsCurl::MultipartRenameRequest(const char* from, const char* to, headers_t
|
|||
S3fsCurl* s3fscurl_para = new S3fsCurl(true);
|
||||
s3fscurl_para->b_from = SAFESTRPTR(from);
|
||||
s3fscurl_para->b_meta = meta;
|
||||
s3fscurl_para->partdata.add_etag_list(&list);
|
||||
s3fscurl_para->partdata.add_etag_list(list);
|
||||
|
||||
// initiate upload part for parallel
|
||||
if(0 != (result = s3fscurl_para->CopyMultipartPostSetup(from, to, static_cast<int>(list.size()), upload_id, meta))){
|
||||
if(0 != (result = s3fscurl_para->CopyMultipartPostSetup(from, to, s3fscurl_para->partdata.get_part_number(), upload_id, meta))){
|
||||
S3FS_PRN_ERR("failed uploading part setup(%d)", result);
|
||||
delete s3fscurl_para;
|
||||
return result;
|
||||
|
|
|
|||
|
|
@ -399,7 +399,7 @@ class S3fsCurl
|
|||
int MultipartListRequest(std::string& body);
|
||||
int AbortMultipartUpload(const char* tpath, const std::string& upload_id);
|
||||
int MultipartHeadRequest(const char* tpath, off_t size, headers_t& meta, bool is_copy);
|
||||
int MultipartUploadRequest(const std::string& upload_id, const char* tpath, int fd, off_t offset, off_t size, int part_num, std::string* petag);
|
||||
int MultipartUploadRequest(const std::string& upload_id, const char* tpath, int fd, off_t offset, off_t size, etagpair* petagpair);
|
||||
int MultipartRenameRequest(const char* from, const char* to, headers_t& meta, off_t size);
|
||||
|
||||
// methods(variables)
|
||||
|
|
|
|||
|
|
@ -1265,14 +1265,13 @@ int FdEntity::NoCacheMultipartPost(PseudoFdInfo* pseudo_obj, int tgfd, off_t sta
|
|||
}
|
||||
|
||||
// append new part and get it's etag string pointer
|
||||
int partnum = 0;
|
||||
std::string* petag = NULL;
|
||||
if(!pseudo_obj->AppendUploadPart(start, size, false, &partnum, &petag)){
|
||||
etagpair* petagpair = NULL;
|
||||
if(!pseudo_obj->AppendUploadPart(start, size, false, &petagpair)){
|
||||
return -EIO;
|
||||
}
|
||||
|
||||
S3fsCurl s3fscurl(true);
|
||||
return s3fscurl.MultipartUploadRequest(upload_id, path.c_str(), tgfd, start, size, partnum, petag);
|
||||
return s3fscurl.MultipartUploadRequest(upload_id, path.c_str(), tgfd, start, size, petagpair);
|
||||
}
|
||||
|
||||
// [NOTE]
|
||||
|
|
|
|||
|
|
@ -177,7 +177,7 @@ bool PseudoFdInfo::GetEtaglist(etaglist_t& list)
|
|||
// An error will occur if it is discontinuous or if it overlaps with an
|
||||
// existing area.
|
||||
//
|
||||
bool PseudoFdInfo::AppendUploadPart(off_t start, off_t size, bool is_copy, int* ppartnum, std::string** ppetag)
|
||||
bool PseudoFdInfo::AppendUploadPart(off_t start, off_t size, bool is_copy, etagpair** ppetag)
|
||||
{
|
||||
if(IsUploading()){
|
||||
S3FS_PRN_ERR("Multipart Upload has not started yet.");
|
||||
|
|
@ -194,21 +194,20 @@ bool PseudoFdInfo::AppendUploadPart(off_t start, off_t size, bool is_copy, int*
|
|||
return false;
|
||||
}
|
||||
|
||||
// add new part
|
||||
etag_entities.push_back(std::string("")); // [NOTE] Create the etag entity and register it in the list.
|
||||
std::string& etag_entity = etag_entities.back();
|
||||
filepart newpart(false, physical_fd, start, size, is_copy, &etag_entity);
|
||||
upload_list.push_back(newpart);
|
||||
// make part number
|
||||
int partnumber = static_cast<int>(upload_list.size()) + 1;
|
||||
|
||||
// set part number
|
||||
if(ppartnum){
|
||||
*ppartnum = static_cast<int>(upload_list.size());
|
||||
}
|
||||
// add new part
|
||||
etag_entities.push_back(etagpair(NULL, partnumber)); // [NOTE] Create the etag entity and register it in the list.
|
||||
etagpair& etag_entity = etag_entities.back();
|
||||
filepart newpart(false, physical_fd, start, size, is_copy, &etag_entity);
|
||||
upload_list.push_back(newpart);
|
||||
|
||||
// set etag pointer
|
||||
if(ppetag){
|
||||
*ppetag = &etag_entity;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -35,7 +35,7 @@ class PseudoFdInfo
|
|||
std::string upload_id;
|
||||
filepart_list_t upload_list;
|
||||
UntreatedParts untreated_list; // list of untreated parts that have been written and not yet uploaded(for streamupload)
|
||||
etaglist_t etag_entities; // list of etag string entities(to maintain the etag entity even if MPPART_INFO is destroyed)
|
||||
etaglist_t etag_entities; // list of etag string and part number entities(to maintain the etag entity even if MPPART_INFO is destroyed)
|
||||
|
||||
bool is_lock_init;
|
||||
pthread_mutex_t upload_list_lock; // protects upload_id and upload_list
|
||||
|
|
@ -61,7 +61,7 @@ class PseudoFdInfo
|
|||
bool GetUploadId(std::string& id) const;
|
||||
bool GetEtaglist(etaglist_t& list);
|
||||
|
||||
bool AppendUploadPart(off_t start, off_t size, bool is_copy = false, int* ppartnum = NULL, std::string** ppetag = NULL);
|
||||
bool AppendUploadPart(off_t start, off_t size, bool is_copy = false, etagpair** ppetag = NULL);
|
||||
|
||||
void ClearUntreated(bool lock_already_held = false);
|
||||
bool ClearUntreated(off_t start, off_t size);
|
||||
|
|
|
|||
47
src/types.h
47
src/types.h
|
|
@ -174,7 +174,29 @@ enum signature_type_t {
|
|||
//----------------------------------------------
|
||||
// etaglist_t / filepart / untreatedpart
|
||||
//----------------------------------------------
|
||||
typedef std::list<std::string> etaglist_t;
|
||||
//
|
||||
// Etag string and part number pair
|
||||
//
|
||||
struct etagpair
|
||||
{
|
||||
std::string etag; // expected etag value
|
||||
int part_num; // part number
|
||||
|
||||
etagpair(const char* petag = NULL, int part = -1) : etag(petag ? petag : ""), part_num(part) {}
|
||||
|
||||
~etagpair()
|
||||
{
|
||||
clear();
|
||||
}
|
||||
|
||||
void clear()
|
||||
{
|
||||
etag.erase();
|
||||
part_num = -1;
|
||||
}
|
||||
};
|
||||
|
||||
typedef std::list<etagpair> etaglist_t;
|
||||
|
||||
//
|
||||
// Each part information for Multipart upload
|
||||
|
|
@ -187,9 +209,9 @@ struct filepart
|
|||
off_t startpos; // seek fd point for uploading
|
||||
off_t size; // uploading size
|
||||
bool is_copy; // whether is copy multipart
|
||||
std::string* petag; // use only parallel upload
|
||||
etagpair* petag; // use only parallel upload
|
||||
|
||||
filepart(bool is_uploaded = false, int _fd = -1, off_t part_start = 0, off_t part_size = -1, bool is_copy_part = false, std::string* petag = NULL) : uploaded(false), fd(_fd), startpos(part_start), size(part_size), is_copy(is_copy_part), petag(petag) {}
|
||||
filepart(bool is_uploaded = false, int _fd = -1, off_t part_start = 0, off_t part_size = -1, bool is_copy_part = false, etagpair* petagpair = NULL) : uploaded(false), fd(_fd), startpos(part_start), size(part_size), is_copy(is_copy_part), petag(petagpair) {}
|
||||
|
||||
~filepart()
|
||||
{
|
||||
|
|
@ -207,16 +229,27 @@ struct filepart
|
|||
petag = NULL;
|
||||
}
|
||||
|
||||
void add_etag_list(etaglist_t* list)
|
||||
void add_etag_list(etaglist_t& list, int partnum = -1)
|
||||
{
|
||||
list->push_back(std::string());
|
||||
petag = &list->back();
|
||||
if(-1 == partnum){
|
||||
partnum = list.size() + 1;
|
||||
}
|
||||
list.push_back(etagpair(NULL, partnum));
|
||||
petag = &list.back();
|
||||
}
|
||||
|
||||
void add_etag(std::string* petagobj)
|
||||
void set_etag(etagpair* petagobj)
|
||||
{
|
||||
petag = petagobj;
|
||||
}
|
||||
|
||||
int get_part_number()
|
||||
{
|
||||
if(!petag){
|
||||
return -1;
|
||||
}
|
||||
return petag->part_num;
|
||||
}
|
||||
};
|
||||
|
||||
typedef std::list<filepart> filepart_list_t;
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user