diff --git a/src/Makefile.am b/src/Makefile.am index b78e37d..7a254f0 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -40,7 +40,6 @@ s3fs_SOURCES = \ curl_handlerpool.cpp \ curl_multi.cpp \ curl_util.cpp \ - bodydata.cpp \ s3objlist.cpp \ cache.cpp \ string_util.cpp \ diff --git a/src/bodydata.cpp b/src/bodydata.cpp deleted file mode 100644 index 86d53e6..0000000 --- a/src/bodydata.cpp +++ /dev/null @@ -1,122 +0,0 @@ -/* - * s3fs - FUSE-based file system backed by Amazon S3 - * - * Copyright(C) 2007 Randy Rizun - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - -#include -#include -#include - -#include "s3fs_logger.h" -#include "bodydata.h" - -//------------------------------------------------------------------- -// Variables -//------------------------------------------------------------------- -static const int BODYDATA_RESIZE_APPEND_MIN = 1024; -static const int BODYDATA_RESIZE_APPEND_MID = 1024 * 1024; -static const int BODYDATA_RESIZE_APPEND_MAX = 10 * 1024 * 1024; - -//------------------------------------------------------------------- -// Utility Functions -//------------------------------------------------------------------- -static size_t adjust_block(size_t bytes, size_t block) -{ - return ((bytes / block) + ((bytes % block) ? 1 : 0)) * block; -} - -//------------------------------------------------------------------- -// Class BodyData -//------------------------------------------------------------------- -bool BodyData::Resize(size_t addbytes) -{ - if(IsSafeSize(addbytes)){ - return true; - } - - // New size - size_t need_size = adjust_block((lastpos + addbytes + 1) - bufsize, sizeof(off_t)); - - if(BODYDATA_RESIZE_APPEND_MAX < bufsize){ - need_size = (BODYDATA_RESIZE_APPEND_MAX < need_size ? need_size : BODYDATA_RESIZE_APPEND_MAX); - }else if(BODYDATA_RESIZE_APPEND_MID < bufsize){ - need_size = (BODYDATA_RESIZE_APPEND_MID < need_size ? need_size : BODYDATA_RESIZE_APPEND_MID); - }else if(BODYDATA_RESIZE_APPEND_MIN < bufsize){ - need_size = ((bufsize * 2) < need_size ? need_size : (bufsize * 2)); - }else{ - need_size = (BODYDATA_RESIZE_APPEND_MIN < need_size ? need_size : BODYDATA_RESIZE_APPEND_MIN); - } - // realloc - char* newtext; - if(nullptr == (newtext = reinterpret_cast(realloc(text, (bufsize + need_size))))){ - S3FS_PRN_CRIT("not enough memory (realloc returned nullptr)"); - free(text); - text = nullptr; - return false; - } - text = newtext; - bufsize += need_size; - - return true; -} - -void BodyData::Clear() -{ - if(text){ - free(text); - text = nullptr; - } - lastpos = 0; - bufsize = 0; -} - -bool BodyData::Append(void* ptr, size_t bytes) -{ - if(!ptr){ - return false; - } - if(0 == bytes){ - return true; - } - if(!Resize(bytes)){ - return false; - } - memcpy(&text[lastpos], ptr, bytes); - lastpos += bytes; - text[lastpos] = '\0'; - - return true; -} - -const char* BodyData::str() const -{ - if(!text){ - static const char strnull[] = ""; - return strnull; - } - return text; -} - -/* -* Local variables: -* tab-width: 4 -* c-basic-offset: 4 -* End: -* vim600: expandtab sw=4 ts=4 fdm=marker -* vim<600: expandtab sw=4 ts=4 -*/ diff --git a/src/bodydata.h b/src/bodydata.h deleted file mode 100644 index c2639c7..0000000 --- a/src/bodydata.h +++ /dev/null @@ -1,72 +0,0 @@ -/* - * s3fs - FUSE-based file system backed by Amazon S3 - * - * Copyright(C) 2007 Randy Rizun - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - -#ifndef S3FS_BODYDATA_H_ -#define S3FS_BODYDATA_H_ - -//---------------------------------------------- -// Class BodyData -//---------------------------------------------- -// memory class for curl write memory callback -// -class BodyData -{ - private: - char* text; - size_t lastpos; - size_t bufsize; - - private: - bool IsSafeSize(size_t addbytes) const - { - return ((lastpos + addbytes + 1) > bufsize ? false : true); - } - bool Resize(size_t addbytes); - - public: - BodyData() : text(nullptr), lastpos(0), bufsize(0) {} - ~BodyData() - { - Clear(); - } - - void Clear(); - bool Append(void* ptr, size_t bytes); - bool Append(void* ptr, size_t blockSize, size_t numBlocks) - { - return Append(ptr, (blockSize * numBlocks)); - } - const char* str() const; - size_t size() const - { - return lastpos; - } -}; - -#endif // S3FS_BODYDATA_H_ - -/* -* Local variables: -* tab-width: 4 -* c-basic-offset: 4 -* End: -* vim600: expandtab sw=4 ts=4 fdm=marker -* vim<600: expandtab sw=4 ts=4 -*/ diff --git a/src/curl.cpp b/src/curl.cpp index 162284f..aa6b0a1 100644 --- a/src/curl.cpp +++ b/src/curl.cpp @@ -593,13 +593,8 @@ bool S3fsCurl::LocateBundle() size_t S3fsCurl::WriteMemoryCallback(void* ptr, size_t blockSize, size_t numBlocks, void* data) { - BodyData* body = static_cast(data); - - if(!body->Append(ptr, blockSize, numBlocks)){ - S3FS_PRN_CRIT("BodyData.Append() returned false."); - S3FS_FUSE_EXIT(); - return -1; - } + std::string* body = static_cast(data); + body->append(static_cast(ptr), blockSize * numBlocks); return (blockSize * numBlocks); } @@ -688,7 +683,7 @@ size_t S3fsCurl::DownloadWriteCallback(void* ptr, size_t size, size_t nmemb, voi // Buffer initial bytes in case it is an XML error response. if(pCurl->bodydata.size() < GET_OBJECT_RESPONSE_LIMIT){ - pCurl->bodydata.Append(ptr, std::min(size * nmemb, GET_OBJECT_RESPONSE_LIMIT - pCurl->bodydata.size())); + pCurl->bodydata.append(static_cast(ptr), std::min(size * nmemb, GET_OBJECT_RESPONSE_LIMIT - pCurl->bodydata.size())); } // write size @@ -1322,9 +1317,9 @@ int S3fsCurl::MapPutErrorResponse(int result) // BHzLOATeDuvN8Es1wI8IcERq4kl4dc2A9tOB8Yqr39Ys6fl7N4EJ8sjGiVvu6wLP // // - const char* pstrbody = bodydata.str(); + const char* pstrbody = bodydata.c_str(); std::string code; - if(!pstrbody || simple_parse_xml(pstrbody, bodydata.size(), "Code", code)){ + if(simple_parse_xml(pstrbody, bodydata.size(), "Code", code)){ S3FS_PRN_ERR("Put request get 200 status response, but it included error body(or nullptr). The request failed during copying the object in S3. Code: %s", code.c_str()); // TODO: parse more specific error from result = -EIO; @@ -2087,8 +2082,8 @@ bool S3fsCurl::ClearInternalData() requestHeaders = nullptr; } responseHeaders.clear(); - bodydata.Clear(); - headdata.Clear(); + bodydata.clear(); + headdata.clear(); LastResponseCode = S3FSCURL_RESPONSECODE_NOTSET; postdata = nullptr; postdata_remaining = 0; @@ -2156,8 +2151,8 @@ bool S3fsCurl::RemakeHandle() // reinitialize internal data requestHeaders = curl_slist_remove(requestHeaders, "Authorization"); responseHeaders.clear(); - bodydata.Clear(); - headdata.Clear(); + bodydata.clear(); + headdata.clear(); LastResponseCode = S3FSCURL_RESPONSECODE_NOTSET; // count up(only use for multipart) @@ -2500,7 +2495,7 @@ int S3fsCurl::RequestPerform(bool dontAddAuthHeaders /*=false*/) { // Try to parse more specific AWS error code otherwise fall back to HTTP error code. std::string value; - if(simple_parse_xml(bodydata.str(), bodydata.size(), "Code", value)){ + if(simple_parse_xml(bodydata.c_str(), bodydata.size(), "Code", value)){ // TODO: other error codes if(value == "EntityTooLarge"){ result = -EFBIG; @@ -2519,7 +2514,7 @@ int S3fsCurl::RequestPerform(bool dontAddAuthHeaders /*=false*/) switch(responseCode){ case 301: case 307: - S3FS_PRN_ERR("HTTP response code 301(Moved Permanently: also happens when bucket's region is incorrect), returning EIO. Body Text: %s", bodydata.str()); + S3FS_PRN_ERR("HTTP response code 301(Moved Permanently: also happens when bucket's region is incorrect), returning EIO. Body Text: %s", bodydata.c_str()); S3FS_PRN_ERR("The options of url and endpoint may be useful for solving, please try to use both options."); result = -EIO; break; @@ -2533,19 +2528,19 @@ int S3fsCurl::RequestPerform(bool dontAddAuthHeaders /*=false*/) S3FS_PRN_ERR("HEAD HTTP response code %ld, returning EPERM.", responseCode); result = -EPERM; }else{ - S3FS_PRN_ERR("HTTP response code %ld, returning EIO. Body Text: %s", responseCode, bodydata.str()); + S3FS_PRN_ERR("HTTP response code %ld, returning EIO. Body Text: %s", responseCode, bodydata.c_str()); result = -EIO; } break; case 403: - S3FS_PRN_ERR("HTTP response code %ld, returning EPERM. Body Text: %s", responseCode, bodydata.str()); + S3FS_PRN_ERR("HTTP response code %ld, returning EPERM. Body Text: %s", responseCode, bodydata.c_str()); result = -EPERM; break; case 404: S3FS_PRN_INFO3("HTTP response code 404 was returned, returning ENOENT"); - S3FS_PRN_DBG("Body Text: %s", bodydata.str()); + S3FS_PRN_DBG("Body Text: %s", bodydata.c_str()); result = -ENOENT; break; @@ -2556,21 +2551,21 @@ int S3fsCurl::RequestPerform(bool dontAddAuthHeaders /*=false*/) case 501: S3FS_PRN_INFO3("HTTP response code 501 was returned, returning ENOTSUP"); - S3FS_PRN_DBG("Body Text: %s", bodydata.str()); + S3FS_PRN_DBG("Body Text: %s", bodydata.c_str()); result = -ENOTSUP; break; case 500: case 503: { S3FS_PRN_INFO3("HTTP response code %ld was returned, slowing down", responseCode); - S3FS_PRN_DBG("Body Text: %s", bodydata.str()); + S3FS_PRN_DBG("Body Text: %s", bodydata.c_str()); // Add jitter to avoid thundering herd. unsigned int sleep_time = 2 << retry_count; sleep(sleep_time + static_cast(random()) % sleep_time); break; } default: - S3FS_PRN_ERR("HTTP response code %ld, returning EIO. Body Text: %s", responseCode, bodydata.str()); + S3FS_PRN_ERR("HTTP response code %ld, returning EIO. Body Text: %s", responseCode, bodydata.c_str()); result = -EIO; break; } @@ -2980,7 +2975,7 @@ int S3fsCurl::GetIAMv2ApiToken(const char* token_url, int token_ttl, const char* } requestHeaders = nullptr; responseHeaders.clear(); - bodydata.Clear(); + bodydata.clear(); std::string ttlstr = std::to_string(token_ttl); requestHeaders = curl_slist_sort_insert(requestHeaders, token_ttl_hdr, ttlstr.c_str()); @@ -3016,11 +3011,11 @@ int S3fsCurl::GetIAMv2ApiToken(const char* token_url, int token_ttl, const char* int result = RequestPerform(true); if(0 == result){ - response = bodydata.str(); + response.swap(bodydata); }else{ S3FS_PRN_ERR("Error(%d) occurred, could not get IAMv2 api token.", result); } - bodydata.Clear(); + bodydata.clear(); return result; } @@ -3046,7 +3041,7 @@ bool S3fsCurl::GetIAMCredentials(const char* cred_url, const char* iam_v2_token, } requestHeaders = nullptr; responseHeaders.clear(); - bodydata.Clear(); + bodydata.clear(); std::string postContent; if(ibm_secret_access_key){ @@ -3103,11 +3098,11 @@ bool S3fsCurl::GetIAMCredentials(const char* cred_url, const char* iam_v2_token, // analyzing response if(0 == result){ - response = bodydata.str(); + response.swap(bodydata); }else{ S3FS_PRN_ERR("Error(%d) occurred, could not get IAM role name.", result); } - bodydata.Clear(); + bodydata.clear(); return (0 == result); } @@ -3134,7 +3129,7 @@ bool S3fsCurl::GetIAMRoleFromMetaData(const char* cred_url, const char* iam_v2_t } requestHeaders = nullptr; responseHeaders.clear(); - bodydata.Clear(); + bodydata.clear(); if(iam_v2_token){ requestHeaders = curl_slist_sort_insert(requestHeaders, S3fsCred::IAMv2_token_hdr, iam_v2_token); @@ -3161,11 +3156,11 @@ bool S3fsCurl::GetIAMRoleFromMetaData(const char* cred_url, const char* iam_v2_t // analyzing response if(0 == result){ - token = bodydata.str(); + token.swap(bodydata); }else{ S3FS_PRN_ERR("Error(%d) occurred, could not get IAM role name from meta data.", result); } - bodydata.Clear(); + bodydata.clear(); return (0 == result); } @@ -3328,7 +3323,7 @@ int S3fsCurl::PutHeadRequest(const char* tpath, headers_t& meta, bool is_copy) path = get_realpath(tpath); requestHeaders = nullptr; responseHeaders.clear(); - bodydata.Clear(); + bodydata.clear(); std::string contype = S3fsCurl::LookupMimeType(tpath); requestHeaders = curl_slist_sort_insert(requestHeaders, "Content-Type", contype.c_str()); @@ -3403,7 +3398,7 @@ int S3fsCurl::PutHeadRequest(const char* tpath, headers_t& meta, bool is_copy) int result = RequestPerform(); result = MapPutErrorResponse(result); - bodydata.Clear(); + bodydata.clear(); return result; } @@ -3454,7 +3449,7 @@ int S3fsCurl::PutRequest(const char* tpath, headers_t& meta, int fd) path = get_realpath(tpath); requestHeaders = nullptr; responseHeaders.clear(); - bodydata.Clear(); + bodydata.clear(); // Make request headers if(S3fsCurl::is_content_md5){ @@ -3545,7 +3540,7 @@ int S3fsCurl::PutRequest(const char* tpath, headers_t& meta, int fd) int result = RequestPerform(); result = MapPutErrorResponse(result); - bodydata.Clear(); + bodydata.clear(); if(file){ fclose(file); } @@ -3682,7 +3677,7 @@ int S3fsCurl::CheckBucket(const char* check_path, bool compat_dir) path = strCheckPath; requestHeaders = nullptr; responseHeaders.clear(); - bodydata.Clear(); + bodydata.clear(); op = "GET"; type = REQTYPE::CHKBUCKET; @@ -3706,7 +3701,7 @@ int S3fsCurl::CheckBucket(const char* check_path, bool compat_dir) int result = RequestPerform(); if (result != 0) { - S3FS_PRN_ERR("Check bucket failed, S3 response: %s", bodydata.str()); + S3FS_PRN_ERR("Check bucket failed, S3 response: %s", bodydata.c_str()); } return result; } @@ -3734,7 +3729,7 @@ int S3fsCurl::ListBucketRequest(const char* tpath, const char* query) path = get_realpath(tpath); requestHeaders = nullptr; responseHeaders.clear(); - bodydata.Clear(); + bodydata.clear(); op = "GET"; type = REQTYPE::LISTBUCKET; @@ -3789,7 +3784,7 @@ int S3fsCurl::PreMultipartPostRequest(const char* tpath, headers_t& meta, std::s url = prepare_url(turl.c_str()); path = get_realpath(tpath); requestHeaders = nullptr; - bodydata.Clear(); + bodydata.clear(); responseHeaders.clear(); std::string contype = S3fsCurl::LookupMimeType(tpath); @@ -3860,16 +3855,16 @@ int S3fsCurl::PreMultipartPostRequest(const char* tpath, headers_t& meta, std::s // request int result; if(0 != (result = RequestPerform())){ - bodydata.Clear(); + bodydata.clear(); return result; } - if(!simple_parse_xml(bodydata.str(), bodydata.size(), "UploadId", upload_id)){ - bodydata.Clear(); + if(!simple_parse_xml(bodydata.c_str(), bodydata.size(), "UploadId", upload_id)){ + bodydata.clear(); return -EIO; } - bodydata.Clear(); + bodydata.clear(); return 0; } @@ -3920,7 +3915,7 @@ int S3fsCurl::CompleteMultipartPostRequest(const char* tpath, const std::string& url = prepare_url(turl.c_str()); path = get_realpath(tpath); requestHeaders = nullptr; - bodydata.Clear(); + bodydata.clear(); responseHeaders.clear(); std::string contype = "application/xml"; @@ -3970,7 +3965,7 @@ int S3fsCurl::CompleteMultipartPostRequest(const char* tpath, const std::string& // request int result = RequestPerform(); - bodydata.Clear(); + bodydata.clear(); postdata = nullptr; b_postdata = nullptr; @@ -3994,7 +3989,7 @@ int S3fsCurl::MultipartListRequest(std::string& body) url = prepare_url(turl.c_str()); requestHeaders = nullptr; responseHeaders.clear(); - bodydata.Clear(); + bodydata.clear(); requestHeaders = curl_slist_sort_insert(requestHeaders, "Accept", nullptr); @@ -4017,11 +4012,11 @@ int S3fsCurl::MultipartListRequest(std::string& body) int result; if(0 == (result = RequestPerform()) && 0 < bodydata.size()){ - body = bodydata.str(); + body.swap(bodydata); }else{ body = ""; } - bodydata.Clear(); + bodydata.clear(); return result; } @@ -4118,8 +4113,8 @@ int S3fsCurl::UploadMultipartPostSetup(const char* tpath, int part_num, const st turl += urlargs; url = prepare_url(turl.c_str()); path = get_realpath(tpath); - bodydata.Clear(); - headdata.Clear(); + bodydata.clear(); + headdata.clear(); responseHeaders.clear(); // SSE-C @@ -4164,8 +4159,8 @@ int S3fsCurl::UploadMultipartPostRequest(const char* tpath, int part_num, const } // closing - bodydata.Clear(); - headdata.Clear(); + bodydata.clear(); + headdata.clear(); return result; } @@ -4192,8 +4187,8 @@ int S3fsCurl::CopyMultipartPostSetup(const char* from, const char* to, int part_ path = get_realpath(to); requestHeaders = nullptr; responseHeaders.clear(); - bodydata.Clear(); - headdata.Clear(); + bodydata.clear(); + headdata.clear(); std::string contype = S3fsCurl::LookupMimeType(to); requestHeaders = curl_slist_sort_insert(requestHeaders, "Content-Type", contype.c_str()); @@ -4278,15 +4273,15 @@ bool S3fsCurl::CopyMultipartPostCallback(S3fsCurl* s3fscurl, void* param) bool S3fsCurl::CopyMultipartPostComplete() { std::string etag; - partdata.uploaded = simple_parse_xml(bodydata.str(), bodydata.size(), "ETag", etag); + partdata.uploaded = simple_parse_xml(bodydata.c_str(), bodydata.size(), "ETag", etag); if(etag.size() >= 2 && *etag.begin() == '"' && *etag.rbegin() == '"'){ etag.erase(etag.size() - 1); etag.erase(0, 1); } partdata.petag->etag = etag; - bodydata.Clear(); - headdata.Clear(); + bodydata.clear(); + headdata.clear(); return true; } diff --git a/src/curl.h b/src/curl.h index 9224d1b..bf74514 100644 --- a/src/curl.h +++ b/src/curl.h @@ -27,7 +27,6 @@ #include #include "autolock.h" -#include "bodydata.h" #include "metaheader.h" #include "fdcache_page.h" @@ -167,8 +166,8 @@ class S3fsCurl std::string url; // target object path(url) struct curl_slist* requestHeaders; headers_t responseHeaders; // header data by HeaderCallback - BodyData bodydata; // body data by WriteMemoryCallback - BodyData headdata; // header data by WriteMemoryCallback + std::string bodydata; // body data by WriteMemoryCallback + std::string headdata; // header data by WriteMemoryCallback long LastResponseCode; const unsigned char* postdata; // use by post method and read callback function. off_t postdata_remaining; // use by post method and read callback function. @@ -377,8 +376,8 @@ class S3fsCurl std::string GetUrl() const { return url; } std::string GetOp() const { return op; } const headers_t* GetResponseHeaders() const { return &responseHeaders; } - const BodyData* GetBodyData() const { return &bodydata; } - const BodyData* GetHeadData() const { return &headdata; } + const std::string* GetBodyData() const { return &bodydata; } + const std::string* GetHeadData() const { return &headdata; } CURLcode GetCurlCode() const { return curlCode; } long GetLastResponseCode() const { return LastResponseCode; } bool SetUseAhbe(bool ahbe); diff --git a/src/s3fs.cpp b/src/s3fs.cpp index c6f05cf..824f29e 100644 --- a/src/s3fs.cpp +++ b/src/s3fs.cpp @@ -3489,14 +3489,14 @@ static int list_bucket(const char* path, S3ObjList& head, const char* delimiter, S3FS_PRN_ERR("ListBucketRequest returns with error."); return result; } - const BodyData* body = s3fscurl.GetBodyData(); + const std::string* body = s3fscurl.GetBodyData(); // [NOTE] // CR code(\r) is replaced with LF(\n) by xmlReadMemory() function. // To prevent that, only CR code is encoded by following function. // The encoded CR code is decoded with append_objects_from_xml(_ex). // - std::string encbody = get_encoded_cr_code(body->str()); + std::string encbody = get_encoded_cr_code(body->c_str()); // xmlDocPtr if(nullptr == (doc = xmlReadMemory(encbody.c_str(), static_cast(encbody.size()), "", nullptr, 0))){ @@ -4410,10 +4410,10 @@ static int s3fs_check_service() if(300 <= responseCode && responseCode < 500){ // check region error(for putting message or retrying) - const BodyData* body = s3fscurl.GetBodyData(); + const std::string* body = s3fscurl.GetBodyData(); std::string expectregion; std::string expectendpoint; - if(check_region_error(body->str(), body->size(), expectregion)){ + if(check_region_error(body->c_str(), body->size(), expectregion)){ // [NOTE] // If endpoint is not specified(using us-east-1 region) and // an error is encountered accessing a different region, we @@ -4455,7 +4455,7 @@ static int s3fs_check_service() S3FS_PRN_CRIT("The bucket region is not '%s'(default), it is correctly '%s'. You should specify endpoint(%s) option.", endpoint.c_str(), expectregion.c_str(), expectregion.c_str()); } - }else if(check_endpoint_error(body->str(), body->size(), expectendpoint)){ + }else if(check_endpoint_error(body->c_str(), body->size(), expectendpoint)){ // redirect error if(pathrequeststyle){ S3FS_PRN_CRIT("S3 service returned PermanentRedirect (current is url(%s) and endpoint(%s)). You need to specify correct url(http(s)://s3-.amazonaws.com) and endpoint option with use_path_request_style option.", s3host.c_str(), endpoint.c_str());