octoleo/s3fs-fuse
1
0
Fork 0
mirror of https://github.com/s3fs-fuse/s3fs-fuse.git synced 2024-06-15 15:22:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

Convert BodyData to std::string

Browse Source 
This is simpler and avoids some copies.
This commit is contained in:
Andrew Gaul 2023-08-17 11:20:04 +09:00 committed by Andrew Gaul
parent 235bccced5
commit 41d3a09931
6 changed files with 61 additions and 262 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/Makefile.am
View File

@ -40,7 +40,6 @@ s3fs_SOURCES = \
curl_handlerpool.cpp \
curl_multi.cpp \
curl_util.cpp \
bodydata.cpp \
s3objlist.cpp \
cache.cpp \
string_util.cpp \

122
src/bodydata.cpp
View File

@ -1,122 +0,0 @@
/*
* s3fs - FUSE-based file system backed by Amazon S3
*
* Copyright(C) 2007 Randy Rizun <rrizun@gmail.com>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include "s3fs_logger.h"
#include "bodydata.h"
//-------------------------------------------------------------------
// Variables
//-------------------------------------------------------------------
static const int BODYDATA_RESIZE_APPEND_MIN = 1024;
static const int BODYDATA_RESIZE_APPEND_MID = 1024 * 1024;
static const int BODYDATA_RESIZE_APPEND_MAX = 10 * 1024 * 1024;
//-------------------------------------------------------------------
// Utility Functions
//-------------------------------------------------------------------
static size_t adjust_block(size_t bytes, size_t block)
{
return ((bytes / block) + ((bytes % block) ? 1 : 0)) * block;
}
//-------------------------------------------------------------------
// Class BodyData
//-------------------------------------------------------------------
bool BodyData::Resize(size_t addbytes)
{
if(IsSafeSize(addbytes)){
return true;
}
// New size
size_t need_size = adjust_block((lastpos + addbytes + 1) - bufsize, sizeof(off_t));
if(BODYDATA_RESIZE_APPEND_MAX < bufsize){
need_size = (BODYDATA_RESIZE_APPEND_MAX < need_size ? need_size : BODYDATA_RESIZE_APPEND_MAX);
}else if(BODYDATA_RESIZE_APPEND_MID < bufsize){
need_size = (BODYDATA_RESIZE_APPEND_MID < need_size ? need_size : BODYDATA_RESIZE_APPEND_MID);
}else if(BODYDATA_RESIZE_APPEND_MIN < bufsize){
need_size = ((bufsize * 2) < need_size ? need_size : (bufsize * 2));
}else{
need_size = (BODYDATA_RESIZE_APPEND_MIN < need_size ? need_size : BODYDATA_RESIZE_APPEND_MIN);
}
// realloc
char* newtext;
if(nullptr == (newtext = reinterpret_cast<char*>(realloc(text, (bufsize + need_size))))){
S3FS_PRN_CRIT("not enough memory (realloc returned nullptr)");
free(text);
text = nullptr;
return false;
}
text = newtext;
bufsize += need_size;
return true;
}
void BodyData::Clear()
{
if(text){
free(text);
text = nullptr;
}
lastpos = 0;
bufsize = 0;
}
bool BodyData::Append(void* ptr, size_t bytes)
{
if(!ptr){
return false;
}
if(0 == bytes){
return true;
}
if(!Resize(bytes)){
return false;
}
memcpy(&text[lastpos], ptr, bytes);
lastpos += bytes;
text[lastpos] = '\0';
return true;
}
const char* BodyData::str() const
{
if(!text){
static const char strnull[] = "";
return strnull;
}
return text;
}
/*
* Local variables:
* tab-width: 4
* c-basic-offset: 4
* End:
* vim600: expandtab sw=4 ts=4 fdm=marker
* vim<600: expandtab sw=4 ts=4
*/

72
src/bodydata.h
View File

@ -1,72 +0,0 @@
/*
* s3fs - FUSE-based file system backed by Amazon S3
*
* Copyright(C) 2007 Randy Rizun <rrizun@gmail.com>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
#ifndef S3FS_BODYDATA_H_
#define S3FS_BODYDATA_H_
//----------------------------------------------
// Class BodyData
//----------------------------------------------
// memory class for curl write memory callback
//
class BodyData
{
private:
char* text;
size_t lastpos;
size_t bufsize;
private:
bool IsSafeSize(size_t addbytes) const
{
return ((lastpos + addbytes + 1) > bufsize ? false : true);
}
bool Resize(size_t addbytes);
public:
BodyData() : text(nullptr), lastpos(0), bufsize(0) {}
~BodyData()
{
Clear();
}
void Clear();
bool Append(void* ptr, size_t bytes);
bool Append(void* ptr, size_t blockSize, size_t numBlocks)
{
return Append(ptr, (blockSize * numBlocks));
}
const char* str() const;
size_t size() const
{
return lastpos;
}
};
#endif // S3FS_BODYDATA_H_
/*
* Local variables:
* tab-width: 4
* c-basic-offset: 4
* End:
* vim600: expandtab sw=4 ts=4 fdm=marker
* vim<600: expandtab sw=4 ts=4
*/

109
src/curl.cpp
View File

@ -593,13 +593,8 @@ bool S3fsCurl::LocateBundle()
size_t S3fsCurl::WriteMemoryCallback(void* ptr, size_t blockSize, size_t numBlocks, void* data)
{
BodyData* body = static_cast<BodyData*>(data);
if(!body->Append(ptr, blockSize, numBlocks)){
S3FS_PRN_CRIT("BodyData.Append() returned false.");
S3FS_FUSE_EXIT();
return -1;
}
std::string* body = static_cast<std::string*>(data);
body->append(static_cast<const char*>(ptr), blockSize * numBlocks);
return (blockSize * numBlocks);
}
@ -688,7 +683,7 @@ size_t S3fsCurl::DownloadWriteCallback(void* ptr, size_t size, size_t nmemb, voi
// Buffer initial bytes in case it is an XML error response.
if(pCurl->bodydata.size() < GET_OBJECT_RESPONSE_LIMIT){
pCurl->bodydata.Append(ptr, std::min(size * nmemb, GET_OBJECT_RESPONSE_LIMIT - pCurl->bodydata.size()));
pCurl->bodydata.append(static_cast<const char*>(ptr), std::min(size * nmemb, GET_OBJECT_RESPONSE_LIMIT - pCurl->bodydata.size()));
}
// write size
@ -1326,9 +1321,9 @@ int S3fsCurl::MapPutErrorResponse(int result)
// <HostId>BHzLOATeDuvN8Es1wI8IcERq4kl4dc2A9tOB8Yqr39Ys6fl7N4EJ8sjGiVvu6wLP</HostId>
// </Error>
//
const char* pstrbody = bodydata.str();
const char* pstrbody = bodydata.c_str();
std::string code;
if(!pstrbody || simple_parse_xml(pstrbody, bodydata.size(), "Code", code)){
if(simple_parse_xml(pstrbody, bodydata.size(), "Code", code)){
S3FS_PRN_ERR("Put request get 200 status response, but it included error body(or nullptr). The request failed during copying the object in S3. Code: %s", code.c_str());
// TODO: parse more specific error from <Code>
result = -EIO;
@ -2092,8 +2087,8 @@ bool S3fsCurl::ClearInternalData()
requestHeaders = nullptr;
}
responseHeaders.clear();
bodydata.Clear();
headdata.Clear();
bodydata.clear();
headdata.clear();
LastResponseCode = S3FSCURL_RESPONSECODE_NOTSET;
postdata = nullptr;
postdata_remaining = 0;
@ -2161,8 +2156,8 @@ bool S3fsCurl::RemakeHandle()
// reinitialize internal data
requestHeaders = curl_slist_remove(requestHeaders, "Authorization");
responseHeaders.clear();
bodydata.Clear();
headdata.Clear();
bodydata.clear();
headdata.clear();
LastResponseCode = S3FSCURL_RESPONSECODE_NOTSET;
// count up(only use for multipart)
@ -2505,7 +2500,7 @@ int S3fsCurl::RequestPerform(bool dontAddAuthHeaders /*=false*/)
{
// Try to parse more specific AWS error code otherwise fall back to HTTP error code.
std::string value;
if(simple_parse_xml(bodydata.str(), bodydata.size(), "Code", value)){
if(simple_parse_xml(bodydata.c_str(), bodydata.size(), "Code", value)){
// TODO: other error codes
if(value == "EntityTooLarge"){
result = -EFBIG;
@ -2524,7 +2519,7 @@ int S3fsCurl::RequestPerform(bool dontAddAuthHeaders /*=false*/)
switch(responseCode){
case 301:
case 307:
S3FS_PRN_ERR("HTTP response code 301(Moved Permanently: also happens when bucket's region is incorrect), returning EIO. Body Text: %s", bodydata.str());
S3FS_PRN_ERR("HTTP response code 301(Moved Permanently: also happens when bucket's region is incorrect), returning EIO. Body Text: %s", bodydata.c_str());
S3FS_PRN_ERR("The options of url and endpoint may be useful for solving, please try to use both options.");
result = -EIO;
break;
@ -2538,19 +2533,19 @@ int S3fsCurl::RequestPerform(bool dontAddAuthHeaders /*=false*/)
S3FS_PRN_ERR("HEAD HTTP response code %ld, returning EPERM.", responseCode);
result = -EPERM;
}else{
S3FS_PRN_ERR("HTTP response code %ld, returning EIO. Body Text: %s", responseCode, bodydata.str());
S3FS_PRN_ERR("HTTP response code %ld, returning EIO. Body Text: %s", responseCode, bodydata.c_str());
result = -EIO;
}
break;
case 403:
S3FS_PRN_ERR("HTTP response code %ld, returning EPERM. Body Text: %s", responseCode, bodydata.str());
S3FS_PRN_ERR("HTTP response code %ld, returning EPERM. Body Text: %s", responseCode, bodydata.c_str());
result = -EPERM;
break;
case 404:
S3FS_PRN_INFO3("HTTP response code 404 was returned, returning ENOENT");
S3FS_PRN_DBG("Body Text: %s", bodydata.str());
S3FS_PRN_DBG("Body Text: %s", bodydata.c_str());
result = -ENOENT;
break;
@ -2561,21 +2556,21 @@ int S3fsCurl::RequestPerform(bool dontAddAuthHeaders /*=false*/)
case 501:
S3FS_PRN_INFO3("HTTP response code 501 was returned, returning ENOTSUP");
S3FS_PRN_DBG("Body Text: %s", bodydata.str());
S3FS_PRN_DBG("Body Text: %s", bodydata.c_str());
result = -ENOTSUP;
break;
case 500:
case 503: {
S3FS_PRN_INFO3("HTTP response code %ld was returned, slowing down", responseCode);
S3FS_PRN_DBG("Body Text: %s", bodydata.str());
S3FS_PRN_DBG("Body Text: %s", bodydata.c_str());
// Add jitter to avoid thundering herd.
unsigned int sleep_time = 2 << retry_count;
sleep(sleep_time + static_cast<unsigned int>(random()) % sleep_time);
break;
}
default:
S3FS_PRN_ERR("HTTP response code %ld, returning EIO. Body Text: %s", responseCode, bodydata.str());
S3FS_PRN_ERR("HTTP response code %ld, returning EIO. Body Text: %s", responseCode, bodydata.c_str());
result = -EIO;
break;
}
@ -2985,7 +2980,7 @@ int S3fsCurl::GetIAMv2ApiToken(const char* token_url, int token_ttl, const char*
}
requestHeaders = nullptr;
responseHeaders.clear();
bodydata.Clear();
bodydata.clear();
std::string ttlstr = std::to_string(token_ttl);
requestHeaders = curl_slist_sort_insert(requestHeaders, token_ttl_hdr, ttlstr.c_str());
@ -3021,11 +3016,11 @@ int S3fsCurl::GetIAMv2ApiToken(const char* token_url, int token_ttl, const char*
int result = RequestPerform(true);
if(0 == result){
response = bodydata.str();
response.swap(bodydata);
}else{
S3FS_PRN_ERR("Error(%d) occurred, could not get IAMv2 api token.", result);
}
bodydata.Clear();
bodydata.clear();
return result;
}
@ -3051,7 +3046,7 @@ bool S3fsCurl::GetIAMCredentials(const char* cred_url, const char* iam_v2_token,
}
requestHeaders = nullptr;
responseHeaders.clear();
bodydata.Clear();
bodydata.clear();
std::string postContent;
if(ibm_secret_access_key){
@ -3107,11 +3102,11 @@ bool S3fsCurl::GetIAMCredentials(const char* cred_url, const char* iam_v2_token,
// analyzing response
if(0 == result){
response = bodydata.str();
response.swap(bodydata);
}else{
S3FS_PRN_ERR("Error(%d) occurred, could not get IAM role name.", result);
}
bodydata.Clear();
bodydata.clear();
return (0 == result);
}
@ -3138,7 +3133,7 @@ bool S3fsCurl::GetIAMRoleFromMetaData(const char* cred_url, const char* iam_v2_t
}
requestHeaders = nullptr;
responseHeaders.clear();
bodydata.Clear();
bodydata.clear();
if(iam_v2_token){
requestHeaders = curl_slist_sort_insert(requestHeaders, S3fsCred::IAMv2_token_hdr, iam_v2_token);
@ -3165,11 +3160,11 @@ bool S3fsCurl::GetIAMRoleFromMetaData(const char* cred_url, const char* iam_v2_t
// analyzing response
if(0 == result){
token = bodydata.str();
token.swap(bodydata);
}else{
S3FS_PRN_ERR("Error(%d) occurred, could not get IAM role name from meta data.", result);
}
bodydata.Clear();
bodydata.clear();
return (0 == result);
}
@ -3332,7 +3327,7 @@ int S3fsCurl::PutHeadRequest(const char* tpath, headers_t& meta, bool is_copy)
path = get_realpath(tpath);
requestHeaders = nullptr;
responseHeaders.clear();
bodydata.Clear();
bodydata.clear();
std::string contype = S3fsCurl::LookupMimeType(std::string(tpath));
requestHeaders = curl_slist_sort_insert(requestHeaders, "Content-Type", contype.c_str());
@ -3407,7 +3402,7 @@ int S3fsCurl::PutHeadRequest(const char* tpath, headers_t& meta, bool is_copy)
int result = RequestPerform();
result = MapPutErrorResponse(result);
bodydata.Clear();
bodydata.clear();
return result;
}
@ -3458,7 +3453,7 @@ int S3fsCurl::PutRequest(const char* tpath, headers_t& meta, int fd)
path = get_realpath(tpath);
requestHeaders = nullptr;
responseHeaders.clear();
bodydata.Clear();
bodydata.clear();
// Make request headers
if(S3fsCurl::is_content_md5){
@ -3549,7 +3544,7 @@ int S3fsCurl::PutRequest(const char* tpath, headers_t& meta, int fd)
int result = RequestPerform();
result = MapPutErrorResponse(result);
bodydata.Clear();
bodydata.clear();
if(file){
fclose(file);
}
@ -3686,7 +3681,7 @@ int S3fsCurl::CheckBucket(const char* check_path, bool compat_dir)
path = strCheckPath;
requestHeaders = nullptr;
responseHeaders.clear();
bodydata.Clear();
bodydata.clear();
op = "GET";
type = REQTYPE::CHKBUCKET;
@ -3710,7 +3705,7 @@ int S3fsCurl::CheckBucket(const char* check_path, bool compat_dir)
int result = RequestPerform();
if (result != 0) {
S3FS_PRN_ERR("Check bucket failed, S3 response: %s", bodydata.str());
S3FS_PRN_ERR("Check bucket failed, S3 response: %s", bodydata.c_str());
}
return result;
}
@ -3738,7 +3733,7 @@ int S3fsCurl::ListBucketRequest(const char* tpath, const char* query)
path = get_realpath(tpath);
requestHeaders = nullptr;
responseHeaders.clear();
bodydata.Clear();
bodydata.clear();
op = "GET";
type = REQTYPE::LISTBUCKET;
@ -3793,7 +3788,7 @@ int S3fsCurl::PreMultipartPostRequest(const char* tpath, headers_t& meta, std::s
url = prepare_url(turl.c_str());
path = get_realpath(tpath);
requestHeaders = nullptr;
bodydata.Clear();
bodydata.clear();
responseHeaders.clear();
std::string contype = S3fsCurl::LookupMimeType(std::string(tpath));
@ -3864,16 +3859,16 @@ int S3fsCurl::PreMultipartPostRequest(const char* tpath, headers_t& meta, std::s
// request
int result;
if(0 != (result = RequestPerform())){
bodydata.Clear();
bodydata.clear();
return result;
}
if(!simple_parse_xml(bodydata.str(), bodydata.size(), "UploadId", upload_id)){
bodydata.Clear();
if(!simple_parse_xml(bodydata.c_str(), bodydata.size(), "UploadId", upload_id)){
bodydata.clear();
return -EIO;
}
bodydata.Clear();
bodydata.clear();
return 0;
}
@ -3924,7 +3919,7 @@ int S3fsCurl::CompleteMultipartPostRequest(const char* tpath, const std::string&
url = prepare_url(turl.c_str());
path = get_realpath(tpath);
requestHeaders = nullptr;
bodydata.Clear();
bodydata.clear();
responseHeaders.clear();
std::string contype = "application/xml";
@ -3974,7 +3969,7 @@ int S3fsCurl::CompleteMultipartPostRequest(const char* tpath, const std::string&
// request
int result = RequestPerform();
bodydata.Clear();
bodydata.clear();
postdata = nullptr;
b_postdata = nullptr;
@ -3998,7 +3993,7 @@ int S3fsCurl::MultipartListRequest(std::string& body)
url = prepare_url(turl.c_str());
requestHeaders = nullptr;
responseHeaders.clear();
bodydata.Clear();
bodydata.clear();
requestHeaders = curl_slist_sort_insert(requestHeaders, "Accept", nullptr);
@ -4021,11 +4016,11 @@ int S3fsCurl::MultipartListRequest(std::string& body)
int result;
if(0 == (result = RequestPerform()) && 0 < bodydata.size()){
body = bodydata.str();
body.swap(bodydata);
}else{
body = "";
}
bodydata.Clear();
bodydata.clear();
return result;
}
@ -4122,8 +4117,8 @@ int S3fsCurl::UploadMultipartPostSetup(const char* tpath, int part_num, const st
turl += urlargs;
url = prepare_url(turl.c_str());
path = get_realpath(tpath);
bodydata.Clear();
headdata.Clear();
bodydata.clear();
headdata.clear();
responseHeaders.clear();
// SSE-C
@ -4168,8 +4163,8 @@ int S3fsCurl::UploadMultipartPostRequest(const char* tpath, int part_num, const
}
// closing
bodydata.Clear();
headdata.Clear();
bodydata.clear();
headdata.clear();
return result;
}
@ -4196,8 +4191,8 @@ int S3fsCurl::CopyMultipartPostSetup(const char* from, const char* to, int part_
path = get_realpath(to);
requestHeaders = nullptr;
responseHeaders.clear();
bodydata.Clear();
headdata.Clear();
bodydata.clear();
headdata.clear();
std::string contype = S3fsCurl::LookupMimeType(std::string(to));
requestHeaders = curl_slist_sort_insert(requestHeaders, "Content-Type", contype.c_str());
@ -4282,15 +4277,15 @@ bool S3fsCurl::CopyMultipartPostCallback(S3fsCurl* s3fscurl, void* param)
bool S3fsCurl::CopyMultipartPostComplete()
{
std::string etag;
partdata.uploaded = simple_parse_xml(bodydata.str(), bodydata.size(), "ETag", etag);
partdata.uploaded = simple_parse_xml(bodydata.c_str(), bodydata.size(), "ETag", etag);
if(etag.size() >= 2 && *etag.begin() == '"' && *etag.rbegin() == '"'){
etag.erase(etag.size() - 1);
etag.erase(0, 1);
}
partdata.petag->etag = etag;
bodydata.Clear();
headdata.Clear();
bodydata.clear();
headdata.clear();
return true;
}

9
src/curl.h
View File

@ -27,7 +27,6 @@
#include <vector>
#include "autolock.h"
#include "bodydata.h"
#include "metaheader.h"
#include "fdcache_page.h"
@ -167,8 +166,8 @@ class S3fsCurl
std::string url; // target object path(url)
struct curl_slist* requestHeaders;
headers_t responseHeaders; // header data by HeaderCallback
BodyData bodydata; // body data by WriteMemoryCallback
BodyData headdata; // header data by WriteMemoryCallback
std::string bodydata; // body data by WriteMemoryCallback
std::string headdata; // header data by WriteMemoryCallback
long LastResponseCode;
const unsigned char* postdata; // use by post method and read callback function.
off_t postdata_remaining; // use by post method and read callback function.
@ -377,8 +376,8 @@ class S3fsCurl
std::string GetUrl() const { return url; }
std::string GetOp() const { return op; }
const headers_t* GetResponseHeaders() const { return &responseHeaders; }
const BodyData* GetBodyData() const { return &bodydata; }
const BodyData* GetHeadData() const { return &headdata; }
const std::string* GetBodyData() const { return &bodydata; }
const std::string* GetHeadData() const { return &headdata; }
CURLcode GetCurlCode() const { return curlCode; }
long GetLastResponseCode() const { return LastResponseCode; }
bool SetUseAhbe(bool ahbe);

10
src/s3fs.cpp
View File

@ -3490,14 +3490,14 @@ static int list_bucket(const char* path, S3ObjList& head, const char* delimiter,
S3FS_PRN_ERR("ListBucketRequest returns with error.");
return result;
}
const BodyData* body = s3fscurl.GetBodyData();
const std::string* body = s3fscurl.GetBodyData();
// [NOTE]
// CR code(\r) is replaced with LF(\n) by xmlReadMemory() function.
// To prevent that, only CR code is encoded by following function.
// The encoded CR code is decoded with append_objects_from_xml(_ex).
//
std::string encbody = get_encoded_cr_code(body->str());
std::string encbody = get_encoded_cr_code(body->c_str());
// xmlDocPtr
if(nullptr == (doc = xmlReadMemory(encbody.c_str(), static_cast<int>(encbody.size()), "", nullptr, 0))){
@ -4411,10 +4411,10 @@ static int s3fs_check_service()
if(300 <= responseCode && responseCode < 500){
// check region error(for putting message or retrying)
const BodyData* body = s3fscurl.GetBodyData();
const std::string* body = s3fscurl.GetBodyData();
std::string expectregion;
std::string expectendpoint;
if(check_region_error(body->str(), body->size(), expectregion)){
if(check_region_error(body->c_str(), body->size(), expectregion)){
// [NOTE]
// If endpoint is not specified(using us-east-1 region) and
// an error is encountered accessing a different region, we
@ -4456,7 +4456,7 @@ static int s3fs_check_service()
S3FS_PRN_CRIT("The bucket region is not '%s'(default), it is correctly '%s'. You should specify endpoint(%s) option.", endpoint.c_str(), expectregion.c_str(), expectregion.c_str());
}
}else if(check_endpoint_error(body->str(), body->size(), expectendpoint)){
}else if(check_endpoint_error(body->c_str(), body->size(), expectendpoint)){
// redirect error
if(pathrequeststyle){
S3FS_PRN_CRIT("S3 service returned PermanentRedirect (current is url(%s) and endpoint(%s)). You need to specify correct url(http(s)://s3-<endpoint>.amazonaws.com) and endpoint option with use_path_request_style option.", s3host.c_str(), endpoint.c_str());