octoleo/s3fs-fuse
1
0
Fork 0
mirror of https://github.com/s3fs-fuse/s3fs-fuse.git synced 2025-01-05 14:02:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

Convert BodyData to std::string (#2276)

Browse Source 
This is simpler and avoids some copies.
This commit is contained in:
Andrew Gaul 2023-08-17 22:49:41 +09:00 committed by GitHub
parent c65ce8a42c
commit 2518ff3568
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 61 additions and 262 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/Makefile.am
View File

@ -40,7 +40,6 @@ s3fs_SOURCES = \
curl_handlerpool.cpp \ curl_handlerpool.cpp \
curl_multi.cpp \ curl_multi.cpp \
curl_util.cpp \ curl_util.cpp \
bodydata.cpp \
s3objlist.cpp \ s3objlist.cpp \
cache.cpp \ cache.cpp \
string_util.cpp \ string_util.cpp \

122
src/bodydata.cpp
View File

@ -1,122 +0,0 @@
/*
* s3fs - FUSE-based file system backed by Amazon S3
*
* Copyright(C) 2007 Randy Rizun <rrizun@gmail.com>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include "s3fs_logger.h"
#include "bodydata.h"
//-------------------------------------------------------------------
// Variables
//-------------------------------------------------------------------
static const int BODYDATA_RESIZE_APPEND_MIN = 1024;
static const int BODYDATA_RESIZE_APPEND_MID = 1024 * 1024;
static const int BODYDATA_RESIZE_APPEND_MAX = 10 * 1024 * 1024;
//-------------------------------------------------------------------
// Utility Functions
//-------------------------------------------------------------------
static size_t adjust_block(size_t bytes, size_t block)
{
return ((bytes / block) + ((bytes % block) ? 1 : 0)) * block;
}
//-------------------------------------------------------------------
// Class BodyData
//-------------------------------------------------------------------
bool BodyData::Resize(size_t addbytes)
{
if(IsSafeSize(addbytes)){
return true;
}
// New size
size_t need_size = adjust_block((lastpos + addbytes + 1) - bufsize, sizeof(off_t));
if(BODYDATA_RESIZE_APPEND_MAX < bufsize){
need_size = (BODYDATA_RESIZE_APPEND_MAX < need_size ? need_size : BODYDATA_RESIZE_APPEND_MAX);
}else if(BODYDATA_RESIZE_APPEND_MID < bufsize){
need_size = (BODYDATA_RESIZE_APPEND_MID < need_size ? need_size : BODYDATA_RESIZE_APPEND_MID);
}else if(BODYDATA_RESIZE_APPEND_MIN < bufsize){
need_size = ((bufsize * 2) < need_size ? need_size : (bufsize * 2));
}else{
need_size = (BODYDATA_RESIZE_APPEND_MIN < need_size ? need_size : BODYDATA_RESIZE_APPEND_MIN);
}
// realloc
char* newtext;
if(nullptr == (newtext = reinterpret_cast<char*>(realloc(text, (bufsize + need_size))))){
S3FS_PRN_CRIT("not enough memory (realloc returned nullptr)");
free(text);
text = nullptr;
return false;
}
text = newtext;
bufsize += need_size;
return true;
}
void BodyData::Clear()
{
if(text){
free(text);
text = nullptr;
}
lastpos = 0;
bufsize = 0;
}
bool BodyData::Append(void* ptr, size_t bytes)
{
if(!ptr){
return false;
}
if(0 == bytes){
return true;
}
if(!Resize(bytes)){
return false;
}
memcpy(&text[lastpos], ptr, bytes);
lastpos += bytes;
text[lastpos] = '\0';
return true;
}
const char* BodyData::str() const
{
if(!text){
static const char strnull[] = "";
return strnull;
}
return text;
}
/*
* Local variables:
* tab-width: 4
* c-basic-offset: 4
* End:
* vim600: expandtab sw=4 ts=4 fdm=marker
* vim<600: expandtab sw=4 ts=4
*/

72
src/bodydata.h
View File

@ -1,72 +0,0 @@
/*
* s3fs - FUSE-based file system backed by Amazon S3
*
* Copyright(C) 2007 Randy Rizun <rrizun@gmail.com>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
#ifndef S3FS_BODYDATA_H_
#define S3FS_BODYDATA_H_
//----------------------------------------------
// Class BodyData
//----------------------------------------------
// memory class for curl write memory callback
//
class BodyData
{
private:
char* text;
size_t lastpos;
size_t bufsize;
private:
bool IsSafeSize(size_t addbytes) const
{
return ((lastpos + addbytes + 1) > bufsize ? false : true);
}
bool Resize(size_t addbytes);
public:
BodyData() : text(nullptr), lastpos(0), bufsize(0) {}
~BodyData()
{
Clear();
}
void Clear();
bool Append(void* ptr, size_t bytes);
bool Append(void* ptr, size_t blockSize, size_t numBlocks)
{
return Append(ptr, (blockSize * numBlocks));
}
const char* str() const;
size_t size() const
{
return lastpos;
}
};
#endif // S3FS_BODYDATA_H_
/*
* Local variables:
* tab-width: 4
* c-basic-offset: 4
* End:
* vim600: expandtab sw=4 ts=4 fdm=marker
* vim<600: expandtab sw=4 ts=4
*/

109
src/curl.cpp
View File

@ -593,13 +593,8 @@ bool S3fsCurl::LocateBundle()
size_t S3fsCurl::WriteMemoryCallback(void* ptr, size_t blockSize, size_t numBlocks, void* data) size_t S3fsCurl::WriteMemoryCallback(void* ptr, size_t blockSize, size_t numBlocks, void* data)
{ {
BodyData* body = static_cast<BodyData*>(data); std::string* body = static_cast<std::string*>(data);
body->append(static_cast<const char*>(ptr), blockSize * numBlocks);
if(!body->Append(ptr, blockSize, numBlocks)){
S3FS_PRN_CRIT("BodyData.Append() returned false.");
S3FS_FUSE_EXIT();
return -1;
}
return (blockSize * numBlocks); return (blockSize * numBlocks);
} }
@ -688,7 +683,7 @@ size_t S3fsCurl::DownloadWriteCallback(void* ptr, size_t size, size_t nmemb, voi
// Buffer initial bytes in case it is an XML error response. // Buffer initial bytes in case it is an XML error response.
if(pCurl->bodydata.size() < GET_OBJECT_RESPONSE_LIMIT){ if(pCurl->bodydata.size() < GET_OBJECT_RESPONSE_LIMIT){
pCurl->bodydata.Append(ptr, std::min(size * nmemb, GET_OBJECT_RESPONSE_LIMIT - pCurl->bodydata.size())); pCurl->bodydata.append(static_cast<const char*>(ptr), std::min(size * nmemb, GET_OBJECT_RESPONSE_LIMIT - pCurl->bodydata.size()));
} }
// write size // write size
@ -1322,9 +1317,9 @@ int S3fsCurl::MapPutErrorResponse(int result)
// <HostId>BHzLOATeDuvN8Es1wI8IcERq4kl4dc2A9tOB8Yqr39Ys6fl7N4EJ8sjGiVvu6wLP</HostId> // <HostId>BHzLOATeDuvN8Es1wI8IcERq4kl4dc2A9tOB8Yqr39Ys6fl7N4EJ8sjGiVvu6wLP</HostId>
// </Error> // </Error>
// //
const char* pstrbody = bodydata.str(); const char* pstrbody = bodydata.c_str();
std::string code; std::string code;
if(!pstrbody || simple_parse_xml(pstrbody, bodydata.size(), "Code", code)){ if(simple_parse_xml(pstrbody, bodydata.size(), "Code", code)){
S3FS_PRN_ERR("Put request get 200 status response, but it included error body(or nullptr). The request failed during copying the object in S3. Code: %s", code.c_str()); S3FS_PRN_ERR("Put request get 200 status response, but it included error body(or nullptr). The request failed during copying the object in S3. Code: %s", code.c_str());
// TODO: parse more specific error from <Code> // TODO: parse more specific error from <Code>
result = -EIO; result = -EIO;
@ -2087,8 +2082,8 @@ bool S3fsCurl::ClearInternalData()
requestHeaders = nullptr; requestHeaders = nullptr;
} }
responseHeaders.clear(); responseHeaders.clear();
bodydata.Clear(); bodydata.clear();
headdata.Clear(); headdata.clear();
LastResponseCode = S3FSCURL_RESPONSECODE_NOTSET; LastResponseCode = S3FSCURL_RESPONSECODE_NOTSET;
postdata = nullptr; postdata = nullptr;
postdata_remaining = 0; postdata_remaining = 0;
@ -2156,8 +2151,8 @@ bool S3fsCurl::RemakeHandle()
// reinitialize internal data // reinitialize internal data
requestHeaders = curl_slist_remove(requestHeaders, "Authorization"); requestHeaders = curl_slist_remove(requestHeaders, "Authorization");
responseHeaders.clear(); responseHeaders.clear();
bodydata.Clear(); bodydata.clear();
headdata.Clear(); headdata.clear();
LastResponseCode = S3FSCURL_RESPONSECODE_NOTSET; LastResponseCode = S3FSCURL_RESPONSECODE_NOTSET;
// count up(only use for multipart) // count up(only use for multipart)
@ -2500,7 +2495,7 @@ int S3fsCurl::RequestPerform(bool dontAddAuthHeaders /*=false*/)
{ {
// Try to parse more specific AWS error code otherwise fall back to HTTP error code. // Try to parse more specific AWS error code otherwise fall back to HTTP error code.
std::string value; std::string value;
if(simple_parse_xml(bodydata.str(), bodydata.size(), "Code", value)){ if(simple_parse_xml(bodydata.c_str(), bodydata.size(), "Code", value)){
// TODO: other error codes // TODO: other error codes
if(value == "EntityTooLarge"){ if(value == "EntityTooLarge"){
result = -EFBIG; result = -EFBIG;
@ -2519,7 +2514,7 @@ int S3fsCurl::RequestPerform(bool dontAddAuthHeaders /*=false*/)
switch(responseCode){ switch(responseCode){
case 301: case 301:
case 307: case 307:
S3FS_PRN_ERR("HTTP response code 301(Moved Permanently: also happens when bucket's region is incorrect), returning EIO. Body Text: %s", bodydata.str()); S3FS_PRN_ERR("HTTP response code 301(Moved Permanently: also happens when bucket's region is incorrect), returning EIO. Body Text: %s", bodydata.c_str());
S3FS_PRN_ERR("The options of url and endpoint may be useful for solving, please try to use both options."); S3FS_PRN_ERR("The options of url and endpoint may be useful for solving, please try to use both options.");
result = -EIO; result = -EIO;
break; break;
@ -2533,19 +2528,19 @@ int S3fsCurl::RequestPerform(bool dontAddAuthHeaders /*=false*/)
S3FS_PRN_ERR("HEAD HTTP response code %ld, returning EPERM.", responseCode); S3FS_PRN_ERR("HEAD HTTP response code %ld, returning EPERM.", responseCode);
result = -EPERM; result = -EPERM;
}else{ }else{
S3FS_PRN_ERR("HTTP response code %ld, returning EIO. Body Text: %s", responseCode, bodydata.str()); S3FS_PRN_ERR("HTTP response code %ld, returning EIO. Body Text: %s", responseCode, bodydata.c_str());
result = -EIO; result = -EIO;
} }
break; break;
case 403: case 403:
S3FS_PRN_ERR("HTTP response code %ld, returning EPERM. Body Text: %s", responseCode, bodydata.str()); S3FS_PRN_ERR("HTTP response code %ld, returning EPERM. Body Text: %s", responseCode, bodydata.c_str());
result = -EPERM; result = -EPERM;
break; break;
case 404: case 404:
S3FS_PRN_INFO3("HTTP response code 404 was returned, returning ENOENT"); S3FS_PRN_INFO3("HTTP response code 404 was returned, returning ENOENT");
S3FS_PRN_DBG("Body Text: %s", bodydata.str()); S3FS_PRN_DBG("Body Text: %s", bodydata.c_str());
result = -ENOENT; result = -ENOENT;
break; break;
@ -2556,21 +2551,21 @@ int S3fsCurl::RequestPerform(bool dontAddAuthHeaders /*=false*/)
case 501: case 501:
S3FS_PRN_INFO3("HTTP response code 501 was returned, returning ENOTSUP"); S3FS_PRN_INFO3("HTTP response code 501 was returned, returning ENOTSUP");
S3FS_PRN_DBG("Body Text: %s", bodydata.str()); S3FS_PRN_DBG("Body Text: %s", bodydata.c_str());
result = -ENOTSUP; result = -ENOTSUP;
break; break;
case 500: case 500:
case 503: { case 503: {
S3FS_PRN_INFO3("HTTP response code %ld was returned, slowing down", responseCode); S3FS_PRN_INFO3("HTTP response code %ld was returned, slowing down", responseCode);
S3FS_PRN_DBG("Body Text: %s", bodydata.str()); S3FS_PRN_DBG("Body Text: %s", bodydata.c_str());
// Add jitter to avoid thundering herd. // Add jitter to avoid thundering herd.
unsigned int sleep_time = 2 << retry_count; unsigned int sleep_time = 2 << retry_count;
sleep(sleep_time + static_cast<unsigned int>(random()) % sleep_time); sleep(sleep_time + static_cast<unsigned int>(random()) % sleep_time);
break; break;
} }
default: default:
S3FS_PRN_ERR("HTTP response code %ld, returning EIO. Body Text: %s", responseCode, bodydata.str()); S3FS_PRN_ERR("HTTP response code %ld, returning EIO. Body Text: %s", responseCode, bodydata.c_str());
result = -EIO; result = -EIO;
break; break;
} }
@ -2980,7 +2975,7 @@ int S3fsCurl::GetIAMv2ApiToken(const char* token_url, int token_ttl, const char*
} }
requestHeaders = nullptr; requestHeaders = nullptr;
responseHeaders.clear(); responseHeaders.clear();
bodydata.Clear(); bodydata.clear();
std::string ttlstr = std::to_string(token_ttl); std::string ttlstr = std::to_string(token_ttl);
requestHeaders = curl_slist_sort_insert(requestHeaders, token_ttl_hdr, ttlstr.c_str()); requestHeaders = curl_slist_sort_insert(requestHeaders, token_ttl_hdr, ttlstr.c_str());
@ -3016,11 +3011,11 @@ int S3fsCurl::GetIAMv2ApiToken(const char* token_url, int token_ttl, const char*
int result = RequestPerform(true); int result = RequestPerform(true);
if(0 == result){ if(0 == result){
response = bodydata.str(); response.swap(bodydata);
}else{ }else{
S3FS_PRN_ERR("Error(%d) occurred, could not get IAMv2 api token.", result); S3FS_PRN_ERR("Error(%d) occurred, could not get IAMv2 api token.", result);
} }
bodydata.Clear(); bodydata.clear();
return result; return result;
} }
@ -3046,7 +3041,7 @@ bool S3fsCurl::GetIAMCredentials(const char* cred_url, const char* iam_v2_token,
} }
requestHeaders = nullptr; requestHeaders = nullptr;
responseHeaders.clear(); responseHeaders.clear();
bodydata.Clear(); bodydata.clear();
std::string postContent; std::string postContent;
if(ibm_secret_access_key){ if(ibm_secret_access_key){
@ -3103,11 +3098,11 @@ bool S3fsCurl::GetIAMCredentials(const char* cred_url, const char* iam_v2_token,
// analyzing response // analyzing response
if(0 == result){ if(0 == result){
response = bodydata.str(); response.swap(bodydata);
}else{ }else{
S3FS_PRN_ERR("Error(%d) occurred, could not get IAM role name.", result); S3FS_PRN_ERR("Error(%d) occurred, could not get IAM role name.", result);
} }
bodydata.Clear(); bodydata.clear();
return (0 == result); return (0 == result);
} }
@ -3134,7 +3129,7 @@ bool S3fsCurl::GetIAMRoleFromMetaData(const char* cred_url, const char* iam_v2_t
} }
requestHeaders = nullptr; requestHeaders = nullptr;
responseHeaders.clear(); responseHeaders.clear();
bodydata.Clear(); bodydata.clear();
if(iam_v2_token){ if(iam_v2_token){
requestHeaders = curl_slist_sort_insert(requestHeaders, S3fsCred::IAMv2_token_hdr, iam_v2_token); requestHeaders = curl_slist_sort_insert(requestHeaders, S3fsCred::IAMv2_token_hdr, iam_v2_token);
@ -3161,11 +3156,11 @@ bool S3fsCurl::GetIAMRoleFromMetaData(const char* cred_url, const char* iam_v2_t
// analyzing response // analyzing response
if(0 == result){ if(0 == result){
token = bodydata.str(); token.swap(bodydata);
}else{ }else{
S3FS_PRN_ERR("Error(%d) occurred, could not get IAM role name from meta data.", result); S3FS_PRN_ERR("Error(%d) occurred, could not get IAM role name from meta data.", result);
} }
bodydata.Clear(); bodydata.clear();
return (0 == result); return (0 == result);
} }
@ -3328,7 +3323,7 @@ int S3fsCurl::PutHeadRequest(const char* tpath, headers_t& meta, bool is_copy)
path = get_realpath(tpath); path = get_realpath(tpath);
requestHeaders = nullptr; requestHeaders = nullptr;
responseHeaders.clear(); responseHeaders.clear();
bodydata.Clear(); bodydata.clear();
std::string contype = S3fsCurl::LookupMimeType(tpath); std::string contype = S3fsCurl::LookupMimeType(tpath);
requestHeaders = curl_slist_sort_insert(requestHeaders, "Content-Type", contype.c_str()); requestHeaders = curl_slist_sort_insert(requestHeaders, "Content-Type", contype.c_str());
@ -3403,7 +3398,7 @@ int S3fsCurl::PutHeadRequest(const char* tpath, headers_t& meta, bool is_copy)
int result = RequestPerform(); int result = RequestPerform();
result = MapPutErrorResponse(result); result = MapPutErrorResponse(result);
bodydata.Clear(); bodydata.clear();
return result; return result;
} }
@ -3454,7 +3449,7 @@ int S3fsCurl::PutRequest(const char* tpath, headers_t& meta, int fd)
path = get_realpath(tpath); path = get_realpath(tpath);
requestHeaders = nullptr; requestHeaders = nullptr;
responseHeaders.clear(); responseHeaders.clear();
bodydata.Clear(); bodydata.clear();
// Make request headers // Make request headers
if(S3fsCurl::is_content_md5){ if(S3fsCurl::is_content_md5){
@ -3545,7 +3540,7 @@ int S3fsCurl::PutRequest(const char* tpath, headers_t& meta, int fd)
int result = RequestPerform(); int result = RequestPerform();
result = MapPutErrorResponse(result); result = MapPutErrorResponse(result);
bodydata.Clear(); bodydata.clear();
if(file){ if(file){
fclose(file); fclose(file);
} }
@ -3682,7 +3677,7 @@ int S3fsCurl::CheckBucket(const char* check_path, bool compat_dir)
path = strCheckPath; path = strCheckPath;
requestHeaders = nullptr; requestHeaders = nullptr;
responseHeaders.clear(); responseHeaders.clear();
bodydata.Clear(); bodydata.clear();
op = "GET"; op = "GET";
type = REQTYPE::CHKBUCKET; type = REQTYPE::CHKBUCKET;
@ -3706,7 +3701,7 @@ int S3fsCurl::CheckBucket(const char* check_path, bool compat_dir)
int result = RequestPerform(); int result = RequestPerform();
if (result != 0) { if (result != 0) {
S3FS_PRN_ERR("Check bucket failed, S3 response: %s", bodydata.str()); S3FS_PRN_ERR("Check bucket failed, S3 response: %s", bodydata.c_str());
} }
return result; return result;
} }
@ -3734,7 +3729,7 @@ int S3fsCurl::ListBucketRequest(const char* tpath, const char* query)
path = get_realpath(tpath); path = get_realpath(tpath);
requestHeaders = nullptr; requestHeaders = nullptr;
responseHeaders.clear(); responseHeaders.clear();
bodydata.Clear(); bodydata.clear();
op = "GET"; op = "GET";
type = REQTYPE::LISTBUCKET; type = REQTYPE::LISTBUCKET;
@ -3789,7 +3784,7 @@ int S3fsCurl::PreMultipartPostRequest(const char* tpath, headers_t& meta, std::s
url = prepare_url(turl.c_str()); url = prepare_url(turl.c_str());
path = get_realpath(tpath); path = get_realpath(tpath);
requestHeaders = nullptr; requestHeaders = nullptr;
bodydata.Clear(); bodydata.clear();
responseHeaders.clear(); responseHeaders.clear();
std::string contype = S3fsCurl::LookupMimeType(tpath); std::string contype = S3fsCurl::LookupMimeType(tpath);
@ -3860,16 +3855,16 @@ int S3fsCurl::PreMultipartPostRequest(const char* tpath, headers_t& meta, std::s
// request // request
int result; int result;
if(0 != (result = RequestPerform())){ if(0 != (result = RequestPerform())){
bodydata.Clear(); bodydata.clear();
return result; return result;
} }
if(!simple_parse_xml(bodydata.str(), bodydata.size(), "UploadId", upload_id)){ if(!simple_parse_xml(bodydata.c_str(), bodydata.size(), "UploadId", upload_id)){
bodydata.Clear(); bodydata.clear();
return -EIO; return -EIO;
} }
bodydata.Clear(); bodydata.clear();
return 0; return 0;
} }
@ -3920,7 +3915,7 @@ int S3fsCurl::CompleteMultipartPostRequest(const char* tpath, const std::string&
url = prepare_url(turl.c_str()); url = prepare_url(turl.c_str());
path = get_realpath(tpath); path = get_realpath(tpath);
requestHeaders = nullptr; requestHeaders = nullptr;
bodydata.Clear(); bodydata.clear();
responseHeaders.clear(); responseHeaders.clear();
std::string contype = "application/xml"; std::string contype = "application/xml";
@ -3970,7 +3965,7 @@ int S3fsCurl::CompleteMultipartPostRequest(const char* tpath, const std::string&
// request // request
int result = RequestPerform(); int result = RequestPerform();
bodydata.Clear(); bodydata.clear();
postdata = nullptr; postdata = nullptr;
b_postdata = nullptr; b_postdata = nullptr;
@ -3994,7 +3989,7 @@ int S3fsCurl::MultipartListRequest(std::string& body)
url = prepare_url(turl.c_str()); url = prepare_url(turl.c_str());
requestHeaders = nullptr; requestHeaders = nullptr;
responseHeaders.clear(); responseHeaders.clear();
bodydata.Clear(); bodydata.clear();
requestHeaders = curl_slist_sort_insert(requestHeaders, "Accept", nullptr); requestHeaders = curl_slist_sort_insert(requestHeaders, "Accept", nullptr);
@ -4017,11 +4012,11 @@ int S3fsCurl::MultipartListRequest(std::string& body)
int result; int result;
if(0 == (result = RequestPerform()) && 0 < bodydata.size()){ if(0 == (result = RequestPerform()) && 0 < bodydata.size()){
body = bodydata.str(); body.swap(bodydata);
}else{ }else{
body = ""; body = "";
} }
bodydata.Clear(); bodydata.clear();
return result; return result;
} }
@ -4118,8 +4113,8 @@ int S3fsCurl::UploadMultipartPostSetup(const char* tpath, int part_num, const st
turl += urlargs; turl += urlargs;
url = prepare_url(turl.c_str()); url = prepare_url(turl.c_str());
path = get_realpath(tpath); path = get_realpath(tpath);
bodydata.Clear(); bodydata.clear();
headdata.Clear(); headdata.clear();
responseHeaders.clear(); responseHeaders.clear();
// SSE-C // SSE-C
@ -4164,8 +4159,8 @@ int S3fsCurl::UploadMultipartPostRequest(const char* tpath, int part_num, const
} }
// closing // closing
bodydata.Clear(); bodydata.clear();
headdata.Clear(); headdata.clear();
return result; return result;
} }
@ -4192,8 +4187,8 @@ int S3fsCurl::CopyMultipartPostSetup(const char* from, const char* to, int part_
path = get_realpath(to); path = get_realpath(to);
requestHeaders = nullptr; requestHeaders = nullptr;
responseHeaders.clear(); responseHeaders.clear();
bodydata.Clear(); bodydata.clear();
headdata.Clear(); headdata.clear();
std::string contype = S3fsCurl::LookupMimeType(to); std::string contype = S3fsCurl::LookupMimeType(to);
requestHeaders = curl_slist_sort_insert(requestHeaders, "Content-Type", contype.c_str()); requestHeaders = curl_slist_sort_insert(requestHeaders, "Content-Type", contype.c_str());
@ -4278,15 +4273,15 @@ bool S3fsCurl::CopyMultipartPostCallback(S3fsCurl* s3fscurl, void* param)
bool S3fsCurl::CopyMultipartPostComplete() bool S3fsCurl::CopyMultipartPostComplete()
{ {
std::string etag; std::string etag;
partdata.uploaded = simple_parse_xml(bodydata.str(), bodydata.size(), "ETag", etag); partdata.uploaded = simple_parse_xml(bodydata.c_str(), bodydata.size(), "ETag", etag);
if(etag.size() >= 2 && *etag.begin() == '"' && *etag.rbegin() == '"'){ if(etag.size() >= 2 && *etag.begin() == '"' && *etag.rbegin() == '"'){
etag.erase(etag.size() - 1); etag.erase(etag.size() - 1);
etag.erase(0, 1); etag.erase(0, 1);
} }
partdata.petag->etag = etag; partdata.petag->etag = etag;
bodydata.Clear(); bodydata.clear();
headdata.Clear(); headdata.clear();
return true; return true;
} }

9
src/curl.h
View File

@ -27,7 +27,6 @@
#include <vector> #include <vector>
#include "autolock.h" #include "autolock.h"
#include "bodydata.h"
#include "metaheader.h" #include "metaheader.h"
#include "fdcache_page.h" #include "fdcache_page.h"
@ -167,8 +166,8 @@ class S3fsCurl
std::string url; // target object path(url) std::string url; // target object path(url)
struct curl_slist* requestHeaders; struct curl_slist* requestHeaders;
headers_t responseHeaders; // header data by HeaderCallback headers_t responseHeaders; // header data by HeaderCallback
BodyData bodydata; // body data by WriteMemoryCallback std::string bodydata; // body data by WriteMemoryCallback
BodyData headdata; // header data by WriteMemoryCallback std::string headdata; // header data by WriteMemoryCallback
long LastResponseCode; long LastResponseCode;
const unsigned char* postdata; // use by post method and read callback function. const unsigned char* postdata; // use by post method and read callback function.
off_t postdata_remaining; // use by post method and read callback function. off_t postdata_remaining; // use by post method and read callback function.
@ -377,8 +376,8 @@ class S3fsCurl
std::string GetUrl() const { return url; } std::string GetUrl() const { return url; }
std::string GetOp() const { return op; } std::string GetOp() const { return op; }
const headers_t* GetResponseHeaders() const { return &responseHeaders; } const headers_t* GetResponseHeaders() const { return &responseHeaders; }
const BodyData* GetBodyData() const { return &bodydata; } const std::string* GetBodyData() const { return &bodydata; }
const BodyData* GetHeadData() const { return &headdata; } const std::string* GetHeadData() const { return &headdata; }
CURLcode GetCurlCode() const { return curlCode; } CURLcode GetCurlCode() const { return curlCode; }
long GetLastResponseCode() const { return LastResponseCode; } long GetLastResponseCode() const { return LastResponseCode; }
bool SetUseAhbe(bool ahbe); bool SetUseAhbe(bool ahbe);

10
src/s3fs.cpp
View File

@ -3489,14 +3489,14 @@ static int list_bucket(const char* path, S3ObjList& head, const char* delimiter,
S3FS_PRN_ERR("ListBucketRequest returns with error."); S3FS_PRN_ERR("ListBucketRequest returns with error.");
return result; return result;
} }
const BodyData* body = s3fscurl.GetBodyData(); const std::string* body = s3fscurl.GetBodyData();
// [NOTE] // [NOTE]
// CR code(\r) is replaced with LF(\n) by xmlReadMemory() function. // CR code(\r) is replaced with LF(\n) by xmlReadMemory() function.
// To prevent that, only CR code is encoded by following function. // To prevent that, only CR code is encoded by following function.
// The encoded CR code is decoded with append_objects_from_xml(_ex). // The encoded CR code is decoded with append_objects_from_xml(_ex).
// //
std::string encbody = get_encoded_cr_code(body->str()); std::string encbody = get_encoded_cr_code(body->c_str());
// xmlDocPtr // xmlDocPtr
if(nullptr == (doc = xmlReadMemory(encbody.c_str(), static_cast<int>(encbody.size()), "", nullptr, 0))){ if(nullptr == (doc = xmlReadMemory(encbody.c_str(), static_cast<int>(encbody.size()), "", nullptr, 0))){
@ -4410,10 +4410,10 @@ static int s3fs_check_service()
if(300 <= responseCode && responseCode < 500){ if(300 <= responseCode && responseCode < 500){
// check region error(for putting message or retrying) // check region error(for putting message or retrying)
const BodyData* body = s3fscurl.GetBodyData(); const std::string* body = s3fscurl.GetBodyData();
std::string expectregion; std::string expectregion;
std::string expectendpoint; std::string expectendpoint;
if(check_region_error(body->str(), body->size(), expectregion)){ if(check_region_error(body->c_str(), body->size(), expectregion)){
// [NOTE] // [NOTE]
// If endpoint is not specified(using us-east-1 region) and // If endpoint is not specified(using us-east-1 region) and
// an error is encountered accessing a different region, we // an error is encountered accessing a different region, we
@ -4455,7 +4455,7 @@ static int s3fs_check_service()
S3FS_PRN_CRIT("The bucket region is not '%s'(default), it is correctly '%s'. You should specify endpoint(%s) option.", endpoint.c_str(), expectregion.c_str(), expectregion.c_str()); S3FS_PRN_CRIT("The bucket region is not '%s'(default), it is correctly '%s'. You should specify endpoint(%s) option.", endpoint.c_str(), expectregion.c_str(), expectregion.c_str());
} }
}else if(check_endpoint_error(body->str(), body->size(), expectendpoint)){ }else if(check_endpoint_error(body->c_str(), body->size(), expectendpoint)){
// redirect error // redirect error
if(pathrequeststyle){ if(pathrequeststyle){
S3FS_PRN_CRIT("S3 service returned PermanentRedirect (current is url(%s) and endpoint(%s)). You need to specify correct url(http(s)://s3-<endpoint>.amazonaws.com) and endpoint option with use_path_request_style option.", s3host.c_str(), endpoint.c_str()); S3FS_PRN_CRIT("S3 service returned PermanentRedirect (current is url(%s) and endpoint(%s)). You need to specify correct url(http(s)://s3-<endpoint>.amazonaws.com) and endpoint option with use_path_request_style option.", s3host.c_str(), endpoint.c_str());