octoleo/s3fs-fuse
1
0
Fork 0
mirror of https://github.com/s3fs-fuse/s3fs-fuse.git synced 2024-12-22 08:48:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

Added new class for curl share handle

Browse Source 
Added new class for curl share handle.
And, paired the curl handle(S3fsCurl) with the worker thread.
Changed that each thread has its own SSL session cache to prevent data
races.
So OpenSSL suppression for ThreadSanitizer is no longer necessary, so
reverted it.
This commit is contained in:
Takeshi Nakatani 2024-07-15 06:40:05 +00:00 committed by Andrew Gaul
parent bfc3ea767a
commit 956e8c5750
13 changed files with 419 additions and 181 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.github/workflows/ci.yml vendored
View File

@ -246,7 +246,7 @@ jobs:
elif [ "${{ matrix.checktype }}" = "sanitize_thread" ]; then
echo 'CXX=clang++'
echo "CXXFLAGS=${COMMON_CXXFLAGS} -O0 -fsanitize=thread"
echo 'TSAN_OPTIONS=halt_on_error=1,suppressions=threadsanitizer_suppressions.txt'
echo 'TSAN_OPTIONS=halt_on_error=1'
# [NOTE]
# Set this to avoid following error when running configure.
# "FATAL: ThreadSanitizer: unexpected memory mapping"
@ -269,7 +269,7 @@ jobs:
- name: Build
run: |
./autogen.sh
/bin/sh -c "CXX=${CXX} CXXFLAGS=\"${CXXFLAGS}\" LDFLAGS=\"${LDFLAGS}\" TSAN_OPTIONS=\"\" ./configure --prefix=/usr --with-openssl"
/bin/sh -c "CXX=${CXX} CXXFLAGS=\"${CXXFLAGS}\" LDFLAGS=\"${LDFLAGS}\" ./configure --prefix=/usr --with-openssl"
make
- name: Test suite

1
src/Makefile.am
View File

@ -36,6 +36,7 @@ s3fs_SOURCES = \
metaheader.cpp \
mpu_util.cpp \
curl.cpp \
curl_share.cpp \
curl_util.cpp \
s3objlist.cpp \
cache.cpp \

120
src/curl.cpp
View File

@ -35,6 +35,7 @@
#include "s3fs.h"
#include "s3fs_logger.h"
#include "curl.h"
#include "curl_share.h"
#include "curl_util.h"
#include "s3fs_auth.h"
#include "s3fs_cred.h"
@ -98,10 +99,7 @@ constexpr char S3fsCurl::S3FS_SSL_PRIVKEY_PASSWORD[];
std::mutex S3fsCurl::curl_handles_lock;
S3fsCurl::callback_locks_t S3fsCurl::callback_locks;
bool S3fsCurl::is_initglobal_done = false;
CURLSH* S3fsCurl::hCurlShare = nullptr;
bool S3fsCurl::is_cert_check = true; // default
bool S3fsCurl::is_dns_cache = true; // default
bool S3fsCurl::is_ssl_session_cache= false;// default(This turns OFF now, but turns ON again last PR)
long S3fsCurl::connect_timeout = 300; // default
time_t S3fsCurl::readwrite_timeout = 120; // default
int S3fsCurl::retries = 5; // default
@ -153,9 +151,6 @@ bool S3fsCurl::InitS3fsCurl()
if(!S3fsCurl::InitGlobalCurl()){
return false;
}
if(!S3fsCurl::InitShareCurl()){
return false;
}
if(!S3fsCurl::InitCryptMutex()){
return false;
}
@ -169,9 +164,6 @@ bool S3fsCurl::DestroyS3fsCurl()
if(!S3fsCurl::DestroyCryptMutex()){
result = false;
}
if(!S3fsCurl::DestroyShareCurl()){
result = false;
}
if(!S3fsCurl::DestroyGlobalCurl()){
result = false;
}
@ -201,97 +193,6 @@ bool S3fsCurl::DestroyGlobalCurl()
return true;
}
bool S3fsCurl::InitShareCurl()
{
CURLSHcode nSHCode;
if(!S3fsCurl::is_dns_cache && !S3fsCurl::is_ssl_session_cache){
S3FS_PRN_INFO("Curl does not share DNS data.");
return true;
}
if(S3fsCurl::hCurlShare){
S3FS_PRN_WARN("already initiated.");
return false;
}
if(nullptr == (S3fsCurl::hCurlShare = curl_share_init())){
S3FS_PRN_ERR("curl_share_init failed");
return false;
}
if(CURLSHE_OK != (nSHCode = curl_share_setopt(S3fsCurl::hCurlShare, CURLSHOPT_LOCKFUNC, S3fsCurl::LockCurlShare))){
S3FS_PRN_ERR("curl_share_setopt(LOCKFUNC) returns %d(%s)", nSHCode, curl_share_strerror(nSHCode));
return false;
}
if(CURLSHE_OK != (nSHCode = curl_share_setopt(S3fsCurl::hCurlShare, CURLSHOPT_UNLOCKFUNC, S3fsCurl::UnlockCurlShare))){
S3FS_PRN_ERR("curl_share_setopt(UNLOCKFUNC) returns %d(%s)", nSHCode, curl_share_strerror(nSHCode));
return false;
}
if(S3fsCurl::is_dns_cache){
nSHCode = curl_share_setopt(S3fsCurl::hCurlShare, CURLSHOPT_SHARE, CURL_LOCK_DATA_DNS);
if(CURLSHE_OK != nSHCode && CURLSHE_BAD_OPTION != nSHCode && CURLSHE_NOT_BUILT_IN != nSHCode){
S3FS_PRN_ERR("curl_share_setopt(DNS) returns %d(%s)", nSHCode, curl_share_strerror(nSHCode));
return false;
}else if(CURLSHE_BAD_OPTION == nSHCode || CURLSHE_NOT_BUILT_IN == nSHCode){
S3FS_PRN_WARN("curl_share_setopt(DNS) returns %d(%s), but continue without shared dns data.", nSHCode, curl_share_strerror(nSHCode));
}
}
if(S3fsCurl::is_ssl_session_cache){
nSHCode = curl_share_setopt(S3fsCurl::hCurlShare, CURLSHOPT_SHARE, CURL_LOCK_DATA_SSL_SESSION);
if(CURLSHE_OK != nSHCode && CURLSHE_BAD_OPTION != nSHCode && CURLSHE_NOT_BUILT_IN != nSHCode){
S3FS_PRN_ERR("curl_share_setopt(SSL SESSION) returns %d(%s)", nSHCode, curl_share_strerror(nSHCode));
return false;
}else if(CURLSHE_BAD_OPTION == nSHCode || CURLSHE_NOT_BUILT_IN == nSHCode){
S3FS_PRN_WARN("curl_share_setopt(SSL SESSION) returns %d(%s), but continue without shared ssl session data.", nSHCode, curl_share_strerror(nSHCode));
}
}
if(CURLSHE_OK != (nSHCode = curl_share_setopt(S3fsCurl::hCurlShare, CURLSHOPT_USERDATA, &S3fsCurl::callback_locks))){
S3FS_PRN_ERR("curl_share_setopt(USERDATA) returns %d(%s)", nSHCode, curl_share_strerror(nSHCode));
return false;
}
return true;
}
bool S3fsCurl::DestroyShareCurl()
{
if(!S3fsCurl::hCurlShare){
if(!S3fsCurl::is_dns_cache && !S3fsCurl::is_ssl_session_cache){
return true;
}
S3FS_PRN_WARN("already destroy share curl.");
return false;
}
if(CURLSHE_OK != curl_share_cleanup(S3fsCurl::hCurlShare)){
return false;
}
S3fsCurl::hCurlShare = nullptr;
return true;
}
void S3fsCurl::LockCurlShare(CURL* handle, curl_lock_data nLockData, curl_lock_access laccess, void* useptr)
{
if(!hCurlShare){
return;
}
auto* locks = static_cast<S3fsCurl::callback_locks_t*>(useptr);
if(CURL_LOCK_DATA_DNS == nLockData){
locks->dns.lock();
}else if(CURL_LOCK_DATA_SSL_SESSION == nLockData){
locks->ssl_session.lock();
}
}
void S3fsCurl::UnlockCurlShare(CURL* handle, curl_lock_data nLockData, void* useptr)
{
if(!hCurlShare){
return;
}
auto* locks = static_cast<S3fsCurl::callback_locks_t*>(useptr);
if(CURL_LOCK_DATA_DNS == nLockData){
locks->dns.unlock();
}else if(CURL_LOCK_DATA_SSL_SESSION == nLockData){
locks->ssl_session.unlock();
}
}
bool S3fsCurl::InitCryptMutex()
{
return s3fs_init_crypt_mutex();
@ -667,26 +568,12 @@ bool S3fsCurl::SetCheckCertificate(bool isCertCheck)
return old;
}
bool S3fsCurl::SetDnsCache(bool isCache)
{
bool old = S3fsCurl::is_dns_cache;
S3fsCurl::is_dns_cache = isCache;
return old;
}
void S3fsCurl::ResetOffset(S3fsCurl* pCurl)
{
pCurl->partdata.startpos = pCurl->b_partdata_startpos;
pCurl->partdata.size = pCurl->b_partdata_size;
}
bool S3fsCurl::SetSslSessionCache(bool isCache)
{
bool old = S3fsCurl::is_ssl_session_cache;
S3fsCurl::is_ssl_session_cache = isCache;
return old;
}
long S3fsCurl::SetConnectTimeout(long timeout)
{
long old = S3fsCurl::connect_timeout;
@ -1583,11 +1470,10 @@ bool S3fsCurl::ResetHandle()
}
}
if((S3fsCurl::is_dns_cache || S3fsCurl::is_ssl_session_cache) && S3fsCurl::hCurlShare){
if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_SHARE, S3fsCurl::hCurlShare)){
if(!S3fsCurlShare::SetCurlShareHandle(hCurl.get())){
return false;
}
}
if(!S3fsCurl::is_cert_check) {
S3FS_PRN_DBG("'no_check_certificate' option in effect.");
S3FS_PRN_DBG("The server certificate won't be checked against the available certificate authorities.");

9
src/curl.h
View File

@ -127,10 +127,7 @@ class S3fsCurl
std::mutex ssl_session;
} callback_locks;
static bool is_initglobal_done;
static CURLSH* hCurlShare;
static bool is_cert_check;
static bool is_dns_cache;
static bool is_ssl_session_cache;
static long connect_timeout;
static time_t readwrite_timeout;
static int retries;
@ -211,10 +208,6 @@ class S3fsCurl
// class methods
static bool InitGlobalCurl();
static bool DestroyGlobalCurl();
static bool InitShareCurl();
static bool DestroyShareCurl();
static void LockCurlShare(CURL* handle, curl_lock_data nLockData, curl_lock_access laccess, void* useptr) NO_THREAD_SAFETY_ANALYSIS;
static void UnlockCurlShare(CURL* handle, curl_lock_data nLockData, void* useptr) NO_THREAD_SAFETY_ANALYSIS;
static bool InitCryptMutex();
static bool DestroyCryptMutex();
static int CurlProgress(void *clientp, double dltotal, double dlnow, double ultotal, double ulnow);
@ -274,8 +267,6 @@ class S3fsCurl
// class methods(variables)
static std::string LookupMimeType(const std::string& name);
static bool SetCheckCertificate(bool isCertCheck);
static bool SetDnsCache(bool isCache);
static bool SetSslSessionCache(bool isCache);
static long SetConnectTimeout(long timeout);
static time_t SetReadwriteTimeout(time_t timeout);
static time_t GetReadwriteTimeout() { return S3fsCurl::readwrite_timeout; }

239
src/curl_share.cpp Normal file
View File

@ -0,0 +1,239 @@
/*
* s3fs - FUSE-based file system backed by Amazon S3
*
* Copyright(C) 2007 Randy Rizun <rrizun@gmail.com>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
#include "common.h"
#include "s3fs.h"
#include "s3fs_logger.h"
#include "curl_share.h"
//-------------------------------------------------------------------
// Class S3fsCurlShare
//-------------------------------------------------------------------
bool S3fsCurlShare::is_dns_cache = true; // default
bool S3fsCurlShare::is_ssl_cache = true; // default
std::mutex S3fsCurlShare::curl_share_lock;
std::map<std::thread::id, CurlSharePtr> S3fsCurlShare::ShareHandles;
std::map<std::thread::id, ShareLocksPtr> S3fsCurlShare::ShareLocks;
//-------------------------------------------------------------------
// Class methods for S3fsCurlShare
//-------------------------------------------------------------------
bool S3fsCurlShare::SetDnsCache(bool isCache)
{
bool old = S3fsCurlShare::is_dns_cache;
S3fsCurlShare::is_dns_cache = isCache;
return old;
}
bool S3fsCurlShare::SetSslSessionCache(bool isCache)
{
bool old = S3fsCurlShare::is_ssl_cache;
S3fsCurlShare::is_ssl_cache = isCache;
return old;
}
void S3fsCurlShare::LockCurlShare(CURL* handle, curl_lock_data nLockData, curl_lock_access laccess, void* useptr)
{
auto* pLocks = static_cast<curl_share_locks*>(useptr);
if(CURL_LOCK_DATA_DNS == nLockData){
pLocks->lock_dns.lock();
}else if(CURL_LOCK_DATA_SSL_SESSION == nLockData){
pLocks->lock_session.lock();
}
}
void S3fsCurlShare::UnlockCurlShare(CURL* handle, curl_lock_data nLockData, void* useptr)
{
auto* pLocks = static_cast<curl_share_locks*>(useptr);
if(CURL_LOCK_DATA_DNS == nLockData){
pLocks->lock_dns.unlock();
}else if(CURL_LOCK_DATA_SSL_SESSION == nLockData){
pLocks->lock_session.unlock();
}
}
bool S3fsCurlShare::SetCurlShareHandle(CURL* hCurl)
{
if(!hCurl){
S3FS_PRN_ERR("Curl handle is null");
return false;
}
// get curl share handle
S3fsCurlShare CurlShareObj;
CURLSH* hCurlShare = CurlShareObj.GetCurlShareHandle();
if(!hCurlShare){
// a case of not to use CurlShare
return true;
}
// set share handle to curl handle
if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_SHARE, hCurlShare)){
S3FS_PRN_ERR("Failed to set Curl share handle to curl handle.");
return false;
}
return true;
}
bool S3fsCurlShare::DestroyCurlShareHandleForThread()
{
S3fsCurlShare CurlShareObj;
return CurlShareObj.DestroyCurlShareHandle();
}
bool S3fsCurlShare::InitializeCurlShare(const CurlSharePtr& hShare, const ShareLocksPtr& ShareLock)
{
CURLSHcode nSHCode;
// set lock handlers
if(CURLSHE_OK != (nSHCode = curl_share_setopt(hShare.get(), CURLSHOPT_LOCKFUNC, S3fsCurlShare::LockCurlShare))){
S3FS_PRN_ERR("curl_share_setopt(LOCKFUNC) returns %d(%s)", nSHCode, curl_share_strerror(nSHCode));
return false;
}
if(CURLSHE_OK != (nSHCode = curl_share_setopt(hShare.get(), CURLSHOPT_UNLOCKFUNC, S3fsCurlShare::UnlockCurlShare))){
S3FS_PRN_ERR("curl_share_setopt(UNLOCKFUNC) returns %d(%s)", nSHCode, curl_share_strerror(nSHCode));
return false;
}
// set user data for lock functions
if(CURLSHE_OK != (nSHCode = curl_share_setopt(hShare.get(), CURLSHOPT_USERDATA, ShareLock.get()))){
S3FS_PRN_ERR("curl_share_setopt(USERDATA) returns %d(%s)", nSHCode, curl_share_strerror(nSHCode));
return false;
}
// set share type
if(S3fsCurlShare::is_dns_cache){
nSHCode = curl_share_setopt(hShare.get(), CURLSHOPT_SHARE, CURL_LOCK_DATA_DNS);
if(CURLSHE_OK != nSHCode && CURLSHE_BAD_OPTION != nSHCode && CURLSHE_NOT_BUILT_IN != nSHCode){
S3FS_PRN_ERR("curl_share_setopt(DNS) returns %d(%s)", nSHCode, curl_share_strerror(nSHCode));
return false;
}else if(CURLSHE_BAD_OPTION == nSHCode || CURLSHE_NOT_BUILT_IN == nSHCode){
S3FS_PRN_WARN("curl_share_setopt(DNS) returns %d(%s), but continue without shared dns data.", nSHCode, curl_share_strerror(nSHCode));
}
}
if(S3fsCurlShare::is_ssl_cache){
nSHCode = curl_share_setopt(hShare.get(), CURLSHOPT_SHARE, CURL_LOCK_DATA_SSL_SESSION);
if(CURLSHE_OK != nSHCode && CURLSHE_BAD_OPTION != nSHCode && CURLSHE_NOT_BUILT_IN != nSHCode){
S3FS_PRN_ERR("curl_share_setopt(SSL SESSION) returns %d(%s)", nSHCode, curl_share_strerror(nSHCode));
return false;
}else if(CURLSHE_BAD_OPTION == nSHCode || CURLSHE_NOT_BUILT_IN == nSHCode){
S3FS_PRN_WARN("curl_share_setopt(SSL SESSION) returns %d(%s), but continue without shared ssl session data.", nSHCode, curl_share_strerror(nSHCode));
}
}
return true;
}
//-------------------------------------------------------------------
// Methods for S3fsCurlShare
//-------------------------------------------------------------------
// [NOTE]
// set current thread id(std style) to ThreadId
//
S3fsCurlShare::S3fsCurlShare() : ThreadId(std::this_thread::get_id())
{
}
bool S3fsCurlShare::DestroyCurlShareHandle()
{
if(!S3fsCurlShare::is_dns_cache && !S3fsCurlShare::is_ssl_cache){
// Any curl share handle does not exist
return true;
}
const std::lock_guard<std::mutex> lock(S3fsCurlShare::curl_share_lock);
// find existed handle and cleanup it
auto handle_iter = S3fsCurlShare::ShareHandles.find(ThreadId);
if(handle_iter == S3fsCurlShare::ShareHandles.end()){
S3FS_PRN_WARN("Not found curl share handle");
}else{
if(CURLSHE_OK != curl_share_cleanup(handle_iter->second.get())){
S3FS_PRN_ERR("Failed to cleanup curl share handle");
return false;
}
S3fsCurlShare::ShareHandles.erase(handle_iter);
}
// find lock and cleanup it
auto locks_iter = S3fsCurlShare::ShareLocks.find(ThreadId);
if(locks_iter == S3fsCurlShare::ShareLocks.end()){
S3FS_PRN_WARN("Not found locks of curl share handle");
}else{
S3fsCurlShare::ShareLocks.erase(locks_iter);
}
return true;
}
CURLSH* S3fsCurlShare::GetCurlShareHandle()
{
if(!S3fsCurlShare::is_dns_cache && !S3fsCurlShare::is_ssl_cache){
// Any curl share handle does not exist
return nullptr;
}
const std::lock_guard<std::mutex> lock(S3fsCurlShare::curl_share_lock);
// find existed handle
auto handle_iter = S3fsCurlShare::ShareHandles.find(ThreadId);
if(handle_iter != S3fsCurlShare::ShareHandles.end()){
// Already created share handle for this thread.
return handle_iter->second.get();
}
// create new curl share handle and locks
CurlSharePtr hShare = {nullptr, curl_share_cleanup};
hShare.reset(curl_share_init());
if(!hShare){
S3FS_PRN_ERR("Failed to create curl share handle");
return nullptr;
}
ShareLocksPtr pLocks(new curl_share_locks);
// Initialize curl share handle
if(!S3fsCurlShare::InitializeCurlShare(hShare, pLocks)){
S3FS_PRN_ERR("Failed to initialize curl share handle");
return nullptr;
}
// set map
S3fsCurlShare::ShareHandles.emplace(ThreadId, std::move(hShare));
S3fsCurlShare::ShareLocks.emplace(ThreadId, std::move(pLocks));
// For clang-tidy measures
handle_iter = S3fsCurlShare::ShareHandles.find(ThreadId);
if(handle_iter == S3fsCurlShare::ShareHandles.end()){
S3FS_PRN_ERR("Failed to insert curl share to map.");
return nullptr;
}
return handle_iter->second.get();
}
/*
* Local variables:
* tab-width: 4
* c-basic-offset: 4
* End:
* vim600: expandtab sw=4 ts=4 fdm=marker
* vim<600: expandtab sw=4 ts=4
*/

89
src/curl_share.h Normal file
View File

@ -0,0 +1,89 @@
/*
* s3fs - FUSE-based file system backed by Amazon S3
*
* Copyright(C) 2007 Randy Rizun <rrizun@gmail.com>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
#ifndef S3FS_CURL_SHARE_H_
#define S3FS_CURL_SHARE_H_
#include <curl/curl.h>
#include <map>
#include <memory>
#include <mutex>
#include <thread>
#include "common.h"
//----------------------------------------------
// Structure / Typedefs
//----------------------------------------------
struct curl_share_locks {
std::mutex lock_dns;
std::mutex lock_session;
};
typedef std::unique_ptr<CURLSH, decltype(&curl_share_cleanup)> CurlSharePtr;
typedef std::unique_ptr<curl_share_locks> ShareLocksPtr;
//----------------------------------------------
// class S3fsCurlShare
//----------------------------------------------
class S3fsCurlShare
{
private:
static bool is_dns_cache;
static bool is_ssl_cache;
static std::mutex curl_share_lock;
static std::map<std::thread::id, CurlSharePtr> ShareHandles GUARDED_BY(curl_share_lock);
static std::map<std::thread::id, ShareLocksPtr> ShareLocks GUARDED_BY(curl_share_lock);
std::thread::id ThreadId;
private:
static void LockCurlShare(CURL* handle, curl_lock_data nLockData, curl_lock_access laccess, void* useptr) NO_THREAD_SAFETY_ANALYSIS;
static void UnlockCurlShare(CURL* handle, curl_lock_data nLockData, void* useptr) NO_THREAD_SAFETY_ANALYSIS;
static bool InitializeCurlShare(const CurlSharePtr& hShare, const ShareLocksPtr& ShareLock) REQUIRES(curl_share_lock);
bool DestroyCurlShareHandle();
CURLSH* GetCurlShareHandle();
public:
static bool SetDnsCache(bool isCache);
static bool SetSslSessionCache(bool isCache);
static bool SetCurlShareHandle(CURL* hCurl);
static bool DestroyCurlShareHandleForThread();
// constructor/destructor
explicit S3fsCurlShare();
~S3fsCurlShare() = default;
S3fsCurlShare(const S3fsCurlShare&) = delete;
S3fsCurlShare(S3fsCurlShare&&) = delete;
S3fsCurlShare& operator=(const S3fsCurlShare&) = delete;
S3fsCurlShare& operator=(S3fsCurlShare&&) = delete;
};
#endif // S3FS_CURL_SHARE_H_
/*
* Local variables:
* tab-width: 4
* c-basic-offset: 4
* End:
* vim600: expandtab sw=4 ts=4 fdm=marker
* vim<600: expandtab sw=4 ts=4
*/

5
src/s3fs.cpp
View File

@ -43,6 +43,7 @@
#include "fdcache_auto.h"
#include "fdcache_stat.h"
#include "curl.h"
#include "curl_share.h"
#include "curl_util.h"
#include "s3objlist.h"
#include "cache.h"
@ -4933,11 +4934,11 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
return 0;
}
else if(0 == strcmp(arg, "nodnscache")){
S3fsCurl::SetDnsCache(false);
S3fsCurlShare::SetDnsCache(false);
return 0;
}
else if(0 == strcmp(arg, "nosscache")){
S3fsCurl::SetSslSessionCache(false);
S3fsCurlShare::SetSslSessionCache(false);
return 0;
}
else if(is_prefix(arg, "parallel_count=") || is_prefix(arg, "parallel_upload=")){

70
src/s3fs_threadreqs.cpp
View File

@ -36,7 +36,7 @@
//
// Thread Worker function for head request
//
void* head_req_threadworker(void* arg)
void* head_req_threadworker(S3fsCurl& s3fscurl, void* arg)
{
auto* pthparam = static_cast<head_req_thparam*>(arg);
if(!pthparam || !pthparam->pmeta){
@ -44,7 +44,8 @@ void* head_req_threadworker(void* arg)
}
S3FS_PRN_INFO3("Head Request [path=%s][pmeta=%p]", pthparam->path.c_str(), pthparam->pmeta);
S3fsCurl s3fscurl;
s3fscurl.SetUseAhbe(false);
pthparam->result = s3fscurl.HeadRequest(pthparam->path.c_str(), *(pthparam->pmeta));
return reinterpret_cast<void*>(pthparam->result);
@ -53,7 +54,7 @@ void* head_req_threadworker(void* arg)
//
// Thread Worker function for multi head request
//
void* multi_head_req_threadworker(void* arg)
void* multi_head_req_threadworker(S3fsCurl& s3fscurl, void* arg)
{
std::unique_ptr<multi_head_req_thparam> pthparam(static_cast<multi_head_req_thparam*>(arg));
if(!pthparam || !pthparam->psyncfiller || !pthparam->pthparam_lock || !pthparam->pretrycount || !pthparam->pnotfound_list || !pthparam->presult){
@ -72,8 +73,9 @@ void* multi_head_req_threadworker(void* arg)
}
}
s3fscurl.SetUseAhbe(false);
// loop for head request
S3fsCurl s3fscurl;
int result = 0;
headers_t meta; // this value is not used
while(true){
@ -197,7 +199,7 @@ void* multi_head_req_threadworker(void* arg)
//
// Thread Worker function for delete request
//
void* delete_req_threadworker(void* arg)
void* delete_req_threadworker(S3fsCurl& s3fscurl, void* arg)
{
auto* pthparam = static_cast<delete_req_thparam*>(arg);
if(!pthparam){
@ -205,7 +207,8 @@ void* delete_req_threadworker(void* arg)
}
S3FS_PRN_INFO3("Delete Request [path=%s]", pthparam->path.c_str());
S3fsCurl s3fscurl;
s3fscurl.SetUseAhbe(false);
pthparam->result = s3fscurl.DeleteRequest(pthparam->path.c_str());
return reinterpret_cast<void*>(pthparam->result);
@ -214,7 +217,7 @@ void* delete_req_threadworker(void* arg)
//
// Thread Worker function for put head request
//
void* put_head_req_threadworker(void* arg)
void* put_head_req_threadworker(S3fsCurl& s3fscurl, void* arg)
{
auto* pthparam = static_cast<put_head_req_thparam*>(arg);
if(!pthparam){
@ -222,7 +225,8 @@ void* put_head_req_threadworker(void* arg)
}
S3FS_PRN_INFO3("Put Head Request [path=%s][meta count=%lu][is copy=%s]", pthparam->path.c_str(), pthparam->meta.size(), (pthparam->isCopy ? "true" : "false"));
S3fsCurl s3fscurl(true);
s3fscurl.SetUseAhbe(true);
pthparam->result = s3fscurl.PutHeadRequest(pthparam->path.c_str(), pthparam->meta, pthparam->isCopy);
return reinterpret_cast<void*>(pthparam->result);
@ -231,7 +235,7 @@ void* put_head_req_threadworker(void* arg)
//
// Thread Worker function for put request
//
void* put_req_threadworker(void* arg)
void* put_req_threadworker(S3fsCurl& s3fscurl, void* arg)
{
auto* pthparam = static_cast<put_req_thparam*>(arg);
if(!pthparam){
@ -239,7 +243,8 @@ void* put_req_threadworker(void* arg)
}
S3FS_PRN_INFO3("Put Request [path=%s][meta count=%lu][fd=%d][use_ahbe=%s]", pthparam->path.c_str(), pthparam->meta.size(), pthparam->fd, (pthparam->ahbe ? "true" : "false"));
S3fsCurl s3fscurl(pthparam->ahbe);
s3fscurl.SetUseAhbe(pthparam->ahbe);
pthparam->result = s3fscurl.PutRequest(pthparam->path.c_str(), pthparam->meta, pthparam->fd);
return reinterpret_cast<void*>(pthparam->result);
@ -248,7 +253,7 @@ void* put_req_threadworker(void* arg)
//
// Thread Worker function for list bucket request
//
void* list_bucket_req_threadworker(void* arg)
void* list_bucket_req_threadworker(S3fsCurl& s3fscurl, void* arg)
{
auto* pthparam = static_cast<list_bucket_req_thparam*>(arg);
if(!pthparam || !(pthparam->presponseBody)){
@ -256,7 +261,8 @@ void* list_bucket_req_threadworker(void* arg)
}
S3FS_PRN_INFO3("List Bucket Request [path=%s][query=%s]", pthparam->path.c_str(), pthparam->query.c_str());
S3fsCurl s3fscurl;
s3fscurl.SetUseAhbe(false);
if(0 == (pthparam->result = s3fscurl.ListBucketRequest(pthparam->path.c_str(), pthparam->query.c_str()))){
*(pthparam->presponseBody) = s3fscurl.GetBodyData();
}
@ -266,7 +272,7 @@ void* list_bucket_req_threadworker(void* arg)
//
// Thread Worker function for check service request
//
void* check_service_req_threadworker(void* arg)
void* check_service_req_threadworker(S3fsCurl& s3fscurl, void* arg)
{
auto* pthparam = static_cast<check_service_req_thparam*>(arg);
if(!pthparam || !(pthparam->presponseCode) || !(pthparam->presponseBody)){
@ -274,7 +280,8 @@ void* check_service_req_threadworker(void* arg)
}
S3FS_PRN_INFO3("Check Service Request [path=%s][support compat dir=%s][force No SSE=%s]", pthparam->path.c_str(), (pthparam->support_compat_dir ? "true" : "false"), (pthparam->forceNoSSE ? "true" : "false"));
S3fsCurl s3fscurl;
s3fscurl.SetUseAhbe(false);
if(0 == (pthparam->result = s3fscurl.CheckBucket(pthparam->path.c_str(), pthparam->support_compat_dir, pthparam->forceNoSSE))){
*(pthparam->presponseCode) = s3fscurl.GetLastResponseCode();
*(pthparam->presponseBody) = s3fscurl.GetBodyData();
@ -285,7 +292,7 @@ void* check_service_req_threadworker(void* arg)
//
// Worker function for pre multipart upload request
//
void* pre_multipart_upload_req_threadworker(void* arg)
void* pre_multipart_upload_req_threadworker(S3fsCurl& s3fscurl, void* arg)
{
auto* pthparam = static_cast<pre_multipart_upload_req_thparam*>(arg);
if(!pthparam){
@ -293,7 +300,8 @@ void* pre_multipart_upload_req_threadworker(void* arg)
}
S3FS_PRN_INFO3("Pre Multipart Upload Request [path=%s][meta count=%lu]", pthparam->path.c_str(), pthparam->meta.size());
S3fsCurl s3fscurl(true);
s3fscurl.SetUseAhbe(true);
pthparam->result = s3fscurl.PreMultipartUploadRequest(pthparam->path.c_str(), pthparam->meta, pthparam->upload_id);
return reinterpret_cast<void*>(pthparam->result);
@ -302,7 +310,7 @@ void* pre_multipart_upload_req_threadworker(void* arg)
//
// Worker function for pre multipart upload part request
//
void* multipart_upload_part_req_threadworker(void* arg)
void* multipart_upload_part_req_threadworker(S3fsCurl& s3fscurl, void* arg)
{
auto* pthparam = static_cast<multipart_upload_part_req_thparam*>(arg);
if(!pthparam || !pthparam->pthparam_lock || !pthparam->petag || !pthparam->presult){
@ -321,10 +329,11 @@ void* multipart_upload_part_req_threadworker(void* arg)
}
}
s3fscurl.SetUseAhbe(true);
//
// Request
//
S3fsCurl s3fscurl(true);
int result;
if(0 != (result = s3fscurl.MultipartUploadPartRequest(pthparam->path.c_str(), pthparam->upload_fd, pthparam->start, pthparam->size, pthparam->part_num, pthparam->upload_id, pthparam->petag, pthparam->is_copy))){
S3FS_PRN_ERR("Failed Multipart Upload Part Worker with error(%d) [path=%s][upload_id=%s][upload_fd=%d][start=%lld][size=%lld][is_copy=%s][part_num=%d]", result, pthparam->path.c_str(), pthparam->upload_id.c_str(), pthparam->upload_fd, static_cast<long long int>(pthparam->start), static_cast<long long int>(pthparam->size), (pthparam->is_copy ? "true" : "false"), pthparam->part_num);
@ -342,7 +351,7 @@ void* multipart_upload_part_req_threadworker(void* arg)
//
// Worker function for complete multipart upload request
//
void* complete_multipart_upload_threadworker(void* arg)
void* complete_multipart_upload_threadworker(S3fsCurl& s3fscurl, void* arg)
{
auto* pthparam = static_cast<complete_multipart_upload_req_thparam*>(arg);
if(!pthparam){
@ -350,7 +359,8 @@ void* complete_multipart_upload_threadworker(void* arg)
}
S3FS_PRN_INFO3("Complete Multipart Upload Request [path=%s][upload id=%s][etaglist=%lu]", pthparam->path.c_str(), pthparam->upload_id.c_str(), pthparam->etaglist.size());
S3fsCurl s3fscurl(true);
s3fscurl.SetUseAhbe(true);
pthparam->result = s3fscurl.MultipartUploadComplete(pthparam->path.c_str(), pthparam->upload_id, pthparam->etaglist);
return reinterpret_cast<void*>(pthparam->result);
@ -359,7 +369,7 @@ void* complete_multipart_upload_threadworker(void* arg)
//
// Worker function for abort multipart upload request
//
void* abort_multipart_upload_req_threadworker(void* arg)
void* abort_multipart_upload_req_threadworker(S3fsCurl& s3fscurl, void* arg)
{
auto* pthparam = static_cast<abort_multipart_upload_req_thparam*>(arg);
if(!pthparam){
@ -367,7 +377,8 @@ void* abort_multipart_upload_req_threadworker(void* arg)
}
S3FS_PRN_INFO3("Abort Multipart Upload Request [path=%s][upload id=%s]", pthparam->path.c_str(), pthparam->upload_id.c_str());
S3fsCurl s3fscurl(true);
s3fscurl.SetUseAhbe(true);
pthparam->result = s3fscurl.AbortMultipartUpload(pthparam->path.c_str(), pthparam->upload_id);
return reinterpret_cast<void*>(pthparam->result);
@ -376,7 +387,7 @@ void* abort_multipart_upload_req_threadworker(void* arg)
//
// Thread Worker function for get object request
//
void* get_object_req_threadworker(void* arg)
void* get_object_req_threadworker(S3fsCurl& s3fscurl, void* arg)
{
auto* pthparam = static_cast<get_object_req_thparam*>(arg);
if(!pthparam){
@ -390,7 +401,8 @@ void* get_object_req_threadworker(void* arg)
S3FS_PRN_WARN("Failed to get SSE type for file(%s).", pthparam->path.c_str());
}
S3fsCurl s3fscurl;
s3fscurl.SetUseAhbe(false);
pthparam->result = s3fscurl.GetObjectRequest(pthparam->path.c_str(), pthparam->fd, pthparam->start, pthparam->size, ssetype, ssevalue);
return reinterpret_cast<void*>(pthparam->result);
@ -399,7 +411,7 @@ void* get_object_req_threadworker(void* arg)
//
// Thread Worker function for multipart put head request
//
void* multipart_put_head_req_threadworker(void* arg)
void* multipart_put_head_req_threadworker(S3fsCurl& s3fscurl, void* arg)
{
auto* pthparam = static_cast<multipart_put_head_req_thparam*>(arg);
if(!pthparam || !pthparam->ppartdata || !pthparam->pthparam_lock || !pthparam->pretrycount || !pthparam->presult){
@ -418,7 +430,8 @@ void* multipart_put_head_req_threadworker(void* arg)
}
}
S3fsCurl s3fscurl(true);
s3fscurl.SetUseAhbe(true);
int result = 0;
while(true){
// Request
@ -516,7 +529,7 @@ void* multipart_put_head_req_threadworker(void* arg)
//
// Thread Worker function for parallel get object request
//
void* parallel_get_object_req_threadworker(void* arg)
void* parallel_get_object_req_threadworker(S3fsCurl& s3fscurl, void* arg)
{
auto* pthparam = static_cast<parallel_get_object_req_thparam*>(arg);
if(!pthparam || !pthparam->pthparam_lock || !pthparam->pretrycount || !pthparam->presult){
@ -535,7 +548,8 @@ void* parallel_get_object_req_threadworker(void* arg)
}
}
S3fsCurl s3fscurl(true);
s3fscurl.SetUseAhbe(true);
int result = 0;
while(true){
// Request

28
src/s3fs_threadreqs.h
View File

@ -209,20 +209,20 @@ struct get_object_req_thparam
//-------------------------------------------------------------------
// Thread Worker functions for MultiThread Request
//-------------------------------------------------------------------
void* head_req_threadworker(void* arg);
void* multi_head_req_threadworker(void* arg);
void* delete_req_threadworker(void* arg);
void* put_head_req_threadworker(void* arg);
void* put_req_threadworker(void* arg);
void* list_bucket_req_threadworker(void* arg);
void* check_service_req_threadworker(void* arg);
void* pre_multipart_upload_req_threadworker(void* arg);
void* multipart_upload_part_req_threadworker(void* arg);
void* complete_multipart_upload_threadworker(void* arg);
void* abort_multipart_upload_req_threadworker(void* arg);
void* multipart_put_head_req_threadworker(void* arg);
void* parallel_get_object_req_threadworker(void* arg);
void* get_object_req_threadworker(void* arg);
void* head_req_threadworker(S3fsCurl& s3fscurl, void* arg);
void* multi_head_req_threadworker(S3fsCurl& s3fscurl, void* arg);
void* delete_req_threadworker(S3fsCurl& s3fscurl, void* arg);
void* put_head_req_threadworker(S3fsCurl& s3fscurl, void* arg);
void* put_req_threadworker(S3fsCurl& s3fscurl, void* arg);
void* list_bucket_req_threadworker(S3fsCurl& s3fscurl, void* arg);
void* check_service_req_threadworker(S3fsCurl& s3fscurl, void* arg);
void* pre_multipart_upload_req_threadworker(S3fsCurl& s3fscurl, void* arg);
void* multipart_upload_part_req_threadworker(S3fsCurl& s3fscurl, void* arg);
void* complete_multipart_upload_threadworker(S3fsCurl& s3fscurl, void* arg);
void* abort_multipart_upload_req_threadworker(S3fsCurl& s3fscurl, void* arg);
void* multipart_put_head_req_threadworker(S3fsCurl& s3fscurl, void* arg);
void* parallel_get_object_req_threadworker(S3fsCurl& s3fscurl, void* arg);
void* get_object_req_threadworker(S3fsCurl& s3fscurl, void* arg);
//-------------------------------------------------------------------
// Utility functions

20
src/threadpoolman.cpp
View File

@ -28,6 +28,8 @@
#include "s3fs_logger.h"
#include "threadpoolman.h"
#include "curl.h"
#include "curl_share.h"
//------------------------------------------------
// ThreadPoolMan class variables
@ -105,6 +107,9 @@ void ThreadPoolMan::Worker(ThreadPoolMan* psingleton, std::promise<int> promise)
}
S3FS_PRN_INFO3("Start worker thread in ThreadPoolMan.");
// The only object in this thread worker
S3fsCurl s3fscurl(true);
while(!psingleton->IsExit()){
// wait
psingleton->thpoolman_sem.acquire();
@ -113,6 +118,12 @@ void ThreadPoolMan::Worker(ThreadPoolMan* psingleton, std::promise<int> promise)
break;
}
// reset curl handle
if(!s3fscurl.CreateCurlHandle(true)){
S3FS_PRN_ERR("Failed to re-create curl handle.");
break;
}
// get instruction
thpoolman_param param;
{
@ -127,8 +138,9 @@ void ThreadPoolMan::Worker(ThreadPoolMan* psingleton, std::promise<int> promise)
}
}
void* retval = param.pfunc(param.args);
if(nullptr != retval){
// run function
void* retval;
if(nullptr != (retval = param.pfunc(s3fscurl, param.args))){
S3FS_PRN_WARN("The instruction function returned with something error code(%ld).", reinterpret_cast<long>(retval));
}
if(param.psem){
@ -136,6 +148,10 @@ void ThreadPoolMan::Worker(ThreadPoolMan* psingleton, std::promise<int> promise)
}
}
if(!S3fsCurlShare::DestroyCurlShareHandleForThread()){
S3FS_PRN_WARN("Failed to destory curl share handle for this thread, but continue...");
}
promise.set_value(0);
}

4
src/threadpoolman.h
View File

@ -33,10 +33,12 @@
//------------------------------------------------
// Typedefs for functions and structures
//------------------------------------------------
class S3fsCurl;
//
// Prototype function
//
typedef void* (*thpoolman_worker)(void*);
typedef void* (*thpoolman_worker)(S3fsCurl&, void*);
//
// Parameter structure

2
test/run_tests_using_sanitizers.sh
View File

@ -49,7 +49,7 @@ ALL_TESTS=1 ASAN_OPTIONS='detect_leaks=1,detect_stack_use_after_return=1' make c
make clean
./configure CXX=clang++ CXXFLAGS="$COMMON_FLAGS -fsanitize=thread"
make --jobs="$(nproc)"
ALL_TESTS=1 TSAN_OPTIONS='halt_on_error=1,suppressions=threadsanitizer_suppressions.txt' make check -C test/
ALL_TESTS=1 TSAN_OPTIONS='halt_on_error=1' make check -C test/
# run tests under UndefinedBehaviorSanitizer, https://clang.llvm.org/docs/UndefinedBehaviorSanitizer.html
make clean

1
test/threadsanitizer_suppressions.txt
View File

@ -1 +0,0 @@
race:OPENSSL_sk_free