Merge 96f1c012db
into c97f7a2a13
This commit is contained in:
commit
e148e9e1c0
|
@ -66,6 +66,7 @@ AWSCLI_ZIP_FILE="awscliv2.zip"
|
|||
#-----------------------------------------------------------
|
||||
# Parameters for configure(set environments)
|
||||
#-----------------------------------------------------------
|
||||
CXX="g++"
|
||||
CXXFLAGS="-O -DS3FS_PTHREAD_ERRORCHECK=1"
|
||||
CONFIGURE_OPTIONS="--prefix=/usr --with-openssl"
|
||||
|
||||
|
@ -188,9 +189,11 @@ elif [ "${CONTAINER_FULLNAME}" = "fedora:39" ]; then
|
|||
PACKAGE_UPDATE_OPTIONS="update -y -qq"
|
||||
PACKAGE_INSTALL_OPTIONS="install -y"
|
||||
|
||||
INSTALL_PACKAGES="clang-tools-extra curl-devel fuse fuse-devel gcc libstdc++-devel gcc-c++ glibc-langpack-en java-latest-openjdk-headless jq libxml2-devel mailcap git automake make openssl openssl-devel curl attr diffutils procps python3-pip unzip"
|
||||
INSTALL_PACKAGES="clang clang-tools-extra curl-devel fuse fuse-devel gcc libstdc++-devel gcc-c++ glibc-langpack-en java-latest-openjdk-headless jq libxml2-devel mailcap git automake make openssl openssl-devel curl attr diffutils procps python3-pip unzip"
|
||||
INSTALL_CHECKER_PKGS="cppcheck ShellCheck"
|
||||
INSTALL_CHECKER_PKG_OPTIONS=""
|
||||
CXX="clang++"
|
||||
CXXFLAGS="-O -Wthread-safety"
|
||||
|
||||
elif [ "${CONTAINER_FULLNAME}" = "fedora:38" ]; then
|
||||
PACKAGE_MANAGER_BIN="dnf"
|
||||
|
@ -290,8 +293,11 @@ fi
|
|||
#-----------------------------------------------------------
|
||||
echo "${PRGNAME} [INFO] Set environment for configure options"
|
||||
|
||||
echo "CXXFLAGS=${CXXFLAGS}" >> "${GITHUB_ENV}"
|
||||
echo "CONFIGURE_OPTIONS=${CONFIGURE_OPTIONS}" >> "${GITHUB_ENV}"
|
||||
cat << EOF > "${GITHUB_ENV}"
|
||||
CXX=${CXX}
|
||||
CXXFLAGS=${CXXFLAGS}
|
||||
CONFIGURE_OPTIONS=${CONFIGURE_OPTIONS}
|
||||
EOF
|
||||
|
||||
echo "${PRGNAME} [INFO] Finish Linux helper for installing packages."
|
||||
|
||||
|
|
36
src/curl.cpp
36
src/curl.cpp
|
@ -84,7 +84,7 @@ static constexpr char SPECIAL_DARWIN_MIME_FILE[] = "/etc/apache2/mime.typ
|
|||
// Class S3fsCurl
|
||||
//-------------------------------------------------------------------
|
||||
pthread_mutex_t S3fsCurl::curl_warnings_lock;
|
||||
pthread_mutex_t S3fsCurl::curl_handles_lock;
|
||||
Mutex S3fsCurl::curl_handles_lock;
|
||||
S3fsCurl::callback_locks_t S3fsCurl::callback_locks;
|
||||
bool S3fsCurl::is_initglobal_done = false;
|
||||
CurlHandlerPool* S3fsCurl::sCurlPool = nullptr;
|
||||
|
@ -144,9 +144,6 @@ bool S3fsCurl::InitS3fsCurl()
|
|||
if(0 != pthread_mutex_init(&S3fsCurl::curl_warnings_lock, &attr)){
|
||||
return false;
|
||||
}
|
||||
if(0 != pthread_mutex_init(&S3fsCurl::curl_handles_lock, &attr)){
|
||||
return false;
|
||||
}
|
||||
if(0 != pthread_mutex_init(&S3fsCurl::callback_locks.dns, &attr)){
|
||||
return false;
|
||||
}
|
||||
|
@ -199,9 +196,6 @@ bool S3fsCurl::DestroyS3fsCurl()
|
|||
if(0 != pthread_mutex_destroy(&S3fsCurl::callback_locks.ssl_session)){
|
||||
result = false;
|
||||
}
|
||||
if(0 != pthread_mutex_destroy(&S3fsCurl::curl_handles_lock)){
|
||||
result = false;
|
||||
}
|
||||
if(0 != pthread_mutex_destroy(&S3fsCurl::curl_warnings_lock)){
|
||||
result = false;
|
||||
}
|
||||
|
@ -353,7 +347,7 @@ int S3fsCurl::CurlProgress(void *clientp, double dltotal, double dlnow, double u
|
|||
time_t now = time(nullptr);
|
||||
progress_t p(dlnow, ulnow);
|
||||
|
||||
AutoLock lock(&S3fsCurl::curl_handles_lock);
|
||||
MutexLocker auto_lock(&S3fsCurl::curl_handles_lock);
|
||||
|
||||
// any progress?
|
||||
if(p != S3fsCurl::curl_progress[curl]){
|
||||
|
@ -1910,7 +1904,7 @@ S3fsCurl::~S3fsCurl()
|
|||
DestroyCurlHandle();
|
||||
}
|
||||
|
||||
bool S3fsCurl::ResetHandle(AutoLock::Type locktype)
|
||||
bool S3fsCurl::ResetHandle()
|
||||
{
|
||||
bool run_once;
|
||||
{
|
||||
|
@ -2003,7 +1997,6 @@ bool S3fsCurl::ResetHandle(AutoLock::Type locktype)
|
|||
}
|
||||
}
|
||||
|
||||
AutoLock lock(&S3fsCurl::curl_handles_lock, locktype);
|
||||
S3fsCurl::curl_times[hCurl] = time(nullptr);
|
||||
S3fsCurl::curl_progress[hCurl] = progress_t(-1, -1);
|
||||
|
||||
|
@ -2012,10 +2005,10 @@ bool S3fsCurl::ResetHandle(AutoLock::Type locktype)
|
|||
|
||||
bool S3fsCurl::CreateCurlHandle(bool only_pool, bool remake)
|
||||
{
|
||||
AutoLock lock(&S3fsCurl::curl_handles_lock);
|
||||
MutexLocker auto_lock(&S3fsCurl::curl_handles_lock);
|
||||
|
||||
if(hCurl && remake){
|
||||
if(!DestroyCurlHandle(false, true, AutoLock::ALREADY_LOCKED)){
|
||||
if(!DestroyCurlHandleUnlocked(false, true)){
|
||||
S3FS_PRN_ERR("could not destroy handle.");
|
||||
return false;
|
||||
}
|
||||
|
@ -2035,12 +2028,18 @@ bool S3fsCurl::CreateCurlHandle(bool only_pool, bool remake)
|
|||
}
|
||||
}
|
||||
}
|
||||
ResetHandle(AutoLock::ALREADY_LOCKED);
|
||||
ResetHandle();
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool S3fsCurl::DestroyCurlHandle(bool restore_pool, bool clear_internal_data, AutoLock::Type locktype)
|
||||
bool S3fsCurl::DestroyCurlHandle(bool restore_pool, bool clear_internal_data)
|
||||
{
|
||||
MutexLocker auto_lock(&S3fsCurl::curl_handles_lock);
|
||||
return DestroyCurlHandleUnlocked(restore_pool, clear_internal_data);
|
||||
}
|
||||
|
||||
bool S3fsCurl::DestroyCurlHandleUnlocked(bool restore_pool, bool clear_internal_data)
|
||||
{
|
||||
// [NOTE]
|
||||
// If type is REQTYPE::IAMCRED or REQTYPE::IAMROLE, do not clear type.
|
||||
|
@ -2051,8 +2050,6 @@ bool S3fsCurl::DestroyCurlHandle(bool restore_pool, bool clear_internal_data, Au
|
|||
type = REQTYPE::UNSET;
|
||||
}
|
||||
|
||||
AutoLock lock(&S3fsCurl::curl_handles_lock, locktype);
|
||||
|
||||
if(clear_internal_data){
|
||||
ClearInternalData();
|
||||
}
|
||||
|
@ -2170,7 +2167,10 @@ bool S3fsCurl::RemakeHandle()
|
|||
partdata.size = b_partdata_size;
|
||||
|
||||
// reset handle
|
||||
ResetHandle();
|
||||
{
|
||||
MutexLocker auto_lock(&S3fsCurl::curl_handles_lock);
|
||||
ResetHandle();
|
||||
}
|
||||
|
||||
// set options
|
||||
switch(type){
|
||||
|
@ -2605,7 +2605,7 @@ int S3fsCurl::RequestPerform(bool dontAddAuthHeaders /*=false*/)
|
|||
S3FS_PRN_ERR("### CURLE_ABORTED_BY_CALLBACK");
|
||||
sleep(4);
|
||||
{
|
||||
AutoLock lock(&S3fsCurl::curl_handles_lock);
|
||||
MutexLocker auto_lock(&S3fsCurl::curl_handles_lock);
|
||||
S3fsCurl::curl_times[hCurl] = time(nullptr);
|
||||
}
|
||||
break;
|
||||
|
|
13
src/curl.h
13
src/curl.h
|
@ -24,10 +24,12 @@
|
|||
#include <curl/curl.h>
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
#include <vector>
|
||||
|
||||
#include "autolock.h"
|
||||
#include "metaheader.h"
|
||||
#include "mutex.h"
|
||||
#include "fdcache_page.h"
|
||||
|
||||
//----------------------------------------------
|
||||
|
@ -113,7 +115,7 @@ class S3fsCurl
|
|||
// class variables
|
||||
static pthread_mutex_t curl_warnings_lock;
|
||||
static bool curl_warnings_once; // emit older curl warnings only once
|
||||
static pthread_mutex_t curl_handles_lock;
|
||||
static Mutex curl_handles_lock;
|
||||
static struct callback_locks_t {
|
||||
pthread_mutex_t dns;
|
||||
pthread_mutex_t ssl_session;
|
||||
|
@ -139,8 +141,8 @@ class S3fsCurl
|
|||
static bool is_dump_body;
|
||||
static S3fsCred* ps3fscred;
|
||||
static long ssl_verify_hostname;
|
||||
static curltime_t curl_times;
|
||||
static curlprogress_t curl_progress;
|
||||
static curltime_t curl_times GUARDED_BY(curl_handles_lock);
|
||||
static curlprogress_t curl_progress GUARDED_BY(curl_handles_lock);
|
||||
static std::string curl_ca_bundle;
|
||||
static mimes_t mimeTypes;
|
||||
static std::string userAgent;
|
||||
|
@ -249,7 +251,7 @@ class S3fsCurl
|
|||
static int RawCurlDebugFunc(const CURL* hcurl, curl_infotype type, char* data, size_t size, void* userptr, curl_infotype datatype);
|
||||
|
||||
// methods
|
||||
bool ResetHandle(AutoLock::Type locktype = AutoLock::NONE);
|
||||
bool ResetHandle() REQUIRES(S3fsCurl::curl_handles_lock);
|
||||
bool RemakeHandle();
|
||||
bool ClearInternalData();
|
||||
void insertV4Headers(const std::string& access_key_id, const std::string& secret_access_key, const std::string& access_token);
|
||||
|
@ -343,7 +345,8 @@ class S3fsCurl
|
|||
|
||||
// methods
|
||||
bool CreateCurlHandle(bool only_pool = false, bool remake = false);
|
||||
bool DestroyCurlHandle(bool restore_pool = true, bool clear_internal_data = true, AutoLock::Type locktype = AutoLock::NONE);
|
||||
bool DestroyCurlHandle(bool restore_pool = true, bool clear_internal_data = true);
|
||||
bool DestroyCurlHandleUnlocked(bool restore_pool = true, bool clear_internal_data = true) REQUIRES(S3fsCurl::curl_handles_lock);
|
||||
|
||||
bool GetIAMCredentials(const char* cred_url, const char* iam_v2_token, const char* ibm_secret_access_key, std::string& response);
|
||||
bool GetIAMRoleFromMetaData(const char* cred_url, const char* iam_v2_token, std::string& token);
|
||||
|
|
|
@ -0,0 +1,150 @@
|
|||
/*
|
||||
* s3fs - FUSE-based file system backed by Amazon S3
|
||||
*
|
||||
* Copyright(C) 2007 Randy Rizun <rrizun@gmail.com>
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU General Public License
|
||||
* as published by the Free Software Foundation; either version 2
|
||||
* of the License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
||||
*/
|
||||
|
||||
#ifndef S3FS_MUTEX_H_
|
||||
#define S3FS_MUTEX_H_
|
||||
|
||||
#include <mutex>
|
||||
|
||||
// Taken from: https://clang.llvm.org/docs/ThreadSafetyAnalysis.html
|
||||
|
||||
#if defined(__clang__)
|
||||
#define THREAD_ANNOTATION_ATTRIBUTE__(x) __attribute__((x))
|
||||
#else
|
||||
#define THREAD_ANNOTATION_ATTRIBUTE__(x) // no-op
|
||||
#endif
|
||||
|
||||
// These wrappers are necessary with GNU libstdc++. clang libc++ should have these already.
|
||||
#define CAPABILITY(x) \
|
||||
THREAD_ANNOTATION_ATTRIBUTE__(capability(x))
|
||||
|
||||
#define SCOPED_CAPABILITY \
|
||||
THREAD_ANNOTATION_ATTRIBUTE__(scoped_lockable)
|
||||
|
||||
#define GUARDED_BY(x) \
|
||||
THREAD_ANNOTATION_ATTRIBUTE__(guarded_by(x))
|
||||
|
||||
#define PT_GUARDED_BY(x) \
|
||||
THREAD_ANNOTATION_ATTRIBUTE__(pt_guarded_by(x))
|
||||
|
||||
#define ACQUIRED_BEFORE(...) \
|
||||
THREAD_ANNOTATION_ATTRIBUTE__(acquired_before(__VA_ARGS__))
|
||||
|
||||
#define ACQUIRED_AFTER(...) \
|
||||
THREAD_ANNOTATION_ATTRIBUTE__(acquired_after(__VA_ARGS__))
|
||||
|
||||
#define REQUIRES(...) \
|
||||
THREAD_ANNOTATION_ATTRIBUTE__(requires_capability(__VA_ARGS__))
|
||||
|
||||
#define ACQUIRE(...) \
|
||||
THREAD_ANNOTATION_ATTRIBUTE__(acquire_capability(__VA_ARGS__))
|
||||
|
||||
#define RELEASE(...) \
|
||||
THREAD_ANNOTATION_ATTRIBUTE__(release_capability(__VA_ARGS__))
|
||||
|
||||
#define RELEASE_GENERIC(...) \
|
||||
THREAD_ANNOTATION_ATTRIBUTE__(release_generic_capability(__VA_ARGS__))
|
||||
|
||||
#define TRY_ACQUIRE(...) \
|
||||
THREAD_ANNOTATION_ATTRIBUTE__(try_acquire_capability(__VA_ARGS__))
|
||||
|
||||
|
||||
// Defines an annotated interface for mutexes.
|
||||
// These methods can be implemented to use any internal mutex implementation.
|
||||
class CAPABILITY("mutex") Mutex {
|
||||
public:
|
||||
// Acquire/lock this mutex exclusively. Only one thread can have exclusive
|
||||
// access at any one time. Write operations to guarded data require an
|
||||
// exclusive lock.
|
||||
void Lock() ACQUIRE() {
|
||||
lock.lock();
|
||||
}
|
||||
|
||||
// Release/unlock an exclusive mutex.
|
||||
void Unlock() RELEASE();
|
||||
|
||||
// Generic unlock, can unlock exclusive and shared mutexes.
|
||||
void GenericUnlock() RELEASE_GENERIC() {
|
||||
lock.unlock();
|
||||
}
|
||||
|
||||
// Try to acquire the mutex. Returns true on success, and false on failure.
|
||||
bool TryLock() TRY_ACQUIRE(true) {
|
||||
return lock.try_lock();
|
||||
}
|
||||
|
||||
Mutex() {}
|
||||
Mutex(const Mutex&) = delete;
|
||||
Mutex(Mutex&&) = delete;
|
||||
Mutex& operator=(const Mutex&) = delete;
|
||||
Mutex& operator=(Mutex&&) = delete;
|
||||
|
||||
private:
|
||||
std::mutex lock;
|
||||
};
|
||||
|
||||
// MutexLocker is an RAII class that acquires a mutex in its constructor, and
|
||||
// releases it in its destructor.
|
||||
class SCOPED_CAPABILITY MutexLocker {
|
||||
private:
|
||||
Mutex* mut;
|
||||
bool locked;
|
||||
|
||||
public:
|
||||
// Acquire mu, implicitly acquire *this and associate it with mu.
|
||||
explicit MutexLocker(Mutex *mu) ACQUIRE(mu) : mut(mu), locked(true) {
|
||||
mu->Lock();
|
||||
}
|
||||
|
||||
// Release *this and all associated mutexes, if they are still held.
|
||||
// There is no warning if the scope was already unlocked before.
|
||||
~MutexLocker() RELEASE() {
|
||||
if (locked)
|
||||
mut->GenericUnlock();
|
||||
}
|
||||
|
||||
// Acquire all associated mutexes exclusively.
|
||||
void Lock() ACQUIRE() {
|
||||
mut->Lock();
|
||||
locked = true;
|
||||
}
|
||||
|
||||
// Try to acquire all associated mutexes exclusively.
|
||||
bool TryLock() TRY_ACQUIRE(true) {
|
||||
return locked = mut->TryLock();
|
||||
}
|
||||
|
||||
// Release all associated mutexes. Warn on double unlock.
|
||||
void Unlock() RELEASE() {
|
||||
mut->Unlock();
|
||||
locked = false;
|
||||
}
|
||||
};
|
||||
|
||||
#endif // S3FS_MUTEX_H_
|
||||
|
||||
/*
|
||||
* Local variables:
|
||||
* tab-width: 4
|
||||
* c-basic-offset: 4
|
||||
* End:
|
||||
* vim600: expandtab sw=4 ts=4 fdm=marker
|
||||
* vim<600: expandtab sw=4 ts=4
|
||||
*/
|
Loading…
Reference in New Issue