octoleo
/
s3fs-fuse
mirror of https://github.com/s3fs-fuse/s3fs-fuse.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Demonstrate static checking with curl_handles_lock 

This can find missing locks at compile-time.
This commit is contained in:
Andrew Gaul 2023-12-01 21:54:06 +09:00
parent b139507ae6
commit 96f1c012db
4 changed files with 185 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
.github/workflows/linux-ci-helper.sh vendored
View File

@ -66,6 +66,7 @@ AWSCLI_ZIP_FILE="awscliv2.zip"
#-----------------------------------------------------------
# Parameters for configure(set environments)
#-----------------------------------------------------------
CXX="g++"
CXXFLAGS="-O -DS3FS_PTHREAD_ERRORCHECK=1"
CONFIGURE_OPTIONS="--prefix=/usr --with-openssl"
@ -188,9 +189,11 @@ elif [ "${CONTAINER_FULLNAME}" = "fedora:39" ]; then
PACKAGE_UPDATE_OPTIONS="update -y -qq"
PACKAGE_INSTALL_OPTIONS="install -y"
INSTALL_PACKAGES="clang-tools-extra curl-devel fuse fuse-devel gcc libstdc++-devel gcc-c++ glibc-langpack-en java-latest-openjdk-headless jq libxml2-devel mailcap git automake make openssl openssl-devel curl attr diffutils procps python3-pip unzip"
INSTALL_PACKAGES="clang clang-tools-extra curl-devel fuse fuse-devel gcc libstdc++-devel gcc-c++ glibc-langpack-en java-latest-openjdk-headless jq libxml2-devel mailcap git automake make openssl openssl-devel curl attr diffutils procps python3-pip unzip"
INSTALL_CHECKER_PKGS="cppcheck ShellCheck"
INSTALL_CHECKER_PKG_OPTIONS=""
CXX="clang++"
CXXFLAGS="-O -Wthread-safety"
elif [ "${CONTAINER_FULLNAME}" = "fedora:38" ]; then
PACKAGE_MANAGER_BIN="dnf"
@ -290,8 +293,11 @@ fi
#-----------------------------------------------------------
echo "${PRGNAME} [INFO] Set environment for configure options"
echo "CXXFLAGS=${CXXFLAGS}" >> "${GITHUB_ENV}"
echo "CONFIGURE_OPTIONS=${CONFIGURE_OPTIONS}" >> "${GITHUB_ENV}"
cat << EOF > "${GITHUB_ENV}"
CXX=${CXX}
CXXFLAGS=${CXXFLAGS}
CONFIGURE_OPTIONS=${CONFIGURE_OPTIONS}
EOF
echo "${PRGNAME} [INFO] Finish Linux helper for installing packages."

36
src/curl.cpp
View File

@ -84,7 +84,7 @@ static constexpr char SPECIAL_DARWIN_MIME_FILE[] = "/etc/apache2/mime.typ
// Class S3fsCurl
//-------------------------------------------------------------------
pthread_mutex_t S3fsCurl::curl_warnings_lock;
pthread_mutex_t S3fsCurl::curl_handles_lock;
Mutex S3fsCurl::curl_handles_lock;
S3fsCurl::callback_locks_t S3fsCurl::callback_locks;
bool S3fsCurl::is_initglobal_done = false;
CurlHandlerPool* S3fsCurl::sCurlPool = nullptr;
@ -144,9 +144,6 @@ bool S3fsCurl::InitS3fsCurl()
if(0 != pthread_mutex_init(&S3fsCurl::curl_warnings_lock, &attr)){
return false;
}
if(0 != pthread_mutex_init(&S3fsCurl::curl_handles_lock, &attr)){
return false;
}
if(0 != pthread_mutex_init(&S3fsCurl::callback_locks.dns, &attr)){
return false;
}
@ -199,9 +196,6 @@ bool S3fsCurl::DestroyS3fsCurl()
if(0 != pthread_mutex_destroy(&S3fsCurl::callback_locks.ssl_session)){
result = false;
}
if(0 != pthread_mutex_destroy(&S3fsCurl::curl_handles_lock)){
result = false;
}
if(0 != pthread_mutex_destroy(&S3fsCurl::curl_warnings_lock)){
result = false;
}
@ -353,7 +347,7 @@ int S3fsCurl::CurlProgress(void *clientp, double dltotal, double dlnow, double u
time_t now = time(nullptr);
progress_t p(dlnow, ulnow);
AutoLock lock(&S3fsCurl::curl_handles_lock);
MutexLocker auto_lock(&S3fsCurl::curl_handles_lock);
// any progress?
if(p != S3fsCurl::curl_progress[curl]){
@ -1903,7 +1897,7 @@ S3fsCurl::~S3fsCurl()
DestroyCurlHandle();
}
bool S3fsCurl::ResetHandle(AutoLock::Type locktype)
bool S3fsCurl::ResetHandle()
{
bool run_once;
{
@ -1996,7 +1990,6 @@ bool S3fsCurl::ResetHandle(AutoLock::Type locktype)
}
}
AutoLock lock(&S3fsCurl::curl_handles_lock, locktype);
S3fsCurl::curl_times[hCurl] = time(nullptr);
S3fsCurl::curl_progress[hCurl] = progress_t(-1, -1);
@ -2005,10 +1998,10 @@ bool S3fsCurl::ResetHandle(AutoLock::Type locktype)
bool S3fsCurl::CreateCurlHandle(bool only_pool, bool remake)
{
AutoLock lock(&S3fsCurl::curl_handles_lock);
MutexLocker auto_lock(&S3fsCurl::curl_handles_lock);
if(hCurl && remake){
if(!DestroyCurlHandle(false, true, AutoLock::ALREADY_LOCKED)){
if(!DestroyCurlHandleUnlocked(false, true)){
S3FS_PRN_ERR("could not destroy handle.");
return false;
}
@ -2028,12 +2021,18 @@ bool S3fsCurl::CreateCurlHandle(bool only_pool, bool remake)
}
}
}
ResetHandle(AutoLock::ALREADY_LOCKED);
ResetHandle();
return true;
}
bool S3fsCurl::DestroyCurlHandle(bool restore_pool, bool clear_internal_data, AutoLock::Type locktype)
bool S3fsCurl::DestroyCurlHandle(bool restore_pool, bool clear_internal_data)
{
MutexLocker auto_lock(&S3fsCurl::curl_handles_lock);
return DestroyCurlHandleUnlocked(restore_pool, clear_internal_data);
}
bool S3fsCurl::DestroyCurlHandleUnlocked(bool restore_pool, bool clear_internal_data)
{
// [NOTE]
// If type is REQTYPE::IAMCRED or REQTYPE::IAMROLE, do not clear type.
@ -2044,8 +2043,6 @@ bool S3fsCurl::DestroyCurlHandle(bool restore_pool, bool clear_internal_data, Au
type = REQTYPE::UNSET;
}
AutoLock lock(&S3fsCurl::curl_handles_lock, locktype);
if(clear_internal_data){
ClearInternalData();
}
@ -2163,7 +2160,10 @@ bool S3fsCurl::RemakeHandle()
partdata.size = b_partdata_size;
// reset handle
ResetHandle();
{
MutexLocker auto_lock(&S3fsCurl::curl_handles_lock);
ResetHandle();
}
// set options
switch(type){
@ -2598,7 +2598,7 @@ int S3fsCurl::RequestPerform(bool dontAddAuthHeaders /*=false*/)
S3FS_PRN_ERR("### CURLE_ABORTED_BY_CALLBACK");
sleep(4);
{
AutoLock lock(&S3fsCurl::curl_handles_lock);
MutexLocker auto_lock(&S3fsCurl::curl_handles_lock);
S3fsCurl::curl_times[hCurl] = time(nullptr);
}
break;

13
src/curl.h
View File

@ -24,10 +24,12 @@
#include <curl/curl.h>
#include <map>
#include <memory>
#include <mutex>
#include <vector>
#include "autolock.h"
#include "metaheader.h"
#include "mutex.h"
#include "fdcache_page.h"
//----------------------------------------------
@ -113,7 +115,7 @@ class S3fsCurl
// class variables
static pthread_mutex_t curl_warnings_lock;
static bool curl_warnings_once; // emit older curl warnings only once
static pthread_mutex_t curl_handles_lock;
static Mutex curl_handles_lock;
static struct callback_locks_t {
pthread_mutex_t dns;
pthread_mutex_t ssl_session;
@ -139,8 +141,8 @@ class S3fsCurl
static bool is_dump_body;
static S3fsCred* ps3fscred;
static long ssl_verify_hostname;
static curltime_t curl_times;
static curlprogress_t curl_progress;
static curltime_t curl_times GUARDED_BY(curl_handles_lock);
static curlprogress_t curl_progress GUARDED_BY(curl_handles_lock);
static std::string curl_ca_bundle;
static mimes_t mimeTypes;
static std::string userAgent;
@ -249,7 +251,7 @@ class S3fsCurl
static int RawCurlDebugFunc(const CURL* hcurl, curl_infotype type, char* data, size_t size, void* userptr, curl_infotype datatype);
// methods
bool ResetHandle(AutoLock::Type locktype = AutoLock::NONE);
bool ResetHandle() REQUIRES(S3fsCurl::curl_handles_lock);
bool RemakeHandle();
bool ClearInternalData();
void insertV4Headers(const std::string& access_key_id, const std::string& secret_access_key, const std::string& access_token);
@ -343,7 +345,8 @@ class S3fsCurl
// methods
bool CreateCurlHandle(bool only_pool = false, bool remake = false);
bool DestroyCurlHandle(bool restore_pool = true, bool clear_internal_data = true, AutoLock::Type locktype = AutoLock::NONE);
bool DestroyCurlHandle(bool restore_pool = true, bool clear_internal_data = true);
bool DestroyCurlHandleUnlocked(bool restore_pool = true, bool clear_internal_data = true) REQUIRES(S3fsCurl::curl_handles_lock);
bool GetIAMCredentials(const char* cred_url, const char* iam_v2_token, const char* ibm_secret_access_key, std::string& response);
bool GetIAMRoleFromMetaData(const char* cred_url, const char* iam_v2_token, std::string& token);

150
src/mutex.h Normal file
View File

@ -0,0 +1,150 @@
/*
* s3fs - FUSE-based file system backed by Amazon S3
*
* Copyright(C) 2007 Randy Rizun <rrizun@gmail.com>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
#ifndef S3FS_MUTEX_H_
#define S3FS_MUTEX_H_
#include <mutex>
// Taken from: https://clang.llvm.org/docs/ThreadSafetyAnalysis.html
#if defined(__clang__)
#define THREAD_ANNOTATION_ATTRIBUTE__(x) __attribute__((x))
#else
#define THREAD_ANNOTATION_ATTRIBUTE__(x) // no-op
#endif
// These wrappers are necessary with GNU libstdc++. clang libc++ should have these already.
#define CAPABILITY(x) \
THREAD_ANNOTATION_ATTRIBUTE__(capability(x))
#define SCOPED_CAPABILITY \
THREAD_ANNOTATION_ATTRIBUTE__(scoped_lockable)
#define GUARDED_BY(x) \
THREAD_ANNOTATION_ATTRIBUTE__(guarded_by(x))
#define PT_GUARDED_BY(x) \
THREAD_ANNOTATION_ATTRIBUTE__(pt_guarded_by(x))
#define ACQUIRED_BEFORE(...) \
THREAD_ANNOTATION_ATTRIBUTE__(acquired_before(__VA_ARGS__))
#define ACQUIRED_AFTER(...) \
THREAD_ANNOTATION_ATTRIBUTE__(acquired_after(__VA_ARGS__))
#define REQUIRES(...) \
THREAD_ANNOTATION_ATTRIBUTE__(requires_capability(__VA_ARGS__))
#define ACQUIRE(...) \
THREAD_ANNOTATION_ATTRIBUTE__(acquire_capability(__VA_ARGS__))
#define RELEASE(...) \
THREAD_ANNOTATION_ATTRIBUTE__(release_capability(__VA_ARGS__))
#define RELEASE_GENERIC(...) \
THREAD_ANNOTATION_ATTRIBUTE__(release_generic_capability(__VA_ARGS__))
#define TRY_ACQUIRE(...) \
THREAD_ANNOTATION_ATTRIBUTE__(try_acquire_capability(__VA_ARGS__))
// Defines an annotated interface for mutexes.
// These methods can be implemented to use any internal mutex implementation.
class CAPABILITY("mutex") Mutex {
public:
// Acquire/lock this mutex exclusively. Only one thread can have exclusive
// access at any one time. Write operations to guarded data require an
// exclusive lock.
void Lock() ACQUIRE() {
lock.lock();
}
// Release/unlock an exclusive mutex.
void Unlock() RELEASE();
// Generic unlock, can unlock exclusive and shared mutexes.
void GenericUnlock() RELEASE_GENERIC() {
lock.unlock();
}
// Try to acquire the mutex. Returns true on success, and false on failure.
bool TryLock() TRY_ACQUIRE(true) {
return lock.try_lock();
}
Mutex() {}
Mutex(const Mutex&) = delete;
Mutex(Mutex&&) = delete;
Mutex& operator=(const Mutex&) = delete;
Mutex& operator=(Mutex&&) = delete;
private:
std::mutex lock;
};
// MutexLocker is an RAII class that acquires a mutex in its constructor, and
// releases it in its destructor.
class SCOPED_CAPABILITY MutexLocker {
private:
Mutex* mut;
bool locked;
public:
// Acquire mu, implicitly acquire *this and associate it with mu.
explicit MutexLocker(Mutex *mu) ACQUIRE(mu) : mut(mu), locked(true) {
mu->Lock();
}
// Release *this and all associated mutexes, if they are still held.
// There is no warning if the scope was already unlocked before.
~MutexLocker() RELEASE() {
if (locked)
mut->GenericUnlock();
}
// Acquire all associated mutexes exclusively.
void Lock() ACQUIRE() {
mut->Lock();
locked = true;
}
// Try to acquire all associated mutexes exclusively.
bool TryLock() TRY_ACQUIRE(true) {
return locked = mut->TryLock();
}
// Release all associated mutexes. Warn on double unlock.
void Unlock() RELEASE() {
mut->Unlock();
locked = false;
}
};
#endif // S3FS_MUTEX_H_
/*
* Local variables:
* tab-width: 4
* c-basic-offset: 4
* End:
* vim600: expandtab sw=4 ts=4 fdm=marker
* vim<600: expandtab sw=4 ts=4
*/