Upgrade CI to Ubuntu 24.04 LTS (#2456)

commit e3b50ad introduce smart pointer to manage FdEntity

But in ChangeEntityToTempPath, we should not destroy the entity.

We should move the entry to the temp ky

Signed-off-by: liubingrun <liubr1@chinatelecom.cn>

.clang-tidy

@@ -61,6 +61,7 @@ Checks: '
     -performance-no-int-to-ptr,
     portability-*,
     readability-*,
+    -readability-avoid-nested-conditional-operator,
     -readability-braces-around-statements,
     -readability-else-after-return,
     -readability-function-cognitive-complexity,

.github/workflows/ci.yml

@@ -50,7 +50,7 @@ jobs:
       #
       matrix:
         container:
-          - ubuntu:23.10
+          - ubuntu:24.04
           - ubuntu:22.04
           - ubuntu:20.04
           - debian:bookworm
@@ -59,10 +59,10 @@ jobs:
           - rockylinux:9
           - rockylinux:8
           - centos:centos7
+          - fedora:40
           - fedora:39
-          - fedora:38
           - opensuse/leap:15
-          - alpine:3.18
+          - alpine:3.19
 
     container:
       image: ${{ matrix.container }}
@@ -225,7 +225,7 @@ jobs:
           - valgrind
 
     container:
-      image: fedora:39
+      image: fedora:40
 
       options: "--privileged --cap-add SYS_ADMIN --device /dev/fuse"
 
@@ -235,7 +235,7 @@ jobs:
 
       - name: Install packages
         run: |
-          .github/workflows/linux-ci-helper.sh fedora:39
+          .github/workflows/linux-ci-helper.sh fedora:40
 
       - name: Install clang
         run: |
@@ -263,6 +263,10 @@ jobs:
               echo 'CXX=clang++'                                       >> $GITHUB_ENV
               echo "CXXFLAGS=${COMMON_CXXFLAGS} -O0 -fsanitize=thread" >> $GITHUB_ENV
               echo 'TSAN_OPTIONS=halt_on_error=1'                      >> $GITHUB_ENV
+              # [NOTE]
+              # Set this to avoid following error when running configure.
+              # "FATAL: ThreadSanitizer: unexpected memory mapping"
+              sysctl vm.mmap_rnd_bits=28
           elif [ "${{ matrix.checktype }}" = "sanitize_others" ]; then
               echo 'CXX=clang++'                                       >> $GITHUB_ENV
               echo "CXXFLAGS=${COMMON_CXXFLAGS} -O1 -fsanitize=undefined,implicit-conversion,local-bounds,unsigned-integer-overflow" >> $GITHUB_ENV

.github/workflows/linux-ci-helper.sh

@@ -80,12 +80,12 @@ PACKAGE_INSTALL_ADDITIONAL_OPTIONS=""
 SHELLCHECK_DIRECT_INSTALL=0
 AWSCLI_DIRECT_INSTALL=1
 
-if [ "${CONTAINER_FULLNAME}" = "ubuntu:23.10" ]; then
+if [ "${CONTAINER_FULLNAME}" = "ubuntu:24.04" ]; then
     PACKAGE_MANAGER_BIN="apt-get"
     PACKAGE_UPDATE_OPTIONS="update -y -qq"
     PACKAGE_INSTALL_OPTIONS="install -y"
 
-    INSTALL_PACKAGES="autoconf autotools-dev clang-tidy openjdk-21-jre-headless fuse jq libfuse-dev libcurl4-openssl-dev libxml2-dev locales-all mime-support libtool pkg-config libssl-dev attr curl python3-pip unzip"
+    INSTALL_PACKAGES="autoconf autotools-dev clang-tidy openjdk-21-jre-headless fuse jq libfuse-dev libcurl4-openssl-dev libxml2-dev locales-all mailcap libtool pkg-config libssl-dev attr curl python3-pip unzip"
     INSTALL_CHECKER_PKGS="cppcheck shellcheck"
     INSTALL_CHECKER_PKG_OPTIONS=""
 
@@ -183,7 +183,7 @@ elif [ "${CONTAINER_FULLNAME}" = "centos:centos7" ]; then
     INSTALL_CHECKER_PKGS="cppcheck jq"
     INSTALL_CHECKER_PKG_OPTIONS="--enablerepo=epel"
 
-elif [ "${CONTAINER_FULLNAME}" = "fedora:39" ]; then
+elif [ "${CONTAINER_FULLNAME}" = "fedora:40" ]; then
     PACKAGE_MANAGER_BIN="dnf"
     PACKAGE_UPDATE_OPTIONS="update -y -qq"
     PACKAGE_INSTALL_OPTIONS="install -y"
@@ -192,7 +192,7 @@ elif [ "${CONTAINER_FULLNAME}" = "fedora:39" ]; then
     INSTALL_CHECKER_PKGS="cppcheck ShellCheck"
     INSTALL_CHECKER_PKG_OPTIONS=""
 
-elif [ "${CONTAINER_FULLNAME}" = "fedora:38" ]; then
+elif [ "${CONTAINER_FULLNAME}" = "fedora:39" ]; then
     PACKAGE_MANAGER_BIN="dnf"
     PACKAGE_UPDATE_OPTIONS="update -y -qq"
     PACKAGE_INSTALL_OPTIONS="install -y"
@@ -210,7 +210,7 @@ elif [ "${CONTAINER_FULLNAME}" = "opensuse/leap:15" ]; then
     INSTALL_CHECKER_PKGS="cppcheck ShellCheck"
     INSTALL_CHECKER_PKG_OPTIONS=""
 
-elif [ "${CONTAINER_FULLNAME}" = "alpine:3.18" ]; then
+elif [ "${CONTAINER_FULLNAME}" = "alpine:3.19" ]; then
     PACKAGE_MANAGER_BIN="apk"
     PACKAGE_UPDATE_OPTIONS="update --no-progress"
     PACKAGE_INSTALL_OPTIONS="add --no-progress --no-cache"

COMPILATION.md

@@ -6,7 +6,9 @@ If you want specific instructions for some distributions, check the [wiki](https
 
 Keep in mind using the pre-built packages when available.
 
-1. Ensure your system satisfies build and runtime dependencies for:
+## Compilation on Linux
+
+### Ensure your system satisfies build and runtime dependencies for:
 
 * fuse >= 2.8.4
 * automake
@@ -14,7 +16,10 @@ Keep in mind using the pre-built packages when available.
 * make
 * libcurl
 * libxml2
-* openssl
+* openssl/gnutls/nss
+    * Please prepare the library according to the OS on which you will compile.
+    * It is necessary to match the library used by libcurl.
+    * Install the OpenSSL, GnuTLS or NSS devel package.
 * mime.types (the package providing depends on the OS)
 	* s3fs tries to detect `/etc/mime.types` as default regardless of the OS
 	* Else s3fs tries to detect `/etc/apache2/mime.types` if OS is macOS
@@ -22,16 +27,38 @@ Keep in mind using the pre-built packages when available.
 	* Alternatively, you can set mime.types file path with `mime` option without detecting these default files
 * pkg-config (or your OS equivalent)
 
-2. Then compile from master via the following commands:
+* NOTE  
+    If you have any trouble about details on required packages, see `INSTALL_PACKAGES` in [linux-ci-helper.sh](https://github.com/s3fs-fuse/s3fs-fuse/blob/master/.github/workflows/linux-ci-helper.sh).
+
+### Then compile from master via the following commands:
+1. Clone the source code:
+    ```sh
+    git clone https://github.com/s3fs-fuse/s3fs-fuse.git
+    ```
+2. Configuration:
+    ```sh
+    cd s3fs-fuse
+    ./autogen.sh
+    ./configure
+    ```
+    Depending on the TLS library (OpenSSL/GnuTLS/NSS), add `--with-openssl`, `--with-gnutls` or `--with-nss` when executing `configure`. (If omitted, it is equivalent to `--with-openssl`.)
+3. Bulding:
+    ```sh
+    make
+    ```
+4. Installing:
+    ```sh
+    sudo make install
+    ```
+
+### NOTE - The required libraries/components required to run s3fs are:
+
+* fuse >= 2.8.4
+* libcurl
+* libxml2
+* openssl/gnutls/nss
+* mime.types (the package providing depends on the OS)
 
-```
-git clone https://github.com/s3fs-fuse/s3fs-fuse.git
-cd s3fs-fuse
-./autogen.sh
-./configure
-make
-sudo make install
-```
 
 ## Compilation on Windows (using MSYS2)
 

README.md

@@ -4,9 +4,11 @@ s3fs allows Linux, macOS, and FreeBSD to mount an S3 bucket via [FUSE(Filesystem
 s3fs makes you operate files and directories in S3 bucket like a local file system.  
 s3fs preserves the native object format for files, allowing use of other tools like [AWS CLI](https://github.com/aws/aws-cli).  
 
-[![s3fs-fuse CI](https://github.com/s3fs-fuse/s3fs-fuse/workflows/s3fs-fuse%20CI/badge.svg)](https://github.com/s3fs-fuse/s3fs-fuse/actions)
+[![s3fs-fuse CI](https://github.com/s3fs-fuse/s3fs-fuse/actions/workflows/ci.yml/badge.svg)](https://github.com/s3fs-fuse/s3fs-fuse/actions/workflows/ci.yml)
 [![Twitter Follow](https://img.shields.io/twitter/follow/s3fsfuse.svg?style=social&label=Follow)](https://twitter.com/s3fsfuse)
 
+![s3fs-fuse](https://github.com/ggtakec/s3fs-fuse-images/blob/master/images/s3fslogo.png)
+
 ## Features
 
 * large subset of POSIX including reading/writing files, directories, symlinks, mode, uid/gid, and extended attributes

doc/man/s3fs.1.in

@@ -179,6 +179,18 @@ server certificate won't be checked against the available certificate authoritie
 \fB\-o\fR ssl_verify_hostname (default="2")
 When 0, do not verify the SSL certificate against the hostname.
 .TP
+\fB\-o\fR ssl_client_cert (default="")
+Specify an SSL client certificate.
+Specify this optional parameter in the following format:
+ "<SSL Cert>[:<Cert Type>[:<Private Key>[:<Key Type>
+                                        [:<Password>]]]]"
+ <SSL Cert>: Client certificate.
+             Specify the file path or NickName(for NSS, etc.).
+ <Cert Type>: Type of certificate, default is "PEM"(optional).
+ <Private Key>: Certificate's private key file(optional).
+ <Key Type>: Type of private key, default is "PEM"(optional).
+ <Password>: Passphrase of the private key(optional). It is also possible to omit this value and specify it using the environment variable "S3FS_SSL_PRIVKEY_PASSWORD".
+.TP
 \fB\-o\fR nodnscache - disable DNS cache.
 s3fs is always using DNS cache, this option make DNS cache disable.
 .TP
@@ -418,6 +430,12 @@ Username and passphrase are valid only for HTTP schema.
 If the HTTP proxy does not require authentication, this option is not required.
 Separate the username and passphrase with a ':' character and specify each as a URL-encoded string.
 .TP
+\fB\-o\fR ipresolve (default="whatever")
+Select what type of IP addresses to use when establishing a connection.
+Default('whatever') can use addresses of all IP versions(IPv4 and IPv6) that your system allows.
+If you specify 'IPv4', only IPv4 addresses are used.
+And when 'IPv6' is specified, only IPv6 addresses will be used.
+.TP
 \fB\-o\fR logfile - specify the log output file.
 s3fs outputs the log file to syslog. Alternatively, if s3fs is started with the "-f" option specified, the log will be output to the stdout/stderr.
 You can use this option to specify the log file that s3fs outputs.

src/curl.cpp

@@ -83,6 +83,7 @@ static constexpr char SPECIAL_DARWIN_MIME_FILE[]        = "/etc/apache2/mime.typ
 //-------------------------------------------------------------------
 // Class S3fsCurl
 //-------------------------------------------------------------------
+constexpr char   S3fsCurl::S3FS_SSL_PRIVKEY_PASSWORD[];
 pthread_mutex_t  S3fsCurl::curl_warnings_lock;
 pthread_mutex_t  S3fsCurl::curl_handles_lock;
 S3fsCurl::callback_locks_t S3fsCurl::callback_locks;
@@ -107,6 +108,12 @@ bool             S3fsCurl::is_verbose          = false;
 bool             S3fsCurl::is_dump_body        = false;
 S3fsCred*        S3fsCurl::ps3fscred           = nullptr;
 long             S3fsCurl::ssl_verify_hostname = 1;    // default(original code...)
+// SSL client cert options
+std::string      S3fsCurl::client_cert;
+std::string      S3fsCurl::client_cert_type;
+std::string      S3fsCurl::client_priv_key;
+std::string      S3fsCurl::client_priv_key_type;
+std::string      S3fsCurl::client_key_password;
 
 // protected by curl_warnings_lock
 bool             S3fsCurl::curl_warnings_once = false;
@@ -130,6 +137,7 @@ bool             S3fsCurl::requester_pays      = false;          // default
 std::string      S3fsCurl::proxy_url;
 bool             S3fsCurl::proxy_http          = false;
 std::string      S3fsCurl::proxy_userpwd;
+long             S3fsCurl::ipresolve_type      = CURL_IPRESOLVE_WHATEVER;
 
 //-------------------------------------------------------------------
 // Class methods for S3fsCurl
@@ -609,7 +617,7 @@ size_t S3fsCurl::ReadCallback(void* ptr, size_t size, size_t nmemb, void* userp)
     memcpy(ptr, pCurl->postdata, copysize);
 
     pCurl->postdata_remaining = (pCurl->postdata_remaining > static_cast<off_t>(copysize) ? (pCurl->postdata_remaining - copysize) : 0);
-    pCurl->postdata          += static_cast<size_t>(copysize);
+    pCurl->postdata          += copysize;
 
     return copysize;
 }
@@ -963,7 +971,7 @@ bool S3fsCurl::GetSseKey(std::string& md5, std::string& ssekey)
 
 bool S3fsCurl::GetSseKeyMd5(size_t pos, std::string& md5)
 {
-    if(S3fsCurl::sseckeys.size() <= static_cast<size_t>(pos)){
+    if(S3fsCurl::sseckeys.size() <= pos){
         return false;
     }
     size_t cnt = 0;
@@ -1012,6 +1020,75 @@ long S3fsCurl::SetSslVerifyHostname(long value)
     return old;
 }
 
+bool S3fsCurl::SetSSLClientCertOptions(const std::string& values)
+{
+    // Parse values:
+    //   <values> = "<SSL Client Cert>:<SSL Cert Type>:<SSL Cert Private Key>:<SSL Cert Private Type>:<Key Password>"
+    //
+    if(values.empty()){
+        return false;
+    }
+
+    std::list<std::string> valarr;
+    std::string::size_type   start_pos = 0;
+    std::string::size_type   pos;
+    do{
+        if(std::string::npos == (pos = values.find(':', start_pos))){
+            valarr.push_back(values.substr(start_pos));
+            start_pos = pos;
+        }else{
+            if(0 < (pos - start_pos)){
+                valarr.push_back(values.substr(start_pos, (pos - start_pos)));
+            }else{
+                valarr.emplace_back("");
+            }
+            start_pos = ++pos;
+        }
+    }while(std::string::npos != start_pos);
+
+    // set client cert
+    if(!valarr.empty() && !valarr.front().empty()){
+        S3fsCurl::client_cert = valarr.front();
+        valarr.pop_front();
+
+        // set client cert type
+        if(!valarr.empty()){
+            S3fsCurl::client_cert_type = valarr.front();                // allow empty(default: PEM)
+            valarr.pop_front();
+
+            // set client private key
+            if(!valarr.empty()){
+                S3fsCurl::client_priv_key = valarr.front();             // allow empty
+                valarr.pop_front();
+
+                // set client private key type
+                if(!valarr.empty()){
+                    S3fsCurl::client_priv_key_type = valarr.front();    // allow empty(default: PEM)
+                    valarr.pop_front();
+
+                    // set key password
+                    if(!valarr.empty()){
+                        S3fsCurl::client_key_password = valarr.front(); // allow empty
+                    }
+                }
+            }
+        }
+    }
+
+    // [NOTE]
+    // If the private key is set but the password is not set,
+    // check the environment variables.
+    //
+    if(!S3fsCurl::client_priv_key.empty() && S3fsCurl::client_key_password.empty()){
+        const char* pass = std::getenv(S3fsCurl::S3FS_SSL_PRIVKEY_PASSWORD);
+        if(pass != nullptr){
+            S3fsCurl::client_key_password = pass;
+        }
+    }
+
+    return true;
+}
+
 bool S3fsCurl::SetMultipartSize(off_t size)
 {
     size = size * 1024 * 1024;
@@ -1174,6 +1251,23 @@ bool S3fsCurl::SetProxyUserPwd(const char* file)
     return true;
 }
 
+bool S3fsCurl::SetIPResolveType(const char* value)
+{
+    if(!value){
+        return false;
+    }
+    if(0 == strcasecmp(value, "ipv4")){
+        S3fsCurl::ipresolve_type = CURL_IPRESOLVE_V4;
+    }else if(0 == strcasecmp(value, "ipv6")){
+        S3fsCurl::ipresolve_type = CURL_IPRESOLVE_V6;
+    }else if(0 == strcasecmp(value, "whatever")){       // = default type
+        S3fsCurl::ipresolve_type = CURL_IPRESOLVE_WHATEVER;
+    }else{
+        return false;
+    }
+    return true;
+}
+
 // cppcheck-suppress unmatchedSuppression
 // cppcheck-suppress constParameter
 // cppcheck-suppress constParameterCallback
@@ -1526,7 +1620,7 @@ int S3fsCurl::ParallelMixMultipartUploadRequest(const char* tpath, headers_t& me
             for(off_t i = 0, bytes = 0; i < iter->bytes; i += bytes){
                 std::unique_ptr<S3fsCurl> s3fscurl_para(new S3fsCurl(true));
 
-                bytes = std::min(static_cast<off_t>(GetMultipartCopySize()), iter->bytes - i);
+                bytes = std::min(GetMultipartCopySize(), iter->bytes - i);
                 /* every part should be larger than MIN_MULTIPART_SIZE and smaller than FIVE_GB */
                 off_t remain_bytes = iter->bytes - i - bytes;
 
@@ -1949,7 +2043,11 @@ bool S3fsCurl::ResetHandle(AutoLock::Type locktype)
     if(CURLE_OK != curl_easy_setopt(hCurl, S3FS_CURLOPT_KEEP_SENDING_ON_ERROR, 1) && !run_once){
         S3FS_PRN_WARN("The S3FS_CURLOPT_KEEP_SENDING_ON_ERROR option could not be set. For maximize performance you need to enable this option and you should use libcurl 7.51.0 or later.");
     }
-
+    if(CURL_IPRESOLVE_WHATEVER != S3fsCurl::ipresolve_type){    // CURL_IPRESOLVE_WHATEVER is default, so not need to set.
+        if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_IPRESOLVE, S3fsCurl::ipresolve_type)){
+            return false;
+        }
+    }
     if(type != REQTYPE::IAMCRED && type != REQTYPE::IAMROLE){
         // REQTYPE::IAMCRED and REQTYPE::IAMROLE are always HTTP
         if(0 == S3fsCurl::ssl_verify_hostname){
@@ -1963,6 +2061,36 @@ bool S3fsCurl::ResetHandle(AutoLock::Type locktype)
             }
         }
     }
+    // SSL Client Cert
+    if(!S3fsCurl::client_cert.empty()){
+        if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_SSLCERT, S3fsCurl::client_cert.c_str())){
+            return false;
+        }
+        if(!S3fsCurl::client_cert_type.empty() && 0 != strcasecmp(S3fsCurl::client_cert_type.c_str(), "PEM")){              // "PEM" is default
+            if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_SSLCERTTYPE, S3fsCurl::client_cert_type.c_str())){
+                return false;
+            }
+        }
+
+        // Private key
+        if(!S3fsCurl::client_priv_key.empty()){
+            if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_SSLKEY, S3fsCurl::client_priv_key.c_str())){
+                return false;
+            }
+            if(!S3fsCurl::client_priv_key_type.empty() && 0 != strcasecmp(S3fsCurl::client_priv_key_type.c_str(), "PEM")){  // "PEM" is default
+                if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_SSLKEYTYPE, S3fsCurl::client_priv_key_type.c_str())){
+                    return false;
+                }
+            }
+            // Password
+            if(!S3fsCurl::client_key_password.empty()){
+                if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_KEYPASSWD, S3fsCurl::client_key_password.c_str())){
+                    return false;
+                }
+            }
+        }
+    }
+
     if((S3fsCurl::is_dns_cache || S3fsCurl::is_ssl_session_cache) && S3fsCurl::hCurlShare){
         if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_SHARE, S3fsCurl::hCurlShare)){
             return false;
@@ -2560,6 +2688,7 @@ int S3fsCurl::RequestPerform(bool dontAddAuthHeaders /*=false*/)
                         result = -ENOTSUP;
                         break;
 
+                    case 429:
                     case 500:
                     case 503: {
                         S3FS_PRN_INFO3("HTTP response code %ld was returned, slowing down", responseCode);
@@ -3447,7 +3576,7 @@ int S3fsCurl::PutRequest(const char* tpath, headers_t& meta, int fd)
         std::string strMD5;
         if(-1 != fd){
             strMD5 = s3fs_get_content_md5(fd);
-            if(0 == strMD5.length()){
+            if(strMD5.empty()){
                 S3FS_PRN_ERR("Failed to make MD5.");
                 return -EIO;
             }

src/curl.h

@@ -110,6 +110,9 @@ class S3fsCurl
             IAMROLE
         };
 
+        // Environment name
+        static constexpr char   S3FS_SSL_PRIVKEY_PASSWORD[] = "S3FS_SSL_PRIVKEY_PASSWORD";
+
         // class variables
         static pthread_mutex_t  curl_warnings_lock;
         static bool             curl_warnings_once;  // emit older curl warnings only once
@@ -139,6 +142,11 @@ class S3fsCurl
         static bool             is_dump_body;
         static S3fsCred*        ps3fscred;
         static long             ssl_verify_hostname;
+        static std::string      client_cert;
+        static std::string      client_cert_type;
+        static std::string      client_priv_key;
+        static std::string      client_priv_key_type;
+        static std::string      client_key_password;
         static curltime_t       curl_times;
         static curlprogress_t   curl_progress;
         static std::string      curl_ca_bundle;
@@ -156,6 +164,7 @@ class S3fsCurl
         static std::string      proxy_url;
         static bool             proxy_http;
         static std::string      proxy_userpwd;     // load from file(<username>:<passphrase>)
+        static long             ipresolve_type;    // this value is a libcurl symbol.
 
         // variables
         CURL*                hCurl;
@@ -316,6 +325,7 @@ class S3fsCurl
         static bool IsDumpBody() { return S3fsCurl::is_dump_body; }
         static long SetSslVerifyHostname(long value);
         static long GetSslVerifyHostname() { return S3fsCurl::ssl_verify_hostname; }
+        static bool SetSSLClientCertOptions(const std::string& values);
         static void ResetOffset(S3fsCurl* pCurl);
         // maximum parallel GET and PUT requests
         static int SetMaxParallelCount(int value);
@@ -340,6 +350,7 @@ class S3fsCurl
         static bool IsRequesterPays() { return S3fsCurl::requester_pays; }
         static bool SetProxy(const char* url);
         static bool SetProxyUserPwd(const char* userpwd);
+        static bool SetIPResolveType(const char* value);
 
         // methods
         bool CreateCurlHandle(bool only_pool = false, bool remake = false);

src/fdcache.cpp

@@ -775,7 +775,7 @@ bool FdManager::Close(FdEntity* ent, int fd)
     return false;
 }
 
-bool FdManager::ChangeEntityToTempPath(FdEntity* ent, const char* path)
+bool FdManager::ChangeEntityToTempPath(const FdEntity* ent, const char* path)
 {
     AutoLock auto_lock(&FdManager::fd_manager_lock);
 
@@ -783,8 +783,10 @@ bool FdManager::ChangeEntityToTempPath(FdEntity* ent, const char* path)
         if(iter->second.get() == ent){
             std::string tmppath;
             FdManager::MakeRandomTempPath(path, tmppath);
-            iter->second.reset(ent);
-            break;
+            // Move the entry to the new key
+            fent[tmppath] = std::move(iter->second);
+            iter = fent.erase(iter);
+            return true;
         }else{
             ++iter;
         }

src/fdcache.h

@@ -96,7 +96,7 @@ class FdManager
       FdEntity* OpenExistFdEntity(const char* path, int& fd, int flags = O_RDONLY);
       void Rename(const std::string &from, const std::string &to);
       bool Close(FdEntity* ent, int fd);
-      bool ChangeEntityToTempPath(FdEntity* ent, const char* path);
+      bool ChangeEntityToTempPath(const FdEntity* ent, const char* path);
       void CleanupCacheDir();
 
       bool CheckAllCache();

src/fdcache_entity.cpp

@@ -1180,10 +1180,10 @@ int FdEntity::NoCacheLoadAndPost(PseudoFdInfo* pseudo_obj, off_t start, off_t si
             break;
         }
         // download each multipart size(default 10MB) in unit
-        for(off_t oneread = 0, totalread = (iter->offset < start ? start : 0); totalread < static_cast<off_t>(iter->bytes); totalread += oneread){
+        for(off_t oneread = 0, totalread = (iter->offset < start ? start : 0); totalread < iter->bytes; totalread += oneread){
             int   upload_fd = physical_fd;
             off_t offset    = iter->offset + totalread;
-            oneread         = std::min(static_cast<off_t>(iter->bytes) - totalread, S3fsCurl::GetMultipartSize());
+            oneread         = std::min(iter->bytes - totalread, S3fsCurl::GetMultipartSize());
 
             // check rest size is over minimum part size
             //

src/fdcache_fdinfo.cpp

@@ -449,13 +449,13 @@ bool PseudoFdInfo::ParallelMultipartUpload(const char* path, const mp_part_list_
         thargs->petag                = petag;
 
         // make parameter for thread pool
-        std::unique_ptr<thpoolman_param> ppoolparam(new thpoolman_param);
-        ppoolparam->args             = thargs;
-        ppoolparam->psem             = &uploaded_sem;
-        ppoolparam->pfunc            = PseudoFdInfo::MultipartUploadThreadWorker;
+        thpoolman_param ppoolparam;
+        ppoolparam.args             = thargs;
+        ppoolparam.psem             = &uploaded_sem;
+        ppoolparam.pfunc            = PseudoFdInfo::MultipartUploadThreadWorker;
 
         // setup instruction
-        if(!ThreadPoolMan::Instruct(std::move(ppoolparam))){
+        if(!ThreadPoolMan::Instruct(ppoolparam)){
             S3FS_PRN_ERR("failed setup instruction for uploading.");
             delete thargs;
             return false;

src/mpu_util.cpp

@@ -115,15 +115,15 @@ int s3fs_utility_processing(time_t abort_time)
         // parse result(incomplete multipart upload information)
         S3FS_PRN_DBG("response body = {\n%s\n}", body.c_str());
 
-        xmlDocPtr doc;
-        if(nullptr == (doc = xmlReadMemory(body.c_str(), static_cast<int>(body.size()), "", nullptr, 0))){
+        std::unique_ptr<xmlDoc, decltype(&xmlFreeDoc)> doc(xmlReadMemory(body.c_str(), static_cast<int>(body.size()), "", nullptr, 0), xmlFreeDoc);
+        if(nullptr == doc){
             S3FS_PRN_DBG("xmlReadMemory exited with error.");
             result = EXIT_FAILURE;
 
         }else{
             // make incomplete uploads list
             incomp_mpu_list_t list;
-            if(!get_incomp_mpu_list(doc, list)){
+            if(!get_incomp_mpu_list(doc.get(), list)){
                 S3FS_PRN_DBG("get_incomp_mpu_list exited with error.");
                 result = EXIT_FAILURE;
 
@@ -139,7 +139,6 @@ int s3fs_utility_processing(time_t abort_time)
                     }
                 }
             }
-            S3FS_XMLFREEDOC(doc);
         }
     }
 

src/s3fs.cpp

@@ -60,7 +60,7 @@
 #define ENOATTR                   ENODATA
 #endif
 
-enum class dirtype {
+enum class dirtype : int8_t {
     UNKNOWN = -1,
     NEW = 0,
     OLD = 1,
@@ -3459,7 +3459,6 @@ static int list_bucket(const char* path, S3ObjList& head, const char* delimiter,
     std::string next_marker;
     bool truncated = true;
     S3fsCurl  s3fscurl;
-    xmlDocPtr doc;
 
     S3FS_PRN_INFO1("[path=%s]", path);
 
@@ -3519,20 +3518,20 @@ static int list_bucket(const char* path, S3ObjList& head, const char* delimiter,
         std::string encbody = get_encoded_cr_code(body->c_str());
 
         // xmlDocPtr
-        if(nullptr == (doc = xmlReadMemory(encbody.c_str(), static_cast<int>(encbody.size()), "", nullptr, 0))){
+        std::unique_ptr<xmlDoc, decltype(&xmlFreeDoc)> doc(xmlReadMemory(encbody.c_str(), static_cast<int>(encbody.size()), "", nullptr, 0), xmlFreeDoc);
+        if(nullptr == doc){
             S3FS_PRN_ERR("xmlReadMemory returns with error.");
             return -EIO;
         }
-        if(0 != append_objects_from_xml(path, doc, head)){
+        if(0 != append_objects_from_xml(path, doc.get(), head)){
             S3FS_PRN_ERR("append_objects_from_xml returns with error.");
-            xmlFreeDoc(doc);
             return -EIO;
         }
-        if(true == (truncated = is_truncated(doc))){
-            auto tmpch = get_next_continuation_token(doc);
+        if(true == (truncated = is_truncated(doc.get()))){
+            auto tmpch = get_next_continuation_token(doc.get());
             if(nullptr != tmpch){
                 next_continuation_token = reinterpret_cast<const char*>(tmpch.get());
-            }else if(nullptr != (tmpch = get_next_marker(doc))){
+            }else if(nullptr != (tmpch = get_next_marker(doc.get()))){
                 next_marker = reinterpret_cast<const char*>(tmpch.get());
             }
 
@@ -3553,7 +3552,6 @@ static int list_bucket(const char* path, S3ObjList& head, const char* delimiter,
                 }
             }
         }
-        S3FS_XMLFREEDOC(doc);
 
         // reset(initialize) curl object
         s3fscurl.DestroyCurlHandle();
@@ -4042,7 +4040,7 @@ static int s3fs_getxattr(const char* path, const char* name, char* value, size_t
     const char* pvalue = xiter->second.c_str();
 
     if(0 < size){
-        if(static_cast<size_t>(size) < length){
+        if(size < length){
             // over buffer size
             return -ERANGE;
         }
@@ -4298,12 +4296,12 @@ static void* s3fs_init(struct fuse_conn_info* conn)
 
     // Investigate system capabilities
     #ifndef __APPLE__
-    if((unsigned int)conn->capable & FUSE_CAP_ATOMIC_O_TRUNC){
+    if(conn->capable & FUSE_CAP_ATOMIC_O_TRUNC){
          conn->want |= FUSE_CAP_ATOMIC_O_TRUNC;
     }
     #endif
 
-    if((unsigned int)conn->capable & FUSE_CAP_BIG_WRITES){
+    if(conn->capable & FUSE_CAP_BIG_WRITES){
          conn->want |= FUSE_CAP_BIG_WRITES;
     }
 
@@ -4727,7 +4725,7 @@ static fsblkcnt_t parse_bucket_size(char* max_size)
         if(!isdigit(*ptr)){
             return 0;   // wrong number
         }
-        n_bytes = static_cast<unsigned long long>(strtoull(max_size, nullptr, 10));
+        n_bytes = strtoull(max_size, nullptr, 10);
         if((INT64_MAX / scale) < n_bytes){
             return 0;   // overflow
         }
@@ -5087,6 +5085,14 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
             }
             return 0;
         }
+        else if(is_prefix(arg, "ssl_client_cert=")){
+            std::string values = strchr(arg, '=') + sizeof(char);
+            if(!S3fsCurl::SetSSLClientCertOptions(values)){
+                S3FS_PRN_EXIT("failed to set SSL client certification options.");
+                return -1;
+            }
+            return 0;
+        }
         //
         // Detect options for credential
         //
@@ -5201,7 +5207,7 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
             return 0;
         }
         else if(is_prefix(arg, "multipart_size=")){
-            off_t size = static_cast<off_t>(cvt_strtoofft(strchr(arg, '=') + sizeof(char), /*base=*/ 10));
+            off_t size = cvt_strtoofft(strchr(arg, '=') + sizeof(char), /*base=*/ 10);
             if(!S3fsCurl::SetMultipartSize(size)){
                 S3FS_PRN_EXIT("multipart_size option must be at least 5 MB.");
                 return -1;
@@ -5209,7 +5215,7 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
             return 0;
         }
         else if(is_prefix(arg, "multipart_copy_size=")){
-            off_t size = static_cast<off_t>(cvt_strtoofft(strchr(arg, '=') + sizeof(char), /*base=*/ 10));
+            off_t size = cvt_strtoofft(strchr(arg, '=') + sizeof(char), /*base=*/ 10);
             if(!S3fsCurl::SetMultipartCopySize(size)){
                 S3FS_PRN_EXIT("multipart_copy_size option must be at least 5 MB.");
                 return -1;
@@ -5217,7 +5223,7 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
             return 0;
         }
         else if(is_prefix(arg, "max_dirty_data=")){
-            off_t size = static_cast<off_t>(cvt_strtoofft(strchr(arg, '=') + sizeof(char), /*base=*/ 10));
+            off_t size = cvt_strtoofft(strchr(arg, '=') + sizeof(char), /*base=*/ 10);
             if(size >= 50){
                 size *= 1024 * 1024;
             }else if(size != -1){
@@ -5436,6 +5442,14 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
             }
             return 0;
         }
+        else if(is_prefix(arg, "ipresolve=")){
+            const char* pipresolve = &arg[strlen("ipresolve=")];
+            if(!S3fsCurl::SetIPResolveType(pipresolve)){
+                S3FS_PRN_EXIT("failed to ip resolve option value(%s).", pipresolve);
+                return -1;
+            }
+            return 0;
+        }
         //
         // log file option
         //

src/s3fs.h

@@ -59,27 +59,6 @@
 #define S3FS_MALLOCTRIM(pad)
 #endif  // S3FS_MALLOC_TRIM
 
-#define S3FS_XMLFREEDOC(doc) \
-        do{ \
-          xmlFreeDoc(doc); \
-          S3FS_MALLOCTRIM(0); \
-        }while(0)
-#define S3FS_XMLFREE(ptr) \
-        do{ \
-          xmlFree(ptr); \
-          S3FS_MALLOCTRIM(0); \
-        }while(0)
-#define S3FS_XMLXPATHFREECONTEXT(ctx) \
-        do{ \
-          xmlXPathFreeContext(ctx); \
-          S3FS_MALLOCTRIM(0); \
-        }while(0)
-#define S3FS_XMLXPATHFREEOBJECT(obj) \
-        do{ \
-          xmlXPathFreeObject(obj); \
-          S3FS_MALLOCTRIM(0); \
-        }while(0)
-
 #endif // S3FS_S3FS_H_
 
 /*

src/s3fs_help.cpp

@@ -217,6 +217,21 @@ static constexpr char help_string[] =
     "   ssl_verify_hostname (default=\"2\")\n"
     "      - When 0, do not verify the SSL certificate against the hostname.\n"
     "\n"
+    "   ssl_client_cert (default=\"\")\n"
+    "      - Specify an SSL client certificate.\n"
+    "      Specify this optional parameter in the following format:\n"
+    "      \"<SSL Cert>[:<Cert Type>[:<Private Key>[:<Key Type>\n"
+    "                                                   [:<Password>]]]]\"\n"
+    "      <SSL Cert>:    Client certificate.\n"
+    "                     Specify the file path or NickName(for NSS, etc.).\n"
+    "      <Cert Type>:   Type of certificate, default is \"PEM\"(optional).\n"
+    "      <Private Key>: Certificate's private key file(optional).\n"
+    "      <Key Type>:    Type of private key, default is \"PEM\"(optional).\n"
+    "      <Password>:    Passphrase of the private key(optional).\n"
+    "                     It is also possible to omit this value and specify\n"
+    "                     it using the environment variable\n"
+    "                     \"S3FS_SSL_PRIVKEY_PASSWORD\".\n"
+    "\n"
     "   nodnscache (disable DNS cache)\n"
     "      - s3fs is always using DNS cache, this option make DNS cache disable.\n"
     "\n"
@@ -527,6 +542,14 @@ static constexpr char help_string[] =
     "        Separate the username and passphrase with a ':' character and\n"
     "        specify each as a URL-encoded string.\n"
     "\n"
+    "   ipresolve (default=\"whatever\")\n"
+    "        Select what type of IP addresses to use when establishing a\n"
+    "        connection.\n"
+    "        Default('whatever') can use addresses of all IP versions(IPv4 and\n"
+    "        IPv6) that your system allows. If you specify 'IPv4', only IPv4\n"
+    "        addresses are used. And when 'IPv6'is specified, only IPv6 addresses\n"
+    "        will be used.\n"
+    "\n"
     "   logfile - specify the log output file.\n"
     "        s3fs outputs the log file to syslog. Alternatively, if s3fs is\n"
     "        started with the \"-f\" option specified, the log will be output\n"

src/s3fs_xml.cpp

@@ -65,15 +65,14 @@ static bool GetXmlNsUrl(xmlDocPtr doc, std::string& nsurl)
             strNs  = "";
             xmlNodePtr pRootNode = xmlDocGetRootElement(doc);
             if(pRootNode){
-                xmlNsPtr* nslist = xmlGetNsList(doc, pRootNode);
+                std::unique_ptr<xmlNsPtr, decltype(xmlFree)> nslist(xmlGetNsList(doc, pRootNode), xmlFree);
                 if(nslist){
-                    if(nslist[0] && nslist[0]->href){
-                        int len = xmlStrlen(nslist[0]->href);
+                    if(*nslist && (*nslist)[0].href){
+                        int len = xmlStrlen((*nslist)[0].href);
                         if(0 < len){
-                            strNs = std::string(reinterpret_cast<const char*>(nslist[0]->href), len);
+                            strNs = std::string(reinterpret_cast<const char*>((*nslist)[0].href), len);
                         }
                     }
-                    S3FS_XMLFREE(nslist);
                 }
             }
         }

src/s3objlist.cpp

@@ -197,7 +197,7 @@ bool S3ObjList::GetLastName(std::string& lastname) const
     bool result = false;
     lastname = "";
     for(s3obj_t::const_iterator iter = objects.begin(); iter != objects.end(); ++iter){
-        if((*iter).second.orgname.length()){
+        if(!iter->second.orgname.empty()){
             if(lastname.compare(iter->second.orgname) < 0){
                 lastname = (*iter).second.orgname;
                 result = true;

src/threadpoolman.cpp

@@ -53,13 +53,14 @@ void ThreadPoolMan::Destroy()
     }
 }
 
-bool ThreadPoolMan::Instruct(std::unique_ptr<thpoolman_param> pparam)
+bool ThreadPoolMan::Instruct(const thpoolman_param& param)
 {
     if(!ThreadPoolMan::singleton){
         S3FS_PRN_WARN("The singleton object is not initialized yet.");
         return false;
     }
-    return ThreadPoolMan::singleton->SetInstruction(std::move(pparam));
+    ThreadPoolMan::singleton->SetInstruction(param);
+    return true;
 }
 
 //
@@ -84,30 +85,25 @@ void* ThreadPoolMan::Worker(void* arg)
         }
 
         // get instruction
-        std::unique_ptr<thpoolman_param> pparam;
+        thpoolman_param param;
         {
             AutoLock auto_lock(&(psingleton->thread_list_lock));
 
-            if(!psingleton->instruction_list.empty()){
-                pparam = std::move(psingleton->instruction_list.front());
-                psingleton->instruction_list.pop_front();
-                if(!pparam){
-                    S3FS_PRN_WARN("Got a semaphore, but the instruction is empty.");
-                }
+            if(psingleton->instruction_list.empty()){
+                S3FS_PRN_DBG("Got a semaphore, but the instruction is empty.");
+                continue;
             }else{
-                S3FS_PRN_WARN("Got a semaphore, but there is no instruction.");
-                pparam = nullptr;
+                param = psingleton->instruction_list.front();
+                psingleton->instruction_list.pop_front();
             }
         }
 
-        if(pparam){
-            void* retval     = pparam->pfunc(pparam->args);
-            if(nullptr != retval){
-                S3FS_PRN_WARN("The instruction function returned with somthign error code(%ld).", reinterpret_cast<long>(retval));
-            }
-            if(pparam->psem){
-                pparam->psem->post();
-            }
+        void* retval = param.pfunc(param.args);
+        if(nullptr != retval){
+            S3FS_PRN_WARN("The instruction function returned with somthign error code(%ld).", reinterpret_cast<long>(retval));
+        }
+        if(param.psem){
+            param.psem->post();
         }
     }
 
@@ -235,23 +231,16 @@ bool ThreadPoolMan::StartThreads(int count)
     return true;
 }
 
-bool ThreadPoolMan::SetInstruction(std::unique_ptr<thpoolman_param> pparam)
+void ThreadPoolMan::SetInstruction(const thpoolman_param& param)
 {
-    if(!pparam){
-        S3FS_PRN_ERR("The parameter value is nullptr.");
-        return false;
-    }
-
     // set parameter to list
     {
         AutoLock auto_lock(&thread_list_lock);
-        instruction_list.push_back(std::move(pparam));
+        instruction_list.push_back(param);
     }
 
     // run thread
     thpoolman_sem.post();
-
-    return true;
 }
 
 /*

src/threadpoolman.h

@@ -23,7 +23,6 @@
 
 #include <atomic>
 #include <list>
-#include <memory>
 #include <vector>
 
 #include "psemaphore.h"
@@ -53,7 +52,7 @@ struct thpoolman_param
     thpoolman_param() : args(nullptr), psem(nullptr), pfunc(nullptr) {}
 };
 
-typedef std::list<std::unique_ptr<thpoolman_param>> thpoolman_params_t;
+typedef std::list<thpoolman_param> thpoolman_params_t;
 
 typedef std::vector<pthread_t> thread_list_t;
 
@@ -89,12 +88,12 @@ class ThreadPoolMan
 
         bool StopThreads();
         bool StartThreads(int count);
-        bool SetInstruction(std::unique_ptr<thpoolman_param> pparam);
+        void SetInstruction(const thpoolman_param& pparam);
 
     public:
         static bool Initialize(int count);
         static void Destroy();
-        static bool Instruct(std::unique_ptr<thpoolman_param> pparam);
+        static bool Instruct(const thpoolman_param& pparam);
 };
 
 #endif // S3FS_THREADPOOLMAN_H_