Added proxy and proxy_cred_file option

doc/man/s3fs.1.in

@@ -382,6 +382,20 @@ Specify the path of the mime.types file.
 If this option is not specified, the existence of "/etc/mime.types" is checked, and that file is loaded as mime information.
 If this file does not exist on macOS, then "/etc/apache2/mime.types" is checked as well.
 .TP
+\fB\-o\fR proxy (default="")
+This option specifies a proxy to S3 server.
+Specify the proxy with '[<scheme://]hostname(fqdn)[:<port>]' formatted.
+'<schema>://' can be omitted, and 'http://' is used when omitted.
+Also, ':<port>' can also be omitted. If omitted, port 443 is used for HTTPS schema, and port 1080 is used otherwise.
+This option is the same as the curl command's '--proxy(-x)' option and libcurl's 'CURLOPT_PROXY' flag.
+This option is equivalent to and takes precedence over the environment variables 'http_proxy', 'all_proxy', etc.
+.TP
+\fB\-o\fR proxy_cred_file (default="")
+This option specifies the file that describes the username and passphrase for authentication of the proxy when the HTTP schema proxy is specified by the 'proxy' option.
+Username and passphrase are valid only for HTTP schema.
+If the HTTP proxy does not require authentication, this option is not required.
+Separate the username and passphrase with a ':' character and specify each as a URL-encoded string.
+.TP
 \fB\-o\fR logfile - specify the log output file.
 s3fs outputs the log file to syslog. Alternatively, if s3fs is started with the "-f" option specified, the log will be output to the stdout/stderr.
 You can use this option to specify the log file that s3fs outputs.

src/curl.cpp

@@ -123,6 +123,9 @@ bool             S3fsCurl::is_unsigned_payload = false;          // default
 bool             S3fsCurl::is_ua               = true;           // default
 bool             S3fsCurl::listobjectsv2       = false;          // default
 bool             S3fsCurl::requester_pays      = false;          // default
+std::string      S3fsCurl::proxy_url;
+bool             S3fsCurl::proxy_http          = false;
+std::string      S3fsCurl::proxy_userpwd;
 
 //-------------------------------------------------------------------
 // Class methods for S3fsCurl
@@ -1053,6 +1056,134 @@ int S3fsCurl::SetMaxMultiRequest(int max)
     return old;
 }
 
+// [NOTE]
+// This proxy setting is as same as the "--proxy" option of the curl command,
+// and equivalent to the "CURLOPT_PROXY" option of the curl_easy_setopt()
+// function.
+// However, currently s3fs does not provide another option to set the schema
+// and port, so you need to specify these it in this function. (Other than
+// this function, there is no means of specifying the schema and port.)
+// Therefore, it should be specified "url" as "[<schema>://]<fqdn>[:<port>]".
+// s3fs passes this string to curl_easy_setopt() function with "CURLOPT_PROXY".
+// If no "schema" is specified, "http" will be used as default, and if no port
+// is specified, "443" will be used for "HTTPS" and "1080" otherwise.
+// (See the description of "CURLOPT_PROXY" in libcurl document.)
+//
+bool S3fsCurl::SetProxy(const char* url)
+{
+    if(!url || '\0' == url[0]){
+        return false;
+    }
+    std::string tmpurl = url;
+
+    // check schema
+    bool   is_http = true;
+    size_t pos = 0;
+    if(std::string::npos != (pos = tmpurl.find("://", pos))){
+        if(0 == pos){
+            // no schema string before "://"
+            return false;
+        }
+        pos += strlen("://");
+
+        // Check if it is other than "http://"
+        if(0 != tmpurl.find("http://", 0)){
+            is_http = false;
+        }
+    }else{
+        // not have schema string
+        pos = 0;
+    }
+    // check fqdn and port number string
+    if(std::string::npos != (pos = tmpurl.find(":", pos))){
+        // specify port
+        if(0 == pos){
+            // no fqdn(hostname) string before ":"
+            return false;
+        }
+        pos += strlen(":");
+        if(std::string::npos != tmpurl.find(":", pos)){
+            // found wrong separator
+            return false;
+        }
+    }
+
+    S3fsCurl::proxy_url  = tmpurl;
+    S3fsCurl::proxy_http = is_http;
+    return true;
+}
+
+// [NOTE]
+// This function loads proxy credentials(username and  passphrase)
+// from a file.
+// The loaded values is set to "CURLOPT_PROXYUSERPWD" in the
+// curl_easy_setopt() function. (But only used if the proxy is HTTP
+// schema.)
+//
+// The file is expected to contain only one valid line:
+//     ------------------------
+//     # comment line
+//     <username>:<passphrase>
+//     ------------------------
+// Lines starting with a '#' character are treated as comments.
+// Lines with only space characters and blank lines are ignored.
+// If the user name contains spaces, it must be url encoded(ex. %20).
+//
+bool S3fsCurl::SetProxyUserPwd(const char* file)
+{
+    if(!file || '\0' == file[0]){
+        return false;
+    }
+    if(!S3fsCurl::proxy_userpwd.empty()){
+        S3FS_PRN_WARN("Already set username and passphrase for proxy.");
+        return false;
+    }
+
+    std::ifstream credFileStream(file);
+    if(!credFileStream.good()){
+        S3FS_PRN_WARN("Could not load username and passphrase for proxy from %s.", file);
+        return false;
+    }
+
+    std::string userpwd;
+    std::string line;
+    while(getline(credFileStream, line)){
+        line = trim(line);
+        if(line.empty()){
+            continue;
+        }
+        if(line[0]=='#'){
+            continue;
+        }
+        if(!userpwd.empty()){
+            S3FS_PRN_WARN("Multiple valid username and passphrase found in %s file. Should specify only one pair.", file);
+            return false;
+        }
+        // check separator for username and passphrase
+        size_t pos = 0;
+        if(std::string::npos == (pos = line.find(':', pos))){
+            S3FS_PRN_WARN("Found string for username and passphrase in %s file does not have separator ':'.", file);
+            return false;
+        }
+        if(0 == pos || (pos + 1) == line.length()){
+            S3FS_PRN_WARN("Found string for username or passphrase in %s file is empty.", file);
+            return false;
+        }
+        if(std::string::npos != line.find(':', ++pos)){
+            S3FS_PRN_WARN("Found string for username and passphrase in %s file has multiple separator ':'.", file);
+            return false;
+        }
+        userpwd = line;
+    }
+    if(userpwd.empty()){
+        S3FS_PRN_WARN("No valid username and passphrase found in %s.", file);
+        return false;
+    }
+
+    S3fsCurl::proxy_userpwd = userpwd;
+    return true;
+}
+
 // cppcheck-suppress unmatchedSuppression
 // cppcheck-suppress constParameter
 bool S3fsCurl::UploadMultipartPostCallback(S3fsCurl* s3fscurl, void* param)
@@ -1879,6 +2010,20 @@ bool S3fsCurl::ResetHandle(AutoLock::Type locktype)
             return false;
         }
     }
+    if(!S3fsCurl::proxy_url.empty()){
+        if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_PROXY, S3fsCurl::proxy_url.c_str())){
+            return false;
+        }
+        if(S3fsCurl::proxy_http){
+            if(!S3fsCurl::proxy_userpwd.empty()){
+                if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_PROXYUSERPWD, S3fsCurl::proxy_userpwd.c_str())){
+                    return false;
+                }
+            }
+        }else if(!S3fsCurl::proxy_userpwd.empty()){
+            S3FS_PRN_DBG("Username and passphrase are specified even though proxy is not 'http' scheme, so skip to set those.");
+        }
+    }
 
     AutoLock lock(&S3fsCurl::curl_handles_lock, locktype);
     S3fsCurl::curl_times[hCurl]    = time(0);

src/curl.h

@@ -154,6 +154,9 @@ class S3fsCurl
         static bool             is_ua;             // User-Agent
         static bool             listobjectsv2;
         static bool             requester_pays;
+        static std::string      proxy_url;
+        static bool             proxy_http;
+        static std::string      proxy_userpwd;     // load from file(<username>:<passphrase>)
 
         // variables
         CURL*                hCurl;
@@ -332,6 +335,8 @@ class S3fsCurl
         static bool IsListObjectsV2() { return S3fsCurl::listobjectsv2; }
         static bool SetRequesterPays(bool flag) { bool old_flag = S3fsCurl::requester_pays; S3fsCurl::requester_pays = flag; return old_flag; }
         static bool IsRequesterPays() { return S3fsCurl::requester_pays; }
+        static bool SetProxy(const char* url);
+        static bool SetProxyUserPwd(const char* userpwd);
 
         // methods
         bool CreateCurlHandle(bool only_pool = false, bool remake = false);

src/s3fs.cpp

@@ -4843,6 +4843,22 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
             mimetype_file = strchr(arg, '=') + sizeof(char);
             return 0;
         }
+        if(is_prefix(arg, "proxy=")){
+            const char* url = &arg[strlen("proxy=")];
+            if(!S3fsCurl::SetProxy(url)){
+                S3FS_PRN_EXIT("failed to set proxy(%s).", url);
+                return -1;
+            }
+            return 0;
+        }
+        if(is_prefix(arg, "proxy_cred_file=")){
+            const char* file = &arg[strlen("proxy_cred_file=")];
+            if(!S3fsCurl::SetProxyUserPwd(file)){
+                S3FS_PRN_EXIT("failed to set proxy user and passphrase from file(%s).", file);
+                return -1;
+            }
+            return 0;
+        }
         //
         // log file option
         //

src/s3fs_help.cpp

@@ -483,6 +483,26 @@ static const char help_string[] =
     "        If this file does not exist on macOS, then \"/etc/apache2/mime.types\"\n"
     "        is checked as well.\n"
     "\n"
+    "   proxy (default=\"\")\n"
+    "        This option specifies a proxy to S3 server.\n"
+    "        Specify the proxy with '[<scheme://]hostname(fqdn)[:<port>]' formatted.\n"
+    "        '<schema>://' can be omitted, and 'http://' is used when omitted.\n"
+    "        Also, ':<port>' can also be omitted. If omitted, port 443 is used for\n"
+    "        HTTPS schema, and port 1080 is used otherwise.\n"
+    "        This option is the same as the curl command's '--proxy(-x)' option and\n"
+    "        libcurl's 'CURLOPT_PROXY' flag.\n"
+    "        This option is equivalent to and takes precedence over the environment\n"
+    "        variables 'http_proxy', 'all_proxy', etc.\n"
+    "\n"
+    "   proxy_cred_file (default=\"\")\n"
+    "        This option specifies the file that describes the username and\n"
+    "        passphrase for authentication of the proxy when the HTTP schema\n"
+    "        proxy is specified by the 'proxy' option.\n"
+    "        Username and passphrase are valid only for HTTP schema. If the HTTP\n"
+    "        proxy does not require authentication, this option is not required.\n"
+    "        Separate the username and passphrase with a ':' character and\n"
+    "        specify each as a URL-encoded string.\n"
+    "\n"
     "   logfile - specify the log output file.\n"
     "        s3fs outputs the log file to syslog. Alternatively, if s3fs is\n"
     "        started with the \"-f\" option specified, the log will be output\n"

test/integration-test-common.sh

@@ -34,6 +34,8 @@
 # S3_ENDPOINT="us-east-1"            Specify region
 # TMPDIR="/var/tmp"                  Set to use a temporary directory different
 #                                    from /var/tmp
+# CHAOS_HTTP_PROXY=1                 Test proxy(environment) by CHAOS HTTP PROXY
+# CHAOS_HTTP_PROXY_OPT=1             Test proxy(option) by CHAOS HTTP PROXY
 #
 # Example of running against Amazon S3 using a bucket named "bucket":
 #
@@ -66,7 +68,15 @@ set -o pipefail
 S3FS=../src/s3fs
 
 # Allow these defaulted values to be overridden
-: "${S3_URL:="https://127.0.0.1:8080"}"
+#
+# [NOTE]
+# CHAOS HTTP PROXY does not support HTTPS.
+#
+if [ -z "${CHAOS_HTTP_PROXY}" ] && [ -z "${CHAOS_HTTP_PROXY_OPT}" ]; then
+    : "${S3_URL:="https://127.0.0.1:8080"}"
+else
+    : "${S3_URL:="http://127.0.0.1:8080"}"
+fi
 : "${S3_ENDPOINT:="us-east-1"}"
 : "${S3FS_CREDENTIALS_FILE:="passwd-s3fs"}"
 : "${TEST_BUCKET_1:="s3fs-integration-test"}"
@@ -135,7 +145,11 @@ function start_s3proxy {
     if [ -n "${PUBLIC}" ]; then
         local S3PROXY_CONFIG="s3proxy-noauth.conf"
     else
-        local S3PROXY_CONFIG="s3proxy.conf"
+        if [ -z "${CHAOS_HTTP_PROXY}" ] && [ -z "${CHAOS_HTTP_PROXY_OPT}" ]; then
+            local S3PROXY_CONFIG="s3proxy.conf"
+        else
+            local S3PROXY_CONFIG="s3proxy_http.conf"
+        fi
     fi
 
     if [ -n "${S3PROXY_BINARY}" ]
@@ -147,9 +161,18 @@ function start_s3proxy {
         fi
 
         # generate self-signed SSL certificate
-        rm -f /tmp/keystore.jks /tmp/keystore.pem
+        #
-        echo -e 'password\npassword\n\n\n\n\n\n\nyes' | keytool -genkey -keystore /tmp/keystore.jks -keyalg RSA -keysize 2048 -validity 365 -ext SAN=IP:127.0.0.1
+        # [NOTE]
-        echo password | keytool -exportcert -keystore /tmp/keystore.jks -rfc -file /tmp/keystore.pem
+        # The PROXY test is HTTP only, so do not create CA certificates.
+        #
+        if [ -z "${CHAOS_HTTP_PROXY}" ] && [ -z "${CHAOS_HTTP_PROXY_OPT}" ]; then
+            S3PROXY_CACERT_FILE="/tmp/keystore.pem"
+            rm -f /tmp/keystore.jks "${S3PROXY_CACERT_FILE}"
+            echo -e 'password\npassword\n\n\n\n\n\n\nyes' | keytool -genkey -keystore /tmp/keystore.jks -keyalg RSA -keysize 2048 -validity 365 -ext SAN=IP:127.0.0.1
+            echo password | keytool -exportcert -keystore /tmp/keystore.jks -rfc -file "${S3PROXY_CACERT_FILE}"
+        else
+            S3PROXY_CACERT_FILE=""
+        fi
 
         "${STDBUF_BIN}" -oL -eL java -jar "${S3PROXY_BINARY}" --properties "${S3PROXY_CONFIG}" &
         S3PROXY_PID=$!
@@ -158,7 +181,7 @@ function start_s3proxy {
         wait_for_port 8080
     fi
 
-    if [ -n "${CHAOS_HTTP_PROXY}" ]; then
+    if [ -n "${CHAOS_HTTP_PROXY}" ] || [ -n "${CHAOS_HTTP_PROXY_OPT}" ]; then
         if [ ! -e "${CHAOS_HTTP_PROXY_BINARY}" ]; then
             curl "https://github.com/bouncestorage/chaos-http-proxy/releases/download/chaos-http-proxy-${CHAOS_HTTP_PROXY_VERSION}/chaos-http-proxy" \
                 --fail --location --silent --output "${CHAOS_HTTP_PROXY_BINARY}"
@@ -212,8 +235,16 @@ function start_s3fs {
        local DIRECT_IO_OPT=""
     fi
 
+    # Set environment variables or options for proxy.
+    # And the PROXY test is HTTP only and does not set CA certificates.
+    #
     if [ -n "${CHAOS_HTTP_PROXY}" ]; then
         export http_proxy="127.0.0.1:1080"
+        S3FS_HTTP_PROXY_OPT=""
+    elif [ -n "${CHAOS_HTTP_PROXY_OPT}" ]; then
+        S3FS_HTTP_PROXY_OPT="-o proxy=http://127.0.0.1:1080"
+    else
+        S3FS_HTTP_PROXY_OPT=""
     fi
 
     # [NOTE]
@@ -247,7 +278,7 @@ function start_s3fs {
     # shellcheck disable=SC2086
     (
         set -x 
-        CURL_CA_BUNDLE=/tmp/keystore.pem \
+        CURL_CA_BUNDLE="${S3PROXY_CACERT_FILE}" \
         ${VIA_STDBUF_CMDLINE} \
             ${VALGRIND_EXEC} \
             ${S3FS} \
@@ -260,6 +291,7 @@ function start_s3fs {
             -o enable_unsigned_payload \
             ${AUTH_OPT} \
             ${DIRECT_IO_OPT} \
+            ${S3FS_HTTP_PROXY_OPT} \
             -o stat_cache_expire=1 \
             -o stat_cache_interval_expire=1 \
             -o dbglevel="${DBGLEVEL:=info}" \

test/s3proxy_http.conf

@@ -0,0 +1,8 @@
+s3proxy.endpoint=http://127.0.0.1:8080
+s3proxy.authorization=aws-v2-or-v4
+s3proxy.identity=local-identity
+s3proxy.credential=local-credential
+
+jclouds.provider=transient
+jclouds.identity=remote-identity
+jclouds.credential=remote-credential