From d1388ff446b74e82483f8a09b1d576cd958d4d64 Mon Sep 17 00:00:00 2001 From: Takeshi Nakatani Date: Sun, 4 Dec 2022 16:09:09 +0000 Subject: [PATCH] Added proxy and proxy_cred_file option --- doc/man/s3fs.1.in | 14 +++ src/curl.cpp | 145 ++++++++++++++++++++++++++++++++ src/curl.h | 5 ++ src/s3fs.cpp | 16 ++++ src/s3fs_help.cpp | 20 +++++ test/integration-test-common.sh | 46 ++++++++-- test/s3proxy_http.conf | 8 ++ 7 files changed, 247 insertions(+), 7 deletions(-) create mode 100644 test/s3proxy_http.conf diff --git a/doc/man/s3fs.1.in b/doc/man/s3fs.1.in index ef3b673..72690e5 100644 --- a/doc/man/s3fs.1.in +++ b/doc/man/s3fs.1.in @@ -382,6 +382,20 @@ Specify the path of the mime.types file. If this option is not specified, the existence of "/etc/mime.types" is checked, and that file is loaded as mime information. If this file does not exist on macOS, then "/etc/apache2/mime.types" is checked as well. .TP +\fB\-o\fR proxy (default="") +This option specifies a proxy to S3 server. +Specify the proxy with '[]' formatted. +'://' can be omitted, and 'http://' is used when omitted. +Also, ':' can also be omitted. If omitted, port 443 is used for HTTPS schema, and port 1080 is used otherwise. +This option is the same as the curl command's '--proxy(-x)' option and libcurl's 'CURLOPT_PROXY' flag. +This option is equivalent to and takes precedence over the environment variables 'http_proxy', 'all_proxy', etc. +.TP +\fB\-o\fR proxy_cred_file (default="") +This option specifies the file that describes the username and passphrase for authentication of the proxy when the HTTP schema proxy is specified by the 'proxy' option. +Username and passphrase are valid only for HTTP schema. +If the HTTP proxy does not require authentication, this option is not required. +Separate the username and passphrase with a ':' character and specify each as a URL-encoded string. +.TP \fB\-o\fR logfile - specify the log output file. s3fs outputs the log file to syslog. Alternatively, if s3fs is started with the "-f" option specified, the log will be output to the stdout/stderr. You can use this option to specify the log file that s3fs outputs. diff --git a/src/curl.cpp b/src/curl.cpp index bac9f4a..cb308d7 100644 --- a/src/curl.cpp +++ b/src/curl.cpp @@ -123,6 +123,9 @@ bool S3fsCurl::is_unsigned_payload = false; // default bool S3fsCurl::is_ua = true; // default bool S3fsCurl::listobjectsv2 = false; // default bool S3fsCurl::requester_pays = false; // default +std::string S3fsCurl::proxy_url; +bool S3fsCurl::proxy_http = false; +std::string S3fsCurl::proxy_userpwd; //------------------------------------------------------------------- // Class methods for S3fsCurl @@ -1053,6 +1056,134 @@ int S3fsCurl::SetMaxMultiRequest(int max) return old; } +// [NOTE] +// This proxy setting is as same as the "--proxy" option of the curl command, +// and equivalent to the "CURLOPT_PROXY" option of the curl_easy_setopt() +// function. +// However, currently s3fs does not provide another option to set the schema +// and port, so you need to specify these it in this function. (Other than +// this function, there is no means of specifying the schema and port.) +// Therefore, it should be specified "url" as "[://][:]". +// s3fs passes this string to curl_easy_setopt() function with "CURLOPT_PROXY". +// If no "schema" is specified, "http" will be used as default, and if no port +// is specified, "443" will be used for "HTTPS" and "1080" otherwise. +// (See the description of "CURLOPT_PROXY" in libcurl document.) +// +bool S3fsCurl::SetProxy(const char* url) +{ + if(!url || '\0' == url[0]){ + return false; + } + std::string tmpurl = url; + + // check schema + bool is_http = true; + size_t pos = 0; + if(std::string::npos != (pos = tmpurl.find("://", pos))){ + if(0 == pos){ + // no schema string before "://" + return false; + } + pos += strlen("://"); + + // Check if it is other than "http://" + if(0 != tmpurl.find("http://", 0)){ + is_http = false; + } + }else{ + // not have schema string + pos = 0; + } + // check fqdn and port number string + if(std::string::npos != (pos = tmpurl.find(":", pos))){ + // specify port + if(0 == pos){ + // no fqdn(hostname) string before ":" + return false; + } + pos += strlen(":"); + if(std::string::npos != tmpurl.find(":", pos)){ + // found wrong separator + return false; + } + } + + S3fsCurl::proxy_url = tmpurl; + S3fsCurl::proxy_http = is_http; + return true; +} + +// [NOTE] +// This function loads proxy credentials(username and passphrase) +// from a file. +// The loaded values is set to "CURLOPT_PROXYUSERPWD" in the +// curl_easy_setopt() function. (But only used if the proxy is HTTP +// schema.) +// +// The file is expected to contain only one valid line: +// ------------------------ +// # comment line +// : +// ------------------------ +// Lines starting with a '#' character are treated as comments. +// Lines with only space characters and blank lines are ignored. +// If the user name contains spaces, it must be url encoded(ex. %20). +// +bool S3fsCurl::SetProxyUserPwd(const char* file) +{ + if(!file || '\0' == file[0]){ + return false; + } + if(!S3fsCurl::proxy_userpwd.empty()){ + S3FS_PRN_WARN("Already set username and passphrase for proxy."); + return false; + } + + std::ifstream credFileStream(file); + if(!credFileStream.good()){ + S3FS_PRN_WARN("Could not load username and passphrase for proxy from %s.", file); + return false; + } + + std::string userpwd; + std::string line; + while(getline(credFileStream, line)){ + line = trim(line); + if(line.empty()){ + continue; + } + if(line[0]=='#'){ + continue; + } + if(!userpwd.empty()){ + S3FS_PRN_WARN("Multiple valid username and passphrase found in %s file. Should specify only one pair.", file); + return false; + } + // check separator for username and passphrase + size_t pos = 0; + if(std::string::npos == (pos = line.find(':', pos))){ + S3FS_PRN_WARN("Found string for username and passphrase in %s file does not have separator ':'.", file); + return false; + } + if(0 == pos || (pos + 1) == line.length()){ + S3FS_PRN_WARN("Found string for username or passphrase in %s file is empty.", file); + return false; + } + if(std::string::npos != line.find(':', ++pos)){ + S3FS_PRN_WARN("Found string for username and passphrase in %s file has multiple separator ':'.", file); + return false; + } + userpwd = line; + } + if(userpwd.empty()){ + S3FS_PRN_WARN("No valid username and passphrase found in %s.", file); + return false; + } + + S3fsCurl::proxy_userpwd = userpwd; + return true; +} + // cppcheck-suppress unmatchedSuppression // cppcheck-suppress constParameter bool S3fsCurl::UploadMultipartPostCallback(S3fsCurl* s3fscurl, void* param) @@ -1879,6 +2010,20 @@ bool S3fsCurl::ResetHandle(AutoLock::Type locktype) return false; } } + if(!S3fsCurl::proxy_url.empty()){ + if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_PROXY, S3fsCurl::proxy_url.c_str())){ + return false; + } + if(S3fsCurl::proxy_http){ + if(!S3fsCurl::proxy_userpwd.empty()){ + if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_PROXYUSERPWD, S3fsCurl::proxy_userpwd.c_str())){ + return false; + } + } + }else if(!S3fsCurl::proxy_userpwd.empty()){ + S3FS_PRN_DBG("Username and passphrase are specified even though proxy is not 'http' scheme, so skip to set those."); + } + } AutoLock lock(&S3fsCurl::curl_handles_lock, locktype); S3fsCurl::curl_times[hCurl] = time(0); diff --git a/src/curl.h b/src/curl.h index 075bbd5..67b64ac 100644 --- a/src/curl.h +++ b/src/curl.h @@ -154,6 +154,9 @@ class S3fsCurl static bool is_ua; // User-Agent static bool listobjectsv2; static bool requester_pays; + static std::string proxy_url; + static bool proxy_http; + static std::string proxy_userpwd; // load from file(:) // variables CURL* hCurl; @@ -332,6 +335,8 @@ class S3fsCurl static bool IsListObjectsV2() { return S3fsCurl::listobjectsv2; } static bool SetRequesterPays(bool flag) { bool old_flag = S3fsCurl::requester_pays; S3fsCurl::requester_pays = flag; return old_flag; } static bool IsRequesterPays() { return S3fsCurl::requester_pays; } + static bool SetProxy(const char* url); + static bool SetProxyUserPwd(const char* userpwd); // methods bool CreateCurlHandle(bool only_pool = false, bool remake = false); diff --git a/src/s3fs.cpp b/src/s3fs.cpp index 2fe5913..f5e4f43 100644 --- a/src/s3fs.cpp +++ b/src/s3fs.cpp @@ -4843,6 +4843,22 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar mimetype_file = strchr(arg, '=') + sizeof(char); return 0; } + if(is_prefix(arg, "proxy=")){ + const char* url = &arg[strlen("proxy=")]; + if(!S3fsCurl::SetProxy(url)){ + S3FS_PRN_EXIT("failed to set proxy(%s).", url); + return -1; + } + return 0; + } + if(is_prefix(arg, "proxy_cred_file=")){ + const char* file = &arg[strlen("proxy_cred_file=")]; + if(!S3fsCurl::SetProxyUserPwd(file)){ + S3FS_PRN_EXIT("failed to set proxy user and passphrase from file(%s).", file); + return -1; + } + return 0; + } // // log file option // diff --git a/src/s3fs_help.cpp b/src/s3fs_help.cpp index 5db4b3d..843a772 100644 --- a/src/s3fs_help.cpp +++ b/src/s3fs_help.cpp @@ -483,6 +483,26 @@ static const char help_string[] = " If this file does not exist on macOS, then \"/etc/apache2/mime.types\"\n" " is checked as well.\n" "\n" + " proxy (default=\"\")\n" + " This option specifies a proxy to S3 server.\n" + " Specify the proxy with '[]' formatted.\n" + " '://' can be omitted, and 'http://' is used when omitted.\n" + " Also, ':' can also be omitted. If omitted, port 443 is used for\n" + " HTTPS schema, and port 1080 is used otherwise.\n" + " This option is the same as the curl command's '--proxy(-x)' option and\n" + " libcurl's 'CURLOPT_PROXY' flag.\n" + " This option is equivalent to and takes precedence over the environment\n" + " variables 'http_proxy', 'all_proxy', etc.\n" + "\n" + " proxy_cred_file (default=\"\")\n" + " This option specifies the file that describes the username and\n" + " passphrase for authentication of the proxy when the HTTP schema\n" + " proxy is specified by the 'proxy' option.\n" + " Username and passphrase are valid only for HTTP schema. If the HTTP\n" + " proxy does not require authentication, this option is not required.\n" + " Separate the username and passphrase with a ':' character and\n" + " specify each as a URL-encoded string.\n" + "\n" " logfile - specify the log output file.\n" " s3fs outputs the log file to syslog. Alternatively, if s3fs is\n" " started with the \"-f\" option specified, the log will be output\n" diff --git a/test/integration-test-common.sh b/test/integration-test-common.sh index 0a11985..b3c4cf7 100644 --- a/test/integration-test-common.sh +++ b/test/integration-test-common.sh @@ -34,6 +34,8 @@ # S3_ENDPOINT="us-east-1" Specify region # TMPDIR="/var/tmp" Set to use a temporary directory different # from /var/tmp +# CHAOS_HTTP_PROXY=1 Test proxy(environment) by CHAOS HTTP PROXY +# CHAOS_HTTP_PROXY_OPT=1 Test proxy(option) by CHAOS HTTP PROXY # # Example of running against Amazon S3 using a bucket named "bucket": # @@ -66,7 +68,15 @@ set -o pipefail S3FS=../src/s3fs # Allow these defaulted values to be overridden -: "${S3_URL:="https://127.0.0.1:8080"}" +# +# [NOTE] +# CHAOS HTTP PROXY does not support HTTPS. +# +if [ -z "${CHAOS_HTTP_PROXY}" ] && [ -z "${CHAOS_HTTP_PROXY_OPT}" ]; then + : "${S3_URL:="https://127.0.0.1:8080"}" +else + : "${S3_URL:="http://127.0.0.1:8080"}" +fi : "${S3_ENDPOINT:="us-east-1"}" : "${S3FS_CREDENTIALS_FILE:="passwd-s3fs"}" : "${TEST_BUCKET_1:="s3fs-integration-test"}" @@ -135,7 +145,11 @@ function start_s3proxy { if [ -n "${PUBLIC}" ]; then local S3PROXY_CONFIG="s3proxy-noauth.conf" else - local S3PROXY_CONFIG="s3proxy.conf" + if [ -z "${CHAOS_HTTP_PROXY}" ] && [ -z "${CHAOS_HTTP_PROXY_OPT}" ]; then + local S3PROXY_CONFIG="s3proxy.conf" + else + local S3PROXY_CONFIG="s3proxy_http.conf" + fi fi if [ -n "${S3PROXY_BINARY}" ] @@ -147,9 +161,18 @@ function start_s3proxy { fi # generate self-signed SSL certificate - rm -f /tmp/keystore.jks /tmp/keystore.pem - echo -e 'password\npassword\n\n\n\n\n\n\nyes' | keytool -genkey -keystore /tmp/keystore.jks -keyalg RSA -keysize 2048 -validity 365 -ext SAN=IP:127.0.0.1 - echo password | keytool -exportcert -keystore /tmp/keystore.jks -rfc -file /tmp/keystore.pem + # + # [NOTE] + # The PROXY test is HTTP only, so do not create CA certificates. + # + if [ -z "${CHAOS_HTTP_PROXY}" ] && [ -z "${CHAOS_HTTP_PROXY_OPT}" ]; then + S3PROXY_CACERT_FILE="/tmp/keystore.pem" + rm -f /tmp/keystore.jks "${S3PROXY_CACERT_FILE}" + echo -e 'password\npassword\n\n\n\n\n\n\nyes' | keytool -genkey -keystore /tmp/keystore.jks -keyalg RSA -keysize 2048 -validity 365 -ext SAN=IP:127.0.0.1 + echo password | keytool -exportcert -keystore /tmp/keystore.jks -rfc -file "${S3PROXY_CACERT_FILE}" + else + S3PROXY_CACERT_FILE="" + fi "${STDBUF_BIN}" -oL -eL java -jar "${S3PROXY_BINARY}" --properties "${S3PROXY_CONFIG}" & S3PROXY_PID=$! @@ -158,7 +181,7 @@ function start_s3proxy { wait_for_port 8080 fi - if [ -n "${CHAOS_HTTP_PROXY}" ]; then + if [ -n "${CHAOS_HTTP_PROXY}" ] || [ -n "${CHAOS_HTTP_PROXY_OPT}" ]; then if [ ! -e "${CHAOS_HTTP_PROXY_BINARY}" ]; then curl "https://github.com/bouncestorage/chaos-http-proxy/releases/download/chaos-http-proxy-${CHAOS_HTTP_PROXY_VERSION}/chaos-http-proxy" \ --fail --location --silent --output "${CHAOS_HTTP_PROXY_BINARY}" @@ -212,8 +235,16 @@ function start_s3fs { local DIRECT_IO_OPT="" fi + # Set environment variables or options for proxy. + # And the PROXY test is HTTP only and does not set CA certificates. + # if [ -n "${CHAOS_HTTP_PROXY}" ]; then export http_proxy="127.0.0.1:1080" + S3FS_HTTP_PROXY_OPT="" + elif [ -n "${CHAOS_HTTP_PROXY_OPT}" ]; then + S3FS_HTTP_PROXY_OPT="-o proxy=http://127.0.0.1:1080" + else + S3FS_HTTP_PROXY_OPT="" fi # [NOTE] @@ -247,7 +278,7 @@ function start_s3fs { # shellcheck disable=SC2086 ( set -x - CURL_CA_BUNDLE=/tmp/keystore.pem \ + CURL_CA_BUNDLE="${S3PROXY_CACERT_FILE}" \ ${VIA_STDBUF_CMDLINE} \ ${VALGRIND_EXEC} \ ${S3FS} \ @@ -260,6 +291,7 @@ function start_s3fs { -o enable_unsigned_payload \ ${AUTH_OPT} \ ${DIRECT_IO_OPT} \ + ${S3FS_HTTP_PROXY_OPT} \ -o stat_cache_expire=1 \ -o stat_cache_interval_expire=1 \ -o dbglevel="${DBGLEVEL:=info}" \ diff --git a/test/s3proxy_http.conf b/test/s3proxy_http.conf new file mode 100644 index 0000000..62f7023 --- /dev/null +++ b/test/s3proxy_http.conf @@ -0,0 +1,8 @@ +s3proxy.endpoint=http://127.0.0.1:8080 +s3proxy.authorization=aws-v2-or-v4 +s3proxy.identity=local-identity +s3proxy.credential=local-credential + +jclouds.provider=transient +jclouds.identity=remote-identity +jclouds.credential=remote-credential