octoleo/s3fs-fuse
1
0
Fork 0
mirror of https://github.com/s3fs-fuse/s3fs-fuse.git synced 2024-12-22 00:38:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

Added proxy and proxy_cred_file option

Browse Source
This commit is contained in:
Takeshi Nakatani 2022-12-04 16:09:09 +00:00 committed by Andrew Gaul
parent 38e8a830c9
commit d1388ff446
7 changed files with 247 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
doc/man/s3fs.1.in
View File

@ -382,6 +382,20 @@ Specify the path of the mime.types file.
If this option is not specified, the existence of "/etc/mime.types" is checked, and that file is loaded as mime information.
If this file does not exist on macOS, then "/etc/apache2/mime.types" is checked as well.
.TP
\fB\-o\fR proxy (default="")
This option specifies a proxy to S3 server.
Specify the proxy with '[<scheme://]hostname(fqdn)[:<port>]' formatted.
'<schema>://' can be omitted, and 'http://' is used when omitted.
Also, ':<port>' can also be omitted. If omitted, port 443 is used for HTTPS schema, and port 1080 is used otherwise.
This option is the same as the curl command's '--proxy(-x)' option and libcurl's 'CURLOPT_PROXY' flag.
This option is equivalent to and takes precedence over the environment variables 'http_proxy', 'all_proxy', etc.
.TP
\fB\-o\fR proxy_cred_file (default="")
This option specifies the file that describes the username and passphrase for authentication of the proxy when the HTTP schema proxy is specified by the 'proxy' option.
Username and passphrase are valid only for HTTP schema.
If the HTTP proxy does not require authentication, this option is not required.
Separate the username and passphrase with a ':' character and specify each as a URL-encoded string.
.TP
\fB\-o\fR logfile - specify the log output file.
s3fs outputs the log file to syslog. Alternatively, if s3fs is started with the "-f" option specified, the log will be output to the stdout/stderr.
You can use this option to specify the log file that s3fs outputs.

145
src/curl.cpp
View File

@ -123,6 +123,9 @@ bool S3fsCurl::is_unsigned_payload = false; // default
bool S3fsCurl::is_ua = true; // default
bool S3fsCurl::listobjectsv2 = false; // default
bool S3fsCurl::requester_pays = false; // default
std::string S3fsCurl::proxy_url;
bool S3fsCurl::proxy_http = false;
std::string S3fsCurl::proxy_userpwd;
//-------------------------------------------------------------------
// Class methods for S3fsCurl
@ -1053,6 +1056,134 @@ int S3fsCurl::SetMaxMultiRequest(int max)
return old;
}
// [NOTE]
// This proxy setting is as same as the "--proxy" option of the curl command,
// and equivalent to the "CURLOPT_PROXY" option of the curl_easy_setopt()
// function.
// However, currently s3fs does not provide another option to set the schema
// and port, so you need to specify these it in this function. (Other than
// this function, there is no means of specifying the schema and port.)
// Therefore, it should be specified "url" as "[<schema>://]<fqdn>[:<port>]".
// s3fs passes this string to curl_easy_setopt() function with "CURLOPT_PROXY".
// If no "schema" is specified, "http" will be used as default, and if no port
// is specified, "443" will be used for "HTTPS" and "1080" otherwise.
// (See the description of "CURLOPT_PROXY" in libcurl document.)
//
bool S3fsCurl::SetProxy(const char* url)
{
if(!url || '\0' == url[0]){
return false;
}
std::string tmpurl = url;
// check schema
bool is_http = true;
size_t pos = 0;
if(std::string::npos != (pos = tmpurl.find("://", pos))){
if(0 == pos){
// no schema string before "://"
return false;
}
pos += strlen("://");
// Check if it is other than "http://"
if(0 != tmpurl.find("http://", 0)){
is_http = false;
}
}else{
// not have schema string
pos = 0;
}
// check fqdn and port number string
if(std::string::npos != (pos = tmpurl.find(":", pos))){
// specify port
if(0 == pos){
// no fqdn(hostname) string before ":"
return false;
}
pos += strlen(":");
if(std::string::npos != tmpurl.find(":", pos)){
// found wrong separator
return false;
}
}
S3fsCurl::proxy_url = tmpurl;
S3fsCurl::proxy_http = is_http;
return true;
}
// [NOTE]
// This function loads proxy credentials(username and passphrase)
// from a file.
// The loaded values is set to "CURLOPT_PROXYUSERPWD" in the
// curl_easy_setopt() function. (But only used if the proxy is HTTP
// schema.)
//
// The file is expected to contain only one valid line:
// ------------------------
// # comment line
// <username>:<passphrase>
// ------------------------
// Lines starting with a '#' character are treated as comments.
// Lines with only space characters and blank lines are ignored.
// If the user name contains spaces, it must be url encoded(ex. %20).
//
bool S3fsCurl::SetProxyUserPwd(const char* file)
{
if(!file || '\0' == file[0]){
return false;
}
if(!S3fsCurl::proxy_userpwd.empty()){
S3FS_PRN_WARN("Already set username and passphrase for proxy.");
return false;
}
std::ifstream credFileStream(file);
if(!credFileStream.good()){
S3FS_PRN_WARN("Could not load username and passphrase for proxy from %s.", file);
return false;
}
std::string userpwd;
std::string line;
while(getline(credFileStream, line)){
line = trim(line);
if(line.empty()){
continue;
}
if(line[0]=='#'){
continue;
}
if(!userpwd.empty()){
S3FS_PRN_WARN("Multiple valid username and passphrase found in %s file. Should specify only one pair.", file);
return false;
}
// check separator for username and passphrase
size_t pos = 0;
if(std::string::npos == (pos = line.find(':', pos))){
S3FS_PRN_WARN("Found string for username and passphrase in %s file does not have separator ':'.", file);
return false;
}
if(0 == pos || (pos + 1) == line.length()){
S3FS_PRN_WARN("Found string for username or passphrase in %s file is empty.", file);
return false;
}
if(std::string::npos != line.find(':', ++pos)){
S3FS_PRN_WARN("Found string for username and passphrase in %s file has multiple separator ':'.", file);
return false;
}
userpwd = line;
}
if(userpwd.empty()){
S3FS_PRN_WARN("No valid username and passphrase found in %s.", file);
return false;
}
S3fsCurl::proxy_userpwd = userpwd;
return true;
}
// cppcheck-suppress unmatchedSuppression
// cppcheck-suppress constParameter
bool S3fsCurl::UploadMultipartPostCallback(S3fsCurl* s3fscurl, void* param)
@ -1879,6 +2010,20 @@ bool S3fsCurl::ResetHandle(AutoLock::Type locktype)
return false;
}
}
if(!S3fsCurl::proxy_url.empty()){
if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_PROXY, S3fsCurl::proxy_url.c_str())){
return false;
}
if(S3fsCurl::proxy_http){
if(!S3fsCurl::proxy_userpwd.empty()){
if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_PROXYUSERPWD, S3fsCurl::proxy_userpwd.c_str())){
return false;
}
}
}else if(!S3fsCurl::proxy_userpwd.empty()){
S3FS_PRN_DBG("Username and passphrase are specified even though proxy is not 'http' scheme, so skip to set those.");
}
}
AutoLock lock(&S3fsCurl::curl_handles_lock, locktype);
S3fsCurl::curl_times[hCurl] = time(0);

5
src/curl.h
View File

@ -154,6 +154,9 @@ class S3fsCurl
static bool is_ua; // User-Agent
static bool listobjectsv2;
static bool requester_pays;
static std::string proxy_url;
static bool proxy_http;
static std::string proxy_userpwd; // load from file(<username>:<passphrase>)
// variables
CURL* hCurl;
@ -332,6 +335,8 @@ class S3fsCurl
static bool IsListObjectsV2() { return S3fsCurl::listobjectsv2; }
static bool SetRequesterPays(bool flag) { bool old_flag = S3fsCurl::requester_pays; S3fsCurl::requester_pays = flag; return old_flag; }
static bool IsRequesterPays() { return S3fsCurl::requester_pays; }
static bool SetProxy(const char* url);
static bool SetProxyUserPwd(const char* userpwd);
// methods
bool CreateCurlHandle(bool only_pool = false, bool remake = false);

16
src/s3fs.cpp
View File

@ -4843,6 +4843,22 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
mimetype_file = strchr(arg, '=') + sizeof(char);
return 0;
}
if(is_prefix(arg, "proxy=")){
const char* url = &arg[strlen("proxy=")];
if(!S3fsCurl::SetProxy(url)){
S3FS_PRN_EXIT("failed to set proxy(%s).", url);
return -1;
}
return 0;
}
if(is_prefix(arg, "proxy_cred_file=")){
const char* file = &arg[strlen("proxy_cred_file=")];
if(!S3fsCurl::SetProxyUserPwd(file)){
S3FS_PRN_EXIT("failed to set proxy user and passphrase from file(%s).", file);
return -1;
}
return 0;
}
//
// log file option
//

20
src/s3fs_help.cpp
View File

@ -483,6 +483,26 @@ static const char help_string[] =
" If this file does not exist on macOS, then \"/etc/apache2/mime.types\"\n"
" is checked as well.\n"
"\n"
" proxy (default=\"\")\n"
" This option specifies a proxy to S3 server.\n"
" Specify the proxy with '[<scheme://]hostname(fqdn)[:<port>]' formatted.\n"
" '<schema>://' can be omitted, and 'http://' is used when omitted.\n"
" Also, ':<port>' can also be omitted. If omitted, port 443 is used for\n"
" HTTPS schema, and port 1080 is used otherwise.\n"
" This option is the same as the curl command's '--proxy(-x)' option and\n"
" libcurl's 'CURLOPT_PROXY' flag.\n"
" This option is equivalent to and takes precedence over the environment\n"
" variables 'http_proxy', 'all_proxy', etc.\n"
"\n"
" proxy_cred_file (default=\"\")\n"
" This option specifies the file that describes the username and\n"
" passphrase for authentication of the proxy when the HTTP schema\n"
" proxy is specified by the 'proxy' option.\n"
" Username and passphrase are valid only for HTTP schema. If the HTTP\n"
" proxy does not require authentication, this option is not required.\n"
" Separate the username and passphrase with a ':' character and\n"
" specify each as a URL-encoded string.\n"
"\n"
" logfile - specify the log output file.\n"
" s3fs outputs the log file to syslog. Alternatively, if s3fs is\n"
" started with the \"-f\" option specified, the log will be output\n"

46
test/integration-test-common.sh
View File

@ -34,6 +34,8 @@
# S3_ENDPOINT="us-east-1" Specify region
# TMPDIR="/var/tmp" Set to use a temporary directory different
# from /var/tmp
# CHAOS_HTTP_PROXY=1 Test proxy(environment) by CHAOS HTTP PROXY
# CHAOS_HTTP_PROXY_OPT=1 Test proxy(option) by CHAOS HTTP PROXY
#
# Example of running against Amazon S3 using a bucket named "bucket":
#
@ -66,7 +68,15 @@ set -o pipefail
S3FS=../src/s3fs
# Allow these defaulted values to be overridden
: "${S3_URL:="https://127.0.0.1:8080"}"
#
# [NOTE]
# CHAOS HTTP PROXY does not support HTTPS.
#
if [ -z "${CHAOS_HTTP_PROXY}" ] && [ -z "${CHAOS_HTTP_PROXY_OPT}" ]; then
: "${S3_URL:="https://127.0.0.1:8080"}"
else
: "${S3_URL:="http://127.0.0.1:8080"}"
fi
: "${S3_ENDPOINT:="us-east-1"}"
: "${S3FS_CREDENTIALS_FILE:="passwd-s3fs"}"
: "${TEST_BUCKET_1:="s3fs-integration-test"}"
@ -135,7 +145,11 @@ function start_s3proxy {
if [ -n "${PUBLIC}" ]; then
local S3PROXY_CONFIG="s3proxy-noauth.conf"
else
local S3PROXY_CONFIG="s3proxy.conf"
if [ -z "${CHAOS_HTTP_PROXY}" ] && [ -z "${CHAOS_HTTP_PROXY_OPT}" ]; then
local S3PROXY_CONFIG="s3proxy.conf"
else
local S3PROXY_CONFIG="s3proxy_http.conf"
fi
fi
if [ -n "${S3PROXY_BINARY}" ]
@ -147,9 +161,18 @@ function start_s3proxy {
fi
# generate self-signed SSL certificate
rm -f /tmp/keystore.jks /tmp/keystore.pem
echo -e 'password\npassword\n\n\n\n\n\n\nyes' | keytool -genkey -keystore /tmp/keystore.jks -keyalg RSA -keysize 2048 -validity 365 -ext SAN=IP:127.0.0.1
echo password | keytool -exportcert -keystore /tmp/keystore.jks -rfc -file /tmp/keystore.pem
#
# [NOTE]
# The PROXY test is HTTP only, so do not create CA certificates.
#
if [ -z "${CHAOS_HTTP_PROXY}" ] && [ -z "${CHAOS_HTTP_PROXY_OPT}" ]; then
S3PROXY_CACERT_FILE="/tmp/keystore.pem"
rm -f /tmp/keystore.jks "${S3PROXY_CACERT_FILE}"
echo -e 'password\npassword\n\n\n\n\n\n\nyes' | keytool -genkey -keystore /tmp/keystore.jks -keyalg RSA -keysize 2048 -validity 365 -ext SAN=IP:127.0.0.1
echo password | keytool -exportcert -keystore /tmp/keystore.jks -rfc -file "${S3PROXY_CACERT_FILE}"
else
S3PROXY_CACERT_FILE=""
fi
"${STDBUF_BIN}" -oL -eL java -jar "${S3PROXY_BINARY}" --properties "${S3PROXY_CONFIG}" &
S3PROXY_PID=$!
@ -158,7 +181,7 @@ function start_s3proxy {
wait_for_port 8080
fi
if [ -n "${CHAOS_HTTP_PROXY}" ]; then
if [ -n "${CHAOS_HTTP_PROXY}" ] || [ -n "${CHAOS_HTTP_PROXY_OPT}" ]; then
if [ ! -e "${CHAOS_HTTP_PROXY_BINARY}" ]; then
curl "https://github.com/bouncestorage/chaos-http-proxy/releases/download/chaos-http-proxy-${CHAOS_HTTP_PROXY_VERSION}/chaos-http-proxy" \
--fail --location --silent --output "${CHAOS_HTTP_PROXY_BINARY}"
@ -212,8 +235,16 @@ function start_s3fs {
local DIRECT_IO_OPT=""
fi
# Set environment variables or options for proxy.
# And the PROXY test is HTTP only and does not set CA certificates.
#
if [ -n "${CHAOS_HTTP_PROXY}" ]; then
export http_proxy="127.0.0.1:1080"
S3FS_HTTP_PROXY_OPT=""
elif [ -n "${CHAOS_HTTP_PROXY_OPT}" ]; then
S3FS_HTTP_PROXY_OPT="-o proxy=http://127.0.0.1:1080"
else
S3FS_HTTP_PROXY_OPT=""
fi
# [NOTE]
@ -247,7 +278,7 @@ function start_s3fs {
# shellcheck disable=SC2086
(
set -x
CURL_CA_BUNDLE=/tmp/keystore.pem \
CURL_CA_BUNDLE="${S3PROXY_CACERT_FILE}" \
${VIA_STDBUF_CMDLINE} \
${VALGRIND_EXEC} \
${S3FS} \
@ -260,6 +291,7 @@ function start_s3fs {
-o enable_unsigned_payload \
${AUTH_OPT} \
${DIRECT_IO_OPT} \
${S3FS_HTTP_PROXY_OPT} \
-o stat_cache_expire=1 \
-o stat_cache_interval_expire=1 \
-o dbglevel="${DBGLEVEL:=info}" \

8
test/s3proxy_http.conf Normal file
View File

@ -0,0 +1,8 @@
s3proxy.endpoint=http://127.0.0.1:8080
s3proxy.authorization=aws-v2-or-v4
s3proxy.identity=local-identity
s3proxy.credential=local-credential
jclouds.provider=transient
jclouds.identity=remote-identity
jclouds.credential=remote-credential