mirror of
https://github.com/s3fs-fuse/s3fs-fuse.git
synced 2024-12-22 08:48:55 +00:00
Added proxy and proxy_cred_file option
This commit is contained in:
Takeshi Nakatani
Andrew Gaul
parent
38e8a830c9
commit
d1388ff446
@ -382,6 +382,20 @@ Specify the path of the mime.types file.
|
||||
If this option is not specified, the existence of "/etc/mime.types" is checked, and that file is loaded as mime information.
|
||||
If this file does not exist on macOS, then "/etc/apache2/mime.types" is checked as well.
|
||||
.TP
|
||||
\fB\-o\fR proxy (default="")
|
||||
This option specifies a proxy to S3 server.
|
||||
Specify the proxy with '[<scheme://]hostname(fqdn)[:<port>]' formatted.
|
||||
'<schema>://' can be omitted, and 'http://' is used when omitted.
|
||||
Also, ':<port>' can also be omitted. If omitted, port 443 is used for HTTPS schema, and port 1080 is used otherwise.
|
||||
This option is the same as the curl command's '--proxy(-x)' option and libcurl's 'CURLOPT_PROXY' flag.
|
||||
This option is equivalent to and takes precedence over the environment variables 'http_proxy', 'all_proxy', etc.
|
||||
.TP
|
||||
\fB\-o\fR proxy_cred_file (default="")
|
||||
This option specifies the file that describes the username and passphrase for authentication of the proxy when the HTTP schema proxy is specified by the 'proxy' option.
|
||||
Username and passphrase are valid only for HTTP schema.
|
||||
If the HTTP proxy does not require authentication, this option is not required.
|
||||
Separate the username and passphrase with a ':' character and specify each as a URL-encoded string.
|
||||
.TP
|
||||
\fB\-o\fR logfile - specify the log output file.
|
||||
s3fs outputs the log file to syslog. Alternatively, if s3fs is started with the "-f" option specified, the log will be output to the stdout/stderr.
|
||||
You can use this option to specify the log file that s3fs outputs.
|
||||
|
||||
145
src/curl.cpp
145
src/curl.cpp
@ -123,6 +123,9 @@ bool S3fsCurl::is_unsigned_payload = false; // default
|
||||
bool S3fsCurl::is_ua = true; // default
|
||||
bool S3fsCurl::listobjectsv2 = false; // default
|
||||
bool S3fsCurl::requester_pays = false; // default
|
||||
std::string S3fsCurl::proxy_url;
|
||||
bool S3fsCurl::proxy_http = false;
|
||||
std::string S3fsCurl::proxy_userpwd;
|
||||
|
||||
//-------------------------------------------------------------------
|
||||
// Class methods for S3fsCurl
|
||||
@ -1053,6 +1056,134 @@ int S3fsCurl::SetMaxMultiRequest(int max)
|
||||
return old;
|
||||
}
|
||||
|
||||
// [NOTE]
|
||||
// This proxy setting is as same as the "--proxy" option of the curl command,
|
||||
// and equivalent to the "CURLOPT_PROXY" option of the curl_easy_setopt()
|
||||
// function.
|
||||
// However, currently s3fs does not provide another option to set the schema
|
||||
// and port, so you need to specify these it in this function. (Other than
|
||||
// this function, there is no means of specifying the schema and port.)
|
||||
// Therefore, it should be specified "url" as "[<schema>://]<fqdn>[:<port>]".
|
||||
// s3fs passes this string to curl_easy_setopt() function with "CURLOPT_PROXY".
|
||||
// If no "schema" is specified, "http" will be used as default, and if no port
|
||||
// is specified, "443" will be used for "HTTPS" and "1080" otherwise.
|
||||
// (See the description of "CURLOPT_PROXY" in libcurl document.)
|
||||
//
|
||||
bool S3fsCurl::SetProxy(const char* url)
|
||||
{
|
||||
if(!url || '\0' == url[0]){
|
||||
return false;
|
||||
}
|
||||
std::string tmpurl = url;
|
||||
|
||||
// check schema
|
||||
bool is_http = true;
|
||||
size_t pos = 0;
|
||||
if(std::string::npos != (pos = tmpurl.find("://", pos))){
|
||||
if(0 == pos){
|
||||
// no schema string before "://"
|
||||
return false;
|
||||
}
|
||||
pos += strlen("://");
|
||||
|
||||
// Check if it is other than "http://"
|
||||
if(0 != tmpurl.find("http://", 0)){
|
||||
is_http = false;
|
||||
}
|
||||
}else{
|
||||
// not have schema string
|
||||
pos = 0;
|
||||
}
|
||||
// check fqdn and port number string
|
||||
if(std::string::npos != (pos = tmpurl.find(":", pos))){
|
||||
// specify port
|
||||
if(0 == pos){
|
||||
// no fqdn(hostname) string before ":"
|
||||
return false;
|
||||
}
|
||||
pos += strlen(":");
|
||||
if(std::string::npos != tmpurl.find(":", pos)){
|
||||
// found wrong separator
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
S3fsCurl::proxy_url = tmpurl;
|
||||
S3fsCurl::proxy_http = is_http;
|
||||
return true;
|
||||
}
|
||||
|
||||
// [NOTE]
|
||||
// This function loads proxy credentials(username and passphrase)
|
||||
// from a file.
|
||||
// The loaded values is set to "CURLOPT_PROXYUSERPWD" in the
|
||||
// curl_easy_setopt() function. (But only used if the proxy is HTTP
|
||||
// schema.)
|
||||
//
|
||||
// The file is expected to contain only one valid line:
|
||||
// ------------------------
|
||||
// # comment line
|
||||
// <username>:<passphrase>
|
||||
// ------------------------
|
||||
// Lines starting with a '#' character are treated as comments.
|
||||
// Lines with only space characters and blank lines are ignored.
|
||||
// If the user name contains spaces, it must be url encoded(ex. %20).
|
||||
//
|
||||
bool S3fsCurl::SetProxyUserPwd(const char* file)
|
||||
{
|
||||
if(!file || '\0' == file[0]){
|
||||
return false;
|
||||
}
|
||||
if(!S3fsCurl::proxy_userpwd.empty()){
|
||||
S3FS_PRN_WARN("Already set username and passphrase for proxy.");
|
||||
return false;
|
||||
}
|
||||
|
||||
std::ifstream credFileStream(file);
|
||||
if(!credFileStream.good()){
|
||||
S3FS_PRN_WARN("Could not load username and passphrase for proxy from %s.", file);
|
||||
return false;
|
||||
}
|
||||
|
||||
std::string userpwd;
|
||||
std::string line;
|
||||
while(getline(credFileStream, line)){
|
||||
line = trim(line);
|
||||
if(line.empty()){
|
||||
continue;
|
||||
}
|
||||
if(line[0]=='#'){
|
||||
continue;
|
||||
}
|
||||
if(!userpwd.empty()){
|
||||
S3FS_PRN_WARN("Multiple valid username and passphrase found in %s file. Should specify only one pair.", file);
|
||||
return false;
|
||||
}
|
||||
// check separator for username and passphrase
|
||||
size_t pos = 0;
|
||||
if(std::string::npos == (pos = line.find(':', pos))){
|
||||
S3FS_PRN_WARN("Found string for username and passphrase in %s file does not have separator ':'.", file);
|
||||
return false;
|
||||
}
|
||||
if(0 == pos || (pos + 1) == line.length()){
|
||||
S3FS_PRN_WARN("Found string for username or passphrase in %s file is empty.", file);
|
||||
return false;
|
||||
}
|
||||
if(std::string::npos != line.find(':', ++pos)){
|
||||
S3FS_PRN_WARN("Found string for username and passphrase in %s file has multiple separator ':'.", file);
|
||||
return false;
|
||||
}
|
||||
userpwd = line;
|
||||
}
|
||||
if(userpwd.empty()){
|
||||
S3FS_PRN_WARN("No valid username and passphrase found in %s.", file);
|
||||
return false;
|
||||
}
|
||||
|
||||
S3fsCurl::proxy_userpwd = userpwd;
|
||||
return true;
|
||||
}
|
||||
|
||||
// cppcheck-suppress unmatchedSuppression
|
||||
// cppcheck-suppress constParameter
|
||||
bool S3fsCurl::UploadMultipartPostCallback(S3fsCurl* s3fscurl, void* param)
|
||||
@ -1879,6 +2010,20 @@ bool S3fsCurl::ResetHandle(AutoLock::Type locktype)
|
||||
return false;
|
||||
}
|
||||
}
|
||||
if(!S3fsCurl::proxy_url.empty()){
|
||||
if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_PROXY, S3fsCurl::proxy_url.c_str())){
|
||||
return false;
|
||||
}
|
||||
if(S3fsCurl::proxy_http){
|
||||
if(!S3fsCurl::proxy_userpwd.empty()){
|
||||
if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_PROXYUSERPWD, S3fsCurl::proxy_userpwd.c_str())){
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}else if(!S3fsCurl::proxy_userpwd.empty()){
|
||||
S3FS_PRN_DBG("Username and passphrase are specified even though proxy is not 'http' scheme, so skip to set those.");
|
||||
}
|
||||
}
|
||||
|
||||
AutoLock lock(&S3fsCurl::curl_handles_lock, locktype);
|
||||
S3fsCurl::curl_times[hCurl] = time(0);
|
||||
|
||||
@ -154,6 +154,9 @@ class S3fsCurl
|
||||
static bool is_ua; // User-Agent
|
||||
static bool listobjectsv2;
|
||||
static bool requester_pays;
|
||||
static std::string proxy_url;
|
||||
static bool proxy_http;
|
||||
static std::string proxy_userpwd; // load from file(<username>:<passphrase>)
|
||||
|
||||
// variables
|
||||
CURL* hCurl;
|
||||
@ -332,6 +335,8 @@ class S3fsCurl
|
||||
static bool IsListObjectsV2() { return S3fsCurl::listobjectsv2; }
|
||||
static bool SetRequesterPays(bool flag) { bool old_flag = S3fsCurl::requester_pays; S3fsCurl::requester_pays = flag; return old_flag; }
|
||||
static bool IsRequesterPays() { return S3fsCurl::requester_pays; }
|
||||
static bool SetProxy(const char* url);
|
||||
static bool SetProxyUserPwd(const char* userpwd);
|
||||
|
||||
// methods
|
||||
bool CreateCurlHandle(bool only_pool = false, bool remake = false);
|
||||
|
||||
16
src/s3fs.cpp
16
src/s3fs.cpp
@ -4843,6 +4843,22 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
|
||||
mimetype_file = strchr(arg, '=') + sizeof(char);
|
||||
return 0;
|
||||
}
|
||||
if(is_prefix(arg, "proxy=")){
|
||||
const char* url = &arg[strlen("proxy=")];
|
||||
if(!S3fsCurl::SetProxy(url)){
|
||||
S3FS_PRN_EXIT("failed to set proxy(%s).", url);
|
||||
return -1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
if(is_prefix(arg, "proxy_cred_file=")){
|
||||
const char* file = &arg[strlen("proxy_cred_file=")];
|
||||
if(!S3fsCurl::SetProxyUserPwd(file)){
|
||||
S3FS_PRN_EXIT("failed to set proxy user and passphrase from file(%s).", file);
|
||||
return -1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
//
|
||||
// log file option
|
||||
//
|
||||
|
||||
@ -483,6 +483,26 @@ static const char help_string[] =
|
||||
" If this file does not exist on macOS, then \"/etc/apache2/mime.types\"\n"
|
||||
" is checked as well.\n"
|
||||
"\n"
|
||||
" proxy (default=\"\")\n"
|
||||
" This option specifies a proxy to S3 server.\n"
|
||||
" Specify the proxy with '[<scheme://]hostname(fqdn)[:<port>]' formatted.\n"
|
||||
" '<schema>://' can be omitted, and 'http://' is used when omitted.\n"
|
||||
" Also, ':<port>' can also be omitted. If omitted, port 443 is used for\n"
|
||||
" HTTPS schema, and port 1080 is used otherwise.\n"
|
||||
" This option is the same as the curl command's '--proxy(-x)' option and\n"
|
||||
" libcurl's 'CURLOPT_PROXY' flag.\n"
|
||||
" This option is equivalent to and takes precedence over the environment\n"
|
||||
" variables 'http_proxy', 'all_proxy', etc.\n"
|
||||
"\n"
|
||||
" proxy_cred_file (default=\"\")\n"
|
||||
" This option specifies the file that describes the username and\n"
|
||||
" passphrase for authentication of the proxy when the HTTP schema\n"
|
||||
" proxy is specified by the 'proxy' option.\n"
|
||||
" Username and passphrase are valid only for HTTP schema. If the HTTP\n"
|
||||
" proxy does not require authentication, this option is not required.\n"
|
||||
" Separate the username and passphrase with a ':' character and\n"
|
||||
" specify each as a URL-encoded string.\n"
|
||||
"\n"
|
||||
" logfile - specify the log output file.\n"
|
||||
" s3fs outputs the log file to syslog. Alternatively, if s3fs is\n"
|
||||
" started with the \"-f\" option specified, the log will be output\n"
|
||||
|
||||
@ -34,6 +34,8 @@
|
||||
# S3_ENDPOINT="us-east-1" Specify region
|
||||
# TMPDIR="/var/tmp" Set to use a temporary directory different
|
||||
# from /var/tmp
|
||||
# CHAOS_HTTP_PROXY=1 Test proxy(environment) by CHAOS HTTP PROXY
|
||||
# CHAOS_HTTP_PROXY_OPT=1 Test proxy(option) by CHAOS HTTP PROXY
|
||||
#
|
||||
# Example of running against Amazon S3 using a bucket named "bucket":
|
||||
#
|
||||
@ -66,7 +68,15 @@ set -o pipefail
|
||||
S3FS=../src/s3fs
|
||||
|
||||
# Allow these defaulted values to be overridden
|
||||
: "${S3_URL:="https://127.0.0.1:8080"}"
|
||||
#
|
||||
# [NOTE]
|
||||
# CHAOS HTTP PROXY does not support HTTPS.
|
||||
#
|
||||
if [ -z "${CHAOS_HTTP_PROXY}" ] && [ -z "${CHAOS_HTTP_PROXY_OPT}" ]; then
|
||||
: "${S3_URL:="https://127.0.0.1:8080"}"
|
||||
else
|
||||
: "${S3_URL:="http://127.0.0.1:8080"}"
|
||||
fi
|
||||
: "${S3_ENDPOINT:="us-east-1"}"
|
||||
: "${S3FS_CREDENTIALS_FILE:="passwd-s3fs"}"
|
||||
: "${TEST_BUCKET_1:="s3fs-integration-test"}"
|
||||
@ -135,7 +145,11 @@ function start_s3proxy {
|
||||
if [ -n "${PUBLIC}" ]; then
|
||||
local S3PROXY_CONFIG="s3proxy-noauth.conf"
|
||||
else
|
||||
local S3PROXY_CONFIG="s3proxy.conf"
|
||||
if [ -z "${CHAOS_HTTP_PROXY}" ] && [ -z "${CHAOS_HTTP_PROXY_OPT}" ]; then
|
||||
local S3PROXY_CONFIG="s3proxy.conf"
|
||||
else
|
||||
local S3PROXY_CONFIG="s3proxy_http.conf"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ -n "${S3PROXY_BINARY}" ]
|
||||
@ -147,9 +161,18 @@ function start_s3proxy {
|
||||
fi
|
||||
|
||||
# generate self-signed SSL certificate
|
||||
rm -f /tmp/keystore.jks /tmp/keystore.pem
|
||||
echo -e 'password\npassword\n\n\n\n\n\n\nyes' | keytool -genkey -keystore /tmp/keystore.jks -keyalg RSA -keysize 2048 -validity 365 -ext SAN=IP:127.0.0.1
|
||||
echo password | keytool -exportcert -keystore /tmp/keystore.jks -rfc -file /tmp/keystore.pem
|
||||
#
|
||||
# [NOTE]
|
||||
# The PROXY test is HTTP only, so do not create CA certificates.
|
||||
#
|
||||
if [ -z "${CHAOS_HTTP_PROXY}" ] && [ -z "${CHAOS_HTTP_PROXY_OPT}" ]; then
|
||||
S3PROXY_CACERT_FILE="/tmp/keystore.pem"
|
||||
rm -f /tmp/keystore.jks "${S3PROXY_CACERT_FILE}"
|
||||
echo -e 'password\npassword\n\n\n\n\n\n\nyes' | keytool -genkey -keystore /tmp/keystore.jks -keyalg RSA -keysize 2048 -validity 365 -ext SAN=IP:127.0.0.1
|
||||
echo password | keytool -exportcert -keystore /tmp/keystore.jks -rfc -file "${S3PROXY_CACERT_FILE}"
|
||||
else
|
||||
S3PROXY_CACERT_FILE=""
|
||||
fi
|
||||
|
||||
"${STDBUF_BIN}" -oL -eL java -jar "${S3PROXY_BINARY}" --properties "${S3PROXY_CONFIG}" &
|
||||
S3PROXY_PID=$!
|
||||
@ -158,7 +181,7 @@ function start_s3proxy {
|
||||
wait_for_port 8080
|
||||
fi
|
||||
|
||||
if [ -n "${CHAOS_HTTP_PROXY}" ]; then
|
||||
if [ -n "${CHAOS_HTTP_PROXY}" ] || [ -n "${CHAOS_HTTP_PROXY_OPT}" ]; then
|
||||
if [ ! -e "${CHAOS_HTTP_PROXY_BINARY}" ]; then
|
||||
curl "https://github.com/bouncestorage/chaos-http-proxy/releases/download/chaos-http-proxy-${CHAOS_HTTP_PROXY_VERSION}/chaos-http-proxy" \
|
||||
--fail --location --silent --output "${CHAOS_HTTP_PROXY_BINARY}"
|
||||
@ -212,8 +235,16 @@ function start_s3fs {
|
||||
local DIRECT_IO_OPT=""
|
||||
fi
|
||||
|
||||
# Set environment variables or options for proxy.
|
||||
# And the PROXY test is HTTP only and does not set CA certificates.
|
||||
#
|
||||
if [ -n "${CHAOS_HTTP_PROXY}" ]; then
|
||||
export http_proxy="127.0.0.1:1080"
|
||||
S3FS_HTTP_PROXY_OPT=""
|
||||
elif [ -n "${CHAOS_HTTP_PROXY_OPT}" ]; then
|
||||
S3FS_HTTP_PROXY_OPT="-o proxy=http://127.0.0.1:1080"
|
||||
else
|
||||
S3FS_HTTP_PROXY_OPT=""
|
||||
fi
|
||||
|
||||
# [NOTE]
|
||||
@ -247,7 +278,7 @@ function start_s3fs {
|
||||
# shellcheck disable=SC2086
|
||||
(
|
||||
set -x
|
||||
CURL_CA_BUNDLE=/tmp/keystore.pem \
|
||||
CURL_CA_BUNDLE="${S3PROXY_CACERT_FILE}" \
|
||||
${VIA_STDBUF_CMDLINE} \
|
||||
${VALGRIND_EXEC} \
|
||||
${S3FS} \
|
||||
@ -260,6 +291,7 @@ function start_s3fs {
|
||||
-o enable_unsigned_payload \
|
||||
${AUTH_OPT} \
|
||||
${DIRECT_IO_OPT} \
|
||||
${S3FS_HTTP_PROXY_OPT} \
|
||||
-o stat_cache_expire=1 \
|
||||
-o stat_cache_interval_expire=1 \
|
||||
-o dbglevel="${DBGLEVEL:=info}" \
|
||||
|
||||
8
test/s3proxy_http.conf
Normal file
8
test/s3proxy_http.conf
Normal file
@ -0,0 +1,8 @@
|
||||
s3proxy.endpoint=http://127.0.0.1:8080
|
||||
s3proxy.authorization=aws-v2-or-v4
|
||||
s3proxy.identity=local-identity
|
||||
s3proxy.credential=local-credential
|
||||
|
||||
jclouds.provider=transient
|
||||
jclouds.identity=remote-identity
|
||||
jclouds.credential=remote-credential
|
||||
Loading…
Reference in New Issue
Block a user