octoleo/s3fs-fuse
1
0
Fork 0
mirror of https://github.com/s3fs-fuse/s3fs-fuse.git synced 2024-05-29 07:00:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #671 from psyvision/master

Browse Source 
Add support for ECS metadata endpoint
This commit is contained in:
Takeshi Nakatani 2017-11-19 14:43:06 +09:00 committed by GitHub
commit bad48ab59a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 76 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.travis.yml
View File

@ -8,7 +8,7 @@ matrix:
cache: apt
before_install:
- sudo apt-get update -qq
- sudo apt-get install -qq cppcheck libfuse-dev openjdk-7-jdk
- sudo apt-get install -qq cppcheck libfuse-dev openjdk-7-jdk libjsoncpp-dev
- sudo update-alternatives --set java /usr/lib/jvm/java-7-openjdk-amd64/jre/bin/java
script:
- ./autogen.sh
@ -24,7 +24,7 @@ matrix:
osx_image: xcode8.3
before_install:
- brew update
- brew install truncate
- brew install truncate jsoncpp
- brew tap caskroom/cask
- brew cask install osxfuse
- if [ -f /Library/Filesystems/osxfusefs.fs/Support/load_osxfusefs ]; then sudo chmod +s /Library/Filesystems/osxfusefs.fs/Support/load_osxfusefs ; elif [ -f /Library/Filesystems/osxfuse.fs/Contents/Resources/load_osxfuse ]; then sudo chmod +s /Library/Filesystems/osxfuse.fs/Contents/Resources/load_osxfuse ; fi

10
configure.ac
View File

@ -176,13 +176,13 @@ dnl
dnl For PKG_CONFIG before checking nss/gnutls.
dnl this is redundant checking, but we need checking before following.
dnl
PKG_CHECK_MODULES([common_lib_checking], [fuse >= ${min_fuse_version} libcurl >= 7.0 libxml-2.0 >= 2.6])
PKG_CHECK_MODULES([common_lib_checking], [fuse >= ${min_fuse_version} libcurl >= 7.0 libxml-2.0 >= 2.6 jsoncpp >= 0.6 ])
AC_MSG_CHECKING([compile s3fs with])
case "${auth_lib}" in
openssl)
AC_MSG_RESULT(OpenSSL)
PKG_CHECK_MODULES([DEPS], [fuse >= ${min_fuse_version} libcurl >= 7.0 libxml-2.0 >= 2.6 libcrypto >= 0.9])
PKG_CHECK_MODULES([DEPS], [fuse >= ${min_fuse_version} libcurl >= 7.0 libxml-2.0 >= 2.6 libcrypto >= 0.9 jsoncpp >= 0.6 ])
;;
gnutls)
AC_MSG_RESULT(GnuTLS-gcrypt)
@ -191,7 +191,7 @@ gnutls)
AS_IF([test "$gnutls_nettle" = ""], [AC_CHECK_LIB(gcrypt, gcry_control, [gnutls_nettle=0])])
AS_IF([test $gnutls_nettle = 0],
[
PKG_CHECK_MODULES([DEPS], [fuse >= ${min_fuse_version} libcurl >= 7.0 libxml-2.0 >= 2.6 gnutls >= 2.12.0 ])
PKG_CHECK_MODULES([DEPS], [fuse >= ${min_fuse_version} libcurl >= 7.0 libxml-2.0 >= 2.6 gnutls >= 2.12.0 jsoncpp >= 0.6 ])
LIBS="-lgnutls -lgcrypt $LIBS"
AC_MSG_CHECKING([gnutls is build with])
AC_MSG_RESULT(gcrypt)
@ -205,7 +205,7 @@ nettle)
AS_IF([test "$gnutls_nettle" = ""], [AC_CHECK_LIB(nettle, nettle_MD5Init, [gnutls_nettle=1])])
AS_IF([test $gnutls_nettle = 1],
[
PKG_CHECK_MODULES([DEPS], [fuse >= ${min_fuse_version} libcurl >= 7.0 libxml-2.0 >= 2.6 nettle >= 2.7.1 ])
PKG_CHECK_MODULES([DEPS], [fuse >= ${min_fuse_version} libcurl >= 7.0 libxml-2.0 >= 2.6 nettle >= 2.7.1 jsoncpp >= 0.6 ])
LIBS="-lgnutls -lnettle $LIBS"
AC_MSG_CHECKING([gnutls is build with])
AC_MSG_RESULT(nettle)
@ -214,7 +214,7 @@ nettle)
;;
nss)
AC_MSG_RESULT(NSS)
PKG_CHECK_MODULES([DEPS], [fuse >= ${min_fuse_version} libcurl >= 7.0 libxml-2.0 >= 2.6 nss >= 3.15.0 ])
PKG_CHECK_MODULES([DEPS], [fuse >= ${min_fuse_version} libcurl >= 7.0 libxml-2.0 >= 2.6 nss >= 3.15.0 jsoncpp >= 0.6 ])
;;
*)
AC_MSG_ERROR([unknown ssl library type.])

92
src/curl.cpp
View File

@ -41,6 +41,8 @@
#include <algorithm>
#include <list>
#include <vector>
#include <json/json.h>
#include <json/reader.h>
#include "common.h"
#include "curl.h"
@ -320,12 +322,16 @@ void CurlHandlerPool::ReturnHandler(CURL* h)
#define MAX_MULTI_COPY_SOURCE_SIZE 524288000 // 500MB
#define IAM_EXPIRE_MERGIN (20 * 60) // update timing
#define IAM_CRED_URL_ECS "http://169.254.170.2"
#define IAM_CRED_URL "http://169.254.169.254/latest/meta-data/iam/security-credentials/"
#define ECS_IAM_ENV_VAR "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"
#define IAMCRED_ACCESSKEYID "AccessKeyId"
#define IAMCRED_SECRETACCESSKEY "SecretAccessKey"
#define IAMCRED_ACCESSTOKEN "Token"
#define IAMCRED_EXPIRATION "Expiration"
#define IAMCRED_ROLEARN "RoleArn"
#define IAMCRED_KEYCOUNT 4
#define IAMCRED_KEYCOUNT_ECS 5
// [NOTICE]
// This symbol is for libcurl under 7.23.0
@ -357,6 +363,7 @@ string S3fsCurl::AWSAccessKeyId;
string S3fsCurl::AWSSecretAccessKey;
string S3fsCurl::AWSAccessToken;
time_t S3fsCurl::AWSAccessTokenExpire= 0;
bool S3fsCurl::is_ecs = false;
string S3fsCurl::IAM_role;
long S3fsCurl::ssl_verify_hostname = 1; // default(original code...)
curltime_t S3fsCurl::curl_times;
@ -1159,6 +1166,13 @@ long S3fsCurl::SetSslVerifyHostname(long value)
return old;
}
bool S3fsCurl::SetIsECS(bool flag)
{
bool old = S3fsCurl::is_ecs;
S3fsCurl::is_ecs = flag;
return old;
}
string S3fsCurl::SetIAMRole(const char* role)
{
string old = S3fsCurl::IAM_role;
@ -1410,37 +1424,23 @@ bool S3fsCurl::ParseIAMCredentialResponse(const char* response, iamcredmap_t& ke
if(!response){
return false;
}
istringstream sscred(response);
string oneline;
keyval.clear();
while(getline(sscred, oneline, '\n')){
string::size_type pos;
string key;
string val;
if(string::npos != (pos = oneline.find(IAMCRED_ACCESSKEYID))){
key = IAMCRED_ACCESSKEYID;
}else if(string::npos != (pos = oneline.find(IAMCRED_SECRETACCESSKEY))){
key = IAMCRED_SECRETACCESSKEY;
}else if(string::npos != (pos = oneline.find(IAMCRED_ACCESSTOKEN))){
key = IAMCRED_ACCESSTOKEN;
}else if(string::npos != (pos = oneline.find(IAMCRED_EXPIRATION))){
key = IAMCRED_EXPIRATION;
}else{
continue;
}
if(string::npos == (pos = oneline.find(':', pos + key.length()))){
continue;
}
if(string::npos == (pos = oneline.find('\"', pos))){
continue;
}
oneline = oneline.substr(pos + sizeof(char));
if(string::npos == (pos = oneline.find('\"'))){
continue;
}
val = oneline.substr(0, pos);
keyval[key] = val;
Json::Value root;
Json::Reader reader;
if (!reader.parse(response, root)) {
return false;
}
keyval[string(IAMCRED_ACCESSKEYID)] = root.get(IAMCRED_ACCESSKEYID, "").asString();
keyval[string(IAMCRED_SECRETACCESSKEY)] = root.get(IAMCRED_SECRETACCESSKEY, "").asString();
keyval[string(IAMCRED_ACCESSTOKEN)] = root.get(IAMCRED_ACCESSTOKEN, "").asString();
keyval[string(IAMCRED_EXPIRATION)] = root.get(IAMCRED_EXPIRATION, "").asString();
if (S3fsCurl::is_ecs) {
keyval[string(IAMCRED_ROLEARN)] = root.get(IAMCRED_ROLEARN, "").asString();
}
return true;
}
@ -1453,7 +1453,8 @@ bool S3fsCurl::SetIAMCredentials(const char* response)
if(!ParseIAMCredentialResponse(response, keyval)){
return false;
}
if(IAMCRED_KEYCOUNT != keyval.size()){
if((S3fsCurl::is_ecs ? IAMCRED_KEYCOUNT_ECS : IAMCRED_KEYCOUNT) != keyval.size()){
return false;
}
@ -1467,7 +1468,7 @@ bool S3fsCurl::SetIAMCredentials(const char* response)
bool S3fsCurl::CheckIAMCredentialUpdate(void)
{
if(0 == S3fsCurl::IAM_role.size()){
if(0 == S3fsCurl::IAM_role.size() && !S3fsCurl::is_ecs){
return true;
}
if(time(NULL) + IAM_EXPIRE_MERGIN <= S3fsCurl::AWSAccessTokenExpire){
@ -2133,7 +2134,7 @@ string S3fsCurl::CalcSignatureV2(const string& method, const string& strMD5, con
string Signature;
string StringToSign;
if(0 < S3fsCurl::IAM_role.size()){
if(0 < S3fsCurl::IAM_role.size() || S3fsCurl::is_ecs){
if(!S3fsCurl::CheckIAMCredentialUpdate()){
S3FS_PRN_ERR("Something error occurred in checking IAM credential.");
return Signature; // returns empty string, then it occurs error.
@ -2175,7 +2176,7 @@ string S3fsCurl::CalcSignature(const string& method, const string& canonical_uri
string Signature, StringCQ, StringToSign;
string uriencode;
if(0 < S3fsCurl::IAM_role.size()){
if(0 < S3fsCurl::IAM_role.size() || S3fsCurl::is_ecs){
if(!S3fsCurl::CheckIAMCredentialUpdate()){
S3FS_PRN_ERR("Something error occurred in checking IAM credential.");
return Signature; // returns empty string, then it occurs error.
@ -2335,7 +2336,7 @@ void S3fsCurl::insertV4Headers()
requestHeaders = curl_slist_sort_insert(requestHeaders, "host", get_bucket_host().c_str());
requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-content-sha256", contentSHA256.c_str());
requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-date", date8601.c_str());
if(!S3fsCurl::IsPublicBucket()){
string Signature = CalcSignature(op, realpath, query_string + (type == REQTYPE_PREMULTIPOST ? "=" : ""), strdate, contentSHA256, date8601);
string auth = "AWS4-HMAC-SHA256 Credential=" + AWSAccessKeyId + "/" + strdate + "/" + endpoint +
@ -2412,12 +2413,15 @@ int S3fsCurl::DeleteRequest(const char* tpath)
//
int S3fsCurl::GetIAMCredentials(void)
{
S3FS_PRN_INFO3("[IAM role=%s]", S3fsCurl::IAM_role.c_str());
if (!S3fsCurl::is_ecs) {
S3FS_PRN_INFO3("[IAM role=%s]", S3fsCurl::IAM_role.c_str());
if(0 == S3fsCurl::IAM_role.size()){
S3FS_PRN_ERR("IAM role name is empty.");
return -EIO;
if(0 == S3fsCurl::IAM_role.size()) {
S3FS_PRN_ERR("IAM role name is empty.");
return -EIO;
}
}
// at first set type for handle
type = REQTYPE_IAMCRED;
@ -2426,7 +2430,13 @@ int S3fsCurl::GetIAMCredentials(void)
}
// url
url = string(IAM_CRED_URL) + S3fsCurl::IAM_role;
if (is_ecs) {
url = string(IAM_CRED_URL_ECS) + std::getenv(ECS_IAM_ENV_VAR);
}
else {
url = string(IAM_CRED_URL) + S3fsCurl::IAM_role;
}
requestHeaders = NULL;
responseHeaders.clear();
bodydata = new BodyData();
@ -2463,7 +2473,7 @@ bool S3fsCurl::LoadIAMRoleFromMetaData(void)
}
// url
url = IAM_CRED_URL;
url = string(IAM_CRED_URL);
requestHeaders = NULL;
responseHeaders.clear();
bodydata = new BodyData();

2
src/curl.h
View File

@ -230,6 +230,7 @@ class S3fsCurl
static std::string AWSSecretAccessKey;
static std::string AWSAccessToken;
static time_t AWSAccessTokenExpire;
static bool is_ecs;
static std::string IAM_role;
static long ssl_verify_hostname;
static curltime_t curl_times;
@ -374,6 +375,7 @@ class S3fsCurl
static long GetSslVerifyHostname(void) { return S3fsCurl::ssl_verify_hostname; }
static int SetMaxParallelCount(int value);
static int GetMaxParallelCount(void) { return S3fsCurl::max_parallel_cnt; }
static bool SetIsECS(bool flag);
static std::string SetIAMRole(const char* role);
static const char* GetIAMRole(void) { return S3fsCurl::IAM_role.c_str(); }
static bool SetMultipartSize(off_t size);

14
src/s3fs.cpp
View File

@ -122,6 +122,7 @@ static bool is_s3fs_uid = false;// default does not set.
static bool is_s3fs_gid = false;// default does not set.
static bool is_s3fs_umask = false;// default does not set.
static bool is_remove_cache = false;
static bool is_ecs = false;
static bool is_use_xattr = false;
static bool create_bucket = false;
static int64_t singlepart_copy_limit = FIVE_GB;
@ -4073,7 +4074,7 @@ static int get_access_keys(void)
}
// access key loading is deferred
if(load_iamrole){
if(load_iamrole || is_ecs){
return EXIT_SUCCESS;
}
@ -4508,7 +4509,16 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
passwd_file = strchr(arg, '=') + sizeof(char);
return 0;
}
if(0 == strcmp(arg, "ecs")){
S3fsCurl::SetIsECS(true);
is_ecs = true;
return 0;
}
if(0 == STR2NCMP(arg, "iam_role")){
if (is_ecs) {
S3FS_PRN_EXIT("option iam_role cannot be used in conjunction with ecs");
return -1;
}
if(0 == strcmp(arg, "iam_role") || 0 == strcmp(arg, "iam_role=auto")){
// loading IAM role name in s3fs_init(), because we need to wait initializing curl.
//
@ -4908,7 +4918,7 @@ int main(int argc, char* argv[])
S3FS_PRN_EXIT("specifying both passwd_file and the access keys options is invalid.");
exit(EXIT_FAILURE);
}
if(!S3fsCurl::IsPublicBucket() && !load_iamrole){
if(!S3fsCurl::IsPublicBucket() && !load_iamrole && !is_ecs){
if(EXIT_SUCCESS != get_access_keys()){
exit(EXIT_FAILURE);
}

4
src/s3fs_util.cpp
View File

@ -1167,6 +1167,10 @@ void show_help (void)
" enable_content_md5 (default is disable)\n"
" - ensure data integrity during writes with MD5 hash.\n"
"\n"
" ecs\n"
" - This option instructs s3fs to query the ECS container credential\n"
" metadata address instead of the instance metadata address.\n"
"\n"
" iam_role (default is no IAM role)\n"
" - This option requires the IAM role name or \"auto\". If you specify\n"
" \"auto\", s3fs will automatically use the IAM role names that are set\n"