From 50f1ad51c814ee5f59f356d8360e6a5335d21332 Mon Sep 17 00:00:00 2001 From: Takeshi Nakatani Date: Fri, 6 May 2016 04:37:32 +0000 Subject: [PATCH] loading IAM role name automatically(iam_role option) - #387 --- doc/man/s3fs.1 | 4 +- src/curl.cpp | 115 ++++++++++++++++++++++++++++++++++++++++++++-- src/curl.h | 6 ++- src/s3fs.cpp | 31 +++++++++++-- src/s3fs_util.cpp | 8 ++-- 5 files changed, 150 insertions(+), 14 deletions(-) diff --git a/doc/man/s3fs.1 b/doc/man/s3fs.1 index 80eb937..7fd840c 100644 --- a/doc/man/s3fs.1 +++ b/doc/man/s3fs.1 @@ -197,8 +197,8 @@ Enable to send "Content-MD5" header when uploading a object without multipart po If this option is enabled, it has some influences on a performance of s3fs when uploading small object. Because s3fs always checks MD5 when uploading large object, this option does not affect on large object. .TP -\fB\-o\fR iam_role ( default is no role ) -set the IAM Role that will supply the credentials from the instance meta-data. +\fB\-o\fR iam_role ( default is no IAM role ) +This option requires the IAM role name or "auto". If you specify "auto", s3fs will automatically use the IAM role names that are set to an instance. If you specify this option without any argument, it is the same as that you have specified the "auto". .TP \fB\-o\fR noxmlns - disable registing xml name space. disable registing xml name space for response of ListBucketResult and ListVersionsResult etc. Default name space is looked up from "http://s3.amazonaws.com/doc/2006-03-01". diff --git a/src/curl.cpp b/src/curl.cpp index 6795b6b..da91e1f 100644 --- a/src/curl.cpp +++ b/src/curl.cpp @@ -315,6 +315,9 @@ void CurlHandlerPool::ReturnHandler(CURL* h) #define IAMCRED_ACCESSTOKEN "Token" #define IAMCRED_EXPIRATION "Expiration" #define IAMCRED_KEYCOUNT 4 +#define IAM_DEFAULT_ROLE_URL "http://169.254.169.254/latest/meta-data/iam/info" +#define IAMDEFROLE_PROFARN "InstanceProfileArn" +#define IAMDEFROLE_PROFARN_PART ":instance-profile/" // [NOTICE] // This symbol is for libcurl under 7.23.0 @@ -1436,6 +1439,66 @@ bool S3fsCurl::CheckIAMCredentialUpdate(void) return true; } +bool S3fsCurl::ParseIAMRoleFromMetaDataResponse(const char* response, string& rolename) +{ + if(!response){ + return false; + } + // [NOTE] + // expected following strings. + // + // { + // "Code" : "Success", + // "LastUpdated" : "2016-01-01T00:00:00Z", + // "InstanceProfileArn" : "arn:aws:iam::111111111111:instance-profile/myrolename", + // "InstanceProfileId" : "AAAAAAAAAAAAAAAAAAAAA" + // } + // + istringstream ssrole(response); + string oneline; + while(getline(ssrole, oneline, '\n')){ + string::size_type pos; + if(string::npos != (pos = oneline.find(IAMDEFROLE_PROFARN))){ + if(string::npos == (pos = oneline.find(':', pos + strlen(IAMDEFROLE_PROFARN)))){ + continue; + } + if(string::npos == (pos = oneline.find('\"', pos))){ + continue; + } + + // value + oneline = oneline.substr(pos + sizeof(char)); + if(string::npos == (pos = oneline.find('\"'))){ + continue; + } + oneline = oneline.substr(0, pos); + + // role name + if(string::npos == (pos = oneline.find(IAMDEFROLE_PROFARN_PART))){ + continue; + } + rolename = oneline.substr(pos + strlen(IAMDEFROLE_PROFARN_PART)); + + return !rolename.empty(); + } + } + return false; +} + +bool S3fsCurl::SetIAMRoleFromMetaData(const char* response) +{ + S3FS_PRN_INFO3("IAM role name response = \"%s\"", response); + + string rolename; + + if(!S3fsCurl::ParseIAMRoleFromMetaDataResponse(response, rolename)){ + return false; + } + + SetIAMRole(rolename.c_str()); + return true; +} + bool S3fsCurl::AddUserAgent(CURL* hCurl) { if(!hCurl){ @@ -1523,8 +1586,8 @@ bool S3fsCurl::ResetHandle(void) curl_easy_setopt(hCurl, CURLOPT_PROGRESSDATA, hCurl); // curl_easy_setopt(hCurl, CURLOPT_FORBID_REUSE, 1); - if(type != REQTYPE_IAMCRED){ - // REQTYPE_IAMCRED is always HTTP + if(type != REQTYPE_IAMCRED && type != REQTYPE_IAMROLE){ + // REQTYPE_IAMCRED and REQTYPE_IAMROLE are always HTTP if(0 == S3fsCurl::ssl_verify_hostname){ curl_easy_setopt(hCurl, CURLOPT_SSL_VERIFYHOST, 0); } @@ -1575,11 +1638,11 @@ bool S3fsCurl::CreateCurlHandle(bool force) } // [NOTE] - // If type is REQTYPE_IAMCRED, do not clear type. + // If type is REQTYPE_IAMCRED or REQTYPE_IAMROLE, do not clear type. // Because that type only uses HTTP protocol, then the special // logic in ResetHandle function. // - if(type != REQTYPE_IAMCRED){ + if(type != REQTYPE_IAMCRED && type != REQTYPE_IAMROLE){ type = REQTYPE_UNSET; } @@ -1834,6 +1897,12 @@ bool S3fsCurl::RemakeHandle(void) curl_easy_setopt(hCurl, CURLOPT_HTTPHEADER, requestHeaders); break; + case REQTYPE_IAMROLE: + curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str()); + curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, (void*)bodydata); + curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback); + break; + default: S3FS_PRN_ERR("request type is unknown(%d)", type); return false; @@ -2300,6 +2369,44 @@ int S3fsCurl::GetIAMCredentials(void) return result; } +// +// Get IAM role name automatically. +// +bool S3fsCurl::LoadIAMRoleFromMetaData(void) +{ + S3FS_PRN_INFO3("Get IAM Role name"); + + // at first set type for handle + type = REQTYPE_IAMROLE; + + if(!CreateCurlHandle(true)){ + return false; + } + + // url + url = IAM_DEFAULT_ROLE_URL; + requestHeaders = NULL; + responseHeaders.clear(); + bodydata = new BodyData(); + + curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str()); + curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, (void*)bodydata); + curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback); + S3fsCurl::AddUserAgent(hCurl); // put User-Agent + + int result = RequestPerform(); + + // analizing response + if(0 == result && !S3fsCurl::SetIAMRoleFromMetaData(bodydata->str())){ + S3FS_PRN_ERR("Something error occurred, could not get IAM role name."); + result = -EIO; + } + delete bodydata; + bodydata = NULL; + + return (0 == result); +} + bool S3fsCurl::AddSseRequestHead(sse_type_t ssetype, string& ssevalue, bool is_only_c, bool is_copy) { if(SSE_S3 == ssetype){ diff --git a/src/curl.h b/src/curl.h index d174f16..78e19a0 100644 --- a/src/curl.h +++ b/src/curl.h @@ -201,7 +201,8 @@ class S3fsCurl REQTYPE_COPYMULTIPOST, REQTYPE_MULTILIST, REQTYPE_IAMCRED, - REQTYPE_ABORTMULTIUPLOAD + REQTYPE_ABORTMULTIUPLOAD, + REQTYPE_IAMROLE }; // class variables @@ -297,6 +298,8 @@ class S3fsCurl static bool ParseIAMCredentialResponse(const char* response, iamcredmap_t& keyval); static bool SetIAMCredentials(const char* response); + static bool ParseIAMRoleFromMetaDataResponse(const char* response, std::string& rolename); + static bool SetIAMRoleFromMetaData(const char* response); static bool LoadEnvSseCKeys(void); static bool LoadEnvSseKmsid(void); static bool PushbackSseKeys(std::string& onekey); @@ -378,6 +381,7 @@ class S3fsCurl bool CreateCurlHandle(bool force = false); bool DestroyCurlHandle(void); + bool LoadIAMRoleFromMetaData(void); bool AddSseRequestHead(sse_type_t ssetype, std::string& ssevalue, bool is_only_c, bool is_copy); bool GetResponseCode(long& responseCode); int RequestPerform(void); diff --git a/src/s3fs.cpp b/src/s3fs.cpp index 093777d..ed2ba0b 100644 --- a/src/s3fs.cpp +++ b/src/s3fs.cpp @@ -113,6 +113,7 @@ static bool nocopyapi = false; static bool norenameapi = false; static bool nonempty = false; static bool allow_other = false; +static bool load_iamrole = false; static uid_t s3fs_uid = 0; static gid_t s3fs_gid = 0; static mode_t s3fs_umask = 0; @@ -3351,6 +3352,19 @@ static void* s3fs_init(struct fuse_conn_info* conn) return NULL; } + // check loading IAM role name + if(load_iamrole){ + // load IAM role name from http://169.254.169.254/latest/meta-data/iam/info + // + S3fsCurl s3fscurl; + if(!s3fscurl.LoadIAMRoleFromMetaData()){ + S3FS_PRN_CRIT("could not load IAM role name from meta data."); + s3fs_exit_fuseloop(EXIT_FAILURE); + return NULL; + } + S3FS_PRN_INFO("loaded IAM role name = %s", S3fsCurl::GetIAMRole()); + } + if (create_bucket){ do_create_bucket(); } @@ -4447,10 +4461,19 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar passwd_file = strchr(arg, '=') + sizeof(char); return 0; } - if(0 == STR2NCMP(arg, "iam_role=")){ - const char* role = strchr(arg, '=') + sizeof(char); - S3fsCurl::SetIAMRole(role); - return 0; + if(0 == STR2NCMP(arg, "iam_role")){ + if(0 == strcmp(arg, "iam_role") || 0 == strcmp(arg, "iam_role=auto")){ + // loading IAM role name in s3fs_init(), because we need to wait initializing curl. + // + load_iamrole = true; + return 0; + + }else if(0 == STR2NCMP(arg, "iam_role=")){ + const char* role = strchr(arg, '=') + sizeof(char); + S3fsCurl::SetIAMRole(role); + load_iamrole = false; + return 0; + } } if(0 == STR2NCMP(arg, "public_bucket=")){ off_t pubbucket = s3fs_strtoofft(strchr(arg, '=') + sizeof(char)); diff --git a/src/s3fs_util.cpp b/src/s3fs_util.cpp index 4c5d151..2999da9 100644 --- a/src/s3fs_util.cpp +++ b/src/s3fs_util.cpp @@ -1077,9 +1077,11 @@ void show_help (void) " enable_content_md5 (default is disable)\n" " - ensure data integrity during writes with MD5 hash.\n" "\n" - " iam_role (default is no role)\n" - " - set the IAM Role that will supply the credentials from the \n" - " instance meta-data.\n" + " iam_role (default is no IAM role)\n" + " - This option requires the IAM role name or \"auto\". If you specify\n" + " \"auto\", s3fs will automatically use the IAM role names that are set\n" + " to an instance. If you specify this option without any argument, it\n" + " is the same as that you have specified the \"auto\".\n" "\n" " noxmlns (disable registering xml name space)\n" " disable registering xml name space for response of \n"