Merge pull request #411 from ggtakec/master

loading IAM role name automatically(iam_role option) - #387
This commit is contained in:
Takeshi Nakatani 2016-05-06 13:57:32 +09:00
commit d16d616f34
5 changed files with 150 additions and 14 deletions

View File

@ -197,8 +197,8 @@ Enable to send "Content-MD5" header when uploading a object without multipart po
If this option is enabled, it has some influences on a performance of s3fs when uploading small object. If this option is enabled, it has some influences on a performance of s3fs when uploading small object.
Because s3fs always checks MD5 when uploading large object, this option does not affect on large object. Because s3fs always checks MD5 when uploading large object, this option does not affect on large object.
.TP .TP
\fB\-o\fR iam_role ( default is no role ) \fB\-o\fR iam_role ( default is no IAM role )
set the IAM Role that will supply the credentials from the instance meta-data. This option requires the IAM role name or "auto". If you specify "auto", s3fs will automatically use the IAM role names that are set to an instance. If you specify this option without any argument, it is the same as that you have specified the "auto".
.TP .TP
\fB\-o\fR noxmlns - disable registing xml name space. \fB\-o\fR noxmlns - disable registing xml name space.
disable registing xml name space for response of ListBucketResult and ListVersionsResult etc. Default name space is looked up from "http://s3.amazonaws.com/doc/2006-03-01". disable registing xml name space for response of ListBucketResult and ListVersionsResult etc. Default name space is looked up from "http://s3.amazonaws.com/doc/2006-03-01".

View File

@ -315,6 +315,9 @@ void CurlHandlerPool::ReturnHandler(CURL* h)
#define IAMCRED_ACCESSTOKEN "Token" #define IAMCRED_ACCESSTOKEN "Token"
#define IAMCRED_EXPIRATION "Expiration" #define IAMCRED_EXPIRATION "Expiration"
#define IAMCRED_KEYCOUNT 4 #define IAMCRED_KEYCOUNT 4
#define IAM_DEFAULT_ROLE_URL "http://169.254.169.254/latest/meta-data/iam/info"
#define IAMDEFROLE_PROFARN "InstanceProfileArn"
#define IAMDEFROLE_PROFARN_PART ":instance-profile/"
// [NOTICE] // [NOTICE]
// This symbol is for libcurl under 7.23.0 // This symbol is for libcurl under 7.23.0
@ -1436,6 +1439,66 @@ bool S3fsCurl::CheckIAMCredentialUpdate(void)
return true; return true;
} }
bool S3fsCurl::ParseIAMRoleFromMetaDataResponse(const char* response, string& rolename)
{
if(!response){
return false;
}
// [NOTE]
// expected following strings.
//
// {
// "Code" : "Success",
// "LastUpdated" : "2016-01-01T00:00:00Z",
// "InstanceProfileArn" : "arn:aws:iam::111111111111:instance-profile/myrolename",
// "InstanceProfileId" : "AAAAAAAAAAAAAAAAAAAAA"
// }
//
istringstream ssrole(response);
string oneline;
while(getline(ssrole, oneline, '\n')){
string::size_type pos;
if(string::npos != (pos = oneline.find(IAMDEFROLE_PROFARN))){
if(string::npos == (pos = oneline.find(':', pos + strlen(IAMDEFROLE_PROFARN)))){
continue;
}
if(string::npos == (pos = oneline.find('\"', pos))){
continue;
}
// value
oneline = oneline.substr(pos + sizeof(char));
if(string::npos == (pos = oneline.find('\"'))){
continue;
}
oneline = oneline.substr(0, pos);
// role name
if(string::npos == (pos = oneline.find(IAMDEFROLE_PROFARN_PART))){
continue;
}
rolename = oneline.substr(pos + strlen(IAMDEFROLE_PROFARN_PART));
return !rolename.empty();
}
}
return false;
}
bool S3fsCurl::SetIAMRoleFromMetaData(const char* response)
{
S3FS_PRN_INFO3("IAM role name response = \"%s\"", response);
string rolename;
if(!S3fsCurl::ParseIAMRoleFromMetaDataResponse(response, rolename)){
return false;
}
SetIAMRole(rolename.c_str());
return true;
}
bool S3fsCurl::AddUserAgent(CURL* hCurl) bool S3fsCurl::AddUserAgent(CURL* hCurl)
{ {
if(!hCurl){ if(!hCurl){
@ -1523,8 +1586,8 @@ bool S3fsCurl::ResetHandle(void)
curl_easy_setopt(hCurl, CURLOPT_PROGRESSDATA, hCurl); curl_easy_setopt(hCurl, CURLOPT_PROGRESSDATA, hCurl);
// curl_easy_setopt(hCurl, CURLOPT_FORBID_REUSE, 1); // curl_easy_setopt(hCurl, CURLOPT_FORBID_REUSE, 1);
if(type != REQTYPE_IAMCRED){ if(type != REQTYPE_IAMCRED && type != REQTYPE_IAMROLE){
// REQTYPE_IAMCRED is always HTTP // REQTYPE_IAMCRED and REQTYPE_IAMROLE are always HTTP
if(0 == S3fsCurl::ssl_verify_hostname){ if(0 == S3fsCurl::ssl_verify_hostname){
curl_easy_setopt(hCurl, CURLOPT_SSL_VERIFYHOST, 0); curl_easy_setopt(hCurl, CURLOPT_SSL_VERIFYHOST, 0);
} }
@ -1575,11 +1638,11 @@ bool S3fsCurl::CreateCurlHandle(bool force)
} }
// [NOTE] // [NOTE]
// If type is REQTYPE_IAMCRED, do not clear type. // If type is REQTYPE_IAMCRED or REQTYPE_IAMROLE, do not clear type.
// Because that type only uses HTTP protocol, then the special // Because that type only uses HTTP protocol, then the special
// logic in ResetHandle function. // logic in ResetHandle function.
// //
if(type != REQTYPE_IAMCRED){ if(type != REQTYPE_IAMCRED && type != REQTYPE_IAMROLE){
type = REQTYPE_UNSET; type = REQTYPE_UNSET;
} }
@ -1834,6 +1897,12 @@ bool S3fsCurl::RemakeHandle(void)
curl_easy_setopt(hCurl, CURLOPT_HTTPHEADER, requestHeaders); curl_easy_setopt(hCurl, CURLOPT_HTTPHEADER, requestHeaders);
break; break;
case REQTYPE_IAMROLE:
curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str());
curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, (void*)bodydata);
curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
break;
default: default:
S3FS_PRN_ERR("request type is unknown(%d)", type); S3FS_PRN_ERR("request type is unknown(%d)", type);
return false; return false;
@ -2300,6 +2369,44 @@ int S3fsCurl::GetIAMCredentials(void)
return result; return result;
} }
//
// Get IAM role name automatically.
//
bool S3fsCurl::LoadIAMRoleFromMetaData(void)
{
S3FS_PRN_INFO3("Get IAM Role name");
// at first set type for handle
type = REQTYPE_IAMROLE;
if(!CreateCurlHandle(true)){
return false;
}
// url
url = IAM_DEFAULT_ROLE_URL;
requestHeaders = NULL;
responseHeaders.clear();
bodydata = new BodyData();
curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str());
curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, (void*)bodydata);
curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
S3fsCurl::AddUserAgent(hCurl); // put User-Agent
int result = RequestPerform();
// analizing response
if(0 == result && !S3fsCurl::SetIAMRoleFromMetaData(bodydata->str())){
S3FS_PRN_ERR("Something error occurred, could not get IAM role name.");
result = -EIO;
}
delete bodydata;
bodydata = NULL;
return (0 == result);
}
bool S3fsCurl::AddSseRequestHead(sse_type_t ssetype, string& ssevalue, bool is_only_c, bool is_copy) bool S3fsCurl::AddSseRequestHead(sse_type_t ssetype, string& ssevalue, bool is_only_c, bool is_copy)
{ {
if(SSE_S3 == ssetype){ if(SSE_S3 == ssetype){

View File

@ -201,7 +201,8 @@ class S3fsCurl
REQTYPE_COPYMULTIPOST, REQTYPE_COPYMULTIPOST,
REQTYPE_MULTILIST, REQTYPE_MULTILIST,
REQTYPE_IAMCRED, REQTYPE_IAMCRED,
REQTYPE_ABORTMULTIUPLOAD REQTYPE_ABORTMULTIUPLOAD,
REQTYPE_IAMROLE
}; };
// class variables // class variables
@ -297,6 +298,8 @@ class S3fsCurl
static bool ParseIAMCredentialResponse(const char* response, iamcredmap_t& keyval); static bool ParseIAMCredentialResponse(const char* response, iamcredmap_t& keyval);
static bool SetIAMCredentials(const char* response); static bool SetIAMCredentials(const char* response);
static bool ParseIAMRoleFromMetaDataResponse(const char* response, std::string& rolename);
static bool SetIAMRoleFromMetaData(const char* response);
static bool LoadEnvSseCKeys(void); static bool LoadEnvSseCKeys(void);
static bool LoadEnvSseKmsid(void); static bool LoadEnvSseKmsid(void);
static bool PushbackSseKeys(std::string& onekey); static bool PushbackSseKeys(std::string& onekey);
@ -378,6 +381,7 @@ class S3fsCurl
bool CreateCurlHandle(bool force = false); bool CreateCurlHandle(bool force = false);
bool DestroyCurlHandle(void); bool DestroyCurlHandle(void);
bool LoadIAMRoleFromMetaData(void);
bool AddSseRequestHead(sse_type_t ssetype, std::string& ssevalue, bool is_only_c, bool is_copy); bool AddSseRequestHead(sse_type_t ssetype, std::string& ssevalue, bool is_only_c, bool is_copy);
bool GetResponseCode(long& responseCode); bool GetResponseCode(long& responseCode);
int RequestPerform(void); int RequestPerform(void);

View File

@ -113,6 +113,7 @@ static bool nocopyapi = false;
static bool norenameapi = false; static bool norenameapi = false;
static bool nonempty = false; static bool nonempty = false;
static bool allow_other = false; static bool allow_other = false;
static bool load_iamrole = false;
static uid_t s3fs_uid = 0; static uid_t s3fs_uid = 0;
static gid_t s3fs_gid = 0; static gid_t s3fs_gid = 0;
static mode_t s3fs_umask = 0; static mode_t s3fs_umask = 0;
@ -3351,6 +3352,19 @@ static void* s3fs_init(struct fuse_conn_info* conn)
return NULL; return NULL;
} }
// check loading IAM role name
if(load_iamrole){
// load IAM role name from http://169.254.169.254/latest/meta-data/iam/info
//
S3fsCurl s3fscurl;
if(!s3fscurl.LoadIAMRoleFromMetaData()){
S3FS_PRN_CRIT("could not load IAM role name from meta data.");
s3fs_exit_fuseloop(EXIT_FAILURE);
return NULL;
}
S3FS_PRN_INFO("loaded IAM role name = %s", S3fsCurl::GetIAMRole());
}
if (create_bucket){ if (create_bucket){
do_create_bucket(); do_create_bucket();
} }
@ -4447,11 +4461,20 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
passwd_file = strchr(arg, '=') + sizeof(char); passwd_file = strchr(arg, '=') + sizeof(char);
return 0; return 0;
} }
if(0 == STR2NCMP(arg, "iam_role=")){ if(0 == STR2NCMP(arg, "iam_role")){
if(0 == strcmp(arg, "iam_role") || 0 == strcmp(arg, "iam_role=auto")){
// loading IAM role name in s3fs_init(), because we need to wait initializing curl.
//
load_iamrole = true;
return 0;
}else if(0 == STR2NCMP(arg, "iam_role=")){
const char* role = strchr(arg, '=') + sizeof(char); const char* role = strchr(arg, '=') + sizeof(char);
S3fsCurl::SetIAMRole(role); S3fsCurl::SetIAMRole(role);
load_iamrole = false;
return 0; return 0;
} }
}
if(0 == STR2NCMP(arg, "public_bucket=")){ if(0 == STR2NCMP(arg, "public_bucket=")){
off_t pubbucket = s3fs_strtoofft(strchr(arg, '=') + sizeof(char)); off_t pubbucket = s3fs_strtoofft(strchr(arg, '=') + sizeof(char));
if(1 == pubbucket){ if(1 == pubbucket){

View File

@ -1077,9 +1077,11 @@ void show_help (void)
" enable_content_md5 (default is disable)\n" " enable_content_md5 (default is disable)\n"
" - ensure data integrity during writes with MD5 hash.\n" " - ensure data integrity during writes with MD5 hash.\n"
"\n" "\n"
" iam_role (default is no role)\n" " iam_role (default is no IAM role)\n"
" - set the IAM Role that will supply the credentials from the \n" " - This option requires the IAM role name or \"auto\". If you specify\n"
" instance meta-data.\n" " \"auto\", s3fs will automatically use the IAM role names that are set\n"
" to an instance. If you specify this option without any argument, it\n"
" is the same as that you have specified the \"auto\".\n"
"\n" "\n"
" noxmlns (disable registering xml name space)\n" " noxmlns (disable registering xml name space)\n"
" disable registering xml name space for response of \n" " disable registering xml name space for response of \n"