octoleo/s3fs-fuse
1
0
Fork 0
mirror of https://github.com/s3fs-fuse/s3fs-fuse.git synced 2025-02-08 21:48:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

Changed handling the credential in S3fsCred more robust

Browse Source
This commit is contained in:
Takeshi Nakatani 2022-02-23 15:03:08 +00:00 committed by Andrew Gaul
parent afb0897553
commit 684ced5a41
5 changed files with 424 additions and 238 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

215
src/curl.cpp
View File

@ -2552,13 +2552,13 @@ int S3fsCurl::RequestPerform(bool dontAddAuthHeaders /*=false*/)
// @param date e.g., get_date_rfc850() // @param date e.g., get_date_rfc850()
// @param resource e.g., "/pub" // @param resource e.g., "/pub"
// //
std::string S3fsCurl::CalcSignatureV2(const std::string& method, const std::string& strMD5, const std::string& content_type, const std::string& date, const std::string& resource) std::string S3fsCurl::CalcSignatureV2(const std::string& method, const std::string& strMD5, const std::string& content_type, const std::string& date, const std::string& resource, const std::string& secret_access_key, const std::string& access_token)
{ {
std::string Signature; std::string Signature;
std::string StringToSign; std::string StringToSign;
if(!S3fsCurl::ps3fscred->GetIAMRole().empty() || S3fsCurl::ps3fscred->IsECS() || S3fsCurl::ps3fscred->IsUseSessionToken()){ if(!access_token.empty()){
requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-security-token", S3fsCurl::ps3fscred->GetAccessToken().c_str()); requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-security-token", access_token.c_str());
} }
StringToSign += method + "\n"; StringToSign += method + "\n";
@ -2568,8 +2568,8 @@ std::string S3fsCurl::CalcSignatureV2(const std::string& method, const std::stri
StringToSign += get_canonical_headers(requestHeaders, true); StringToSign += get_canonical_headers(requestHeaders, true);
StringToSign += resource; StringToSign += resource;
const void* key = S3fsCurl::ps3fscred->GetSecretAccessKey().data(); const void* key = secret_access_key.data();
size_t key_len = S3fsCurl::ps3fscred->GetSecretAccessKey().size(); size_t key_len = secret_access_key.size();
const unsigned char* sdata = reinterpret_cast<const unsigned char*>(StringToSign.data()); const unsigned char* sdata = reinterpret_cast<const unsigned char*>(StringToSign.data());
size_t sdata_len = StringToSign.size(); size_t sdata_len = StringToSign.size();
unsigned char* md = NULL; unsigned char* md = NULL;
@ -2590,13 +2590,13 @@ std::string S3fsCurl::CalcSignatureV2(const std::string& method, const std::stri
return Signature; return Signature;
} }
std::string S3fsCurl::CalcSignature(const std::string& method, const std::string& canonical_uri, const std::string& query_string, const std::string& strdate, const std::string& payload_hash, const std::string& date8601) std::string S3fsCurl::CalcSignature(const std::string& method, const std::string& canonical_uri, const std::string& query_string, const std::string& strdate, const std::string& payload_hash, const std::string& date8601, const std::string& secret_access_key, const std::string& access_token)
{ {
std::string Signature, StringCQ, StringToSign; std::string Signature, StringCQ, StringToSign;
std::string uriencode; std::string uriencode;
if(!S3fsCurl::ps3fscred->GetIAMRole().empty() || S3fsCurl::ps3fscred->IsECS() || S3fsCurl::ps3fscred->IsUseSessionToken()){ if(!access_token.empty()){
requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-security-token", S3fsCurl::ps3fscred->GetAccessToken().c_str()); requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-security-token", access_token.c_str());
} }
uriencode = urlEncode(canonical_uri); uriencode = urlEncode(canonical_uri);
@ -2617,7 +2617,7 @@ std::string S3fsCurl::CalcSignature(const std::string& method, const std::string
StringCQ += get_sorted_header_keys(requestHeaders) + "\n"; StringCQ += get_sorted_header_keys(requestHeaders) + "\n";
StringCQ += payload_hash; StringCQ += payload_hash;
std::string kSecret = "AWS4" + S3fsCurl::ps3fscred->GetSecretAccessKey(); std::string kSecret = "AWS4" + secret_access_key;
unsigned char *kDate, *kRegion, *kService, *kSigning, *sRequest = NULL; unsigned char *kDate, *kRegion, *kService, *kSigning, *sRequest = NULL;
unsigned int kDate_len,kRegion_len, kService_len, kSigning_len, sRequest_len = 0; unsigned int kDate_len,kRegion_len, kService_len, kSigning_len, sRequest_len = 0;
@ -2653,7 +2653,7 @@ std::string S3fsCurl::CalcSignature(const std::string& method, const std::string
return Signature; return Signature;
} }
void S3fsCurl::insertV4Headers() void S3fsCurl::insertV4Headers(const std::string& access_key_id, const std::string& secret_access_key, const std::string& access_token)
{ {
std::string server_path = type == REQTYPE_LISTBUCKET ? "/" : path; std::string server_path = type == REQTYPE_LISTBUCKET ? "/" : path;
std::string payload_hash; std::string payload_hash;
@ -2711,13 +2711,13 @@ void S3fsCurl::insertV4Headers()
} }
if(!S3fsCurl::IsPublicBucket()){ if(!S3fsCurl::IsPublicBucket()){
std::string Signature = CalcSignature(op, realpath, query_string + (type == REQTYPE_PREMULTIPOST || type == REQTYPE_MULTILIST ? "=" : ""), strdate, contentSHA256, date8601); std::string Signature = CalcSignature(op, realpath, query_string + (type == REQTYPE_PREMULTIPOST || type == REQTYPE_MULTILIST ? "=" : ""), strdate, contentSHA256, date8601, secret_access_key, access_token);
std::string auth = "AWS4-HMAC-SHA256 Credential=" + S3fsCurl::ps3fscred->GetAccessKeyID() + "/" + strdate + "/" + endpoint + "/s3/aws4_request, SignedHeaders=" + get_sorted_header_keys(requestHeaders) + ", Signature=" + Signature; std::string auth = "AWS4-HMAC-SHA256 Credential=" + access_key_id + "/" + strdate + "/" + endpoint + "/s3/aws4_request, SignedHeaders=" + get_sorted_header_keys(requestHeaders) + ", Signature=" + Signature;
requestHeaders = curl_slist_sort_insert(requestHeaders, "Authorization", auth.c_str()); requestHeaders = curl_slist_sort_insert(requestHeaders, "Authorization", auth.c_str());
} }
} }
void S3fsCurl::insertV2Headers() void S3fsCurl::insertV2Headers(const std::string& access_key_id, const std::string& secret_access_key, const std::string& access_token)
{ {
std::string resource; std::string resource;
std::string turl; std::string turl;
@ -2734,34 +2734,39 @@ void S3fsCurl::insertV2Headers()
} }
if(!S3fsCurl::IsPublicBucket()){ if(!S3fsCurl::IsPublicBucket()){
std::string Signature = CalcSignatureV2(op, get_header_value(requestHeaders, "Content-MD5"), get_header_value(requestHeaders, "Content-Type"), date, resource); std::string Signature = CalcSignatureV2(op, get_header_value(requestHeaders, "Content-MD5"), get_header_value(requestHeaders, "Content-Type"), date, resource, secret_access_key, access_token);
requestHeaders = curl_slist_sort_insert(requestHeaders, "Authorization", std::string("AWS " + S3fsCurl::ps3fscred->GetAccessKeyID() + ":" + Signature).c_str()); requestHeaders = curl_slist_sort_insert(requestHeaders, "Authorization", std::string("AWS " + access_key_id + ":" + Signature).c_str());
} }
} }
void S3fsCurl::insertIBMIAMHeaders() void S3fsCurl::insertIBMIAMHeaders(const std::string& access_key_id, const std::string& access_token)
{ {
requestHeaders = curl_slist_sort_insert(requestHeaders, "Authorization", ("Bearer " + S3fsCurl::ps3fscred->GetAccessToken()).c_str()); requestHeaders = curl_slist_sort_insert(requestHeaders, "Authorization", ("Bearer " + access_token).c_str());
if(op == "PUT" && path == mount_prefix + "/"){ if(op == "PUT" && path == mount_prefix + "/"){
// ibm-service-instance-id header is required for bucket creation requests // ibm-service-instance-id header is required for bucket creation requests
requestHeaders = curl_slist_sort_insert(requestHeaders, "ibm-service-instance-id", S3fsCurl::ps3fscred->GetAccessKeyID().c_str()); requestHeaders = curl_slist_sort_insert(requestHeaders, "ibm-service-instance-id", access_key_id.c_str());
} }
} }
void S3fsCurl::insertAuthHeaders() void S3fsCurl::insertAuthHeaders()
{ {
if(!S3fsCurl::ps3fscred->CheckIAMCredentialUpdate()){ std::string access_key_id;
std::string secret_access_key;
std::string access_token;
// check and get credential variables
if(!S3fsCurl::ps3fscred->CheckIAMCredentialUpdate(&access_key_id, &secret_access_key, &access_token)){
S3FS_PRN_ERR("An error occurred in checking IAM credential."); S3FS_PRN_ERR("An error occurred in checking IAM credential.");
return; // do not insert auth headers on error return; // do not insert auth headers on error
} }
if(S3fsCurl::ps3fscred->IsIBMIAMAuth()){ if(S3fsCurl::ps3fscred->IsIBMIAMAuth()){
insertIBMIAMHeaders(); insertIBMIAMHeaders(access_key_id, access_token);
}else if(S3fsCurl::signature_type == V2_ONLY){ }else if(S3fsCurl::signature_type == V2_ONLY){
insertV2Headers(); insertV2Headers(access_key_id, secret_access_key, access_token);
}else{ }else{
insertV4Headers(); insertV4Headers(access_key_id, secret_access_key, access_token);
} }
} }
@ -2800,12 +2805,14 @@ int S3fsCurl::DeleteRequest(const char* tpath)
return RequestPerform(); return RequestPerform();
} }
// int S3fsCurl::GetIAMv2ApiToken(const char* token_url, int token_ttl, const char* token_ttl_hdr, std::string& response)
// Get the token that we need to pass along with AWS IMDSv2 API requests
//
int S3fsCurl::GetIAMv2ApiToken()
{ {
url = std::string(S3fsCred::IAMv2_token_url); if(!token_url || !token_ttl_hdr){
S3FS_PRN_ERR("IAMv2 token url(%s) or ttl_hdr(%s) parameter are wrong.", token_url ? token_url : "null", token_ttl_hdr ? token_ttl_hdr : "null");
return -EIO;
}
response.erase();
url = std::string(token_url);
if(!CreateCurlHandle()){ if(!CreateCurlHandle()){
return -EIO; return -EIO;
} }
@ -2813,8 +2820,8 @@ int S3fsCurl::GetIAMv2ApiToken()
responseHeaders.clear(); responseHeaders.clear();
bodydata.Clear(); bodydata.Clear();
std::string ttlstr = str(S3fsCred::IAMv2_token_ttl); std::string ttlstr = str(token_ttl);
requestHeaders = curl_slist_sort_insert(requestHeaders, S3fsCred::IAMv2_token_ttl_hdr, ttlstr.c_str()); requestHeaders = curl_slist_sort_insert(requestHeaders, token_ttl_hdr, ttlstr.c_str());
// Curl appends an "Expect: 100-continue" header to the token request, // Curl appends an "Expect: 100-continue" header to the token request,
// and aws responds with a 417 Expectation Failed. This ensures the // and aws responds with a 417 Expectation Failed. This ensures the
@ -2837,16 +2844,18 @@ int S3fsCurl::GetIAMv2ApiToken()
return -EIO; return -EIO;
} }
// [NOTE]
// Be sure to give "dontAddAuthHeaders=true".
// If set to false(default), it will deadlock in S3fsCred.
//
int result = RequestPerform(true); int result = RequestPerform(true);
if(0 == result && !S3fsCurl::ps3fscred->SetIAMv2APIToken(bodydata.str())){ if(0 == result){
S3FS_PRN_ERR("Error storing IMDSv2 API token."); response = bodydata.str();
result = -EIO; }else{
S3FS_PRN_ERR("Error(%d) occurred, could not get IAMv2 api token.", result);
} }
bodydata.Clear(); bodydata.Clear();
if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_PUT, false)){
return -EIO;
}
return result; return result;
} }
@ -2855,64 +2864,31 @@ int S3fsCurl::GetIAMv2ApiToken()
// Get AccessKeyId/SecretAccessKey/AccessToken/Expiration by IAM role, // Get AccessKeyId/SecretAccessKey/AccessToken/Expiration by IAM role,
// and Set these value to class variable. // and Set these value to class variable.
// //
int S3fsCurl::GetIAMCredentials() bool S3fsCurl::GetIAMCredentials(const char* cred_url, const char* iam_v2_token, const char* ibm_secret_access_key, std::string& response)
{ {
if (!S3fsCurl::ps3fscred->IsECS() && !S3fsCurl::ps3fscred->IsIBMIAMAuth()) { if(!cred_url){
S3FS_PRN_INFO3("[IAM role=%s]", S3fsCurl::ps3fscred->GetIAMRole().c_str()); S3FS_PRN_ERR("url is null.");
return false;
if(S3fsCurl::ps3fscred->GetIAMRole().empty()) {
S3FS_PRN_ERR("IAM role name is empty.");
return -EIO;
}
} }
url = cred_url;
response.erase();
// at first set type for handle // at first set type for handle
type = REQTYPE_IAMCRED; type = REQTYPE_IAMCRED;
if(!CreateCurlHandle()){ if(!CreateCurlHandle()){
return -EIO; return false;
} }
// url
if(S3fsCurl::ps3fscred->IsECS()){
const char *env = std::getenv(S3fsCred::ECS_IAM_ENV_VAR);
if(env == NULL){
S3FS_PRN_ERR("%s is not set.", S3fsCred::ECS_IAM_ENV_VAR);
return -EIO;
}
url = S3fsCurl::ps3fscred->GetIAMCredentialsURL() + env;
}else{
if(S3fsCurl::ps3fscred->GetIMDSVersion() > 1){
int result = GetIAMv2ApiToken();
if(-ENOENT == result){
// If we get a 404 back when requesting the token service,
// then it's highly likely we're running in an environment
// that doesn't support the AWS IMDSv2 API, so we'll skip
// the token retrieval in the future.
S3fsCurl::ps3fscred->SetIMDSVersion(1);
}else if(result != 0){
// If we get an unexpected error when retrieving the API
// token, log it but continue. Requirement for including
// an API token with the metadata request may or may not
// be required, so we should not abort here.
S3FS_PRN_ERR("AWS IMDSv2 token retrieval failed: %d", result);
}
}
url = S3fsCurl::ps3fscred->GetIAMCredentialsURL() + S3fsCurl::ps3fscred->GetIAMRole();
}
requestHeaders = NULL; requestHeaders = NULL;
responseHeaders.clear(); responseHeaders.clear();
bodydata.Clear(); bodydata.Clear();
std::string postContent; std::string postContent;
if(S3fsCurl::ps3fscred->IsIBMIAMAuth()){ if(ibm_secret_access_key){
url = S3fsCurl::ps3fscred->GetIAMCredentialsURL();
// make contents // make contents
postContent += "grant_type=urn:ibm:params:oauth:grant-type:apikey"; postContent += "grant_type=urn:ibm:params:oauth:grant-type:apikey";
postContent += "&response_type=cloud_iam"; postContent += "&response_type=cloud_iam";
postContent += "&apikey=" + S3fsCurl::ps3fscred->GetSecretAccessKey(); postContent += "&apikey=" + std::string(ibm_secret_access_key);
// set postdata // set postdata
postdata = reinterpret_cast<const unsigned char*>(postContent.c_str()); postdata = reinterpret_cast<const unsigned char*>(postContent.c_str());
@ -2923,53 +2899,65 @@ int S3fsCurl::GetIAMCredentials()
requestHeaders = curl_slist_sort_insert(requestHeaders, "Authorization", "Basic Yng6Yng="); requestHeaders = curl_slist_sort_insert(requestHeaders, "Authorization", "Basic Yng6Yng=");
if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_POST, true)){ // POST if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_POST, true)){ // POST
return -EIO; return false;
} }
if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_POSTFIELDSIZE, static_cast<curl_off_t>(postdata_remaining))){ if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_POSTFIELDSIZE, static_cast<curl_off_t>(postdata_remaining))){
return -EIO; return false;
} }
if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_READDATA, (void*)this)){ if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_READDATA, (void*)this)){
return -EIO; return false;
} }
if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_READFUNCTION, S3fsCurl::ReadCallback)){ if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_READFUNCTION, S3fsCurl::ReadCallback)){
return -EIO; return false;
} }
} }
if(S3fsCurl::ps3fscred->GetIMDSVersion() > 1){ if(iam_v2_token){
requestHeaders = curl_slist_sort_insert(requestHeaders, S3fsCred::IAMv2_token_hdr, S3fsCurl::ps3fscred->GetIAMv2APIToken().c_str()); requestHeaders = curl_slist_sort_insert(requestHeaders, S3fsCred::IAMv2_token_hdr, iam_v2_token);
} }
if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str())){ if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str())){
return -EIO; return false;
} }
if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, (void*)&bodydata)){ if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, (void*)&bodydata)){
return -EIO; return false;
} }
if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback)){ if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback)){
return -EIO; return false;
} }
if(!S3fsCurl::AddUserAgent(hCurl)){ // put User-Agent if(!S3fsCurl::AddUserAgent(hCurl)){ // put User-Agent
return -EIO; return false;
} }
// [NOTE]
// Be sure to give "dontAddAuthHeaders=true".
// If set to false(default), it will deadlock in S3fsCred.
//
int result = RequestPerform(true); int result = RequestPerform(true);
// analyzing response // analyzing response
if(0 == result && !S3fsCurl::ps3fscred->SetIAMCredentials(bodydata.str())){ if(0 == result){
S3FS_PRN_ERR("Something error occurred, could not get IAM credential."); response = bodydata.str();
result = -EIO; }else{
S3FS_PRN_ERR("Error(%d) occurred, could not get IAM role name.", result);
} }
bodydata.Clear(); bodydata.Clear();
return result; return (0 == result);
} }
// //
// Get IAM role name automatically. // Get IAM role name automatically.
// //
bool S3fsCurl::LoadIAMRoleFromMetaData() bool S3fsCurl::GetIAMRoleFromMetaData(const char* cred_url, const char* iam_v2_token, std::string& token)
{ {
if(!cred_url){
S3FS_PRN_ERR("url is null.");
return false;
}
url = cred_url;
token.erase();
S3FS_PRN_INFO3("Get IAM Role name"); S3FS_PRN_INFO3("Get IAM Role name");
// at first set type for handle // at first set type for handle
@ -2978,40 +2966,12 @@ bool S3fsCurl::LoadIAMRoleFromMetaData()
if(!CreateCurlHandle()){ if(!CreateCurlHandle()){
return false; return false;
} }
// url
if(S3fsCurl::ps3fscred->IsECS()){
const char *env = std::getenv(S3fsCred::ECS_IAM_ENV_VAR);
if(env == NULL){
S3FS_PRN_ERR("%s is not set.", S3fsCred::ECS_IAM_ENV_VAR);
return -EIO;
}
url = S3fsCurl::ps3fscred->GetIAMCredentialsURL() + env;
}else{
if(S3fsCurl::ps3fscred->GetIMDSVersion() > 1){
int result = GetIAMv2ApiToken();
if(-ENOENT == result){
// If we get a 404 back when requesting the token service,
// then it's highly likely we're running in an environment
// that doesn't support the AWS IMDSv2 API, so we'll skip
// the token retrieval in the future.
S3fsCurl::ps3fscred->SetIMDSVersion(1);
}else if(result != 0){
// If we get an unexpected error when retrieving the API
// token, log it but continue. Requirement for including
// an API token with the metadata request may or may not
// be enforced, so we should not abort here.
S3FS_PRN_ERR("AWS IMDSv2 token retrieval failed: %d", result);
}
}
url = S3fsCurl::ps3fscred->GetIAMCredentialsURL();
}
requestHeaders = NULL; requestHeaders = NULL;
responseHeaders.clear(); responseHeaders.clear();
bodydata.Clear(); bodydata.Clear();
if(S3fsCurl::ps3fscred->GetIMDSVersion() > 1){ if(iam_v2_token){
requestHeaders = curl_slist_sort_insert(requestHeaders, S3fsCred::IAMv2_token_hdr, S3fsCurl::ps3fscred->GetIAMv2APIToken().c_str()); requestHeaders = curl_slist_sort_insert(requestHeaders, S3fsCred::IAMv2_token_hdr, iam_v2_token);
} }
if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str())){ if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str())){
@ -3027,12 +2987,17 @@ bool S3fsCurl::LoadIAMRoleFromMetaData()
return false; return false;
} }
// [NOTE]
// Be sure to give "dontAddAuthHeaders=true".
// If set to false(default), it will deadlock in S3fsCred.
//
int result = RequestPerform(true); int result = RequestPerform(true);
// analyzing response // analyzing response
if(0 == result && !S3fsCurl::ps3fscred->SetIAMRoleFromMetaData(bodydata.str())){ if(0 == result){
S3FS_PRN_ERR("Something error occurred, could not get IAM role name."); token = bodydata.str();
result = -EIO; }else{
S3FS_PRN_ERR("Error(%d) occurred, could not get IAM role name from meta data.", result);
} }
bodydata.Clear(); bodydata.Clear();

16
src/curl.h
View File

@ -249,13 +249,12 @@ class S3fsCurl
bool ResetHandle(bool lock_already_held = false); bool ResetHandle(bool lock_already_held = false);
bool RemakeHandle(); bool RemakeHandle();
bool ClearInternalData(); bool ClearInternalData();
void insertV4Headers(); void insertV4Headers(const std::string& access_key_id, const std::string& secret_access_key, const std::string& access_token);
void insertV2Headers(); void insertV2Headers(const std::string& access_key_id, const std::string& secret_access_key, const std::string& access_token);
void insertIBMIAMHeaders(); void insertIBMIAMHeaders(const std::string& access_key_id, const std::string& access_token);
void insertAuthHeaders(); void insertAuthHeaders();
std::string CalcSignatureV2(const std::string& method, const std::string& strMD5, const std::string& content_type, const std::string& date, const std::string& resource); std::string CalcSignatureV2(const std::string& method, const std::string& strMD5, const std::string& content_type, const std::string& date, const std::string& resource, const std::string& secret_access_key, const std::string& access_token);
std::string CalcSignature(const std::string& method, const std::string& canonical_uri, const std::string& query_string, const std::string& strdate, const std::string& payload_hash, const std::string& date8601); std::string CalcSignature(const std::string& method, const std::string& canonical_uri, const std::string& query_string, const std::string& strdate, const std::string& payload_hash, const std::string& date8601, const std::string& secret_access_key, const std::string& access_token);
int GetIAMv2ApiToken();
int UploadMultipartPostSetup(const char* tpath, int part_num, const std::string& upload_id); int UploadMultipartPostSetup(const char* tpath, int part_num, const std::string& upload_id);
int CopyMultipartPostSetup(const char* from, const char* to, int part_num, const std::string& upload_id, headers_t& meta); int CopyMultipartPostSetup(const char* from, const char* to, int part_num, const std::string& upload_id, headers_t& meta);
bool UploadMultipartPostComplete(); bool UploadMultipartPostComplete();
@ -337,12 +336,13 @@ class S3fsCurl
bool CreateCurlHandle(bool only_pool = false, bool remake = false); bool CreateCurlHandle(bool only_pool = false, bool remake = false);
bool DestroyCurlHandle(bool restore_pool = true, bool clear_internal_data = true); bool DestroyCurlHandle(bool restore_pool = true, bool clear_internal_data = true);
int GetIAMCredentials(); bool GetIAMCredentials(const char* cred_url, const char* iam_v2_token, const char* ibm_secret_access_key, std::string& response);
bool LoadIAMRoleFromMetaData(); bool GetIAMRoleFromMetaData(const char* cred_url, const char* iam_v2_token, std::string& token);
bool AddSseRequestHead(sse_type_t ssetype, const std::string& ssevalue, bool is_only_c, bool is_copy); bool AddSseRequestHead(sse_type_t ssetype, const std::string& ssevalue, bool is_only_c, bool is_copy);
bool GetResponseCode(long& responseCode, bool from_curl_handle = true); bool GetResponseCode(long& responseCode, bool from_curl_handle = true);
int RequestPerform(bool dontAddAuthHeaders=false); int RequestPerform(bool dontAddAuthHeaders=false);
int DeleteRequest(const char* tpath); int DeleteRequest(const char* tpath);
int GetIAMv2ApiToken(const char* token_url, int token_ttl, const char* token_ttl_hdr, std::string& response);
bool PreHeadRequest(const char* tpath, const char* bpath = NULL, const char* savedpath = NULL, size_t ssekey_pos = -1); bool PreHeadRequest(const char* tpath, const char* bpath = NULL, const char* savedpath = NULL, size_t ssekey_pos = -1);
bool PreHeadRequest(const std::string& tpath, const std::string& bpath, const std::string& savedpath, size_t ssekey_pos = -1) { bool PreHeadRequest(const std::string& tpath, const std::string& bpath, const std::string& savedpath, size_t ssekey_pos = -1) {
return PreHeadRequest(tpath.c_str(), bpath.c_str(), savedpath.c_str(), ssekey_pos); return PreHeadRequest(tpath.c_str(), bpath.c_str(), savedpath.c_str(), ssekey_pos);

16
src/s3fs.cpp
View File

@ -3380,16 +3380,10 @@ static void* s3fs_init(struct fuse_conn_info* conn)
} }
// check loading IAM role name // check loading IAM role name
if(ps3fscred->IsIAMRoleMetadataType()){ if(!ps3fscred->LoadIAMRoleFromMetaData()){
// load IAM role name from http://169.254.169.254/latest/meta-data/iam/security-credentials S3FS_PRN_CRIT("could not load IAM role name from meta data.");
// s3fs_exit_fuseloop(EXIT_FAILURE);
S3fsCurl s3fscurl; return NULL;
if(!s3fscurl.LoadIAMRoleFromMetaData()){
S3FS_PRN_CRIT("could not load IAM role name from meta data.");
s3fs_exit_fuseloop(EXIT_FAILURE);
return NULL;
}
S3FS_PRN_INFO("loaded IAM role name = %s", ps3fscred->GetIAMRole().c_str());
} }
// Check Bucket // Check Bucket
@ -3502,7 +3496,7 @@ static int s3fs_check_service()
// At first time for access S3, we check IAM role if it sets. // At first time for access S3, we check IAM role if it sets.
if(!ps3fscred->CheckIAMCredentialUpdate()){ if(!ps3fscred->CheckIAMCredentialUpdate()){
S3FS_PRN_CRIT("Failed to check IAM role name(%s).", ps3fscred->GetIAMRole().c_str()); S3FS_PRN_CRIT("Failed to initialize IAM credential.");
return EXIT_FAILURE; return EXIT_FAILURE;
} }

346
src/s3fs_cred.cpp
View File

@ -97,6 +97,7 @@ bool S3fsCred::ParseIAMRoleFromMetaDataResponse(const char* response, std::strin
// Methods : Constructor / Destructor // Methods : Constructor / Destructor
//------------------------------------------------------------------- //-------------------------------------------------------------------
S3fsCred::S3fsCred() : S3fsCred::S3fsCred() :
is_lock_init(false),
passwd_file(""), passwd_file(""),
aws_profile(DEFAULT_AWS_PROFILE_NAME), aws_profile(DEFAULT_AWS_PROFILE_NAME),
load_iamrole(false), load_iamrole(false),
@ -115,10 +116,29 @@ S3fsCred::S3fsCred() :
IAM_expiry_field("Expiration"), IAM_expiry_field("Expiration"),
IAM_role("") IAM_role("")
{ {
pthread_mutexattr_t attr;
pthread_mutexattr_init(&attr);
#if S3FS_PTHREAD_ERRORCHECK
pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_ERRORCHECK);
#endif
int result;
if(0 != (result = pthread_mutex_init(&token_lock, &attr))){
S3FS_PRN_CRIT("failed to init token_lock: %d", result);
abort();
}
is_lock_init = true;
} }
S3fsCred::~S3fsCred() S3fsCred::~S3fsCred()
{ {
if(is_lock_init){
int result;
if(0 != (result = pthread_mutex_destroy(&token_lock))){
S3FS_PRN_CRIT("failed to destroy token_lock: %d", result);
abort();
}
is_lock_init = false;
}
} }
//------------------------------------------------------------------- //-------------------------------------------------------------------
@ -156,8 +176,10 @@ bool S3fsCred::SetIAMRoleMetadataType(bool flag)
return old; return old;
} }
bool S3fsCred::SetAccessKey(const char* AccessKeyId, const char* SecretAccessKey) bool S3fsCred::SetAccessKey(const char* AccessKeyId, const char* SecretAccessKey, AutoLock::Type type)
{ {
AutoLock auto_lock(&token_lock, type);
if((!is_ibm_iam_auth && (!AccessKeyId || '\0' == AccessKeyId[0])) || !SecretAccessKey || '\0' == SecretAccessKey[0]){ if((!is_ibm_iam_auth && (!AccessKeyId || '\0' == AccessKeyId[0])) || !SecretAccessKey || '\0' == SecretAccessKey[0]){
return false; return false;
} }
@ -167,8 +189,10 @@ bool S3fsCred::SetAccessKey(const char* AccessKeyId, const char* SecretAccessKey
return true; return true;
} }
bool S3fsCred::SetAccessKeyWithSessionToken(const char* AccessKeyId, const char* SecretAccessKey, const char * SessionToken) bool S3fsCred::SetAccessKeyWithSessionToken(const char* AccessKeyId, const char* SecretAccessKey, const char * SessionToken, AutoLock::Type type)
{ {
AutoLock auto_lock(&token_lock, type);
bool access_key_is_empty = !AccessKeyId || '\0' == AccessKeyId[0]; bool access_key_is_empty = !AccessKeyId || '\0' == AccessKeyId[0];
bool secret_access_key_is_empty = !SecretAccessKey || '\0' == SecretAccessKey[0]; bool secret_access_key_is_empty = !SecretAccessKey || '\0' == SecretAccessKey[0];
bool session_token_is_empty = !SessionToken || '\0' == SessionToken[0]; bool session_token_is_empty = !SessionToken || '\0' == SessionToken[0];
@ -184,9 +208,11 @@ bool S3fsCred::SetAccessKeyWithSessionToken(const char* AccessKeyId, const char*
return true; return true;
} }
bool S3fsCred::IsSetAccessKeys() const bool S3fsCred::IsSetAccessKeys(AutoLock::Type type)
{ {
return !IAM_role.empty() || ((!AWSAccessKeyId.empty() || is_ibm_iam_auth) && !AWSSecretAccessKey.empty()); AutoLock auto_lock(&token_lock, type);
return IsSetIAMRole(AutoLock::ALREADY_LOCKED) || ((!AWSAccessKeyId.empty() || is_ibm_iam_auth) && !AWSSecretAccessKey.empty());
} }
bool S3fsCred::SetIsECS(bool flag) bool S3fsCred::SetIsECS(bool flag)
@ -210,11 +236,26 @@ bool S3fsCred::SetIsIBMIAMAuth(bool flag)
return old; return old;
} }
std::string S3fsCred::SetIAMRole(const char* role) bool S3fsCred::SetIAMRole(const char* role, AutoLock::Type type)
{ {
std::string old = IAM_role; AutoLock auto_lock(&token_lock, type);
IAM_role = role ? role : ""; IAM_role = role ? role : "";
return old; return true;
}
std::string S3fsCred::GetIAMRole(AutoLock::Type type)
{
AutoLock auto_lock(&token_lock, type);
return IAM_role;
}
bool S3fsCred::IsSetIAMRole(AutoLock::Type type)
{
AutoLock auto_lock(&token_lock, type);
return !IAM_role.empty();
} }
size_t S3fsCred::SetIAMFieldCount(size_t field_count) size_t S3fsCred::SetIAMFieldCount(size_t field_count)
@ -245,24 +286,189 @@ std::string S3fsCred::SetIAMExpiryField(const char* expiry_field)
return old; return old;
} }
int S3fsCred::SetIMDSVersion(int version) bool S3fsCred::GetIAMCredentialsURL(std::string& url, bool check_iam_role, AutoLock::Type type)
{ {
// check
if(check_iam_role && !is_ecs && !IsIBMIAMAuth()){
if(!IsSetIAMRole(type)) {
S3FS_PRN_ERR("IAM role name is empty.");
return false;
}
S3FS_PRN_INFO3("[IAM role=%s]", GetIAMRole(type).c_str());
}
if(is_ecs){
const char *env = std::getenv(S3fsCred::ECS_IAM_ENV_VAR);
if(env == NULL){
S3FS_PRN_ERR("%s is not set.", S3fsCred::ECS_IAM_ENV_VAR);
return false;
}
url = IAM_cred_url + env;
}else if(IsIBMIAMAuth()){
url = IAM_cred_url;
}else{
// [NOTE]
// To avoid deadlocking, do not manipulate the S3fsCred object
// in the S3fsCurl::GetIAMv2ApiToken method (when retrying).
//
AutoLock auto_lock(&token_lock, type); // Lock for IAM_api_version, IAMv2_api_token
if(GetIMDSVersion(AutoLock::ALREADY_LOCKED) > 1){
S3fsCurl s3fscurl;
std::string token;
int result = s3fscurl.GetIAMv2ApiToken(S3fsCred::IAMv2_token_url, S3fsCred::IAMv2_token_ttl, S3fsCred::IAMv2_token_ttl_hdr, token);
if(-ENOENT == result){
// If we get a 404 back when requesting the token service,
// then it's highly likely we're running in an environment
// that doesn't support the AWS IMDSv2 API, so we'll skip
// the token retrieval in the future.
SetIMDSVersion(1, AutoLock::ALREADY_LOCKED);
}else if(result != 0){
// If we get an unexpected error when retrieving the API
// token, log it but continue. Requirement for including
// an API token with the metadata request may or may not
// be required, so we should not abort here.
S3FS_PRN_ERR("AWS IMDSv2 token retrieval failed: %d", result);
}else{
// Set token
if(!SetIAMv2APIToken(token, AutoLock::ALREADY_LOCKED)){
S3FS_PRN_ERR("Error storing IMDSv2 API token(%s).", token.c_str());
}
}
}
if(check_iam_role){
url = IAM_cred_url + GetIAMRole(AutoLock::ALREADY_LOCKED);
}else{
url = IAM_cred_url;
}
}
return true;
}
int S3fsCred::SetIMDSVersion(int version, AutoLock::Type type)
{
AutoLock auto_lock(&token_lock, type);
int old = IAM_api_version; int old = IAM_api_version;
IAM_api_version = version; IAM_api_version = version;
return old; return old;
} }
bool S3fsCred::SetIAMv2APIToken(const char* response) int S3fsCred::GetIMDSVersion(AutoLock::Type type)
{ {
S3FS_PRN_INFO3("Setting AWS IMDSv2 API token to %s", response ? response : "(null)"); AutoLock auto_lock(&token_lock, type);
if(!response){
return IAM_api_version;
}
bool S3fsCred::SetIAMv2APIToken(const std::string& token, AutoLock::Type type)
{
S3FS_PRN_INFO3("Setting AWS IMDSv2 API token to %s", token.c_str());
AutoLock auto_lock(&token_lock, type);
if(token.empty()){
return false; return false;
} }
IAMv2_api_token = std::string(response); IAMv2_api_token = token;
return true; return true;
} }
bool S3fsCred::SetIAMCredentials(const char* response) std::string S3fsCred::GetIAMv2APIToken(AutoLock::Type type)
{
AutoLock auto_lock(&token_lock, type);
return IAMv2_api_token;
}
// [NOTE]
// Currently, token_lock is always locked before calling this method,
// and this method calls the S3fsCurl::GetIAMCredentials method.
// Currently, when the request fails and retries in the process of
// S3fsCurl::GetIAMCredentials, does not use the S3fsCred object in
// retry logic.
// Be careful not to deadlock whenever you change this logic.
//
bool S3fsCred::LoadIAMCredentials(AutoLock::Type type)
{
// url(check iam role)
std::string url;
AutoLock auto_lock(&token_lock, type);
if(!GetIAMCredentialsURL(url, true, AutoLock::ALREADY_LOCKED)){
return false;
}
const char* iam_v2_token = NULL;
std::string str_iam_v2_token;
if(GetIMDSVersion(AutoLock::ALREADY_LOCKED) > 1){
str_iam_v2_token = GetIAMv2APIToken(AutoLock::ALREADY_LOCKED);
iam_v2_token = str_iam_v2_token.c_str();
}
const char* ibm_secret_access_key = NULL;
std::string str_ibm_secret_access_key;
if(IsIBMIAMAuth()){
str_ibm_secret_access_key = AWSSecretAccessKey;
ibm_secret_access_key = str_ibm_secret_access_key.c_str();
}
S3fsCurl s3fscurl;
std::string response;
if(!s3fscurl.GetIAMCredentials(url.c_str(), iam_v2_token, ibm_secret_access_key, response)){
return false;
}
if(!SetIAMCredentials(response.c_str(), AutoLock::ALREADY_LOCKED)){
S3FS_PRN_ERR("Something error occurred, could not set IAM role name.");
return false;
}
return true;
}
//
// load IAM role name from http://169.254.169.254/latest/meta-data/iam/security-credentials
//
bool S3fsCred::LoadIAMRoleFromMetaData()
{
AutoLock auto_lock(&token_lock);
if(load_iamrole){
// url(not check iam role)
std::string url;
if(!GetIAMCredentialsURL(url, false, AutoLock::ALREADY_LOCKED)){
return false;
}
const char* iam_v2_token = NULL;
std::string str_iam_v2_token;
if(GetIMDSVersion(AutoLock::ALREADY_LOCKED) > 1){
str_iam_v2_token = GetIAMv2APIToken(AutoLock::ALREADY_LOCKED);
iam_v2_token = str_iam_v2_token.c_str();
}
S3fsCurl s3fscurl;
std::string token;
if(!s3fscurl.GetIAMRoleFromMetaData(url.c_str(), iam_v2_token, token)){
return false;
}
if(!SetIAMRoleFromMetaData(token.c_str(), AutoLock::ALREADY_LOCKED)){
S3FS_PRN_ERR("Something error occurred, could not set IAM role name.");
return false;
}
S3FS_PRN_INFO("loaded IAM role name = %s", GetIAMRole(AutoLock::ALREADY_LOCKED).c_str());
}
return true;
}
bool S3fsCred::SetIAMCredentials(const char* response, AutoLock::Type type)
{ {
S3FS_PRN_INFO3("IAM credential response = \"%s\"", response); S3FS_PRN_INFO3("IAM credential response = \"%s\"", response);
@ -276,6 +482,8 @@ bool S3fsCred::SetIAMCredentials(const char* response)
return false; return false;
} }
AutoLock auto_lock(&token_lock, type);
AWSAccessToken = keyval[IAM_token_field]; AWSAccessToken = keyval[IAM_token_field];
if(is_ibm_iam_auth){ if(is_ibm_iam_auth){
@ -292,7 +500,7 @@ bool S3fsCred::SetIAMCredentials(const char* response)
return true; return true;
} }
bool S3fsCred::SetIAMRoleFromMetaData(const char* response) bool S3fsCred::SetIAMRoleFromMetaData(const char* response, AutoLock::Type type)
{ {
S3FS_PRN_INFO3("IAM role name response = \"%s\"", response ? response : "(null)"); S3FS_PRN_INFO3("IAM role name response = \"%s\"", response ? response : "(null)");
@ -301,11 +509,10 @@ bool S3fsCred::SetIAMRoleFromMetaData(const char* response)
return false; return false;
} }
SetIAMRole(rolename.c_str()); SetIAMRole(rolename.c_str(), type);
return true; return true;
} }
//------------------------------------------------------------------- //-------------------------------------------------------------------
// Methods : for Credentials // Methods : for Credentials
//------------------------------------------------------------------- //-------------------------------------------------------------------
@ -504,7 +711,7 @@ bool S3fsCred::ParseS3fsPasswdFile(bucketkvmap_t& resmap)
// //
// only one default key pair is allowed, but not required // only one default key pair is allowed, but not required
// //
bool S3fsCred::ReadS3fsPasswdFile() bool S3fsCred::ReadS3fsPasswdFile(AutoLock::Type type)
{ {
bucketkvmap_t bucketmap; bucketkvmap_t bucketmap;
kvmap_t keyval; kvmap_t keyval;
@ -529,11 +736,17 @@ bool S3fsCred::ReadS3fsPasswdFile()
bucketkvmap_t::iterator it = bucketmap.find(S3fsCred::KEYVAL_FIELDS_TYPE); bucketkvmap_t::iterator it = bucketmap.find(S3fsCred::KEYVAL_FIELDS_TYPE);
if(bucketmap.end() != it){ if(bucketmap.end() != it){
// aws format // aws format
int result = CheckS3fsCredentialAwsFormat(it->second); std::string access_key_id;
std::string secret_access_key;
int result = CheckS3fsCredentialAwsFormat(it->second, access_key_id, secret_access_key);
if(-1 == result){ if(-1 == result){
return false; return false;
}else if(1 == result){ }else if(1 == result){
// success to set // found ascess(secret) keys
if(!SetAccessKey(access_key_id.c_str(), secret_access_key.c_str(), type)){
S3FS_PRN_EXIT("failed to set access key/secret key.");
return false;
}
return true; return true;
} }
} }
@ -556,7 +769,7 @@ bool S3fsCred::ReadS3fsPasswdFile()
return false; return false;
} }
if(!SetAccessKey(aws_accesskeyid_it->second.c_str(), aws_secretkey_it->second.c_str())){ if(!SetAccessKey(aws_accesskeyid_it->second.c_str(), aws_secretkey_it->second.c_str(), type)){
S3FS_PRN_EXIT("failed to set internal data for access key/secret key from passwd file."); S3FS_PRN_EXIT("failed to set internal data for access key/secret key from passwd file.");
return false; return false;
} }
@ -568,7 +781,7 @@ bool S3fsCred::ReadS3fsPasswdFile()
// 0 - NG(could not read) // 0 - NG(could not read)
// -1 - Should shutdown immediately // -1 - Should shutdown immediately
// //
int S3fsCred::CheckS3fsCredentialAwsFormat(const kvmap_t& kvmap) int S3fsCred::CheckS3fsCredentialAwsFormat(const kvmap_t& kvmap, std::string& access_key_id, std::string& secret_access_key)
{ {
std::string str1(S3fsCred::AWS_ACCESSKEYID); std::string str1(S3fsCred::AWS_ACCESSKEYID);
std::string str2(S3fsCred::AWS_SECRETKEY); std::string str2(S3fsCred::AWS_SECRETKEY);
@ -585,17 +798,16 @@ int S3fsCred::CheckS3fsCredentialAwsFormat(const kvmap_t& kvmap)
S3FS_PRN_EXIT("AWSAccesskey or AWSSecretkey is not specified."); S3FS_PRN_EXIT("AWSAccesskey or AWSSecretkey is not specified.");
return -1; return -1;
} }
if(!SetAccessKey(str1_it->second.c_str(), str2_it->second.c_str())){ access_key_id = str1_it->second;
S3FS_PRN_EXIT("failed to set access key/secret key."); secret_access_key = str2_it->second;
return -1;
}
return 1; return 1;
} }
// //
// Read Aws Credential File // Read Aws Credential File
// //
bool S3fsCred::ReadAwsCredentialFile(const std::string &filename) bool S3fsCred::ReadAwsCredentialFile(const std::string &filename, AutoLock::Type type)
{ {
// open passwd file // open passwd file
std::ifstream PF(filename.c_str()); std::ifstream PF(filename.c_str());
@ -652,12 +864,12 @@ bool S3fsCred::ReadAwsCredentialFile(const std::string &filename)
S3FS_PRN_EXIT("AWS session token was expected but wasn't provided in aws/credentials file for profile: %s.", aws_profile.c_str()); S3FS_PRN_EXIT("AWS session token was expected but wasn't provided in aws/credentials file for profile: %s.", aws_profile.c_str());
return false; return false;
} }
if(!SetAccessKey(accesskey.c_str(), secret.c_str())){ if(!SetAccessKey(accesskey.c_str(), secret.c_str(), type)){
S3FS_PRN_EXIT("failed to set internal data for access key/secret key from aws credential file."); S3FS_PRN_EXIT("failed to set internal data for access key/secret key from aws credential file.");
return false; return false;
} }
}else{ }else{
if(!SetAccessKeyWithSessionToken(accesskey.c_str(), secret.c_str(), session_token.c_str())){ if(!SetAccessKeyWithSessionToken(accesskey.c_str(), secret.c_str(), session_token.c_str(), type)){
S3FS_PRN_EXIT("session token is invalid."); S3FS_PRN_EXIT("session token is invalid.");
return false; return false;
} }
@ -695,13 +907,13 @@ bool S3fsCred::InitialS3fsCredentials()
} }
// 1 - keys specified on the command line // 1 - keys specified on the command line
if(IsSetAccessKeys()){ if(IsSetAccessKeys(AutoLock::NONE)){
return true; return true;
} }
// 2 - was specified on the command line // 2 - was specified on the command line
if(IsSetPasswdFile()){ if(IsSetPasswdFile()){
if(!ReadS3fsPasswdFile()){ if(!ReadS3fsPasswdFile(AutoLock::NONE)){
return false; return false;
} }
return true; return true;
@ -721,7 +933,7 @@ bool S3fsCred::InitialS3fsCredentials()
S3FS_PRN_INFO2("access key from env variables"); S3FS_PRN_INFO2("access key from env variables");
if(AWSSESSIONTOKEN != NULL){ if(AWSSESSIONTOKEN != NULL){
S3FS_PRN_INFO2("session token is available"); S3FS_PRN_INFO2("session token is available");
if(!SetAccessKeyWithSessionToken(AWSACCESSKEYID, AWSSECRETACCESSKEY, AWSSESSIONTOKEN)){ if(!SetAccessKeyWithSessionToken(AWSACCESSKEYID, AWSSECRETACCESSKEY, AWSSESSIONTOKEN, AutoLock::NONE)){
S3FS_PRN_EXIT("session token is invalid."); S3FS_PRN_EXIT("session token is invalid.");
return false; return false;
} }
@ -732,7 +944,7 @@ bool S3fsCred::InitialS3fsCredentials()
return false; return false;
} }
} }
if(!SetAccessKey(AWSACCESSKEYID, AWSSECRETACCESSKEY)){ if(!SetAccessKey(AWSACCESSKEYID, AWSSECRETACCESSKEY, AutoLock::NONE)){
S3FS_PRN_EXIT("if one access key is specified, both keys need to be specified."); S3FS_PRN_EXIT("if one access key is specified, both keys need to be specified.");
return false; return false;
} }
@ -748,7 +960,7 @@ bool S3fsCred::InitialS3fsCredentials()
S3FS_PRN_EXIT("AWS_CREDENTIAL_FILE: \"%s\" is not readable.", passwd_file.c_str()); S3FS_PRN_EXIT("AWS_CREDENTIAL_FILE: \"%s\" is not readable.", passwd_file.c_str());
return false; return false;
} }
if(!ReadS3fsPasswdFile()){ if(!ReadS3fsPasswdFile(AutoLock::NONE)){
return false; return false;
} }
return true; return true;
@ -757,7 +969,7 @@ bool S3fsCred::InitialS3fsCredentials()
// 3b - check ${HOME}/.aws/credentials // 3b - check ${HOME}/.aws/credentials
std::string aws_credentials = std::string(getpwuid(getuid())->pw_dir) + "/.aws/credentials"; std::string aws_credentials = std::string(getpwuid(getuid())->pw_dir) + "/.aws/credentials";
if(ReadAwsCredentialFile(aws_credentials)){ if(ReadAwsCredentialFile(aws_credentials, AutoLock::NONE)){
return true; return true;
}else if(aws_profile != DEFAULT_AWS_PROFILE_NAME){ }else if(aws_profile != DEFAULT_AWS_PROFILE_NAME){
S3FS_PRN_EXIT("Could not find profile: %s in file: %s", aws_profile.c_str(), aws_credentials.c_str()); S3FS_PRN_EXIT("Could not find profile: %s in file: %s", aws_profile.c_str(), aws_credentials.c_str());
@ -770,14 +982,14 @@ bool S3fsCred::InitialS3fsCredentials()
passwd_file = HOME; passwd_file = HOME;
passwd_file += "/.passwd-s3fs"; passwd_file += "/.passwd-s3fs";
if(IsReadableS3fsPasswdFile()){ if(IsReadableS3fsPasswdFile()){
if(!ReadS3fsPasswdFile()){ if(!ReadS3fsPasswdFile(AutoLock::NONE)){
return false; return false;
} }
// It is possible that the user's file was there but // It is possible that the user's file was there but
// contained no key pairs i.e. commented out // contained no key pairs i.e. commented out
// in that case, go look in the final location // in that case, go look in the final location
if(IsSetAccessKeys()){ if(IsSetAccessKeys(AutoLock::NONE)){
return true; return true;
} }
} }
@ -786,7 +998,7 @@ bool S3fsCred::InitialS3fsCredentials()
// 5 - from the system default location // 5 - from the system default location
passwd_file = "/etc/passwd-s3fs"; passwd_file = "/etc/passwd-s3fs";
if(IsReadableS3fsPasswdFile()){ if(IsReadableS3fsPasswdFile()){
if(!ReadS3fsPasswdFile()){ if(!ReadS3fsPasswdFile(AutoLock::NONE)){
return false; return false;
} }
return true; return true;
@ -854,23 +1066,37 @@ bool S3fsCred::ParseIAMCredentialResponse(const char* response, iamcredmap_t& ke
return true; return true;
} }
bool S3fsCred::CheckIAMCredentialUpdate() bool S3fsCred::CheckIAMCredentialUpdate(std::string* access_key_id, std::string* secret_access_key, std::string* access_token)
{ {
if(IAM_role.empty() && !is_ecs && !is_ibm_iam_auth){ AutoLock auto_lock(&token_lock);
return true;
}
if(time(NULL) + S3fsCred::IAM_EXPIRE_MERGIN <= AWSAccessTokenExpire){
return true;
}
S3FS_PRN_INFO("IAM Access Token refreshing...");
// update if(IsSetIAMRole(AutoLock::ALREADY_LOCKED) || is_ecs || is_ibm_iam_auth){
S3fsCurl s3fscurl; if(AWSAccessTokenExpire < (time(NULL) + S3fsCred::IAM_EXPIRE_MERGIN)){
if(0 != s3fscurl.GetIAMCredentials()){ S3FS_PRN_INFO("IAM Access Token refreshing...");
S3FS_PRN_ERR("IAM Access Token refresh failed");
return false; // update
if(!LoadIAMCredentials(AutoLock::ALREADY_LOCKED)){
S3FS_PRN_ERR("IAM Access Token refresh failed");
return false;
}
S3FS_PRN_INFO("IAM Access Token refreshed");
}
}
// set
if(access_key_id){
*access_key_id = AWSAccessKeyId;
}
if(secret_access_key){
*secret_access_key = AWSSecretAccessKey;
}
if(access_token){
if(IsIBMIAMAuth() || IsSetIAMRole(AutoLock::ALREADY_LOCKED) || is_ecs || is_use_session_token){
*access_token = AWSAccessToken;
}else{
access_token->erase();
}
} }
S3FS_PRN_INFO("IAM Access Token refreshed");
return true; return true;
} }
@ -900,7 +1126,7 @@ int S3fsCred::DetectParam(const char* arg)
SetIAMTokenField("\"access_token\""); SetIAMTokenField("\"access_token\"");
SetIAMExpiryField("\"expiration\""); SetIAMExpiryField("\"expiration\"");
SetIAMFieldCount(2); SetIAMFieldCount(2);
SetIMDSVersion(1); SetIMDSVersion(1, AutoLock::NONE);
return 0; return 0;
} }
@ -924,7 +1150,7 @@ int S3fsCred::DetectParam(const char* arg)
} }
if(0 == strcmp(arg, "imdsv1only")){ if(0 == strcmp(arg, "imdsv1only")){
SetIMDSVersion(1); SetIMDSVersion(1, AutoLock::NONE);
return 0; return 0;
} }
@ -934,14 +1160,14 @@ int S3fsCred::DetectParam(const char* arg)
return -1; return -1;
} }
SetIsECS(true); SetIsECS(true);
SetIMDSVersion(1); SetIMDSVersion(1, AutoLock::NONE);
SetIAMCredentialsURL("http://169.254.170.2"); SetIAMCredentialsURL("http://169.254.170.2");
SetIAMFieldCount(5); SetIAMFieldCount(5);
return 0; return 0;
} }
if(is_prefix(arg, "iam_role")){ if(is_prefix(arg, "iam_role")){
if(IsECS() || IsIBMIAMAuth()){ if(is_ecs || IsIBMIAMAuth()){
S3FS_PRN_EXIT("option iam_role cannot be used in conjunction with ecs or ibm"); S3FS_PRN_EXIT("option iam_role cannot be used in conjunction with ecs or ibm");
return -1; return -1;
} }
@ -953,7 +1179,7 @@ int S3fsCred::DetectParam(const char* arg)
}else if(is_prefix(arg, "iam_role=")){ }else if(is_prefix(arg, "iam_role=")){
const char* role = strchr(arg, '=') + sizeof(char); const char* role = strchr(arg, '=') + sizeof(char);
SetIAMRole(role); SetIAMRole(role, AutoLock::NONE);
SetIAMRoleMetadataType(false); SetIAMRoleMetadataType(false);
return 0; return 0;
} }
@ -1014,21 +1240,21 @@ bool S3fsCred::CheckAllParams()
} }
// error checking of command line arguments for compatibility // error checking of command line arguments for compatibility
if(S3fsCurl::IsPublicBucket() && IsSetAccessKeys()){ if(S3fsCurl::IsPublicBucket() && IsSetAccessKeys(AutoLock::NONE)){
S3FS_PRN_EXIT("specifying both public_bucket and the access keys options is invalid."); S3FS_PRN_EXIT("specifying both public_bucket and the access keys options is invalid.");
return false; return false;
} }
if(IsSetPasswdFile() && IsSetAccessKeys()){ if(IsSetPasswdFile() && IsSetAccessKeys(AutoLock::NONE)){
S3FS_PRN_EXIT("specifying both passwd_file and the access keys options is invalid."); S3FS_PRN_EXIT("specifying both passwd_file and the access keys options is invalid.");
return false; return false;
} }
if(!S3fsCurl::IsPublicBucket() && !IsIAMRoleMetadataType() && !IsECS()){ if(!S3fsCurl::IsPublicBucket() && !load_iamrole && !is_ecs){
if(!InitialS3fsCredentials()){ if(!InitialS3fsCredentials()){
return false; return false;
} }
if(!IsSetAccessKeys()){ if(!IsSetAccessKeys(AutoLock::NONE)){
S3FS_PRN_EXIT("could not establish security credentials, check documentation."); S3FS_PRN_EXIT("could not establish security credentials, check documentation.");
return false; return false;
} }

69
src/s3fs_cred.h
View File

@ -21,6 +21,8 @@
#ifndef S3FS_CRED_H_ #ifndef S3FS_CRED_H_
#define S3FS_CRED_H_ #define S3FS_CRED_H_
#include "autolock.h"
//---------------------------------------------- //----------------------------------------------
// Typedefs // Typedefs
//---------------------------------------------- //----------------------------------------------
@ -43,37 +45,39 @@ class S3fsCred
static const char* AWS_SECRETKEY; static const char* AWS_SECRETKEY;
static const int IAM_EXPIRE_MERGIN; static const int IAM_EXPIRE_MERGIN;
static const char* ECS_IAM_ENV_VAR;
static const char* IAMCRED_ACCESSKEYID; static const char* IAMCRED_ACCESSKEYID;
static const char* IAMCRED_SECRETACCESSKEY; static const char* IAMCRED_SECRETACCESSKEY;
static const char* IAMCRED_ROLEARN; static const char* IAMCRED_ROLEARN;
static std::string bucket_name; static std::string bucket_name;
pthread_mutex_t token_lock;
bool is_lock_init;
std::string passwd_file; std::string passwd_file;
std::string aws_profile; std::string aws_profile;
bool load_iamrole; bool load_iamrole;
std::string AWSAccessKeyId; std::string AWSAccessKeyId; // Protect exclusively
std::string AWSSecretAccessKey; std::string AWSSecretAccessKey; // Protect exclusively
std::string AWSAccessToken; std::string AWSAccessToken; // Protect exclusively
time_t AWSAccessTokenExpire; time_t AWSAccessTokenExpire; // Protect exclusively
bool is_ecs; bool is_ecs;
bool is_use_session_token; bool is_use_session_token;
bool is_ibm_iam_auth; bool is_ibm_iam_auth;
std::string IAM_cred_url; std::string IAM_cred_url;
int IAM_api_version; int IAM_api_version; // Protect exclusively
std::string IAMv2_api_token; std::string IAMv2_api_token; // Protect exclusively
size_t IAM_field_count; size_t IAM_field_count;
std::string IAM_token_field; std::string IAM_token_field;
std::string IAM_expiry_field; std::string IAM_expiry_field;
std::string IAM_role; std::string IAM_role; // Protect exclusively
public: public:
static const char* ECS_IAM_ENV_VAR;
static const char* IAMv2_token_url; static const char* IAMv2_token_url;
static int IAMv2_token_ttl; static int IAMv2_token_ttl;
static const char* IAMv2_token_ttl_hdr; static const char* IAMv2_token_ttl_hdr;
@ -87,16 +91,24 @@ class S3fsCred
bool SetAwsProfileName(const char* profile_name); bool SetAwsProfileName(const char* profile_name);
bool SetIAMRoleMetadataType(bool flag); bool SetIAMRoleMetadataType(bool flag);
bool SetAccessKey(const char* AccessKeyId, const char* SecretAccessKey); bool SetAccessKey(const char* AccessKeyId, const char* SecretAccessKey, AutoLock::Type type);
bool SetAccessKeyWithSessionToken(const char* AccessKeyId, const char* SecretAccessKey, const char * SessionToken); bool SetAccessKeyWithSessionToken(const char* AccessKeyId, const char* SecretAccessKey, const char * SessionToken, AutoLock::Type type);
bool IsSetAccessKeys() const; bool IsSetAccessKeys(AutoLock::Type type);
bool SetIsECS(bool flag); bool SetIsECS(bool flag);
bool SetIsUseSessionToken(bool flag); bool SetIsUseSessionToken(bool flag);
bool SetIsIBMIAMAuth(bool flag); bool SetIsIBMIAMAuth(bool flag);
std::string SetIAMRole(const char* role); int SetIMDSVersion(int version, AutoLock::Type type);
int GetIMDSVersion(AutoLock::Type type);
bool SetIAMv2APIToken(const std::string& token, AutoLock::Type type);
std::string GetIAMv2APIToken(AutoLock::Type type);
bool SetIAMRole(const char* role, AutoLock::Type type);
std::string GetIAMRole(AutoLock::Type type);
bool IsSetIAMRole(AutoLock::Type type);
size_t SetIAMFieldCount(size_t field_count); size_t SetIAMFieldCount(size_t field_count);
std::string SetIAMCredentialsURL(const char* url); std::string SetIAMCredentialsURL(const char* url);
std::string SetIAMTokenField(const char* token_field); std::string SetIAMTokenField(const char* token_field);
@ -105,14 +117,19 @@ class S3fsCred
bool IsReadableS3fsPasswdFile(); bool IsReadableS3fsPasswdFile();
bool CheckS3fsPasswdFilePerms(); bool CheckS3fsPasswdFilePerms();
bool ParseS3fsPasswdFile(bucketkvmap_t& resmap); bool ParseS3fsPasswdFile(bucketkvmap_t& resmap);
bool ReadS3fsPasswdFile(); bool ReadS3fsPasswdFile(AutoLock::Type type);
int CheckS3fsCredentialAwsFormat(const kvmap_t& kvmap); int CheckS3fsCredentialAwsFormat(const kvmap_t& kvmap, std::string& access_key_id, std::string& secret_access_key);
bool ReadAwsCredentialFile(const std::string &filename); bool ReadAwsCredentialFile(const std::string &filename, AutoLock::Type type);
bool InitialS3fsCredentials(); bool InitialS3fsCredentials();
bool ParseIAMCredentialResponse(const char* response, iamcredmap_t& keyval); bool ParseIAMCredentialResponse(const char* response, iamcredmap_t& keyval);
bool GetIAMCredentialsURL(std::string& url, bool check_iam_role, AutoLock::Type type);
bool LoadIAMCredentials(AutoLock::Type type);
bool SetIAMCredentials(const char* response, AutoLock::Type type);
bool SetIAMRoleFromMetaData(const char* response, AutoLock::Type type);
bool CheckForbiddenBucketParams(); bool CheckForbiddenBucketParams();
public: public:
@ -122,27 +139,11 @@ class S3fsCred
S3fsCred(); S3fsCred();
~S3fsCred(); ~S3fsCred();
bool IsIAMRoleMetadataType() const { return load_iamrole; }
const std::string& GetAccessKeyID() const { return AWSAccessKeyId; }
const std::string& GetSecretAccessKey() const { return AWSSecretAccessKey; }
const std::string& GetAccessToken() const { return AWSAccessToken; }
bool IsECS() const { return is_ecs; }
bool IsUseSessionToken() const { return is_use_session_token; }
bool IsIBMIAMAuth() const { return is_ibm_iam_auth; } bool IsIBMIAMAuth() const { return is_ibm_iam_auth; }
const std::string& GetIAMRole() const { return IAM_role; } bool LoadIAMRoleFromMetaData();
const std::string& GetIAMCredentialsURL() const { return IAM_cred_url; }
int SetIMDSVersion(int version);
int GetIMDSVersion() const { return IAM_api_version; }
bool SetIAMv2APIToken(const char* response); bool CheckIAMCredentialUpdate(std::string* access_key_id = NULL, std::string* secret_access_key = NULL, std::string* access_token = NULL);
const std::string& GetIAMv2APIToken() const { return IAMv2_api_token; }
bool SetIAMCredentials(const char* response);
bool SetIAMRoleFromMetaData(const char* response);
bool CheckIAMCredentialUpdate();
int DetectParam(const char* arg); int DetectParam(const char* arg);
bool CheckAllParams(); bool CheckAllParams();