octoleo/s3fs-fuse
1
0
Fork 0
mirror of https://github.com/s3fs-fuse/s3fs-fuse.git synced 2024-06-07 11:30:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Added S3fsCred class and moved Credential related processing in it

Browse Source
This commit is contained in:
Takeshi Nakatani 2022-01-30 08:04:33 +00:00 committed by Andrew Gaul
parent d7e929e0a8
commit 1678803566
13 changed files with 1254 additions and 917 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/Makefile.am
View File

@ -41,6 +41,7 @@ s3fs_SOURCES = \
s3objlist.cpp \
cache.cpp \
string_util.cpp \
s3fs_cred.cpp \
s3fs_util.cpp \
fdcache.cpp \
fdcache_entity.cpp \

2
src/common.h
View File

@ -42,12 +42,10 @@ extern bool noxmlns;
extern std::string program_name;
extern std::string service_path;
extern std::string s3host;
extern std::string bucket;
extern std::string mount_prefix;
extern std::string endpoint;
extern std::string cipher_suites;
extern std::string instance_name;
extern std::string aws_profile;
#endif // S3FS_COMMON_H_

361
src/curl.cpp
View File

@ -49,12 +49,6 @@ static const char EMPTY_MD5_BASE64_HASH[] = "1B2M2Y8AsgTpgAmY7PhCfg=="
static const int MULTIPART_SIZE = 10 * 1024 * 1024;
static const int GET_OBJECT_RESPONSE_LIMIT = 1024;
static const int IAM_EXPIRE_MERGIN = 20 * 60; // update timing
static const char ECS_IAM_ENV_VAR[] = "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI";
static const char IAMCRED_ACCESSKEYID[] = "AccessKeyId";
static const char IAMCRED_SECRETACCESSKEY[] = "SecretAccessKey";
static const char IAMCRED_ROLEARN[] = "RoleArn";
// [NOTE] about default mime.types file
// If no mime.types file is specified in the mime option, s3fs
// will look for /etc/mime.types on all operating systems and
@ -103,23 +97,7 @@ sse_type_t S3fsCurl::ssetype = sse_type_t::SSE_DISABLE;
bool S3fsCurl::is_content_md5 = false;
bool S3fsCurl::is_verbose = false;
bool S3fsCurl::is_dump_body = false;
std::string S3fsCurl::AWSAccessKeyId;
std::string S3fsCurl::AWSSecretAccessKey;
std::string S3fsCurl::AWSAccessToken;
time_t S3fsCurl::AWSAccessTokenExpire= 0;
bool S3fsCurl::is_ecs = false;
bool S3fsCurl::is_ibm_iam_auth = false;
std::string S3fsCurl::IAM_cred_url = "http://169.254.169.254/latest/meta-data/iam/security-credentials/";
std::string S3fsCurl::IAMv2_token_url = "http://169.254.169.254/latest/api/token";
std::string S3fsCurl::IAMv2_token_ttl_hdr = "X-aws-ec2-metadata-token-ttl-seconds";
std::string S3fsCurl::IAMv2_token_hdr = "X-aws-ec2-metadata-token";
int S3fsCurl::IAMv2_token_ttl = 21600;
size_t S3fsCurl::IAM_field_count = 4;
std::string S3fsCurl::IAM_token_field = "Token";
std::string S3fsCurl::IAM_expiry_field = "Expiration";
std::string S3fsCurl::IAM_role;
std::string S3fsCurl::IAMv2_api_token;
int S3fsCurl::IAM_api_version = 2;
S3fsCred* S3fsCurl::ps3fscred = NULL;
long S3fsCurl::ssl_verify_hostname = 1; // default(original code...)
// protected by curl_warnings_lock
@ -140,7 +118,6 @@ signature_type_t S3fsCurl::signature_type = V2_OR_V4; // default
bool S3fsCurl::is_unsigned_payload = false; // default
bool S3fsCurl::is_ua = true; // default
bool S3fsCurl::listobjectsv2 = false; // default
bool S3fsCurl::is_use_session_token= false; // default
bool S3fsCurl::requester_pays = false; // default
//-------------------------------------------------------------------
@ -383,6 +360,18 @@ int S3fsCurl::CurlProgress(void *clientp, double dltotal, double dlnow, double u
return 0;
}
bool S3fsCurl::InitCredentialObject(S3fsCred* pcredobj)
{
// Set the only Credential object
if(!pcredobj || S3fsCurl::ps3fscred){
S3FS_PRN_ERR("Unable to set the only Credential object.");
return false;
}
S3fsCurl::ps3fscred = pcredobj;
return true;
}
bool S3fsCurl::InitMimeType(const std::string& strFile)
{
std::string MimeFile;
@ -1013,32 +1002,6 @@ bool S3fsCurl::SetDumpBody(bool flag)
return old;
}
bool S3fsCurl::SetAccessKey(const char* AccessKeyId, const char* SecretAccessKey)
{
if((!S3fsCurl::is_ibm_iam_auth && (!AccessKeyId || '\0' == AccessKeyId[0])) || !SecretAccessKey || '\0' == SecretAccessKey[0]){
return false;
}
AWSAccessKeyId = AccessKeyId;
AWSSecretAccessKey = SecretAccessKey;
return true;
}
bool S3fsCurl::SetAccessKeyWithSessionToken(const char* AccessKeyId, const char* SecretAccessKey, const char* SessionToken)
{
bool access_key_is_empty = !AccessKeyId || '\0' == AccessKeyId[0];
bool secret_access_key_is_empty = !SecretAccessKey || '\0' == SecretAccessKey[0];
bool session_token_is_empty = !SessionToken || '\0' == SessionToken[0];
if((!S3fsCurl::is_ibm_iam_auth && access_key_is_empty) || secret_access_key_is_empty || session_token_is_empty){
return false;
}
AWSAccessKeyId = AccessKeyId;
AWSSecretAccessKey = SecretAccessKey;
AWSAccessToken = SessionToken;
S3fsCurl::is_use_session_token = true;
return true;
}
long S3fsCurl::SetSslVerifyHostname(long value)
{
if(0 != value && 1 != value){
@ -1049,61 +1012,6 @@ long S3fsCurl::SetSslVerifyHostname(long value)
return old;
}
bool S3fsCurl::SetIsIBMIAMAuth(bool flag)
{
bool old = S3fsCurl::is_ibm_iam_auth;
S3fsCurl::is_ibm_iam_auth = flag;
return old;
}
bool S3fsCurl::SetIsECS(bool flag)
{
bool old = S3fsCurl::is_ecs;
S3fsCurl::is_ecs = flag;
return old;
}
std::string S3fsCurl::SetIAMRole(const char* role)
{
std::string old = S3fsCurl::IAM_role;
S3fsCurl::IAM_role = role ? role : "";
return old;
}
size_t S3fsCurl::SetIAMFieldCount(size_t field_count)
{
size_t old = S3fsCurl::IAM_field_count;
S3fsCurl::IAM_field_count = field_count;
return old;
}
std::string S3fsCurl::SetIAMCredentialsURL(const char* url)
{
std::string old = S3fsCurl::IAM_cred_url;
S3fsCurl::IAM_cred_url = url ? url : "";
return old;
}
std::string S3fsCurl::SetIAMTokenField(const char* token_field)
{
std::string old = S3fsCurl::IAM_token_field;
S3fsCurl::IAM_token_field = token_field ? token_field : "";
return old;
}
std::string S3fsCurl::SetIAMExpiryField(const char* expiry_field)
{
std::string old = S3fsCurl::IAM_expiry_field;
S3fsCurl::IAM_expiry_field = expiry_field ? expiry_field : "";
return old;
}
bool S3fsCurl::SetIMDSVersion(int version)
{
S3fsCurl::IAM_api_version = version;
return true;
}
bool S3fsCurl::SetMultipartSize(off_t size)
{
size = size * 1024 * 1024;
@ -1735,150 +1643,6 @@ bool S3fsCurl::PreHeadRequestSetCurlOpts(S3fsCurl* s3fscurl)
return true;
}
bool S3fsCurl::ParseIAMCredentialResponse(const char* response, iamcredmap_t& keyval)
{
if(!response){
return false;
}
std::istringstream sscred(response);
std::string oneline;
keyval.clear();
while(getline(sscred, oneline, ',')){
std::string::size_type pos;
std::string key;
std::string val;
if(std::string::npos != (pos = oneline.find(IAMCRED_ACCESSKEYID))){
key = IAMCRED_ACCESSKEYID;
}else if(std::string::npos != (pos = oneline.find(IAMCRED_SECRETACCESSKEY))){
key = IAMCRED_SECRETACCESSKEY;
}else if(std::string::npos != (pos = oneline.find(S3fsCurl::IAM_token_field))){
key = S3fsCurl::IAM_token_field;
}else if(std::string::npos != (pos = oneline.find(S3fsCurl::IAM_expiry_field))){
key = S3fsCurl::IAM_expiry_field;
}else if(std::string::npos != (pos = oneline.find(IAMCRED_ROLEARN))){
key = IAMCRED_ROLEARN;
}else{
continue;
}
if(std::string::npos == (pos = oneline.find(':', pos + key.length()))){
continue;
}
if(S3fsCurl::is_ibm_iam_auth && key == S3fsCurl::IAM_expiry_field){
// parse integer value
if(std::string::npos == (pos = oneline.find_first_of("0123456789", pos))){
continue;
}
oneline.erase(0, pos);
if(std::string::npos == (pos = oneline.find_last_of("0123456789"))){
continue;
}
val = oneline.substr(0, pos+1);
}else{
// parse std::string value (starts and ends with quotes)
if(std::string::npos == (pos = oneline.find('\"', pos))){
continue;
}
oneline.erase(0, pos+1);
if(std::string::npos == (pos = oneline.find('\"'))){
continue;
}
val = oneline.substr(0, pos);
}
keyval[key] = val;
}
return true;
}
bool S3fsCurl::SetIAMv2APIToken(const char* response)
{
S3FS_PRN_INFO3("Setting AWS IMDSv2 API token to %s", response);
S3fsCurl::IAMv2_api_token = std::string(response);
return true;
}
bool S3fsCurl::SetIAMCredentials(const char* response)
{
S3FS_PRN_INFO3("IAM credential response = \"%s\"", response);
iamcredmap_t keyval;
if(!ParseIAMCredentialResponse(response, keyval)){
return false;
}
if(S3fsCurl::IAM_field_count != keyval.size()){
return false;
}
S3fsCurl::AWSAccessToken = keyval[std::string(S3fsCurl::IAM_token_field)];
if(S3fsCurl::is_ibm_iam_auth){
off_t tmp_expire = 0;
if(!s3fs_strtoofft(&tmp_expire, keyval[std::string(S3fsCurl::IAM_expiry_field)].c_str(), /*base=*/ 10)){
return false;
}
S3fsCurl::AWSAccessTokenExpire = static_cast<time_t>(tmp_expire);
}else{
S3fsCurl::AWSAccessKeyId = keyval[std::string(IAMCRED_ACCESSKEYID)];
S3fsCurl::AWSSecretAccessKey = keyval[std::string(IAMCRED_SECRETACCESSKEY)];
S3fsCurl::AWSAccessTokenExpire = cvtIAMExpireStringToTime(keyval[S3fsCurl::IAM_expiry_field].c_str());
}
return true;
}
bool S3fsCurl::CheckIAMCredentialUpdate()
{
if(S3fsCurl::IAM_role.empty() && !S3fsCurl::is_ecs && !S3fsCurl::is_ibm_iam_auth){
return true;
}
if(time(NULL) + IAM_EXPIRE_MERGIN <= S3fsCurl::AWSAccessTokenExpire){
return true;
}
S3FS_PRN_INFO("IAM Access Token refreshing...");
// update
S3fsCurl s3fscurl;
if(0 != s3fscurl.GetIAMCredentials()){
S3FS_PRN_ERR("IAM Access Token refresh failed");
return false;
}
S3FS_PRN_INFO("IAM Access Token refreshed");
return true;
}
bool S3fsCurl::ParseIAMRoleFromMetaDataResponse(const char* response, std::string& rolename)
{
if(!response){
return false;
}
// [NOTE]
// expected following strings.
//
// myrolename
//
std::istringstream ssrole(response);
std::string oneline;
if (getline(ssrole, oneline, '\n')){
rolename = oneline;
return !rolename.empty();
}
return false;
}
bool S3fsCurl::SetIAMRoleFromMetaData(const char* response)
{
S3FS_PRN_INFO3("IAM role name response = \"%s\"", response);
std::string rolename;
if(!S3fsCurl::ParseIAMRoleFromMetaDataResponse(response, rolename)){
return false;
}
SetIAMRole(rolename.c_str());
return true;
}
bool S3fsCurl::AddUserAgent(CURL* hCurl)
{
if(!hCurl){
@ -1982,6 +1746,10 @@ S3fsCurl::S3fsCurl(bool ahbe) :
b_ssekey_pos(-1), b_ssetype(sse_type_t::SSE_DISABLE),
sem(NULL), completed_tids_lock(NULL), completed_tids(NULL), fpLazySetup(NULL), curlCode(CURLE_OK)
{
if(!S3fsCurl::ps3fscred){
S3FS_PRN_CRIT("The object of S3fs Credential class is not initialized.");
abort();
}
}
S3fsCurl::~S3fsCurl()
@ -2462,7 +2230,7 @@ bool S3fsCurl::RemakeHandle()
if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback)){
return false;
}
if(S3fsCurl::is_ibm_iam_auth){
if(S3fsCurl::ps3fscred->IsIBMIAMAuth()){
if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_POST, true)){
return false;
}
@ -2712,7 +2480,7 @@ int S3fsCurl::RequestPerform(bool dontAddAuthHeaders /*=false*/)
case CURLE_PEER_FAILED_VERIFICATION:
S3FS_PRN_ERR("### CURLE_PEER_FAILED_VERIFICATION");
first_pos = bucket.find_first_of('.');
first_pos = S3fsCred::GetBucket().find_first_of('.');
if(first_pos != std::string::npos){
S3FS_PRN_INFO("curl returned a CURL_PEER_FAILED_VERIFICATION error");
S3FS_PRN_INFO("security issue found: buckets with periods in their name are incompatible with http");
@ -2789,8 +2557,8 @@ std::string S3fsCurl::CalcSignatureV2(const std::string& method, const std::stri
std::string Signature;
std::string StringToSign;
if(!S3fsCurl::IAM_role.empty() || S3fsCurl::is_ecs || S3fsCurl::is_use_session_token){
requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-security-token", S3fsCurl::AWSAccessToken.c_str());
if(!S3fsCurl::ps3fscred->GetIAMRole().empty() || S3fsCurl::ps3fscred->IsECS() || S3fsCurl::ps3fscred->IsUseSessionToken()){
requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-security-token", S3fsCurl::ps3fscred->GetAccessToken().c_str());
}
StringToSign += method + "\n";
@ -2800,8 +2568,8 @@ std::string S3fsCurl::CalcSignatureV2(const std::string& method, const std::stri
StringToSign += get_canonical_headers(requestHeaders, true);
StringToSign += resource;
const void* key = S3fsCurl::AWSSecretAccessKey.data();
size_t key_len = S3fsCurl::AWSSecretAccessKey.size();
const void* key = S3fsCurl::ps3fscred->GetSecretAccessKey().data();
size_t key_len = S3fsCurl::ps3fscred->GetSecretAccessKey().size();
const unsigned char* sdata = reinterpret_cast<const unsigned char*>(StringToSign.data());
size_t sdata_len = StringToSign.size();
unsigned char* md = NULL;
@ -2827,8 +2595,8 @@ std::string S3fsCurl::CalcSignature(const std::string& method, const std::string
std::string Signature, StringCQ, StringToSign;
std::string uriencode;
if(!S3fsCurl::IAM_role.empty() || S3fsCurl::is_ecs || S3fsCurl::is_use_session_token){
requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-security-token", S3fsCurl::AWSAccessToken.c_str());
if(!S3fsCurl::ps3fscred->GetIAMRole().empty() || S3fsCurl::ps3fscred->IsECS() || S3fsCurl::ps3fscred->IsUseSessionToken()){
requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-security-token", S3fsCurl::ps3fscred->GetAccessToken().c_str());
}
uriencode = urlEncode(canonical_uri);
@ -2849,7 +2617,7 @@ std::string S3fsCurl::CalcSignature(const std::string& method, const std::string
StringCQ += get_sorted_header_keys(requestHeaders) + "\n";
StringCQ += payload_hash;
std::string kSecret = "AWS4" + S3fsCurl::AWSSecretAccessKey;
std::string kSecret = "AWS4" + S3fsCurl::ps3fscred->GetSecretAccessKey();
unsigned char *kDate, *kRegion, *kService, *kSigning, *sRequest = NULL;
unsigned int kDate_len,kRegion_len, kService_len, kSigning_len, sRequest_len = 0;
@ -2931,7 +2699,7 @@ void S3fsCurl::insertV4Headers()
get_date_sigv3(strdate, date8601);
std::string contentSHA256 = payload_hash.empty() ? EMPTY_PAYLOAD_HASH : payload_hash;
const std::string realpath = pathrequeststyle ? "/" + bucket + server_path : server_path;
const std::string realpath = pathrequeststyle ? "/" + S3fsCred::GetBucket() + server_path : server_path;
//string canonical_headers, signed_headers;
requestHeaders = curl_slist_sort_insert(requestHeaders, "host", get_bucket_host().c_str());
@ -2944,7 +2712,7 @@ void S3fsCurl::insertV4Headers()
if(!S3fsCurl::IsPublicBucket()){
std::string Signature = CalcSignature(op, realpath, query_string + (type == REQTYPE_PREMULTIPOST || type == REQTYPE_MULTILIST ? "=" : ""), strdate, contentSHA256, date8601);
std::string auth = "AWS4-HMAC-SHA256 Credential=" + AWSAccessKeyId + "/" + strdate + "/" + endpoint + "/s3/aws4_request, SignedHeaders=" + get_sorted_header_keys(requestHeaders) + ", Signature=" + Signature;
std::string auth = "AWS4-HMAC-SHA256 Credential=" + S3fsCurl::ps3fscred->GetAccessKeyID() + "/" + strdate + "/" + endpoint + "/s3/aws4_request, SignedHeaders=" + get_sorted_header_keys(requestHeaders) + ", Signature=" + Signature;
requestHeaders = curl_slist_sort_insert(requestHeaders, "Authorization", auth.c_str());
}
}
@ -2967,28 +2735,28 @@ void S3fsCurl::insertV2Headers()
if(!S3fsCurl::IsPublicBucket()){
std::string Signature = CalcSignatureV2(op, get_header_value(requestHeaders, "Content-MD5"), get_header_value(requestHeaders, "Content-Type"), date, resource);
requestHeaders = curl_slist_sort_insert(requestHeaders, "Authorization", std::string("AWS " + AWSAccessKeyId + ":" + Signature).c_str());
requestHeaders = curl_slist_sort_insert(requestHeaders, "Authorization", std::string("AWS " + S3fsCurl::ps3fscred->GetAccessKeyID() + ":" + Signature).c_str());
}
}
void S3fsCurl::insertIBMIAMHeaders()
{
requestHeaders = curl_slist_sort_insert(requestHeaders, "Authorization", ("Bearer " + S3fsCurl::AWSAccessToken).c_str());
requestHeaders = curl_slist_sort_insert(requestHeaders, "Authorization", ("Bearer " + S3fsCurl::ps3fscred->GetAccessToken()).c_str());
if(op == "PUT" && path == mount_prefix + "/"){
// ibm-service-instance-id header is required for bucket creation requests
requestHeaders = curl_slist_sort_insert(requestHeaders, "ibm-service-instance-id", S3fsCurl::AWSAccessKeyId.c_str());
requestHeaders = curl_slist_sort_insert(requestHeaders, "ibm-service-instance-id", S3fsCurl::ps3fscred->GetAccessKeyID().c_str());
}
}
void S3fsCurl::insertAuthHeaders()
{
if(!S3fsCurl::CheckIAMCredentialUpdate()){
if(!S3fsCurl::ps3fscred->CheckIAMCredentialUpdate()){
S3FS_PRN_ERR("An error occurred in checking IAM credential.");
return; // do not insert auth headers on error
}
if(S3fsCurl::is_ibm_iam_auth){
if(S3fsCurl::ps3fscred->IsIBMIAMAuth()){
insertIBMIAMHeaders();
}else if(S3fsCurl::signature_type == V2_ONLY){
insertV2Headers();
@ -3037,7 +2805,7 @@ int S3fsCurl::DeleteRequest(const char* tpath)
//
int S3fsCurl::GetIAMv2ApiToken()
{
url = std::string(S3fsCurl::IAMv2_token_url);
url = std::string(S3fsCred::IAMv2_token_url);
if(!CreateCurlHandle()){
return -EIO;
}
@ -3045,9 +2813,8 @@ int S3fsCurl::GetIAMv2ApiToken()
responseHeaders.clear();
bodydata.Clear();
std::string ttlstr = str(S3fsCurl::IAMv2_token_ttl);
requestHeaders = curl_slist_sort_insert(requestHeaders, S3fsCurl::IAMv2_token_ttl_hdr.c_str(),
ttlstr.c_str());
std::string ttlstr = str(S3fsCred::IAMv2_token_ttl);
requestHeaders = curl_slist_sort_insert(requestHeaders, S3fsCred::IAMv2_token_ttl_hdr, ttlstr.c_str());
// Curl appends an "Expect: 100-continue" header to the token request,
// and aws responds with a 417 Expectation Failed. This ensures the
@ -3072,7 +2839,7 @@ int S3fsCurl::GetIAMv2ApiToken()
int result = RequestPerform(true);
if(0 == result && !S3fsCurl::SetIAMv2APIToken(bodydata.str())){
if(0 == result && !S3fsCurl::ps3fscred->SetIAMv2APIToken(bodydata.str())){
S3FS_PRN_ERR("Error storing IMDSv2 API token.");
result = -EIO;
}
@ -3090,10 +2857,10 @@ int S3fsCurl::GetIAMv2ApiToken()
//
int S3fsCurl::GetIAMCredentials()
{
if (!S3fsCurl::is_ecs && !S3fsCurl::is_ibm_iam_auth) {
S3FS_PRN_INFO3("[IAM role=%s]", S3fsCurl::IAM_role.c_str());
if (!S3fsCurl::ps3fscred->IsECS() && !S3fsCurl::ps3fscred->IsIBMIAMAuth()) {
S3FS_PRN_INFO3("[IAM role=%s]", S3fsCurl::ps3fscred->GetIAMRole().c_str());
if(S3fsCurl::IAM_role.empty()) {
if(S3fsCurl::ps3fscred->GetIAMRole().empty()) {
S3FS_PRN_ERR("IAM role name is empty.");
return -EIO;
}
@ -3107,22 +2874,22 @@ int S3fsCurl::GetIAMCredentials()
}
// url
if(is_ecs){
const char *env = std::getenv(ECS_IAM_ENV_VAR);
if(S3fsCurl::ps3fscred->IsECS()){
const char *env = std::getenv(S3fsCred::ECS_IAM_ENV_VAR);
if(env == NULL){
S3FS_PRN_ERR("%s is not set.", ECS_IAM_ENV_VAR);
S3FS_PRN_ERR("%s is not set.", S3fsCred::ECS_IAM_ENV_VAR);
return -EIO;
}
url = std::string(S3fsCurl::IAM_cred_url) + env;
url = S3fsCurl::ps3fscred->GetIAMCredentialsURL() + env;
}else{
if(S3fsCurl::IAM_api_version > 1){
if(S3fsCurl::ps3fscred->GetIMDSVersion() > 1){
int result = GetIAMv2ApiToken();
if(-ENOENT == result){
// If we get a 404 back when requesting the token service,
// then it's highly likely we're running in an environment
// that doesn't support the AWS IMDSv2 API, so we'll skip
// the token retrieval in the future.
SetIMDSVersion(1);
S3fsCurl::ps3fscred->SetIMDSVersion(1);
}else if(result != 0){
// If we get an unexpected error when retrieving the API
// token, log it but continue. Requirement for including
@ -3131,8 +2898,7 @@ int S3fsCurl::GetIAMCredentials()
S3FS_PRN_ERR("AWS IMDSv2 token retrieval failed: %d", result);
}
}
url = std::string(S3fsCurl::IAM_cred_url) + S3fsCurl::IAM_role;
url = S3fsCurl::ps3fscred->GetIAMCredentialsURL() + S3fsCurl::ps3fscred->GetIAMRole();
}
requestHeaders = NULL;
@ -3140,13 +2906,13 @@ int S3fsCurl::GetIAMCredentials()
bodydata.Clear();
std::string postContent;
if(S3fsCurl::is_ibm_iam_auth){
url = std::string(S3fsCurl::IAM_cred_url);
if(S3fsCurl::ps3fscred->IsIBMIAMAuth()){
url = S3fsCurl::ps3fscred->GetIAMCredentialsURL();
// make contents
postContent += "grant_type=urn:ibm:params:oauth:grant-type:apikey";
postContent += "&response_type=cloud_iam";
postContent += "&apikey=" + S3fsCurl::AWSSecretAccessKey;
postContent += "&apikey=" + S3fsCurl::ps3fscred->GetSecretAccessKey();
// set postdata
postdata = reinterpret_cast<const unsigned char*>(postContent.c_str());
@ -3170,8 +2936,8 @@ int S3fsCurl::GetIAMCredentials()
}
}
if(S3fsCurl::IAM_api_version > 1){
requestHeaders = curl_slist_sort_insert(requestHeaders, S3fsCurl::IAMv2_token_hdr.c_str(), S3fsCurl::IAMv2_api_token.c_str());
if(S3fsCurl::ps3fscred->GetIMDSVersion() > 1){
requestHeaders = curl_slist_sort_insert(requestHeaders, S3fsCred::IAMv2_token_hdr, S3fsCurl::ps3fscred->GetIAMv2APIToken().c_str());
}
if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str())){
@ -3190,7 +2956,7 @@ int S3fsCurl::GetIAMCredentials()
int result = RequestPerform(true);
// analyzing response
if(0 == result && !S3fsCurl::SetIAMCredentials(bodydata.str())){
if(0 == result && !S3fsCurl::ps3fscred->SetIAMCredentials(bodydata.str())){
S3FS_PRN_ERR("Something error occurred, could not get IAM credential.");
result = -EIO;
}
@ -3214,22 +2980,22 @@ bool S3fsCurl::LoadIAMRoleFromMetaData()
}
// url
if(is_ecs){
const char *env = std::getenv(ECS_IAM_ENV_VAR);
if(S3fsCurl::ps3fscred->IsECS()){
const char *env = std::getenv(S3fsCred::ECS_IAM_ENV_VAR);
if(env == NULL){
S3FS_PRN_ERR("%s is not set.", ECS_IAM_ENV_VAR);
S3FS_PRN_ERR("%s is not set.", S3fsCred::ECS_IAM_ENV_VAR);
return -EIO;
}
url = std::string(S3fsCurl::IAM_cred_url) + env;
url = S3fsCurl::ps3fscred->GetIAMCredentialsURL() + env;
}else{
if(S3fsCurl::IAM_api_version > 1){
if(S3fsCurl::ps3fscred->GetIMDSVersion() > 1){
int result = GetIAMv2ApiToken();
if(-ENOENT == result){
// If we get a 404 back when requesting the token service,
// then it's highly likely we're running in an environment
// that doesn't support the AWS IMDSv2 API, so we'll skip
// the token retrieval in the future.
SetIMDSVersion(1);
S3fsCurl::ps3fscred->SetIMDSVersion(1);
}else if(result != 0){
// If we get an unexpected error when retrieving the API
// token, log it but continue. Requirement for including
@ -3238,15 +3004,14 @@ bool S3fsCurl::LoadIAMRoleFromMetaData()
S3FS_PRN_ERR("AWS IMDSv2 token retrieval failed: %d", result);
}
}
url = std::string(S3fsCurl::IAM_cred_url);
url = S3fsCurl::ps3fscred->GetIAMCredentialsURL();
}
requestHeaders = NULL;
responseHeaders.clear();
bodydata.Clear();
if(S3fsCurl::IAM_api_version > 1){
requestHeaders = curl_slist_sort_insert(requestHeaders, S3fsCurl::IAMv2_token_hdr.c_str(), S3fsCurl::IAMv2_api_token.c_str());
if(S3fsCurl::ps3fscred->GetIMDSVersion() > 1){
requestHeaders = curl_slist_sort_insert(requestHeaders, S3fsCred::IAMv2_token_hdr, S3fsCurl::ps3fscred->GetIAMv2APIToken().c_str());
}
if(CURLE_OK != curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str())){
@ -3265,7 +3030,7 @@ bool S3fsCurl::LoadIAMRoleFromMetaData()
int result = RequestPerform(true);
// analyzing response
if(0 == result && !S3fsCurl::SetIAMRoleFromMetaData(bodydata.str())){
if(0 == result && !S3fsCurl::ps3fscred->SetIAMRoleFromMetaData(bodydata.str())){
S3FS_PRN_ERR("Something error occurred, could not get IAM role name.");
result = -EIO;
}

50
src/curl.h
View File

@ -34,6 +34,7 @@
#include "psemaphore.h"
#include "metaheader.h"
#include "fdcache_page.h"
#include "s3fs_cred.h"
//----------------------------------------------
// Avoid dependency on libcurl version
@ -83,7 +84,6 @@ class S3fsCurl;
// Prototype function for lazy setup options for curl handle
typedef bool (*s3fscurl_lazy_setup)(S3fsCurl* s3fscurl);
typedef std::map<std::string, std::string> iamcredmap_t;
typedef std::map<std::string, std::string> sseckeymap_t;
typedef std::list<sseckeymap_t> sseckeylist_t;
@ -140,24 +140,7 @@ class S3fsCurl
static bool is_content_md5;
static bool is_verbose;
static bool is_dump_body;
static std::string AWSAccessKeyId;
static std::string AWSSecretAccessKey;
static std::string AWSAccessToken;
static time_t AWSAccessTokenExpire;
static bool is_ecs;
static bool is_use_session_token;
static bool is_ibm_iam_auth;
static std::string IAM_cred_url;
static int IAM_api_version;
static std::string IAMv2_token_url;
static int IAMv2_token_ttl;
static std::string IAMv2_token_ttl_hdr;
static std::string IAMv2_token_hdr;
static std::string IAMv2_api_token;
static size_t IAM_field_count;
static std::string IAM_token_field;
static std::string IAM_expiry_field;
static std::string IAM_role;
static S3fsCred* ps3fscred;
static long ssl_verify_hostname;
static curltime_t curl_times;
static curlprogress_t curl_progress;
@ -252,11 +235,6 @@ class S3fsCurl
static bool PreGetObjectRequestSetCurlOpts(S3fsCurl* s3fscurl);
static bool PreHeadRequestSetCurlOpts(S3fsCurl* s3fscurl);
static bool ParseIAMCredentialResponse(const char* response, iamcredmap_t& keyval);
static bool SetIAMCredentials(const char* response);
static bool SetIAMv2APIToken(const char* response);
static bool ParseIAMRoleFromMetaDataResponse(const char* response, std::string& rolename);
static bool SetIAMRoleFromMetaData(const char* response);
static bool LoadEnvSseCKeys();
static bool LoadEnvSseKmsid();
static bool PushbackSseKeys(const std::string& onekey);
@ -278,8 +256,6 @@ class S3fsCurl
std::string CalcSignatureV2(const std::string& method, const std::string& strMD5, const std::string& content_type, const std::string& date, const std::string& resource);
std::string CalcSignature(const std::string& method, const std::string& canonical_uri, const std::string& query_string, const std::string& strdate, const std::string& payload_hash, const std::string& date8601);
int GetIAMv2ApiToken();
int GetIAMCredentials();
int UploadMultipartPostSetup(const char* tpath, int part_num, const std::string& upload_id);
int CopyMultipartPostSetup(const char* from, const char* to, int part_num, const std::string& upload_id, headers_t& meta);
bool UploadMultipartPostComplete();
@ -290,12 +266,12 @@ class S3fsCurl
public:
// class methods
static bool InitS3fsCurl();
static bool InitCredentialObject(S3fsCred* pcredobj);
static bool InitMimeType(const std::string& strFile);
static bool DestroyS3fsCurl();
static int ParallelMultipartUploadRequest(const char* tpath, headers_t& meta, int fd);
static int ParallelMixMultipartUploadRequest(const char* tpath, headers_t& meta, int fd, const fdpage_list_t& mixuppages);
static int ParallelGetObjectRequest(const char* tpath, int fd, off_t start, off_t size);
static bool CheckIAMCredentialUpdate();
// class methods(variables)
static std::string LookupMimeType(const std::string& name);
@ -332,16 +308,6 @@ class S3fsCurl
static bool GetVerbose() { return S3fsCurl::is_verbose; }
static bool SetDumpBody(bool flag);
static bool IsDumpBody() { return S3fsCurl::is_dump_body; }
static bool SetAccessKey(const char* AccessKeyId, const char* SecretAccessKey);
static bool SetAccessKeyWithSessionToken(const char* AccessKeyId, const char* SecretAccessKey, const char * SessionToken);
static bool IsSetAccessKeyID()
{
return !S3fsCurl::AWSAccessKeyId.empty();
}
static bool IsSetAccessKeys()
{
return !S3fsCurl::IAM_role.empty() || ((!S3fsCurl::AWSAccessKeyId.empty() || S3fsCurl::is_ibm_iam_auth) && !S3fsCurl::AWSSecretAccessKey.empty());
}
static long SetSslVerifyHostname(long value);
static long GetSslVerifyHostname() { return S3fsCurl::ssl_verify_hostname; }
static void ResetOffset(S3fsCurl* pCurl);
@ -351,14 +317,6 @@ class S3fsCurl
// maximum parallel HEAD requests
static int SetMaxMultiRequest(int max);
static int GetMaxMultiRequest() { return S3fsCurl::max_multireq; }
static bool SetIsECS(bool flag);
static bool SetIsIBMIAMAuth(bool flag);
static size_t SetIAMFieldCount(size_t field_count);
static std::string SetIAMCredentialsURL(const char* url);
static std::string SetIAMTokenField(const char* token_field);
static std::string SetIAMExpiryField(const char* expiry_field);
static std::string SetIAMRole(const char* role);
static const char* GetIAMRole() { return S3fsCurl::IAM_role.c_str(); }
static bool SetMultipartSize(off_t size);
static off_t GetMultipartSize() { return S3fsCurl::multipart_size; }
static bool SetMultipartCopySize(off_t size);
@ -374,12 +332,12 @@ class S3fsCurl
static bool IsListObjectsV2() { return S3fsCurl::listobjectsv2; }
static bool SetRequesterPays(bool flag) { bool old_flag = S3fsCurl::requester_pays; S3fsCurl::requester_pays = flag; return old_flag; }
static bool IsRequesterPays() { return S3fsCurl::requester_pays; }
static bool SetIMDSVersion(int version);
// methods
bool CreateCurlHandle(bool only_pool = false, bool remake = false);
bool DestroyCurlHandle(bool restore_pool = true, bool clear_internal_data = true);
int GetIAMCredentials();
bool LoadIAMRoleFromMetaData();
bool AddSseRequestHead(sse_type_t ssetype, const std::string& ssevalue, bool is_only_c, bool is_copy);
bool GetResponseCode(long& responseCode, bool from_curl_handle = true);

11
src/curl_util.cpp
View File

@ -27,6 +27,7 @@
#include "curl_util.h"
#include "string_util.h"
#include "s3fs_auth.h"
#include "s3fs_cred.h"
//-------------------------------------------------------------------
// Utility Functions
@ -244,7 +245,7 @@ bool MakeUrlResource(const char* realpath, std::string& resourcepath, std::strin
if(!realpath){
return false;
}
resourcepath = urlEncode(service_path + bucket + realpath);
resourcepath = urlEncode(service_path + S3fsCred::GetBucket() + realpath);
url = s3host + resourcepath;
return true;
}
@ -257,7 +258,7 @@ std::string prepare_url(const char* url)
std::string hostname;
std::string path;
std::string url_str = std::string(url);
std::string token = std::string("/") + bucket;
std::string token = std::string("/") + S3fsCred::GetBucket();
size_t bucket_pos;
size_t bucket_length = token.size();
size_t uri_length = 0;
@ -271,7 +272,7 @@ std::string prepare_url(const char* url)
bucket_pos = url_str.find(token, uri_length);
if(!pathrequeststyle){
hostname = bucket + "." + url_str.substr(uri_length, bucket_pos - uri_length);
hostname = S3fsCred::GetBucket() + "." + url_str.substr(uri_length, bucket_pos - uri_length);
path = url_str.substr((bucket_pos + bucket_length));
}else{
hostname = url_str.substr(uri_length, bucket_pos - uri_length);
@ -279,7 +280,7 @@ std::string prepare_url(const char* url)
if('/' != part[0]){
part = "/" + part;
}
path = "/" + bucket + part;
path = "/" + S3fsCred::GetBucket() + part;
}
url_str = uri + hostname + path;
@ -354,7 +355,7 @@ std::string url_to_host(const std::string &url)
std::string get_bucket_host()
{
if(!pathrequeststyle){
return bucket + "." + url_to_host(s3host);
return S3fsCred::GetBucket() + "." + url_to_host(s3host);
}
return url_to_host(s3host);
}

11
src/fdcache.cpp
View File

@ -31,6 +31,7 @@
#include "fdcache_pseudofd.h"
#include "s3fs_util.h"
#include "s3fs_logger.h"
#include "s3fs_cred.h"
#include "string_util.h"
#include "autolock.h"
@ -125,7 +126,7 @@ bool FdManager::DeleteCacheDirectory()
return false;
}
std::string mirror_path = FdManager::cache_dir + "/." + bucket + ".mirror";
std::string mirror_path = FdManager::cache_dir + "/." + S3fsCred::GetBucket() + ".mirror";
if(!delete_files_in_dir(mirror_path.c_str(), true)){
return false;
}
@ -181,10 +182,10 @@ bool FdManager::MakeCachePath(const char* path, std::string& cache_path, bool is
std::string resolved_path(FdManager::cache_dir);
if(!is_mirror_path){
resolved_path += "/";
resolved_path += bucket;
resolved_path += S3fsCred::GetBucket();
}else{
resolved_path += "/.";
resolved_path += bucket;
resolved_path += S3fsCred::GetBucket();
resolved_path += ".mirror";
}
@ -208,7 +209,7 @@ bool FdManager::CheckCacheTopDir()
if(FdManager::cache_dir.empty()){
return true;
}
std::string toppath(FdManager::cache_dir + "/" + bucket);
std::string toppath(FdManager::cache_dir + "/" + S3fsCred::GetBucket());
return check_exist_dir_permission(toppath.c_str());
}
@ -749,7 +750,7 @@ void FdManager::CleanupCacheDirInternal(const std::string &path)
{
DIR* dp;
struct dirent* dent;
std::string abs_path = cache_dir + "/" + bucket + path;
std::string abs_path = cache_dir + "/" + S3fsCred::GetBucket() + path;
if(NULL == (dp = opendir(abs_path.c_str()))){
S3FS_PRN_ERR("could not open cache dir(%s) - errno(%d)", abs_path.c_str(), errno);

2
src/fdcache_entity.cpp
View File

@ -2090,7 +2090,7 @@ int FdEntity::UploadPendingMeta()
}
headers_t updatemeta = orgmeta;
updatemeta["x-amz-copy-source"] = urlEncode(service_path + bucket + get_realpath(path.c_str()));
updatemeta["x-amz-copy-source"] = urlEncode(service_path + S3fsCred::GetBucket() + get_realpath(path.c_str()));
updatemeta["x-amz-metadata-directive"] = "REPLACE";
// put headers, no need to update mtime to avoid dead lock
int result = put_headers(path.c_str(), updatemeta, true);

5
src/fdcache_stat.cpp
View File

@ -29,6 +29,7 @@
#include "fdcache_stat.h"
#include "fdcache.h"
#include "s3fs_util.h"
#include "s3fs_cred.h"
#include "string_util.h"
//------------------------------------------------
@ -37,14 +38,14 @@
std::string CacheFileStat::GetCacheFileStatTopDir()
{
std::string top_path;
if(!FdManager::IsCacheDir() || bucket.empty()){
if(!FdManager::IsCacheDir() || S3fsCred::GetBucket().empty()){
return top_path;
}
// stat top dir( "/<cache_dir>/.<bucket_name>.stat" )
top_path += FdManager::GetCacheDir();
top_path += "/.";
top_path += bucket;
top_path += S3fsCred::GetBucket();
top_path += ".stat";
return top_path;
}

624
src/s3fs.cpp
View File

@ -72,13 +72,12 @@ static mode_t mp_mode = 0; // mode of mount point
static mode_t mp_umask = 0; // umask for mount point
static bool is_mp_umask = false;// default does not set.
static std::string mountpoint;
static std::string passwd_file;
static S3fsCred* ps3fscred = NULL; // using only in this file
static std::string mimetype_file;
static bool nocopyapi = false;
static bool norenameapi = false;
static bool nonempty = false;
static bool allow_other = false;
static bool load_iamrole = false;
static uid_t s3fs_uid = 0;
static gid_t s3fs_gid = 0;
static mode_t s3fs_umask = 0;
@ -86,10 +85,7 @@ static bool is_s3fs_uid = false;// default does not set.
static bool is_s3fs_gid = false;// default does not set.
static bool is_s3fs_umask = false;// default does not set.
static bool is_remove_cache = false;
static bool is_ecs = false;
static bool is_ibm_iam_auth = false;
static bool is_use_xattr = false;
static bool is_use_session_token = false;
static off_t multipart_threshold = 25 * 1024 * 1024;
static int64_t singlepart_copy_limit = 512 * 1024 * 1024;
static bool is_specified_endpoint = false;
@ -100,11 +96,6 @@ static off_t max_dirty_data = 5LL * 1024LL * 1024LL * 1024LL;
static bool use_wtf8 = false;
static off_t fake_diskfree_size = -1; // default is not set(-1)
static const char ALLBUCKET_FIELDS_TYPE[] = ""; // special key for mapping(This name is absolutely not used as a bucket name)
static const char KEYVAL_FIELDS_TYPE[] = "\t"; // special key for mapping(This name is absolutely not used as a bucket name)
static const char AWS_ACCESSKEYID[] = "AWSAccessKeyId";
static const char AWS_SECRETKEY[] = "AWSSecretKey";
//-------------------------------------------------------------------
// Global functions : prototype
//-------------------------------------------------------------------
@ -139,12 +130,6 @@ static bool parse_xattr_keyval(const std::string& xattrpair, std::string& key, P
static size_t parse_xattrs(const std::string& strxattrs, xattrs_t& xattrs);
static std::string build_xattrs(const xattrs_t& xattrs);
static int s3fs_check_service();
static int parse_passwd_file(bucketkvmap_t& resmap);
static int check_for_aws_format(const kvmap_t& kvmap);
static int check_passwd_file_perms();
static int read_aws_credentials_file(const std::string &filename);
static int read_passwd_file();
static int get_access_keys();
static bool set_mountpoint_attribute(struct stat& mpst);
static int set_bucket(const char* arg);
static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_args* outargs);
@ -1207,7 +1192,7 @@ static int rename_object(const char* from, const char* to, bool update_ctime)
if(update_ctime){
meta["x-amz-meta-ctime"] = str(time(NULL));
}
meta["x-amz-copy-source"] = urlEncode(service_path + bucket + s3_realpath);
meta["x-amz-copy-source"] = urlEncode(service_path + S3fsCred::GetBucket() + s3_realpath);
meta["Content-Type"] = S3fsCurl::LookupMimeType(std::string(to));
meta["x-amz-metadata-directive"] = "REPLACE";
@ -1626,7 +1611,7 @@ static int s3fs_chmod(const char* _path, mode_t mode)
headers_t updatemeta;
updatemeta["x-amz-meta-ctime"] = str(time(NULL));
updatemeta["x-amz-meta-mode"] = str(mode);
updatemeta["x-amz-copy-source"] = urlEncode(service_path + bucket + get_realpath(strpath.c_str()));
updatemeta["x-amz-copy-source"] = urlEncode(service_path + S3fsCred::GetBucket() + get_realpath(strpath.c_str()));
updatemeta["x-amz-metadata-directive"] = "REPLACE";
// check opened file handle.
@ -1803,7 +1788,7 @@ static int s3fs_chown(const char* _path, uid_t uid, gid_t gid)
updatemeta["x-amz-meta-ctime"] = str(time(NULL));
updatemeta["x-amz-meta-uid"] = str(uid);
updatemeta["x-amz-meta-gid"] = str(gid);
updatemeta["x-amz-copy-source"] = urlEncode(service_path + bucket + get_realpath(strpath.c_str()));
updatemeta["x-amz-copy-source"] = urlEncode(service_path + S3fsCred::GetBucket() + get_realpath(strpath.c_str()));
updatemeta["x-amz-metadata-directive"] = "REPLACE";
// check opened file handle.
@ -2007,7 +1992,7 @@ static int s3fs_utimens(const char* _path, const struct timespec ts[2])
updatemeta["x-amz-meta-mtime"] = str(mtime);
updatemeta["x-amz-meta-ctime"] = str(actime);
updatemeta["x-amz-meta-atime"] = str(actime);
updatemeta["x-amz-copy-source"] = urlEncode(service_path + bucket + get_realpath(strpath.c_str()));
updatemeta["x-amz-copy-source"] = urlEncode(service_path + S3fsCred::GetBucket() + get_realpath(strpath.c_str()));
updatemeta["x-amz-metadata-directive"] = "REPLACE";
// check opened file handle.
@ -3028,7 +3013,7 @@ static int s3fs_setxattr(const char* path, const char* name, const char* value,
// set xattr all object
headers_t updatemeta;
updatemeta["x-amz-meta-ctime"] = str(time(NULL));
updatemeta["x-amz-copy-source"] = urlEncode(service_path + bucket + get_realpath(strpath.c_str()));
updatemeta["x-amz-copy-source"] = urlEncode(service_path + S3fsCred::GetBucket() + get_realpath(strpath.c_str()));
updatemeta["x-amz-metadata-directive"] = "REPLACE";
// check opened file handle.
@ -3309,7 +3294,7 @@ static int s3fs_removexattr(const char* path, const char* name)
// set xattr all object
headers_t updatemeta;
updatemeta["x-amz-copy-source"] = urlEncode(service_path + bucket + get_realpath(strpath.c_str()));
updatemeta["x-amz-copy-source"] = urlEncode(service_path + S3fsCred::GetBucket() + get_realpath(strpath.c_str()));
updatemeta["x-amz-metadata-directive"] = "REPLACE";
if(!xattrs.empty()){
updatemeta["x-amz-meta-xattr"] = build_xattrs(xattrs);
@ -3380,7 +3365,7 @@ static void* s3fs_init(struct fuse_conn_info* conn)
}
// check loading IAM role name
if(load_iamrole){
if(ps3fscred->IsIAMRoleMetadataType()){
// load IAM role name from http://169.254.169.254/latest/meta-data/iam/security-credentials
//
S3fsCurl s3fscurl;
@ -3389,7 +3374,7 @@ static void* s3fs_init(struct fuse_conn_info* conn)
s3fs_exit_fuseloop(EXIT_FAILURE);
return NULL;
}
S3FS_PRN_INFO("loaded IAM role name = %s", S3fsCurl::GetIAMRole());
S3FS_PRN_INFO("loaded IAM role name = %s", ps3fscred->GetIAMRole().c_str());
}
// Check Bucket
@ -3501,8 +3486,8 @@ static int s3fs_check_service()
S3FS_PRN_INFO("check services.");
// At first time for access S3, we check IAM role if it sets.
if(!S3fsCurl::CheckIAMCredentialUpdate()){
S3FS_PRN_CRIT("Failed to check IAM role name(%s).", S3fsCurl::GetIAMRole());
if(!ps3fscred->CheckIAMCredentialUpdate()){
S3FS_PRN_CRIT("Failed to check IAM role name(%s).", ps3fscred->GetIAMRole().c_str());
return EXIT_FAILURE;
}
@ -3598,474 +3583,6 @@ static int s3fs_check_service()
return EXIT_SUCCESS;
}
//
// Read and Parse passwd file
//
// The line of the password file is one of the following formats:
// (1) "accesskey:secretkey" : AWS format for default(all) access key/secret key
// (2) "bucket:accesskey:secretkey" : AWS format for bucket's access key/secret key
// (3) "key=value" : Content-dependent KeyValue contents
//
// This function sets result into bucketkvmap_t, it bucket name and key&value mapping.
// If bucket name is empty(1 or 3 format), bucket name for mapping is set "\t" or "".
//
// Return: 1 - OK(could parse and set mapping etc.)
// 0 - NG(could not read any value)
// -1 - Should shutdown immediately
//
static int parse_passwd_file(bucketkvmap_t& resmap)
{
std::string line;
size_t first_pos;
readline_t linelist;
readline_t::iterator iter;
// open passwd file
std::ifstream PF(passwd_file.c_str());
if(!PF.good()){
S3FS_PRN_EXIT("could not open passwd file : %s", passwd_file.c_str());
return -1;
}
// read each line
while(getline(PF, line)){
line = trim(line);
if(line.empty()){
continue;
}
if('#' == line[0]){
continue;
}
if(std::string::npos != line.find_first_of(" \t")){
S3FS_PRN_EXIT("invalid line in passwd file, found whitespace character.");
return -1;
}
if('[' == line[0]){
S3FS_PRN_EXIT("invalid line in passwd file, found a bracket \"[\" character.");
return -1;
}
linelist.push_back(line);
}
// read '=' type
kvmap_t kv;
for(iter = linelist.begin(); iter != linelist.end(); ++iter){
first_pos = iter->find_first_of('=');
if(first_pos == std::string::npos){
continue;
}
// formatted by "key=val"
std::string key = trim(iter->substr(0, first_pos));
std::string val = trim(iter->substr(first_pos + 1, std::string::npos));
if(key.empty()){
continue;
}
if(kv.end() != kv.find(key)){
S3FS_PRN_WARN("same key name(%s) found in passwd file, skip this.", key.c_str());
continue;
}
kv[key] = val;
}
// set special key name
resmap[KEYVAL_FIELDS_TYPE] = kv;
// read ':' type
for(iter = linelist.begin(); iter != linelist.end(); ++iter){
first_pos = iter->find_first_of(':');
size_t last_pos = iter->find_last_of(':');
if(first_pos == std::string::npos){
continue;
}
std::string bucketname;
std::string accesskey;
std::string secret;
if(first_pos != last_pos){
// formatted by "bucket:accesskey:secretkey"
bucketname = trim(iter->substr(0, first_pos));
accesskey = trim(iter->substr(first_pos + 1, last_pos - first_pos - 1));
secret = trim(iter->substr(last_pos + 1, std::string::npos));
}else{
// formatted by "accesskey:secretkey"
bucketname = ALLBUCKET_FIELDS_TYPE;
accesskey = trim(iter->substr(0, first_pos));
secret = trim(iter->substr(first_pos + 1, std::string::npos));
}
if(resmap.end() != resmap.find(bucketname)){
S3FS_PRN_EXIT("there are multiple entries for the same bucket(%s) in the passwd file.", (bucketname.empty() ? "default" : bucketname.c_str()));
return -1;
}
kv.clear();
kv[AWS_ACCESSKEYID] = accesskey;
kv[AWS_SECRETKEY] = secret;
resmap[bucketname] = kv;
}
return (resmap.empty() ? 0 : 1);
}
//
// Return: 1 - OK(could read and set accesskey etc.)
// 0 - NG(could not read)
// -1 - Should shutdown immediately
//
static int check_for_aws_format(const kvmap_t& kvmap)
{
std::string str1(AWS_ACCESSKEYID);
std::string str2(AWS_SECRETKEY);
if(kvmap.empty()){
return 0;
}
kvmap_t::const_iterator str1_it = kvmap.find(str1);
kvmap_t::const_iterator str2_it = kvmap.find(str2);
if(kvmap.end() == str1_it && kvmap.end() == str2_it){
return 0;
}
if(kvmap.end() == str1_it || kvmap.end() == str2_it){
S3FS_PRN_EXIT("AWSAccesskey or AWSSecretkey is not specified.");
return -1;
}
if(!S3fsCurl::SetAccessKey(str1_it->second.c_str(), str2_it->second.c_str())){
S3FS_PRN_EXIT("failed to set access key/secret key.");
return -1;
}
return 1;
}
//
// check_passwd_file_perms
//
// expect that global passwd_file variable contains
// a non-empty value and is readable by the current user
//
// Check for too permissive access to the file
// help save users from themselves via a security hole
//
// only two options: return or error out
//
static int check_passwd_file_perms()
{
struct stat info;
// let's get the file info
if(stat(passwd_file.c_str(), &info) != 0){
S3FS_PRN_EXIT("unexpected error from stat(%s).", passwd_file.c_str());
return EXIT_FAILURE;
}
// return error if any file has others permissions
if( (info.st_mode & S_IROTH) ||
(info.st_mode & S_IWOTH) ||
(info.st_mode & S_IXOTH)) {
S3FS_PRN_EXIT("credentials file %s should not have others permissions.", passwd_file.c_str());
return EXIT_FAILURE;
}
// Any local file should not have any group permissions
// /etc/passwd-s3fs can have group permissions
if(passwd_file != "/etc/passwd-s3fs"){
if( (info.st_mode & S_IRGRP) ||
(info.st_mode & S_IWGRP) ||
(info.st_mode & S_IXGRP)) {
S3FS_PRN_EXIT("credentials file %s should not have group permissions.", passwd_file.c_str());
return EXIT_FAILURE;
}
}else{
// "/etc/passwd-s3fs" does not allow group write.
if((info.st_mode & S_IWGRP)){
S3FS_PRN_EXIT("credentials file %s should not have group writable permissions.", passwd_file.c_str());
return EXIT_FAILURE;
}
}
if((info.st_mode & S_IXUSR) || (info.st_mode & S_IXGRP)){
S3FS_PRN_EXIT("credentials file %s should not have executable permissions.", passwd_file.c_str());
return EXIT_FAILURE;
}
return EXIT_SUCCESS;
}
static int read_aws_credentials_file(const std::string &filename)
{
// open passwd file
std::ifstream PF(filename.c_str());
if(!PF.good()){
return -1;
}
std::string profile;
std::string accesskey;
std::string secret;
std::string session_token;
// read each line
std::string line;
while(getline(PF, line)){
line = trim(line);
if(line.empty()){
continue;
}
if('#' == line[0]){
continue;
}
if(line.size() > 2 && line[0] == '[' && line[line.size() - 1] == ']') {
if(profile == aws_profile){
break;
}
profile = line.substr(1, line.size() - 2);
accesskey.clear();
secret.clear();
session_token.clear();
}
size_t pos = line.find_first_of('=');
if(pos == std::string::npos){
continue;
}
std::string key = trim(line.substr(0, pos));
std::string value = trim(line.substr(pos + 1, std::string::npos));
if(key == "aws_access_key_id"){
accesskey = value;
}else if(key == "aws_secret_access_key"){
secret = value;
}else if(key == "aws_session_token"){
session_token = value;
}
}
if(profile != aws_profile){
return EXIT_FAILURE;
}
if (session_token.empty()) {
if (is_use_session_token) {
S3FS_PRN_EXIT("AWS session token was expected but wasn't provided in aws/credentials file for profile: %s.", aws_profile.c_str());
return EXIT_FAILURE;
}
if(!S3fsCurl::SetAccessKey(accesskey.c_str(), secret.c_str())){
S3FS_PRN_EXIT("failed to set internal data for access key/secret key from aws credential file.");
return EXIT_FAILURE;
}
} else {
if (!S3fsCurl::SetAccessKeyWithSessionToken(accesskey.c_str(), secret.c_str(), session_token.c_str())) {
S3FS_PRN_EXIT("session token is invalid.");
return EXIT_FAILURE;
}
}
return EXIT_SUCCESS;
}
//
// read_passwd_file
//
// Support for per bucket credentials
//
// Format for the credentials file:
// [bucket:]AccessKeyId:SecretAccessKey
//
// Lines beginning with # are considered comments
// and ignored, as are empty lines
//
// Uncommented lines without the ":" character are flagged as
// an error, so are lines with spaces or tabs
//
// only one default key pair is allowed, but not required
//
static int read_passwd_file()
{
bucketkvmap_t bucketmap;
kvmap_t keyval;
int result;
// if you got here, the password file
// exists and is readable by the
// current user, check for permissions
if(EXIT_SUCCESS != check_passwd_file_perms()){
return EXIT_FAILURE;
}
//
// parse passwd file
//
result = parse_passwd_file(bucketmap);
if(-1 == result){
return EXIT_FAILURE;
}
//
// check key=value type format.
//
bucketkvmap_t::iterator it = bucketmap.find(KEYVAL_FIELDS_TYPE);
if(bucketmap.end() != it){
// aws format
result = check_for_aws_format(it->second);
if(-1 == result){
return EXIT_FAILURE;
}else if(1 == result){
// success to set
return EXIT_SUCCESS;
}
}
std::string bucket_key = ALLBUCKET_FIELDS_TYPE;
if(!bucket.empty() && bucketmap.end() != bucketmap.find(bucket)){
bucket_key = bucket;
}
it = bucketmap.find(bucket_key);
if(bucketmap.end() == it){
S3FS_PRN_EXIT("Not found access key/secret key in passwd file.");
return EXIT_FAILURE;
}
keyval = it->second;
kvmap_t::iterator aws_accesskeyid_it = keyval.find(AWS_ACCESSKEYID);
kvmap_t::iterator aws_secretkey_it = keyval.find(AWS_SECRETKEY);
if(keyval.end() == aws_accesskeyid_it || keyval.end() == aws_secretkey_it){
S3FS_PRN_EXIT("Not found access key/secret key in passwd file.");
return EXIT_FAILURE;
}
if(!S3fsCurl::SetAccessKey(aws_accesskeyid_it->second.c_str(), aws_secretkey_it->second.c_str())){
S3FS_PRN_EXIT("failed to set internal data for access key/secret key from passwd file.");
return EXIT_FAILURE;
}
return EXIT_SUCCESS;
}
//
// get_access_keys
//
// called only when were are not mounting a
// public bucket
//
// Here is the order precedence for getting the
// keys:
//
// 1 - from the command line (security risk)
// 2 - from a password file specified on the command line
// 3 - from environment variables
// 3a - from the AWS_CREDENTIAL_FILE environment variable
// 3b - from ${HOME}/.aws/credentials
// 4 - from the users ~/.passwd-s3fs
// 5 - from /etc/passwd-s3fs
//
static int get_access_keys()
{
// should be redundant
if(S3fsCurl::IsPublicBucket()){
return EXIT_SUCCESS;
}
// access key loading is deferred
if(load_iamrole || is_ecs){
return EXIT_SUCCESS;
}
// 1 - keys specified on the command line
if(S3fsCurl::IsSetAccessKeys()){
return EXIT_SUCCESS;
}
// 2 - was specified on the command line
if(!passwd_file.empty()){
std::ifstream PF(passwd_file.c_str());
if(PF.good()){
PF.close();
return read_passwd_file();
}else{
S3FS_PRN_EXIT("specified passwd_file is not readable.");
return EXIT_FAILURE;
}
}
// 3 - environment variables
char* AWSACCESSKEYID = getenv("AWS_ACCESS_KEY_ID") ? getenv("AWS_ACCESS_KEY_ID") : getenv("AWSACCESSKEYID");
char* AWSSECRETACCESSKEY = getenv("AWS_SECRET_ACCESS_KEY") ? getenv("AWS_SECRET_ACCESS_KEY") : getenv("AWSSECRETACCESSKEY");
char* AWSSESSIONTOKEN = getenv("AWS_SESSION_TOKEN") ? getenv("AWS_SESSION_TOKEN") : getenv("AWSSESSIONTOKEN");
if(AWSACCESSKEYID != NULL || AWSSECRETACCESSKEY != NULL){
if( (AWSACCESSKEYID == NULL && AWSSECRETACCESSKEY != NULL) ||
(AWSACCESSKEYID != NULL && AWSSECRETACCESSKEY == NULL) ){
S3FS_PRN_EXIT("both environment variables AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY must be set together.");
return EXIT_FAILURE;
}
S3FS_PRN_INFO2("access key from env variables");
if (AWSSESSIONTOKEN != NULL) {
S3FS_PRN_INFO2("session token is available");
if (!S3fsCurl::SetAccessKeyWithSessionToken(AWSACCESSKEYID, AWSSECRETACCESSKEY, AWSSESSIONTOKEN)) {
S3FS_PRN_EXIT("session token is invalid.");
return EXIT_FAILURE;
}
} else {
S3FS_PRN_INFO2("session token is not available");
if (is_use_session_token) {
S3FS_PRN_EXIT("environment variable AWS_SESSION_TOKEN is expected to be set.");
return EXIT_FAILURE;
}
}
if(!S3fsCurl::SetAccessKey(AWSACCESSKEYID, AWSSECRETACCESSKEY)){
S3FS_PRN_EXIT("if one access key is specified, both keys need to be specified.");
return EXIT_FAILURE;
}
return EXIT_SUCCESS;
}
// 3a - from the AWS_CREDENTIAL_FILE environment variable
char * AWS_CREDENTIAL_FILE;
AWS_CREDENTIAL_FILE = getenv("AWS_CREDENTIAL_FILE");
if(AWS_CREDENTIAL_FILE != NULL){
passwd_file = AWS_CREDENTIAL_FILE;
if(!passwd_file.empty()){
std::ifstream PF(passwd_file.c_str());
if(PF.good()){
PF.close();
return read_passwd_file();
}else{
S3FS_PRN_EXIT("AWS_CREDENTIAL_FILE: \"%s\" is not readable.", passwd_file.c_str());
return EXIT_FAILURE;
}
}
}
// 3b - check ${HOME}/.aws/credentials
std::string aws_credentials = std::string(getpwuid(getuid())->pw_dir) + "/.aws/credentials";
if(read_aws_credentials_file(aws_credentials) == EXIT_SUCCESS) {
return EXIT_SUCCESS;
}else if(aws_profile != "default"){
S3FS_PRN_EXIT("Could not find profile: %s in file: %s", aws_profile.c_str(), aws_credentials.c_str());
return EXIT_FAILURE;
}
// 4 - from the default location in the users home directory
char * HOME;
HOME = getenv ("HOME");
if(HOME != NULL){
passwd_file = HOME;
passwd_file += "/.passwd-s3fs";
std::ifstream PF(passwd_file.c_str());
if(PF.good()){
PF.close();
if(EXIT_SUCCESS != read_passwd_file()){
return EXIT_FAILURE;
}
// It is possible that the user's file was there but
// contained no key pairs i.e. commented out
// in that case, go look in the final location
if(S3fsCurl::IsSetAccessKeys()){
return EXIT_SUCCESS;
}
}
}
// 5 - from the system default location
passwd_file = "/etc/passwd-s3fs";
std::ifstream PF(passwd_file.c_str());
if(PF.good()){
PF.close();
return read_passwd_file();
}
S3FS_PRN_EXIT("could not determine how to establish security credentials.");
return EXIT_FAILURE;
}
//
// Check & Set attributes for mount point.
//
@ -4112,7 +3629,10 @@ static int set_bucket(const char* arg)
S3FS_PRN_EXIT("bucket name and path(\"%s\") is wrong, it must be \"bucket[:/path]\".", arg);
return -1;
}
bucket = strtok(bucket_name, ":");
if(!S3fsCred::SetBucket(strtok(bucket_name, ":"))){
S3FS_PRN_EXIT("bucket name and path(\"%s\") is wrong, it must be \"bucket[:/path]\".", arg);
return -1;
}
char* pmount_prefix = strtok(NULL, "");
if(pmount_prefix){
if(0 == strlen(pmount_prefix) || '/' != pmount_prefix[0]){
@ -4126,7 +3646,10 @@ static int set_bucket(const char* arg)
}
}
}else{
bucket = arg;
if(!S3fsCred::SetBucket(arg)){
S3FS_PRN_EXIT("bucket name and path(\"%s\") is wrong, it must be \"bucket[:/path]\".", arg);
return -1;
}
}
return 0;
}
@ -4142,7 +3665,7 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
int ret;
if(key == FUSE_OPT_KEY_NONOPT){
// the first NONOPT option is the bucket name
if(bucket.empty()){
if(S3fsCred::GetBucket().empty()){
if ((ret = set_bucket(arg))){
return ret;
}
@ -4427,21 +3950,21 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
return 0;
}
if(is_prefix(arg, "passwd_file=")){
passwd_file = strchr(arg, '=') + sizeof(char);
ps3fscred->SetS3fsPasswdFile(strchr(arg, '=') + sizeof(char));
return 0;
}
if(0 == strcmp(arg, "ibm_iam_auth")){
S3fsCurl::SetIsIBMIAMAuth(true);
S3fsCurl::SetIAMCredentialsURL("https://iam.cloud.ibm.com/identity/token");
S3fsCurl::SetIAMTokenField("\"access_token\"");
S3fsCurl::SetIAMExpiryField("\"expiration\"");
S3fsCurl::SetIAMFieldCount(2);
S3fsCurl::SetIMDSVersion(1);
is_ibm_iam_auth = true;
ps3fscred->SetIsIBMIAMAuth(true);
ps3fscred->SetIAMCredentialsURL("https://iam.cloud.ibm.com/identity/token");
ps3fscred->SetIAMTokenField("\"access_token\"");
ps3fscred->SetIAMExpiryField("\"expiration\"");
ps3fscred->SetIAMFieldCount(2);
ps3fscred->SetIMDSVersion(1);
ps3fscred->SetIsIBMIAMAuth(true);
return 0;
}
if (0 == strcmp(arg, "use_session_token")) {
is_use_session_token = true;
ps3fscred->SetIsUseSessionToken(true);
return 0;
}
if(is_prefix(arg, "ibm_iam_endpoint=")){
@ -4453,45 +3976,45 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
return -1;
}
endpoint_url = std::string(iam_endpoint) + "/identity/token";
S3fsCurl::SetIAMCredentialsURL(endpoint_url.c_str());
ps3fscred->SetIAMCredentialsURL(endpoint_url.c_str());
return 0;
}
if(0 == strcmp(arg, "imdsv1only")){
S3fsCurl::SetIMDSVersion(1);
ps3fscred->SetIMDSVersion(1);
return 0;
}
if(0 == strcmp(arg, "ecs")){
if (is_ibm_iam_auth) {
if(ps3fscred->IsIBMIAMAuth()){
S3FS_PRN_EXIT("option ecs cannot be used in conjunction with ibm");
return -1;
}
S3fsCurl::SetIsECS(true);
S3fsCurl::SetIMDSVersion(1);
S3fsCurl::SetIAMCredentialsURL("http://169.254.170.2");
S3fsCurl::SetIAMFieldCount(5);
is_ecs = true;
ps3fscred->SetIsECS(true);
ps3fscred->SetIMDSVersion(1);
ps3fscred->SetIAMCredentialsURL("http://169.254.170.2");
ps3fscred->SetIAMFieldCount(5);
ps3fscred->SetIsECS(true);
return 0;
}
if(is_prefix(arg, "iam_role")){
if (is_ecs || is_ibm_iam_auth) {
if(ps3fscred->IsECS() || ps3fscred->IsIBMIAMAuth()){
S3FS_PRN_EXIT("option iam_role cannot be used in conjunction with ecs or ibm");
return -1;
}
if(0 == strcmp(arg, "iam_role") || 0 == strcmp(arg, "iam_role=auto")){
// loading IAM role name in s3fs_init(), because we need to wait initializing curl.
//
load_iamrole = true;
ps3fscred->SetIAMRoleMetadataType(true);
return 0;
}else if(is_prefix(arg, "iam_role=")){
const char* role = strchr(arg, '=') + sizeof(char);
S3fsCurl::SetIAMRole(role);
load_iamrole = false;
ps3fscred->SetIAMRole(role);
ps3fscred->SetIAMRoleMetadataType(false);
return 0;
}
}
if(is_prefix(arg, "profile=")){
aws_profile = strchr(arg, '=') + sizeof(char);
ps3fscred->SetAwsProfileName(strchr(arg, '=') + sizeof(char));
return 0;
}
if(is_prefix(arg, "public_bucket=")){
@ -4917,6 +4440,18 @@ int main(int argc, char* argv[])
program_name.replace(0, found+1, "");
}
// set credential object
//
// This local variable is the only credential object.
// It is also set in the S3fsCurl class and this object is used.
//
S3fsCred s3fscredObj;
ps3fscred = &s3fscredObj;
if(!S3fsCurl::InitCredentialObject(&s3fscredObj)){
S3FS_PRN_EXIT("Failed to setup credential object to s3fs curl.");
exit(EXIT_FAILURE);
}
while((ch = getopt_long(argc, argv, "dho:fsu", long_opts, &option_index)) != -1){
switch(ch){
case 0:
@ -5057,9 +4592,10 @@ int main(int argc, char* argv[])
max_dirty_data = -1;
}
// The first plain argument is the bucket
if(bucket.empty()){
S3FS_PRN_EXIT("missing BUCKET argument.");
//
// Checking forbidden parameters for bucket
//
if(!ps3fscred->CheckForbiddenBucketParams()){
show_usage();
S3fsCurl::DestroyS3fsCurl();
s3fs_destroy_global_ssl();
@ -5067,33 +4603,6 @@ int main(int argc, char* argv[])
exit(EXIT_FAILURE);
}
// bucket names cannot contain upper case characters in virtual-hosted style
if((!pathrequeststyle) && (lower(bucket) != bucket)){
S3FS_PRN_EXIT("BUCKET %s, name not compatible with virtual-hosted style.", bucket.c_str());
S3fsCurl::DestroyS3fsCurl();
s3fs_destroy_global_ssl();
destroy_parser_xml_lock();
exit(EXIT_FAILURE);
}
// check bucket name for illegal characters
found = bucket.find_first_of("/:\\;!@#$%^&*?|+=");
if(found != std::string::npos){
S3FS_PRN_EXIT("BUCKET %s -- bucket name contains an illegal character.", bucket.c_str());
S3fsCurl::DestroyS3fsCurl();
s3fs_destroy_global_ssl();
destroy_parser_xml_lock();
exit(EXIT_FAILURE);
}
if(!pathrequeststyle && is_prefix(s3host.c_str(), "https://") && bucket.find_first_of('.') != std::string::npos) {
S3FS_PRN_EXIT("BUCKET %s -- cannot mount bucket with . while using HTTPS without use_path_request_style", bucket.c_str());
S3fsCurl::DestroyS3fsCurl();
s3fs_destroy_global_ssl();
destroy_parser_xml_lock();
exit(EXIT_FAILURE);
}
// The second plain argument is the mountpoint
// if the option was given, we all ready checked for a
// readable, non-empty directory, this checks determines
@ -5110,28 +4619,31 @@ int main(int argc, char* argv[])
}
// error checking of command line arguments for compatibility
if(S3fsCurl::IsPublicBucket() && S3fsCurl::IsSetAccessKeys()){
if(S3fsCurl::IsPublicBucket() && ps3fscred->IsSetAccessKeys()){
S3FS_PRN_EXIT("specifying both public_bucket and the access keys options is invalid.");
S3fsCurl::DestroyS3fsCurl();
s3fs_destroy_global_ssl();
destroy_parser_xml_lock();
exit(EXIT_FAILURE);
}
if(!passwd_file.empty() && S3fsCurl::IsSetAccessKeys()){
if(!ps3fscred->IsSetPasswdFile() && ps3fscred->IsSetAccessKeys()){
S3FS_PRN_EXIT("specifying both passwd_file and the access keys options is invalid.");
S3fsCurl::DestroyS3fsCurl();
s3fs_destroy_global_ssl();
destroy_parser_xml_lock();
exit(EXIT_FAILURE);
}
if(!S3fsCurl::IsPublicBucket() && !load_iamrole && !is_ecs){
if(EXIT_SUCCESS != get_access_keys()){
if(!S3fsCurl::IsPublicBucket() && !ps3fscred->IsIAMRoleMetadataType() && !ps3fscred->IsECS()){
if(!ps3fscred->InitialS3fsCredentials()){
S3fsCurl::DestroyS3fsCurl();
s3fs_destroy_global_ssl();
destroy_parser_xml_lock();
exit(EXIT_FAILURE);
}
if(!S3fsCurl::IsSetAccessKeys()){
if(!ps3fscred->IsSetAccessKeys()){
S3FS_PRN_EXIT("could not establish security credentials, check documentation.");
S3fsCurl::DestroyS3fsCurl();
s3fs_destroy_global_ssl();
@ -5166,7 +4678,7 @@ int main(int argc, char* argv[])
}
// check IBM IAM requirements
if(is_ibm_iam_auth){
if(ps3fscred->IsIBMIAMAuth()){
// check that default ACL is either public-read or private
acl_t defaultACL = S3fsCurl::GetDefaultAcl();
if(defaultACL != acl_t::PRIVATE && defaultACL != acl_t::PUBLIC_READ){
@ -5197,7 +4709,7 @@ int main(int argc, char* argv[])
// See issue #128strncasecmp
/*
if(1 == S3fsCurl::GetSslVerifyHostname()){
found = bucket.find_first_of('.');
found = S3fsCred::GetBucket().find_first_of('.');
if(found != std::string::npos){
found = s3host.find("https:");
if(found != std::string::npos){

920
src/s3fs_cred.cpp Normal file
View File

@ -0,0 +1,920 @@
/*
* s3fs - FUSE-based file system backed by Amazon S3
*
* Copyright(C) 2007 Randy Rizun <rrizun@gmail.com>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
#include <unistd.h>
#include <pwd.h>
#include <sys/types.h>
#include <fstream>
#include <sstream>
#include "common.h"
#include "s3fs.h"
#include "s3fs_cred.h"
#include "curl.h"
#include "string_util.h"
#include "metaheader.h"
//-------------------------------------------------------------------
// Symbols
//-------------------------------------------------------------------
#define DEFAULT_AWS_PROFILE_NAME "default"
//-------------------------------------------------------------------
// Class Variables
//-------------------------------------------------------------------
const char* S3fsCred::ALLBUCKET_FIELDS_TYPE = "";
const char* S3fsCred::KEYVAL_FIELDS_TYPE = "\t";
const char* S3fsCred::AWS_ACCESSKEYID = "AWSAccessKeyId";
const char* S3fsCred::AWS_SECRETKEY = "AWSSecretKey";
const int S3fsCred::IAM_EXPIRE_MERGIN = 20 * 60; // update timing
const char* S3fsCred::ECS_IAM_ENV_VAR = "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI";
const char* S3fsCred::IAMCRED_ACCESSKEYID = "AccessKeyId";
const char* S3fsCred::IAMCRED_SECRETACCESSKEY = "SecretAccessKey";
const char* S3fsCred::IAMCRED_ROLEARN = "RoleArn";
const char* S3fsCred::IAMv2_token_url = "http://169.254.169.254/latest/api/token";
int S3fsCred::IAMv2_token_ttl = 21600;
const char* S3fsCred::IAMv2_token_ttl_hdr = "X-aws-ec2-metadata-token-ttl-seconds";
const char* S3fsCred::IAMv2_token_hdr = "X-aws-ec2-metadata-token";
std::string S3fsCred::bucket_name;
//-------------------------------------------------------------------
// Class Methods
//-------------------------------------------------------------------
bool S3fsCred::SetBucket(const char* bucket)
{
if(!bucket || strlen(bucket) == 0){
return false;
}
S3fsCred::bucket_name = bucket;
return true;
}
const std::string& S3fsCred::GetBucket()
{
return S3fsCred::bucket_name;
}
bool S3fsCred::ParseIAMRoleFromMetaDataResponse(const char* response, std::string& rolename)
{
if(!response){
return false;
}
// [NOTE]
// expected following strings.
//
// myrolename
//
std::istringstream ssrole(response);
std::string oneline;
if (getline(ssrole, oneline, '\n')){
rolename = oneline;
return !rolename.empty();
}
return false;
}
//-------------------------------------------------------------------
// Methods : Constructor / Destructor
//-------------------------------------------------------------------
S3fsCred::S3fsCred() :
passwd_file(""),
aws_profile(DEFAULT_AWS_PROFILE_NAME),
load_iamrole(false),
AWSAccessKeyId(""),
AWSSecretAccessKey(""),
AWSAccessToken(""),
AWSAccessTokenExpire(0),
is_ecs(false),
is_use_session_token(false),
is_ibm_iam_auth(false),
IAM_cred_url("http://169.254.169.254/latest/meta-data/iam/security-credentials/"),
IAM_api_version(2),
IAMv2_api_token(""),
IAM_field_count(4),
IAM_token_field("Token"),
IAM_expiry_field("Expiration"),
IAM_role("")
{
}
S3fsCred::~S3fsCred()
{
}
//-------------------------------------------------------------------
// Methods : Access member variables
//-------------------------------------------------------------------
bool S3fsCred::SetS3fsPasswdFile(const char* file)
{
if(!file || strlen(file) == 0){
return false;
}
passwd_file = file;
return true;
}
bool S3fsCred::IsSetPasswdFile()
{
return !passwd_file.empty();
}
bool S3fsCred::SetAwsProfileName(const char* name)
{
if(!name || strlen(name) == 0){
return false;
}
aws_profile = name;
return true;
}
bool S3fsCred::SetIAMRoleMetadataType(bool flag)
{
bool old = load_iamrole;
load_iamrole = flag;
return old;
}
bool S3fsCred::SetAccessKey(const char* AccessKeyId, const char* SecretAccessKey)
{
if((!is_ibm_iam_auth && (!AccessKeyId || '\0' == AccessKeyId[0])) || !SecretAccessKey || '\0' == SecretAccessKey[0]){
return false;
}
AWSAccessKeyId = AccessKeyId;
AWSSecretAccessKey = SecretAccessKey;
return true;
}
bool S3fsCred::SetAccessKeyWithSessionToken(const char* AccessKeyId, const char* SecretAccessKey, const char * SessionToken)
{
bool access_key_is_empty = !AccessKeyId || '\0' == AccessKeyId[0];
bool secret_access_key_is_empty = !SecretAccessKey || '\0' == SecretAccessKey[0];
bool session_token_is_empty = !SessionToken || '\0' == SessionToken[0];
if((!is_ibm_iam_auth && access_key_is_empty) || secret_access_key_is_empty || session_token_is_empty){
return false;
}
AWSAccessKeyId = AccessKeyId;
AWSSecretAccessKey = SecretAccessKey;
AWSAccessToken = SessionToken;
is_use_session_token= true;
return true;
}
bool S3fsCred::IsSetAccessKeyID() const
{
return !AWSAccessKeyId.empty();
}
bool S3fsCred::IsSetAccessKeys() const
{
return !IAM_role.empty() || ((!AWSAccessKeyId.empty() || is_ibm_iam_auth) && !AWSSecretAccessKey.empty());
}
bool S3fsCred::SetIsECS(bool flag)
{
bool old = is_ecs;
is_ecs = flag;
return old;
}
bool S3fsCred::SetIsUseSessionToken(bool flag)
{
bool old = is_use_session_token;
is_use_session_token = flag;
return old;
}
bool S3fsCred::SetIsIBMIAMAuth(bool flag)
{
bool old = is_ibm_iam_auth;
is_ibm_iam_auth = flag;
return old;
}
std::string S3fsCred::SetIAMRole(const char* role)
{
std::string old = IAM_role;
IAM_role = role ? role : "";
return old;
}
size_t S3fsCred::SetIAMFieldCount(size_t field_count)
{
size_t old = IAM_field_count;
IAM_field_count = field_count;
return old;
}
std::string S3fsCred::SetIAMCredentialsURL(const char* url)
{
std::string old = IAM_cred_url;
IAM_cred_url = url ? url : "";
return old;
}
std::string S3fsCred::SetIAMTokenField(const char* token_field)
{
std::string old = IAM_token_field;
IAM_token_field = token_field ? token_field : "";
return old;
}
std::string S3fsCred::SetIAMExpiryField(const char* expiry_field)
{
std::string old = IAM_expiry_field;
IAM_expiry_field = expiry_field ? expiry_field : "";
return old;
}
int S3fsCred::SetIMDSVersion(int version)
{
int old = IAM_api_version;
IAM_api_version = version;
return old;
}
bool S3fsCred::SetIAMv2APIToken(const char* response)
{
S3FS_PRN_INFO3("Setting AWS IMDSv2 API token to %s", response ? response : "(null)");
if(!response){
return false;
}
IAMv2_api_token = std::string(response);
return true;
}
bool S3fsCred::SetIAMCredentials(const char* response)
{
S3FS_PRN_INFO3("IAM credential response = \"%s\"", response);
iamcredmap_t keyval;
if(!ParseIAMCredentialResponse(response, keyval)){
return false;
}
if(IAM_field_count != keyval.size()){
return false;
}
AWSAccessToken = keyval[IAM_token_field];
if(is_ibm_iam_auth){
off_t tmp_expire = 0;
if(!s3fs_strtoofft(&tmp_expire, keyval[IAM_expiry_field].c_str(), /*base=*/ 10)){
return false;
}
AWSAccessTokenExpire = static_cast<time_t>(tmp_expire);
}else{
AWSAccessKeyId = keyval[std::string(S3fsCred::IAMCRED_ACCESSKEYID)];
AWSSecretAccessKey = keyval[std::string(S3fsCred::IAMCRED_SECRETACCESSKEY)];
AWSAccessTokenExpire = cvtIAMExpireStringToTime(keyval[IAM_expiry_field].c_str());
}
return true;
}
bool S3fsCred::SetIAMRoleFromMetaData(const char* response)
{
S3FS_PRN_INFO3("IAM role name response = \"%s\"", response ? response : "(null)");
std::string rolename;
if(!S3fsCred::ParseIAMRoleFromMetaDataResponse(response, rolename)){
return false;
}
SetIAMRole(rolename.c_str());
return true;
}
//-------------------------------------------------------------------
// Methods : for Credentials
//-------------------------------------------------------------------
//
// Check passwd file readable
//
bool S3fsCred::IsReadableS3fsPasswdFile()
{
if(passwd_file.empty()){
return false;
}
std::ifstream PF(passwd_file.c_str());
if(!PF.good()){
return false;
}
PF.close();
return true;
}
//
// S3fsCred::CheckS3fsPasswdFilePerms
//
// expect that global passwd_file variable contains
// a non-empty value and is readable by the current user
//
// Check for too permissive access to the file
// help save users from themselves via a security hole
//
// only two options: return or error out
//
bool S3fsCred::CheckS3fsPasswdFilePerms()
{
struct stat info;
// let's get the file info
if(stat(passwd_file.c_str(), &info) != 0){
S3FS_PRN_EXIT("unexpected error from stat(%s).", passwd_file.c_str());
return false;
}
// Check readable
if(!IsReadableS3fsPasswdFile()){
S3FS_PRN_EXIT("S3fs passwd file \"%s\" is not readable.", passwd_file.c_str());
return false;
}
// return error if any file has others permissions
if( (info.st_mode & S_IROTH) ||
(info.st_mode & S_IWOTH) ||
(info.st_mode & S_IXOTH)) {
S3FS_PRN_EXIT("credentials file %s should not have others permissions.", passwd_file.c_str());
return false;
}
// Any local file should not have any group permissions
// /etc/passwd-s3fs can have group permissions
if(passwd_file != "/etc/passwd-s3fs"){
if( (info.st_mode & S_IRGRP) ||
(info.st_mode & S_IWGRP) ||
(info.st_mode & S_IXGRP)) {
S3FS_PRN_EXIT("credentials file %s should not have group permissions.", passwd_file.c_str());
return false;
}
}else{
// "/etc/passwd-s3fs" does not allow group write.
if((info.st_mode & S_IWGRP)){
S3FS_PRN_EXIT("credentials file %s should not have group writable permissions.", passwd_file.c_str());
return false;
}
}
if((info.st_mode & S_IXUSR) || (info.st_mode & S_IXGRP)){
S3FS_PRN_EXIT("credentials file %s should not have executable permissions.", passwd_file.c_str());
return false;
}
return true;
}
//
// Read and Parse passwd file
//
// The line of the password file is one of the following formats:
// (1) "accesskey:secretkey" : AWS format for default(all) access key/secret key
// (2) "bucket:accesskey:secretkey" : AWS format for bucket's access key/secret key
// (3) "key=value" : Content-dependent KeyValue contents
//
// This function sets result into bucketkvmap_t, it bucket name and key&value mapping.
// If bucket name is empty(1 or 3 format), bucket name for mapping is set "\t" or "".
//
// Return: true - Succeed parsing
// false - Should shutdown immediately
//
bool S3fsCred::ParseS3fsPasswdFile(bucketkvmap_t& resmap)
{
std::string line;
size_t first_pos;
readline_t linelist;
readline_t::iterator iter;
// open passwd file
std::ifstream PF(passwd_file.c_str());
if(!PF.good()){
S3FS_PRN_EXIT("could not open passwd file : %s", passwd_file.c_str());
return false;;
}
// read each line
while(getline(PF, line)){
line = trim(line);
if(line.empty()){
continue;
}
if('#' == line[0]){
continue;
}
if(std::string::npos != line.find_first_of(" \t")){
S3FS_PRN_EXIT("invalid line in passwd file, found whitespace character.");
return false;;
}
if('[' == line[0]){
S3FS_PRN_EXIT("invalid line in passwd file, found a bracket \"[\" character.");
return false;;
}
linelist.push_back(line);
}
// read '=' type
kvmap_t kv;
for(iter = linelist.begin(); iter != linelist.end(); ++iter){
first_pos = iter->find_first_of('=');
if(first_pos == std::string::npos){
continue;
}
// formatted by "key=val"
std::string key = trim(iter->substr(0, first_pos));
std::string val = trim(iter->substr(first_pos + 1, std::string::npos));
if(key.empty()){
continue;
}
if(kv.end() != kv.find(key)){
S3FS_PRN_WARN("same key name(%s) found in passwd file, skip this.", key.c_str());
continue;
}
kv[key] = val;
}
// set special key name
resmap[S3fsCred::KEYVAL_FIELDS_TYPE] = kv;
// read ':' type
for(iter = linelist.begin(); iter != linelist.end(); ++iter){
first_pos = iter->find_first_of(':');
size_t last_pos = iter->find_last_of(':');
if(first_pos == std::string::npos){
continue;
}
std::string bucketname;
std::string accesskey;
std::string secret;
if(first_pos != last_pos){
// formatted by "bucket:accesskey:secretkey"
bucketname= trim(iter->substr(0, first_pos));
accesskey = trim(iter->substr(first_pos + 1, last_pos - first_pos - 1));
secret = trim(iter->substr(last_pos + 1, std::string::npos));
}else{
// formatted by "accesskey:secretkey"
bucketname= S3fsCred::ALLBUCKET_FIELDS_TYPE;
accesskey = trim(iter->substr(0, first_pos));
secret = trim(iter->substr(first_pos + 1, std::string::npos));
}
if(resmap.end() != resmap.find(bucketname)){
S3FS_PRN_EXIT("there are multiple entries for the same bucket(%s) in the passwd file.", (bucketname.empty() ? "default" : bucketname.c_str()));
return false;;
}
kv.clear();
kv[S3fsCred::AWS_ACCESSKEYID] = accesskey;
kv[S3fsCred::AWS_SECRETKEY] = secret;
resmap[bucketname] = kv;
}
return true;
}
//
// ReadS3fsPasswdFile
//
// Support for per bucket credentials
//
// Format for the credentials file:
// [bucket:]AccessKeyId:SecretAccessKey
//
// Lines beginning with # are considered comments
// and ignored, as are empty lines
//
// Uncommented lines without the ":" character are flagged as
// an error, so are lines with spaces or tabs
//
// only one default key pair is allowed, but not required
//
bool S3fsCred::ReadS3fsPasswdFile()
{
bucketkvmap_t bucketmap;
kvmap_t keyval;
// if you got here, the password file
// exists and is readable by the
// current user, check for permissions
if(!CheckS3fsPasswdFilePerms()){
return false;
}
//
// parse passwd file
//
if(!ParseS3fsPasswdFile(bucketmap)){
return false;
}
//
// check key=value type format.
//
bucketkvmap_t::iterator it = bucketmap.find(S3fsCred::KEYVAL_FIELDS_TYPE);
if(bucketmap.end() != it){
// aws format
int result = CheckS3fsCredentialAwsFormat(it->second);
if(-1 == result){
return false;
}else if(1 == result){
// success to set
return true;
}
}
std::string bucket_key = S3fsCred::ALLBUCKET_FIELDS_TYPE;
if(!S3fsCred::bucket_name.empty() && bucketmap.end() != bucketmap.find(S3fsCred::bucket_name)){
bucket_key = S3fsCred::bucket_name;
}
it = bucketmap.find(bucket_key);
if(bucketmap.end() == it){
S3FS_PRN_EXIT("Not found access key/secret key in passwd file.");
return false;
}
keyval = it->second;
kvmap_t::iterator aws_accesskeyid_it = keyval.find(S3fsCred::AWS_ACCESSKEYID);
kvmap_t::iterator aws_secretkey_it = keyval.find(S3fsCred::AWS_SECRETKEY);
if(keyval.end() == aws_accesskeyid_it || keyval.end() == aws_secretkey_it){
S3FS_PRN_EXIT("Not found access key/secret key in passwd file.");
return false;
}
if(!SetAccessKey(aws_accesskeyid_it->second.c_str(), aws_secretkey_it->second.c_str())){
S3FS_PRN_EXIT("failed to set internal data for access key/secret key from passwd file.");
return false;
}
return true;
}
//
// Return: 1 - OK(could read and set accesskey etc.)
// 0 - NG(could not read)
// -1 - Should shutdown immediately
//
int S3fsCred::CheckS3fsCredentialAwsFormat(const kvmap_t& kvmap)
{
std::string str1(S3fsCred::AWS_ACCESSKEYID);
std::string str2(S3fsCred::AWS_SECRETKEY);
if(kvmap.empty()){
return 0;
}
kvmap_t::const_iterator str1_it = kvmap.find(str1);
kvmap_t::const_iterator str2_it = kvmap.find(str2);
if(kvmap.end() == str1_it && kvmap.end() == str2_it){
return 0;
}
if(kvmap.end() == str1_it || kvmap.end() == str2_it){
S3FS_PRN_EXIT("AWSAccesskey or AWSSecretkey is not specified.");
return -1;
}
if(!SetAccessKey(str1_it->second.c_str(), str2_it->second.c_str())){
S3FS_PRN_EXIT("failed to set access key/secret key.");
return -1;
}
return 1;
}
//
// Read Aws Credential File
//
bool S3fsCred::ReadAwsCredentialFile(const std::string &filename)
{
// open passwd file
std::ifstream PF(filename.c_str());
if(!PF.good()){
return false;
}
std::string profile;
std::string accesskey;
std::string secret;
std::string session_token;
// read each line
std::string line;
while(getline(PF, line)){
line = trim(line);
if(line.empty()){
continue;
}
if('#' == line[0]){
continue;
}
if(line.size() > 2 && line[0] == '[' && line[line.size() - 1] == ']') {
if(profile == aws_profile){
break;
}
profile = line.substr(1, line.size() - 2);
accesskey.clear();
secret.clear();
session_token.clear();
}
size_t pos = line.find_first_of('=');
if(pos == std::string::npos){
continue;
}
std::string key = trim(line.substr(0, pos));
std::string value = trim(line.substr(pos + 1, std::string::npos));
if(key == "aws_access_key_id"){
accesskey = value;
}else if(key == "aws_secret_access_key"){
secret = value;
}else if(key == "aws_session_token"){
session_token = value;
}
}
if(profile != aws_profile){
return false;
}
if(session_token.empty()){
if(is_use_session_token){
S3FS_PRN_EXIT("AWS session token was expected but wasn't provided in aws/credentials file for profile: %s.", aws_profile.c_str());
return false;
}
if(!SetAccessKey(accesskey.c_str(), secret.c_str())){
S3FS_PRN_EXIT("failed to set internal data for access key/secret key from aws credential file.");
return false;
}
}else{
if(!SetAccessKeyWithSessionToken(accesskey.c_str(), secret.c_str(), session_token.c_str())){
S3FS_PRN_EXIT("session token is invalid.");
return false;
}
}
return true;
}
//
// InitialS3fsCredentials
//
// called only when were are not mounting a
// public bucket
//
// Here is the order precedence for getting the
// keys:
//
// 1 - from the command line (security risk)
// 2 - from a password file specified on the command line
// 3 - from environment variables
// 3a - from the AWS_CREDENTIAL_FILE environment variable
// 3b - from ${HOME}/.aws/credentials
// 4 - from the users ~/.passwd-s3fs
// 5 - from /etc/passwd-s3fs
//
bool S3fsCred::InitialS3fsCredentials()
{
// should be redundant
if(S3fsCurl::IsPublicBucket()){
return true;
}
// access key loading is deferred
if(load_iamrole || is_ecs){
return true;
}
// 1 - keys specified on the command line
if(IsSetAccessKeys()){
return true;
}
// 2 - was specified on the command line
if(!IsSetPasswdFile()){
if(!ReadS3fsPasswdFile()){
return false;;
}
return true;
}
// 3 - environment variables
char* AWSACCESSKEYID = getenv("AWS_ACCESS_KEY_ID") ? getenv("AWS_ACCESS_KEY_ID") : getenv("AWSACCESSKEYID");
char* AWSSECRETACCESSKEY = getenv("AWS_SECRET_ACCESS_KEY") ? getenv("AWS_SECRET_ACCESS_KEY") : getenv("AWSSECRETACCESSKEY");
char* AWSSESSIONTOKEN = getenv("AWS_SESSION_TOKEN") ? getenv("AWS_SESSION_TOKEN") : getenv("AWSSESSIONTOKEN");
if(AWSACCESSKEYID != NULL || AWSSECRETACCESSKEY != NULL){
if( (AWSACCESSKEYID == NULL && AWSSECRETACCESSKEY != NULL) ||
(AWSACCESSKEYID != NULL && AWSSECRETACCESSKEY == NULL) ){
S3FS_PRN_EXIT("both environment variables AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY must be set together.");
return false;;
}
S3FS_PRN_INFO2("access key from env variables");
if(AWSSESSIONTOKEN != NULL){
S3FS_PRN_INFO2("session token is available");
if(!SetAccessKeyWithSessionToken(AWSACCESSKEYID, AWSSECRETACCESSKEY, AWSSESSIONTOKEN)){
S3FS_PRN_EXIT("session token is invalid.");
return false;;
}
}else{
S3FS_PRN_INFO2("session token is not available");
if(is_use_session_token){
S3FS_PRN_EXIT("environment variable AWS_SESSION_TOKEN is expected to be set.");
return false;;
}
}
if(!SetAccessKey(AWSACCESSKEYID, AWSSECRETACCESSKEY)){
S3FS_PRN_EXIT("if one access key is specified, both keys need to be specified.");
return false;;
}
return true;
}
// 3a - from the AWS_CREDENTIAL_FILE environment variable
char* AWS_CREDENTIAL_FILE = getenv("AWS_CREDENTIAL_FILE");
if(AWS_CREDENTIAL_FILE != NULL){
passwd_file = AWS_CREDENTIAL_FILE;
if(!passwd_file.empty()){
if(!ReadS3fsPasswdFile()){
return false;;
}
return true;
}
}
// 3b - check ${HOME}/.aws/credentials
std::string aws_credentials = std::string(getpwuid(getuid())->pw_dir) + "/.aws/credentials";
if(ReadAwsCredentialFile(aws_credentials)){
return true;
}else if(aws_profile != DEFAULT_AWS_PROFILE_NAME){
S3FS_PRN_EXIT("Could not find profile: %s in file: %s", aws_profile.c_str(), aws_credentials.c_str());
return false;;
}
// 4 - from the default location in the users home directory
char* HOME = getenv("HOME");
if(HOME != NULL){
passwd_file = HOME;
passwd_file += "/.passwd-s3fs";
if(IsReadableS3fsPasswdFile()){
if(!ReadS3fsPasswdFile()){
return false;;
}
// It is possible that the user's file was there but
// contained no key pairs i.e. commented out
// in that case, go look in the final location
if(IsSetAccessKeys()){
return true;
}
}
}
// 5 - from the system default location
passwd_file = "/etc/passwd-s3fs";
if(IsReadableS3fsPasswdFile()){
if(!ReadS3fsPasswdFile()){
return false;;
}
return true;
}
S3FS_PRN_EXIT("could not determine how to establish security credentials.");
return false;;
}
//-------------------------------------------------------------------
// Methods : for IAM
//-------------------------------------------------------------------
bool S3fsCred::ParseIAMCredentialResponse(const char* response, iamcredmap_t& keyval)
{
if(!response){
return false;
}
std::istringstream sscred(response);
std::string oneline;
keyval.clear();
while(getline(sscred, oneline, ',')){
std::string::size_type pos;
std::string key;
std::string val;
if(std::string::npos != (pos = oneline.find(S3fsCred::IAMCRED_ACCESSKEYID))){
key = S3fsCred::IAMCRED_ACCESSKEYID;
}else if(std::string::npos != (pos = oneline.find(S3fsCred::IAMCRED_SECRETACCESSKEY))){
key = S3fsCred::IAMCRED_SECRETACCESSKEY;
}else if(std::string::npos != (pos = oneline.find(IAM_token_field))){
key = IAM_token_field;
}else if(std::string::npos != (pos = oneline.find(IAM_expiry_field))){
key = IAM_expiry_field;
}else if(std::string::npos != (pos = oneline.find(S3fsCred::IAMCRED_ROLEARN))){
key = S3fsCred::IAMCRED_ROLEARN;
}else{
continue;
}
if(std::string::npos == (pos = oneline.find(':', pos + key.length()))){
continue;
}
if(is_ibm_iam_auth && key == IAM_expiry_field){
// parse integer value
if(std::string::npos == (pos = oneline.find_first_of("0123456789", pos))){
continue;
}
oneline.erase(0, pos);
if(std::string::npos == (pos = oneline.find_last_of("0123456789"))){
continue;
}
val = oneline.substr(0, pos+1);
}else{
// parse std::string value (starts and ends with quotes)
if(std::string::npos == (pos = oneline.find('\"', pos))){
continue;
}
oneline.erase(0, pos+1);
if(std::string::npos == (pos = oneline.find('\"'))){
continue;
}
val = oneline.substr(0, pos);
}
keyval[key] = val;
}
return true;
}
bool S3fsCred::CheckIAMCredentialUpdate()
{
if(IAM_role.empty() && !is_ecs && !is_ibm_iam_auth){
return true;
}
if(time(NULL) + S3fsCred::IAM_EXPIRE_MERGIN <= AWSAccessTokenExpire){
return true;
}
S3FS_PRN_INFO("IAM Access Token refreshing...");
// update
S3fsCurl s3fscurl;
if(0 != s3fscurl.GetIAMCredentials()){
S3FS_PRN_ERR("IAM Access Token refresh failed");
return false;
}
S3FS_PRN_INFO("IAM Access Token refreshed");
return true;
}
//-------------------------------------------------------------------
// Methods : Checking forbidden parameters
//-------------------------------------------------------------------
//
// Checking forbidden parameters for bucket
//
bool S3fsCred::CheckForbiddenBucketParams()
{
// The first plain argument is the bucket
if(bucket_name.empty()){
S3FS_PRN_EXIT("missing BUCKET argument.");
return false;
}
// bucket names cannot contain upper case characters in virtual-hosted style
if(!pathrequeststyle && (lower(bucket_name) != bucket_name)){
S3FS_PRN_EXIT("BUCKET %s, name not compatible with virtual-hosted style.", bucket_name.c_str());
return false;
}
// check bucket name for illegal characters
size_t found = bucket_name.find_first_of("/:\\;!@#$%^&*?|+=");
if(found != std::string::npos){
S3FS_PRN_EXIT("BUCKET %s -- bucket name contains an illegal character.", bucket_name.c_str());
return false;
}
if(!pathrequeststyle && is_prefix(s3host.c_str(), "https://") && bucket_name.find_first_of('.') != std::string::npos) {
S3FS_PRN_EXIT("BUCKET %s -- cannot mount bucket with . while using HTTPS without use_path_request_style", bucket_name.c_str());
return false;
}
return true;
}
/*
* Local variables:
* tab-width: 4
* c-basic-offset: 4
* End:
* vim600: expandtab sw=4 ts=4 fdm=marker
* vim<600: expandtab sw=4 ts=4
*/

154
src/s3fs_cred.h Normal file
View File

@ -0,0 +1,154 @@
/*
* s3fs - FUSE-based file system backed by Amazon S3
*
* Copyright(C) 2007 Randy Rizun <rrizun@gmail.com>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
#ifndef S3FS_CRED_H_
#define S3FS_CRED_H_
//----------------------------------------------
// Typedefs
//----------------------------------------------
typedef std::map<std::string, std::string> iamcredmap_t;
//------------------------------------------------
// class S3fsCred
//------------------------------------------------
// This is a class for operating and managing Credentials(accesskey,
// secret key, tokens, etc.) used by S3fs.
// Operations related to Credentials are aggregated in this class.
//
// cppcheck-suppress ctuOneDefinitionRuleViolation ; for stub in test_curl_util.cpp
class S3fsCred
{
private:
static const char* ALLBUCKET_FIELDS_TYPE; // special key for mapping(This name is absolutely not used as a bucket name)
static const char* KEYVAL_FIELDS_TYPE; // special key for mapping(This name is absolutely not used as a bucket name)
static const char* AWS_ACCESSKEYID;
static const char* AWS_SECRETKEY;
static const int IAM_EXPIRE_MERGIN;
static const char* IAMCRED_ACCESSKEYID;
static const char* IAMCRED_SECRETACCESSKEY;
static const char* IAMCRED_ROLEARN;
static std::string bucket_name;
std::string passwd_file;
std::string aws_profile;
bool load_iamrole;
std::string AWSAccessKeyId;
std::string AWSSecretAccessKey;
std::string AWSAccessToken;
time_t AWSAccessTokenExpire;
bool is_ecs;
bool is_use_session_token;
bool is_ibm_iam_auth;
std::string IAM_cred_url;
int IAM_api_version;
std::string IAMv2_api_token;
size_t IAM_field_count;
std::string IAM_token_field;
std::string IAM_expiry_field;
std::string IAM_role;
public:
static const char* ECS_IAM_ENV_VAR;
static const char* IAMv2_token_url;
static int IAMv2_token_ttl;
static const char* IAMv2_token_ttl_hdr;
static const char* IAMv2_token_hdr;
private:
static bool ParseIAMRoleFromMetaDataResponse(const char* response, std::string& rolename);
bool IsReadableS3fsPasswdFile();
bool CheckS3fsPasswdFilePerms();
bool ParseS3fsPasswdFile(bucketkvmap_t& resmap);
bool ReadS3fsPasswdFile();
int CheckS3fsCredentialAwsFormat(const kvmap_t& kvmap);
bool ReadAwsCredentialFile(const std::string &filename);
bool ParseIAMCredentialResponse(const char* response, iamcredmap_t& keyval);
public:
static bool SetBucket(const char* bucket);
static const std::string& GetBucket();
S3fsCred();
~S3fsCred();
bool SetS3fsPasswdFile(const char* file);
bool IsSetPasswdFile();
bool SetAwsProfileName(const char* profile_name);
bool SetIAMRoleMetadataType(bool flag);
bool IsIAMRoleMetadataType() const { return load_iamrole; }
bool SetAccessKey(const char* AccessKeyId, const char* SecretAccessKey);
bool SetAccessKeyWithSessionToken(const char* AccessKeyId, const char* SecretAccessKey, const char * SessionToken);
bool IsSetAccessKeyID() const;
bool IsSetAccessKeys() const;
const std::string& GetAccessKeyID() const { return AWSAccessKeyId; }
const std::string& GetSecretAccessKey() const { return AWSSecretAccessKey; }
const std::string& GetAccessToken() const { return AWSAccessToken; }
bool SetIsECS(bool flag);
bool IsECS() const { return is_ecs; }
bool SetIsUseSessionToken(bool flag);
bool IsUseSessionToken() const { return is_use_session_token; }
bool SetIsIBMIAMAuth(bool flag);
bool IsIBMIAMAuth() const { return is_ibm_iam_auth; }
std::string SetIAMRole(const char* role);
const std::string& GetIAMRole() const { return IAM_role; }
size_t SetIAMFieldCount(size_t field_count);
std::string SetIAMCredentialsURL(const char* url);
const std::string& GetIAMCredentialsURL() const { return IAM_cred_url; }
std::string SetIAMTokenField(const char* token_field);
std::string SetIAMExpiryField(const char* expiry_field);
int SetIMDSVersion(int version);
int GetIMDSVersion() const { return IAM_api_version; }
bool SetIAMv2APIToken(const char* response);
const std::string& GetIAMv2APIToken() const { return IAMv2_api_token; }
bool SetIAMCredentials(const char* response);
bool SetIAMRoleFromMetaData(const char* response);
bool InitialS3fsCredentials();
bool CheckIAMCredentialUpdate();
bool CheckForbiddenBucketParams();
};
#endif // S3FS_CRED_H_
/*
* Local variables:
* tab-width: 4
* c-basic-offset: 4
* End:
* vim600: expandtab sw=4 ts=4 fdm=marker
* vim<600: expandtab sw=4 ts=4
*/

2
src/s3fs_global.cpp
View File

@ -32,11 +32,9 @@ bool noxmlns = false;
std::string program_name;
std::string service_path = "/";
std::string s3host = "https://s3.amazonaws.com";
std::string bucket;
std::string endpoint = "us-east-1";
std::string cipher_suites;
std::string instance_name;
std::string aws_profile = "default";
/*
* Local variables:

28
src/test_curl_util.cpp
View File

@ -24,6 +24,34 @@
#include "curl_util.h"
#include "test_util.h"
//---------------------------------------------------------
// S3fsCred Stub
//
// [NOTE]
// This test program links curl_util.cpp just to use the
// curl_slist_sort_insert function.
// This file has a call to S3fsCred::GetBucket(), which
// results in a link error. That method is not used in
// this test file, so define a stub class. Linking all
// implementation of the S3fsCred class or making all
// stubs is not practical, so this is the best answer.
//
class S3fsCred
{
private:
static std::string bucket_name;
public:
static const std::string& GetBucket();
};
std::string S3fsCred::bucket_name;
const std::string& S3fsCred::GetBucket()
{
return S3fsCred::bucket_name;
}
//---------------------------------------------------------
#define ASSERT_IS_SORTED(x) assert_is_sorted((x), __FILE__, __LINE__)
void assert_is_sorted(struct curl_slist* list, const char *file, int line)