octoleo/s3fs-fuse
1
0
Fork 0
mirror of https://github.com/s3fs-fuse/s3fs-fuse.git synced 2024-11-13 07:56:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

Implement AWS IMDSv2 support

Browse Source 
AWS IMDSv2 is a session oriented method for retrieving instance metadata,
including IAM credentials, in Amazon EC2.  It is enabled by default in
non-enforcing mode in AWS (meaning it retains backwards compatibility with
existing IMDSv1 clients), but can be switched to enforcing mode, in which
clients are required to return API tokens with requests.

With this change, we implement support for IMDSv2 and enable it by default when
IAM roles are our source for authentication credentials.  In the event that
s3fs is running in cloud environment offering an IMDSv1-compatible API, we
support graceful fallback to that mode.  It can also be selected explicitly via
the imdsv1only mount option.

More details on IMDSv2 are available at
https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instancedata-data-retrieval.html
and
https://aws.amazon.com/blogs/security/defense-in-depth-open-firewalls-reverse-proxies-ssrf-vulnerabilities-ec2-instance-metadata-service/

Signed-off-by: Noah Meyerhans <nmeyerha@amazon.com>
This commit is contained in:
Noah Meyerhans 2020-10-15 10:18:19 -07:00 committed by Andrew Gaul
parent 81ad3ce0ae
commit f2f930300a
4 changed files with 98 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

76
src/curl.cpp
View File

@ -112,10 +112,16 @@ time_t S3fsCurl::AWSAccessTokenExpire= 0;
bool S3fsCurl::is_ecs = false;
bool S3fsCurl::is_ibm_iam_auth = false;
std::string S3fsCurl::IAM_cred_url = "http://169.254.169.254/latest/meta-data/iam/security-credentials/";
std::string S3fsCurl::IAMv2_token_url = "http://169.254.169.254/latest/api/token";
std::string S3fsCurl::IAMv2_token_ttl_hdr = "X-aws-ec2-metadata-token-ttl-seconds";
std::string S3fsCurl::IAMv2_token_hdr = "X-aws-ec2-metadata-token";
int S3fsCurl::IAMv2_token_ttl = 21600;
size_t S3fsCurl::IAM_field_count = 4;
std::string S3fsCurl::IAM_token_field = "Token";
std::string S3fsCurl::IAM_expiry_field = "Expiration";
std::string S3fsCurl::IAM_role;
std::string S3fsCurl::IAMv2_api_token;
int S3fsCurl::IAM_api_version = 2;
long S3fsCurl::ssl_verify_hostname = 1; // default(original code...)
// protected by curl_warnings_lock
@ -1085,6 +1091,12 @@ std::string S3fsCurl::SetIAMExpiryField(const char* expiry_field)
return old;
}
bool S3fsCurl::SetIMDSVersion(int version)
{
S3fsCurl::IAM_api_version = version;
return true;
}
bool S3fsCurl::SetMultipartSize(off_t size)
{
size = size * 1024 * 1024;
@ -1656,6 +1668,13 @@ bool S3fsCurl::ParseIAMCredentialResponse(const char* response, iamcredmap_t& ke
return true;
}
bool S3fsCurl::SetIAMv2APIToken(const char* response)
{
S3FS_PRN_INFO3("Setting AWS IMDSv2 API token to %s", response);
S3fsCurl::IAMv2_api_token = std::string(response);
return true;
}
bool S3fsCurl::SetIAMCredentials(const char* response)
{
S3FS_PRN_INFO3("IAM credential response = \"%s\"", response);
@ -2670,6 +2689,42 @@ int S3fsCurl::DeleteRequest(const char* tpath)
return RequestPerform();
}
//
// Get the token that we need to pass along with AWS IMDSv2 API requests
//
int S3fsCurl::GetIAMv2ApiToken()
{
url = std::string(S3fsCurl::IAMv2_token_url);
if(!CreateCurlHandle()){
return -EIO;
}
requestHeaders = NULL;
responseHeaders.clear();
bodydata.Clear();
// maximum allowed value is 21600, so 6 bytes for the C string
char ttlstr[6];
snprintf(ttlstr, sizeof(ttlstr), "%d", S3fsCurl::IAMv2_token_ttl);
requestHeaders = curl_slist_sort_insert(requestHeaders, S3fsCurl::IAMv2_token_ttl_hdr.c_str(),
ttlstr);
curl_easy_setopt(hCurl, CURLOPT_PUT, true);
curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str());
curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, (void*)&bodydata);
curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
S3fsCurl::AddUserAgent(hCurl);
int result = RequestPerform(true);
if(0 == result && !S3fsCurl::SetIAMv2APIToken(bodydata.str())){
S3FS_PRN_ERR("Error storing IMDSv2 API token.");
result = -EIO;
}
bodydata.Clear();
curl_easy_setopt(hCurl, CURLOPT_PUT, false);
return result;
}
//
// Get AccessKeyId/SecretAccessKey/AccessToken/Expiration by IAM role,
// and Set these value to class variable.
@ -2701,6 +2756,23 @@ int S3fsCurl::GetIAMCredentials()
}
url = std::string(S3fsCurl::IAM_cred_url) + env;
}else{
if(S3fsCurl::IAM_api_version > 1){
int result = GetIAMv2ApiToken();
if(-ENOENT == result){
// If we get a 404 back when requesting the token service,
// then it's highly likely we're running in an environment
// that doesn't support the AWS IMDSv2 API, so we'll skip
// the token retrieval in the future.
SetIMDSVersion(1);
}else if(result != 0){
// If we get an unexpected error when retrieving the API
// token, log it but continue. Requirement for including
// an API token with the metadata request may or may not
// be required, so we should not abort here.
S3FS_PRN_ERR("AWS IMDSv2 token retrieval failed: %d", result);
}
}
url = std::string(S3fsCurl::IAM_cred_url) + S3fsCurl::IAM_role;
}
@ -2731,6 +2803,10 @@ int S3fsCurl::GetIAMCredentials()
curl_easy_setopt(hCurl, CURLOPT_READFUNCTION, S3fsCurl::ReadCallback);
}
if(S3fsCurl::IAM_api_version > 1){
requestHeaders = curl_slist_sort_insert(requestHeaders, S3fsCurl::IAMv2_token_hdr.c_str(), S3fsCurl::IAMv2_api_token.c_str());
}
curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str());
curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, (void*)&bodydata);
curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);

9
src/curl.h
View File

@ -142,6 +142,12 @@ class S3fsCurl
static bool is_use_session_token;
static bool is_ibm_iam_auth;
static std::string IAM_cred_url;
static int IAM_api_version;
static std::string IAMv2_token_url;
static int IAMv2_token_ttl;
static std::string IAMv2_token_ttl_hdr;
static std::string IAMv2_token_hdr;
static std::string IAMv2_api_token;
static size_t IAM_field_count;
static std::string IAM_token_field;
static std::string IAM_expiry_field;
@ -238,6 +244,7 @@ class S3fsCurl
static bool ParseIAMCredentialResponse(const char* response, iamcredmap_t& keyval);
static bool SetIAMCredentials(const char* response);
static bool SetIAMv2APIToken(const char* response);
static bool ParseIAMRoleFromMetaDataResponse(const char* response, std::string& rolename);
static bool SetIAMRoleFromMetaData(const char* response);
static bool LoadEnvSseCKeys();
@ -260,6 +267,7 @@ class S3fsCurl
void insertAuthHeaders();
std::string CalcSignatureV2(const std::string& method, const std::string& strMD5, const std::string& content_type, const std::string& date, const std::string& resource);
std::string CalcSignature(const std::string& method, const std::string& canonical_uri, const std::string& query_string, const std::string& strdate, const std::string& payload_hash, const std::string& date8601);
int GetIAMv2ApiToken();
int GetIAMCredentials();
int UploadMultipartPostSetup(const char* tpath, int part_num, const std::string& upload_id);
@ -348,6 +356,7 @@ class S3fsCurl
static void InitUserAgent();
static bool SetRequesterPays(bool flag) { bool old_flag = S3fsCurl::requester_pays; S3fsCurl::requester_pays = flag; return old_flag; }
static bool IsRequesterPays() { return S3fsCurl::requester_pays; }
static bool SetIMDSVersion(int version);
// methods
bool CreateCurlHandle(bool only_pool = false, bool remake = false);

6
src/s3fs.cpp
View File

@ -4329,6 +4329,7 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
S3fsCurl::SetIAMTokenField("\"access_token\"");
S3fsCurl::SetIAMExpiryField("\"expiration\"");
S3fsCurl::SetIAMFieldCount(2);
S3fsCurl::SetIMDSVersion(1);
is_ibm_iam_auth = true;
return 0;
}
@ -4348,12 +4349,17 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
S3fsCurl::SetIAMCredentialsURL(endpoint_url.c_str());
return 0;
}
if(0 == strcmp(arg, "imdsv1only")){
S3fsCurl::SetIMDSVersion(1);
return 0;
}
if(0 == strcmp(arg, "ecs")){
if (is_ibm_iam_auth) {
S3FS_PRN_EXIT("option ecs cannot be used in conjunction with ibm");
return -1;
}
S3fsCurl::SetIsECS(true);
S3fsCurl::SetIMDSVersion(1);
S3fsCurl::SetIAMCredentialsURL("http://169.254.170.2");
S3fsCurl::SetIAMFieldCount(5);
is_ecs = true;

7
src/s3fs_help.cpp
View File

@ -313,6 +313,13 @@ static const char help_string[] =
" \"auto\", s3fs will automatically use the IAM role names that are set\n"
" to an instance. If you specify this option without any argument, it\n"
" is the same as that you have specified the \"auto\".\n"
"\n"
" imdsv1only (default is to use IMDSv2)\n"
" - AWS instance metadata service, used IAM role authentication\n"
" supports the use of an API token. If you're using an IAM role\n"
" in an environment that does not support IMDSv2, setting this flag\n"
" will skip retrieval and usage of the API token when retrieving\n"
" IAM credentials.\n"
"\n"
" ibm_iam_auth (default is not using IBM IAM authentication)\n"
" - This option instructs s3fs to use IBM IAM authentication.\n"