mirror of
https://github.com/s3fs-fuse/s3fs-fuse.git
synced 2024-12-22 16:58:55 +00:00
Changes codes
1) Supported IAM role Supports IAM role by option, that is instead of AccessKeyID/ SecretAccessKey. Adds new option "iam_role" which is specified as IAM role name.(like s3fs-c) git-svn-id: http://s3fs.googlecode.com/svn/trunk@490 df820570-a93a-0410-bd06-b72b767a4274
This commit is contained in:
ggtakec@gmail.com
parent
44468ba00f
commit
99db6d13af
@ -145,6 +145,9 @@ Enable to send "Content-MD5" header when uploading a object without multipart po
|
||||
If this option is enabled, it has some influences on a performance of s3fs when uploading small object.
|
||||
Because s3fs always checks MD5 when uploading large object, this option does not affect on large object.
|
||||
.TP
|
||||
\fB\-o\fR iam_role ( default is no role )
|
||||
set the IAM Role that will supply the credentials from the instance meta-data.
|
||||
.TP
|
||||
\fB\-o\fR noxmlns - disable registing xml name space.
|
||||
disable registing xml name space for response of ListBucketResult and ListVersionsResult etc. Default name space is looked up from "http://s3.amazonaws.com/doc/2006-03-01".
|
||||
This option should not be specified now, because s3fs looks up xmlns automatically after v1.66.
|
||||
|
||||
162
src/curl.cpp
162
src/curl.cpp
@ -138,6 +138,14 @@ const char* BodyData::str(void) const
|
||||
#define MULTIPART_SIZE 10485760 // 10MB
|
||||
#define MAX_MULTI_COPY_SOURCE_SIZE 524288000 // 500MB
|
||||
|
||||
#define IAM_EXPIRE_MERGIN (20 * 60) // update timming
|
||||
#define IAM_CRED_URL "http://169.254.169.254/latest/meta-data/iam/security-credentials/"
|
||||
#define IAMCRED_ACCESSKEYID "AccessKeyId"
|
||||
#define IAMCRED_SECRETACCESSKEY "SecretAccessKey"
|
||||
#define IAMCRED_ACCESSTOKEN "Token"
|
||||
#define IAMCRED_EXPIRATION "Expiration"
|
||||
#define IAMCRED_KEYCOUNT 4
|
||||
|
||||
pthread_mutex_t S3fsCurl::curl_handles_lock;
|
||||
pthread_mutex_t S3fsCurl::curl_share_lock[SHARE_MUTEX_MAX];
|
||||
pthread_mutex_t* S3fsCurl::crypt_mutex = NULL;
|
||||
@ -156,6 +164,9 @@ bool S3fsCurl::is_content_md5 = false;
|
||||
bool S3fsCurl::is_verbose = false;
|
||||
string S3fsCurl::AWSAccessKeyId;
|
||||
string S3fsCurl::AWSSecretAccessKey;
|
||||
string S3fsCurl::AWSAccessToken;
|
||||
time_t S3fsCurl::AWSAccessTokenExpire= 0;
|
||||
string S3fsCurl::IAM_role;
|
||||
long S3fsCurl::ssl_verify_hostname = 1; // default(original code...)
|
||||
const EVP_MD* S3fsCurl::evp_md = EVP_sha1();
|
||||
curltime_t S3fsCurl::curl_times;
|
||||
@ -804,6 +815,13 @@ long S3fsCurl::SetSslVerifyHostname(long value)
|
||||
return old;
|
||||
}
|
||||
|
||||
string S3fsCurl::SetIAMRole(const char* role)
|
||||
{
|
||||
string old = S3fsCurl::IAM_role;
|
||||
S3fsCurl::IAM_role = role ? role : "";
|
||||
return old;
|
||||
}
|
||||
|
||||
int S3fsCurl::SetMaxParallelCount(int value)
|
||||
{
|
||||
int old = S3fsCurl::max_parallel_cnt;
|
||||
@ -1017,6 +1035,82 @@ int S3fsCurl::ParallelGetObjectRequest(const char* tpath, int fd, off_t start, s
|
||||
return result;
|
||||
}
|
||||
|
||||
bool S3fsCurl::ParseIAMCredentialResponse(const char* response, iamcredmap_t& keyval)
|
||||
{
|
||||
if(!response){
|
||||
return false;
|
||||
}
|
||||
istringstream sscred(response);
|
||||
string oneline;
|
||||
keyval.clear();
|
||||
while(getline(sscred, oneline, '\n')){
|
||||
string::size_type pos;
|
||||
string key;
|
||||
string val;
|
||||
if(string::npos != (pos = oneline.find(IAMCRED_ACCESSKEYID))){
|
||||
key = IAMCRED_ACCESSKEYID;
|
||||
}else if(string::npos != (pos = oneline.find(IAMCRED_SECRETACCESSKEY))){
|
||||
key = IAMCRED_SECRETACCESSKEY;
|
||||
}else if(string::npos != (pos = oneline.find(IAMCRED_ACCESSTOKEN))){
|
||||
key = IAMCRED_ACCESSTOKEN;
|
||||
}else if(string::npos != (pos = oneline.find(IAMCRED_EXPIRATION))){
|
||||
key = IAMCRED_EXPIRATION;
|
||||
}else{
|
||||
continue;
|
||||
}
|
||||
if(string::npos == (pos = oneline.find(':', pos + key.length()))){
|
||||
continue;
|
||||
}
|
||||
if(string::npos == (pos = oneline.find('\"', pos))){
|
||||
continue;
|
||||
}
|
||||
oneline = oneline.substr(pos + sizeof(char));
|
||||
if(string::npos == (pos = oneline.find('\"'))){
|
||||
continue;
|
||||
}
|
||||
val = oneline.substr(0, pos);
|
||||
keyval[key] = val;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool S3fsCurl::SetIAMCredentials(const char* response)
|
||||
{
|
||||
FPRNINFO("IAM credential response = \"%s\"", response);
|
||||
|
||||
iamcredmap_t keyval;
|
||||
|
||||
if(!ParseIAMCredentialResponse(response, keyval)){
|
||||
return false;
|
||||
}
|
||||
if(IAMCRED_KEYCOUNT != keyval.size()){
|
||||
return false;
|
||||
}
|
||||
|
||||
S3fsCurl::AWSAccessKeyId = keyval[string(IAMCRED_ACCESSKEYID)];
|
||||
S3fsCurl::AWSSecretAccessKey = keyval[string(IAMCRED_SECRETACCESSKEY)];
|
||||
S3fsCurl::AWSAccessToken = keyval[string(IAMCRED_ACCESSTOKEN)];
|
||||
S3fsCurl::AWSAccessTokenExpire = cvtIAMExpireStringToTime(keyval[string(IAMCRED_EXPIRATION)].c_str());
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool S3fsCurl::CheckIAMCredentialUpdate(void)
|
||||
{
|
||||
if(0 == S3fsCurl::IAM_role.size()){
|
||||
return true;
|
||||
}
|
||||
if(time(NULL) + IAM_EXPIRE_MERGIN <= S3fsCurl::AWSAccessTokenExpire){
|
||||
return true;
|
||||
}
|
||||
// update
|
||||
S3fsCurl s3fscurl;
|
||||
if(0 != s3fscurl.GetIAMCredentials()){
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
//-------------------------------------------------------------------
|
||||
// Methods for S3fsCurl
|
||||
//-------------------------------------------------------------------
|
||||
@ -1044,11 +1138,14 @@ bool S3fsCurl::ResetHandle(void)
|
||||
curl_easy_setopt(hCurl, CURLOPT_PROGRESSDATA, hCurl);
|
||||
// curl_easy_setopt(hCurl, CURLOPT_FORBID_REUSE, 1);
|
||||
|
||||
if(0 == S3fsCurl::ssl_verify_hostname){
|
||||
curl_easy_setopt(hCurl, CURLOPT_SSL_VERIFYHOST, 0);
|
||||
}
|
||||
if(S3fsCurl::curl_ca_bundle.size() != 0){
|
||||
curl_easy_setopt(hCurl, CURLOPT_CAINFO, S3fsCurl::curl_ca_bundle.c_str());
|
||||
if(type != REQTYPE_IAMCRED){
|
||||
// REQTYPE_IAMCRED is always HTTP
|
||||
if(0 == S3fsCurl::ssl_verify_hostname){
|
||||
curl_easy_setopt(hCurl, CURLOPT_SSL_VERIFYHOST, 0);
|
||||
}
|
||||
if(S3fsCurl::curl_ca_bundle.size() != 0){
|
||||
curl_easy_setopt(hCurl, CURLOPT_CAINFO, S3fsCurl::curl_ca_bundle.c_str());
|
||||
}
|
||||
}
|
||||
if((S3fsCurl::is_dns_cache || S3fsCurl::is_ssl_session_cache) && S3fsCurl::hCurlShare){
|
||||
curl_easy_setopt(hCurl, CURLOPT_SHARE, S3fsCurl::hCurlShare);
|
||||
@ -1320,6 +1417,12 @@ bool S3fsCurl::RemakeHandle(void)
|
||||
curl_easy_setopt(hCurl, CURLOPT_HTTPHEADER, requestHeaders);
|
||||
break;
|
||||
|
||||
case REQTYPE_IAMCRED:
|
||||
curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str());
|
||||
curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, (void*)bodydata);
|
||||
curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
|
||||
break;
|
||||
|
||||
default:
|
||||
DPRNNN("request type is unknown(%d)", type);
|
||||
return false;
|
||||
@ -1513,6 +1616,14 @@ string S3fsCurl::CalcSignature(string method, string strMD5, string content_type
|
||||
string Signature;
|
||||
string StringToSign;
|
||||
|
||||
if(0 < S3fsCurl::IAM_role.size()){
|
||||
if(!S3fsCurl::CheckIAMCredentialUpdate()){
|
||||
DPRN("Something error occurred in checking IAM credential.");
|
||||
return Signature; // returns empty string, then it occures error.
|
||||
}
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, string("x-amz-security-token:" + S3fsCurl::AWSAccessToken).c_str());
|
||||
}
|
||||
|
||||
StringToSign += method + "\n";
|
||||
StringToSign += strMD5 + "\n"; // md5
|
||||
StringToSign += content_type + "\n";
|
||||
@ -1683,6 +1794,47 @@ int S3fsCurl::DeleteRequest(const char* tpath)
|
||||
return RequestPerform();
|
||||
}
|
||||
|
||||
//
|
||||
// Get AccessKeyId/SecretAccessKey/AccessToken/Expiration by IAM role,
|
||||
// and Set these value to class valiable.
|
||||
//
|
||||
int S3fsCurl::GetIAMCredentials(void)
|
||||
{
|
||||
FPRNINFO("[IAM role=%s]", S3fsCurl::IAM_role.c_str());
|
||||
|
||||
if(0 == S3fsCurl::IAM_role.size()){
|
||||
DPRN("IAM role name is empty.");
|
||||
return -EIO;
|
||||
}
|
||||
// at first set type for handle
|
||||
type = REQTYPE_IAMCRED;
|
||||
|
||||
if(!CreateCurlHandle(true)){
|
||||
return -EIO;
|
||||
}
|
||||
|
||||
// url
|
||||
url = string(IAM_CRED_URL) + S3fsCurl::IAM_role;
|
||||
requestHeaders = NULL;
|
||||
responseHeaders.clear();
|
||||
bodydata = new BodyData();
|
||||
|
||||
curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str());
|
||||
curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, (void*)bodydata);
|
||||
curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
|
||||
|
||||
int result = RequestPerform();
|
||||
|
||||
// analizing response
|
||||
if(0 == result && !S3fsCurl::SetIAMCredentials(bodydata->str())){
|
||||
DPRN("Something error occured, could not get IAM credential.");
|
||||
}
|
||||
delete bodydata;
|
||||
bodydata = NULL;
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
//
|
||||
// tpath : target path for head request
|
||||
// bpath : saved into base_path
|
||||
|
||||
19
src/curl.h
19
src/curl.h
@ -99,6 +99,8 @@ class S3fsMultiCurl;
|
||||
//----------------------------------------------
|
||||
// class S3fsCurl
|
||||
//----------------------------------------------
|
||||
typedef std::map<std::string, std::string> iamcredmap_t;
|
||||
|
||||
// share
|
||||
#define SHARE_MUTEX_DNS 0
|
||||
#define SHARE_MUTEX_SSL_SESSION 1
|
||||
@ -130,7 +132,8 @@ class S3fsCurl
|
||||
REQTYPE_COMPLETEMULTIPOST,
|
||||
REQTYPE_UPLOADMULTIPOST,
|
||||
REQTYPE_COPYMULTIPOST,
|
||||
REQTYPE_MULTILIST
|
||||
REQTYPE_MULTILIST,
|
||||
REQTYPE_IAMCRED
|
||||
};
|
||||
|
||||
// class variables
|
||||
@ -152,6 +155,9 @@ class S3fsCurl
|
||||
static bool is_verbose;
|
||||
static std::string AWSAccessKeyId;
|
||||
static std::string AWSSecretAccessKey;
|
||||
static std::string AWSAccessToken;
|
||||
static time_t AWSAccessTokenExpire;
|
||||
static std::string IAM_role;
|
||||
static long ssl_verify_hostname;
|
||||
static const EVP_MD* evp_md;
|
||||
static curltime_t curl_times;
|
||||
@ -216,12 +222,16 @@ class S3fsCurl
|
||||
static S3fsCurl* UploadMultipartPostRetryCallback(S3fsCurl* s3fscurl);
|
||||
static S3fsCurl* ParallelGetObjectRetryCallback(S3fsCurl* s3fscurl);
|
||||
|
||||
static bool ParseIAMCredentialResponse(const char* response, iamcredmap_t& keyval);
|
||||
static bool SetIAMCredentials(const char* response);
|
||||
|
||||
// methods
|
||||
bool ResetHandle(void);
|
||||
bool RemakeHandle(void);
|
||||
bool ClearInternalData(void);
|
||||
std::string CalcSignature(std::string method, std::string strMD5, std::string content_type, std::string date, std::string resource);
|
||||
bool GetUploadId(std::string& upload_id);
|
||||
int GetIAMCredentials(void);
|
||||
|
||||
int PreMultipartPostRequest(const char* tpath, headers_t& meta, std::string& upload_id, bool ow_sse_flg);
|
||||
int CompleteMultipartPostRequest(const char* tpath, std::string& upload_id, etaglist_t& parts);
|
||||
@ -235,6 +245,7 @@ class S3fsCurl
|
||||
static bool DestroyS3fsCurl(void);
|
||||
static int ParallelMultipartUploadRequest(const char* tpath, headers_t& meta, int fd, bool ow_sse_flg);
|
||||
static int ParallelGetObjectRequest(const char* tpath, int fd, off_t start, ssize_t size);
|
||||
static bool CheckIAMCredentialUpdate(void);
|
||||
|
||||
// class methods(valiables)
|
||||
static std::string LookupMimeType(std::string name);
|
||||
@ -255,10 +266,14 @@ class S3fsCurl
|
||||
static bool SetVerbose(bool flag);
|
||||
static bool GetVerbose(void) { return S3fsCurl::is_verbose; }
|
||||
static bool SetAccessKey(const char* AccessKeyId, const char* SecretAccessKey);
|
||||
static bool IsSetAccessKeyId(void) { return (0 < S3fsCurl::AWSAccessKeyId.size() && 0 < S3fsCurl::AWSSecretAccessKey.size()); }
|
||||
static bool IsSetAccessKeyId(void){
|
||||
return (0 < S3fsCurl::IAM_role.size() || (0 < S3fsCurl::AWSAccessKeyId.size() && 0 < S3fsCurl::AWSSecretAccessKey.size()));
|
||||
}
|
||||
static long SetSslVerifyHostname(long value);
|
||||
static long GetSslVerifyHostname(void) { return S3fsCurl::ssl_verify_hostname; }
|
||||
static int SetMaxParallelCount(int value);
|
||||
static std::string SetIAMRole(const char* role);
|
||||
static const char* GetIAMRole(void) { return S3fsCurl::IAM_role.c_str(); }
|
||||
|
||||
// methods
|
||||
bool CreateCurlHandle(bool force = false);
|
||||
|
||||
11
src/s3fs.cpp
11
src/s3fs.cpp
@ -2664,6 +2664,12 @@ static int s3fs_check_service(void)
|
||||
{
|
||||
FPRN("check services.");
|
||||
|
||||
// At first time for access S3, we check IAM role if it sets.
|
||||
if(!S3fsCurl::CheckIAMCredentialUpdate()){
|
||||
fprintf(stderr, "%s: Failed to check IAM role name(%s).\n", program_name.c_str(), S3fsCurl::GetIAMRole());
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
|
||||
S3fsCurl s3fscurl;
|
||||
if(0 != s3fscurl.CheckBucket()){
|
||||
fprintf(stderr, "%s: Failed to access bucket.\n", program_name.c_str());
|
||||
@ -3256,6 +3262,11 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
|
||||
passwd_file = strchr(arg, '=') + sizeof(char);
|
||||
return 0;
|
||||
}
|
||||
if(0 == STR2NCMP(arg, "iam_role=")){
|
||||
const char* role = strchr(arg, '=') + sizeof(char);
|
||||
S3fsCurl::SetIAMRole(role);
|
||||
return 0;
|
||||
}
|
||||
if(0 == STR2NCMP(arg, "public_bucket=")){
|
||||
long pubbucket = strtol(strchr(arg, '=') + sizeof(char), 0, 10);
|
||||
if(1 == pubbucket){
|
||||
|
||||
@ -743,6 +743,17 @@ blkcnt_t get_blocks(off_t size)
|
||||
return size / 512 + 1;
|
||||
}
|
||||
|
||||
time_t cvtIAMExpireStringToTime(const char* s)
|
||||
{
|
||||
struct tm tm;
|
||||
if(!s){
|
||||
return 0L;
|
||||
}
|
||||
memset(&tm, 0, sizeof(struct tm));
|
||||
strptime(s, "%Y-%m-%dT%H:%M:%S", &tm);
|
||||
return mktime(&tm); // GMT
|
||||
}
|
||||
|
||||
time_t get_lastmodified(const char* s)
|
||||
{
|
||||
struct tm tm;
|
||||
@ -927,6 +938,10 @@ void show_help (void)
|
||||
" enable_content_md5 (default is disable)\n"
|
||||
" - verifying uploaded object without multipart by content-md5 header.\n"
|
||||
"\n"
|
||||
" iam_role (default is no role)\n"
|
||||
" - set the IAM Role that will supply the credentials from the \n"
|
||||
" instance meta-data.\n"
|
||||
"\n"
|
||||
" noxmlns (disable registing xml name space)\n"
|
||||
" disable registing xml name space for response of \n"
|
||||
" ListBucketResult and ListVersionsResult etc. Default name \n"
|
||||
|
||||
@ -106,6 +106,7 @@ uid_t get_uid(headers_t& meta);
|
||||
gid_t get_gid(const char *s);
|
||||
gid_t get_gid(headers_t& meta);
|
||||
blkcnt_t get_blocks(off_t size);
|
||||
time_t cvtIAMExpireStringToTime(const char* s);
|
||||
time_t get_lastmodified(const char* s);
|
||||
time_t get_lastmodified(headers_t& meta);
|
||||
bool is_need_check_obj_detail(headers_t& meta);
|
||||
|
||||
Loading…
Reference in New Issue
Block a user