Changes codes

1) Supported IAM role
   Supports IAM role by option, that is instead of AccessKeyID/
   SecretAccessKey.
   Adds new option "iam_role" which is specified as IAM role
   name.(like s3fs-c)



git-svn-id: http://s3fs.googlecode.com/svn/trunk@490 df820570-a93a-0410-bd06-b72b767a4274

doc/man/s3fs.1

@@ -145,6 +145,9 @@ Enable to send "Content-MD5" header when uploading a object without multipart po
 If this option is enabled, it has some influences on a performance of s3fs when uploading small object.
 Because s3fs always checks MD5 when uploading large object, this option does not affect on large object.
 .TP
+\fB\-o\fR iam_role ( default is no role )
+set the IAM Role that will supply the credentials from the instance meta-data.
+.TP
 \fB\-o\fR noxmlns - disable registing xml name space.
 disable registing xml name space for response of ListBucketResult and ListVersionsResult etc. Default name space is looked up from "http://s3.amazonaws.com/doc/2006-03-01".
 This option should not be specified now, because s3fs looks up xmlns automatically after v1.66.

src/curl.cpp

@@ -138,6 +138,14 @@ const char* BodyData::str(void) const
 #define MULTIPART_SIZE              10485760          // 10MB
 #define MAX_MULTI_COPY_SOURCE_SIZE  524288000         // 500MB
 
+#define	IAM_EXPIRE_MERGIN           (20 * 60)         // update timming
+#define	IAM_CRED_URL                "http://169.254.169.254/latest/meta-data/iam/security-credentials/"
+#define IAMCRED_ACCESSKEYID         "AccessKeyId"
+#define IAMCRED_SECRETACCESSKEY     "SecretAccessKey"
+#define IAMCRED_ACCESSTOKEN         "Token"
+#define IAMCRED_EXPIRATION          "Expiration"
+#define IAMCRED_KEYCOUNT            4
+
 pthread_mutex_t  S3fsCurl::curl_handles_lock;
 pthread_mutex_t  S3fsCurl::curl_share_lock[SHARE_MUTEX_MAX];
 pthread_mutex_t* S3fsCurl::crypt_mutex         = NULL;
@@ -156,6 +164,9 @@ bool             S3fsCurl::is_content_md5      = false;
 bool             S3fsCurl::is_verbose          = false;
 string           S3fsCurl::AWSAccessKeyId;
 string           S3fsCurl::AWSSecretAccessKey;
+string           S3fsCurl::AWSAccessToken;
+time_t           S3fsCurl::AWSAccessTokenExpire= 0;
+string           S3fsCurl::IAM_role;
 long             S3fsCurl::ssl_verify_hostname = 1;    // default(original code...)
 const EVP_MD*    S3fsCurl::evp_md              = EVP_sha1();
 curltime_t       S3fsCurl::curl_times;
@@ -804,6 +815,13 @@ long S3fsCurl::SetSslVerifyHostname(long value)
   return old;
 }
 
+string S3fsCurl::SetIAMRole(const char* role)
+{
+  string old = S3fsCurl::IAM_role;
+  S3fsCurl::IAM_role = role ? role : "";
+  return old;
+}
+
 int S3fsCurl::SetMaxParallelCount(int value)
 {
   int old = S3fsCurl::max_parallel_cnt;
@@ -1017,6 +1035,82 @@ int S3fsCurl::ParallelGetObjectRequest(const char* tpath, int fd, off_t start, s
   return result;
 }
 
+bool S3fsCurl::ParseIAMCredentialResponse(const char* response, iamcredmap_t& keyval)
+{
+  if(!response){
+    return false;
+  }
+  istringstream sscred(response);
+  string        oneline;
+  keyval.clear();
+  while(getline(sscred, oneline, '\n')){
+    string::size_type pos;
+    string            key;
+    string            val;
+    if(string::npos != (pos = oneline.find(IAMCRED_ACCESSKEYID))){
+      key = IAMCRED_ACCESSKEYID;
+    }else if(string::npos != (pos = oneline.find(IAMCRED_SECRETACCESSKEY))){
+      key = IAMCRED_SECRETACCESSKEY;
+    }else if(string::npos != (pos = oneline.find(IAMCRED_ACCESSTOKEN))){
+      key = IAMCRED_ACCESSTOKEN;
+    }else if(string::npos != (pos = oneline.find(IAMCRED_EXPIRATION))){
+      key = IAMCRED_EXPIRATION;
+    }else{
+      continue;
+    }
+    if(string::npos == (pos = oneline.find(':', pos + key.length()))){
+      continue;
+    }
+    if(string::npos == (pos = oneline.find('\"', pos))){
+      continue;
+    }
+    oneline = oneline.substr(pos + sizeof(char));
+    if(string::npos == (pos = oneline.find('\"'))){
+      continue;
+    }
+    val = oneline.substr(0, pos);
+    keyval[key] = val;
+  }
+  return true;
+}
+
+bool S3fsCurl::SetIAMCredentials(const char* response)
+{
+  FPRNINFO("IAM credential response = \"%s\"", response);
+
+  iamcredmap_t keyval;
+
+  if(!ParseIAMCredentialResponse(response, keyval)){
+    return false;
+  }
+  if(IAMCRED_KEYCOUNT != keyval.size()){
+    return false;
+  }
+
+  S3fsCurl::AWSAccessKeyId       = keyval[string(IAMCRED_ACCESSKEYID)];
+  S3fsCurl::AWSSecretAccessKey   = keyval[string(IAMCRED_SECRETACCESSKEY)];
+  S3fsCurl::AWSAccessToken       = keyval[string(IAMCRED_ACCESSTOKEN)];
+  S3fsCurl::AWSAccessTokenExpire = cvtIAMExpireStringToTime(keyval[string(IAMCRED_EXPIRATION)].c_str());
+
+  return true;
+}
+
+bool S3fsCurl::CheckIAMCredentialUpdate(void)
+{
+  if(0 == S3fsCurl::IAM_role.size()){
+    return true;
+  }
+  if(time(NULL) + IAM_EXPIRE_MERGIN <= S3fsCurl::AWSAccessTokenExpire){
+    return true;
+  }
+  // update
+  S3fsCurl s3fscurl;
+  if(0 != s3fscurl.GetIAMCredentials()){
+    return false;
+  }
+  return true;
+}
+
 //-------------------------------------------------------------------
 // Methods for S3fsCurl
 //-------------------------------------------------------------------
@@ -1044,11 +1138,14 @@ bool S3fsCurl::ResetHandle(void)
   curl_easy_setopt(hCurl, CURLOPT_PROGRESSDATA, hCurl);
   // curl_easy_setopt(hCurl, CURLOPT_FORBID_REUSE, 1);
 
-  if(0 == S3fsCurl::ssl_verify_hostname){
+  if(type != REQTYPE_IAMCRED){
-    curl_easy_setopt(hCurl, CURLOPT_SSL_VERIFYHOST, 0);
+    // REQTYPE_IAMCRED is always HTTP
-  }
+    if(0 == S3fsCurl::ssl_verify_hostname){
-  if(S3fsCurl::curl_ca_bundle.size() != 0){
+      curl_easy_setopt(hCurl, CURLOPT_SSL_VERIFYHOST, 0);
-    curl_easy_setopt(hCurl, CURLOPT_CAINFO, S3fsCurl::curl_ca_bundle.c_str());
+    }
+    if(S3fsCurl::curl_ca_bundle.size() != 0){
+      curl_easy_setopt(hCurl, CURLOPT_CAINFO, S3fsCurl::curl_ca_bundle.c_str());
+    }
   }
   if((S3fsCurl::is_dns_cache || S3fsCurl::is_ssl_session_cache) && S3fsCurl::hCurlShare){
     curl_easy_setopt(hCurl, CURLOPT_SHARE, S3fsCurl::hCurlShare);
@@ -1320,6 +1417,12 @@ bool S3fsCurl::RemakeHandle(void)
       curl_easy_setopt(hCurl, CURLOPT_HTTPHEADER, requestHeaders);
       break;
 
+    case REQTYPE_IAMCRED:
+      curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str());
+      curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, (void*)bodydata);
+      curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
+      break;
+
     default:
       DPRNNN("request type is unknown(%d)", type);
       return false;
@@ -1513,6 +1616,14 @@ string S3fsCurl::CalcSignature(string method, string strMD5, string content_type
   string Signature;
   string StringToSign;
 
+  if(0 < S3fsCurl::IAM_role.size()){
+    if(!S3fsCurl::CheckIAMCredentialUpdate()){
+      DPRN("Something error occurred in checking IAM credential.");  
+      return Signature;  // returns empty string, then it occures error.
+    }
+    requestHeaders = curl_slist_sort_insert(requestHeaders, string("x-amz-security-token:" + S3fsCurl::AWSAccessToken).c_str());
+  }
+
   StringToSign += method + "\n";
   StringToSign += strMD5 + "\n";        // md5
   StringToSign += content_type + "\n";
@@ -1683,6 +1794,47 @@ int S3fsCurl::DeleteRequest(const char* tpath)
   return RequestPerform();
 }
 
+//
+// Get AccessKeyId/SecretAccessKey/AccessToken/Expiration by IAM role,
+// and Set these value to class valiable.
+//
+int S3fsCurl::GetIAMCredentials(void)
+{
+  FPRNINFO("[IAM role=%s]", S3fsCurl::IAM_role.c_str());
+
+  if(0 == S3fsCurl::IAM_role.size()){
+    DPRN("IAM role name is empty.");
+    return -EIO;
+  }
+  // at first set type for handle
+  type = REQTYPE_IAMCRED;
+
+  if(!CreateCurlHandle(true)){
+    return -EIO;
+  }
+
+  // url
+  url             = string(IAM_CRED_URL) + S3fsCurl::IAM_role;
+  requestHeaders  = NULL;
+  responseHeaders.clear();
+  bodydata        = new BodyData();
+
+  curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str());
+  curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, (void*)bodydata);
+  curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
+
+  int result = RequestPerform();
+
+  // analizing response
+  if(0 == result && !S3fsCurl::SetIAMCredentials(bodydata->str())){
+    DPRN("Something error occured, could not get IAM credential.");
+  }
+  delete bodydata;
+  bodydata = NULL;
+
+  return result;
+}
+
 //
 // tpath :      target path for head request
 // bpath :      saved into base_path

src/curl.h

@@ -99,6 +99,8 @@ class S3fsMultiCurl;
 //----------------------------------------------
 // class S3fsCurl
 //----------------------------------------------
+typedef std::map<std::string, std::string> iamcredmap_t;
+
 // share
 #define	SHARE_MUTEX_DNS         0
 #define	SHARE_MUTEX_SSL_SESSION 1
@@ -130,7 +132,8 @@ class S3fsCurl
       REQTYPE_COMPLETEMULTIPOST,
       REQTYPE_UPLOADMULTIPOST,
       REQTYPE_COPYMULTIPOST,
-      REQTYPE_MULTILIST
+      REQTYPE_MULTILIST,
+      REQTYPE_IAMCRED
     };
 
     // class variables
@@ -152,6 +155,9 @@ class S3fsCurl
     static bool             is_verbose;
     static std::string      AWSAccessKeyId;
     static std::string      AWSSecretAccessKey;
+    static std::string      AWSAccessToken;
+    static time_t           AWSAccessTokenExpire;
+    static std::string      IAM_role;
     static long             ssl_verify_hostname;
     static const EVP_MD*    evp_md;
     static curltime_t       curl_times;
@@ -216,12 +222,16 @@ class S3fsCurl
     static S3fsCurl* UploadMultipartPostRetryCallback(S3fsCurl* s3fscurl);
     static S3fsCurl* ParallelGetObjectRetryCallback(S3fsCurl* s3fscurl);
 
+    static bool ParseIAMCredentialResponse(const char* response, iamcredmap_t& keyval);
+    static bool SetIAMCredentials(const char* response);
+
     // methods
     bool ResetHandle(void);
     bool RemakeHandle(void);
     bool ClearInternalData(void);
     std::string CalcSignature(std::string method, std::string strMD5, std::string content_type, std::string date, std::string resource);
     bool GetUploadId(std::string& upload_id);
+    int GetIAMCredentials(void);
 
     int PreMultipartPostRequest(const char* tpath, headers_t& meta, std::string& upload_id, bool ow_sse_flg);
     int CompleteMultipartPostRequest(const char* tpath, std::string& upload_id, etaglist_t& parts);
@@ -235,6 +245,7 @@ class S3fsCurl
     static bool DestroyS3fsCurl(void);
     static int ParallelMultipartUploadRequest(const char* tpath, headers_t& meta, int fd, bool ow_sse_flg);
     static int ParallelGetObjectRequest(const char* tpath, int fd, off_t start, ssize_t size);
+    static bool CheckIAMCredentialUpdate(void);
 
     // class methods(valiables)
     static std::string LookupMimeType(std::string name);
@@ -255,10 +266,14 @@ class S3fsCurl
     static bool SetVerbose(bool flag);
     static bool GetVerbose(void) { return S3fsCurl::is_verbose; }
     static bool SetAccessKey(const char* AccessKeyId, const char* SecretAccessKey);
-    static bool IsSetAccessKeyId(void) { return (0 < S3fsCurl::AWSAccessKeyId.size() && 0 < S3fsCurl::AWSSecretAccessKey.size()); }
+    static bool IsSetAccessKeyId(void){
+                  return (0 < S3fsCurl::IAM_role.size() || (0 < S3fsCurl::AWSAccessKeyId.size() && 0 < S3fsCurl::AWSSecretAccessKey.size()));
+                }
     static long SetSslVerifyHostname(long value);
     static long GetSslVerifyHostname(void) { return S3fsCurl::ssl_verify_hostname; }
     static int SetMaxParallelCount(int value);
+    static std::string SetIAMRole(const char* role);
+    static const char* GetIAMRole(void) { return S3fsCurl::IAM_role.c_str(); }
 
     // methods
     bool CreateCurlHandle(bool force = false);

src/s3fs.cpp

@@ -2664,6 +2664,12 @@ static int s3fs_check_service(void)
 {
   FPRN("check services.");
 
+  // At first time for access S3, we check IAM role if it sets.
+  if(!S3fsCurl::CheckIAMCredentialUpdate()){
+    fprintf(stderr, "%s: Failed to check IAM role name(%s).\n", program_name.c_str(), S3fsCurl::GetIAMRole());
+    return EXIT_FAILURE;
+  }
+
   S3fsCurl s3fscurl;
   if(0 != s3fscurl.CheckBucket()){
     fprintf(stderr, "%s: Failed to access bucket.\n", program_name.c_str());
@@ -3256,6 +3262,11 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
       passwd_file = strchr(arg, '=') + sizeof(char);
       return 0;
     }
+    if(0 == STR2NCMP(arg, "iam_role=")){
+      const char* role = strchr(arg, '=') + sizeof(char);
+      S3fsCurl::SetIAMRole(role);
+      return 0;
+    }
     if(0 == STR2NCMP(arg, "public_bucket=")){
       long pubbucket = strtol(strchr(arg, '=') + sizeof(char), 0, 10);
       if(1 == pubbucket){

src/s3fs_util.cpp

@@ -743,6 +743,17 @@ blkcnt_t get_blocks(off_t size)
   return size / 512 + 1;
 }
 
+time_t cvtIAMExpireStringToTime(const char* s)
+{
+  struct tm tm;
+  if(!s){
+    return 0L;
+  }
+  memset(&tm, 0, sizeof(struct tm));
+  strptime(s, "%Y-%m-%dT%H:%M:%S", &tm);
+  return mktime(&tm);      // GMT
+}
+
 time_t get_lastmodified(const char* s)
 {
   struct tm tm;
@@ -927,6 +938,10 @@ void show_help (void)
     "   enable_content_md5 (default is disable)\n"
     "      - verifying uploaded object without multipart by content-md5 header.\n"
     "\n"
+    "   iam_role (default is no role)\n"
+    "      - set the IAM Role that will supply the credentials from the \n"
+    "      instance meta-data.\n"
+    "\n"
     "   noxmlns (disable registing xml name space)\n"
     "        disable registing xml name space for response of \n"
     "        ListBucketResult and ListVersionsResult etc. Default name \n"

src/s3fs_util.h

@@ -106,6 +106,7 @@ uid_t get_uid(headers_t& meta);
 gid_t get_gid(const char *s);
 gid_t get_gid(headers_t& meta);
 blkcnt_t get_blocks(off_t size);
+time_t cvtIAMExpireStringToTime(const char* s);
 time_t get_lastmodified(const char* s);
 time_t get_lastmodified(headers_t& meta);
 bool is_need_check_obj_detail(headers_t& meta);