Merge pull request #411 from ggtakec/master

loading IAM role name automatically(iam_role option) - #387

doc/man/s3fs.1

@@ -197,8 +197,8 @@ Enable to send "Content-MD5" header when uploading a object without multipart po
 If this option is enabled, it has some influences on a performance of s3fs when uploading small object.
 Because s3fs always checks MD5 when uploading large object, this option does not affect on large object.
 .TP
-\fB\-o\fR iam_role ( default is no role )
+\fB\-o\fR iam_role ( default is no IAM role )
-set the IAM Role that will supply the credentials from the instance meta-data.
+This option requires the IAM role name or "auto". If you specify "auto", s3fs will automatically use the IAM role names that are set to an instance. If you specify this option without any argument, it is the same as that you have specified the "auto".
 .TP
 \fB\-o\fR noxmlns - disable registing xml name space.
 disable registing xml name space for response of ListBucketResult and ListVersionsResult etc. Default name space is looked up from "http://s3.amazonaws.com/doc/2006-03-01".

src/curl.cpp

@@ -315,6 +315,9 @@ void CurlHandlerPool::ReturnHandler(CURL* h)
 #define IAMCRED_ACCESSTOKEN         "Token"
 #define IAMCRED_EXPIRATION          "Expiration"
 #define IAMCRED_KEYCOUNT            4
+#define	IAM_DEFAULT_ROLE_URL        "http://169.254.169.254/latest/meta-data/iam/info"
+#define IAMDEFROLE_PROFARN          "InstanceProfileArn"
+#define IAMDEFROLE_PROFARN_PART     ":instance-profile/"
 
 // [NOTICE]
 // This symbol is for libcurl under 7.23.0
@@ -1436,6 +1439,66 @@ bool S3fsCurl::CheckIAMCredentialUpdate(void)
   return true;
 }
 
+bool S3fsCurl::ParseIAMRoleFromMetaDataResponse(const char* response, string& rolename)
+{
+  if(!response){
+    return false;
+  }
+  // [NOTE]
+  // expected following strings.
+  // 
+  // {
+  //   "Code" : "Success",
+  //   "LastUpdated" : "2016-01-01T00:00:00Z",
+  //   "InstanceProfileArn" : "arn:aws:iam::111111111111:instance-profile/myrolename",
+  //   "InstanceProfileId" : "AAAAAAAAAAAAAAAAAAAAA"
+  // }
+  //
+  istringstream ssrole(response);
+  string        oneline;
+  while(getline(ssrole, oneline, '\n')){
+    string::size_type pos;
+    if(string::npos != (pos = oneline.find(IAMDEFROLE_PROFARN))){
+      if(string::npos == (pos = oneline.find(':', pos + strlen(IAMDEFROLE_PROFARN)))){
+        continue;
+      }
+      if(string::npos == (pos = oneline.find('\"', pos))){
+        continue;
+      }
+
+      // value
+      oneline = oneline.substr(pos + sizeof(char));
+      if(string::npos == (pos = oneline.find('\"'))){
+        continue;
+      }
+      oneline = oneline.substr(0, pos);
+
+      // role name
+      if(string::npos == (pos = oneline.find(IAMDEFROLE_PROFARN_PART))){
+        continue;
+      }
+      rolename = oneline.substr(pos + strlen(IAMDEFROLE_PROFARN_PART));
+
+      return !rolename.empty();
+    }
+  }
+  return false;
+}
+
+bool S3fsCurl::SetIAMRoleFromMetaData(const char* response)
+{
+  S3FS_PRN_INFO3("IAM role name response = \"%s\"", response);
+
+  string rolename;
+
+  if(!S3fsCurl::ParseIAMRoleFromMetaDataResponse(response, rolename)){
+    return false;
+  }
+
+  SetIAMRole(rolename.c_str());
+  return true;
+}
+
 bool S3fsCurl::AddUserAgent(CURL* hCurl)
 {
   if(!hCurl){
@@ -1523,8 +1586,8 @@ bool S3fsCurl::ResetHandle(void)
   curl_easy_setopt(hCurl, CURLOPT_PROGRESSDATA, hCurl);
   // curl_easy_setopt(hCurl, CURLOPT_FORBID_REUSE, 1);
 
-  if(type != REQTYPE_IAMCRED){
+  if(type != REQTYPE_IAMCRED && type != REQTYPE_IAMROLE){
-    // REQTYPE_IAMCRED is always HTTP
+    // REQTYPE_IAMCRED and REQTYPE_IAMROLE are always HTTP
     if(0 == S3fsCurl::ssl_verify_hostname){
       curl_easy_setopt(hCurl, CURLOPT_SSL_VERIFYHOST, 0);
     }
@@ -1575,11 +1638,11 @@ bool S3fsCurl::CreateCurlHandle(bool force)
   }
 
   // [NOTE]
-  // If type is REQTYPE_IAMCRED, do not clear type.
+  // If type is REQTYPE_IAMCRED or REQTYPE_IAMROLE, do not clear type.
   // Because that type only uses HTTP protocol, then the special
   // logic in ResetHandle function.
   //
-  if(type != REQTYPE_IAMCRED){
+  if(type != REQTYPE_IAMCRED && type != REQTYPE_IAMROLE){
     type = REQTYPE_UNSET;
   }
 
@@ -1834,6 +1897,12 @@ bool S3fsCurl::RemakeHandle(void)
       curl_easy_setopt(hCurl, CURLOPT_HTTPHEADER, requestHeaders);
       break;
 
+    case REQTYPE_IAMROLE:
+      curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str());
+      curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, (void*)bodydata);
+      curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
+      break;
+
     default:
       S3FS_PRN_ERR("request type is unknown(%d)", type);
       return false;
@@ -2300,6 +2369,44 @@ int S3fsCurl::GetIAMCredentials(void)
   return result;
 }
 
+//
+// Get IAM role name automatically.
+//
+bool S3fsCurl::LoadIAMRoleFromMetaData(void)
+{
+  S3FS_PRN_INFO3("Get IAM Role name");
+
+  // at first set type for handle
+  type = REQTYPE_IAMROLE;
+
+  if(!CreateCurlHandle(true)){
+    return false;
+  }
+
+  // url
+  url             = IAM_DEFAULT_ROLE_URL;
+  requestHeaders  = NULL;
+  responseHeaders.clear();
+  bodydata        = new BodyData();
+
+  curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str());
+  curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, (void*)bodydata);
+  curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
+  S3fsCurl::AddUserAgent(hCurl);        // put User-Agent
+
+  int result = RequestPerform();
+
+  // analizing response
+  if(0 == result && !S3fsCurl::SetIAMRoleFromMetaData(bodydata->str())){
+    S3FS_PRN_ERR("Something error occurred, could not get IAM role name.");
+    result = -EIO;
+  }
+  delete bodydata;
+  bodydata = NULL;
+
+  return (0 == result);
+}
+
 bool S3fsCurl::AddSseRequestHead(sse_type_t ssetype, string& ssevalue, bool is_only_c, bool is_copy)
 {
   if(SSE_S3 == ssetype){

src/curl.h

@@ -201,7 +201,8 @@ class S3fsCurl
       REQTYPE_COPYMULTIPOST,
       REQTYPE_MULTILIST,
       REQTYPE_IAMCRED,
-      REQTYPE_ABORTMULTIUPLOAD
+      REQTYPE_ABORTMULTIUPLOAD,
+      REQTYPE_IAMROLE
     };
 
     // class variables
@@ -297,6 +298,8 @@ class S3fsCurl
 
     static bool ParseIAMCredentialResponse(const char* response, iamcredmap_t& keyval);
     static bool SetIAMCredentials(const char* response);
+    static bool ParseIAMRoleFromMetaDataResponse(const char* response, std::string& rolename);
+    static bool SetIAMRoleFromMetaData(const char* response);
     static bool LoadEnvSseCKeys(void);
     static bool LoadEnvSseKmsid(void);
     static bool PushbackSseKeys(std::string& onekey);
@@ -378,6 +381,7 @@ class S3fsCurl
     bool CreateCurlHandle(bool force = false);
     bool DestroyCurlHandle(void);
 
+    bool LoadIAMRoleFromMetaData(void);
     bool AddSseRequestHead(sse_type_t ssetype, std::string& ssevalue, bool is_only_c, bool is_copy);
     bool GetResponseCode(long& responseCode);
     int RequestPerform(void);

src/s3fs.cpp

@@ -113,6 +113,7 @@ static bool nocopyapi             = false;
 static bool norenameapi           = false;
 static bool nonempty              = false;
 static bool allow_other           = false;
+static bool load_iamrole          = false;
 static uid_t s3fs_uid             = 0;
 static gid_t s3fs_gid             = 0;
 static mode_t s3fs_umask          = 0;
@@ -3351,6 +3352,19 @@ static void* s3fs_init(struct fuse_conn_info* conn)
     return NULL;
   }
 
+  // check loading IAM role name
+  if(load_iamrole){
+    // load IAM role name from http://169.254.169.254/latest/meta-data/iam/info
+    //
+    S3fsCurl s3fscurl;
+    if(!s3fscurl.LoadIAMRoleFromMetaData()){
+      S3FS_PRN_CRIT("could not load IAM role name from meta data.");
+      s3fs_exit_fuseloop(EXIT_FAILURE);
+      return NULL;
+    }
+    S3FS_PRN_INFO("loaded IAM role name = %s", S3fsCurl::GetIAMRole());
+  }
+
   if (create_bucket){
     do_create_bucket();
   }
@@ -4447,11 +4461,20 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
       passwd_file = strchr(arg, '=') + sizeof(char);
       return 0;
     }
-    if(0 == STR2NCMP(arg, "iam_role=")){
+    if(0 == STR2NCMP(arg, "iam_role")){
+      if(0 == strcmp(arg, "iam_role") || 0 == strcmp(arg, "iam_role=auto")){
+        // loading IAM role name in s3fs_init(), because we need to wait initializing curl.
+        //
+        load_iamrole = true;
+        return 0;
+
+      }else if(0 == STR2NCMP(arg, "iam_role=")){
         const char* role = strchr(arg, '=') + sizeof(char);
         S3fsCurl::SetIAMRole(role);
+        load_iamrole = false;
         return 0;
       }
+    }
     if(0 == STR2NCMP(arg, "public_bucket=")){
       off_t pubbucket = s3fs_strtoofft(strchr(arg, '=') + sizeof(char));
       if(1 == pubbucket){

src/s3fs_util.cpp

@@ -1077,9 +1077,11 @@ void show_help (void)
     "   enable_content_md5 (default is disable)\n"
     "      - ensure data integrity during writes with MD5 hash.\n"
     "\n"
-    "   iam_role (default is no role)\n"
+    "   iam_role (default is no IAM role)\n"
-    "      - set the IAM Role that will supply the credentials from the \n"
+    "      - This option requires the IAM role name or \"auto\". If you specify\n"
-    "      instance meta-data.\n"
+    "      \"auto\", s3fs will automatically use the IAM role names that are set\n"
+    "      to an instance. If you specify this option without any argument, it\n"
+    "      is the same as that you have specified the \"auto\".\n"
     "\n"
     "   noxmlns (disable registering xml name space)\n"
     "        disable registering xml name space for response of \n"