Merge pull request #671 from psyvision/master

Add support for ECS metadata endpoint

.travis.yml

@@ -8,7 +8,7 @@ matrix:
       cache: apt
       before_install:
         - sudo apt-get update -qq
-        - sudo apt-get install -qq cppcheck libfuse-dev openjdk-7-jdk
+        - sudo apt-get install -qq cppcheck libfuse-dev openjdk-7-jdk libjsoncpp-dev
         - sudo update-alternatives --set java /usr/lib/jvm/java-7-openjdk-amd64/jre/bin/java
       script:
         - ./autogen.sh
@@ -24,7 +24,7 @@ matrix:
       osx_image: xcode8.3
       before_install:
         - brew update
-        - brew install truncate
+        - brew install truncate jsoncpp
         - brew tap caskroom/cask
         - brew cask install osxfuse
         - if [ -f /Library/Filesystems/osxfusefs.fs/Support/load_osxfusefs ]; then sudo chmod +s /Library/Filesystems/osxfusefs.fs/Support/load_osxfusefs ; elif [ -f /Library/Filesystems/osxfuse.fs/Contents/Resources/load_osxfuse ]; then sudo chmod +s /Library/Filesystems/osxfuse.fs/Contents/Resources/load_osxfuse ; fi

configure.ac

@@ -176,13 +176,13 @@ dnl
 dnl For PKG_CONFIG before checking nss/gnutls.
 dnl this is redundant checking, but we need checking before following.
 dnl
-PKG_CHECK_MODULES([common_lib_checking], [fuse >= ${min_fuse_version} libcurl >= 7.0 libxml-2.0 >= 2.6])
+PKG_CHECK_MODULES([common_lib_checking], [fuse >= ${min_fuse_version} libcurl >= 7.0 libxml-2.0 >= 2.6 jsoncpp >= 0.6 ])
 
 AC_MSG_CHECKING([compile s3fs with])
 case "${auth_lib}" in
 openssl)
   AC_MSG_RESULT(OpenSSL)
-  PKG_CHECK_MODULES([DEPS], [fuse >= ${min_fuse_version} libcurl >= 7.0 libxml-2.0 >= 2.6 libcrypto >= 0.9])
+  PKG_CHECK_MODULES([DEPS], [fuse >= ${min_fuse_version} libcurl >= 7.0 libxml-2.0 >= 2.6 libcrypto >= 0.9 jsoncpp >= 0.6 ])
   ;;
 gnutls)
   AC_MSG_RESULT(GnuTLS-gcrypt)
@@ -191,7 +191,7 @@ gnutls)
   AS_IF([test "$gnutls_nettle" = ""], [AC_CHECK_LIB(gcrypt, gcry_control, [gnutls_nettle=0])])
   AS_IF([test $gnutls_nettle = 0],
     [
-      PKG_CHECK_MODULES([DEPS], [fuse >= ${min_fuse_version} libcurl >= 7.0 libxml-2.0 >= 2.6 gnutls >= 2.12.0 ])
+      PKG_CHECK_MODULES([DEPS], [fuse >= ${min_fuse_version} libcurl >= 7.0 libxml-2.0 >= 2.6 gnutls >= 2.12.0 jsoncpp >= 0.6 ])
       LIBS="-lgnutls -lgcrypt $LIBS"
       AC_MSG_CHECKING([gnutls is build with])
       AC_MSG_RESULT(gcrypt)
@@ -205,7 +205,7 @@ nettle)
   AS_IF([test "$gnutls_nettle" = ""], [AC_CHECK_LIB(nettle, nettle_MD5Init, [gnutls_nettle=1])])
   AS_IF([test $gnutls_nettle = 1],
     [
-      PKG_CHECK_MODULES([DEPS], [fuse >= ${min_fuse_version} libcurl >= 7.0 libxml-2.0 >= 2.6 nettle >= 2.7.1 ])
+      PKG_CHECK_MODULES([DEPS], [fuse >= ${min_fuse_version} libcurl >= 7.0 libxml-2.0 >= 2.6 nettle >= 2.7.1 jsoncpp >= 0.6 ])
       LIBS="-lgnutls -lnettle $LIBS"
       AC_MSG_CHECKING([gnutls is build with])
       AC_MSG_RESULT(nettle)
@@ -214,7 +214,7 @@ nettle)
   ;;
 nss)
   AC_MSG_RESULT(NSS)
-  PKG_CHECK_MODULES([DEPS], [fuse >= ${min_fuse_version} libcurl >= 7.0 libxml-2.0 >= 2.6 nss >= 3.15.0 ])
+  PKG_CHECK_MODULES([DEPS], [fuse >= ${min_fuse_version} libcurl >= 7.0 libxml-2.0 >= 2.6 nss >= 3.15.0 jsoncpp >= 0.6 ])
   ;;
 *)
   AC_MSG_ERROR([unknown ssl library type.])

src/curl.cpp

@@ -41,6 +41,8 @@
 #include <algorithm>
 #include <list>
 #include <vector>
+#include <json/json.h>
+#include <json/reader.h>
 
 #include "common.h"
 #include "curl.h"
@@ -320,12 +322,16 @@ void CurlHandlerPool::ReturnHandler(CURL* h)
 #define MAX_MULTI_COPY_SOURCE_SIZE  524288000         // 500MB
 
 #define	IAM_EXPIRE_MERGIN           (20 * 60)         // update timing
+#define IAM_CRED_URL_ECS            "http://169.254.170.2" 
 #define	IAM_CRED_URL                "http://169.254.169.254/latest/meta-data/iam/security-credentials/"
+#define ECS_IAM_ENV_VAR             "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"
 #define IAMCRED_ACCESSKEYID         "AccessKeyId"
 #define IAMCRED_SECRETACCESSKEY     "SecretAccessKey"
 #define IAMCRED_ACCESSTOKEN         "Token"
 #define IAMCRED_EXPIRATION          "Expiration"
+#define IAMCRED_ROLEARN             "RoleArn"
 #define IAMCRED_KEYCOUNT            4
+#define IAMCRED_KEYCOUNT_ECS	    5
 
 // [NOTICE]
 // This symbol is for libcurl under 7.23.0
@@ -357,6 +363,7 @@ string           S3fsCurl::AWSAccessKeyId;
 string           S3fsCurl::AWSSecretAccessKey;
 string           S3fsCurl::AWSAccessToken;
 time_t           S3fsCurl::AWSAccessTokenExpire= 0;
+bool             S3fsCurl::is_ecs              = false;
 string           S3fsCurl::IAM_role;
 long             S3fsCurl::ssl_verify_hostname = 1;    // default(original code...)
 curltime_t       S3fsCurl::curl_times;
@@ -1159,6 +1166,13 @@ long S3fsCurl::SetSslVerifyHostname(long value)
   return old;
 }
 
+bool S3fsCurl::SetIsECS(bool flag)
+{
+  bool old = S3fsCurl::is_ecs;
+  S3fsCurl::is_ecs = flag;
+  return old;
+}
+
 string S3fsCurl::SetIAMRole(const char* role)
 {
   string old = S3fsCurl::IAM_role;
@@ -1410,37 +1424,23 @@ bool S3fsCurl::ParseIAMCredentialResponse(const char* response, iamcredmap_t& ke
   if(!response){
     return false;
   }
-  istringstream sscred(response);
+
-  string        oneline;
+  Json::Value root;
-  keyval.clear();
+  Json::Reader reader;
-  while(getline(sscred, oneline, '\n')){
+
-    string::size_type pos;
+  if (!reader.parse(response, root)) {
-    string            key;
+    return false;
-    string            val;
-    if(string::npos != (pos = oneline.find(IAMCRED_ACCESSKEYID))){
-      key = IAMCRED_ACCESSKEYID;
-    }else if(string::npos != (pos = oneline.find(IAMCRED_SECRETACCESSKEY))){
-      key = IAMCRED_SECRETACCESSKEY;
-    }else if(string::npos != (pos = oneline.find(IAMCRED_ACCESSTOKEN))){
-      key = IAMCRED_ACCESSTOKEN;
-    }else if(string::npos != (pos = oneline.find(IAMCRED_EXPIRATION))){
-      key = IAMCRED_EXPIRATION;
-    }else{
-      continue;
-    }
-    if(string::npos == (pos = oneline.find(':', pos + key.length()))){
-      continue;
-    }
-    if(string::npos == (pos = oneline.find('\"', pos))){
-      continue;
-    }
-    oneline = oneline.substr(pos + sizeof(char));
-    if(string::npos == (pos = oneline.find('\"'))){
-      continue;
-    }
-    val = oneline.substr(0, pos);
-    keyval[key] = val;
   }
+
+  keyval[string(IAMCRED_ACCESSKEYID)] = root.get(IAMCRED_ACCESSKEYID, "").asString();
+  keyval[string(IAMCRED_SECRETACCESSKEY)] = root.get(IAMCRED_SECRETACCESSKEY, "").asString();
+  keyval[string(IAMCRED_ACCESSTOKEN)] = root.get(IAMCRED_ACCESSTOKEN, "").asString();
+  keyval[string(IAMCRED_EXPIRATION)] = root.get(IAMCRED_EXPIRATION, "").asString();
+
+  if (S3fsCurl::is_ecs) {
+    keyval[string(IAMCRED_ROLEARN)] = root.get(IAMCRED_ROLEARN, "").asString();
+  }
+
   return true;
 }
 
@@ -1453,7 +1453,8 @@ bool S3fsCurl::SetIAMCredentials(const char* response)
   if(!ParseIAMCredentialResponse(response, keyval)){
     return false;
   }
-  if(IAMCRED_KEYCOUNT != keyval.size()){
+
+  if((S3fsCurl::is_ecs ? IAMCRED_KEYCOUNT_ECS : IAMCRED_KEYCOUNT) != keyval.size()){
     return false;
   }
 
@@ -1467,7 +1468,7 @@ bool S3fsCurl::SetIAMCredentials(const char* response)
 
 bool S3fsCurl::CheckIAMCredentialUpdate(void)
 {
-  if(0 == S3fsCurl::IAM_role.size()){
+  if(0 == S3fsCurl::IAM_role.size() && !S3fsCurl::is_ecs){
     return true;
   }
   if(time(NULL) + IAM_EXPIRE_MERGIN <= S3fsCurl::AWSAccessTokenExpire){
@@ -2133,7 +2134,7 @@ string S3fsCurl::CalcSignatureV2(const string& method, const string& strMD5, con
   string Signature;
   string StringToSign;
 
-  if(0 < S3fsCurl::IAM_role.size()){
+  if(0 < S3fsCurl::IAM_role.size() || S3fsCurl::is_ecs){
     if(!S3fsCurl::CheckIAMCredentialUpdate()){
       S3FS_PRN_ERR("Something error occurred in checking IAM credential.");
       return Signature;  // returns empty string, then it occurs error.
@@ -2175,7 +2176,7 @@ string S3fsCurl::CalcSignature(const string& method, const string& canonical_uri
   string Signature, StringCQ, StringToSign;
   string uriencode;
 
-  if(0 < S3fsCurl::IAM_role.size()){
+  if(0 < S3fsCurl::IAM_role.size()  || S3fsCurl::is_ecs){
     if(!S3fsCurl::CheckIAMCredentialUpdate()){
       S3FS_PRN_ERR("Something error occurred in checking IAM credential.");
       return Signature;  // returns empty string, then it occurs error.
@@ -2412,12 +2413,15 @@ int S3fsCurl::DeleteRequest(const char* tpath)
 //
 int S3fsCurl::GetIAMCredentials(void)
 {
-  S3FS_PRN_INFO3("[IAM role=%s]", S3fsCurl::IAM_role.c_str());
+  if (!S3fsCurl::is_ecs) {
+    S3FS_PRN_INFO3("[IAM role=%s]", S3fsCurl::IAM_role.c_str());
 
-  if(0 == S3fsCurl::IAM_role.size()){
+    if(0 == S3fsCurl::IAM_role.size()) {
-    S3FS_PRN_ERR("IAM role name is empty.");
+      S3FS_PRN_ERR("IAM role name is empty.");
-    return -EIO;
+      return -EIO;
+    }
   }
+
   // at first set type for handle
   type = REQTYPE_IAMCRED;
 
@@ -2426,7 +2430,13 @@ int S3fsCurl::GetIAMCredentials(void)
   }
 
   // url
-  url             = string(IAM_CRED_URL) + S3fsCurl::IAM_role;
+  if (is_ecs) {
+    url = string(IAM_CRED_URL_ECS) + std::getenv(ECS_IAM_ENV_VAR);
+  }
+  else {
+    url = string(IAM_CRED_URL) + S3fsCurl::IAM_role;
+  }
+
   requestHeaders  = NULL;
   responseHeaders.clear();
   bodydata        = new BodyData();
@@ -2463,7 +2473,7 @@ bool S3fsCurl::LoadIAMRoleFromMetaData(void)
   }
 
   // url
-  url             = IAM_CRED_URL;
+  url             = string(IAM_CRED_URL);
   requestHeaders  = NULL;
   responseHeaders.clear();
   bodydata        = new BodyData();

src/curl.h

@@ -230,6 +230,7 @@ class S3fsCurl
     static std::string      AWSSecretAccessKey;
     static std::string      AWSAccessToken;
     static time_t           AWSAccessTokenExpire;
+    static bool             is_ecs;
     static std::string      IAM_role;
     static long             ssl_verify_hostname;
     static curltime_t       curl_times;
@@ -374,6 +375,7 @@ class S3fsCurl
     static long GetSslVerifyHostname(void) { return S3fsCurl::ssl_verify_hostname; }
     static int SetMaxParallelCount(int value);
     static int GetMaxParallelCount(void) { return S3fsCurl::max_parallel_cnt; }
+    static bool SetIsECS(bool flag);
     static std::string SetIAMRole(const char* role);
     static const char* GetIAMRole(void) { return S3fsCurl::IAM_role.c_str(); }
     static bool SetMultipartSize(off_t size);

src/s3fs.cpp

@@ -122,6 +122,7 @@ static bool is_s3fs_uid           = false;// default does not set.
 static bool is_s3fs_gid           = false;// default does not set.
 static bool is_s3fs_umask         = false;// default does not set.
 static bool is_remove_cache       = false;
+static bool is_ecs                = false;
 static bool is_use_xattr          = false;
 static bool create_bucket         = false;
 static int64_t singlepart_copy_limit = FIVE_GB;
@@ -4073,7 +4074,7 @@ static int get_access_keys(void)
   }
 
   // access key loading is deferred
-  if(load_iamrole){
+  if(load_iamrole || is_ecs){
      return EXIT_SUCCESS;
   }
 
@@ -4508,7 +4509,16 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
       passwd_file = strchr(arg, '=') + sizeof(char);
       return 0;
     }
+    if(0 == strcmp(arg, "ecs")){
+      S3fsCurl::SetIsECS(true);
+      is_ecs = true;
+      return 0;
+    }
     if(0 == STR2NCMP(arg, "iam_role")){
+      if (is_ecs) {
+        S3FS_PRN_EXIT("option iam_role cannot be used in conjunction with ecs");
+        return -1;
+      }
       if(0 == strcmp(arg, "iam_role") || 0 == strcmp(arg, "iam_role=auto")){
         // loading IAM role name in s3fs_init(), because we need to wait initializing curl.
         //
@@ -4908,7 +4918,7 @@ int main(int argc, char* argv[])
     S3FS_PRN_EXIT("specifying both passwd_file and the access keys options is invalid.");
     exit(EXIT_FAILURE);
   }
-  if(!S3fsCurl::IsPublicBucket() && !load_iamrole){
+  if(!S3fsCurl::IsPublicBucket() && !load_iamrole && !is_ecs){
     if(EXIT_SUCCESS != get_access_keys()){
       exit(EXIT_FAILURE);
     }

src/s3fs_util.cpp

@@ -1167,6 +1167,10 @@ void show_help (void)
     "   enable_content_md5 (default is disable)\n"
     "      - ensure data integrity during writes with MD5 hash.\n"
     "\n"
+    "   ecs\n"
+    "      - This option instructs s3fs to query the ECS container credential\n"
+    "      metadata address instead of the instance metadata address.\n"
+    "\n"
     "   iam_role (default is no IAM role)\n"
     "      - This option requires the IAM role name or \"auto\". If you specify\n"
     "      \"auto\", s3fs will automatically use the IAM role names that are set\n"