Merge pull request #671 from psyvision/master

Add support for ECS metadata endpoint
2024-11-13 07:56:30 +00:00 · 2017-11-19 14:43:06 +09:00 · 2017-11-19 14:43:06 +09:00 · bad48ab59a
commit bad48ab59a
parent 259f028490 b95e4acaeb
6 changed files with 76 additions and 50 deletions
--- a/.travis.yml
+++ b/.travis.yml
@ -8,7 +8,7 @@ matrix:
      cache: apt
      before_install:
        - sudo apt-get update -qq
-        - sudo apt-get install -qq cppcheck libfuse-dev openjdk-7-jdk
+        - sudo apt-get install -qq cppcheck libfuse-dev openjdk-7-jdk libjsoncpp-dev
        - sudo update-alternatives --set java /usr/lib/jvm/java-7-openjdk-amd64/jre/bin/java
      script:
        - ./autogen.sh
@ -24,7 +24,7 @@ matrix:
      osx_image: xcode8.3
      before_install:
        - brew update
-        - brew install truncate
+        - brew install truncate jsoncpp
        - brew tap caskroom/cask
        - brew cask install osxfuse
        - if [ -f /Library/Filesystems/osxfusefs.fs/Support/load_osxfusefs ]; then sudo chmod +s /Library/Filesystems/osxfusefs.fs/Support/load_osxfusefs ; elif [ -f /Library/Filesystems/osxfuse.fs/Contents/Resources/load_osxfuse ]; then sudo chmod +s /Library/Filesystems/osxfuse.fs/Contents/Resources/load_osxfuse ; fi
--- a/configure.ac
+++ b/configure.ac
@ -176,13 +176,13 @@ dnl
 dnl For PKG_CONFIG before checking nss/gnutls.
 dnl this is redundant checking, but we need checking before following.
 dnl
-PKG_CHECK_MODULES([common_lib_checking], [fuse >= ${min_fuse_version} libcurl >= 7.0 libxml-2.0 >= 2.6])
+PKG_CHECK_MODULES([common_lib_checking], [fuse >= ${min_fuse_version} libcurl >= 7.0 libxml-2.0 >= 2.6 jsoncpp >= 0.6 ])

 AC_MSG_CHECKING([compile s3fs with])
 case "${auth_lib}" in
 openssl)
  AC_MSG_RESULT(OpenSSL)
-  PKG_CHECK_MODULES([DEPS], [fuse >= ${min_fuse_version} libcurl >= 7.0 libxml-2.0 >= 2.6 libcrypto >= 0.9])
+  PKG_CHECK_MODULES([DEPS], [fuse >= ${min_fuse_version} libcurl >= 7.0 libxml-2.0 >= 2.6 libcrypto >= 0.9 jsoncpp >= 0.6 ])
  ;;
 gnutls)
  AC_MSG_RESULT(GnuTLS-gcrypt)
@ -191,7 +191,7 @@ gnutls)
  AS_IF([test "$gnutls_nettle" = ""], [AC_CHECK_LIB(gcrypt, gcry_control, [gnutls_nettle=0])])
  AS_IF([test $gnutls_nettle = 0],
    [
-      PKG_CHECK_MODULES([DEPS], [fuse >= ${min_fuse_version} libcurl >= 7.0 libxml-2.0 >= 2.6 gnutls >= 2.12.0 ])
+      PKG_CHECK_MODULES([DEPS], [fuse >= ${min_fuse_version} libcurl >= 7.0 libxml-2.0 >= 2.6 gnutls >= 2.12.0 jsoncpp >= 0.6 ])
      LIBS="-lgnutls -lgcrypt $LIBS"
      AC_MSG_CHECKING([gnutls is build with])
      AC_MSG_RESULT(gcrypt)
@ -205,7 +205,7 @@ nettle)
  AS_IF([test "$gnutls_nettle" = ""], [AC_CHECK_LIB(nettle, nettle_MD5Init, [gnutls_nettle=1])])
  AS_IF([test $gnutls_nettle = 1],
    [
-      PKG_CHECK_MODULES([DEPS], [fuse >= ${min_fuse_version} libcurl >= 7.0 libxml-2.0 >= 2.6 nettle >= 2.7.1 ])
+      PKG_CHECK_MODULES([DEPS], [fuse >= ${min_fuse_version} libcurl >= 7.0 libxml-2.0 >= 2.6 nettle >= 2.7.1 jsoncpp >= 0.6 ])
      LIBS="-lgnutls -lnettle $LIBS"
      AC_MSG_CHECKING([gnutls is build with])
      AC_MSG_RESULT(nettle)
@ -214,7 +214,7 @@ nettle)
  ;;
 nss)
  AC_MSG_RESULT(NSS)
-  PKG_CHECK_MODULES([DEPS], [fuse >= ${min_fuse_version} libcurl >= 7.0 libxml-2.0 >= 2.6 nss >= 3.15.0 ])
+  PKG_CHECK_MODULES([DEPS], [fuse >= ${min_fuse_version} libcurl >= 7.0 libxml-2.0 >= 2.6 nss >= 3.15.0 jsoncpp >= 0.6 ])
  ;;
 *)
  AC_MSG_ERROR([unknown ssl library type.])
--- a/src/curl.cpp
+++ b/src/curl.cpp
@ -41,6 +41,8 @@
 #include <algorithm>
 #include <list>
 #include <vector>
+#include <json/json.h>
+#include <json/reader.h>

 #include "common.h"
 #include "curl.h"
@ -320,12 +322,16 @@ void CurlHandlerPool::ReturnHandler(CURL* h)
 #define MAX_MULTI_COPY_SOURCE_SIZE  524288000         // 500MB

 #define	IAM_EXPIRE_MERGIN           (20 * 60)         // update timing
+#define IAM_CRED_URL_ECS            "http://169.254.170.2" 
 #define	IAM_CRED_URL                "http://169.254.169.254/latest/meta-data/iam/security-credentials/"
+#define ECS_IAM_ENV_VAR             "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"
 #define IAMCRED_ACCESSKEYID         "AccessKeyId"
 #define IAMCRED_SECRETACCESSKEY     "SecretAccessKey"
 #define IAMCRED_ACCESSTOKEN         "Token"
 #define IAMCRED_EXPIRATION          "Expiration"
+#define IAMCRED_ROLEARN             "RoleArn"
 #define IAMCRED_KEYCOUNT            4
+#define IAMCRED_KEYCOUNT_ECS	    5

 // [NOTICE]
 // This symbol is for libcurl under 7.23.0
@ -357,6 +363,7 @@ string           S3fsCurl::AWSAccessKeyId;
 string           S3fsCurl::AWSSecretAccessKey;
 string           S3fsCurl::AWSAccessToken;
 time_t           S3fsCurl::AWSAccessTokenExpire= 0;
+bool             S3fsCurl::is_ecs              = false;
 string           S3fsCurl::IAM_role;
 long             S3fsCurl::ssl_verify_hostname = 1;    // default(original code...)
 curltime_t       S3fsCurl::curl_times;
@ -1159,6 +1166,13 @@ long S3fsCurl::SetSslVerifyHostname(long value)
  return old;
 }

+bool S3fsCurl::SetIsECS(bool flag)
+{
+  bool old = S3fsCurl::is_ecs;
+  S3fsCurl::is_ecs = flag;
+  return old;
+}
+
 string S3fsCurl::SetIAMRole(const char* role)
 {
  string old = S3fsCurl::IAM_role;
@ -1410,37 +1424,23 @@ bool S3fsCurl::ParseIAMCredentialResponse(const char* response, iamcredmap_t& ke
  if(!response){
    return false;
  }
-  istringstream sscred(response);
-  string        oneline;
-  keyval.clear();
-  while(getline(sscred, oneline, '\n')){
-    string::size_type pos;
-    string            key;
-    string            val;
-    if(string::npos != (pos = oneline.find(IAMCRED_ACCESSKEYID))){
-      key = IAMCRED_ACCESSKEYID;
-    }else if(string::npos != (pos = oneline.find(IAMCRED_SECRETACCESSKEY))){
-      key = IAMCRED_SECRETACCESSKEY;
-    }else if(string::npos != (pos = oneline.find(IAMCRED_ACCESSTOKEN))){
-      key = IAMCRED_ACCESSTOKEN;
-    }else if(string::npos != (pos = oneline.find(IAMCRED_EXPIRATION))){
-      key = IAMCRED_EXPIRATION;
-    }else{
-      continue;
-    }
-    if(string::npos == (pos = oneline.find(':', pos + key.length()))){
-      continue;
-    }
-    if(string::npos == (pos = oneline.find('\"', pos))){
-      continue;
-    }
-    oneline = oneline.substr(pos + sizeof(char));
-    if(string::npos == (pos = oneline.find('\"'))){
-      continue;
-    }
-    val = oneline.substr(0, pos);
-    keyval[key] = val;
+
+  Json::Value root;
+  Json::Reader reader;
+
+  if (!reader.parse(response, root)) {
+    return false;
  }
+
+  keyval[string(IAMCRED_ACCESSKEYID)] = root.get(IAMCRED_ACCESSKEYID, "").asString();
+  keyval[string(IAMCRED_SECRETACCESSKEY)] = root.get(IAMCRED_SECRETACCESSKEY, "").asString();
+  keyval[string(IAMCRED_ACCESSTOKEN)] = root.get(IAMCRED_ACCESSTOKEN, "").asString();
+  keyval[string(IAMCRED_EXPIRATION)] = root.get(IAMCRED_EXPIRATION, "").asString();
+
+  if (S3fsCurl::is_ecs) {
+    keyval[string(IAMCRED_ROLEARN)] = root.get(IAMCRED_ROLEARN, "").asString();
+  }
+
  return true;
 }

@ -1453,7 +1453,8 @@ bool S3fsCurl::SetIAMCredentials(const char* response)
  if(!ParseIAMCredentialResponse(response, keyval)){
    return false;
  }
-  if(IAMCRED_KEYCOUNT != keyval.size()){
+
+  if((S3fsCurl::is_ecs ? IAMCRED_KEYCOUNT_ECS : IAMCRED_KEYCOUNT) != keyval.size()){
    return false;
  }

@ -1467,7 +1468,7 @@ bool S3fsCurl::SetIAMCredentials(const char* response)

 bool S3fsCurl::CheckIAMCredentialUpdate(void)
 {
-  if(0 == S3fsCurl::IAM_role.size()){
+  if(0 == S3fsCurl::IAM_role.size() && !S3fsCurl::is_ecs){
    return true;
  }
  if(time(NULL) + IAM_EXPIRE_MERGIN <= S3fsCurl::AWSAccessTokenExpire){
@ -2133,7 +2134,7 @@ string S3fsCurl::CalcSignatureV2(const string& method, const string& strMD5, con
  string Signature;
  string StringToSign;

-  if(0 < S3fsCurl::IAM_role.size()){
+  if(0 < S3fsCurl::IAM_role.size() || S3fsCurl::is_ecs){
    if(!S3fsCurl::CheckIAMCredentialUpdate()){
      S3FS_PRN_ERR("Something error occurred in checking IAM credential.");
      return Signature;  // returns empty string, then it occurs error.
@ -2175,7 +2176,7 @@ string S3fsCurl::CalcSignature(const string& method, const string& canonical_uri
  string Signature, StringCQ, StringToSign;
  string uriencode;

-  if(0 < S3fsCurl::IAM_role.size()){
+  if(0 < S3fsCurl::IAM_role.size()  || S3fsCurl::is_ecs){
    if(!S3fsCurl::CheckIAMCredentialUpdate()){
      S3FS_PRN_ERR("Something error occurred in checking IAM credential.");
      return Signature;  // returns empty string, then it occurs error.
@ -2335,7 +2336,7 @@ void S3fsCurl::insertV4Headers()
  requestHeaders = curl_slist_sort_insert(requestHeaders, "host", get_bucket_host().c_str());
  requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-content-sha256", contentSHA256.c_str());
  requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-date", date8601.c_str());
-
+	
  if(!S3fsCurl::IsPublicBucket()){
    string Signature = CalcSignature(op, realpath, query_string + (type == REQTYPE_PREMULTIPOST ? "=" : ""), strdate, contentSHA256, date8601);
    string auth = "AWS4-HMAC-SHA256 Credential=" + AWSAccessKeyId + "/" + strdate + "/" + endpoint +
@ -2412,12 +2413,15 @@ int S3fsCurl::DeleteRequest(const char* tpath)
 //
 int S3fsCurl::GetIAMCredentials(void)
 {
-  S3FS_PRN_INFO3("[IAM role=%s]", S3fsCurl::IAM_role.c_str());
+  if (!S3fsCurl::is_ecs) {
+    S3FS_PRN_INFO3("[IAM role=%s]", S3fsCurl::IAM_role.c_str());

-  if(0 == S3fsCurl::IAM_role.size()){
-    S3FS_PRN_ERR("IAM role name is empty.");
-    return -EIO;
+    if(0 == S3fsCurl::IAM_role.size()) {
+      S3FS_PRN_ERR("IAM role name is empty.");
+      return -EIO;
+    }
  }
+
  // at first set type for handle
  type = REQTYPE_IAMCRED;

@ -2426,7 +2430,13 @@ int S3fsCurl::GetIAMCredentials(void)
  }

  // url
-  url             = string(IAM_CRED_URL) + S3fsCurl::IAM_role;
+  if (is_ecs) {
+    url = string(IAM_CRED_URL_ECS) + std::getenv(ECS_IAM_ENV_VAR);
+  }
+  else {
+    url = string(IAM_CRED_URL) + S3fsCurl::IAM_role;
+  }
+
  requestHeaders  = NULL;
  responseHeaders.clear();
  bodydata        = new BodyData();
@ -2463,7 +2473,7 @@ bool S3fsCurl::LoadIAMRoleFromMetaData(void)
  }

  // url
-  url             = IAM_CRED_URL;
+  url             = string(IAM_CRED_URL);
  requestHeaders  = NULL;
  responseHeaders.clear();
  bodydata        = new BodyData();
--- a/src/curl.h
+++ b/src/curl.h
@ -230,6 +230,7 @@ class S3fsCurl
    static std::string      AWSSecretAccessKey;
    static std::string      AWSAccessToken;
    static time_t           AWSAccessTokenExpire;
+    static bool             is_ecs;
    static std::string      IAM_role;
    static long             ssl_verify_hostname;
    static curltime_t       curl_times;
@ -374,6 +375,7 @@ class S3fsCurl
    static long GetSslVerifyHostname(void) { return S3fsCurl::ssl_verify_hostname; }
    static int SetMaxParallelCount(int value);
    static int GetMaxParallelCount(void) { return S3fsCurl::max_parallel_cnt; }
+    static bool SetIsECS(bool flag);
    static std::string SetIAMRole(const char* role);
    static const char* GetIAMRole(void) { return S3fsCurl::IAM_role.c_str(); }
    static bool SetMultipartSize(off_t size);
--- a/src/s3fs.cpp
+++ b/src/s3fs.cpp
@ -122,6 +122,7 @@ static bool is_s3fs_uid           = false;// default does not set.
 static bool is_s3fs_gid           = false;// default does not set.
 static bool is_s3fs_umask         = false;// default does not set.
 static bool is_remove_cache       = false;
+static bool is_ecs                = false;
 static bool is_use_xattr          = false;
 static bool create_bucket         = false;
 static int64_t singlepart_copy_limit = FIVE_GB;
@ -4073,7 +4074,7 @@ static int get_access_keys(void)
  }

  // access key loading is deferred
-  if(load_iamrole){
+  if(load_iamrole || is_ecs){
     return EXIT_SUCCESS;
  }

@ -4508,7 +4509,16 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
      passwd_file = strchr(arg, '=') + sizeof(char);
      return 0;
    }
+    if(0 == strcmp(arg, "ecs")){
+      S3fsCurl::SetIsECS(true);
+      is_ecs = true;
+      return 0;
+    }
    if(0 == STR2NCMP(arg, "iam_role")){
+      if (is_ecs) {
+        S3FS_PRN_EXIT("option iam_role cannot be used in conjunction with ecs");
+        return -1;
+      }
      if(0 == strcmp(arg, "iam_role") || 0 == strcmp(arg, "iam_role=auto")){
        // loading IAM role name in s3fs_init(), because we need to wait initializing curl.
        //
@ -4908,7 +4918,7 @@ int main(int argc, char* argv[])
    S3FS_PRN_EXIT("specifying both passwd_file and the access keys options is invalid.");
    exit(EXIT_FAILURE);
  }
-  if(!S3fsCurl::IsPublicBucket() && !load_iamrole){
+  if(!S3fsCurl::IsPublicBucket() && !load_iamrole && !is_ecs){
    if(EXIT_SUCCESS != get_access_keys()){
      exit(EXIT_FAILURE);
    }
--- a/src/s3fs_util.cpp
+++ b/src/s3fs_util.cpp
@ -1167,6 +1167,10 @@ void show_help (void)
    "   enable_content_md5 (default is disable)\n"
    "      - ensure data integrity during writes with MD5 hash.\n"
    "\n"
+    "   ecs\n"
+    "      - This option instructs s3fs to query the ECS container credential\n"
+    "      metadata address instead of the instance metadata address.\n"
+    "\n"
    "   iam_role (default is no IAM role)\n"
    "      - This option requires the IAM role name or \"auto\". If you specify\n"
    "      \"auto\", s3fs will automatically use the IAM role names that are set\n"