Supported signature version 4 for GnuTLS/NSS and automatically set endpoint/sigv2

doc/man/s3fs.1

@@ -151,6 +151,10 @@ sets the url to use to access Amazon S3. If you want to use HTTPS, then you can
 .TP
 \fB\-o\fR endpoint (default="us-east-1")
 sets the endpoint to use.
+If this option is not specified, s3fs uses \"us-east-1\" region as the default.
+If the s3fs could not connect to the region specified by this option, s3fs could not run.
+But if you do not specify this option, and if you can not connect with the default region, s3fs will retry to automatically connect to the other region.
+So s3fs can know the correct region name, because s3fs can find it in an error from the S3 server.
 .TP
 \fB\-o\fR sigv2 (default is signature version 4)
 sets signing AWS requests by sing Signature Version 2.

src/curl.cpp

@@ -2577,9 +2577,6 @@ int S3fsCurl::CheckBucket(void)
   if (result != 0) {
     DPRN("Check bucket failed, S3 response: %s", (bodydata ? bodydata->str() : ""));
   }
-  delete bodydata;
-  bodydata = NULL;
-
   return result;
 }
 

src/gnutls_auth.cpp

@@ -120,6 +120,25 @@ bool s3fs_HMAC(const void* key, size_t keylen, const unsigned char* data, size_t
   return true;
 }
 
+bool s3fs_HMAC256(const void* key, size_t keylen, const unsigned char* data, size_t datalen, unsigned char** digest, unsigned int* digestlen)
+{
+  if(!key || 0 >= keylen || !data || 0 >= datalen || !digest || !digestlen){
+    return false;
+  }
+
+  if(NULL == (*digest = (unsigned char*)malloc(SHA256_DIGEST_SIZE))){
+    return false;
+  }
+
+  struct hmac_sha256_ctx ctx_hmac;
+  hmac_sha256_set_key(&ctx_hmac, keylen, reinterpret_cast<const uint8_t*>(key));
+  hmac_sha256_update(&ctx_hmac, datalen, reinterpret_cast<const uint8_t*>(data));
+  hmac_sha256_digest(&ctx_hmac, SHA256_DIGEST_SIZE, reinterpret_cast<uint8_t*>(*digest));
+  *digestlen = SHA256_DIGEST_SIZE;
+
+  return true;
+}
+
 #else	// USE_GNUTLS_NETTLE
 
 bool s3fs_HMAC(const void* key, size_t keylen, const unsigned char* data, size_t datalen, unsigned char** digest, unsigned int* digestlen)
@@ -142,6 +161,26 @@ bool s3fs_HMAC(const void* key, size_t keylen, const unsigned char* data, size_t
   return true;
 }
 
+bool s3fs_HMAC256(const void* key, size_t keylen, const unsigned char* data, size_t datalen, unsigned char** digest, unsigned int* digestlen)
+{
+  if(!key || 0 >= keylen || !data || 0 >= datalen || !digest || !digestlen){
+    return false;
+  }
+
+  if(0 >= (*digestlen = gnutls_hmac_get_len(GNUTLS_MAC_SHA256))){
+    return false;
+  }
+  if(NULL == (*digest = (unsigned char*)malloc(*digestlen + 1))){
+    return false;
+  }
+  if(0 > gnutls_hmac_fast(GNUTLS_MAC_SHA256, key, keylen, data, datalen, *digest)){
+    free(*digest);
+    *digest = NULL;
+    return false;
+  }
+  return true;
+}
+
 #endif	// USE_GNUTLS_NETTLE
 
 //-------------------------------------------------------------------
@@ -256,6 +295,154 @@ unsigned char* s3fs_md5hexsum(int fd, off_t start, ssize_t size)
 
 #endif	// USE_GNUTLS_NETTLE
 
+//-------------------------------------------------------------------
+// Utility Function for SHA256
+//-------------------------------------------------------------------
+#define SHA256_DIGEST_LENGTH     32
+
+size_t get_sha256_digest_length(void)
+{
+  return SHA256_DIGEST_LENGTH;
+}
+
+#ifdef	USE_GNUTLS_NETTLE
+bool s3fs_sha256(const unsigned char* data, unsigned int datalen, unsigned char** digest, unsigned int* digestlen)
+{
+  (*digestlen) = static_cast<unsigned int>(get_sha256_digest_length());
+  if(NULL == ((*digest) = reinterpret_cast<unsigned char*>(malloc(*digestlen)))){
+    return false;
+  }
+
+  struct sha256_ctx ctx_sha256;
+  sha256_init(&ctx_sha256);
+  sha256_update(&ctx_sha256, datalen, data);
+  sha256_digest(&ctx_sha256, *digestlen, *digest);
+
+  return true;
+}
+
+unsigned char* s3fs_sha256hexsum(int fd, off_t start, ssize_t size)
+{
+  struct sha256_ctx ctx_sha256;
+  unsigned char     buf[512];
+  ssize_t           bytes;
+  unsigned char*    result;
+
+  // seek to top of file.
+  if(-1 == lseek(fd, start, SEEK_SET)){
+    return NULL;
+  }
+
+  memset(buf, 0, 512);
+  sha256_init(&ctx_sha256);
+
+  for(ssize_t total = 0; total < size; total += bytes){
+    bytes = 512 < (size - total) ? 512 : (size - total);
+    bytes = read(fd, buf, bytes);
+    if(0 == bytes){
+      // end of file
+      break;
+    }else if(-1 == bytes){
+      // error
+      DPRNNN("file read error(%d)", errno);
+      return NULL;
+    }
+    sha256_update(&ctx_sha256, bytes, buf);
+    memset(buf, 0, 512);
+  }
+  if(NULL == (result = (unsigned char*)malloc(get_sha256_digest_length()))){
+    return NULL;
+  }
+  sha256_digest(&ctx_sha256, get_sha256_digest_length(), result);
+
+  if(-1 == lseek(fd, start, SEEK_SET)){
+    free(result);
+    return NULL;
+  }
+
+  return result;
+}
+
+#else	// USE_GNUTLS_NETTLE
+
+bool s3fs_sha256(const unsigned char* data, unsigned int datalen, unsigned char** digest, unsigned int* digestlen)
+{
+  (*digestlen) = static_cast<unsigned int>(get_sha256_digest_length());
+  if(NULL == ((*digest) = reinterpret_cast<unsigned char*>(malloc(*digestlen)))){
+    return false;
+  }
+
+  gcry_md_hd_t   ctx_sha256;
+  gcry_error_t   err;
+  if(GPG_ERR_NO_ERROR != (err = gcry_md_open(&ctx_sha256, GCRY_MD_SHA256, 0))){
+    DPRNN("SHA256 context creation failure: %s/%s", gcry_strsource(err), gcry_strerror(err));
+    free(*digest);
+    return false;
+  }
+  gcry_md_write(ctx_sha256, data, datalen);
+  memcpy(*digest, gcry_md_read(ctx_sha256, 0), *digestlen);
+  gcry_md_close(ctx_sha256);
+
+  return true;
+}
+
+unsigned char* s3fs_sha256hexsum(int fd, off_t start, ssize_t size)
+{
+  gcry_md_hd_t   ctx_sha256;
+  gcry_error_t   err;
+  char           buf[512];
+  ssize_t        bytes;
+  unsigned char* result;
+
+  if(-1 == size){
+    struct stat st;
+    if(-1 == fstat(fd, &st)){
+      return NULL;
+    }
+    size = static_cast<ssize_t>(st.st_size);
+  }
+
+  // seek to top of file.
+  if(-1 == lseek(fd, start, SEEK_SET)){
+    return NULL;
+  }
+
+  memset(buf, 0, 512);
+  if(GPG_ERR_NO_ERROR != (err = gcry_md_open(&ctx_sha256, GCRY_MD_SHA256, 0))){
+    DPRNN("SHA256 context creation failure: %s/%s", gcry_strsource(err), gcry_strerror(err));
+    return NULL;
+  }
+
+  for(ssize_t total = 0; total < size; total += bytes){
+    bytes = 512 < (size - total) ? 512 : (size - total);
+    bytes = read(fd, buf, bytes);
+    if(0 == bytes){
+      // end of file
+      break;
+    }else if(-1 == bytes){
+      // error
+      DPRNNN("file read error(%d)", errno);
+      return NULL;
+    }
+    gcry_md_write(ctx_sha256, buf, bytes);
+    memset(buf, 0, 512);
+  }
+  if(NULL == (result = (unsigned char*)malloc(get_sha256_digest_length()))){
+    return NULL;
+  }
+  memcpy(result, gcry_md_read(ctx_sha256, 0), get_sha256_digest_length());
+  gcry_md_close(ctx_sha256);
+
+  if(-1 == lseek(fd, start, SEEK_SET)){
+    free(result);
+    return NULL;
+  }
+
+  return result;
+}
+
+#endif	// USE_GNUTLS_NETTLE
+
 /*
 * Local variables:
 * tab-width: 4

src/nss_auth.cpp

@@ -83,7 +83,7 @@ bool s3fs_destroy_crypt_mutex(void)
 //-------------------------------------------------------------------
 // Utility Function for HMAC
 //-------------------------------------------------------------------
-bool s3fs_HMAC(const void* key, size_t keylen, const unsigned char* data, size_t datalen, unsigned char** digest, unsigned int* digestlen)
+static bool s3fs_HMAC_RAW(const void* key, size_t keylen, const unsigned char* data, size_t datalen, unsigned char** digest, unsigned int* digestlen, bool is_sha256)
 {
   if(!key || 0 >= keylen || !data || 0 >= datalen || !digest || !digestlen){
     return false;
@@ -94,17 +94,17 @@ bool s3fs_HMAC(const void* key, size_t keylen, const unsigned char* data, size_t
   PK11Context*  Context;
   SECStatus     SecStatus;
   unsigned char tmpdigest[64];
-  SECItem       KeySecItem   = {siBuffer, reinterpret_cast<unsigned char*>(const_cast<void*>(key)), keylen};
+  SECItem       KeySecItem   = {siBuffer, reinterpret_cast<unsigned char*>(const_cast<void*>(key)), static_cast<unsigned int>(keylen)};
   SECItem       NullSecItem  = {siBuffer, NULL, 0};
 
   if(NULL == (Slot = PK11_GetInternalKeySlot())){
     return false;
   }
-  if(NULL == (pKey = PK11_ImportSymKey(Slot, CKM_SHA_1_HMAC, PK11_OriginUnwrap, CKA_SIGN, &KeySecItem, NULL))){
+  if(NULL == (pKey = PK11_ImportSymKey(Slot, (is_sha256 ? CKM_SHA256_HMAC : CKM_SHA_1_HMAC), PK11_OriginUnwrap, CKA_SIGN, &KeySecItem, NULL))){
     PK11_FreeSlot(Slot);
     return false;
   }
-  if(NULL == (Context = PK11_CreateContextBySymKey(CKM_SHA_1_HMAC, CKA_SIGN, pKey, &NullSecItem))){
+  if(NULL == (Context = PK11_CreateContextBySymKey((is_sha256 ? CKM_SHA256_HMAC : CKM_SHA_1_HMAC), CKA_SIGN, pKey, &NullSecItem))){
     PK11_FreeSymKey(pKey);
     PK11_FreeSlot(Slot);
     return false;
@@ -132,6 +132,16 @@ bool s3fs_HMAC(const void* key, size_t keylen, const unsigned char* data, size_t
   return true;
 }
 
+bool s3fs_HMAC(const void* key, size_t keylen, const unsigned char* data, size_t datalen, unsigned char** digest, unsigned int* digestlen)
+{
+  return s3fs_HMAC_RAW(key, keylen, data, datalen, digest, digestlen, false);
+}
+
+bool s3fs_HMAC256(const void* key, size_t keylen, const unsigned char* data, size_t datalen, unsigned char** digest, unsigned int* digestlen)
+{
+  return s3fs_HMAC_RAW(key, keylen, data, datalen, digest, digestlen, true);
+}
+
 //-------------------------------------------------------------------
 // Utility Function for MD5
 //-------------------------------------------------------------------
@@ -193,6 +203,86 @@ unsigned char* s3fs_md5hexsum(int fd, off_t start, ssize_t size)
   return result;
 }
 
+//-------------------------------------------------------------------
+// Utility Function for SHA256
+//-------------------------------------------------------------------
+size_t get_sha256_digest_length(void)
+{
+  return SHA256_LENGTH;
+}
+
+bool s3fs_sha256(const unsigned char* data, unsigned int datalen, unsigned char** digest, unsigned int* digestlen)
+{
+  (*digestlen) = static_cast<unsigned int>(get_sha256_digest_length());
+  if(NULL == ((*digest) = reinterpret_cast<unsigned char*>(malloc(*digestlen)))){
+    return false;
+  }
+
+  PK11Context*	 sha256ctx;
+  unsigned int   sha256outlen;
+  sha256ctx = PK11_CreateDigestContext(SEC_OID_SHA256);
+
+  PK11_DigestOp(sha256ctx, data, datalen);
+  PK11_DigestFinal(sha256ctx, *digest, &sha256outlen, *digestlen);
+  PK11_DestroyContext(sha256ctx, PR_TRUE);
+  *digestlen = sha256outlen;
+
+  return true;
+}
+
+unsigned char* s3fs_sha256hexsum(int fd, off_t start, ssize_t size)
+{
+  PK11Context*	 sha256ctx;
+  unsigned char  buf[512];
+  ssize_t        bytes;
+  unsigned char* result;
+  unsigned int   sha256outlen;
+
+  if(-1 == size){
+    struct stat st;
+    if(-1 == fstat(fd, &st)){
+      return NULL;
+    }
+    size = static_cast<ssize_t>(st.st_size);
+  }
+
+  // seek to top of file.
+  if(-1 == lseek(fd, start, SEEK_SET)){
+    return NULL;
+  }
+
+  memset(buf, 0, 512);
+  sha256ctx = PK11_CreateDigestContext(SEC_OID_SHA256);
+
+  for(ssize_t total = 0; total < size; total += bytes){
+    bytes = 512 < (size - total) ? 512 : (size - total);
+    bytes = read(fd, buf, bytes);
+    if(0 == bytes){
+      // end of file
+      break;
+    }else if(-1 == bytes){
+      // error
+      DPRNNN("file read error(%d)", errno);
+      return NULL;
+    }
+    PK11_DigestOp(sha256ctx, buf, bytes);
+    memset(buf, 0, 512);
+  }
+  if(NULL == (result = (unsigned char*)malloc(get_sha256_digest_length()))){
+    PK11_DestroyContext(sha256ctx, PR_TRUE);
+    return NULL;
+  }
+  PK11_DigestFinal(sha256ctx, result, &sha256outlen, get_sha256_digest_length());
+  PK11_DestroyContext(sha256ctx, PR_TRUE);
+
+  if(-1 == lseek(fd, start, SEEK_SET)){
+    free(result);
+    return NULL;
+  }
+
+  return result;
+}
+
 /*
 * Local variables:
 * tab-width: 4

src/s3fs.cpp

@@ -84,6 +84,7 @@ bool foreground                   = false;
 bool foreground2                  = false;
 bool nomultipart                  = false;
 bool pathrequeststyle             = false;
+bool is_specified_endpoint        = false;
 std::string program_name;
 std::string service_path          = "/";
 std::string host                  = "http://s3.amazonaws.com";
@@ -2979,6 +2980,41 @@ static int s3fs_utility_mode(void)
   return result;
 }
 
+//
+// If calling with wrong region, s3fs gets following error body as 400 erro code.
+// "<Error><Code>AuthorizationHeaderMalformed</Code><Message>The authorization header is 
+//  malformed; the region 'us-east-1' is wrong; expecting 'ap-northeast-1'</Message>
+//  <Region>ap-northeast-1</Region><RequestId>...</RequestId><HostId>...</HostId>
+//  </Error>"
+//
+// So this is cheep codes but s3fs should get correct reagion automatically.
+//
+static bool check_region_error(const char* pbody, string& expectregion)
+{
+  if(!pbody){
+    return false;
+  }
+  const char* region;
+  const char* regionend;
+  if(NULL == (region = strcasestr(pbody, "<Message>The authorization header is malformed; the region "))){
+    return false;
+  }
+  if(NULL == (region = strcasestr(region, "expecting \'"))){
+    return false;
+  }
+  region += strlen("expecting \'");
+  if(NULL == (regionend = strchr(region, '\''))){
+    return false;
+  }
+  string strtmp(region, (regionend - region));
+  if(0 == strtmp.length()){
+    return false;
+  }
+  expectregion = strtmp;
+
+  return true;
+}
+
 static int s3fs_check_service(void)
 {
   FPRN("check services.");
@@ -2990,12 +3026,61 @@ static int s3fs_check_service(void)
   }
 
   S3fsCurl s3fscurl;
-  if(0 != s3fscurl.CheckBucket()){
+  if(-1 == s3fscurl.CheckBucket()){
     fprintf(stderr, "%s: Failed to access bucket.\n", program_name.c_str());
     return EXIT_FAILURE;
   }
   long responseCode = s3fscurl.GetLastResponseCode();
 
+  if(responseCode == 400){
+    if(!S3fsCurl::IsSignatureV4()){
+      // signature version 2
+      fprintf(stderr, "%s: Bad Request\n", program_name.c_str());
+      return EXIT_FAILURE;
+    }
+    if(is_specified_endpoint){
+      // if specifies endpoint, do not retry to connect.
+      fprintf(stderr, "%s: Bad Request\n", program_name.c_str());
+      return EXIT_FAILURE;
+    }
+
+    // check region error for signature version 4
+    BodyData* body = s3fscurl.GetBodyData();
+    string    expectregion;
+    if(check_region_error(body->str(), expectregion)){
+      // not specified endpoint, so try to connect to expected region.
+      LOWSYSLOGPRINT(LOG_ERR, "Could not connect wrong region %s, so retry to connect region %s.", endpoint.c_str(), expectregion.c_str());
+      FPRN("Could not connect wrong region %s, so retry to connect region %s.", endpoint.c_str(), expectregion.c_str());
+      endpoint = expectregion;
+
+      // retry to check
+      s3fscurl.DestroyCurlHandle();
+      if(-1 == s3fscurl.CheckBucket()){
+        fprintf(stderr, "%s: Failed to access bucket.\n", program_name.c_str());
+        return EXIT_FAILURE;
+      }
+      responseCode = s3fscurl.GetLastResponseCode();
+    }
+
+    if(responseCode == 400){
+      // retry to use sigv2
+      LOWSYSLOGPRINT(LOG_ERR, "Could not connect, so retry to connect by signature version 2.");
+      FPRN("Could not connect, so retry to connect by signature version 2.");
+      S3fsCurl::SetSignatureV4();
+
+      // retry to check
+      s3fscurl.DestroyCurlHandle();
+      if(-1 == s3fscurl.CheckBucket()){
+        fprintf(stderr, "%s: Failed to access bucket.\n", program_name.c_str());
+        return EXIT_FAILURE;
+      }
+      responseCode = s3fscurl.GetLastResponseCode();
+      if(responseCode == 400){
+        fprintf(stderr, "%s: Bad Request\n", program_name.c_str());
+        return EXIT_FAILURE;
+      }
+    }
+  }
   if(responseCode == 403){
     fprintf(stderr, "%s: invalid credentials\n", program_name.c_str());
     return EXIT_FAILURE;
@@ -3770,7 +3855,8 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
       return 0;
     }
     if(0 == STR2NCMP(arg, "endpoint=")){
-      endpoint = strchr(arg, '=') + sizeof(char);
+      endpoint              = strchr(arg, '=') + sizeof(char);
+      is_specified_endpoint = true;
       return 0;
     }
     if(0 == strcmp(arg, "use_path_request_style")){

src/s3fs_util.cpp

@@ -977,7 +977,14 @@ void show_help (void)
     "      - sets the url to use to access amazon s3\n"
     "\n"
     "   endpoint (default=\"us-east-1\")\n"
-    "      - sets the endpoint to use\n"
+    "      - sets the endpoint to use on signatue version 4\n"
+    "      If this option is not specified, s3fs uses \"us-east-1\" region as\n"
+    "      the default. If the s3fs could not connect to the region specified\n"
+    "      by this option, s3fs could not run. But if you do not specify this\n"
+    "      option, and if you can not connect with the default region, s3fs\n"
+    "      will retry to automatically connect to the other region. So s3fs\n"
+    "      can know the correct region name, because s3fs can find it in an\n"
+    "      error from the S3 server.\n"
     "\n"
     "   sigv2 (default is signature version 4)\n"
     "      - sets signing AWS requests by sing Signature Version 2\n"