Enable Content-MD5 during multipart upload part

This allows retries of multi-part uploads instead of discovering a
fatal error during complete multipart upload.  Also enable Content-MD5
for integration tests and refactor hexadecimal code.

src/common_auth.cpp

@@ -30,6 +30,17 @@ using namespace std;
 //-------------------------------------------------------------------
 // Utility Function
 //-------------------------------------------------------------------
+std::string s3fs_hex(const unsigned char* input, size_t length)
+{
+  std::string hex;
+  for(size_t pos = 0; pos < length; ++pos){
+    char hexbuf[3];
+    snprintf(hexbuf, 3, "%02x", input[pos]);
+    hex += hexbuf;
+  }
+  return hex;
+}
+
 char* s3fs_base64(const unsigned char* input, size_t length)
 {
   static const char* base = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=";
@@ -139,22 +150,16 @@ string s3fs_get_content_md5(int fd)
 string s3fs_md5sum(int fd, off_t start, ssize_t size)
 {
   size_t digestlen = get_md5_digest_length();
-  char md5[2 * digestlen + 1];
-  char hexbuf[3];
   unsigned char* md5hex;
 
   if(NULL == (md5hex = s3fs_md5hexsum(fd, start, size))){
     return string("");
   }
 
-  memset(md5, 0, 2 * digestlen + 1);
-  for(size_t pos = 0; pos < digestlen; pos++){
-    snprintf(hexbuf, 3, "%02x", md5hex[pos]);
-    strncat(md5, hexbuf, 2);
-  }
+  std::string md5 = s3fs_hex(md5hex, digestlen);
   free(md5hex);
 
-  return string(md5);
+  return md5;
 }
 
 string s3fs_sha256sum(int fd, off_t start, ssize_t size)

src/curl.cpp

@@ -2971,11 +2971,16 @@ int S3fsCurl::UploadMultipartPostSetup(const char* tpath, int part_num, string&
   }
 
   // make md5 and file pointer
-  partdata.etag = s3fs_md5sum(partdata.fd, partdata.startpos, partdata.size);
-  if(partdata.etag.empty()){
+  unsigned char *md5raw = s3fs_md5hexsum(partdata.fd, partdata.startpos, partdata.size);
+  if(md5raw == NULL){
     DPRN("Could not make md5 for file(part %d)", part_num);
     return -1;
   }
+  partdata.etag = s3fs_hex(md5raw, get_md5_digest_length());
+  char* md5base64p = s3fs_base64(md5raw, get_md5_digest_length());
+  std::string md5base64 = md5base64p;
+  free(md5base64p);
+  free(md5raw);
 
   // create handle
   if(!CreateCurlHandle(true)){
@@ -3003,8 +3008,14 @@ int S3fsCurl::UploadMultipartPostSetup(const char* tpath, int part_num, string&
     requestHeaders = curl_slist_sort_insert(requestHeaders, "Date", date.c_str());
     requestHeaders = curl_slist_sort_insert(requestHeaders, "Accept", NULL);
 
+    string strMD5;
+    if(S3fsCurl::is_content_md5){
+      strMD5 = md5base64;
+      requestHeaders = curl_slist_sort_insert(requestHeaders, "Content-MD5", strMD5.c_str());
+    }
+
     if(!S3fsCurl::IsPublicBucket()){
-      string Signature = CalcSignatureV2("PUT", "", "", date, resource);
+      string Signature = CalcSignatureV2("PUT", strMD5, "", date, resource);
       requestHeaders   = curl_slist_sort_insert(requestHeaders, "Authorization", string("AWS " + AWSAccessKeyId + ":" + Signature).c_str());
     }
 

src/s3fs_auth.h

@@ -26,6 +26,7 @@
 //
 // in common_auth.cpp
 //
+std::string s3fs_hex(const unsigned char* input, size_t length);
 char* s3fs_base64(const unsigned char* input, size_t length);
 unsigned char* s3fs_decode64(const char* input, size_t* plength);
 std::string s3fs_get_content_md5(int fd);

src/s3fs_util.cpp

@@ -1013,7 +1013,7 @@ void show_help (void)
     "   nomultipart (disable multipart uploads)\n"
     "\n"
     "   enable_content_md5 (default is disable)\n"
-    "      - verifying uploaded object without multipart by content-md5 header.\n"
+    "      - ensure data integrity during writes with MD5 hash.\n"
     "\n"
     "   iam_role (default is no role)\n"
     "      - set the IAM Role that will supply the credentials from the \n"

test/small-integration-test.sh

@@ -57,6 +57,7 @@ then
 fi
 stdbuf -oL -eL $S3FS $TEST_BUCKET_1 $TEST_BUCKET_MOUNT_POINT_1 \
     -o createbucket \
+    -o enable_content_md5 \
     -o passwd_file=$S3FS_CREDENTIALS_FILE \
     -o sigv2 \
     -o singlepart_copy_limit=$((10 * 1024)) \