Add option to allow unsigned payloads (#1801)

This reduces CPU usage of sigv4.  This reduces test run-time by 7
seconds per flag.

doc/man/s3fs.1

@@ -262,6 +262,9 @@ that they did not create.
 \fB\-o\fR enable_content_md5 (default is disable)
 Allow S3 server to check data integrity of uploads via the Content-MD5 header.
 This can add CPU overhead to transfers.
+\fB\-o\fR enable_unsigned_payload (default is disable)
+Do not calculate Content-SHA25 for PutObject and UploadPart
+payloads.  This can reduce CPU overhead to transfers.
 .TP
 \fB\-o\fR ecs (default is disable)
 This option instructs s3fs to query the ECS container credential metadata address instead of the instance metadata address.

src/curl.cpp

@@ -137,6 +137,7 @@ int              S3fsCurl::max_multireq        = 20;             // default
 off_t            S3fsCurl::multipart_size      = MULTIPART_SIZE; // default
 off_t            S3fsCurl::multipart_copy_size = 512 * 1024 * 1024;  // default
 signature_type_t S3fsCurl::signature_type      = V2_OR_V4;       // default
+bool             S3fsCurl::is_unsigned_payload = false;          // default
 bool             S3fsCurl::is_ua               = true;           // default
 bool             S3fsCurl::listobjectsv2       = false;          // default
 bool             S3fsCurl::is_use_session_token= false;          // default
@@ -2879,7 +2880,11 @@ void S3fsCurl::insertV4Headers()
     std::string payload_hash;
     switch (type) {
         case REQTYPE_PUT:
+            if(GetUnsignedPayload()){
+                payload_hash = "UNSIGNED-PAYLOAD";
+            }else{
                 payload_hash = s3fs_sha256_hex_fd(b_infile == NULL ? -1 : fileno(b_infile), 0, -1);
+            }
             break;
 
         case REQTYPE_COMPLETEMULTIPOST:
@@ -2894,7 +2899,11 @@ void S3fsCurl::insertV4Headers()
             }
 
         case REQTYPE_UPLOADMULTIPOST:
+            if(GetUnsignedPayload()){
+                payload_hash = "UNSIGNED-PAYLOAD";
+            }else{
                 payload_hash = s3fs_sha256_hex_fd(partdata.fd, partdata.startpos, partdata.size);
+            }
             break;
         default:
             break;

src/curl.h

@@ -169,6 +169,7 @@ class S3fsCurl
         static off_t            multipart_size;
         static off_t            multipart_copy_size;
         static signature_type_t signature_type;
+        static bool             is_unsigned_payload;
         static bool             is_ua;             // User-Agent
         static bool             listobjectsv2;
         static bool             requester_pays;
@@ -364,6 +365,8 @@ class S3fsCurl
         static off_t GetMultipartCopySize() { return S3fsCurl::multipart_copy_size; }
         static signature_type_t SetSignatureType(signature_type_t signature_type) { signature_type_t bresult = S3fsCurl::signature_type; S3fsCurl::signature_type = signature_type; return bresult; }
         static signature_type_t GetSignatureType() { return S3fsCurl::signature_type; }
+        static bool SetUnsignedPayload(bool issset) { bool bresult = S3fsCurl::is_unsigned_payload; S3fsCurl::is_unsigned_payload = issset; return bresult; }
+        static bool GetUnsignedPayload() { return S3fsCurl::is_unsigned_payload; }
         static bool SetUserAgentFlag(bool isset) { bool bresult = S3fsCurl::is_ua; S3fsCurl::is_ua = isset; return bresult; }
         static bool IsUserAgentFlag() { return S3fsCurl::is_ua; }
         static void InitUserAgent();

src/s3fs.cpp

@@ -4677,6 +4677,10 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
             S3fsCurl::SetContentMd5(true);
             return 0;
         }
+        if(0 == strcmp(arg, "enable_unsigned_payload")){
+            S3fsCurl::SetUnsignedPayload(true);
+            return 0;
+        }
         if(is_prefix(arg, "host=")){
             s3host = strchr(arg, '=') + sizeof(char);
             return 0;
@@ -5050,6 +5054,10 @@ int main(int argc, char* argv[])
         exit(EXIT_FAILURE);
     }
 
+    if(S3fsCurl::GetSignatureType() == V2_ONLY && S3fsCurl::GetUnsignedPayload()){
+        S3FS_PRN_WARN("Ignoring enable_unsigned_payload with sigv2");
+    }
+
     if(!FdEntity::GetNoMixMultipart() && max_dirty_data != -1){
         S3FS_PRN_WARN("Setting max_dirty_data to -1 when nomixupload is enabled");
         max_dirty_data = -1;

src/s3fs_help.cpp

@@ -315,9 +315,13 @@ static const char help_string[] =
     "   nomultipart (disable multipart uploads)\n"
     "\n"
     "   enable_content_md5 (default is disable)\n"
-    "      Allow S3 server to check data integrity of uploads via the\n"
+    "      - Allow S3 server to check data integrity of uploads via the\n"
     "      Content-MD5 header.  This can add CPU overhead to transfers.\n"
     "\n"
+    "   enable_unsigned_payload (default is disable)\n"
+    "      - Do not calculate Content-SHA25 for PutObject and UploadPart\n"
+    "      payloads.  This can reduce CPU overhead to transfers.\n"
+    "\n"
     "   ecs (default is disable)\n"
     "      - This option instructs s3fs to query the ECS container credential\n"
     "      metadata address instead of the instance metadata address.\n"

test/integration-test-common.sh

@@ -249,6 +249,7 @@ function start_s3fs {
             -o ssl_verify_hostname=0 \
             -o use_xattr=1 \
             -o createbucket \
+            -o enable_unsigned_payload \
             ${AUTH_OPT} \
             ${DIRECT_IO_OPT} \
             -o stat_cache_expire=1 \