From 3cf00626a29363207030e6bfe67c3e09c37ff9b4 Mon Sep 17 00:00:00 2001 From: Andrew Gaul Date: Mon, 1 Nov 2021 23:33:55 +0900 Subject: [PATCH] Add option to allow unsigned payloads (#1801) This reduces CPU usage of sigv4. This reduces test run-time by 7 seconds per flag. --- doc/man/s3fs.1 | 3 +++ src/curl.cpp | 13 +++++++++++-- src/curl.h | 3 +++ src/s3fs.cpp | 8 ++++++++ src/s3fs_help.cpp | 6 +++++- test/integration-test-common.sh | 1 + 6 files changed, 31 insertions(+), 3 deletions(-) diff --git a/doc/man/s3fs.1 b/doc/man/s3fs.1 index be89d1b..2965f81 100644 --- a/doc/man/s3fs.1 +++ b/doc/man/s3fs.1 @@ -262,6 +262,9 @@ that they did not create. \fB\-o\fR enable_content_md5 (default is disable) Allow S3 server to check data integrity of uploads via the Content-MD5 header. This can add CPU overhead to transfers. +\fB\-o\fR enable_unsigned_payload (default is disable) +Do not calculate Content-SHA25 for PutObject and UploadPart +payloads. This can reduce CPU overhead to transfers. .TP \fB\-o\fR ecs (default is disable) This option instructs s3fs to query the ECS container credential metadata address instead of the instance metadata address. diff --git a/src/curl.cpp b/src/curl.cpp index 0a56905..a2113f7 100644 --- a/src/curl.cpp +++ b/src/curl.cpp @@ -137,6 +137,7 @@ int S3fsCurl::max_multireq = 20; // default off_t S3fsCurl::multipart_size = MULTIPART_SIZE; // default off_t S3fsCurl::multipart_copy_size = 512 * 1024 * 1024; // default signature_type_t S3fsCurl::signature_type = V2_OR_V4; // default +bool S3fsCurl::is_unsigned_payload = false; // default bool S3fsCurl::is_ua = true; // default bool S3fsCurl::listobjectsv2 = false; // default bool S3fsCurl::is_use_session_token= false; // default @@ -2879,7 +2880,11 @@ void S3fsCurl::insertV4Headers() std::string payload_hash; switch (type) { case REQTYPE_PUT: - payload_hash = s3fs_sha256_hex_fd(b_infile == NULL ? -1 : fileno(b_infile), 0, -1); + if(GetUnsignedPayload()){ + payload_hash = "UNSIGNED-PAYLOAD"; + }else{ + payload_hash = s3fs_sha256_hex_fd(b_infile == NULL ? -1 : fileno(b_infile), 0, -1); + } break; case REQTYPE_COMPLETEMULTIPOST: @@ -2894,7 +2899,11 @@ void S3fsCurl::insertV4Headers() } case REQTYPE_UPLOADMULTIPOST: - payload_hash = s3fs_sha256_hex_fd(partdata.fd, partdata.startpos, partdata.size); + if(GetUnsignedPayload()){ + payload_hash = "UNSIGNED-PAYLOAD"; + }else{ + payload_hash = s3fs_sha256_hex_fd(partdata.fd, partdata.startpos, partdata.size); + } break; default: break; diff --git a/src/curl.h b/src/curl.h index b948835..206043b 100644 --- a/src/curl.h +++ b/src/curl.h @@ -169,6 +169,7 @@ class S3fsCurl static off_t multipart_size; static off_t multipart_copy_size; static signature_type_t signature_type; + static bool is_unsigned_payload; static bool is_ua; // User-Agent static bool listobjectsv2; static bool requester_pays; @@ -364,6 +365,8 @@ class S3fsCurl static off_t GetMultipartCopySize() { return S3fsCurl::multipart_copy_size; } static signature_type_t SetSignatureType(signature_type_t signature_type) { signature_type_t bresult = S3fsCurl::signature_type; S3fsCurl::signature_type = signature_type; return bresult; } static signature_type_t GetSignatureType() { return S3fsCurl::signature_type; } + static bool SetUnsignedPayload(bool issset) { bool bresult = S3fsCurl::is_unsigned_payload; S3fsCurl::is_unsigned_payload = issset; return bresult; } + static bool GetUnsignedPayload() { return S3fsCurl::is_unsigned_payload; } static bool SetUserAgentFlag(bool isset) { bool bresult = S3fsCurl::is_ua; S3fsCurl::is_ua = isset; return bresult; } static bool IsUserAgentFlag() { return S3fsCurl::is_ua; } static void InitUserAgent(); diff --git a/src/s3fs.cpp b/src/s3fs.cpp index dbf3f11..6a9148b 100644 --- a/src/s3fs.cpp +++ b/src/s3fs.cpp @@ -4677,6 +4677,10 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar S3fsCurl::SetContentMd5(true); return 0; } + if(0 == strcmp(arg, "enable_unsigned_payload")){ + S3fsCurl::SetUnsignedPayload(true); + return 0; + } if(is_prefix(arg, "host=")){ s3host = strchr(arg, '=') + sizeof(char); return 0; @@ -5050,6 +5054,10 @@ int main(int argc, char* argv[]) exit(EXIT_FAILURE); } + if(S3fsCurl::GetSignatureType() == V2_ONLY && S3fsCurl::GetUnsignedPayload()){ + S3FS_PRN_WARN("Ignoring enable_unsigned_payload with sigv2"); + } + if(!FdEntity::GetNoMixMultipart() && max_dirty_data != -1){ S3FS_PRN_WARN("Setting max_dirty_data to -1 when nomixupload is enabled"); max_dirty_data = -1; diff --git a/src/s3fs_help.cpp b/src/s3fs_help.cpp index 757f4e4..bd79823 100644 --- a/src/s3fs_help.cpp +++ b/src/s3fs_help.cpp @@ -315,9 +315,13 @@ static const char help_string[] = " nomultipart (disable multipart uploads)\n" "\n" " enable_content_md5 (default is disable)\n" - " Allow S3 server to check data integrity of uploads via the\n" + " - Allow S3 server to check data integrity of uploads via the\n" " Content-MD5 header. This can add CPU overhead to transfers.\n" "\n" + " enable_unsigned_payload (default is disable)\n" + " - Do not calculate Content-SHA25 for PutObject and UploadPart\n" + " payloads. This can reduce CPU overhead to transfers.\n" + "\n" " ecs (default is disable)\n" " - This option instructs s3fs to query the ECS container credential\n" " metadata address instead of the instance metadata address.\n" diff --git a/test/integration-test-common.sh b/test/integration-test-common.sh index 056c474..8c59262 100644 --- a/test/integration-test-common.sh +++ b/test/integration-test-common.sh @@ -249,6 +249,7 @@ function start_s3fs { -o ssl_verify_hostname=0 \ -o use_xattr=1 \ -o createbucket \ + -o enable_unsigned_payload \ ${AUTH_OPT} \ ${DIRECT_IO_OPT} \ -o stat_cache_expire=1 \